1 /*
2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.openjdk.skara.bots.mlbridge;
24
25 import org.openjdk.skara.bot.WorkItem;
26 import org.openjdk.skara.email.*;
27 import org.openjdk.skara.host.*;
28 import org.openjdk.skara.mailinglist.*;
29 import org.openjdk.skara.vcs.Repository;
30
31 import java.io.*;
32 import java.net.URI;
33 import java.nio.file.Path;
34 import java.time.Duration;
35 import java.util.*;
36 import java.util.function.*;
37 import java.util.logging.Logger;
38 import java.util.regex.Pattern;
39 import java.util.stream.Collectors;
40
41 class ArchiveWorkItem implements WorkItem {
42 private final PullRequest pr;
43 private final MailingListBridgeBot bot;
44 private final Consumer<RuntimeException> exceptionConsumer;
45 private final Logger log = Logger.getLogger("org.openjdk.skara.bots.mlbridge");
46
47 ArchiveWorkItem(PullRequest pr, MailingListBridgeBot bot, Consumer<RuntimeException> exceptionConsumer) {
48 this.pr = pr;
49 this.bot = bot;
50 this.exceptionConsumer = exceptionConsumer;
51 }
52
53 @Override
54 public String toString() {
55 return "ArchiveWorkItem@" + bot.codeRepo().getName() + "#" + pr.getId();
56 }
57
58 @Override
59 public boolean concurrentWith(WorkItem other) {
60 if (!(other instanceof ArchiveWorkItem)) {
61 return true;
62 }
63 ArchiveWorkItem otherItem = (ArchiveWorkItem)other;
64 if (!pr.getId().equals(otherItem.pr.getId())) {
65 return true;
66 }
67 if (!bot.codeRepo().getName().equals(otherItem.bot.codeRepo().getName())) {
68 return true;
69 }
70 return false;
71 }
72
73 private void pushMbox(Repository localRepo, String message) {
74 try {
75 localRepo.add(localRepo.root().resolve("."));
76 var hash = localRepo.commit(message, bot.emailAddress().fullName().orElseThrow(), bot.emailAddress().address());
77 localRepo.push(hash, bot.archiveRepo().getUrl(), "master");
78 } catch (IOException e) {
79 throw new UncheckedIOException(e);
80 }
81 }
82
83 private static final Pattern replyToPattern = Pattern.compile("^\\s*@([-A-Za-z0-9]+)");
84
85 private Optional<Comment> getParentPost(Comment post, List<Comment> all) {
86 var matcher = replyToPattern.matcher(post.body());
87 if (matcher.find()) {
88 var replyToName = matcher.group(1);
89 var replyToNamePattern = Pattern.compile("^" + replyToName + "$");
90
91 var postIterator = all.listIterator();
92 while (postIterator.hasNext()) {
93 var cur = postIterator.next();
94 if (cur == post) {
95 break;
96 }
97 }
98
99 while (postIterator.hasPrevious()) {
100 var cur = postIterator.previous();
101 var userMatcher = replyToNamePattern.matcher(cur.author().userName());
102 if (userMatcher.matches()) {
103 return Optional.of(cur);
104 }
105 }
106 }
107
108 return Optional.empty();
109 }
110
111 private Repository materializeArchive(Path scratchPath) {
112 try {
113 return Repository.materialize(scratchPath, bot.archiveRepo().getUrl(), pr.getTargetRef());
114 } catch (IOException e) {
115 throw new UncheckedIOException(e);
116 }
117 }
118
119 private final static Pattern commandPattern = Pattern.compile("^/.*$");
120
121 private boolean ignoreComment(HostUserDetails author, String body) {
122 if (pr.repository().host().getCurrentUserDetails().equals(author)) {
123 return true;
124 }
125 if (bot.ignoredUsers().contains(author.userName())) {
126 return true;
127 }
128 var commandMatcher = commandPattern.matcher(body);
129 if (commandMatcher.matches()) {
130 return true;
131 }
132 for (var ignoredCommentPattern : bot.ignoredComments()) {
133 var ignoredCommentMatcher = ignoredCommentPattern.matcher(body);
134 if (ignoredCommentMatcher.find()) {
135 return true;
136 }
137 }
138 return false;
139 }
140
141 private static final String webrevCommentMarker = "<!-- mlbridge webrev comment -->";
142 private static final String webrevHeaderMarker = "<!-- mlbridge webrev header -->";
143 private static final String webrevListMarker = "<!-- mlbridge webrev list -->";
144
145 private void updateWebrevComment(List<Comment> comments, int index, URI fullWebrev, URI incWebrev) {
146 var existing = comments.stream()
147 .filter(comment -> comment.author().equals(pr.repository().host().getCurrentUserDetails()))
148 .filter(comment -> comment.body().contains(webrevCommentMarker))
149 .findAny();
150 var comment = webrevCommentMarker + "\n";
151 comment += webrevHeaderMarker + "\n";
152 comment += "### Webrevs" + "\n";
153 comment += webrevListMarker + "\n";
154 comment += " * " + String.format("%02d", index) + ": [Full](" + fullWebrev.toString() + ")";
155 if (incWebrev != null) {
156 comment += " - [Incremental](" + incWebrev.toString() + ")";
157 }
158 comment += " (" + pr.getHeadHash() + ")\n";
159
160 if (existing.isPresent()) {
161 if (existing.get().body().contains(fullWebrev.toString())) {
162 log.fine("Webrev link already posted - skipping update");
163 return;
164 }
165 var previousListStart = existing.get().body().indexOf(webrevListMarker) + webrevListMarker.length() + 1;
166 var previousList = existing.get().body().substring(previousListStart);
167 comment += previousList;
168 pr.updateComment(existing.get().id(), comment);
169 } else {
170 pr.addComment(comment);
171 }
172 }
173
174 private List<Email> parseArchive(MailingList archive) {
175 var conversations = archive.conversations(Duration.ofDays(365));
176
177 if (conversations.size() == 0) {
178 return new ArrayList<>();
179 } else if (conversations.size() == 1) {
180 var conversation = conversations.get(0);
181 return conversation.allMessages();
182 } else {
183 throw new RuntimeException("Something is wrong with the mbox");
184 }
185 }
186
187 @Override
188 public void run(Path scratchPath) {
189 var path = scratchPath.resolve("mlbridge");
190 var archiveRepo = materializeArchive(path);
191 var mboxBasePath = path.resolve(bot.codeRepo().getName());
192 var mbox = MailingListServerFactory.createMboxFileServer(mboxBasePath);
193 var reviewArchiveList = mbox.getList(pr.getId());
194 var sentMails = parseArchive(reviewArchiveList);
195
196 // First determine if this PR should be inspected further or not
197 if (sentMails.isEmpty()) {
198 var labels = new HashSet<>(pr.getLabels());
199 for (var readyLabel : bot.readyLabels()) {
200 if (!labels.contains(readyLabel)) {
201 log.fine("PR is not yet ready - missing label '" + readyLabel + "'");
202 return;
203 }
204 }
205 }
206
207 // Also inspect comments before making the first post
208 var comments = pr.getComments();
209 if (sentMails.isEmpty()) {
210 for (var readyComment : bot.readyComments().entrySet()) {
211 var commentFound = false;
212 for (var comment : comments) {
213 if (comment.author().userName().equals(readyComment.getKey())) {
214 var matcher = readyComment.getValue().matcher(comment.body());
215 if (matcher.find()) {
216 commentFound = true;
217 break;
218 }
219 }
220 }
221 if (!commentFound) {
222 log.fine("PR is not yet ready - missing ready comment from '" + readyComment.getKey() +
223 "containing '" + readyComment.getValue().pattern() + "'");
224 return;
225 }
226 }
227 }
228
229 var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr);
230 var prInstance = new PullRequestInstance(scratchPath.resolve("mlbridge-mergebase"), pr);
231 var reviewArchive = new ReviewArchive(bot.emailAddress(), prInstance, census, sentMails);
232 var webrevPath = scratchPath.resolve("mlbridge-webrevs");
233 var listServer = MailingListServerFactory.createMailmanServer(bot.listArchive(), bot.smtpServer());
234 var list = listServer.getList(bot.listAddress().address());
235
236 // First post
237 if (sentMails.isEmpty()) {
238 log.fine("Creating new PR review archive");
239 var webrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
240 prInstance.headHash(), "00");
241 reviewArchive.create(webrev);
242 updateWebrevComment(comments, 0, webrev, null);
243 } else {
244 var latestHead = reviewArchive.latestHead();
245
246 // Check if the head has changed
247 if (!pr.getHeadHash().equals(latestHead)) {
248 log.fine("Head hash change detected: current: " + pr.getHeadHash() + " - last: " + latestHead);
249
250 var latestBase = reviewArchive.latestBase();
251 if (!prInstance.baseHash().equals(latestBase)) {
252 // FIXME: Could try harder to make an incremental
253 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
254 prInstance.headHash(), String.format("%02d", reviewArchive.revisionCount()));
255 reviewArchive.addFull(fullWebrev);
256 updateWebrevComment(comments, reviewArchive.revisionCount(), fullWebrev, null);
257 } else {
258 var index = reviewArchive.revisionCount();
259 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
260 prInstance.headHash(), String.format("%02d", index));
261 var incrementalWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, latestHead,
262 prInstance.headHash(), String.format("%02d-%02d", index - 1, index));
263 reviewArchive.addIncremental(fullWebrev, incrementalWebrev);
264 updateWebrevComment(comments, index, fullWebrev, incrementalWebrev);
265 }
266 }
267 }
268
269 // Regular comments
270 for (var comment : comments) {
271 if (ignoreComment(comment.author(), comment.body())) {
272 continue;
273 }
274 reviewArchive.addComment(comment);
275 }
276
277 // Review comments
278 var reviews = pr.getReviews();
279 for (var review : reviews) {
280 if (ignoreComment(review.reviewer(), review.body().orElse(""))) {
281 continue;
282 }
283 reviewArchive.addReview(review);
284 }
285
286 // File specific comments
287 var reviewComments = pr.getReviewComments();
288 for (var reviewComment : reviewComments) {
289 if (ignoreComment(reviewComment.author(), reviewComment.body())) {
290 continue;
291 }
292 reviewArchive.addReviewComment(reviewComment);
293 }
294
295 var newMails = reviewArchive.generatedEmails();
296 if (newMails.isEmpty()) {
297 return;
298 }
299
300 // Push all new mails to the archive repository
301 newMails.forEach(reviewArchiveList::post);
302 pushMbox(archiveRepo, "Adding comments for PR " + bot.codeRepo().getName() + "/" + pr.getId());
303
304 // Finally post all new mails to the actual list
305 for (var newMail : newMails) {
306 var filteredHeaders = newMail.headers().stream()
307 .filter(header -> !header.startsWith("PR-"))
308 .collect(Collectors.toMap(Function.identity(),
309 newMail::headerValue));
310 var filteredEmail = Email.from(newMail)
311 .replaceHeaders(filteredHeaders)
312 .build();
313 list.post(filteredEmail);
314 }
315 }
316
317 @Override
318 public void handleRuntimeException(RuntimeException e) {
319 exceptionConsumer.accept(e);
320 }
321 }