1 /*
2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.openjdk.skara.bots.mlbridge;
24
25 import org.openjdk.skara.bot.WorkItem;
26 import org.openjdk.skara.email.*;
27 import org.openjdk.skara.forge.PullRequest;
28 import org.openjdk.skara.host.*;
29 import org.openjdk.skara.issuetracker.Comment;
30 import org.openjdk.skara.mailinglist.*;
31 import org.openjdk.skara.vcs.Repository;
32
33 import java.io.*;
34 import java.net.URI;
35 import java.nio.file.Path;
36 import java.time.Duration;
37 import java.util.*;
38 import java.util.function.*;
39 import java.util.logging.Logger;
40 import java.util.regex.Pattern;
41 import java.util.stream.Collectors;
42
43 class ArchiveWorkItem implements WorkItem {
44 private final PullRequest pr;
45 private final MailingListBridgeBot bot;
46 private final Consumer<RuntimeException> exceptionConsumer;
47 private final Logger log = Logger.getLogger("org.openjdk.skara.bots.mlbridge");
48
49 ArchiveWorkItem(PullRequest pr, MailingListBridgeBot bot, Consumer<RuntimeException> exceptionConsumer) {
50 this.pr = pr;
51 this.bot = bot;
52 this.exceptionConsumer = exceptionConsumer;
53 }
54
55 @Override
56 public String toString() {
57 return "ArchiveWorkItem@" + bot.codeRepo().name() + "#" + pr.id();
58 }
59
60 @Override
61 public boolean concurrentWith(WorkItem other) {
62 if (!(other instanceof ArchiveWorkItem)) {
63 return true;
64 }
65 ArchiveWorkItem otherItem = (ArchiveWorkItem)other;
66 if (!pr.id().equals(otherItem.pr.id())) {
67 return true;
68 }
69 if (!bot.codeRepo().name().equals(otherItem.bot.codeRepo().name())) {
70 return true;
71 }
72 return false;
73 }
74
75 private void pushMbox(Repository localRepo, String message) {
76 try {
77 localRepo.add(localRepo.root().resolve("."));
78 var hash = localRepo.commit(message, bot.emailAddress().fullName().orElseThrow(), bot.emailAddress().address());
79 localRepo.push(hash, bot.archiveRepo().url(), "master");
80 } catch (IOException e) {
81 throw new UncheckedIOException(e);
82 }
83 }
84
85 private static final Pattern replyToPattern = Pattern.compile("^\\s*@([-A-Za-z0-9]+)");
86
87 private Optional<Comment> getParentPost(Comment post, List<Comment> all) {
88 var matcher = replyToPattern.matcher(post.body());
89 if (matcher.find()) {
90 var replyToName = matcher.group(1);
91 var replyToNamePattern = Pattern.compile("^" + replyToName + "$");
92
93 var postIterator = all.listIterator();
94 while (postIterator.hasNext()) {
95 var cur = postIterator.next();
96 if (cur == post) {
97 break;
98 }
99 }
100
101 while (postIterator.hasPrevious()) {
102 var cur = postIterator.previous();
103 var userMatcher = replyToNamePattern.matcher(cur.author().userName());
104 if (userMatcher.matches()) {
105 return Optional.of(cur);
106 }
107 }
108 }
109
110 return Optional.empty();
111 }
112
113 private Repository materializeArchive(Path scratchPath) {
114 try {
115 return Repository.materialize(scratchPath, bot.archiveRepo().url(),
116 "+" + bot.archiveRef() + ":mlbridge_archive");
117 } catch (IOException e) {
118 throw new UncheckedIOException(e);
119 }
120 }
121
122 private final static Pattern commandPattern = Pattern.compile("^/.*$");
123
124 private boolean ignoreComment(HostUser author, String body) {
125 if (pr.repository().forge().currentUser().equals(author)) {
126 return true;
127 }
128 if (bot.ignoredUsers().contains(author.userName())) {
129 return true;
130 }
131 var commandMatcher = commandPattern.matcher(body);
132 if (commandMatcher.matches()) {
133 return true;
134 }
135 for (var ignoredCommentPattern : bot.ignoredComments()) {
136 var ignoredCommentMatcher = ignoredCommentPattern.matcher(body);
137 if (ignoredCommentMatcher.find()) {
138 return true;
139 }
140 }
141 return false;
142 }
143
144 private static final String webrevCommentMarker = "<!-- mlbridge webrev comment -->";
145 private static final String webrevHeaderMarker = "<!-- mlbridge webrev header -->";
146 private static final String webrevListMarker = "<!-- mlbridge webrev list -->";
147
148 private void updateWebrevComment(List<Comment> comments, int index, URI fullWebrev, URI incWebrev) {
149 var existing = comments.stream()
150 .filter(comment -> comment.author().equals(pr.repository().forge().currentUser()))
151 .filter(comment -> comment.body().contains(webrevCommentMarker))
152 .findAny();
153 var comment = webrevCommentMarker + "\n";
154 comment += webrevHeaderMarker + "\n";
155 comment += "### Webrevs" + "\n";
156 comment += webrevListMarker + "\n";
157 comment += " * " + String.format("%02d", index) + ": [Full](" + fullWebrev.toString() + ")";
158 if (incWebrev != null) {
159 comment += " - [Incremental](" + incWebrev.toString() + ")";
160 }
161 comment += " (" + pr.headHash() + ")\n";
162
163 if (existing.isPresent()) {
164 if (existing.get().body().contains(fullWebrev.toString())) {
165 log.fine("Webrev link already posted - skipping update");
166 return;
167 }
168 var previousListStart = existing.get().body().indexOf(webrevListMarker) + webrevListMarker.length() + 1;
169 var previousList = existing.get().body().substring(previousListStart);
170 comment += previousList;
171 pr.updateComment(existing.get().id(), comment);
172 } else {
173 pr.addComment(comment);
174 }
175 }
176
177 private List<Email> parseArchive(MailingList archive) {
178 var conversations = archive.conversations(Duration.ofDays(365));
179
180 if (conversations.size() == 0) {
181 return new ArrayList<>();
182 } else if (conversations.size() == 1) {
183 var conversation = conversations.get(0);
184 return conversation.allMessages();
185 } else {
186 throw new RuntimeException("Something is wrong with the mbox");
187 }
188 }
189
190 @Override
191 public void run(Path scratchPath) {
192 var path = scratchPath.resolve("mlbridge");
193 var archiveRepo = materializeArchive(path);
194 var mboxBasePath = path.resolve(bot.codeRepo().name());
195 var mbox = MailingListServerFactory.createMboxFileServer(mboxBasePath);
196 var reviewArchiveList = mbox.getList(pr.id());
197 var sentMails = parseArchive(reviewArchiveList);
198
199 // First determine if this PR should be inspected further or not
200 if (sentMails.isEmpty()) {
201 var labels = new HashSet<>(pr.labels());
202 for (var readyLabel : bot.readyLabels()) {
203 if (!labels.contains(readyLabel)) {
204 log.fine("PR is not yet ready - missing label '" + readyLabel + "'");
205 return;
206 }
207 }
208 }
209
210 // Also inspect comments before making the first post
211 var comments = pr.comments();
212 if (sentMails.isEmpty()) {
213 for (var readyComment : bot.readyComments().entrySet()) {
214 var commentFound = false;
215 for (var comment : comments) {
216 if (comment.author().userName().equals(readyComment.getKey())) {
217 var matcher = readyComment.getValue().matcher(comment.body());
218 if (matcher.find()) {
219 commentFound = true;
220 break;
221 }
222 }
223 }
224 if (!commentFound) {
225 log.fine("PR is not yet ready - missing ready comment from '" + readyComment.getKey() +
226 "containing '" + readyComment.getValue().pattern() + "'");
227 return;
228 }
229 }
230 }
231
232 var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr);
233 var jbs = census.configuration().general().jbs();
234 if (jbs == null) {
235 jbs = census.configuration().general().project();
236 }
237 var prInstance = new PullRequestInstance(scratchPath.resolve("mlbridge-mergebase"), pr, bot.issueTracker(),
238 jbs.toUpperCase());
239 var reviewArchive = new ReviewArchive(bot.emailAddress(), prInstance, census, sentMails);
240 var webrevPath = scratchPath.resolve("mlbridge-webrevs");
241 var listServer = MailingListServerFactory.createMailmanServer(bot.listArchive(), bot.smtpServer(), bot.sendInterval());
242 var list = listServer.getList(bot.listAddress().address());
243
244 // First post
245 if (sentMails.isEmpty()) {
246 log.fine("Creating new PR review archive");
247 var webrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
248 prInstance.headHash(), "00");
249 reviewArchive.create(webrev);
250 updateWebrevComment(comments, 0, webrev, null);
251 } else {
252 var latestHead = reviewArchive.latestHead();
253
254 // Check if the head has changed
255 if (!pr.headHash().equals(latestHead)) {
256 log.fine("Head hash change detected: current: " + pr.headHash() + " - last: " + latestHead);
257
258 var latestBase = reviewArchive.latestBase();
259 if (!prInstance.baseHash().equals(latestBase)) {
260 // FIXME: Could try harder to make an incremental
261 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
262 prInstance.headHash(), String.format("%02d", reviewArchive.revisionCount()));
263 reviewArchive.addFull(fullWebrev);
264 updateWebrevComment(comments, reviewArchive.revisionCount(), fullWebrev, null);
265 } else {
266 var index = reviewArchive.revisionCount();
267 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
268 prInstance.headHash(), String.format("%02d", index));
269 var incrementalWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, latestHead,
270 prInstance.headHash(), String.format("%02d-%02d", index - 1, index));
271 reviewArchive.addIncremental(fullWebrev, incrementalWebrev);
272 updateWebrevComment(comments, index, fullWebrev, incrementalWebrev);
273 }
274 }
275 }
276
277 // Regular comments
278 for (var comment : comments) {
279 if (ignoreComment(comment.author(), comment.body())) {
280 continue;
281 }
282 reviewArchive.addComment(comment);
283 }
284
285 // Review comments
286 var reviews = pr.reviews();
287 for (var review : reviews) {
288 if (ignoreComment(review.reviewer(), review.body().orElse(""))) {
289 continue;
290 }
291 reviewArchive.addReview(review);
292 }
293
294 // File specific comments
295 var reviewComments = pr.reviewComments();
296 for (var reviewComment : reviewComments) {
297 if (ignoreComment(reviewComment.author(), reviewComment.body())) {
298 continue;
299 }
300 reviewArchive.addReviewComment(reviewComment);
301 }
302
303 // Review verdict comments
304 for (var review : reviews) {
305 if (ignoreComment(review.reviewer(), review.body().orElse(""))) {
306 continue;
307 }
308 reviewArchive.addReviewVerdict(review);
309 }
310
311 var newMails = reviewArchive.generatedEmails();
312 if (newMails.isEmpty()) {
313 return;
314 }
315
316 // Push all new mails to the archive repository
317 newMails.forEach(reviewArchiveList::post);
318 pushMbox(archiveRepo, "Adding comments for PR " + bot.codeRepo().name() + "/" + pr.id());
319
320 // Finally post all new mails to the actual list
321 for (var newMail : newMails) {
322 var filteredHeaders = newMail.headers().stream()
323 .filter(header -> !header.startsWith("PR-"))
324 .collect(Collectors.toMap(Function.identity(),
325 newMail::headerValue));
326 var filteredEmail = Email.from(newMail)
327 .replaceHeaders(filteredHeaders)
328 .headers(bot.headers())
329 .build();
330 list.post(filteredEmail);
331 }
332 }
333
334 @Override
335 public void handleRuntimeException(RuntimeException e) {
336 exceptionConsumer.accept(e);
337 }
338 }