1 /*
  2  * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 package org.openjdk.skara.bots.mlbridge;
 24 
 25 import org.openjdk.skara.bot.WorkItem;
 26 import org.openjdk.skara.email.*;
 27 import org.openjdk.skara.host.*;
 28 import org.openjdk.skara.mailinglist.*;
 29 import org.openjdk.skara.vcs.Repository;
 30 
 31 import java.io.*;
 32 import java.net.URI;
 33 import java.nio.file.Path;
 34 import java.time.Duration;
 35 import java.util.*;
 36 import java.util.function.*;
 37 import java.util.logging.Logger;
 38 import java.util.regex.Pattern;
 39 import java.util.stream.Collectors;
 40 
 41 class ArchiveWorkItem implements WorkItem {
 42     private final PullRequest pr;
 43     private final MailingListBridgeBot bot;
 44     private final Consumer<RuntimeException> exceptionConsumer;
 45     private final Logger log = Logger.getLogger("org.openjdk.skara.bots.mlbridge");
 46 
 47     ArchiveWorkItem(PullRequest pr, MailingListBridgeBot bot, Consumer<RuntimeException> exceptionConsumer) {
 48         this.pr = pr;
 49         this.bot = bot;
 50         this.exceptionConsumer = exceptionConsumer;
 51     }
 52 
 53     @Override
 54     public String toString() {
 55         return "ArchiveWorkItem@" + bot.codeRepo().getName() + "#" + pr.getId();
 56     }
 57 
 58     @Override
 59     public boolean concurrentWith(WorkItem other) {
 60         if (!(other instanceof ArchiveWorkItem)) {
 61             return true;
 62         }
 63         ArchiveWorkItem otherItem = (ArchiveWorkItem)other;
 64         if (!pr.getId().equals(otherItem.pr.getId())) {
 65             return true;
 66         }
 67         if (!bot.codeRepo().getName().equals(otherItem.bot.codeRepo().getName())) {
 68             return true;
 69         }
 70         return false;
 71     }
 72 
 73     private void pushMbox(Repository localRepo, String message) {
 74         try {
 75             localRepo.add(localRepo.root().resolve("."));
 76             var hash = localRepo.commit(message, bot.emailAddress().fullName().orElseThrow(), bot.emailAddress().address());
 77             localRepo.push(hash, bot.archiveRepo().getUrl(), "master");
 78         } catch (IOException e) {
 79             throw new UncheckedIOException(e);
 80         }
 81     }
 82 
 83     private static final Pattern replyToPattern = Pattern.compile("^\\s*@([-A-Za-z0-9]+)");
 84 
 85     private Optional<Comment> getParentPost(Comment post, List<Comment> all) {
 86         var matcher = replyToPattern.matcher(post.body());
 87         if (matcher.find()) {
 88             var replyToName = matcher.group(1);
 89             var replyToNamePattern = Pattern.compile("^" + replyToName + "$");
 90 
 91             var postIterator = all.listIterator();
 92             while (postIterator.hasNext()) {
 93                 var cur = postIterator.next();
 94                 if (cur == post) {
 95                     break;
 96                 }
 97             }
 98 
 99             while (postIterator.hasPrevious()) {
100                 var cur = postIterator.previous();
101                 var userMatcher = replyToNamePattern.matcher(cur.author().userName());
102                 if (userMatcher.matches()) {
103                     return Optional.of(cur);
104                 }
105             }
106         }
107 
108         return Optional.empty();
109     }
110 
111     private Repository materializeArchive(Path scratchPath) {
112         try {
113             return Repository.materialize(scratchPath, bot.archiveRepo().getUrl(), pr.getTargetRef());
114         } catch (IOException e) {
115             throw new UncheckedIOException(e);
116         }
117     }
118 
119     private final static Pattern commandPattern = Pattern.compile("^/.*$");
120 
121     private boolean ignoreComment(HostUserDetails author, String body) {
122         if (pr.repository().host().getCurrentUserDetails().equals(author)) {
123             return true;
124         }
125         if (bot.ignoredUsers().contains(author.userName())) {
126             return true;
127         }
128         var commandMatcher = commandPattern.matcher(body);
129         if (commandMatcher.matches()) {
130             return true;
131         }
132         for (var ignoredCommentPattern : bot.ignoredComments()) {
133             var ignoredCommentMatcher = ignoredCommentPattern.matcher(body);
134             if (ignoredCommentMatcher.find()) {
135                 return true;
136             }
137         }
138         return false;
139     }
140 
141     private static final String webrevCommentMarker = "<!-- mlbridge webrev comment -->";
142     private static final String webrevHeaderMarker = "<!-- mlbridge webrev header -->";
143     private static final String webrevListMarker = "<!-- mlbridge webrev list -->";
144 
145     private void updateWebrevComment(List<Comment> comments, int index, URI fullWebrev, URI incWebrev) {
146         var existing = comments.stream()
147                                .filter(comment -> comment.author().equals(pr.repository().host().getCurrentUserDetails()))
148                                .filter(comment -> comment.body().contains(webrevCommentMarker))
149                                .findAny();
150         var comment = webrevCommentMarker + "\n";
151         comment += webrevHeaderMarker + "\n";
152         comment += "### Webrevs" + "\n";
153         comment += webrevListMarker + "\n";
154         comment += " * " + String.format("%02d", index) + ": [Full](" + fullWebrev.toString() + ")";
155         if (incWebrev != null) {
156             comment += " - [Incremental](" + incWebrev.toString() + ")";
157         }
158         comment += " (" + pr.getHeadHash() + ")\n";
159 
160         if (existing.isPresent()) {
161             if (existing.get().body().contains(fullWebrev.toString())) {
162                 log.fine("Webrev link already posted - skipping update");
163                 return;
164             }
165             var previousListStart = existing.get().body().indexOf(webrevListMarker) + webrevListMarker.length() + 1;
166             var previousList = existing.get().body().substring(previousListStart);
167             comment += previousList;
168             pr.updateComment(existing.get().id(), comment);
169         } else {
170             pr.addComment(comment);
171         }
172     }
173 
174     private List<Email> parseArchive(MailingList archive) {
175         var conversations = archive.conversations(Duration.ofDays(365));
176 
177         if (conversations.size() == 0) {
178             return new ArrayList<>();
179         } else if (conversations.size() == 1) {
180             var conversation = conversations.get(0);
181             return conversation.allMessages();
182         } else {
183             throw new RuntimeException("Something is wrong with the mbox");
184         }
185     }
186 
187     @Override
188     public void run(Path scratchPath) {
189         var path = scratchPath.resolve("mlbridge");
190         var archiveRepo = materializeArchive(path);
191         var mboxBasePath = path.resolve(bot.codeRepo().getName());
192         var mbox = MailingListServerFactory.createMboxFileServer(mboxBasePath);
193         var reviewArchiveList = mbox.getList(pr.getId());
194         var sentMails = parseArchive(reviewArchiveList);
195 
196         // First determine if this PR should be inspected further or not
197         if (sentMails.isEmpty()) {
198             var labels = new HashSet<>(pr.getLabels());
199             for (var readyLabel : bot.readyLabels()) {
200                 if (!labels.contains(readyLabel)) {
201                     log.fine("PR is not yet ready - missing label '" + readyLabel + "'");
202                     return;
203                 }
204             }
205         }
206 
207         // Also inspect comments before making the first post
208         var comments = pr.getComments();
209         if (sentMails.isEmpty()) {
210             for (var readyComment : bot.readyComments().entrySet()) {
211                 var commentFound = false;
212                 for (var comment : comments) {
213                     if (comment.author().userName().equals(readyComment.getKey())) {
214                         var matcher = readyComment.getValue().matcher(comment.body());
215                         if (matcher.find()) {
216                             commentFound = true;
217                             break;
218                         }
219                     }
220                 }
221                 if (!commentFound) {
222                     log.fine("PR is not yet ready - missing ready comment from '" + readyComment.getKey() +
223                                      "containing '" + readyComment.getValue().pattern() + "'");
224                     return;
225                 }
226             }
227         }
228 
229         var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr);
230         var prInstance = new PullRequestInstance(scratchPath.resolve("mlbridge-mergebase"), pr);
231         var reviewArchive = new ReviewArchive(bot.emailAddress(), prInstance, census, sentMails);
232         var webrevPath = scratchPath.resolve("mlbridge-webrevs");
233         var listServer = MailingListServerFactory.createMailmanServer(bot.listArchive(), bot.smtpServer());
234         var list = listServer.getList(bot.listAddress().address());
235 
236         // First post
237         if (sentMails.isEmpty()) {
238             log.fine("Creating new PR review archive");
239             var webrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
240                                                               prInstance.headHash(), "00");
241             reviewArchive.create(webrev);
242             updateWebrevComment(comments, 0, webrev, null);
243         } else {
244             var latestHead = reviewArchive.latestHead();
245 
246             // Check if the head has changed
247             if (!pr.getHeadHash().equals(latestHead)) {
248                 log.fine("Head hash change detected: current: " + pr.getHeadHash() + " - last: " + latestHead);
249 
250                 var latestBase = reviewArchive.latestBase();
251                 if (!prInstance.baseHash().equals(latestBase)) {
252                     // FIXME: Could try harder to make an incremental
253                     var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
254                                                                           prInstance.headHash(), String.format("%02d", reviewArchive.revisionCount()));
255                     reviewArchive.addFull(fullWebrev);
256                     updateWebrevComment(comments, reviewArchive.revisionCount(), fullWebrev, null);
257                 } else {
258                     var index = reviewArchive.revisionCount();
259                     var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(),
260                                                                           prInstance.headHash(), String.format("%02d", index));
261                     var incrementalWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, latestHead,
262                                                                                  prInstance.headHash(), String.format("%02d-%02d", index - 1, index));
263                     reviewArchive.addIncremental(fullWebrev, incrementalWebrev);
264                     updateWebrevComment(comments, index, fullWebrev, incrementalWebrev);
265                 }
266             }
267         }
268 
269         // Regular comments
270         for (var comment : comments) {
271             if (ignoreComment(comment.author(), comment.body())) {
272                 continue;
273             }
274             reviewArchive.addComment(comment);
275         }
276 
277         // Review comments
278         var reviews = pr.getReviews();
279         for (var review : reviews) {
280             if (ignoreComment(review.reviewer(), review.body().orElse(""))) {
281                 continue;
282             }
283             reviewArchive.addReview(review);
284         }
285 
286         // File specific comments
287         var reviewComments = pr.getReviewComments();
288         for (var reviewComment : reviewComments) {
289             if (ignoreComment(reviewComment.author(), reviewComment.body())) {
290                 continue;
291             }
292             reviewArchive.addReviewComment(reviewComment);
293         }
294 
295         var newMails = reviewArchive.generatedEmails();
296         if (newMails.isEmpty()) {
297             return;
298         }
299 
300         // Push all new mails to the archive repository
301         newMails.forEach(reviewArchiveList::post);
302         pushMbox(archiveRepo, "Adding comments for PR " + bot.codeRepo().getName() + "/" + pr.getId());
303 
304         // Finally post all new mails to the actual list
305         for (var newMail : newMails) {
306             var filteredHeaders = newMail.headers().stream()
307                                          .filter(header -> !header.startsWith("PR-"))
308                                          .collect(Collectors.toMap(Function.identity(),
309                                                                    newMail::headerValue));
310             var filteredEmail = Email.from(newMail)
311                                      .replaceHeaders(filteredHeaders)
312                                      .build();
313             list.post(filteredEmail);
314         }
315     }
316 
317     @Override
318     public void handleRuntimeException(RuntimeException e) {
319         exceptionConsumer.accept(e);
320     }
321 }