1 /* 2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.openjdk.skara.bots.mlbridge; 24 25 import org.openjdk.skara.bot.WorkItem; 26 import org.openjdk.skara.email.*; 27 import org.openjdk.skara.host.*; 28 import org.openjdk.skara.mailinglist.*; 29 import org.openjdk.skara.vcs.Repository; 30 31 import java.io.*; 32 import java.net.URI; 33 import java.nio.file.Path; 34 import java.time.Duration; 35 import java.util.*; 36 import java.util.function.*; 37 import java.util.logging.Logger; 38 import java.util.regex.Pattern; 39 import java.util.stream.Collectors; 40 41 class ArchiveWorkItem implements WorkItem { 42 private final PullRequest pr; 43 private final MailingListBridgeBot bot; 44 private final Consumer<RuntimeException> exceptionConsumer; 45 private final Logger log = Logger.getLogger("org.openjdk.skara.bots.mlbridge"); 46 47 ArchiveWorkItem(PullRequest pr, MailingListBridgeBot bot, Consumer<RuntimeException> exceptionConsumer) { 48 this.pr = pr; 49 this.bot = bot; 50 this.exceptionConsumer = exceptionConsumer; 51 } 52 53 @Override 54 public String toString() { 55 return "ArchiveWorkItem@" + bot.codeRepo().getName() + "#" + pr.getId(); 56 } 57 58 @Override 59 public boolean concurrentWith(WorkItem other) { 60 if (!(other instanceof ArchiveWorkItem)) { 61 return true; 62 } 63 ArchiveWorkItem otherItem = (ArchiveWorkItem)other; 64 if (!pr.getId().equals(otherItem.pr.getId())) { 65 return true; 66 } 67 if (!bot.codeRepo().getName().equals(otherItem.bot.codeRepo().getName())) { 68 return true; 69 } 70 return false; 71 } 72 73 private void pushMbox(Repository localRepo, String message) { 74 try { 75 localRepo.add(localRepo.root().resolve(".")); 76 var hash = localRepo.commit(message, bot.emailAddress().fullName().orElseThrow(), bot.emailAddress().address()); 77 localRepo.push(hash, bot.archiveRepo().getUrl(), "master"); 78 } catch (IOException e) { 79 throw new UncheckedIOException(e); 80 } 81 } 82 83 private static final Pattern replyToPattern = Pattern.compile("^\\s*@([-A-Za-z0-9]+)"); 84 85 private Optional<Comment> getParentPost(Comment post, List<Comment> all) { 86 var matcher = replyToPattern.matcher(post.body()); 87 if (matcher.find()) { 88 var replyToName = matcher.group(1); 89 var replyToNamePattern = Pattern.compile("^" + replyToName + "$"); 90 91 var postIterator = all.listIterator(); 92 while (postIterator.hasNext()) { 93 var cur = postIterator.next(); 94 if (cur == post) { 95 break; 96 } 97 } 98 99 while (postIterator.hasPrevious()) { 100 var cur = postIterator.previous(); 101 var userMatcher = replyToNamePattern.matcher(cur.author().userName()); 102 if (userMatcher.matches()) { 103 return Optional.of(cur); 104 } 105 } 106 } 107 108 return Optional.empty(); 109 } 110 111 private Repository materializeArchive(Path scratchPath) { 112 try { 113 return Repository.materialize(scratchPath, bot.archiveRepo().getUrl(), pr.getTargetRef()); 114 } catch (IOException e) { 115 throw new UncheckedIOException(e); 116 } 117 } 118 119 private final static Pattern commandPattern = Pattern.compile("^/.*$"); 120 121 private boolean ignoreComment(HostUserDetails author, String body) { 122 if (pr.repository().host().getCurrentUserDetails().equals(author)) { 123 return true; 124 } 125 if (bot.ignoredUsers().contains(author.userName())) { 126 return true; 127 } 128 var commandMatcher = commandPattern.matcher(body); 129 if (commandMatcher.matches()) { 130 return true; 131 } 132 for (var ignoredCommentPattern : bot.ignoredComments()) { 133 var ignoredCommentMatcher = ignoredCommentPattern.matcher(body); 134 if (ignoredCommentMatcher.find()) { 135 return true; 136 } 137 } 138 return false; 139 } 140 141 private static final String webrevCommentMarker = "<!-- mlbridge webrev comment -->"; 142 private static final String webrevHeaderMarker = "<!-- mlbridge webrev header -->"; 143 private static final String webrevListMarker = "<!-- mlbridge webrev list -->"; 144 145 private void updateWebrevComment(List<Comment> comments, int index, URI fullWebrev, URI incWebrev) { 146 var existing = comments.stream() 147 .filter(comment -> comment.author().equals(pr.repository().host().getCurrentUserDetails())) 148 .filter(comment -> comment.body().contains(webrevCommentMarker)) 149 .findAny(); 150 var comment = webrevCommentMarker + "\n"; 151 comment += webrevHeaderMarker + "\n"; 152 comment += "### Webrevs" + "\n"; 153 comment += webrevListMarker + "\n"; 154 comment += " * " + String.format("%02d", index) + ": [Full](" + fullWebrev.toString() + ")"; 155 if (incWebrev != null) { 156 comment += " - [Incremental](" + incWebrev.toString() + ")"; 157 } 158 comment += " (" + pr.getHeadHash() + ")\n"; 159 160 if (existing.isPresent()) { 161 if (existing.get().body().contains(fullWebrev.toString())) { 162 log.fine("Webrev link already posted - skipping update"); 163 return; 164 } 165 var previousListStart = existing.get().body().indexOf(webrevListMarker) + webrevListMarker.length() + 1; 166 var previousList = existing.get().body().substring(previousListStart); 167 comment += previousList; 168 pr.updateComment(existing.get().id(), comment); 169 } else { 170 pr.addComment(comment); 171 } 172 } 173 174 private List<Email> parseArchive(MailingList archive) { 175 var conversations = archive.conversations(Duration.ofDays(365)); 176 177 if (conversations.size() == 0) { 178 return new ArrayList<>(); 179 } else if (conversations.size() == 1) { 180 var conversation = conversations.get(0); 181 return conversation.allMessages(); 182 } else { 183 throw new RuntimeException("Something is wrong with the mbox"); 184 } 185 } 186 187 @Override 188 public void run(Path scratchPath) { 189 var path = scratchPath.resolve("mlbridge"); 190 var archiveRepo = materializeArchive(path); 191 var mboxBasePath = path.resolve(bot.codeRepo().getName()); 192 var mbox = MailingListServerFactory.createMboxFileServer(mboxBasePath); 193 var reviewArchiveList = mbox.getList(pr.getId()); 194 var sentMails = parseArchive(reviewArchiveList); 195 196 // First determine if this PR should be inspected further or not 197 if (sentMails.isEmpty()) { 198 var labels = new HashSet<>(pr.getLabels()); 199 for (var readyLabel : bot.readyLabels()) { 200 if (!labels.contains(readyLabel)) { 201 log.fine("PR is not yet ready - missing label '" + readyLabel + "'"); 202 return; 203 } 204 } 205 } 206 207 // Also inspect comments before making the first post 208 var comments = pr.getComments(); 209 if (sentMails.isEmpty()) { 210 for (var readyComment : bot.readyComments().entrySet()) { 211 var commentFound = false; 212 for (var comment : comments) { 213 if (comment.author().userName().equals(readyComment.getKey())) { 214 var matcher = readyComment.getValue().matcher(comment.body()); 215 if (matcher.find()) { 216 commentFound = true; 217 break; 218 } 219 } 220 } 221 if (!commentFound) { 222 log.fine("PR is not yet ready - missing ready comment from '" + readyComment.getKey() + 223 "containing '" + readyComment.getValue().pattern() + "'"); 224 return; 225 } 226 } 227 } 228 229 var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr); 230 var prInstance = new PullRequestInstance(scratchPath.resolve("mlbridge-mergebase"), pr); 231 var reviewArchive = new ReviewArchive(bot.emailAddress(), prInstance, census, sentMails); 232 var webrevPath = scratchPath.resolve("mlbridge-webrevs"); 233 var listServer = MailingListServerFactory.createMailmanServer(bot.listArchive(), bot.smtpServer()); 234 var list = listServer.getList(bot.listAddress().address()); 235 236 // First post 237 if (sentMails.isEmpty()) { 238 log.fine("Creating new PR review archive"); 239 var webrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(), 240 prInstance.headHash(), "00"); 241 reviewArchive.create(webrev); 242 updateWebrevComment(comments, 0, webrev, null); 243 } else { 244 var latestHead = reviewArchive.latestHead(); 245 246 // Check if the head has changed 247 if (!pr.getHeadHash().equals(latestHead)) { 248 log.fine("Head hash change detected: current: " + pr.getHeadHash() + " - last: " + latestHead); 249 250 var latestBase = reviewArchive.latestBase(); 251 if (!prInstance.baseHash().equals(latestBase)) { 252 // FIXME: Could try harder to make an incremental 253 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(), 254 prInstance.headHash(), String.format("%02d", reviewArchive.revisionCount())); 255 reviewArchive.addFull(fullWebrev); 256 updateWebrevComment(comments, reviewArchive.revisionCount(), fullWebrev, null); 257 } else { 258 var index = reviewArchive.revisionCount(); 259 var fullWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, prInstance.baseHash(), 260 prInstance.headHash(), String.format("%02d", index)); 261 var incrementalWebrev = bot.webrevStorage().createAndArchive(prInstance, webrevPath, latestHead, 262 prInstance.headHash(), String.format("%02d-%02d", index - 1, index)); 263 reviewArchive.addIncremental(fullWebrev, incrementalWebrev); 264 updateWebrevComment(comments, index, fullWebrev, incrementalWebrev); 265 } 266 } 267 } 268 269 // Regular comments 270 for (var comment : comments) { 271 if (ignoreComment(comment.author(), comment.body())) { 272 continue; 273 } 274 reviewArchive.addComment(comment); 275 } 276 277 // Review comments 278 var reviews = pr.getReviews(); 279 for (var review : reviews) { 280 if (ignoreComment(review.reviewer(), review.body().orElse(""))) { 281 continue; 282 } 283 reviewArchive.addReview(review); 284 } 285 286 // File specific comments 287 var reviewComments = pr.getReviewComments(); 288 for (var reviewComment : reviewComments) { 289 if (ignoreComment(reviewComment.author(), reviewComment.body())) { 290 continue; 291 } 292 reviewArchive.addReviewComment(reviewComment); 293 } 294 295 var newMails = reviewArchive.generatedEmails(); 296 if (newMails.isEmpty()) { 297 return; 298 } 299 300 // Push all new mails to the archive repository 301 newMails.forEach(reviewArchiveList::post); 302 pushMbox(archiveRepo, "Adding comments for PR " + bot.codeRepo().getName() + "/" + pr.getId()); 303 304 // Finally post all new mails to the actual list 305 for (var newMail : newMails) { 306 var filteredHeaders = newMail.headers().stream() 307 .filter(header -> !header.startsWith("PR-")) 308 .collect(Collectors.toMap(Function.identity(), 309 newMail::headerValue)); 310 var filteredEmail = Email.from(newMail) 311 .replaceHeaders(filteredHeaders) 312 .build(); 313 list.post(filteredEmail); 314 } 315 } 316 317 @Override 318 public void handleRuntimeException(RuntimeException e) { 319 exceptionConsumer.accept(e); 320 } 321 }