1 /*
2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.openjdk.skara.bots.pr;
24
25 import org.openjdk.skara.forge.*;
26 import org.openjdk.skara.host.*;
27 import org.openjdk.skara.issuetracker.*;
28 import org.openjdk.skara.vcs.*;
29 import org.openjdk.skara.vcs.openjdk.Issue;
30
31 import java.io.*;
32 import java.util.*;
33 import java.util.logging.Logger;
34 import java.util.regex.Pattern;
35 import java.util.stream.*;
36
37 class CheckRun {
38 private final CheckWorkItem workItem;
39 private final PullRequest pr;
40 private final PullRequestInstance prInstance;
41 private final List<Comment> comments;
42 private final List<Review> allReviews;
43 private final List<Review> activeReviews;
44 private final Set<String> labels;
45 private final CensusInstance censusInstance;
46 private final Map<String, String> blockingLabels;
47 private final IssueProject issueProject;
48
49 private final Logger log = Logger.getLogger("org.openjdk.skara.bots.pr");
50 private final String progressMarker = "<!-- Anything below this marker will be automatically updated, please do not edit manually! -->";
51 private final String mergeReadyMarker = "<!-- PullRequestBot merge is ready comment -->";
52 private final Pattern mergeSourcePattern = Pattern.compile("^Merge ([-/\\w]+):([-\\w]+$)");
53 private final Set<String> newLabels;
54
55 private CheckRun(CheckWorkItem workItem, PullRequest pr, PullRequestInstance prInstance, List<Comment> comments,
56 List<Review> allReviews, List<Review> activeReviews, Set<String> labels,
57 CensusInstance censusInstance, Map<String, String> blockingLabels, IssueProject issueProject) {
58 this.workItem = workItem;
59 this.pr = pr;
60 this.prInstance = prInstance;
61 this.comments = comments;
62 this.allReviews = allReviews;
63 this.activeReviews = activeReviews;
64 this.labels = new HashSet<>(labels);
65 this.newLabels = new HashSet<>(labels);
66 this.censusInstance = censusInstance;
67 this.blockingLabels = blockingLabels;
68 this.issueProject = issueProject;
69 }
70
71 static void execute(CheckWorkItem workItem, PullRequest pr, PullRequestInstance prInstance, List<Comment> comments,
72 List<Review> allReviews, List<Review> activeReviews, Set<String> labels, CensusInstance censusInstance, Map<String, String> blockingLabels,
73 IssueProject issueProject) {
74 var run = new CheckRun(workItem, pr, prInstance, comments, allReviews, activeReviews, labels, censusInstance, blockingLabels, issueProject);
75 run.checkStatus();
76 }
77
78 // For unknown contributors, check that all commits have the same name and email
79 private boolean checkCommitAuthor(List<Commit> commits) throws IOException {
80 var author = censusInstance.namespace().get(pr.author().id());
81 if (author != null) {
82 return true;
83 }
84
85 var names = new HashSet<String>();
86 var emails = new HashSet<String>();
87
88 for (var commit : commits) {
89 names.add(commit.author().name());
90 emails.add(commit.author().email());
91 }
92
93 return ((names.size() == 1) && emails.size() == 1);
94 }
95
96 private Optional<String> mergeSourceRepository() {
97 var repoMatcher = mergeSourcePattern.matcher(pr.title());
98 if (!repoMatcher.matches()) {
99 return Optional.empty();
100 }
101 return Optional.of(repoMatcher.group(1));
102 }
103
104 private Optional<String> mergeSourceBranch() {
105 var branchMatcher = mergeSourcePattern.matcher(pr.title());
106 if (!branchMatcher.matches()) {
107 return Optional.empty();
108 }
109 var mergeSourceBranch = branchMatcher.group(2);
110 return Optional.of(mergeSourceBranch);
111 }
112
113 // Additional bot-specific checks that are not handled by JCheck
114 private List<String> botSpecificChecks() throws IOException {
115 var ret = new ArrayList<String>();
116
117 var baseHash = prInstance.baseHash();
118 var headHash = pr.headHash();
119 var commits = prInstance.localRepo().commits(baseHash + ".." + headHash).asList();
120
121 if (!checkCommitAuthor(commits)) {
122 var error = "For contributors who are not existing OpenJDK Authors, commit attribution will be taken from " +
123 "the commits in the PR. However, the commits in this PR have inconsistent user names and/or " +
124 "email addresses. Please amend the commits.";
125 ret.add(error);
126 }
127
128 if (pr.title().startsWith("Merge")) {
129 if (commits.size() < 2) {
130 ret.add("A Merge PR must contain at least two commits that are not already present in the target.");
131 } else {
132 if (!commits.get(0).isMerge()) {
133 ret.add("The top commit must be a merge commit.");
134 }
135
136 var sourceRepo = mergeSourceRepository();
137 var sourceBranch = mergeSourceBranch();
138 if (sourceBranch.isPresent() && sourceRepo.isPresent()) {
139 try {
140 var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()).orElseThrow(() ->
141 new RuntimeException("Could not find repository " + sourceRepo.get())
142 );
143 try {
144 var sourceHash = prInstance.localRepo().fetch(mergeSourceRepo.url(), sourceBranch.get());
145 if (!prInstance.localRepo().isAncestor(commits.get(1).hash(), sourceHash)) {
146 ret.add("The merge contains commits that are not ancestors of the source");
147 }
148 } catch (IOException e) {
149 ret.add("Could not fetch branch `" + sourceBranch.get() + "` from project `" +
150 sourceRepo.get() + "` - check that they are correct.");
151 }
152 } catch (RuntimeException e) {
153 ret.add("Could not find project `" +
154 sourceRepo.get() + "` - check that it is correct.");
155 }
156 } else {
157 ret.add("Could not determine the source for this merge. A Merge PR title must be specified on the format: " +
158 "Merge `project`:`branch` to allow verification of the merge contents.");
159 }
160 }
161 }
162
163 for (var blocker : blockingLabels.entrySet()) {
164 if (labels.contains(blocker.getKey())) {
165 ret.add(blocker.getValue());
166 }
167 }
168
169 return ret;
170 }
171
172 private void updateCheckBuilder(CheckBuilder checkBuilder, PullRequestCheckIssueVisitor visitor, List<String> additionalErrors) {
173 if (visitor.isReadyForReview() && additionalErrors.isEmpty()) {
174 checkBuilder.complete(true);
175 } else {
176 checkBuilder.title("Required");
177 var summary = Stream.concat(visitor.getMessages().stream(), additionalErrors.stream())
178 .sorted()
179 .map(m -> "- " + m)
180 .collect(Collectors.joining("\n"));
181 checkBuilder.summary(summary);
182 for (var annotation : visitor.getAnnotations()) {
183 checkBuilder.annotation(annotation);
184 }
185 checkBuilder.complete(false);
186 }
187 }
188
189 private void updateReadyForReview(PullRequestCheckIssueVisitor visitor, List<String> additionalErrors) {
190 // If there are no issues at all, the PR is already reviewed
191 if (visitor.getMessages().isEmpty() && additionalErrors.isEmpty()) {
192 pr.removeLabel("rfr");
193 return;
194 }
195
196 // Additional errors are not allowed
197 if (!additionalErrors.isEmpty()) {
198 newLabels.remove("rfr");
199 return;
200 }
201
202 // Draft requests are not for review
203 if (pr.isDraft()) {
204 newLabels.remove("rfr");
205 return;
206 }
207
208 // Check if the visitor found any issues that should be resolved before reviewing
209 if (visitor.isReadyForReview()) {
210 newLabels.add("rfr");
211 } else {
212 newLabels.remove("rfr");
213 }
214 }
215
216 private String getRole(String username) {
217 var project = censusInstance.project();
218 var version = censusInstance.census().version().format();
219 if (project.isReviewer(username, version)) {
220 return "**Reviewer**";
221 } else if (project.isCommitter(username, version)) {
222 return "Committer";
223 } else if (project.isAuthor(username, version)) {
224 return "Author";
225 } else {
226 return "no project role";
227 }
228 }
229
230 private String formatReviewer(HostUser reviewer) {
231 var namespace = censusInstance.namespace();
232 var contributor = namespace.get(reviewer.id());
233 if (contributor == null) {
234 return reviewer.userName() + " (no known " + namespace.name() + " user name / role)";
235 } else {
236 var userNameLink = "[" + contributor.username() + "](@" + reviewer.userName() + ")";
237 return contributor.fullName().orElse(contributor.username()) + " (" + userNameLink + " - " +
238 getRole(contributor.username()) + ")";
239 }
240 }
241
242 private String getChecksList(PullRequestCheckIssueVisitor visitor) {
243 return visitor.getChecks().entrySet().stream()
244 .map(entry -> "- [" + (entry.getValue() ? "x" : " ") + "] " + entry.getKey())
245 .collect(Collectors.joining("\n"));
246 }
247
248 private Optional<String> getReviewersList(List<Review> reviews) {
249 var reviewers = reviews.stream()
250 .filter(review -> review.verdict() == Review.Verdict.APPROVED)
251 .map(review -> {
252 var entry = " * " + formatReviewer(review.reviewer());
253 if (!review.hash().equals(pr.headHash())) {
254 entry += " **Note!** Review applies to " + review.hash();
255 }
256 return entry;
257 })
258 .collect(Collectors.joining("\n"));
259 if (reviewers.length() > 0) {
260 return Optional.of(reviewers);
261 } else {
262 return Optional.empty();
263 }
264 }
265
266 private String getStatusMessage(List<Comment> comments, List<Review> reviews, PullRequestCheckIssueVisitor visitor) {
267 var progressBody = new StringBuilder();
268 progressBody.append("## Progress\n");
269 progressBody.append(getChecksList(visitor));
270
271 var issue = Issue.fromString(pr.title());
272 if (issueProject != null && issue.isPresent()) {
273 var allIssues = new ArrayList<Issue>();
274 allIssues.add(issue.get());
275 allIssues.addAll(SolvesTracker.currentSolved(pr.repository().forge().currentUser(), comments));
276 progressBody.append("\n\n## Issue");
277 if (allIssues.size() > 1) {
278 progressBody.append("s");
279 }
280 progressBody.append("\n");
281 for (var currentIssue : allIssues) {
282 var iss = issueProject.issue(currentIssue.id());
283 if (iss.isPresent()) {
284 progressBody.append("[");
285 progressBody.append(iss.get().id());
286 progressBody.append("](");
287 progressBody.append(iss.get().webUrl());
288 progressBody.append("): ");
289 progressBody.append(iss.get().title());
290 progressBody.append("\n");
291 } else {
292 progressBody.append("⚠️ Failed to retrieve information on issue `");
293 progressBody.append(currentIssue.id());
294 progressBody.append("`.\n");
295 }
296 }
297 }
298
299 getReviewersList(reviews).ifPresent(reviewers -> {
300 progressBody.append("\n\n## Approvers\n");
301 progressBody.append(reviewers);
302 });
303
304 return progressBody.toString();
305 }
306
307 private String updateStatusMessage(String message) {
308 var description = pr.body();
309 var markerIndex = description.lastIndexOf(progressMarker);
310
311 if (markerIndex >= 0 && description.substring(markerIndex).equals(message)) {
312 log.info("Progress already up to date");
313 return description;
314 }
315 var newBody = (markerIndex < 0 ?
316 description :
317 description.substring(0, markerIndex)).trim() + "\n" + progressMarker + "\n" + message;
318
319 // TODO? Retrieve the body again here to lower the chance of concurrent updates
320 pr.setBody(newBody);
321 return newBody;
322 }
323
324 private String verdictToString(Review.Verdict verdict) {
325 switch (verdict) {
326 case APPROVED:
327 return "changes are approved";
328 case DISAPPROVED:
329 return "more changes needed";
330 case NONE:
331 return "comment added";
332 default:
333 throw new RuntimeException("Unknown verdict: " + verdict);
334 }
335 }
336
337 private void updateReviewedMessages(List<Comment> comments, List<Review> reviews) {
338 var reviewTracker = new ReviewTracker(comments, reviews);
339
340 for (var added : reviewTracker.newReviews().entrySet()) {
341 var body = added.getValue() + "\n" +
342 "This PR has been reviewed by " +
343 formatReviewer(added.getKey().reviewer()) + " - " +
344 verdictToString(added.getKey().verdict()) + ".";
345 pr.addComment(body);
346 }
347 }
348
349 private Optional<Comment> findComment(List<Comment> comments, String marker) {
350 var self = pr.repository().forge().currentUser();
351 return comments.stream()
352 .filter(comment -> comment.author().equals(self))
353 .filter(comment -> comment.body().contains(marker))
354 .findAny();
355 }
356
357 private String getMergeReadyComment(String commitMessage, List<Review> reviews, boolean rebasePossible) {
358 var message = new StringBuilder();
359 message.append("@");
360 message.append(pr.author().userName());
361 message.append(" This change can now be integrated. The commit message will be:\n");
362 message.append("```\n");
363 message.append(commitMessage);
364 message.append("\n```\n");
365
366 message.append("- If you would like to add a summary, use the `/summary` command.\n");
367 message.append("- To list additional contributors, use the `/contributor` command.\n");
368
369 var divergingCommits = prInstance.divergingCommits();
370 if (divergingCommits.size() > 0) {
371 message.append("\n");
372 message.append("Since the source branch of this PR was last updated there ");
373 if (divergingCommits.size() == 1) {
374 message.append("has been 1 commit ");
375 } else {
376 message.append("have been ");
377 message.append(divergingCommits.size());
378 message.append(" commits ");
379 }
380 message.append("pushed to the `");
381 message.append(pr.targetRef());
382 message.append("` branch:\n");
383 var commitList = divergingCommits.stream()
384 .map(commit -> " * " + commit.hash().hex() + ": " + commit.message().get(0))
385 .collect(Collectors.joining("\n"));
386 message.append(commitList);
387 message.append("\n\n");
388 if (rebasePossible) {
389 message.append("Since there are no conflicts, your changes will automatically be rebased on top of the ");
390 message.append("above commits when integrating. If you prefer to do this manually, please merge `");
391 message.append(pr.targetRef());
392 message.append("` into your branch first.\n");
393 } else {
394 message.append("Your changes cannot be rebased automatically without conflicts, so you will need to ");
395 message.append("merge `");
396 message.append(pr.targetRef());
397 message.append("` into your branch before integrating.\n");
398 }
399 }
400
401 if (!ProjectPermissions.mayCommit(censusInstance, pr.author())) {
402 message.append("\n");
403 var contributor = censusInstance.namespace().get(pr.author().id());
404 if (contributor == null) {
405 message.append("As you are not a known OpenJDK [Author](http://openjdk.java.net/bylaws#author), ");
406 } else {
407 message.append("As you do not have Committer status in this project, ");
408 }
409
410 message.append("an existing [Committer](http://openjdk.java.net/bylaws#committer) must agree to ");
411 message.append("[sponsor](http://openjdk.java.net/sponsor/) your change. ");
412 var candidates = reviews.stream()
413 .filter(review -> ProjectPermissions.mayCommit(censusInstance, review.reviewer()))
414 .map(review -> "@" + review.reviewer().userName())
415 .collect(Collectors.joining(", "));
416 if (candidates.length() > 0) {
417 message.append("Possible candidates are the reviewers of this PR (");
418 message.append(candidates);
419 message.append(") but any other Committer may sponsor as well. ");
420 }
421 if (rebasePossible) {
422 message.append("\n\n");
423 message.append("- To flag this PR as ready for integration with the above commit message, type ");
424 message.append("`/integrate` in a new comment. (Afterwards, your sponsor types ");
425 message.append("`/sponsor` in a new comment to perform the integration).\n");
426 }
427 } else if (rebasePossible) {
428 if (divergingCommits.size() > 0) {
429 message.append("\n");
430 }
431 message.append("- To integrate this PR with the above commit message, type ");
432 message.append("`/integrate` in a new comment.\n");
433 }
434 message.append(mergeReadyMarker);
435 return message.toString();
436 }
437
438 private String getMergeNoLongerReadyComment() {
439 var message = new StringBuilder();
440 message.append("@");
441 message.append(pr.author().userName());
442 message.append(" This change is no longer ready for integration - check the PR body for details.\n");
443 message.append(mergeReadyMarker);
444 return message.toString();
445 }
446
447 private void updateMergeReadyComment(boolean isReady, String commitMessage, List<Comment> comments, List<Review> reviews, boolean rebasePossible) {
448 var existing = findComment(comments, mergeReadyMarker);
449 if (isReady) {
450 var message = getMergeReadyComment(commitMessage, reviews, rebasePossible);
451 if (existing.isEmpty()) {
452 pr.addComment(message);
453 } else {
454 pr.updateComment(existing.get().id(), message);
455 }
456 } else {
457 existing.ifPresent(comment -> pr.updateComment(comment.id(), getMergeNoLongerReadyComment()));
458 }
459 }
460
461 private void checkStatus() {
462 var checkBuilder = CheckBuilder.create("jcheck", pr.headHash());
463 var censusDomain = censusInstance.configuration().census().domain();
464 Exception checkException = null;
465
466 try {
467 // Post check in-progress
468 log.info("Starting to run jcheck on PR head");
469 pr.createCheck(checkBuilder.build());
470 var localHash = prInstance.commit(censusInstance.namespace(), censusDomain, null);
471
472 // Try to rebase
473 boolean rebasePossible = true;
474 var ignored = new PrintWriter(new StringWriter());
475 var rebasedHash = prInstance.rebase(localHash, ignored);
476 if (rebasedHash.isEmpty()) {
477 rebasePossible = false;
478 } else {
479 localHash = rebasedHash.get();
480 }
481
482 // Determine current status
483 var visitor = prInstance.executeChecks(localHash, censusInstance);
484 var additionalErrors = botSpecificChecks();
485 updateCheckBuilder(checkBuilder, visitor, additionalErrors);
486 updateReadyForReview(visitor, additionalErrors);
487
488 // Calculate and update the status message if needed
489 var statusMessage = getStatusMessage(comments, activeReviews, visitor);
490 var updatedBody = updateStatusMessage(statusMessage);
491
492 // Post / update approval messages (only needed if the review itself can't contain a body)
493 if (!pr.repository().forge().supportsReviewBody()) {
494 updateReviewedMessages(comments, allReviews);
495 }
496
497 var commit = prInstance.localRepo().lookup(localHash).orElseThrow();
498 var commitMessage = String.join("\n", commit.message());
499 var readyForIntegration = visitor.getMessages().isEmpty() && additionalErrors.isEmpty();
500 updateMergeReadyComment(readyForIntegration, commitMessage, comments, activeReviews, rebasePossible);
501 if (readyForIntegration) {
502 newLabels.add("ready");
503 } else {
504 newLabels.remove("ready");
505 }
506 if (!rebasePossible) {
507 newLabels.add("outdated");
508 } else {
509 newLabels.remove("outdated");
510 }
511
512 // Ensure that the ready for sponsor label is up to date
513 newLabels.remove("sponsor");
514 var readyHash = ReadyForSponsorTracker.latestReadyForSponsor(pr.repository().forge().currentUser(), comments);
515 if (readyHash.isPresent() && readyForIntegration) {
516 var acceptedHash = readyHash.get();
517 if (pr.headHash().equals(acceptedHash)) {
518 newLabels.add("sponsor");
519 }
520 }
521
522 // Calculate current metadata to avoid unnecessary future checks
523 var metadata = workItem.getMetadata(pr.title(), updatedBody, pr.comments(), activeReviews, newLabels, censusInstance, pr.targetHash());
524 checkBuilder.metadata(metadata);
525 } catch (Exception e) {
526 log.throwing("CommitChecker", "checkStatus", e);
527 newLabels.remove("ready");
528 checkBuilder.metadata("invalid");
529 checkBuilder.title("Exception occurred during jcheck - the operation will be retried");
530 checkBuilder.summary(e.getMessage());
531 checkBuilder.complete(false);
532 checkException = e;
533 }
534 var check = checkBuilder.build();
535 pr.updateCheck(check);
536
537 // Synchronize the wanted set of labels
538 for (var newLabel : newLabels) {
539 if (!labels.contains(newLabel)) {
540 pr.addLabel(newLabel);
541 }
542 }
543 for (var oldLabel : labels) {
544 if (!newLabels.contains(oldLabel)) {
545 pr.removeLabel(oldLabel);
546 }
547 }
548
549 // After updating the PR, rethrow any exception to automatically retry on transient errors
550 if (checkException != null) {
551 throw new RuntimeException("Exception during jcheck", checkException);
552 }
553 }
554 }