1 /*
2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.openjdk.skara.bots.pr;
24
25 import org.openjdk.skara.forge.*;
26 import org.openjdk.skara.host.*;
27 import org.openjdk.skara.issuetracker.*;
28 import org.openjdk.skara.vcs.*;
29 import org.openjdk.skara.vcs.openjdk.Issue;
30
31 import java.io.*;
32 import java.util.*;
33 import java.util.logging.Logger;
34 import java.util.regex.Pattern;
35 import java.util.stream.*;
36
37 class CheckRun {
38 private final CheckWorkItem workItem;
39 private final PullRequest pr;
40 private final PullRequestInstance prInstance;
41 private final List<Comment> comments;
42 private final List<Review> allReviews;
43 private final List<Review> activeReviews;
44 private final Set<String> labels;
45 private final CensusInstance censusInstance;
46 private final Map<String, String> blockingLabels;
47 private final IssueProject issueProject;
48
49 private final Logger log = Logger.getLogger("org.openjdk.skara.bots.pr");
50 private final String progressMarker = "<!-- Anything below this marker will be automatically updated, please do not edit manually! -->";
51 private final String mergeReadyMarker = "<!-- PullRequestBot merge is ready comment -->";
52 private final Pattern mergeSourcePattern = Pattern.compile("^Merge ([-/\\w]+):([-\\w]+$)");
53 private final Set<String> newLabels;
54
55 private CheckRun(CheckWorkItem workItem, PullRequest pr, PullRequestInstance prInstance, List<Comment> comments,
56 List<Review> allReviews, List<Review> activeReviews, Set<String> labels,
57 CensusInstance censusInstance, Map<String, String> blockingLabels, IssueProject issueProject) {
58 this.workItem = workItem;
59 this.pr = pr;
60 this.prInstance = prInstance;
61 this.comments = comments;
62 this.allReviews = allReviews;
63 this.activeReviews = activeReviews;
64 this.labels = new HashSet<>(labels);
65 this.newLabels = new HashSet<>(labels);
66 this.censusInstance = censusInstance;
67 this.blockingLabels = blockingLabels;
68 this.issueProject = issueProject;
69 }
70
71 static void execute(CheckWorkItem workItem, PullRequest pr, PullRequestInstance prInstance, List<Comment> comments,
72 List<Review> allReviews, List<Review> activeReviews, Set<String> labels, CensusInstance censusInstance, Map<String, String> blockingLabels,
73 IssueProject issueProject) {
74 var run = new CheckRun(workItem, pr, prInstance, comments, allReviews, activeReviews, labels, censusInstance, blockingLabels, issueProject);
75 run.checkStatus();
76 }
77
78 // For unknown contributors, check that all commits have the same name and email
79 private boolean checkCommitAuthor(List<Commit> commits) throws IOException {
80 var author = censusInstance.namespace().get(pr.author().id());
81 if (author != null) {
82 return true;
83 }
84
85 var names = new HashSet<String>();
86 var emails = new HashSet<String>();
87
88 for (var commit : commits) {
89 names.add(commit.author().name());
90 emails.add(commit.author().email());
91 }
92
93 return ((names.size() == 1) && emails.size() == 1);
94 }
95
96 private Optional<String> mergeSourceRepository() {
97 var repoMatcher = mergeSourcePattern.matcher(pr.title());
98 if (!repoMatcher.matches()) {
99 return Optional.empty();
100 }
101 return Optional.of(repoMatcher.group(1));
102 }
103
104 private Optional<String> mergeSourceBranch() {
105 var branchMatcher = mergeSourcePattern.matcher(pr.title());
106 if (!branchMatcher.matches()) {
107 return Optional.empty();
108 }
109 var mergeSourceBranch = branchMatcher.group(2);
110 return Optional.of(mergeSourceBranch);
111 }
112
113 // Additional bot-specific checks that are not handled by JCheck
114 private List<String> botSpecificChecks() throws IOException {
115 var ret = new ArrayList<String>();
116
117 var baseHash = prInstance.baseHash();
118 var headHash = pr.headHash();
119 var commits = prInstance.localRepo().commits(baseHash + ".." + headHash).asList();
120
121 if (!checkCommitAuthor(commits)) {
122 var error = "For contributors who are not existing OpenJDK Authors, commit attribution will be taken from " +
123 "the commits in the PR. However, the commits in this PR have inconsistent user names and/or " +
124 "email addresses. Please amend the commits.";
125 ret.add(error);
126 }
127
128 if (pr.title().startsWith("Merge")) {
129 if (commits.size() < 2) {
130 ret.add("A Merge PR must contain at least two commits that are not already present in the target.");
131 } else {
132 if (!commits.get(0).isMerge()) {
133 ret.add("The top commit must be a merge commit.");
134 }
135
136 var sourceRepo = mergeSourceRepository();
137 var sourceBranch = mergeSourceBranch();
138 if (sourceBranch.isPresent() && sourceRepo.isPresent()) {
139 try {
140 var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()).orElseThrow(() ->
141 new RuntimeException("Could not find repository " + sourceRepo.get())
142 );
143 try {
144 var sourceHash = prInstance.localRepo().fetch(mergeSourceRepo.url(), sourceBranch.get());
145 if (!prInstance.localRepo().isAncestor(commits.get(1).hash(), sourceHash)) {
146 ret.add("The merge contains commits that are not ancestors of the source");
147 }
148 } catch (IOException e) {
149 ret.add("Could not fetch branch `" + sourceBranch.get() + "` from project `" +
150 sourceRepo.get() + "` - check that they are correct.");
151 }
152 } catch (RuntimeException e) {
153 ret.add("Could not find project `" +
154 sourceRepo.get() + "` - check that it is correct.");
155 }
156 } else {
157 ret.add("Could not determine the source for this merge. A Merge PR title must be specified on the format: " +
158 "Merge `project`:`branch` to allow verification of the merge contents.");
159 }
160 }
161 }
162
163 for (var blocker : blockingLabels.entrySet()) {
164 if (labels.contains(blocker.getKey())) {
165 ret.add(blocker.getValue());
166 }
167 }
168
169 return ret;
170 }
171
172 private void updateCheckBuilder(CheckBuilder checkBuilder, PullRequestCheckIssueVisitor visitor, List<String> additionalErrors) {
173 if (visitor.isReadyForReview() && additionalErrors.isEmpty()) {
174 checkBuilder.complete(true);
175 } else {
176 checkBuilder.title("Required");
177 var summary = Stream.concat(visitor.getMessages().stream(), additionalErrors.stream())
178 .sorted()
179 .map(m -> "- " + m)
180 .collect(Collectors.joining("\n"));
181 checkBuilder.summary(summary);
182 for (var annotation : visitor.getAnnotations()) {
183 checkBuilder.annotation(annotation);
184 }
185 checkBuilder.complete(false);
186 }
187 }
188
189 private void updateReadyForReview(PullRequestCheckIssueVisitor visitor, List<String> additionalErrors) {
190 // If there are no issues at all, the PR is already reviewed
191 if (visitor.getMessages().isEmpty() && additionalErrors.isEmpty()) {
192 pr.removeLabel("rfr");
193 return;
194 }
195
196 // Additional errors are not allowed
197 if (!additionalErrors.isEmpty()) {
198 newLabels.remove("rfr");
199 return;
200 }
201
202 // Draft requests are not for review
203 if (pr.isDraft()) {
204 newLabels.remove("rfr");
205 return;
206 }
207
208 // Check if the visitor found any issues that should be resolved before reviewing
209 if (visitor.isReadyForReview()) {
210 newLabels.add("rfr");
211 } else {
212 newLabels.remove("rfr");
213 }
214 }
215
216 private String getRole(String username) {
217 var project = censusInstance.project();
218 var version = censusInstance.census().version().format();
219 if (project.isReviewer(username, version)) {
220 return "**Reviewer**";
221 } else if (project.isCommitter(username, version)) {
222 return "Committer";
223 } else if (project.isAuthor(username, version)) {
224 return "Author";
225 } else {
226 return "no project role";
227 }
228 }
229
230 private String formatReviewer(HostUser reviewer) {
231 var namespace = censusInstance.namespace();
232 var contributor = namespace.get(reviewer.id());
233 if (contributor == null) {
234 return reviewer.userName() + " (no known " + namespace.name() + " user name / role)";
235 } else {
236 var userNameLink = "[" + contributor.username() + "](@" + reviewer.userName() + ")";
237 return contributor.fullName().orElse(contributor.username()) + " (" + userNameLink + " - " +
238 getRole(contributor.username()) + ")";
239 }
240 }
241
242 private String getChecksList(PullRequestCheckIssueVisitor visitor) {
243 return visitor.getChecks().entrySet().stream()
244 .map(entry -> "- [" + (entry.getValue() ? "x" : " ") + "] " + entry.getKey())
245 .collect(Collectors.joining("\n"));
246 }
247
248 private Optional<String> getReviewersList(List<Review> reviews) {
249 var reviewers = reviews.stream()
250 .filter(review -> review.verdict() == Review.Verdict.APPROVED)
251 .map(review -> {
252 var entry = " * " + formatReviewer(review.reviewer());
253 if (!review.hash().equals(pr.headHash())) {
254 entry += " **Note!** Review applies to " + review.hash();
255 }
256 return entry;
257 })
258 .collect(Collectors.joining("\n"));
259 if (reviewers.length() > 0) {
260 return Optional.of(reviewers);
261 } else {
262 return Optional.empty();
263 }
264 }
265
266 private String getStatusMessage(List<Review> reviews, PullRequestCheckIssueVisitor visitor) {
267 var progressBody = new StringBuilder();
268 progressBody.append("## Progress\n");
269 progressBody.append(getChecksList(visitor));
270
271 var issue = Issue.fromString(pr.title());
272 if (issueProject != null && issue.isPresent()) {
273 progressBody.append("\n\n## Issue\n");
274 var iss = issueProject.issue(issue.get().id());
275 if (iss.isPresent()) {
276 progressBody.append("[");
277 progressBody.append(iss.get().id());
278 progressBody.append("](");
279 progressBody.append(iss.get().webUrl());
280 progressBody.append("): ");
281 progressBody.append(iss.get().title());
282 progressBody.append("\n");
283 } else {
284 progressBody.append("⚠️ Failed to retrieve information on issue `");
285 progressBody.append(issue.get().id());
286 progressBody.append("`.\n");
287 }
288 }
289
290 getReviewersList(reviews).ifPresent(reviewers -> {
291 progressBody.append("\n\n## Approvers\n");
292 progressBody.append(reviewers);
293 });
294
295 return progressBody.toString();
296 }
297
298 private String updateStatusMessage(String message) {
299 var description = pr.body();
300 var markerIndex = description.lastIndexOf(progressMarker);
301
302 if (markerIndex >= 0 && description.substring(markerIndex).equals(message)) {
303 log.info("Progress already up to date");
304 return description;
305 }
306 var newBody = (markerIndex < 0 ?
307 description :
308 description.substring(0, markerIndex)).trim() + "\n" + progressMarker + "\n" + message;
309
310 // TODO? Retrieve the body again here to lower the chance of concurrent updates
311 pr.setBody(newBody);
312 return newBody;
313 }
314
315 private String verdictToString(Review.Verdict verdict) {
316 switch (verdict) {
317 case APPROVED:
318 return "changes are approved";
319 case DISAPPROVED:
320 return "more changes needed";
321 case NONE:
322 return "comment added";
323 default:
324 throw new RuntimeException("Unknown verdict: " + verdict);
325 }
326 }
327
328 private void updateReviewedMessages(List<Comment> comments, List<Review> reviews) {
329 var reviewTracker = new ReviewTracker(comments, reviews);
330
331 for (var added : reviewTracker.newReviews().entrySet()) {
332 var body = added.getValue() + "\n" +
333 "This PR has been reviewed by " +
334 formatReviewer(added.getKey().reviewer()) + " - " +
335 verdictToString(added.getKey().verdict()) + ".";
336 pr.addComment(body);
337 }
338 }
339
340 private Optional<Comment> findComment(List<Comment> comments, String marker) {
341 var self = pr.repository().forge().currentUser();
342 return comments.stream()
343 .filter(comment -> comment.author().equals(self))
344 .filter(comment -> comment.body().contains(marker))
345 .findAny();
346 }
347
348 private String getMergeReadyComment(String commitMessage, List<Review> reviews, boolean rebasePossible) {
349 var message = new StringBuilder();
350 message.append("@");
351 message.append(pr.author().userName());
352 message.append(" This change can now be integrated. The commit message will be:\n");
353 message.append("```\n");
354 message.append(commitMessage);
355 message.append("\n```\n");
356
357 message.append("- If you would like to add a summary, use the `/summary` command.\n");
358 message.append("- To list additional contributors, use the `/contributor` command.\n");
359
360 var divergingCommits = prInstance.divergingCommits();
361 if (divergingCommits.size() > 0) {
362 message.append("\n");
363 message.append("Since the source branch of this PR was last updated there ");
364 if (divergingCommits.size() == 1) {
365 message.append("has been 1 commit ");
366 } else {
367 message.append("have been ");
368 message.append(divergingCommits.size());
369 message.append(" commits ");
370 }
371 message.append("pushed to the `");
372 message.append(pr.targetRef());
373 message.append("` branch:\n");
374 var commitList = divergingCommits.stream()
375 .map(commit -> " * " + commit.hash().hex() + ": " + commit.message().get(0))
376 .collect(Collectors.joining("\n"));
377 message.append(commitList);
378 message.append("\n\n");
379 if (rebasePossible) {
380 message.append("Since there are no conflicts, your changes will automatically be rebased on top of the ");
381 message.append("above commits when integrating. If you prefer to do this manually, please merge `");
382 message.append(pr.targetRef());
383 message.append("` into your branch first.\n");
384 } else {
385 message.append("Your changes cannot be rebased automatically without conflicts, so you will need to ");
386 message.append("merge `");
387 message.append(pr.targetRef());
388 message.append("` into your branch before integrating.\n");
389 }
390 }
391
392 if (!ProjectPermissions.mayCommit(censusInstance, pr.author())) {
393 message.append("\n");
394 var contributor = censusInstance.namespace().get(pr.author().id());
395 if (contributor == null) {
396 message.append("As you are not a known OpenJDK [Author](http://openjdk.java.net/bylaws#author), ");
397 } else {
398 message.append("As you do not have Committer status in this project, ");
399 }
400
401 message.append("an existing [Committer](http://openjdk.java.net/bylaws#committer) must agree to ");
402 message.append("[sponsor](http://openjdk.java.net/sponsor/) your change. ");
403 var candidates = reviews.stream()
404 .filter(review -> ProjectPermissions.mayCommit(censusInstance, review.reviewer()))
405 .map(review -> "@" + review.reviewer().userName())
406 .collect(Collectors.joining(", "));
407 if (candidates.length() > 0) {
408 message.append("Possible candidates are the reviewers of this PR (");
409 message.append(candidates);
410 message.append(") but any other Committer may sponsor as well. ");
411 }
412 if (rebasePossible) {
413 message.append("\n\n");
414 message.append("- To flag this PR as ready for integration with the above commit message, type ");
415 message.append("`/integrate` in a new comment. (Afterwards, your sponsor types ");
416 message.append("`/sponsor` in a new comment to perform the integration).\n");
417 }
418 } else if (rebasePossible) {
419 if (divergingCommits.size() > 0) {
420 message.append("\n");
421 }
422 message.append("- To integrate this PR with the above commit message, type ");
423 message.append("`/integrate` in a new comment.\n");
424 }
425 message.append(mergeReadyMarker);
426 return message.toString();
427 }
428
429 private String getMergeNoLongerReadyComment() {
430 var message = new StringBuilder();
431 message.append("@");
432 message.append(pr.author().userName());
433 message.append(" This change is no longer ready for integration - check the PR body for details.\n");
434 message.append(mergeReadyMarker);
435 return message.toString();
436 }
437
438 private void updateMergeReadyComment(boolean isReady, String commitMessage, List<Comment> comments, List<Review> reviews, boolean rebasePossible) {
439 var existing = findComment(comments, mergeReadyMarker);
440 if (isReady) {
441 var message = getMergeReadyComment(commitMessage, reviews, rebasePossible);
442 if (existing.isEmpty()) {
443 pr.addComment(message);
444 } else {
445 pr.updateComment(existing.get().id(), message);
446 }
447 } else {
448 existing.ifPresent(comment -> pr.updateComment(comment.id(), getMergeNoLongerReadyComment()));
449 }
450 }
451
452 private void checkStatus() {
453 var checkBuilder = CheckBuilder.create("jcheck", pr.headHash());
454 var censusDomain = censusInstance.configuration().census().domain();
455 Exception checkException = null;
456
457 try {
458 // Post check in-progress
459 log.info("Starting to run jcheck on PR head");
460 pr.createCheck(checkBuilder.build());
461 var localHash = prInstance.commit(censusInstance.namespace(), censusDomain, null);
462
463 // Try to rebase
464 boolean rebasePossible = true;
465 var ignored = new PrintWriter(new StringWriter());
466 var rebasedHash = prInstance.rebase(localHash, ignored);
467 if (rebasedHash.isEmpty()) {
468 rebasePossible = false;
469 } else {
470 localHash = rebasedHash.get();
471 }
472
473 // Determine current status
474 var visitor = prInstance.executeChecks(localHash, censusInstance);
475 var additionalErrors = botSpecificChecks();
476 updateCheckBuilder(checkBuilder, visitor, additionalErrors);
477 updateReadyForReview(visitor, additionalErrors);
478
479 // Calculate and update the status message if needed
480 var statusMessage = getStatusMessage(activeReviews, visitor);
481 var updatedBody = updateStatusMessage(statusMessage);
482
483 // Post / update approval messages (only needed if the review itself can't contain a body)
484 if (!pr.repository().forge().supportsReviewBody()) {
485 updateReviewedMessages(comments, allReviews);
486 }
487
488 var commit = prInstance.localRepo().lookup(localHash).orElseThrow();
489 var commitMessage = String.join("\n", commit.message());
490 var readyForIntegration = visitor.getMessages().isEmpty() && additionalErrors.isEmpty();
491 updateMergeReadyComment(readyForIntegration, commitMessage, comments, activeReviews, rebasePossible);
492 if (readyForIntegration) {
493 newLabels.add("ready");
494 } else {
495 newLabels.remove("ready");
496 }
497 if (!rebasePossible) {
498 newLabels.add("outdated");
499 } else {
500 newLabels.remove("outdated");
501 }
502
503 // Ensure that the ready for sponsor label is up to date
504 newLabels.remove("sponsor");
505 var readyHash = ReadyForSponsorTracker.latestReadyForSponsor(pr.repository().forge().currentUser(), comments);
506 if (readyHash.isPresent() && readyForIntegration) {
507 var acceptedHash = readyHash.get();
508 if (pr.headHash().equals(acceptedHash)) {
509 newLabels.add("sponsor");
510 }
511 }
512
513 // Calculate current metadata to avoid unnecessary future checks
514 var metadata = workItem.getMetadata(pr.title(), updatedBody, pr.comments(), activeReviews, newLabels, censusInstance, pr.targetHash());
515 checkBuilder.metadata(metadata);
516 } catch (Exception e) {
517 log.throwing("CommitChecker", "checkStatus", e);
518 newLabels.remove("ready");
519 checkBuilder.metadata("invalid");
520 checkBuilder.title("Exception occurred during jcheck - the operation will be retried");
521 checkBuilder.summary(e.getMessage());
522 checkBuilder.complete(false);
523 checkException = e;
524 }
525 var check = checkBuilder.build();
526 pr.updateCheck(check);
527
528 // Synchronize the wanted set of labels
529 for (var newLabel : newLabels) {
530 if (!labels.contains(newLabel)) {
531 pr.addLabel(newLabel);
532 }
533 }
534 for (var oldLabel : labels) {
535 if (!newLabels.contains(oldLabel)) {
536 pr.removeLabel(oldLabel);
537 }
538 }
539
540 // After updating the PR, rethrow any exception to automatically retry on transient errors
541 if (checkException != null) {
542 throw new RuntimeException("Exception during jcheck", checkException);
543 }
544 }
545 }