1 /* 2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.openjdk.skara.bots.mlbridge; 24 25 import org.openjdk.skara.email.EmailAddress; 26 import org.openjdk.skara.forge.HostedRepository; 27 import org.openjdk.skara.network.URIBuilder; 28 import org.openjdk.skara.vcs.*; 29 import org.openjdk.skara.webrev.Webrev; 30 31 import java.io.*; 32 import java.net.URI; 33 import java.nio.file.*; 34 import java.util.Comparator; 35 import java.util.concurrent.atomic.AtomicInteger; 36 import java.util.stream.Collectors; 37 38 class WebrevStorage { 39 private final HostedRepository storage; 40 private final String storageRef; 41 private final Path baseFolder; 42 private final URI baseUri; 43 private final EmailAddress author; 44 45 WebrevStorage(HostedRepository storage, String ref, Path baseFolder, URI baseUri, EmailAddress author) { 46 this.baseFolder = baseFolder; 47 this.baseUri = baseUri; 48 this.storage = storage; 49 storageRef = ref; 50 this.author = author; 51 } 52 53 private void generate(PullRequestInstance prInstance, Path folder, Hash base, Hash head) throws IOException { 54 Files.createDirectories(folder); 55 Webrev.repository(prInstance.localRepo()).output(folder) 56 .generate(base, head); 57 } 58 59 private void push(Repository localStorage, Path webrevFolder, String identifier) throws IOException { 60 var batchIndex = new AtomicInteger(); 61 try (var files = Files.walk(webrevFolder)) { 62 // Try to push 1000 files at a time 63 var batches = files.filter(Files::isRegularFile) 64 .filter(file -> { 65 // Huge files are not that useful in a webrev - but make an exception for the index 66 try { 67 if (file.getFileName().toString().equals("index.html")) { 68 return true; 69 } else { 70 return Files.size(file) < 1000 * 1000; 71 } 72 } catch (IOException e) { 73 return false; 74 } 75 }) 76 .collect(Collectors.groupingBy(path -> { 77 int curIndex = batchIndex.incrementAndGet(); 78 return Math.floorDiv(curIndex, 1000); 79 })); 80 81 for (var batch : batches.entrySet()) { 82 localStorage.add(batch.getValue()); 83 Hash hash; 84 var message = "Added webrev for " + identifier + 85 (batches.size() > 1 ? " (" + (batch.getKey() + 1) + "/" + batches.size() + ")" : ""); 86 try { 87 hash = localStorage.commit(message, author.fullName().orElseThrow(), author.address()); 88 } catch (IOException e) { 89 // If the commit fails, it probably means that we're resuming a partially completed previous update 90 // where some of the files have already been committed. Ignore it and continue. 91 continue; 92 } 93 localStorage.push(hash, storage.url(), storageRef); 94 } 95 } 96 } 97 98 private static void clearDirectory(Path directory) { 99 try (var files = Files.walk(directory)) { 100 files.map(Path::toFile) 101 .sorted(Comparator.reverseOrder()) 102 .forEach(File::delete); 103 } catch (IOException io) { 104 throw new RuntimeException(io); 105 } 106 } 107 108 URI createAndArchive(PullRequestInstance prInstance, Path scratchPath, Hash base, Hash head, String identifier) { 109 try { 110 var localStorage = Repository.materialize(scratchPath, storage.url(), storageRef); 111 var relativeFolder = baseFolder.resolve(String.format("%s/webrev.%s", prInstance.id(), identifier)); 112 var outputFolder = scratchPath.resolve(relativeFolder); 113 // If a previous operation was interrupted there may be content here already - overwrite if so 114 if (Files.exists(outputFolder)) { 115 clearDirectory(outputFolder); 116 } 117 generate(prInstance, outputFolder, base, head); 118 if (!localStorage.isClean()) { 119 push(localStorage, outputFolder, baseFolder.resolve(prInstance.id()).toString()); 120 } 121 return URIBuilder.base(baseUri).appendPath(relativeFolder.toString().replace('\\', '/')).build(); 122 } catch (IOException e) { 123 throw new UncheckedIOException(e); 124 } 125 } 126 }