Synchronize Maven: resolve using batches
use
RepositorySystem.resolveArtifacts() per plugin instead of
RepositorySystem.resolveArtifact() per file
opens less TCP connections and is reproducible faster.
like ~ 8min -> 3min on my setup
Also use eclipse resource API instead of java file api.
Change-Id: Idf0f1f0265cc775890eaf797a1a2fefdb8db730f
diff --git a/org.eclipse.tea.library.build/src/org/eclipse/tea/library/build/tasks/maven/SynchronizeMavenArtifact.java b/org.eclipse.tea.library.build/src/org/eclipse/tea/library/build/tasks/maven/SynchronizeMavenArtifact.java
index c4a1f96..bb61c45 100644
--- a/org.eclipse.tea.library.build/src/org/eclipse/tea/library/build/tasks/maven/SynchronizeMavenArtifact.java
+++ b/org.eclipse.tea.library.build/src/org/eclipse/tea/library/build/tasks/maven/SynchronizeMavenArtifact.java
@@ -10,20 +10,22 @@
*******************************************************************************/
package org.eclipse.tea.library.build.tasks.maven;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
-import java.util.TreeSet;
import java.util.regex.Matcher;
+import java.util.stream.Collectors;
import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
import org.apache.maven.wagon.ConnectionException;
@@ -42,6 +44,7 @@
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.repository.RepositoryPolicy;
import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.resolution.ArtifactResolutionException;
import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
import org.eclipse.aether.spi.connector.transport.TransporterFactory;
@@ -49,6 +52,7 @@
import org.eclipse.aether.transport.wagon.WagonProvider;
import org.eclipse.aether.transport.wagon.WagonTransporterFactory;
import org.eclipse.core.internal.variables.StringVariableManager;
+import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IFolder;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
@@ -56,7 +60,6 @@
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.NullProgressMonitor;
-import org.eclipse.core.runtime.Status;
import org.eclipse.e4.core.di.annotations.Execute;
import org.eclipse.jdt.apt.core.util.AptConfig;
import org.eclipse.jdt.core.IClasspathEntry;
@@ -213,7 +216,7 @@
System.runFinalization();
for (PluginBuild pb : classpathManipulatorOfPlugin.keySet()) {
- runSingle(log, tracker, pb, system, session, remotes);
+ synchronizePlugin(log, tracker, pb, system, session, remotes);
}
} catch (Exception e) {
log.error("error synchronizing maven artifacts", e);
@@ -251,23 +254,28 @@
return new MavenConfig(file);
}
- private void runSingle(TaskingLog log, TaskProgressTracker tracker, PluginBuild hostPlugin, RepositorySystem system,
- RepositorySystemSession session, List<RemoteRepository> remotes) throws CoreException {
+ private void synchronizePlugin(TaskingLog log, TaskProgressTracker tracker, PluginBuild hostPlugin,
+ RepositorySystem system, RepositorySystemSession session, List<RemoteRepository> remotes)
+ throws CoreException {
+ NullProgressMonitor monitor = new NullProgressMonitor();
IProject prj = hostPlugin.getData().getProject();
- IFolder folder = prj.getFolder(MAVEN_DIRNAME);
- if (!folder.exists()) {
- folder.create(false, true, null);
- log.warn("creating " + folder.getName() + "; make sure to add to the classpath of "
+ IFolder targetFolder = prj.getFolder(MAVEN_DIRNAME);
+ if (!targetFolder.exists()) {
+ targetFolder.create(false, true, monitor);
+ log.warn("creating " + targetFolder.getName() + "; make sure to add to the classpath of "
+ hostPlugin.getPluginName());
+ // write .gitignore
+ IFile gitignore = targetFolder.getFile(".gitignore");
+ gitignore.create(new ByteArrayInputStream("*.jar".getBytes(Charsets.UTF_8)), false, null);
}
- File target = folder.getRawLocation().toFile();
- Set<File> valid = new TreeSet<>(Comparator.comparing(File::getName));
+ List<ArtifactRequest> artifactRequests = new ArrayList<>();
+ log.info("synchronize maven artifacts for '" + hostPlugin.getPluginName() + "': "
+ + hostPlugin.getMavenExternalJarDependencies().stream().map(artifact -> artifact.getCoordinates())
+ .collect(Collectors.joining(", ")));
for (MavenExternalJarBuild artifact : hostPlugin.getMavenExternalJarDependencies()) {
tracker.setTaskName(artifact.getCoordinates());
tracker.worked(1);
- log.info("synchronize nexus coordinate " + artifact.getCoordinates() + " into "
- + hostPlugin.getPluginName());
Coordinate coord = new Coordinate(artifact.getCoordinates());
@@ -289,166 +297,132 @@
{
ArtifactRequest remoterq = new ArtifactRequest().setArtifact(mvn)
.setRepositories(remote ? remotes : null);
- resolveArtifact(log, target, system, session, remoterq, valid);
+ artifactRequests.add(remoterq);
}
// resolve source bundle.
- try {
- Artifact srcmvn = new DefaultArtifact(coord.group, coord.artifact, "sources", coord.extension,
- coord.version);
- ArtifactRequest srcrq = new ArtifactRequest().setArtifact(srcmvn)
- .setRepositories(remote ? remotes : null);
+ Artifact srcmvn = new DefaultArtifact(coord.group, coord.artifact, "sources", coord.extension,
+ coord.version);
+ ArtifactRequest srcrq = new ArtifactRequest().setArtifact(srcmvn).setRepositories(remote ? remotes : null);
+ artifactRequests.add(srcrq);
+ }
- resolveArtifact(log, target, system, session, srcrq, valid);
- } catch (Exception e) {
- log.warn("No sources available for " + artifact.getCoordinates());
+ List<ArtifactResult> results = resolveArtifacts(log, system, session, artifactRequests);
+ Set<IFile> resolvedFiles = new HashSet<>();
+ for (ArtifactResult result : results) {
+ ArtifactRequest rq = result.getRequest();
+ Artifact mvn = rq.getArtifact();
+ String c = mvn.getClassifier();
+ if (result.isMissing() || !result.isResolved() || !result.getExceptions().isEmpty()) {
+ if ("sources".equals(c)) {
+ log.warn("No sources available for " + mvn.getGroupId() + ":" + mvn.getArtifactId() + ":"
+ + mvn.getVersion());
+ } else {
+ String classifier = c == null || c.isEmpty() ? "" : (":" + c);
+ log.error("cannot resolve " + mvn.getGroupId() + ":" + mvn.getArtifactId() + classifier + ":"
+ + mvn.getVersion());
+ }
+ } else {
+ // copy file to maven directory
+ File resolvedFile = result.getArtifact().getFile();
+ IFile targetFile = targetFolder.getFile(resolvedFile.getName());
+ resolvedFiles.add(targetFile);
+
+ Artifact artifact = result.getArtifact();
+ if (needUpdateFileInProjectsMavenFolder(log, targetFile, artifact)) {
+ updateFileInProjectsMavenFolder(log, targetFile, artifact);
+ }
}
}
// cleanup old files
- for (File file : target.listFiles()) {
+ for (IResource file : targetFolder.members()) {
if (file.getName().equals(".gitignore")) {
continue;
}
- boolean isValid = false;
- for (File validFile : valid) {
- if (file.equals(validFile)) {
- isValid = true;
- }
- }
-
- if (!isValid) {
+ if (!resolvedFiles.contains(file)) {
log.info("removing old maven artifact: " + file);
- FileUtils.delete(file);
+ file.delete(true, null);
}
}
-
- // write .gitignore
- File gitignore = new File(target, ".gitignore");
- if (!gitignore.exists()) {
- try {
- FileUtils.writeFileFromString(gitignore, Charsets.UTF_8, "*.jar");
- } catch (IOException e) {
- throw new CoreException(Status.error(e.getMessage(), e));
- }
- }
-
- folder.refreshLocal(IResource.DEPTH_INFINITE, new NullProgressMonitor());
+ return;
}
/**
- * Resolves an {@link ArtifactRequest}. Resolving means that it looks up the
- * bundle on the local repository and all servers. After successful
+ * Resolves multiple {@link ArtifactRequest}. Resolving means that it looks
+ * up the bundle on the local repository and all servers. After successful
* resolution, the bundles is located in the local repository. After that,
* this method copies the according file to the target location.
*
* @param controller
* used for logging
- * @param target
- * the target directory to put the bundle into
* @param system
* the {@link RepositorySystem} providing the resolution
* algorithm
* @param session
* the {@link RepositorySystemSession} to use
- * @param rq
- * the {@link ArtifactRequest} that defines what to resolve
- * @param resolved
- * all resolved artifacts will be added to this list, even if no
- * file has been changed on disk.
+ * @param requests
+ * A Collection of {@link ArtifactRequest} that defines what to
+ * resolve
+ * @returns list of {@link ArtifactResult}
*/
- private void resolveArtifact(TaskingLog log, File target, RepositorySystem system, RepositorySystemSession session,
- ArtifactRequest rq, Set<File> resolved) {
- Artifact mvn = rq.getArtifact();
-
+ private List<ArtifactResult> resolveArtifacts(TaskingLog log, RepositorySystem system,
+ RepositorySystemSession session, Collection<ArtifactRequest> requests) {
+ List<ArtifactResult> results;
try {
- ArtifactResult result = system.resolveArtifact(session, rq);
- if (result.isMissing() || !result.isResolved()) {
- log.warn("cannot resolve " + mvn.getGroupId() + ":" + mvn.getArtifactId() + ":" + mvn.getVersion() + ":"
- + mvn.getClassifier());
- if (!result.getExceptions().isEmpty()) {
- for (Exception e : result.getExceptions()) {
- e.printStackTrace(log.error());
- }
- }
- return;
- }
- if (properties.isVerboseMavenOutput()) {
- if (!result.getExceptions().isEmpty()) {
- for (Exception e : result.getExceptions()) {
- e.printStackTrace(log.debug());
- }
- }
- }
- // download file to maven directory
- File file = result.getArtifact().getFile();
- addFileToProjectsMavenFolder(log, target, resolved, result, file);
- } catch (Exception e) {
- String c = mvn.getClassifier();
- if (!"sources".equals(c)) {
- String classifier = c == null || c.isEmpty() ? "" : (":" + c);
- log.error("cannot resolve " + mvn.getGroupId() + ":" + mvn.getArtifactId() + classifier + ":"
- + mvn.getVersion());
- }
- throw new RuntimeException("failed to synchronize " + rq.getArtifact().getArtifactId(), e);
+ results = system.resolveArtifacts(session, requests);
+ } catch (ArtifactResolutionException e) {
+ results = e.getResults();
}
+ return results;
}
- private static void addFileToProjectsMavenFolder(TaskingLog log, File target, Set<File> resolved,
- ArtifactResult result, File file) throws IOException {
- File targetFile = new File(target, file.getName());
- resolved.add(targetFile);
+ private static boolean needUpdateFileInProjectsMavenFolder(TaskingLog log, IFile targetResource,
+ Artifact artifact) {
+ File targetFile = targetResource.getRawLocation().toFile();
+ File resolvedFile = artifact.getFile();
+ // no need to update if files are equal:
+ return !FileUtils.equals(targetFile, resolvedFile);
+ }
- if (targetFile.exists() && !result.getArtifact().isSnapshot()) {
- // it is a released file. try to update but don't fail if (for
- // example) windows locks
- // the file...
+ private static void updateFileInProjectsMavenFolder(TaskingLog log, IFile targetResource, Artifact artifact)
+ throws CoreException {
+ File targetFile = targetResource.getRawLocation().toFile();
+ File resolvedFile = artifact.getFile();
+ // update = delete existing + copy or link new file:
+ if (targetFile.exists()) {
if (!targetFile.delete()) {
- if (targetFile.length() != file.length()) {
- // would need update but can't
- log.error("cannot update " + targetFile + " to new version, please make sure file is not locked");
- }
-
- return; // don't update
+ log.error("cannot update " + targetFile + ". Please make sure file is not locked");
+ return;
}
}
- boolean copyNeeded = true;
- // refresh of file/link:
try {
- FileUtils.delete(targetFile);
- } catch (java.lang.IllegalStateException e) { // cannot delete
- if (FileUtils.equals(targetFile, file)) {
- // ignore
- log.info("Could not update file. Probably in use. Can be ignored since contents is the same: "
- + targetFile);
- copyNeeded = false;
- } else {
- throw e;
- }
- }
-
- if (copyNeeded) {
- try {
- targetFile = java.nio.file.Files.createSymbolicLink(targetFile.toPath(), file.toPath()).toFile();
- } catch (IOException e) {
- String exName = e.getClass().getName();
- // don't spam missing rights for symlink creation
- if (!Objects.equals(exName, lastExceptionName)) {
- lastExceptionName = exName;
- // Windows 10:
- // "$file: Dem Client fehlt ein erforderliches Recht.\r\n"
- String msg = e.getMessage();
- if (msg != null) {
- msg = msg.replace("\n", "\\n");
- msg = msg.replace("\r", "\\r");
- }
- log.warn("cannot create symlink for: " + targetFile + " (" + exName + " " + msg + ")");
+ java.nio.file.Files.createSymbolicLink(targetFile.toPath(), resolvedFile.toPath());
+ } catch (IOException e) {
+ String exName = e.getClass().getName();
+ // don't spam missing rights for symlink creation
+ if (!Objects.equals(exName, lastExceptionName)) {
+ lastExceptionName = exName;
+ // Windows 10:
+ // "$file: Dem Client fehlt ein erforderliches Recht.\r\n"
+ String msg = e.getMessage();
+ if (msg != null) {
+ msg = msg.replace("\n", "\\n");
+ msg = msg.replace("\r", "\\r");
}
- FileUtils.copyFileToDirectory(file, target);
+ log.warn("cannot create symlink for: " + targetFile + " (" + exName + " " + msg + ")");
+ }
+ try {
+ FileUtils.copyFile(resolvedFile, targetFile);
+ } catch (IOException copyException) {
+ log.error("Could not copy file.", copyException);
+ return;
}
}
+ targetResource.refreshLocal(IResource.DEPTH_ZERO, null);
+ return;
}
private List<RemoteRepository> createRemoteRepositories() {
