Bug 530649 - Make BuildManager able to process build graph in parallel
Implements a parallel graph navigation with Jobs.
Change-Id: Ibfcc12ac2d8c6c39eaca4649a98d72d7e50e2a1f
Signed-off-by: Mickael Istria <mistria@redhat.com>
diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java
index 1b2c196..9d90df5 100644
--- a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java
+++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java
@@ -19,6 +19,7 @@
import java.util.stream.Collectors;
import org.eclipse.core.internal.dtree.DeltaDataTree;
import org.eclipse.core.internal.resources.*;
+import org.eclipse.core.internal.resources.ComputeProjectOrder.Digraph;
import org.eclipse.core.internal.utils.Messages;
import org.eclipse.core.internal.utils.Policy;
import org.eclipse.core.internal.watson.ElementTree;
@@ -103,7 +104,7 @@
//the job for performing background autobuild
final AutoBuildJob autoBuildJob;
- private final Set<IProject> builtProjects = new HashSet<>();
+ private final Set<IProject> builtProjects = Collections.synchronizedSet(new HashSet<>());
//the following four fields only apply for the lifetime of a single builder invocation.
protected final Set<InternalBuilder> currentBuilders;
@@ -392,6 +393,55 @@
}
/**
+ * Runs all builders on all the given project configs, in the order that
+ * they are given.
+ * @param buildJobGroup
+ * @return A status indicating if the build succeeded or failed
+ */
+ public IStatus buildParallel(Digraph<IBuildConfiguration> configs, IBuildConfiguration[] requestedConfigs, int trigger, JobGroup buildJobGroup, IProgressMonitor monitor) {
+ monitor = Policy.monitorFor(monitor);
+ try {
+ monitor.beginTask(Messages.events_building_0, TOTAL_BUILD_WORK);
+ try {
+ builtProjects.clear();
+ hookStartBuild(configs.vertexList.stream().map(vertex -> vertex.id).toArray(IBuildConfiguration[]::new), trigger);
+ MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null);
+ parallelBuildLoop(configs, requestedConfigs, trigger, buildJobGroup, status, monitor);
+ return status;
+ } finally {
+ hookEndBuild(trigger);
+ }
+ } finally {
+ monitor.done();
+ if (trigger == IncrementalProjectBuilder.INCREMENTAL_BUILD || trigger == IncrementalProjectBuilder.FULL_BUILD)
+ autoBuildJob.avoidBuild();
+ }
+ }
+
+ private void parallelBuildLoop(final Digraph<IBuildConfiguration> configs, IBuildConfiguration[] requestedConfigs, int trigger, JobGroup buildJobGroup, MultiStatus status, IProgressMonitor monitor) {
+ final int projectWork = configs.vertexList.size() > 0 ? TOTAL_BUILD_WORK / configs.vertexList.size() : 0;
+ builtProjects.clear();
+ final GraphProcessor<IBuildConfiguration> graphProcessor = new GraphProcessor<>(configs, IBuildConfiguration.class, (config, graphCrawler) -> {
+ IBuildContext context = new BuildContext(config, requestedConfigs, graphCrawler.getSequentialOrder()); // TODO consider passing Digraph to BuildConfig?
+ try {
+ workspace.prepareOperation(null, monitor);
+ workspace.beginOperation(false);
+ basicBuild(config, trigger, context, status, Policy.subMonitorFor(monitor, projectWork));
+ workspace.endOperation(null, false);
+ builtProjects.add(config.getProject());
+ } catch (CoreException ex) {
+ status.add(new Status(IStatus.ERROR, ResourcesPlugin.PI_RESOURCES, ex.getMessage(), ex));
+ }
+ }, buildJobGroup);
+ graphProcessor.processGraphWithParallelJobs();
+ try {
+ Job.getJobManager().join(graphProcessor, monitor);
+ } catch (OperationCanceledException | InterruptedException e) {
+ status.add(new Status(IStatus.ERROR, ResourcesPlugin.PI_RESOURCES, e.getMessage(), e));
+ }
+ }
+
+ /**
* Runs the builder with the given name on the given project config.
* @return A status indicating if the build succeeded or failed
*/
diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/GraphProcessor.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/GraphProcessor.java
new file mode 100644
index 0000000..d3fbba6
--- /dev/null
+++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/GraphProcessor.java
@@ -0,0 +1,135 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Mickael Istria (Red Hat Inc.)
+ *******************************************************************************/
+package org.eclipse.core.internal.events;
+
+import java.util.*;
+import java.util.function.BiConsumer;
+import org.eclipse.core.internal.resources.ComputeProjectOrder;
+import org.eclipse.core.internal.resources.ComputeProjectOrder.Digraph;
+import org.eclipse.core.internal.resources.ComputeProjectOrder.Digraph.Edge;
+import org.eclipse.core.internal.resources.ComputeProjectOrder.VertexOrder;
+import org.eclipse.core.runtime.*;
+import org.eclipse.core.runtime.jobs.Job;
+import org.eclipse.core.runtime.jobs.JobGroup;
+
+/**
+ *
+ */
+class GraphProcessor<T> {
+
+ final private Digraph<T> graph;
+ final private Set<T> toProcess;
+ final private Set<T> processing;
+ final private Set<T> processed;
+ final private VertexOrder<T> sequentialOrder;
+ final private JobGroup buildJobGroup;
+ final BiConsumer<T, GraphProcessor<T>> processor;
+
+ GraphProcessor(Digraph<T> graph1, Class<T> clazz, final BiConsumer<T, GraphProcessor<T>> processor, JobGroup buildJobGroup) {
+ this.graph = graph1;
+ this.processor = processor;
+ this.buildJobGroup = buildJobGroup;
+ toProcess = new HashSet<>(graph.vertexMap.keySet());
+ processing = new HashSet<>();
+ processed = new HashSet<>();
+ sequentialOrder = ComputeProjectOrder.computeVertexOrder(graph, clazz);
+ }
+
+ private boolean complete() {
+ return processed.size() == graph.vertexList.size();
+ }
+
+ private boolean allTriggered() {
+ return toProcess.isEmpty();
+ }
+
+ private void markProcessing(T item) {
+ if (!toProcess.remove(item)) {
+ throw new IllegalArgumentException();
+ }
+ processing.add(item);
+ }
+
+ void markProcessed(T item) {
+ if (!processing.remove(item)) {
+ throw new IllegalArgumentException();
+ }
+ processed.add(item);
+ }
+
+ private Set<T> computeReadyVertexes() {
+ Set<T> res = new HashSet<>(toProcess);
+ for (T item : toProcess) {
+ for (Edge<T> edge : graph.getEdges()) {
+ if (edge.to == item && !processed.contains(edge.from)) {
+ res.remove(item);
+ }
+ }
+ }
+ if (res.isEmpty() && !isProcessing()) { // nothing ready, nothing running: a cycle!
+ for (T id : sequentialOrder.vertexes) {
+ if (!isProcessed(id)) {
+ return Collections.singleton(id);
+ }
+ }
+ }
+ return res;
+ }
+
+ private boolean isProcessing() {
+ return !processing.isEmpty();
+ }
+
+ private boolean isProcessed(T item) {
+ return processed.contains(item);
+ }
+
+ public T[] getSequentialOrder() {
+ return this.sequentialOrder.vertexes;
+ }
+
+ public void processGraphWithParallelJobs() {
+ if (!complete()) {
+ if (!allTriggered()) {
+ Set<T> readyToBuild = computeReadyVertexes();
+ readyToBuild.forEach(item -> triggerJob(item));
+ }
+ }
+ }
+
+ private void triggerJob(T item) {
+ synchronized (this) {
+ markProcessing(item);
+ }
+ Job buildJob = new Job(item.toString()) {
+
+ @Override
+ protected IStatus run(IProgressMonitor monitor) {
+ processor.accept(item, GraphProcessor.this);
+ synchronized (GraphProcessor.this) {
+ markProcessed(item);
+ // do it as part of Job so we're sure following jobs are triggered before this one completes,
+ // so we can safely rely on join(family)
+ processGraphWithParallelJobs();
+ }
+ return Status.OK_STATUS;
+ }
+
+ @Override
+ public boolean belongsTo(Object family) {
+ return super.belongsTo(family) || family == GraphProcessor.this;
+ }
+ };
+ buildJob.setJobGroup(buildJobGroup);
+ buildJob.schedule();
+ }
+
+}
\ No newline at end of file
