org.eclipse.mdm.apicopy/src/main/java/org/eclipse/mdm/apicopy/control/ImportTask.java

/********************************************************************************
 * Copyright (c) 2015-2020 Contributors to the Eclipse Foundation
 *
 * See the NOTICE file(s) distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Public License v. 2.0 which is available at
 * http://www.eclipse.org/legal/epl-2.0.
 *
 * SPDX-License-Identifier: EPL-2.0
 *
 ********************************************************************************/
package org.eclipse.mdm.apicopy.control;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import org.eclipse.mdm.api.base.ServiceNotProvidedException;
import org.eclipse.mdm.api.base.Transaction;
import org.eclipse.mdm.api.base.adapter.EntityType;
import org.eclipse.mdm.api.base.adapter.Relation;
import org.eclipse.mdm.api.base.massdata.ReadRequest;
import org.eclipse.mdm.api.base.massdata.ReadRequest.ValuesMode;
import org.eclipse.mdm.api.base.massdata.WriteRequest;
import org.eclipse.mdm.api.base.model.Channel;
import org.eclipse.mdm.api.base.model.ChannelGroup;
import org.eclipse.mdm.api.base.model.ContextComponent;
import org.eclipse.mdm.api.base.model.ContextRoot;
import org.eclipse.mdm.api.base.model.ContextType;
import org.eclipse.mdm.api.base.model.Entity;
import org.eclipse.mdm.api.base.model.MeasuredValues;
import org.eclipse.mdm.api.base.model.Measurement;
import org.eclipse.mdm.api.base.model.Quantity;
import org.eclipse.mdm.api.base.model.Test;
import org.eclipse.mdm.api.base.model.TestStep;
import org.eclipse.mdm.api.base.model.User;
import org.eclipse.mdm.api.base.model.Value;
import org.eclipse.mdm.api.base.model.VersionState;
import org.eclipse.mdm.api.base.query.Filter;
import org.eclipse.mdm.api.base.search.SearchService;
import org.eclipse.mdm.api.dflt.ApplicationContext;
import org.eclipse.mdm.api.dflt.EntityManager;
import org.eclipse.mdm.api.dflt.model.Pool;
import org.eclipse.mdm.api.dflt.model.Project;
import org.eclipse.mdm.api.dflt.model.TemplateComponent;
import org.eclipse.mdm.api.dflt.model.TemplateRoot;
import org.eclipse.mdm.api.dflt.model.TemplateTest;
import org.eclipse.mdm.api.dflt.model.TemplateTestStep;
import org.eclipse.mdm.api.dflt.model.Versionable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Strings;
import com.google.common.collect.ListMultimap;

public class ImportTask extends TransferBase implements ApiCopyTask {

	/**
	 * Property to configure to overwrite the existing test step
	 * 
	 * Default: false
	 * 
	 * If true and the matching target test step exists, context data will be
	 * overwriten and the measurements will be imported as childs of the existing
	 * teststep.
	 * 
	 * Otherwise a new test step will be created
	 */
	private static final String PROPERTY_OVERWRITE_EXISTING_ELEMENTS = "overwriteExistingElements";
	private static final String DEFAULT_OVERWRITE_EXISTING_ELEMENTS = "false";

	/**
	 * Property to configure that existing measured values will be appended
	 * 
	 * Default: false
	 * 
	 * If true the measured value of the import data set will be appended to the
	 * existing. That means, if the target measurement already exists the importer
	 * will add the measured values of the source column to the target column. If at
	 * the the target measurement the column not exists, the importer create a new
	 * one, add first invalid values with the count of the target submatrix size,
	 * and add there the source values.
	 * 
	 * If false and the target measurement already exists, the import task will be
	 * canceled
	 */
	private static final String PROPERTY_APPEND = "append";
	private static final String DEFAULT_APPEND = "false";

	/**
	 * Property to configure that existing file links will be replaced
	 * 
	 * Default: false
	 * 
	 * If true, file links of the target entities will be replaced, if in the source
	 * the file link exists
	 * 
	 * Otherwise new file links will be added to the existing sequence.
	 */
	private static final String PROPERTY_REPLACE_FILELINKS = "replaceFileLinks";
	private static final String DEFAULT_REPLACE_FILES = "false";

	private static final Logger LOG = LoggerFactory.getLogger(ImportTask.class);

	private SearchService searchServiceDst;
	private ClassificationUtil classificationUtil;
	private TemplateManager templateManager;

	private Map<String, String> quantityMapping = new HashMap<>();

	private Map<String, String> unitMapping = new HashMap<>();

	private Map<String, String> properties = new HashMap<>();

	private boolean replaceFileLinks = false;

	public ImportTask(ApplicationContext src, ApplicationContext dst, TemplateManager templateManager) {
		super(src, dst);

		searchServiceDst = contextDst.getSearchService()
				.orElseThrow(() -> new ServiceNotProvidedException(SearchService.class));

		if (templateManager == null) {
			this.templateManager = new DefaultTemplateManager();
		} else {
			this.templateManager = templateManager;
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.eclipse.mdm.apicopy.control.ApiCopyTask#setUnitMapping(java.util.Map)
	 */
	@Override
	public void setUnitMapping(Map<String, String> unitMapping) {
		this.unitMapping = unitMapping;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.eclipse.mdm.apicopy.control.ApiCopyTask#setQuantityMapping(java.util.Map)
	 */
	@Override
	public void setQuantityMapping(Map<String, String> quantityMapping) {
		this.quantityMapping = quantityMapping;
	}

	@Override
	public void copy(List<? extends Entity> entities) {
		Transaction transaction = entityManagerDst.startTransaction();
		classificationUtil = new ClassificationUtil(transaction, contextDst);
		try {
			mapSrcDstEntities.clear();

			ListMultimap<Class<? extends Entity>, Entity> parents = loadParents(entities);
			LOG.trace("Resolved parents: {}", parents);
			supportedRootEntities.forEach(ec -> parents.get(ec).forEach(e -> copyEntity(e, false, transaction)));

			LOG.trace("Export entities: {}", entities);
			entities.forEach(e -> copyEntity(e, true, transaction));
			transaction.commit();
			deleteFilesOfReplacedFileLinks();
		} catch (Exception exc) {
			try {
				transaction.abort();
			} catch (Exception exc2) {
				LOG.error("Could not abort transaction!");
			}
			deleteFilesOfUploadedFileLinks();

			throw new ApiCopyException("Could not copy data.", exc);
		} finally {
			classificationUtil.clearCache();
			clearReplacedFileLinkCache();
			clearUploadedFileLinkCache();
		}
	}

	private Entity copyEntity(Entity entity, boolean recursive, Transaction transaction) {
		if (entity instanceof Project) {
			return copyProject((Project) entity, recursive, transaction);
		} else if (entity instanceof Pool) {
			return copyPool((Pool) entity, recursive, transaction);
		} else if (entity instanceof Test) {
			return copyTest((Test) entity, recursive, transaction);
		} else if (entity instanceof TestStep) {
			return copyTestStep((TestStep) entity, recursive, transaction);
		} else if (entity instanceof Measurement) {
			return copyMeasurement((Measurement) entity, new ArrayList<ContextRoot>(), recursive, transaction);
		} else {
			throw new ApiCopyException("Unsupported entity: '" + entity.getClass().getName() + "'");
		}
	}

	private Project copyProject(Project projectSrc, boolean recursive, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(projectSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {
			LOG.trace("Importing Project '{}'", projectSrc.getName());
			EntityType etProject = modelManagerDst.getEntityType(Project.class);

			Filter filter = Filter.nameOnly(etProject, projectSrc.getName());

			Project projectDst = fetchOne(searchServiceDst, Project.class, filter)
					.orElseGet(() -> entityFactoryDst.createProject(projectSrc.getName()));

			copyValues(projectSrc, projectDst, Arrays.asList("Id", "Name"), false, replaceFileLinks);

			persist(transaction, projectDst);

			ehDst = new EntityHolder(projectDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);

			if (recursive) {
				entityManagerSrc.loadChildren(projectSrc, Pool.class)
						.forEach(pool -> copyPool(pool, recursive, transaction));
			}
		}

		return (Project) ehDst.getEntity();
	}

	private Pool copyPool(Pool poolSrc, boolean recursive, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(poolSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {
			LOG.trace("Importing Pool '{}'", poolSrc.getName());

			Project projectParentDst = (Project) mapSrcDstEntities
					.get(new EntityHolder(entityManagerSrc.loadParent(poolSrc, Project.class).get(), entityManagerSrc))
					.getEntity();

			EntityType etPool = modelManagerDst.getEntityType(Pool.class);
			EntityType etProject = modelManagerDst.getEntityType(Project.class);

			Relation relProject = etPool.getRelation(etProject);

			if (null == relProject) {
				throw new ApiCopyException("No relation to Project found at Pool!");
			}

			Filter filter = Filter.nameOnly(etPool, poolSrc.getName()).id(relProject, projectParentDst.getID());
			Pool poolDst = fetchOne(searchServiceDst, Pool.class, filter)
					.orElseGet(() -> entityFactoryDst.createPool(poolSrc.getName(), projectParentDst));

			copyValues(poolSrc, poolDst, Arrays.asList("Id", "Name"), false, replaceFileLinks);

			persist(transaction, poolDst);

			ehDst = new EntityHolder(poolDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);

			if (recursive) {
				entityManagerSrc.loadChildren(poolSrc, Test.class)
						.forEach(test -> copyTest(test, recursive, transaction));
			}
		}

		return (Pool) ehDst.getEntity();
	}

	private Test copyTest(Test testSrc, boolean recursive, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(testSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {

			Pool poolParentDst = (Pool) mapSrcDstEntities
					.get(new EntityHolder(entityManagerSrc.loadParent(testSrc, Pool.class).get(), entityManagerSrc))
					.getEntity();

			EntityType etTest = modelManagerDst.getEntityType(Test.class);
			EntityType etPool = modelManagerDst.getEntityType(Pool.class);

			Relation relPool = etTest.getRelation(etPool);

			if (null == relPool) {
				throw new ApiCopyException("No relation to StructureLevel found at Test!");
			}

			Optional<TemplateTest> templateTestDst = loadTemplateTest(testSrc);

			Filter filter = Filter.nameOnly(etTest, testSrc.getName()).id(relPool, poolParentDst.getID());

			String rootProjectName = getProjectName(testSrc);

			Test testDst;

			if (templateTestDst.isPresent()) {
				LOG.trace("Importing Test '{}' using TestTemplate '{}'", testSrc.getName(),
						templateTestDst.get().getName());
				testDst = fetchOne(searchServiceDst, Test.class, filter)
						.orElseGet(() -> entityFactoryDst.createTest(testSrc.getName(), poolParentDst,
								templateTestDst.get(), classificationUtil.getClassification(rootProjectName), false));
			} else {
				LOG.trace("Importing Test '{}' using no TestTemplate", testSrc.getName());
				testDst = fetchOne(searchServiceDst, Test.class, filter)
						.orElseGet(() -> entityFactoryDst.createTest(testSrc.getName(), poolParentDst, null,
								classificationUtil.getClassification(rootProjectName), false));
			}

			copyValues(testSrc, testDst, Arrays.asList("Id", "Name"), true, replaceFileLinks);

			// copy responsible person:
			Optional<User> userSrc = testSrc.getResponsiblePerson();
			if (userSrc.isPresent()) {
				User userDst = entityManagerDst.loadAll(User.class, userSrc.get().getName()).stream().findFirst()
						.orElseThrow(() -> new ApiCopyException(String.format(
								"No User instance with name %s found in destination!", userSrc.get().getName())));

				testDst.setResponsiblePerson(userDst);
			}

			persist(transaction, testDst);

			ehDst = new EntityHolder(testDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);

			if (recursive) {
				EntityType etTestStep = modelManagerDst.getEntityType(TestStep.class);
				Relation relTest = etTestStep.getRelation(etTest);

				if (null == relTest) {
					throw new ApiCopyException("No relation to Test found at TestStep!");
				}

				copyTestSteps(testSrc, recursive, templateTestDst, testDst, transaction);
				entityManagerSrc.loadChildren(testSrc, TestStep.class)
						.forEach(testStep -> copyTestStep(testStep, recursive, transaction));
			}
		}

		return (Test) ehDst.getEntity();
	}

	private TestStep copyTestStep(TestStep testStepSrc, boolean recursive, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(testStepSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {
			LOG.trace("Importing TestStep '{}'", testStepSrc.getName());
			EntityType etTestStep = modelManagerDst.getEntityType(TestStep.class);
			EntityType etTest = modelManagerDst.getEntityType(Test.class);

			Relation relTest = etTestStep.getRelation(etTest);

			if (null == relTest) {
				throw new ApiCopyException("No relation to Test found at TestStep!");
			}

			Test testDst = (Test) mapSrcDstEntities
					.get(new EntityHolder(entityManagerSrc.loadParent(testStepSrc, Test.class).get(), entityManagerSrc))
					.getEntity();

			Optional<TemplateTestStep> templateTestStep = loadTemplateTestStep(testStepSrc);

			Filter filter = Filter.nameOnly(etTestStep, testStepSrc.getName()).id(relTest, testDst.getID());

			String rootProjectName = getProjectName(testStepSrc);

			TestStep testStepDst = fetchOne(searchServiceDst, TestStep.class, filter).orElseGet(() -> {
				if (templateTestStep.isPresent() && hasContextData(testStepSrc)) {
					LOG.trace("Importing TestStep '{}' using TestStepTemplate '{}' and create context roots",
							testStepSrc.getName(), templateTestStep.get().getName());
					return entityFactoryDst.createTestStep(testDst, templateTestStep.get(),
							classificationUtil.getClassification(rootProjectName));
				} else if (templateTestStep.isPresent()) {
					LOG.trace("Importing TestStep '{}' using TestStepTemplate '{}'", testStepSrc.getName(),
							templateTestStep.get().getName());
					return entityFactoryDst.createTestStepWithOutContextRoots(testDst, templateTestStep.get(),
							classificationUtil.getClassification(rootProjectName));
				} else {
					LOG.trace("Importing TestStep '{}' using no TestStepTemplate", testStepSrc.getName());
					TestStep createdTestStep = entityFactoryDst.createTestStep(testStepSrc.getName(), testDst,
							classificationUtil.getClassification(rootProjectName));
					Value value = createdTestStep.getValue("TplTestStep");
					return createdTestStep;
				}
			});

			copyValues(testStepSrc, testStepDst, Arrays.asList("Id"), true, replaceFileLinks);

			persist(transaction, testStepDst);

			ehDst = new EntityHolder(testStepDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);

			Map<ContextType, ContextRoot> mapContextRootsSrc = testStepSrc.loadContexts(entityManagerSrc,
					ContextType.values());
			Map<ContextType, ContextRoot> mapContextRootsDst = testStepDst.loadContexts(entityManagerDst,
					ContextType.values());

			copyContext(mapContextRootsSrc, mapContextRootsDst, templateTestStep, transaction);

			if (recursive) {
				List<ContextRoot> listContextRoots = new ArrayList<>();
				entityManagerSrc.loadChildren(testStepSrc, Measurement.class)
						.forEach(measurement -> copyMeasurement(measurement, listContextRoots, recursive, transaction));
			}
		}
		return (TestStep) ehDst.getEntity();
	}

	private String getProjectName(TestStep testStepSrc) {
		Optional<Test> parentTest = entityManagerSrc.loadParent(testStepSrc, Test.class);

		if (!parentTest.isPresent()) {
			throw new ApiCopyException("Parent of source teststep not found!");
		}

		return getProjectName(parentTest.get());
	}

	private String getProjectName(Test testSrc) {

		Optional<Pool> parentPool = entityManagerSrc.loadParent(testSrc, Pool.class);

		if (!parentPool.isPresent()) {
			throw new ApiCopyException("Parent of source test not found!");
		}

		Optional<Project> parentProject = entityManagerSrc.loadParent(parentPool.get(), Project.class);

		if (!parentProject.isPresent()) {
			throw new ApiCopyException("Parent of source pool not found!");
		}

		return parentProject.get().getName();
	}

	/**
	 * Check if the teststep has context data
	 * 
	 * @param testStepSrc
	 * @return true if the teststep has conteyt data
	 */
	private boolean hasContextData(TestStep testStepSrc) {
		boolean hasContextData = false;

		Optional<EntityManager> entityManager = contextSrc.getEntityManager();
		if (entityManager.isPresent()) {
			Map<ContextType, ContextRoot> loadContexts = entityManager.get().loadContexts(testStepSrc,
					ContextType.UNITUNDERTEST, ContextType.TESTEQUIPMENT, ContextType.TESTSEQUENCE);
			if (!loadContexts.isEmpty()) {
				hasContextData = true;
			}
		}

		return hasContextData;

	}

	private List<TestStep> copyTestSteps(Test testSrc, boolean recursive, Optional<TemplateTest> templateTestDst,
			Test testDst, Transaction transaction) {
		EntityType etTestStep = modelManagerDst.getEntityType(TestStep.class);
		EntityType etTest = modelManagerDst.getEntityType(Test.class);

		Relation relTest = etTestStep.getRelation(etTest);

		if (null == relTest) {
			throw new ApiCopyException("No relation to Test found at TestStep!");
		}

		List<TestStep> listTestStepsSrc = entityManagerSrc.loadChildren(testSrc, TestStep.class);

		List<TestStep> listTestStepsDst = new ArrayList<>();
		for (int i = 0, len = listTestStepsSrc.size(); i < len; ++i) {
			TestStep testStepSrc = listTestStepsSrc.get(i);
			Optional<TemplateTestStep> templateTestStep = loadTemplateTestStep(testStepSrc);

			String rootProjectName = getProjectName(testStepSrc);

			Filter filter = Filter.nameOnly(etTestStep, testStepSrc.getName()).id(relTest, testDst.getID());
			Optional<TestStep> oTestSTepDst = fetchOne(searchServiceDst, TestStep.class, filter);

			TestStep testStepDst;

			boolean overwriteExistingElements = Boolean.valueOf(
					properties.getOrDefault(PROPERTY_OVERWRITE_EXISTING_ELEMENTS, DEFAULT_OVERWRITE_EXISTING_ELEMENTS));

			if (!overwriteExistingElements || (overwriteExistingElements && !oTestSTepDst.isPresent())) {
				if (templateTestStep.isPresent() && hasContextData(testStepSrc)) {
					LOG.trace("Importing TestStep '{}' using TestStepTemplate '{}' and create context roots",
							testStepSrc.getName(), templateTestStep.get().getName());
					testStepDst = entityFactoryDst.createTestStep(testDst, templateTestStep.get(),
							classificationUtil.getClassification(rootProjectName));
					testStepDst.setName(testStepSrc.getName());
				} else if (templateTestStep.isPresent()) {
					LOG.trace("Importing TestStep '{}' using TestStepTemplate '{}' and create context roots",
							testStepSrc.getName(), templateTestStep.get().getName());
					testStepDst = entityFactoryDst.createTestStepWithOutContextRoots(testDst, templateTestStep.get(),
							classificationUtil.getClassification(rootProjectName));
				} else {
					LOG.trace("Importing TestStep '{}' using no TestStepTemplate", testStepSrc.getName());
					testStepDst = entityFactoryDst.createTestStep(testStepSrc.getName(), testDst,
							classificationUtil.getClassification(rootProjectName));
				}
			} else {
				testStepDst = oTestSTepDst.get();
			}

			copyValues(testStepSrc, testStepDst, Arrays.asList("Id"), true, replaceFileLinks);

			listTestStepsDst.add(testStepDst);

			mapSrcDstEntities.put(new EntityHolder(testStepSrc, entityManagerSrc),
					new EntityHolder(testStepDst, entityManagerDst));
		}

		for (TestStep testStepSrc : listTestStepsSrc) {
			Optional<TestStep> testStepDst = listTestStepsDst.stream()
					.filter(n -> n.getName().equals(testStepSrc.getName())).findFirst();

			if (testStepDst.isPresent()) {
				Map<ContextType, ContextRoot> mapContextRootsSrc = testStepSrc.loadContexts(entityManagerSrc,
						ContextType.values());
				Map<ContextType, ContextRoot> mapContextRootsDst = testStepDst.get().loadContexts(entityManagerDst,
						ContextType.values());
				copyContext(mapContextRootsSrc, mapContextRootsDst, loadTemplateTestStep(testStepSrc), transaction);
			}
		}

		persist(transaction, listTestStepsDst);

		if (recursive) {
			for (TestStep testStep : listTestStepsSrc) {
				List<ContextRoot> listContextRoots = new ArrayList<>();
				entityManagerSrc.loadChildren(testStep, Measurement.class)
						.forEach(measurement -> copyMeasurement(measurement, listContextRoots, recursive, transaction));
			}
		}

		return listTestStepsDst;
	}

	private void copyContext(Map<ContextType, ContextRoot> mapContextRootsSrc,
			Map<ContextType, ContextRoot> mapContextRootsDst, Optional<TemplateTestStep> optTemplateTestStep,
			Transaction transaction) {

		for (Map.Entry<ContextType, ContextRoot> me : mapContextRootsSrc.entrySet()) {
			LOG.trace("Importing ContextRoot '{}'", me.getKey());
			ContextRoot contextRootDst = mapContextRootsDst.get(me.getKey());
			if (null != contextRootDst && contextRootDst.getName().equals(me.getValue().getName())) {
				ContextRoot contextRootSrc = me.getValue();
				copyValues(contextRootSrc, contextRootDst, Arrays.asList("Id", "Name"), false, replaceFileLinks);

				TemplateRoot templateRootDst = null;

				if (optTemplateTestStep.isPresent()) {
					Optional<TemplateRoot> optTemplateRoot = optTemplateTestStep.get()
							.getTemplateRoot(contextRootDst.getContextType());

					templateRootDst = optTemplateRoot.orElse(null);
				}

				for (ContextComponent contextComponentSrc : me.getValue().getContextComponents()) {
					LOG.trace("Importing ContextComponent '{}'", contextComponentSrc.getName());
					Optional<ContextComponent> o = contextRootDst.getContextComponent(contextComponentSrc.getName());

					if (o.isPresent()) {
						ContextComponent contextComponentDst = o.get();

						copyValues(contextComponentSrc, contextComponentDst, Arrays.asList("Id", "Name"), true,
								replaceFileLinks);

					} else if (null != templateRootDst) {
						Optional<TemplateComponent> optTemplateComponent = templateRootDst
								.getTemplateComponent(contextComponentSrc.getName());

						if (optTemplateComponent.isPresent()) {
							TemplateComponent templateComponent = optTemplateComponent.get();
							if (templateComponent.isOptional() && !templateComponent.isDefaultActive()) {
								ContextComponent contextComponentDst = entityFactoryDst
										.createContextComponent(contextComponentSrc.getName(), contextRootDst);

								copyValues(contextComponentSrc, contextComponentDst, Arrays.asList("Id", "Name"), true,
										replaceFileLinks);

								persist(transaction, contextComponentDst);
							}
						}
					}
				}
			}
		}
	}

	private Measurement copyMeasurement(Measurement measurementSrc, List<ContextRoot> listContextRootsDst,
			boolean recursive, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(measurementSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		boolean append = Boolean.valueOf(properties.getOrDefault(PROPERTY_APPEND, DEFAULT_APPEND));

		if (null == ehDst) {
			LOG.trace("Importing Measurement '{}'", measurementSrc.getName());
			TestStep testStepSrs = entityManagerSrc.loadParent(measurementSrc, TestStep.class).get();
			TestStep testStepParentDst = (TestStep) mapSrcDstEntities
					.get(new EntityHolder(testStepSrs, entityManagerSrc)).getEntity();

			EntityType etMeasurement = modelManagerDst.getEntityType(Measurement.class);
			EntityType etTestStep = modelManagerDst.getEntityType(TestStep.class);

			Relation relTestStep = etMeasurement.getRelation(etTestStep);

			if (null == relTestStep) {
				throw new ApiCopyException("No relation to TestStep found at MeaResult!");
			}

			Filter filter = Filter.nameOnly(etMeasurement, measurementSrc.getName()).id(relTestStep,
					testStepParentDst.getID());

			Optional<Measurement> optMeasurement = fetchOne(searchServiceDst, Measurement.class, filter);
			Measurement measurementDst = null;

			if (optMeasurement.isPresent() && !append) {
				throw new ApiCopyException(String.format("TestStep '%s' has already a measurement with name '%s'!",
						testStepParentDst.getName(), measurementSrc.getName()));
			} else if (optMeasurement.isPresent()) {
				measurementDst = optMeasurement.get();
			} else {
				append = false;
			}

			Optional<TemplateTestStep> optTemplateTestStep = loadTemplateTestStep(testStepSrs);

			// If no ContextRoots to use with the newly created Measurement are passed into
			// this method, use ContextRoots
			// of any already existing Measurement under the parent test step.
			// First look in destination...
			if (listContextRootsDst.isEmpty()) {
				for (Measurement existingMeasurementDst : entityManagerDst.loadChildren(testStepParentDst,
						Measurement.class)) {
					listContextRootsDst.addAll(
							existingMeasurementDst.loadContexts(entityManagerDst, ContextType.values()).values());

					if (!listContextRootsDst.isEmpty()) {
						break;
					}
				}
			}

			// ...then, if nothing has been found, in source and try to find destination
			// counterpart in cache map:
			if (listContextRootsDst.isEmpty()) {
				for (Measurement existingMeasurementSrc : entityManagerSrc.loadChildren(testStepSrs,
						Measurement.class)) {
					EntityHolder eh = mapSrcDstEntities.get(new EntityHolder(existingMeasurementSrc, entityManagerSrc));
					if (null != eh) {
						listContextRootsDst.addAll(((Measurement) eh.getEntity())
								.loadContexts(entityManagerDst, ContextType.values()).values());
					}

					if (!listContextRootsDst.isEmpty()) {
						break;
					}
				}
			}

			// Still no ContextRoots found? Create them from the test step template:
			if (listContextRootsDst.isEmpty() && optTemplateTestStep.isPresent()) {
				optTemplateTestStep.get().getTemplateRoots()
						.forEach(tr -> listContextRootsDst.add(entityFactoryDst.createContextRoot(tr)));
			}

			if (measurementDst == null) {
				measurementDst = entityFactoryDst.createMeasurement(measurementSrc.getName(), testStepParentDst,
						listContextRootsDst.toArray(new ContextRoot[listContextRootsDst.size()]));
			}

			copyValues(measurementSrc, measurementDst, Arrays.asList("Id", "Name"), true, replaceFileLinks);

			ehDst = new EntityHolder(measurementDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);

			Map<ContextType, ContextRoot> mapContextRootsSrc = testStepParentDst.loadContexts(entityManagerDst,
					ContextType.values());
			Map<ContextType, ContextRoot> mapContextRootsDst = measurementDst.loadContexts(entityManagerDst,
					ContextType.values());

			copyContext(mapContextRootsSrc, mapContextRootsDst, optTemplateTestStep, transaction);

			mapContextRootsSrc = measurementSrc.loadContexts(entityManagerSrc, ContextType.values());
			mapContextRootsDst = measurementDst.loadContexts(entityManagerDst, ContextType.values());
			copyContext(mapContextRootsSrc, mapContextRootsDst, optTemplateTestStep, transaction);

			persist(transaction, measurementDst);

			Iterator<ContextRoot> contextRootIter = mapContextRootsDst.values().iterator();

			while (contextRootIter.hasNext()) {
				ContextRoot contextRootForUpdate = contextRootIter.next();
				persist(transaction, contextRootForUpdate);
			}

			if (recursive) {
				List<WriteRequest> listWriteRequests = new ArrayList<>();

				Map<String, Channel> mapChannels = new HashMap<>();

				if (append && !isSrcChannelsValid(entityManagerSrc.loadChildren(measurementSrc, Channel.class),
						entityManagerDst.loadChildren(measurementDst, Channel.class))) {
					throw new ApiCopyException(
							"Appending not possible. All channels of existing measurement have to be in the source measurement");
				}

				for (Channel channel : entityManagerSrc.loadChildren(measurementSrc, Channel.class)) {
					Channel channelDst = copyChannel(channel, transaction);
					mapChannels.put(channel.getName(), channelDst);
				}

				for (ChannelGroup channelGroup : entityManagerSrc.loadChildren(measurementSrc, ChannelGroup.class)) {
					Map<String, MeasuredValues> existingMeasValuesToAppend = new HashMap<>();

					Integer numberOfValues = 0;
					if (append) {
						Optional<ChannelGroup> optChannelGroup = fetchOneChannelGroup(channelGroup);

						if (optChannelGroup.isPresent()) {
							numberOfValues = optChannelGroup.get().getNumberOfValues();
							existingMeasValuesToAppend = getMeasurementValuesMap(entityManagerDst,
									optChannelGroup.get());
						}

					}

					ChannelGroup channelGroupDst = copyChannelGroup(channelGroup, transaction, append);

					for (MeasuredValues measuredValues : entityManagerSrc.readMeasuredValues(ReadRequest
							.create(channelGroup).valuesMode(ValuesMode.STORAGE).allChannels().allValues())) {
						LOG.trace("Importing MeasuredValues '{}'", measuredValues.getName());
						if (!mapChannels.containsKey(measuredValues.getName())) {
							throw new ApiCopyException(
									String.format("Cannot find Channel %s in destination!", measuredValues.getName()));
						}

						Channel channelDst = mapChannels.get(measuredValues.getName());

						String mappedUnit = unitMapping.get(measuredValues.getUnit());

						if (!Strings.isNullOrEmpty(mappedUnit)) {
							measuredValues = MeasuredValuesHelper.mapUnitOfMeasuredValues(measuredValues, mappedUnit);
						}

						if (append) {

							channelDst = replaceChannelDst(channelDst, measurementDst, ehSrc, transaction);

							MeasuredValues measuredValuesOld = existingMeasValuesToAppend.get(channelDst.getName());

							if (measuredValuesOld == null) {
								measuredValuesOld = MeasuredValuesHelper.createEmptyValues(measuredValues.getName(),
										measuredValues.getScalarType(), measuredValues.getUnit(),
										measuredValues.getSequenceRepresentation(),
										measuredValues.getGenerationParameters(), measuredValues.isIndependent(),
										measuredValues.getAxisType(), numberOfValues);

							}

							measuredValues = MeasuredValuesHelper.appenMeasuredValues(measuredValuesOld,
									measuredValues);

						}

						listWriteRequests.add(createWriteRequest(channelGroupDst, channelDst, measuredValues));

					}
				}

				transaction.writeMeasuredValues(listWriteRequests);
			}
		}

		return (Measurement) ehDst.getEntity();
	}

	private boolean isSrcChannelsValid(List<Channel> channelsSrc, List<Channel> channelsDst) {
		boolean isSrcChannelsValid = true;

		Map<String, Channel> sourceMap = new HashMap<>();
		channelsSrc.forEach(srcChannel -> sourceMap.put(srcChannel.getName(), srcChannel));

		for (Channel channelDst : channelsDst) {
			if (sourceMap.get(channelDst.getName()) == null) {
				isSrcChannelsValid = false;
				break;
			}
		}

		return isSrcChannelsValid;
	}

	private Channel replaceChannelDst(Channel channelDst, Measurement measurementDst, EntityHolder ehSrc,
			Transaction transaction) {
		Channel channelNew = entityFactoryDst.createChannel(channelDst.getName(), measurementDst,
				channelDst.getQuantity());

		copyValues(channelDst, channelNew, Arrays.asList("Id"), false, replaceFileLinks);
		persist(transaction, channelNew);
		mapSrcDstEntities.put(ehSrc, new EntityHolder(channelNew, entityManagerDst));

		transaction.delete(Collections.singletonList(channelDst));
		return channelNew;
	}

	private Map<String, MeasuredValues> getMeasurementValuesMap(EntityManager entityManager,
			ChannelGroup channelGroup) {

		Map<String, MeasuredValues> returnVal = new HashMap<>();

		for (MeasuredValues measuredValues : entityManager.readMeasuredValues(
				ReadRequest.create(channelGroup).valuesMode(ValuesMode.STORAGE).allChannels().allValues())) {
			returnVal.put(measuredValues.getName(), measuredValues);
		}
		return returnVal;
	}

	private Channel copyChannel(Channel channelSrc, Transaction transaction) {
		EntityHolder ehSrc = new EntityHolder(channelSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {
			LOG.trace("Importing Channel '{}'", channelSrc.getName());
			Measurement measurementParentDst = (Measurement) mapSrcDstEntities
					.get(new EntityHolder(entityManagerSrc.loadParent(channelSrc, Measurement.class).get(),
							entityManagerSrc))
					.getEntity();

			EntityType etChannel = modelManagerDst.getEntityType(Channel.class);
			EntityType etMeasurement = modelManagerDst.getEntityType(Measurement.class);

			Relation relMeasurement = etChannel.getRelation(etMeasurement);

			if (null == relMeasurement) {
				throw new ApiCopyException("No relation to MeaResult found at MeaQuantity!");
			}

			// Find a valid quantity with the highest version
			/*
			 * TODO Quantity should actually implement Versionable, then we could just use
			 * org.eclipse.mdm.api.dflt.EntityManager.loadLatestValid(Class<T>, String) Bug:
			 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=553368
			 */

			Quantity quantity = entityManagerDst
					.loadAll(Quantity.class,
							quantityMapping.getOrDefault(channelSrc.getQuantity().getName(),
									channelSrc.getQuantity().getName()))
					.stream().filter(q -> q.getValue(Versionable.ATTR_VERSION_STATE).extract() == VersionState.VALID)
					.sorted(Comparator.comparing(q -> Integer.valueOf(q.getValue(Versionable.ATTR_VERSION).extract())))
					.findFirst().orElseThrow(() -> new ApiCopyException(String
							.format("Cannot find Quantity %s in destination!", channelSrc.getQuantity().getName())));

			Filter filter = Filter.nameOnly(etChannel, channelSrc.getName()).id(relMeasurement,
					measurementParentDst.getID());

			Channel channelDst = fetchOne(searchServiceDst, Channel.class, filter).orElseGet(
					() -> entityFactoryDst.createChannel(channelSrc.getName(), measurementParentDst, quantity));

			copyValues(channelSrc, channelDst, Arrays.asList("Id"), false, replaceFileLinks);

			persist(transaction, channelDst);

			ehDst = new EntityHolder(channelDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);
		}

		return (Channel) ehDst.getEntity();
	}

	private Optional<ChannelGroup> fetchOneChannelGroup(ChannelGroup channelGroupSrc) {
		Measurement measurementParentDst = (Measurement) mapSrcDstEntities
				.get(new EntityHolder(entityManagerSrc.loadParent(channelGroupSrc, Measurement.class).get(),
						entityManagerSrc))
				.getEntity();
		EntityType etChannelGroup = modelManagerDst.getEntityType(ChannelGroup.class);
		EntityType etMeasurement = modelManagerDst.getEntityType(Measurement.class);

		Relation relMeasurement = etChannelGroup.getRelation(etMeasurement);

		if (null == relMeasurement) {
			throw new ApiCopyException("No relation to MeaResult found at SubMatrix!");
		}

		Filter filter = Filter.nameOnly(etChannelGroup, channelGroupSrc.getName()).id(relMeasurement,
				measurementParentDst.getID());

		return fetchOne(searchServiceDst, ChannelGroup.class, filter);
	}

	private ChannelGroup copyChannelGroup(ChannelGroup channelGroupSrc, Transaction transaction, boolean append) {
		EntityHolder ehSrc = new EntityHolder(channelGroupSrc, entityManagerSrc);

		EntityHolder ehDst = mapSrcDstEntities.get(ehSrc);

		if (null == ehDst) {
			LOG.trace("Importing ChannelGroup '{}'", channelGroupSrc.getName());
			Measurement measurementParentDst = (Measurement) mapSrcDstEntities
					.get(new EntityHolder(entityManagerSrc.loadParent(channelGroupSrc, Measurement.class).get(),
							entityManagerSrc))
					.getEntity();

			EntityType etChannelGroup = modelManagerDst.getEntityType(ChannelGroup.class);
			EntityType etMeasurement = modelManagerDst.getEntityType(Measurement.class);

			Relation relMeasurement = etChannelGroup.getRelation(etMeasurement);

			if (null == relMeasurement) {
				throw new ApiCopyException("No relation to MeaResult found at SubMatrix!");
			}

			Optional<ChannelGroup> channelGroupOptional = fetchOneChannelGroup(channelGroupSrc);

			ChannelGroup channelGroupDst = null;

			if (append && channelGroupOptional.isPresent()) {
				transaction.delete(Collections.singletonList(channelGroupOptional.get()));
				channelGroupDst = entityFactoryDst.createChannelGroup(channelGroupSrc.getName(),
						channelGroupSrc.getNumberOfValues(), measurementParentDst);
			} else {
				channelGroupDst = channelGroupOptional
						.orElseGet(() -> entityFactoryDst.createChannelGroup(channelGroupSrc.getName(),
								channelGroupSrc.getNumberOfValues(), measurementParentDst));
			}

			copyValues(channelGroupSrc, channelGroupDst, Arrays.asList("Id"), false, replaceFileLinks);

			persist(transaction, channelGroupDst);

			ehDst = new EntityHolder(channelGroupDst, entityManagerDst);
			mapSrcDstEntities.put(ehSrc, ehDst);
		}

		return (ChannelGroup) ehDst.getEntity();
	}

	private Optional<TemplateTest> loadTemplateTest(Test testSrc) {
		Optional<TemplateTest> returnVal = Optional.empty();

		String templateTestName = templateManager.getTemplateTestName(testSrc);

		if (!Strings.isNullOrEmpty(templateTestName)) {
			returnVal = getCurrentValidTestTemplate(entityManagerDst.loadAll(TemplateTest.class, templateTestName));

		}

		return returnVal;
	}

	private Optional<TemplateTest> getCurrentValidTestTemplate(List<TemplateTest> listTemplateTests) {
		Optional<TemplateTest> returnVal = Optional.empty();

		if (listTemplateTests != null && !listTemplateTests.isEmpty()) {
			int highestVersion = -1;
			TemplateTest highestValidTemplate = null;

			for (TemplateTest templateTest : listTemplateTests) {
				Value value = templateTest.getValue(TemplateTest.ATTR_VERSION_STATE);

				if (VersionState.VALID.equals(value.extract())) {
					int version = Integer.parseInt(templateTest.getValue(TemplateTest.ATTR_VERSION).extract());

					if (version > highestVersion) {
						highestValidTemplate = templateTest;
						highestVersion = version;
					}
				}

			}

			if (highestValidTemplate != null) {
				returnVal = Optional.of(highestValidTemplate);
			}

		}
		return returnVal;
	}

	private Optional<TemplateTestStep> loadTemplateTestStep(TestStep testStepSrc) {
		Optional<TemplateTestStep> returnVal = Optional.empty();

		String templateTestStepName = templateManager.getTemplateTestStepName(testStepSrc);

		if (!Strings.isNullOrEmpty(templateTestStepName)) {
			returnVal = getCurrentValidTestStepTemplate(
					entityManagerDst.loadAll(TemplateTestStep.class, templateTestStepName));
		}

		return returnVal;
	}

	private Optional<TemplateTestStep> getCurrentValidTestStepTemplate(List<TemplateTestStep> listTemplateTestSteps) {
		Optional<TemplateTestStep> returnVal = Optional.empty();

		if (listTemplateTestSteps != null && !listTemplateTestSteps.isEmpty()) {
			int highestVersion = -1;
			TemplateTestStep highestValidTemplate = null;

			for (TemplateTestStep templateTestStep : listTemplateTestSteps) {
				Value value = templateTestStep.getValue(TemplateTestStep.ATTR_VERSION_STATE);

				if (VersionState.VALID.equals(value.extract())) {
					int version = Integer.parseInt(templateTestStep.getValue(TemplateTestStep.ATTR_VERSION).extract());

					if (version > highestVersion) {
						highestValidTemplate = templateTestStep;
						highestVersion = version;
					}
				}

			}

			if (highestValidTemplate != null) {
				returnVal = Optional.of(highestValidTemplate);
			}

		}
		return returnVal;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.eclipse.mdm.apicopy.control.ApiCopyTask#setProperties(java.util.Map)
	 */
	@Override
	public void setProperties(Map<String, String> properties) {
		this.properties = properties;
		this.replaceFileLinks = Boolean
				.valueOf(this.properties.getOrDefault(PROPERTY_REPLACE_FILELINKS, DEFAULT_REPLACE_FILES));
	}

}