blob: 0ba1064ba6139ff6d58ecb0fe22a44489a23db3b [file] [log] [blame]
/********************************************************************************
* Copyright (c) 2015-2020 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*
********************************************************************************/
package org.eclipse.mdm.api.odsadapter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.asam.ods.AoException;
import org.asam.ods.Column;
import org.asam.ods.ElemId;
import org.asam.ods.NameValueSeqUnit;
import org.asam.ods.T_LONGLONG;
import org.asam.ods.ValueMatrix;
import org.asam.ods.ValueMatrixMode;
import org.eclipse.mdm.api.base.adapter.Attribute;
import org.eclipse.mdm.api.base.adapter.EntityType;
import org.eclipse.mdm.api.base.adapter.Relation;
import org.eclipse.mdm.api.base.massdata.ExternalComponent;
import org.eclipse.mdm.api.base.massdata.ExternalComponentEntity;
import org.eclipse.mdm.api.base.massdata.ReadRequest;
import org.eclipse.mdm.api.base.massdata.ReadRequest.ValuesMode;
import org.eclipse.mdm.api.base.model.AxisType;
import org.eclipse.mdm.api.base.model.Channel;
import org.eclipse.mdm.api.base.model.Entity;
import org.eclipse.mdm.api.base.model.MeasuredValues;
import org.eclipse.mdm.api.base.model.ScalarType;
import org.eclipse.mdm.api.base.model.SequenceRepresentation;
import org.eclipse.mdm.api.base.model.Unit;
import org.eclipse.mdm.api.base.model.Value;
import org.eclipse.mdm.api.base.query.ComparisonOperator;
import org.eclipse.mdm.api.base.query.DataAccessException;
import org.eclipse.mdm.api.base.query.Filter;
import org.eclipse.mdm.api.base.query.Query;
import org.eclipse.mdm.api.base.query.QueryService;
import org.eclipse.mdm.api.base.query.Result;
import org.eclipse.mdm.api.odsadapter.query.ODSEntityType;
import org.eclipse.mdm.api.odsadapter.query.ODSModelManager;
import org.eclipse.mdm.api.odsadapter.utils.ODSConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
/**
* Reads mass data specified in {@link ReadRequest}s.
*
* @since 1.0.0
* @author Viktor Stoehr, Gigatronik Ingolstadt GmbH
*/
public final class ReadRequestHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(ReadRequestHandler.class);
public static final class ColumnAttributes {
private String name;
private String id;
private SequenceRepresentation sequenceRepresentation;
private double[] generationParameters;
private boolean independent;
private AxisType axisType;
private Short globalFlag;
private ScalarType dataType;
private ScalarType rawDataType;
private String unit;
private String meaQuantityId;
public ColumnAttributes(String name, String columnId, SequenceRepresentation sequenceRepresentation,
double[] generationParameters, boolean independent, AxisType axisType, Short globalFlag,
ScalarType rawDataType, String meaQuantityId) {
this.name = name;
this.id = columnId;
this.sequenceRepresentation = sequenceRepresentation;
this.generationParameters = generationParameters;
this.independent = independent;
this.axisType = axisType;
this.globalFlag = globalFlag;
this.rawDataType = rawDataType;
this.meaQuantityId = meaQuantityId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public SequenceRepresentation getSequenceRepresentation() {
return sequenceRepresentation;
}
public void setSequenceRepresentation(SequenceRepresentation sequenceRepresentation) {
this.sequenceRepresentation = sequenceRepresentation;
}
public double[] getGenerationParameters() {
return generationParameters;
}
public void setGenerationParameters(double[] generationParameters) {
this.generationParameters = generationParameters;
}
public boolean isIndependentColumn() {
return independent;
}
public void setIndependentColumn(boolean independent) {
this.independent = independent;
}
public AxisType getAxisType() {
return axisType;
}
public void setAxisType(AxisType axisType) {
this.axisType = axisType;
}
public Short getGlobalFlag() {
return globalFlag;
}
public void setGlobalFlag(Short globalFlag) {
this.globalFlag = globalFlag;
}
public ScalarType getRawDataType() {
return rawDataType;
}
public void setRawDataType(ScalarType rawDataType) {
this.rawDataType = rawDataType;
}
public ScalarType getDataType() {
return dataType;
}
public void setDataType(ScalarType dataType) {
this.dataType = dataType;
}
public String getMeaQuantityId() {
return meaQuantityId;
}
public String getUnit() {
return unit;
}
public void setUnit(String unit) {
this.unit = unit;
}
}
// ======================================================================
// Instance variables
// ======================================================================
private final ODSModelManager modelManager;
private final QueryService queryService;
// ======================================================================
// Constructors
// ======================================================================
/**
* Constructor.
*
* @param modelManager Used to gain access to value matrices.
*/
public ReadRequestHandler(ODSModelManager modelManager, QueryService queryService) {
this.modelManager = modelManager;
this.queryService = queryService;
}
// ======================================================================
// Public methods
// ======================================================================
/**
* Loads {@link MeasuredValues} as defined in given {@link ReadRequest}.
*
* @param readRequest The {@code MeasuredValues} request configuration.
* @return The loaded {@code MeasuredValues} are returned.
* @throws DataAccessException Thrown if unable to load {@code
* MeasuredValues} .
*/
public List<MeasuredValues> execute(ReadRequest readRequest) throws DataAccessException {
if (ValuesMode.STORAGE_PRESERVE_EXTCOMPS == readRequest.getValuesMode()) {
if (readRequest.getStartIndex() != 0) {
throw new DataAccessException(String.format(
"Start index of read request is %d, must be 0 for ValuesMode STORAGE_PRESERVE_EXTCOMPS!",
readRequest.getStartIndex()));
}
if (readRequest.getRequestSize() != 0) {
throw new DataAccessException(String.format(
"Request size of read request is %d, must be 0 for ValuesMode STORAGE_PRESERVE_EXTCOMPS!",
readRequest.getRequestSize()));
}
}
// Load ColumnAttributes:
Map<String, ColumnAttributes> mapColumnAttributes = getColumnAttributes(readRequest);
if (mapColumnAttributes.isEmpty()) {
return Collections.emptyList();
}
List<MeasuredValues> listMeasuredValues = new ArrayList<>();
if (ValuesMode.STORAGE_PRESERVE_EXTCOMPS == readRequest.getValuesMode() && externalComponentExists()) {
List<String> processedLcIds = new ArrayList<>();
Map<String, List<ExternalComponent>> mapExternalComponents = loadExternalComponents(mapColumnAttributes);
LOGGER.debug("Loaded external components for {} localcolumns.", mapExternalComponents.size());
// Create MeasuredValues for local columns whose external components have been
// loaded above:
for (Map.Entry<String, List<ExternalComponent>> entry : mapExternalComponents.entrySet()) {
String lcId = entry.getKey();
List<ExternalComponent> listExtComps = entry.getValue();
listMeasuredValues
.add(ODSConverter.fromExternalComponents(listExtComps, mapColumnAttributes.get(lcId)));
processedLcIds.add(lcId);
}
// implict
for (Map.Entry<String, ColumnAttributes> entry : mapColumnAttributes.entrySet()) {
String lcId = entry.getKey();
ColumnAttributes ca = entry.getValue();
if (entry.getValue().getSequenceRepresentation().isImplicit()) {
ScalarType scalarType = ca.getRawDataType();
if (scalarType == ScalarType.UNKNOWN) {
scalarType = ca.getDataType();
}
listMeasuredValues.add(scalarType.createMeasuredValues(ca.getName(), ca.getUnit(),
ca.getSequenceRepresentation(), ca.getGenerationParameters(), ca.isIndependentColumn(),
ca.getAxisType(), ca.getGlobalFlag(), Collections.emptyList()));
processedLcIds.add(lcId);
}
}
// remove processed localcolumns
processedLcIds.forEach(lcId -> mapColumnAttributes.remove(lcId));
}
if (!mapColumnAttributes.isEmpty()) {
// if there are still localcolumns left we load them be valuematrix
LOGGER.debug("Loading {} localcolumns by value matrix.", mapColumnAttributes.size());
ValueMatrix valueMatrix = null;
Column[] arrColumns = null;
try {
valueMatrix = getValueMatrix(readRequest);
List<Pair<Column, ColumnAttributes>> listColumnPairs = getODSColumns(readRequest, valueMatrix,
mapColumnAttributes);
// Get array of all ODS columns for later release:
arrColumns = listColumnPairs.stream().map(Pair::getLeft).toArray(Column[]::new);
// Create MeasuredValues for the remaining local columns:
NameValueSeqUnit[] nvsus = valueMatrix.getValue(arrColumns, readRequest.getStartIndex(),
readRequest.getRequestSize());
listMeasuredValues.addAll(ODSConverter.fromODSMeasuredValuesSeq(nvsus,
listColumnPairs.stream().map(Pair::getRight).toArray(ColumnAttributes[]::new)));
} catch (AoException aoe) {
throw new DataAccessException(aoe.reason, aoe);
} finally {
releaseColumns(arrColumns);
releaseValueMatrix(valueMatrix);
}
}
return listMeasuredValues;
}
/**
* Loads External Components for all ColumnAttributes which satisfy
* {@link SequenceRepresentation#isExternal()}
*
* @param mapColumnAttributes
* @return a map with the LocalColumn.Id as key and a list of its
* ExternalComponents as value
*/
private Map<String, List<ExternalComponent>> loadExternalComponents(
Map<String, ColumnAttributes> mapColumnAttributes) {
Map<String, List<ExternalComponent>> mapExternalComponents;
EntityType localColumnEntityType = modelManager.getEntityType("LocalColumn");
EntityType externalComponentEntityType = modelManager.getEntityType("ExternalComponent");
Relation relExternalComponentToLocalColumn = externalComponentEntityType.getRelation(localColumnEntityType);
List<Attribute> listExtCompAttributes = externalComponentEntityType.getAttributes();
boolean hasExtCompBitCount = (listExtCompAttributes.stream()
.filter(a -> ExternalComponentEntity.ATTR_BITCOUNT.equals(a.getName())).count() > 0);
boolean hasExtCompBitOffset = (listExtCompAttributes.stream()
.filter(a -> ExternalComponentEntity.ATTR_BITOFFSET.equals(a.getName())).count() > 0);
List<String> ids = new ArrayList<>();
for (Map.Entry<String, ColumnAttributes> me : mapColumnAttributes.entrySet()) {
if (me.getValue().getSequenceRepresentation().isExternal()) {
ids.add(me.getKey());
}
}
if (ids.isEmpty()) {
// no external sequence representations -> no externalcomponents to load
return Collections.emptyMap();
}
mapExternalComponents = queryService.createQuery().select(externalComponentEntityType.getAttributes())
.select(relExternalComponentToLocalColumn.getAttribute())
.order(relExternalComponentToLocalColumn.getAttribute(),
externalComponentEntityType.getAttribute(ExternalComponentEntity.ATTR_ORDINALNUMBER))
.fetch(Filter.and()
.add(ComparisonOperator.IN_SET
.create(relExternalComponentToLocalColumn.getAttribute(), ids.toArray(new String[0]))))
.stream()
.map(r -> new ImmutablePair<String, ExternalComponent>(
r.getValue(relExternalComponentToLocalColumn.getAttribute()).extract(),
new ExternalComponent(
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_TYPESPECIFICATION)).extract(),
r.getValue(
externalComponentEntityType.getAttribute(ExternalComponentEntity.ATTR_LENGTH))
.extract(),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_STARTOFFSET)).extract(),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_BLOCKSIZE)).extract(),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_VALUESPERBLOCK)).extract(),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_VALUEOFFSET)).extract(),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_FILENAMEURL))
.extractWithDefault(null),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_FLAGSFILENAMEURL))
.extractWithDefault(null),
r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_FLAGSSTARTOFFSET))
.extractWithDefault(null),
(hasExtCompBitCount
? r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_BITCOUNT)).extract()
: Short.valueOf((short) 0)),
(hasExtCompBitOffset
? r.getValue(externalComponentEntityType
.getAttribute(ExternalComponentEntity.ATTR_BITOFFSET)).extract()
: Short.valueOf((short) 0)))))
.collect(Collectors.groupingBy(Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toList())));
return mapExternalComponents;
}
private boolean externalComponentExists() {
try {
modelManager.getEntityType("ExternalComponent");
return true;
} catch (IllegalArgumentException e) {
return false;
}
}
boolean contains(Set<String> keys, String key) {
return keys.contains(key);
}
// ======================================================================
// Private methods
// ======================================================================
/**
* Gets the {@link Column}s from a {@link ValueMatrix} whose names are present
* in the [@link ColumnAttributes} of mapColumnAttributes.
*
* @param readRequest
* @param valueMatrix
* @param mapColumnAttributes
* @return
* @throws AoException
* @throws DataAccessException
*/
private List<Pair<Column, ColumnAttributes>> getODSColumns(ReadRequest readRequest, ValueMatrix valueMatrix,
Map<String, ColumnAttributes> mapColumnAttributes) throws AoException, DataAccessException {
List<Pair<Column, ColumnAttributes>> listColumnPairs = new ArrayList<>();
Map<String, Column> mapColumns = new HashMap<>();
// We consciously refrain from using valueMatrix.getColumns("*") if
// readRequest.isLoadAllChannels() == true
// and mapColumnAttributes contains entries for all columns of the submatrix as
// we would have to call
// column.getName() for every column and this would actually result in n + 1
// calls to the ODS source
// rather than n (n: number of entries in mapColumnAttributes) in the loop
// below:
for (ColumnAttributes ca : mapColumnAttributes.values()) {
String columnName = ca.getName();
Column[] columns = valueMatrix.getColumns(columnName);
if (columns == null || columns.length != 1) {
List<Column> colsToRelease = new ArrayList<>();
if (columns != null) {
colsToRelease.addAll(Arrays.asList(columns));
}
colsToRelease.addAll(mapColumns.values());
releaseColumns(colsToRelease.toArray(new Column[colsToRelease.size()]));
throw new DataAccessException(
String.format("Zero or more than one column with name '%s' found within submatrix ID %d!",
columnName, Long.valueOf(readRequest.getChannelGroup().getID())));
}
Column column = columns[0];
if (!readRequest.isLoadAllChannels()) {
for (Map.Entry<Channel, Unit> me : readRequest.getChannels().entrySet().stream()
.filter(e -> e.getKey().getName().equals(columnName)).collect(Collectors.toList())) {
Channel channel = me.getKey();
Unit unit = me.getValue();
if (!unit.nameEquals(channel.getUnit().getName())) {
column.setUnit(unit.getName());
}
}
}
mapColumns.put(columnName, column);
}
if (mapColumns.size() > 0) {
for (Map.Entry<String, ColumnAttributes> me : mapColumnAttributes.entrySet()) {
ColumnAttributes ca = me.getValue();
listColumnPairs.add(new ImmutablePair<Column, ColumnAttributes>(mapColumns.get(ca.getName()), ca));
}
}
return listColumnPairs;
}
/**
* Gets the {@link ColumnAttributes} for the local columns (channels) specified
* in a {@link ReadRequest}.
*
* @param readRequest The {@link ReadRequest} to use.
* @return A map with the LocalColumnIDs as keys.
*/
private Map<String, ColumnAttributes> getColumnAttributes(ReadRequest readRequest) {
Map<String, ColumnAttributes> mapColumnAttributes = new HashMap<>();
EntityType localColumnEntityType = modelManager.getEntityType("LocalColumn");
Attribute idAttr = localColumnEntityType.getIDAttribute();
Attribute nameAttr = localColumnEntityType.getNameAttribute();
Attribute submatrixAttr = localColumnEntityType.getAttribute("SubMatrix");
// Don't query GenerationParameters together with other non-ID attributes as
// Avalon dislikes this:
Query query1 = queryService.createQuery().select(Lists.newArrayList(idAttr, nameAttr,
localColumnEntityType.getAttribute("SequenceRepresentation"),
localColumnEntityType.getAttribute("IndependentFlag"), localColumnEntityType.getAttribute("GlobalFlag"),
localColumnEntityType.getAttribute("axistype"), localColumnEntityType.getAttribute("RawDatatype"),
localColumnEntityType.getAttribute("MeaQuantity")));
// OpenATFX can't handle multiple search conditions, so use just one...
Filter filter = Filter.and().add(
ComparisonOperator.EQUAL.create(submatrixAttr, Long.valueOf(readRequest.getChannelGroup().getID())));
Set<String> setColumnNames = readRequest.isLoadAllChannels() ? null
: readRequest.getChannels().keySet().stream().map(c -> c.getName()).collect(Collectors.toSet());
// ...and pick the columns of interest "manually":
for (Result result : query1.fetch(filter)) {
Map<String, Value> mapValues = result.getRecord(localColumnEntityType).getValues();
String columnName = mapValues.get("Name").extract();
if (null == setColumnNames || setColumnNames.contains(columnName)) {
String columnId = result.getRecord(localColumnEntityType).getID();
SequenceRepresentation seqRep = ValuesMode.CALCULATED == readRequest.getValuesMode()
? SequenceRepresentation.EXPLICIT
: mapValues.get("SequenceRepresentation").extract();
ColumnAttributes ca = new ColumnAttributes(columnName, columnId, seqRep, new double[0],
((short) mapValues.get("IndependentFlag").extract() != 0), mapValues.get("axistype").extract(),
mapValues.get("GlobalFlag").extractWithDefault(null), mapValues.get("RawDatatype").extract(),
mapValues.get("MeaQuantity").extract());
mapColumnAttributes.put(mapValues.get("Id").extract(), ca);
}
}
if (ValuesMode.CALCULATED != readRequest.getValuesMode()) {
Query query2 = queryService.createQuery().select(idAttr,
localColumnEntityType.getAttribute("GenerationParameters"));
for (Result result : query2.fetch(filter)) {
Map<String, Value> mapValues = result.getRecord(localColumnEntityType).getValues();
ColumnAttributes ca = mapColumnAttributes.get(mapValues.get("Id").extract());
if (ca != null) {
ca.setGenerationParameters(mapValues.get("GenerationParameters").extract());
}
}
}
applyUnitNamesAndDataTypesFallback(mapColumnAttributes.values());
return mapColumnAttributes;
}
/**
* Loads and sets the unit property of the given ColumnAttributes. This method
* does not use Queries with joins as openATFX does not support it. Instead, two
* queries are used. (LocalColumn.Id, LocalColumn.MeaQuantity) ->
* (MeaQuantity.Id, MeaQuantity.Unit) -> (Unit.Id, Unit.Name)
*
* @param mapColumnAttributes ColumnAttributes to set the unit.
*/
private void applyUnitNamesAndDataTypesFallback(Collection<ColumnAttributes> columnAttributes) {
EntityType meaQuantityEntityType = modelManager.getEntityType(Channel.class);
Query queryMq = queryService.createQuery().select(meaQuantityEntityType.getIDAttribute(),
meaQuantityEntityType.getAttribute("DataType"), meaQuantityEntityType.getAttribute("Unit"));
Filter filterMq = Filter.and().add(ComparisonOperator.IN_SET.create(meaQuantityEntityType.getIDAttribute(),
columnAttributes.stream().map(ca -> ca.getMeaQuantityId()).toArray(String[]::new)));
Map<String, String> mqId2unitId = new HashMap<>();
Map<String, ScalarType> mqId2datatype = new HashMap<>();
for (Result result : queryMq.fetch(filterMq)) {
Map<String, Value> mapValues = result.getRecord(meaQuantityEntityType).getValues();
mqId2unitId.put(result.getRecord(meaQuantityEntityType).getID(), mapValues.get("Unit").extract());
mqId2datatype.put(result.getRecord(meaQuantityEntityType).getID(), mapValues.get("DataType").extract());
}
EntityType unitEntityType = modelManager.getEntityType(Unit.class);
Query queryUnit = queryService.createQuery().select(unitEntityType.getIDAttribute(),
unitEntityType.getNameAttribute());
Filter filterUnit = Filter.and().add(ComparisonOperator.IN_SET.create(unitEntityType.getIDAttribute(),
mqId2unitId.values().stream().distinct().toArray(String[]::new)));
Map<String, String> mqId2unitName = new HashMap<>();
for (Result result : queryUnit.fetch(filterUnit)) {
Map<String, Value> mapValues = result.getRecord(unitEntityType).getValues();
mqId2unitName.put(result.getRecord(unitEntityType).getID(), mapValues.get("Name").extract());
}
columnAttributes.forEach(ca -> {
String unitId = mqId2unitId.get(ca.getMeaQuantityId());
ca.setUnit(mqId2unitName.get(unitId));
ca.setDataType(mqId2datatype.get(ca.getMeaQuantityId()));
});
}
/**
* Returns the {@link ValueMatrix} CORBA service object associated with given
* {@link ReadRequest}.
*
* @param readRequest The {@code ReadRequest}.
* @return The {@code ValueMatrix} is returned.
* @throws AoException Thrown if unable to load the {@code ValueMatrix}.
*/
private ValueMatrix getValueMatrix(ReadRequest readRequest) throws AoException {
Entity entity = readRequest.getChannelGroup();
T_LONGLONG iid = ODSConverter.toODSID(entity.getID());
T_LONGLONG aid = ((ODSEntityType) modelManager.getEntityType(entity)).getODSID();
ValueMatrixMode valueMatrixMode = ValueMatrixMode.CALCULATED;
switch (readRequest.getValuesMode()) {
case CALCULATED:
valueMatrixMode = ValueMatrixMode.CALCULATED;
break;
case STORAGE:
case STORAGE_PRESERVE_EXTCOMPS:
valueMatrixMode = ValueMatrixMode.STORAGE;
break;
default:
throw new DataAccessException(
String.format("Unsupported ValueMode %s!", readRequest.getValuesMode().name()));
}
return modelManager.getApplElemAccess().getValueMatrixInMode(new ElemId(aid, iid), valueMatrixMode);
}
/**
* Releases given {@link ValueMatrix} CORBA object.
*
* @param valueMatrix Will be released.
*/
private void releaseValueMatrix(ValueMatrix valueMatrix) {
if (valueMatrix == null) {
return;
}
try {
valueMatrix.destroy();
} catch (AoException aoe) {
// ignore
} finally {
valueMatrix._release();
}
}
/**
* Releases each CORBA {@link Column} object.
*
* @param columns Will be released.
*/
private void releaseColumns(Column[] columns) {
if (columns == null) {
return;
}
for (Column column : columns) {
try {
column.destroy();
} catch (AoException e) {
// ignore
} catch (Exception e) {
// ignore
} finally {
column._release();
}
}
}
}