Google Git
Sign in
eclipse / mdmbl / org.eclipse.mdm.openatfx.mdf / refs/heads/master / . / src / main / java / org / eclipse / mdm / openatfx / mdf / mdf4 / AoSessionWriter.java
blob: f204daa68c72918d8d80d7c805d460150ae868f0 [file] [log] [blame]
/********************************************************************************
* Copyright (c) 2015-2018 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*
********************************************************************************/
package org.eclipse.mdm.openatfx.mdf.mdf4;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import org.asam.ods.AoException;
import org.asam.ods.ApplicationElement;
import org.asam.ods.ApplicationRelation;
import org.asam.ods.ErrorCode;
import org.asam.ods.InstanceElement;
import org.asam.ods.NameValueUnit;
import org.asam.ods.SeverityFlag;
import org.asam.ods.TS_Union;
import org.asam.ods.TS_Value;
import org.asam.ods.T_ExternalReference;
import org.eclipse.mdm.openatfx.mdf.ConvertException;
import org.eclipse.mdm.openatfx.mdf.util.BitInputStream;
import org.eclipse.mdm.openatfx.mdf.util.ConfigurationHelper;
import org.eclipse.mdm.openatfx.mdf.util.FileUtil;
import org.eclipse.mdm.openatfx.mdf.util.LookupTableHelper;
import org.eclipse.mdm.openatfx.mdf.util.ODSHelper;
import org.eclipse.mdm.openatfx.mdf.util.ODSInsertStatement;
import org.eclipse.mdm.openatfx.mdf.util.ODSModelCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Main class for writing the MDF4 file content into an ASAM ODS session backed
* by an ATFX file.
*
* @author Christian Rechner
*/
public class AoSessionWriter
{
private static final Logger LOG = LoggerFactory.getLogger(AoSessionWriter.class);
private static final int FLAGS_BUFFER_SIZE = 1_000_000;
private static final int MAX_RECORDS_PER_FLAGS_READ = 1000;
private static final String ENDING_TEXTTABLE = ".texttable";
private static final String ENDING_LINEAR = ".linear";
/** The number format having 5 digits used for count formatting */
private final NumberFormat countFormat;
/** The XML parser object used for parsing the embedded XML contents */
private final MDF4XMLParser xmlParser;
/** helper class for lookup tables */
private final LookupTableHelper lookupTableHelper;
/** helper class for previews */
private PreviewHelper previewHelper;
// Properties
// replace channel name characters '[' with '{' and ']' with '}'
private boolean replaceSquareBrackets; // default = false
// whether to use the file name as AoMeasurement name, if false, property
// value of 'result_name' is used
private boolean useFileNameAsResultName = true; // default = true
// the name of the created AoMeasurement (only used if
// use_file_name_as_result_name=false)
private String resultName = "Messdaten"; // default = "Messdaten"
// the name suffix of the created AoMeasurement, e.g. if suffix='_123' then
// 'test.mf4' will be converted to 'test_123.mf4'
private String resultSuffix = ""; // default = null
// the mimetype of the created AoMeasurement
private String resultMimeType = "application/x-asam.aomeasurement.timeseries"; // default
// =
// "application/x-asam.aomeasurement.timeseries"
// whether to add a reference to the original file to the AoMeasurement
// instance
private boolean addMDF3FileAsResultAttachment; // default = false
private boolean skipScaleConversionChannels = false;
private boolean skipUINT64Channels = false;
// skip channels with DT_BYTESTR
// remove this once it is save to import channels with byte stream data
@Deprecated
private boolean skipByteStreamChannels = false;
private boolean writeFlagsFile = true;
private Path flagFile = null;
private boolean readOnlyHeader = false;
private Path customRatConfPath;
// skip empty channels
private boolean skipEmptyChannels = false;
private boolean skipExtraConversionChannels = false;
private boolean ignoreUnsupportedConversionTypes = false;
private boolean skipPreviews = true;
private boolean attachLinks2Tsts = false;
private boolean copyToSub = false;
private boolean ignoreATBLOCKs = false;
private Map<Long, Long> arraySubMatrixIDsByNrOfValues = new HashMap<>();
/**
* Constructor.
*/
public AoSessionWriter()
{
xmlParser = new MDF4XMLParser();
countFormat = new DecimalFormat("00000");
lookupTableHelper = new LookupTableHelper();
previewHelper = new PreviewHelper();
}
/**
* Appends the content of the MDF4 file to the ASAM ODS session.
*
* @param modelCache
* The application model cache.
* @param idBlock
* The IDBLOCK.
* @param props
* Map with additional attributes to be set.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
public void writeTst(ODSModelCache modelCache, IDBLOCK idBlock, Properties props) throws AoException, IOException
{
Path fileName = idBlock.getMdfFilePath().getFileName();
if (fileName == null)
{
throw new IOException("Unable to obtain file name!");
}
try
{
if (props != null)
{
// replace channel name characters '[' with '{' and ']' with '}'
if (props.containsKey("replace_square_brackets"))
{
// default = false
replaceSquareBrackets = Boolean.valueOf(props.getProperty("replace_square_brackets"));
}
// whether to use the file name as AoMeasurement name, if false,
// property value of 'result_name' is used
if (props.containsKey("use_file_name_as_result_name"))
{
// default = true
useFileNameAsResultName = Boolean.valueOf(props.getProperty("use_file_name_as_result_name"));
}
// the name of the created AoMeasurement (only used if
// use_file_name_as_result_name=false)
if (props.containsKey("result_name"))
{
// default = Messdaten
resultName = props.getProperty("result_name");
}
// the name suffix of the created AoMeasurement, e.g. if
// suffix='_123' then 'test.mf4' will be converted to
// 'test_123.mf4'
if (props.containsKey("result_suffix"))
{
// default = null
resultSuffix = props.getProperty("result_suffix");
}
// the mimetype of the created AoMeasurement
if (props.containsKey("result_mimetype"))
{
// default = application/x-asam.aomeasurement.timeseries
resultMimeType = props.getProperty("result_mimetype");
}
// whether to add a reference to the original file to the
// AoMeasurement instance
if (props.containsKey("attach_source_files"))
{
// default = false
addMDF3FileAsResultAttachment = Boolean.valueOf(props.getProperty("attach_source_files"));
}
if (props.containsKey("read_only_header"))
{
readOnlyHeader = Boolean.valueOf(props.getProperty("read_only_header"));
}
if (props.containsKey("skip_scale_conversion_channels"))
{
skipScaleConversionChannels = Boolean.valueOf(props.getProperty("skip_scale_conversion_channels"));
}
if (props.containsKey("skip_uint64_data_channels"))
{
skipUINT64Channels = Boolean.valueOf(props.getProperty("skip_uint64_data_channels"));
}
if (props.containsKey("skip_byte_stream_channels"))
{
skipByteStreamChannels = Boolean.valueOf(props.getProperty("skip_byte_stream_channels"));
}
if (props.containsKey("write_flags_file"))
{
writeFlagsFile = Boolean.valueOf(props.getProperty("write_flags_file"));
}
if (props.containsKey("skip_empty_channels"))
{
skipEmptyChannels = Boolean.valueOf(props.getProperty("skip_empty_channels"));
}
if (props.containsKey("attachments_to_tst"))
{
attachLinks2Tsts = Boolean.valueOf(props.getProperty("attachments_to_tst"));
}
if (props.containsKey("export_to_subfolder"))
{
copyToSub = Boolean.valueOf(props.getProperty("export_to_subfolder"));
}
if (props.containsKey("$${ignore_attachments}"))
{
ignoreATBLOCKs = Boolean.valueOf(props.getProperty("$${ignore_attachments}"));
}
if (props.containsKey("comment_replacement_strings")) {
String replacementStrings = props.getProperty("comment_replacement_strings");
ConfigurationHelper configHelper = ConfigurationHelper.getInstance();
configHelper.readReplacementsFromConfiguredString(replacementStrings);
}
if (props.containsKey("skip_extra_conversion_channels")) {
skipExtraConversionChannels = Boolean.valueOf(props.getProperty("skip_extra_conversion_channels"));
}
if (props.containsKey("ignore_unsupported_conversion_types")) {
ignoreUnsupportedConversionTypes = Boolean.valueOf(props.getProperty("ignore_unsupported_conversion_types"));
}
if (props.containsKey("skip_previews")) {
skipPreviews = Boolean.valueOf(props.getProperty("skip_previews"));
}
}
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "tst");
ins.setStringVal("iname", FileUtil.stripExtension(fileName.toString()));
ins.setStringVal("mdf_file_id", idBlock.getIdFile());
ins.setStringVal("mdf_version_str", idBlock.getIdVers());
ins.setLongVal("mdf_version", idBlock.getIdVer());
ins.setStringVal("mdf_program", idBlock.getIdProg());
ins.setLongVal("mdf_unfin_flags", idBlock.getIdUnfinFlags());
ins.setLongVal("mdf_custom_unfin_flags", idBlock.getIdCustomUnfinFlags());
// write properties
if (props != null)
{
MDF4Util.writeProperites(ins, props);
}
// set Relations, to environment
ins.setLongLongVal("env", 1L);
long iidTst = ins.execute();
previewHelper.setCache(modelCache);
previewHelper.setWriter(this);
// write 'AoMeasurement' instance
writeMea(modelCache, iidTst, idBlock);
}
catch (AoException | IOException e)
{
// make sure file with manually calculated values is deleted in case
// of errors
if (customRatConfPath != null && Files.exists(customRatConfPath))
{
try
{
Files.delete(customRatConfPath);
}
catch (IOException e2)
{
LOG.warn("failed to delete file with manually calculated values: '" + customRatConfPath + "'", e2);
}
}
if (flagFile != null && Files.exists(flagFile))
{
try
{
Files.delete(flagFile);
}
catch (IOException e2)
{
LOG.warn("failed to delete file with exported flags: '" + flagFile + "'", e2);
}
}
// preserve origin error
throw e;
}
}
/**
* Appends the content of the MDF4 file to the ASAM ODS session.
*
* @param modelCache
* The application model cache.
* @param ieTst
* The parent 'AoTest' instance.
* @param idBlock
* The IDBLOCK.
* @return The created AoMeasurement instance.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
private void writeMea(ODSModelCache modelCache, long iidTst, IDBLOCK idBlock) throws AoException, IOException
{
Path fileName = idBlock.getMdfFilePath().getFileName();
if (fileName == null)
{
throw new IOException("Unable to obtain file name!");
}
// create "AoMeasurement" instance
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "mea");
String mrName = null;
if (useFileNameAsResultName)
{
mrName = FileUtil.getResultName(fileName.toString(), resultSuffix);
}
else
{
mrName = resultName + resultSuffix;
}
ins.setStringVal("iname", mrName);
LOG.debug("MeaResult name: " + mrName);
ins.setStringVal("mt", resultMimeType);
if (addMDF3FileAsResultAttachment)
{
ins.setExtRefVal("attachment", new T_ExternalReference("original MDF-File", "", fileName.toString()));
}
// meta information
HDBLOCK hdBlock = idBlock.getHDBlock();
BLOCK block = hdBlock.getMdCommentBlock();
if (block instanceof TXBLOCK)
{
ins.setStringVal("desc", ((TXBLOCK) block).getTxData());
}
else if (block instanceof MDBLOCK)
{
xmlParser.writeHDCommentToMea(ins, ((MDBLOCK) block).getMdData());
}
Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(hdBlock.getStartTimeNs() / 1000000);
if (!hdBlock.isLocalTime() && hdBlock.isTimeFlagsValid())
{ // UTC time
// given,
// calc
// local
cal.add(Calendar.MINUTE, hdBlock.getTzOffsetMin());
cal.add(Calendar.MINUTE, hdBlock.getDstOffsetMin());
}
ins.setDateVal("date_created", ODSHelper.asODSDate(cal.getTime()));
ins.setDateVal("mea_begin", ODSHelper.asODSDate(cal.getTime()));
ins.setLongLongVal("start_time_ns", hdBlock.getStartTimeNs());
ins.setShortVal("local_time", hdBlock.isLocalTime() ? (short) 1 : (short) 0);
ins.setShortVal("time_offsets_valid", hdBlock.isTimeFlagsValid() ? (short) 1 : (short) 0);
ins.setShortVal("tz_offset_min", hdBlock.getTzOffsetMin());
ins.setShortVal("dst_offset_min", hdBlock.getDstOffsetMin());
ins.setEnumVal("time_quality_class", hdBlock.getTimeClass());
ins.setShortVal("start_angle_valid", hdBlock.isStartAngleValid() ? (short) 1 : (short) 0);
ins.setShortVal("start_distance_valid", hdBlock.isStartDistanceValid() ? (short) 1 : (short) 0);
ins.setDoubleVal("start_angle_rad", hdBlock.getStartAngleRad());
ins.setDoubleVal("start_distance_m", hdBlock.getStartDistanceM());
// write attachments (ATBLOCK)
if (!ignoreATBLOCKs && hdBlock.getLnkAtFirst() > 0)
{
Path root = idBlock.getMdfFilePath().getParent();
String name = idBlock.getMdfFilePath().getFileName().toString().replaceAll("\\.[^.]+", "");
T_ExternalReference[] extRefs = hdBlock.getAtFirstBlock().toExtRefs(root, copyToSub ? name : "");
if (attachLinks2Tsts)
{
NameValueUnit nvu = ODSHelper.createExtRefSeqNVU("attachments", extRefs);
modelCache.getApplicationElement("tst").getInstanceById(ODSHelper.asODSLongLong(iidTst)).setValue(nvu);
}
else
{
ins.setExtRefSeq("attachments", extRefs);
}
}
// set Relations, to test
ins.setLongLongVal("tst", iidTst);
long iidMea = ins.execute();
// write file history (FHBLOCK)
Long[] iidFh = writeFh(modelCache, iidTst, hdBlock);
// write channel hierarchy (CHBLOCK): not yet supported!
if (hdBlock.getLnkChFirst() > 0)
{
LOG.warn("Found CHBLOCK, currently not yet supported!");
}
if (!readOnlyHeader)
{
writeEv(modelCache, iidTst, iidFh, hdBlock);
// write submatrices
Map<String, Integer> meqNames = new HashMap<>();
writeSm(modelCache, iidMea, idBlock, hdBlock, meqNames);
}
}
/**
* Writes the content of all FHBLOCKS (file history) to the session.
*
* @param modelCache
* The application model cache.
* @param ieTst
* The parent 'AoTest' instance.
* @param hdBlock
* The HDBLOCK.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
private Long[] writeFh(ODSModelCache modelCache, long iidTst, HDBLOCK hdBlock) throws AoException, IOException
{
int no = 1;
FHBLOCK fhBlock = hdBlock.getFhFirstBlock();
List<Long> iids = new LinkedList<Long>();
while (fhBlock != null)
{
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "fh");
ins.setStringVal("iname", "fh_" + countFormat.format(no));
// meta information
Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(fhBlock.getStartTimeNs() / 1000000);
if (!fhBlock.isLocalTime() && fhBlock.isTimeFlagsValid())
{ // UTC
// time
// given,
// calc
// local
cal.add(Calendar.MINUTE, fhBlock.getTzOffsetMin());
cal.add(Calendar.MINUTE, fhBlock.getDstOffsetMin());
}
ins.setNameValueUnit(ODSHelper.createDateNVU("date", ODSHelper.asODSDate(cal.getTime())));
ins.setLongLongVal("start_time_ns", fhBlock.getStartTimeNs());
ins.setShortVal("local_time", fhBlock.isLocalTime() ? (short) 1 : (short) 0);
ins.setShortVal("time_offsets_valid", fhBlock.isTimeFlagsValid() ? (short) 1 : (short) 0);
ins.setShortVal("tz_offset_min", fhBlock.getTzOffsetMin());
ins.setShortVal("dst_offset_min", fhBlock.getDstOffsetMin());
if (fhBlock.getLnkMdComment() != 0)
{
xmlParser.writeFHCommentToFh(ins, fhBlock.getMdCommentBlock().getMdData());
}
// set Relations.
ins.setLongLongVal("tst", iidTst);
iids.add(ins.execute());
no++;
fhBlock = fhBlock.getFhNextBlock();
}
return iids.toArray(new Long[0]);
}
/**
* Writes the content of all EVBLOCKS (event) to the session.
*
* @param modelCache
* The application model cache.
* @param ieTst
* The parent 'AoTest' instance.
* @param hdBlock
* The HDBLOCK.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
private void writeEv(ODSModelCache modelCache, long iidTst, Long[] iidFh, HDBLOCK hdBlock)
throws AoException, IOException
{
int no = 1;
EVBLOCK evBlock = hdBlock.getEvFirstBlock();
while (evBlock != null)
{
if (evBlock.getLnkEvParent() != 0)
{
LOG.warn("Event Hirachy not supported.");
}
if (evBlock.getLnkEvRange() != 0)
{
LOG.warn("Event Range not supported.");
}
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "ev");
TXBLOCK name;
if ((name = evBlock.getEvTxNameBlock()) != null)
{
ins.setStringVal("iname", name.getTxData());
}
else
{
ins.setStringVal("iname", "ev_" + countFormat.format(no));
}
// Get Descpription from XMLComment block
BLOCK mdblock = evBlock.getMdCommentBlock();
if (mdblock != null)
{
if (mdblock instanceof MDBLOCK)
{
xmlParser.writeEVCommentToEv(ins, ((MDBLOCK) mdblock).getMdData());
}
else if (mdblock instanceof TXBLOCK)
{
ins.setStringVal("desc", ((TXBLOCK) mdblock).getTxData());
}
}
ins.setEnumVal("type", evBlock.getType());
// System.out.println("Synctype: "+evBlock.getSyncType());
ins.setEnumVal("sync_type", evBlock.getSyncType());
ins.setEnumVal("range_type", evBlock.getRangeType());
ins.setEnumVal("cause", evBlock.getCause());
ins.setShortVal("flags", evBlock.getFlags());
ins.setDoubleVal("sync_val", evBlock.getSyncValue());
// set Relations.
ins.setLongLongVal("tst", iidTst);
if (evBlock.getCreatorIndex() < iidFh.length && evBlock.getCreatorIndex() >= 0)
{
ins.setLongLongVal("creator", iidFh[evBlock.getCreatorIndex()]);
}
ins.execute();
no++;
evBlock = evBlock.getEvNextBlock();
}
}
/**
* Write the instances of 'AoSubMatrix'.
*
* @param modelCache
* The application model cache.
* @param iidMea
* The ID of the Parent Measurement.
* @param idBlock
* The IDBLOCK.
* @param hdBlock
* The HDBLOCK.
* @param meqNames
* Map with existing MeasurmentQuantities and their IDs.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
public void writeSm(ODSModelCache modelCache, long iidMea, IDBLOCK idBlock, HDBLOCK hdBlock,
Map<String, Integer> meqNames) throws AoException, IOException
{
Map<String, Long> meqInstances = new HashMap<String, Long>();
Map<String, Long> untInstances = new HashMap<String, Long>();
// iterate over data group blocks
int grpNo = 1;
DGBLOCK dgBlock = hdBlock.getDgFirstBlock();
while (dgBlock != null)
{
// if sorted, only one channel group block is available
CGBLOCK cgBlock = dgBlock.getCgFirstBlock();
if (cgBlock != null && cgBlock.getLnkCgNext() > 0)
{
throw new IOException(
"Only 'sorted' MDF4 files are supported, found 'unsorted' data! [DGBLOCK=" + dgBlock + "]");
}
// skip channel groups having no channels (or optionally no values)
boolean skipNoValues = skipEmptyChannels && cgBlock.getCycleCount() < 1;
if (cgBlock != null && !skipNoValues)
{
// check flags (not yet supported)
if (cgBlock.isBusEventChannel())
{
throw new IOException("Bus event data currently not supported! [DGBLOCK=" + dgBlock + "]");
}
// create SubMatrix instance
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "sm");
String smName = "sm_" + countFormat.format(grpNo);
ins.setStringVal("iname", smName);
LOG.debug("SubMatrix name: " + smName);
// write CGComment
BLOCK mdblock = cgBlock.getMdCommentBlock();
if (mdblock != null)
{
if (mdblock instanceof MDBLOCK)
{
xmlParser.writeCGCommentToCg(ins, ((MDBLOCK) mdblock).getMdData());
}
else if (mdblock instanceof TXBLOCK)
{
ins.setStringVal("desc", ((TXBLOCK) mdblock).getTxData());
}
}
// write DGComment
mdblock = dgBlock.getMdCommentBlock();
if (mdblock != null)
{
if (mdblock instanceof MDBLOCK)
{
xmlParser.writeDGCommentToCg(ins, ((MDBLOCK) mdblock).getMdData());
}
else if (mdblock instanceof TXBLOCK)
{
ins.setStringVal("dg_desc", ((TXBLOCK) mdblock).getTxData());
}
}
TXBLOCK txAcqName = cgBlock.getTxAcqNameBlock();
if (txAcqName != null)
{
ins.setStringVal("acq_name", txAcqName.getTxData());
}
SIBLOCK siAcqSource = cgBlock.getSiAcqSourceBlock();
if (siAcqSource != null)
{
writeSiBlock(ins, siAcqSource);
}
ins.setLongVal("rows", (int) cgBlock.getCycleCount());
// Relation to measurement
ins.setLongLongVal("mea", iidMea);
long iidSm = ins.execute();
ApplicationElement aeMea = modelCache.getApplicationElement("mea");
InstanceElement ieMea = aeMea.getInstanceById(ODSHelper.asODSLongLong(iidMea));
// write instances of
// AoMeasurementQuantity,AoLocalColumn,AoExternalReference
Map<String, Integer> mapMeq = new HashMap<String, Integer>();
SRBLOCK srBlock = cgBlock.getSrFirstBlock();
writeLc(modelCache, ieMea, iidSm, idBlock, dgBlock, cgBlock, srBlock, mapMeq, meqInstances, untInstances);
}
dgBlock = dgBlock.getDgNextBlock();
grpNo++;
}
}
/**
* Write the instances of 'AoLocalColumn' and 'AoMeasurementQuantity'.
*
* @param modelCache
* The application model cache.
* @param ieMea
* The parent 'AoMeasurement' instance.
* @param iidSm
* The parent 'AoSubMatrix' instance id.
* @param idBlock
* The IDBLOCK.
* @param dgBlock
* The DGBLOCK.
* @param cgBlock
* The CGBLOCK.
* @param srBlock
* Possible SRBLOCK (can be null)
* @param meqNames
* The so far cached 'AoMeasurementQuantity' names to the number of
* their occurrence, used for suffixing names with respective running
* number.
* @param meqInstances
* The so far cached 'AoMeasurementQuantity' names to their iids.
* @param untInstances
* The cached unit names to their iid, used in measurement quantity creation.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
private void writeLc(ODSModelCache modelCache, InstanceElement ieMea, long iidSm, IDBLOCK idBlock, DGBLOCK dgBlock,
CGBLOCK cgBlock, SRBLOCK srBlock, Map<String, Integer> meqNames, Map<String, Long> meqInstances,
Map<String, Long> untInstances) throws AoException, IOException
{
// Performance?
ApplicationElement aeLc = modelCache.getApplicationElement("lc");
ApplicationElement aeSm = modelCache.getApplicationElement("sm");
ApplicationRelation relLcSmLookup = modelCache.getApplicationRelation("lc", "sm", "LookupTable");
ApplicationRelation relLcSmPrev = modelCache.getApplicationRelation("lc", "sm", "Previews");
// iterate over channel blocks
CNBLOCK cnBlock = cgBlock.getCnFirstBlock();
List<String> channelNamesToSkip = getChannelNamesToSkip(cnBlock);
long iidMea = ODSHelper.asJLong(ieMea.getId());
while (cnBlock != null)
{
// build signal name
String meqName = readMeqName(cnBlock).trim();
// skip channel if configured
if (channelNamesToSkip.contains(meqName))
{
LOG.debug("Skipping channel " + meqName);
cnBlock = cnBlock.getCnNextBlock();
continue;
}
LOG.debug("CNBLOCK=" + cnBlock);
if ((cnBlock.getFlags() & 0x02) != 0 && cnBlock.getInvalBitPos() > 0)
{
if (writeFlagsFile)
{
// NOTE: flags are exported within writeEc()!
LOG.debug(
"channel with invalid values found, " + "export flags into separate file [CNBLOCK=" + cnBlock + "]");
}
else
{
LOG.debug("skipping channel with invalid values [CNBLOCK=" + cnBlock + "]");
continue;
}
}
// TODO cn_at_reference: attachments
if (cnBlock.getLnkAtReference().length > 0)
{
LOG.warn("Found channel 'cn_at_reference'>0, not yet supported ");
}
// replace channel names (concerto invalid characters)
if (replaceSquareBrackets)
{
meqName = meqName.replaceAll("\\[", "{");
meqName = meqName.replaceAll("\\]", "}");
}
LOG.debug("Channel name: " + meqName);
// check for duplicate signal names and add suffix (except time
// channel)
if (cnBlock.getChannelType() == 0)
{
Integer noChannels = meqNames.get(meqName);
if (noChannels == null)
{
noChannels = 0;
}
noChannels++;
meqNames.put(meqName, noChannels);
if (noChannels > 1)
{
meqName = meqName + "_" + noChannels;
}
}
CCBLOCK ccBlock = cnBlock.getCcConversionBlock();
// check whether channel has to be or shall be skipped
if (skipScaleConversionChannels && ccBlock != null && ccBlock.hasCCRefs())
{
LOG.info("Channel '" + meqName + "' with scale conversion rules in CCBlocks skipped: " + ccBlock);
cnBlock = cnBlock.getCnNextBlock();
continue;
}
else if (skipByteStreamChannels && 10 == cnBlock.getDataType())
{
// remove this block once it is save to import channels with
// byte stream data
LOG.info("Channel '" + meqName + "' with byte stream data skipped: " + ccBlock);
cnBlock = cnBlock.getCnNextBlock();
continue;
}
else if (10 == cnBlock.getDataType() && cnBlock.getLnkComposition() != 0)
{
LOG.info("Channel '" + meqName + "' with composed byte stream data skipped: " + ccBlock);
cnBlock = cnBlock.getCnNextBlock();
continue;
}
else if (64 == cnBlock.getBitCount()
&& (0 /* LEO */ == cnBlock.getDataType() || 1 /* BEO */ == cnBlock.getDataType()))
{
// UNSIGNED 64 bit: if cnBlock represents a time channel (sync_type ==
// 1)
// => it is considered save to interpret data as signed 64 bit
// => otherwise either fail or skip
if (1 != cnBlock.getSyncType())
{
// this is a data channel with 64 bit unsigned data; it is not
// possible to represent such data
// => either throw an error or skip channel
if (skipUINT64Channels)
{
LOG.info("Channel '" + meqName + "' with unsigned 64 bit data skipped: " + cnBlock);
cnBlock = cnBlock.getCnNextBlock();
continue;
}
else
{
throw new IOException("unable to write unsigned 64 bit data channel");
}
}
}
BLOCK compositionBlock = null;
if (cnBlock.getLnkComposition() != 0)
{
LOG.debug("Composition found!");
compositionBlock = cnBlock.getCompositionBlock();
}
// create instance of 'AoMeasurementQuantity' (if not yet existing)
Long iidMeq = meqInstances.get(meqName);
if (iidMeq == null)
{
iidMeq = createMeasurementQuantity(modelCache, cnBlock, ccBlock, compositionBlock, meqName, iidMea, null,
untInstances);
meqInstances.put(meqName, iidMeq);
}
// create 'AoLocalColumn' instance
long iidLc = createLocalColumn(modelCache, dgBlock, cgBlock, cnBlock, ccBlock, compositionBlock, meqName, iidMea,
iidSm, iidMeq, null);
// create previews
if (!skipPreviews && srBlock != null && (cnBlock.getChannelType() < 2 || cnBlock.getChannelType() > 4))
{
// previews should only be created for the data channels, not for any
// master or synchronization channel, therefore only link and create
// those previews
InstanceElement ieLc = aeLc.getInstanceById(ODSHelper.asODSLongLong(iidLc));
previewHelper.createPreviewChannels(ieMea, ieLc, meqName, srBlock, idBlock, cgBlock,
dgBlock, cnBlock, ccBlock, untInstances, relLcSmPrev);
}
// create 'AoExternalComponent' instance (if not a string data type
// (6-9) or VLSD-Channel (Type ==1) or virtual master channel (Type == 3))
Boolean setGlobalFlag = null;
if (!(cnBlock.getDataType() >= 6 && cnBlock.getDataType() <= 9 || cnBlock.getChannelType() == 1
|| cnBlock.getChannelType() == 3))
{
setGlobalFlag = writeEc(modelCache, iidLc, idBlock, dgBlock, cgBlock, cnBlock, ccBlock, compositionBlock,
dgBlock.getLnkData(), 0);
}
// set the global flag to the channel if returned by writeEc
if (setGlobalFlag != null)
{
InstanceElement ieLc = aeLc.getInstanceById(ODSHelper.asODSLongLong(iidLc));
TS_Union union = new TS_Union();
union.shortVal(Boolean.TRUE.equals(setGlobalFlag) ? (short) 15 : 0);
ieLc.setValue(new NameValueUnit("glb", new TS_Value(union, (short) 15), ""));
}
// create Table for Lookup conversion, if conversion type is 4 to 10
InstanceElement ieLc = aeLc.getInstanceById(ODSHelper.asODSLongLong(iidLc));
if (ccBlock != null && ccBlock.getType() >= 4 && ccBlock.getType() <= 10)
{
long iidLookup = createLookupTable(modelCache, ccBlock, ieMea, ieLc);
InstanceElement ieSmLookup = aeSm.getInstanceById(ODSHelper.asODSLongLong(iidLookup));
ieLc.createRelation(relLcSmLookup, ieSmLookup);
}
// jump to next channel
cnBlock = cnBlock.getCnNextBlock();
}
}
/**
* If configured, remove the .texttable and .linear channels which may
* have been added by a special feature of ETAS' INCA application.
* Otherwise those conversions are redundantly created and multiplied.
*
* @param name
* @return
* @throws IOException
*/
private List<String> getChannelNamesToSkip(CNBLOCK firstCnBlock) throws IOException
{
List<String> channelNames = new ArrayList<>();
if (!skipExtraConversionChannels)
{
return channelNames;
}
CNBLOCK cnBlock = firstCnBlock;
while (cnBlock != null)
{
channelNames.add(readMeqName(cnBlock));
cnBlock = cnBlock.getCnNextBlock();
}
List<String> namesToSkip = new ArrayList<>();
for (String currentName : channelNames)
{
String originalChannelName = null;
if (currentName.endsWith(ENDING_TEXTTABLE))
{
int index = currentName.indexOf(ENDING_TEXTTABLE);
originalChannelName = currentName.substring(0, index);
}
else if (currentName.endsWith(ENDING_LINEAR))
{
int index = currentName.indexOf(ENDING_LINEAR);
originalChannelName = currentName.substring(0, index);
}
if (originalChannelName != null && !originalChannelName.isEmpty() && channelNames.contains(originalChannelName))
{
namesToSkip.add(currentName);
}
}
return namesToSkip;
}
/**
* Create and insert the actual MeasurementQuantity This function also creates
* the corresponding unit if it doesn't already exist.
*
* @param modelCache
* Cache with model Object
* @param cnBlock
* The Channel Block.
* @param ccBlock
* The Conversion Block.
* @param compositionBlock
* The composition Block, may be null.
* @param meqName
* The Name the Measurement will have.
* @param iidMea
* The ID of the Measurement this Quantity is related to.
* @param mimeType
* The MIME-Type of this MeasurementQuantity. if mimeType==null, no
* MIME-Type is set.
* @param untInstances
* Map that contains iids of all created units so far. Key is the name
* of the unit
* @return The ID of the created MeasurmentQuantity
* @throws IOException
* If an I/O-error occurs.
* @throws AoException
*/
long createMeasurementQuantity(ODSModelCache modelCache, CNBLOCK cnBlock, CCBLOCK ccBlock, BLOCK compositionBlock,
String meqName, long iidMea, String mimeType, Map<String, Long> untInstances) throws IOException, AoException
{
int seqRep = getSeqRep(cnBlock, ccBlock);
double[] genParams = getGenerationParameters(ccBlock);
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "meq");
ins.setStringVal("iname", meqName);
// Write XML MetaData
BLOCK mdblock = cnBlock.getMdCommentBlock();
if (mdblock instanceof MDBLOCK)
{
xmlParser.writeCNCommentToMeq(ins, ((MDBLOCK) mdblock).getMdData());
}
else if (mdblock instanceof TXBLOCK)
{
ins.setStringVal("desc", ((TXBLOCK) mdblock).getTxData().trim());
}
// MIME-Type
if (mimeType != null)
{
ins.setStringVal("mt", mimeType);
}
boolean expandDataType = genParams != null && genParams.length > 0 && seqRep != 0 && seqRep != 7;
ins.setEnumVal("dt", getDataType(expandDataType, cnBlock, ccBlock));
if (ccBlock != null && ccBlock.isPhysicalRangeValid())
{
ins.setDoubleVal("min", ccBlock.getPhyRangeMin());
ins.setDoubleVal("max", ccBlock.getPhyRangeMax());
}
/*
* // CEBLOCK (extension block) info CEBLOCK ceBlock = cnBlock.getCeblock();
* if (ceBlock != null && ceBlock.getCeBlockDim() != null) { CEBLOCK_DIM ext
* = ceBlock.getCeBlockDim(); ins.setLongVal("NumberOfModule",
* ext.getNumberOfModule()); ins.setLongLongVal("Address",
* ext.getAddress()); ins.setStringVal("DIMDescription",
* ext.getDescription()); ins.setStringVal("ECUIdent", ext.getEcuIdent()); }
* else if (ceBlock != null && ceBlock.getCeBlockVectorCAN() != null) {
* CEBLOCK_VectorCAN ext = ceBlock.getCeBlockVectorCAN();
* ins.setLongLongVal("CANIndex", ext.getCanIndex());
* ins.setLongLongVal("MessageId", ext.getMessageId());
* ins.setStringVal("MessageName", ext.getMessageName());
* ins.setStringVal("SenderName", ext.getSenderName()); }
*/
ins.setLongLongVal("mea", iidMea);
SIBLOCK siBlock = cnBlock.getSiSourceBlock();
if (siBlock != null)
{
writeSiBlock(ins, siBlock);
}
// create 'AoUnit' instance if not yet existing
long uiid = writeUnit(modelCache, cnBlock, ccBlock, untInstances);
if (uiid != -1)
{
ins.setLongLongVal("unt", uiid);
}
// handle composition block and transform into according MeaQuantity
handleCompositionInfo(compositionBlock, ins);
return ins.execute();
}
/**
* Handles composition of blocks. Only array composition is supported so far.
*
* @param compositionBlock
* The composition Block, may be null.
* @param ins
* the insert statement for a MeaQuantity
* @throws IOException
*/
private void handleCompositionInfo(BLOCK compositionBlock, ODSInsertStatement ins) throws IOException
{
if (compositionBlock == null)
{
return;
}
CABLOCK arrayCompositionBlock = null;
if (compositionBlock instanceof CNBLOCK)
{
throw new IOException("Channel Composition based on compact structures not yet supported!");
}
else if (compositionBlock instanceof CABLOCK)
{
LOG.debug("Array block for composition: [" + compositionBlock + "]");
arrayCompositionBlock = (CABLOCK) compositionBlock;
}
if (arrayCompositionBlock != null)
{
handleArrayComposition(arrayCompositionBlock, ins);
}
}
/**
* Handles array compositions. The ods information on rank and dimension is
* read from the CABLOCK. Additionally the inverse layout flag has to be
* transferred as well, otherwise no client may correctly calculate the values
* from the tensor in this channel. This additional flag is not yet specified
* by ODS, but will be written as an application attribute 'columnOriented' to
* the MeaQuantity.
*
* @param arrayCompositionBlock
* @param ins
* the insert statement for a MeaQuantity
* @throws IOException
*/
private void handleArrayComposition(CABLOCK arrayCompositionBlock, ODSInsertStatement ins) throws IOException
{
if (arrayCompositionBlock == null || ins == null)
{
return;
}
// set the rank attribute of the MeaQuantity
int odsRank = arrayCompositionBlock.getNrOfDimensions();
ins.setLongVal(ODSModelCache.ATTR_NAME_MEAQU_RANK, odsRank);
// set the dimension attribute of the MeaQuantity
long[] longDimSizes = arrayCompositionBlock.getDimSizes();
int[] odsDimension = new int[longDimSizes.length];
for (int i = 0; i < longDimSizes.length; i++)
{
long currentDimSize = longDimSizes[i];
if (currentDimSize > Integer.MAX_VALUE)
{
throw new IOException("Found dimension (index=" + i + ") in MDF with more values (" + currentDimSize
+ ") than can be handled by ODS!");
}
odsDimension[i] = (int) currentDimSize;
}
ins.setLongSeq(ODSModelCache.ATTR_NAME_MEAQU_DIMENSION, odsDimension);
// set the ColumnOriented attribute of the MeaQuantity
boolean odsInverseLayout = arrayCompositionBlock.isFlagSet(CABLOCK.FLAG_INVERSE_LAYOUT);
ins.setBooleanVal(ODSModelCache.ATTR_NAME_MEAQU_COL_ORIENTED_TENSOR, odsInverseLayout);
// TODO handle working points
// handle scaling axes
if (arrayCompositionBlock.isFlagSet(CABLOCK.FLAG_SCALING_AXIS_DEFINED))
{
if (arrayCompositionBlock.isFlagSet(CABLOCK.FLAG_FIXED_AXES))
{
// throw new IOException("Handling of fixed scaling axes for arrays is
// not yet implemented!");
}
else
{
// throw new IOException("Handling of variable scaling axes for arrays
// is not yet implemented!");
}
}
}
/**
* Create and insert the actual LocalColumn.
*
* @param modelCache
* Cache with model Object
* @param dgBlock
* The Data Group Block.
* @param cgBlock
* The Channel Group Block.
* @param cnBlock
* The Channel Block.
* @param ccBlock
* The Conversion Block.
* @param compositionBlock
* The composition Block.
* @param lcName
* The Name the Local Column will have.
* @param iidMea
* The ID of the Measurement this LocalColumn's SubMatrix will be
* related to in case of array composition.
* @param iidSm
* The ID of the SubMatrix this LocalColumn is related to.
* @param iidMeq
* The ID of the MeasurementQuantity this LocalColumn is related to.
* @param mimeType
* The MIME-Type of this MeasurementQuantity. if mimeType==null, no
* MIME-Type is set.
* @return The ID of the created MeasurmentQuantity
* @throws IOException
* If an I/O-error occurs.
* @throws AoException
*/
long createLocalColumn(ODSModelCache modelCache, DGBLOCK dgBlock, CGBLOCK cgBlock, CNBLOCK cnBlock, CCBLOCK ccBlock,
BLOCK compositionBlock, String lcName, long iidMea, long iidSm, long iidMeq, String mimeType)
throws IOException, AoException
{
int seqRep = getSeqRep(cnBlock, ccBlock);
double[] genParams = getGenerationParameters(ccBlock);
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "lc");
ins.setStringVal("iname", lcName);
// MIME-Type
if (mimeType != null)
{
ins.setStringVal("mt", mimeType);
}
// global flag
ins.setShortVal("glb", (short) 15);
// sequence_representation: string channels cannot be referenced in ASAM
// ODS: read and write external
if (cnBlock.getDataType() >= 6 && cnBlock.getDataType() <= 9 && cnBlock.getChannelType() != 1)
{ // 6 To 9:
// Datatype
// String
String[] stringDataValues = readStringDataValues(dgBlock, cgBlock, cnBlock);
ins.setEnumVal("srp", 0);
ins.setNameValueUnit(ODSHelper.createStringSeqNVU("val", stringDataValues));
LOG.info("Unable to reference into MDF4, extracting string values. [Channel=" + lcName + "]");
}
else if (cnBlock.getChannelType() == 1)
{
// Read VLSDChannel values.
ins.setEnumVal("srp", 0);
LOG.info("Variable Length Channel! [Channel=" + lcName + "]");
insertVLSDValues(ins, dgBlock, cgBlock, cnBlock);
}
else
{
ins.setEnumVal("srp", seqRep);
}
// independent flag
short idp = cnBlock.getSyncType() > 0 ? (short) 1 : (short) 0;
ins.setShortVal("idp", idp);
// generation parameters
ins.setDoubleSeq("par", genParams);
// raw_datatype
int valueType = getValueType(cnBlock);
int rawDataType = getRawDataTypeForValueType(valueType, cnBlock);
ins.setEnumVal("rdt", rawDataType);
// axistype
int axistype = cnBlock.getSyncType() == 0 ? 1 : 0;
ins.setEnumVal("axistype", axistype);
// minimum/maximum
if (cnBlock.isValueRangeValid())
{
ins.setDoubleVal("min", cnBlock.getValRangeMin());
ins.setDoubleVal("max", cnBlock.getValRangeMax());
}
// identify the actual SubMatrix to connect to
long actualSubMatrixId = identifySubMatrixIdToConnect(modelCache, cgBlock, compositionBlock, iidSm, iidMea);
// relation to submatrix
ins.setLongLongVal("sm", actualSubMatrixId);
ins.setLongLongVal("meq", iidMeq);
long iidLc = ins.execute();
return iidLc;
}
/**
* For array composition, the nrOfValues for the SubMatrix has to be
* corrected. An according array channel will have the following length:
* cycleCount * dimSize1 * ... * dimSizeN Therefore a new SubMatrix with this
* nrOfRows may have to be created. If one with this length was already
* created, its ID is returned. If no composition was passed, the original
* SubMatrix ID is returned.
*
* @param modelCache
* Cache with model Objects.
* @param cgBlock
* The CGBLOCK.
* @param compositionBlock
* The composition Block.
* @param originalSmId
* If compositionBlock is null, this ID will be returned.
* @param iidMea
* The measurement ID to connect a newly created SubMatrix to.
* @return SubMatrix ID to be used for the LocalColumn, for which this method
* was called.
* @throws AoException
*/
private long identifySubMatrixIdToConnect(ODSModelCache modelCache, CGBLOCK cgBlock, BLOCK compositionBlock,
long originalSmId, long iidMea) throws AoException
{
long actualSubMatrixId = originalSmId;
if (compositionBlock instanceof CABLOCK && iidMea > 0)
{
long smNrOfValues = cgBlock.getCycleCount();
long[] dimSizes = ((CABLOCK) compositionBlock).getDimSizes();
for (long dimSize : dimSizes)
{
smNrOfValues *= dimSize;
}
Long existingArraySubmatrixId = arraySubMatrixIDsByNrOfValues.get(smNrOfValues);
if (existingArraySubmatrixId == null)
{
// create array SubMatrix instance
ODSInsertStatement smIns = new ODSInsertStatement(modelCache, "sm");
String smName = "sm_arrays_" + smNrOfValues;
smIns.setStringVal("iname", smName);
LOG.debug("SubMatrix name: " + smName);
smIns.setLongVal("rows", (int) smNrOfValues);
// Relation to measurement
smIns.setLongLongVal("mea", iidMea);
actualSubMatrixId = smIns.execute();
arraySubMatrixIDsByNrOfValues.put(smNrOfValues, actualSubMatrixId);
}
else
{
actualSubMatrixId = existingArraySubmatrixId;
}
}
return actualSubMatrixId;
}
/**
* Read values from an VLSD-Channel and write them directly to file.
*
* @param ins
* The Insertstatement to use.
* @param cnBlock
* The VLSD-Channel
* @throws IOException
*/
private void insertVLSDValues(ODSInsertStatement ins, DGBLOCK dgBlock, CGBLOCK cgBlock, CNBLOCK cnBlock)
throws IOException
{
BLOCK blk = cnBlock.getDataBlock();
if (blk == null)
{
// assume empty SDBLOCK
ins.setNameValueUnit(ODSHelper.createStringSeqNVU("val", new String[0]));
return;
}
else if (!(blk instanceof SDBLOCK))
{
throw new RuntimeException("Found VLSD Block with no valid signal data.");
}
SDBLOCK sdBlock = (SDBLOCK) blk;
// start Reading the values.
SeekableByteChannel sbc = cnBlock.sbc;
long recordSize = dgBlock.getRecIdSize() + cgBlock.getDataBytes() + cgBlock.getInvalBytes();
long recStart = dgBlock.getLnkData() + 24L;
long[] offsets = new long[(int) cgBlock.getCycleCount()];
long valueBitSize = cnBlock.getBitCount() + cnBlock.getBitOffset();
int valueByteSize = (int) (valueBitSize % 8 == 0 ? valueBitSize / 8 : valueBitSize / 8 + 1);
long offset = cnBlock.getByteOffset();
ByteBuffer bb = ByteBuffer.allocate(valueByteSize);
// iterate over records and read offsets
long pos = recStart;
for (int i = 0; i < cgBlock.getCycleCount(); i++)
{
// read offset
bb.rewind();
sbc.position(pos + offset);
sbc.read(bb);
bb.rewind();
offsets[i] = MDF4Util.readValue(cnBlock.getBitOffset(), (int) cnBlock.getBitCount(), bb);
pos += recordSize;
}
// read data
// iterate over records
ByteBuffer signals = ByteBuffer.allocate((int) (sdBlock.getLength() - 24));
long signalpos = cnBlock.getLnkData() + 24L;
sbc.position(signalpos);
sbc.read(signals);
signals.rewind();
LinkedList<String> list = new LinkedList<String>();
for (int i = 0; i < cgBlock.getCycleCount(); i++)
{
signals.position((int) offsets[i]);
int size = (int) MDF4Util.readUInt32(signals.order(ByteOrder.LITTLE_ENDIAN));
// read record
byte[] record = new byte[size];
signals.get(record);
// build string value and append to list.
// Switch for different encodings.
String value;
switch (cnBlock.getDataType())
{
case 6: // ISO-8859
value = MDF4Util.readCharsISO8859(ByteBuffer.wrap(record), record.length);
break;
case 7: // UTF-8
value = MDF4Util.readCharsUTF8(ByteBuffer.wrap(record), record.length);
break;
case 8: // UFT-16LE
value = MDF4Util.readCharsUTF16(ByteBuffer.wrap(record), record.length, true);
break;
case 9: // UTF-16BE
value = MDF4Util.readCharsUTF16(ByteBuffer.wrap(record), record.length, false);
break;
default:
throw new IllegalArgumentException("Illegal String encoding. Other encodings for VLSD not supported.");
}
list.add(value);
}
ins.setNameValueUnit(ODSHelper.createStringSeqNVU("val", list.toArray(new String[0])));
}
/**
* Creates a Lookup-Table from the given conversion Block. (must not be null!)
*
* @param modelCache
* @param cc
* @return The iid of the SubMatrix
* @throws IOException
* @throws AoException
*/
private long createLookupTable(ODSModelCache modelCache, CCBLOCK ccBlock, InstanceElement ieMea, InstanceElement ieLc)
throws IOException, AoException
{
// Scale Lookups are not yet supported.
if (ccBlock.getType() == 4 || ccBlock.getType() == 5)
{
double[] keys = ccBlock.getSecondValues(true);
double[] values = ccBlock.getSecondValues(false);
return lookupTableHelper.createValueToValueTable(modelCache, ieMea, ieLc, keys, values, ccBlock.getType() == 4);
}
else if (ccBlock.getType() == 6)
{
double[] minKeys = ccBlock.getThirdValues(0);
double[] maxKeys = ccBlock.getThirdValues(1);
double[] values = ccBlock.getThirdValues(2);
return lookupTableHelper.createValueRangeToValueTable(modelCache, ieMea, ieLc, minKeys, maxKeys, values, ccBlock);
}
else if (ccBlock.getType() == 7)
{ // Value to Text/Scale lookup
double[] keys = ccBlock.getValues();
String[] values = ccBlock.getValuesForTextTable(ieLc.getName(), ignoreUnsupportedConversionTypes);
if (values.length != keys.length)
{
LOG.warn("Number of values and keys for Lookup-Table are not equal!");
}
return lookupTableHelper.createValueToTextTable(modelCache, ieMea, ieLc, keys, values, ccBlock);
}
else if (ccBlock.getType() == 8)
{ // Value Range to Text/Scale lookup
double[] keysMin = ccBlock.getSecondValues(true);
double[] keysMax = ccBlock.getSecondValues(false);
String[] values = ccBlock.getValuesForTextTable(ieLc.getName(), ignoreUnsupportedConversionTypes);
return lookupTableHelper.createValueRangeToTextTable(modelCache, ieMea, ieLc, keysMin, keysMax, values, ccBlock);
}
else if (ccBlock.getType() == 9)
{
String[] keys = ccBlock.getRefValues(ieLc.getName(), ignoreUnsupportedConversionTypes);
double[] values = ccBlock.getValuesForTextToValueTable();
return lookupTableHelper.createTextToValueTable(modelCache, ieMea, ieLc, keys, values, ccBlock);
}
else if (ccBlock.getType() == 10)
{
String channelName = ieLc.getName();
String[] keys = ccBlock.getSecondTexts(true, channelName, ignoreUnsupportedConversionTypes);
String[] values = ccBlock.getSecondTexts(false, channelName, ignoreUnsupportedConversionTypes);
return lookupTableHelper.createTextToTextTable(modelCache, ieMea, ieLc, keys, values, ccBlock);
}
else
{
LOG.warn("Unsupported Conversion.");
return 0;
}
}
/**
* Reads String values from a string channel.
*
* @param dgBlock
* The data group block.
* @param cgBlock
* The channel group block.
* @param cnBlock
* The channel block.
* @return The values as String-Array.
* @throws IOException
* If an input error occurs.
*/
private static String[] readStringDataValues(DGBLOCK dgBlock, CGBLOCK cgBlock, CNBLOCK cnBlock) throws IOException
{
List<String> list = new ArrayList<>();
SeekableByteChannel sbc = dgBlock.sbc;
int recordIdOffset = dgBlock.getRecIdSize();
long mapStart = dgBlock.getLnkData() + 24L;
long mapSize = (cgBlock.getDataBytes() + recordIdOffset) * cgBlock.getCycleCount();
// For huge string channels it is necessary to split the data block into
// smaller partitions, because otherwise the mapSize would exceed the
// Integer.MAX_VALUE size of the byte buffer allocation and cause an
// Exception. A good enough compromise should be a cycle count of 200.000.
// Also the buffer may exceed the VM's size if too big.
long restSize = mapSize;
int maxReadRecords = 200000; // cycle count of 200.000 allows up to around 10.7KB record size and yields a 100MB buffer in memory at record size 500B
int maxValue = (int) (cgBlock.getDataBytes() + recordIdOffset) * maxReadRecords;
while (restSize > 0)
{
int currentMapSize = maxValue;
int currentRecordCount = maxReadRecords;
if (restSize < maxValue)
{
currentMapSize = (int) restSize;
currentRecordCount = (int) (restSize / (cgBlock.getDataBytes() + recordIdOffset));
restSize = 0;
}
else
{
restSize -= maxValue;
}
ByteBuffer bb = ByteBuffer.allocate(currentMapSize);
sbc.position(mapStart);
sbc.read(bb);
bb.rewind();
mapStart += currentMapSize;
// iterate over records
for (int i = 0; i < currentRecordCount; i++)
{
// read record
byte[] dataRecord = new byte[(int) (cgBlock.getDataBytes() + recordIdOffset)];
bb.get(dataRecord);
// skip first bits and read value
BitInputStream bis = new BitInputStream(dataRecord);
int skipBits = (int) (recordIdOffset * 8L + cnBlock.getByteOffset() * 8L + cnBlock.getBitOffset());
bis.skip(skipBits);
byte[] b = bis.readByteArray((int) cnBlock.getBitCount());
// build string value and append to list.
// Switch for different encodings.
String value;
switch (cnBlock.getDataType())
{
case 6: // ISO-8859
value = MDF4Util.readCharsISO8859(ByteBuffer.wrap(b), b.length);
break;
case 7: // UTF-8
value = MDF4Util.readCharsUTF8(ByteBuffer.wrap(b), b.length);
break;
case 8: // UFT-16LE
value = MDF4Util.readCharsUTF16(ByteBuffer.wrap(b), b.length, true);
break;
case 9: // UTF-16BE
value = MDF4Util.readCharsUTF16(ByteBuffer.wrap(b), b.length, false);
break;
default:
throw new IllegalArgumentException("Illegal String encoding.");
}
list.add(value);
}
}
return list.toArray(new String[0]);
}
/**
* Write the instances of 'AoExternalComponent'. One instance is written for
* each data block.
*
* @param modelCache
* The application model cache.
* @param iidLc
* The instance id of the 'AoLocalColumn' instance.
* @param idBlock
* The IDBLOCK.
* @param dgBlock
* The DGBLOCK.
* @param cgBlock
* The CGBLOCK.
* @param cnBlock
* The CNBLOCK.
* @param ccBlock
* The CCBLOCK.
* @param compositionBlock
* The composition Block, may be null.
* @param sectionstart
* The link to data, can be a DL, RD or DT Block or HL, DZ, but these
* are not supported.
* @param parity
* Used for previews, where only each third record has to be addressed.
* If every record will be addressed, parity has to be set to 0.
* Otherwise valid values are 1, 2 and 3. (e.g. if parity==1 the first
* of each three records will be addressed)
*
* @return true, if the global flag of the channel has to be set to valid, false for invalid, null otherwise
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
Boolean writeEc(ODSModelCache modelCache, long iidLc, IDBLOCK idBlock, DGBLOCK dgBlock, CGBLOCK cgBlock,
CNBLOCK cnBlock, CCBLOCK ccBlock, BLOCK compositionBlock, long sectionstart, int parity)
throws AoException, IOException
{
Boolean setGlobalFlag = null;
if (isRatConv2ExtComp(ccBlock))
{
// NOTE: once CCBLOCK is no longer required, it should be removed
// from this method's signature!
setGlobalFlag = createCustomRatConvEC(modelCache, iidLc, idBlock, dgBlock, cgBlock, cnBlock, ccBlock, sectionstart, parity);
}
else
{
DLBLOCK currdl = null; // current list block
int dlindex = 0; // index in data list block.
int totalindex = 0; // number of blocks read;
long currblock = -1;
switch (BLOCK.getBlockType(idBlock.sbc, sectionstart))
{
case "##DT":
case "##RD":
currblock = sectionstart;
break;
case "##DL":
currdl = DLBLOCK.read(idBlock.sbc, sectionstart);
currblock = currdl.getLnkDlData()[0];
// perform check for records over blockborders.
if (currdl.breaksRecords(cgBlock.getDataBytes() + dgBlock.getRecIdSize() + cgBlock.getInvalBytes()))
{
throw new IOException("This data list cannot be read because records are splitted.");
}
break;
case "##DZ":
case "##HL":
throw new IOException("Zipped blocks cannot be parsed into the ODS-Format.");
}
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "ec");
while (currblock != -1)
{
ins.setStringVal("iname", "ec_" + countFormat.format(++totalindex));
Path mdfFilePath = idBlock.getMdfFilePath().getFileName();
if (mdfFilePath == null)
{
throw new IOException("mdfFilePath must not be null");
}
long startOffset = currblock + 24L;
ins.setLongVal("on", totalindex);
ins.setStringVal("fl", mdfFilePath.toString());
int vt = getValueType(cnBlock);
ins.setEnumVal("vt", vt);
ins.setLongLongVal("so", startOffset);
int valuesPerBlock = 1; // default is one value per block
if (compositionBlock != null)
{
if (compositionBlock instanceof CABLOCK)
{
// for array composition set valuesPerBlock to the product of all
// dim sizes
long[] dimSizes = ((CABLOCK) compositionBlock).getDimSizes();
for (long dimSize : dimSizes)
{
valuesPerBlock *= dimSize;
}
}
}
// TODO Strings, write number of Bytes? Spec4/61
ins.setLongVal("vb", valuesPerBlock);
// Calculate ByteSize and Offset
long bytesize;
long valueOffset;
long recSizeWoInval = cgBlock.getDataBytes() + dgBlock.getRecIdSize();
if (parity == 0)
{
bytesize = recSizeWoInval + cgBlock.getInvalBytes();
valueOffset = dgBlock.getRecIdSize() + cnBlock.getByteOffset() + cnBlock.getBitOffset() / 8;
}
else
{
// inval bytes once at the end
bytesize = 3 * recSizeWoInval + cgBlock.getInvalBytes();
valueOffset = (parity - 1) * recSizeWoInval + dgBlock.getRecIdSize() + cnBlock.getByteOffset()
+ cnBlock.getBitOffset() / 8;
}
ins.setLongVal("bs", (int) bytesize);
ins.setLongVal("vo", (int) valueOffset);
// get Cycle count from Block size.
int cycleCount = -1;
if (BLOCK.getBlockType(idBlock.sbc, currblock).equals(DTBLOCK.BLOCK_ID))
{
cycleCount = (int) ((DTBLOCK.read(idBlock.sbc, currblock).getLength() - 24) / bytesize);
}
else if (BLOCK.getBlockType(idBlock.sbc, currblock).equals(RDBLOCK.BLOCK_ID))
{
cycleCount = (int) ((RDBLOCK.read(idBlock.sbc, currblock).getLength() - 24) / bytesize);
}
else
{
}
// for array composition the component length must be adjusted like for
// the SubMatrix nrOfRows
if (compositionBlock != null)
{
if (compositionBlock instanceof CABLOCK)
{
long[] dimSizes = ((CABLOCK) compositionBlock).getDimSizes();
for (long dimSize : dimSizes)
{
cycleCount *= dimSize;
}
}
}
ins.setLongVal("cl", cycleCount);
// export flags
setGlobalFlag = exportFlags(ins, idBlock, dgBlock, cgBlock, cnBlock, startOffset, cycleCount);
// type spec is of type: dt_bit_* => write bit offset (bo) and bit count
// (bc)
boolean writeBitProps = vt > 26 && vt < 33;
short bitOffset = cnBlock.getBitOffset();
if ((bitOffset != 0 || cnBlock.getBitCount() % 8 != 0) && writeBitProps)
{
ins.setShortVal("bo", bitOffset);
ins.setShortVal("bc", (short) cnBlock.getBitCount());
}
// bind to local column and write instance
ins.setLongLongVal("lc", iidLc);
ins.execute();
// switch to next block or escape
if (currdl == null)
{
currblock = -1; // just a single data block
}
else
{
// search next block in list.
dlindex++;
if (dlindex < currdl.getCount())
{ // more blocks in list
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
// switch to next list, and start with first block?
if (currdl.getLnkDlNext() > 0)
{
currdl = currdl.getDlNextBlock();
dlindex = 0;
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
currblock = -1;
}
}
}
}
}
return setGlobalFlag;
}
/**
* Create a new Unit for a channel, if needed.
*
* @param modelCache
* The ODSModelCache in use.
* @param cnblock
* The Channel Block.
* @param ccBlock
* The Channel's conversion block.
* @param existingUnts
* Map containing Name (key) and ID (Value) of all created units.
* @return The id of the corresponding unit.
*/
private long writeUnit(ODSModelCache modelCache, CNBLOCK cnblock, CCBLOCK ccBlock, Map<String, Long> existingUnts)
throws AoException, IOException
{
// create 'AoUnit' instance if not yet existing
// Get unit name and metadata
String unitName = "";
BLOCK unitblk = cnblock.getMdUnitBlock();
MDBLOCK unitMetadata = null;
// Search for correct unit in channel CNBLOCK first
if (unitblk != null)
{
if (unitblk instanceof MDBLOCK)
{
unitName = xmlParser.extractCommentText(((MDBLOCK) unitblk).getMdData());
unitMetadata = (MDBLOCK) unitblk;
}
else if (unitblk instanceof TXBLOCK)
{
unitName = ((TXBLOCK) unitblk).getTxData().trim();
}
}
else
{ // Then look in CCBLOCK
if (ccBlock != null)
{
unitblk = ccBlock.getMdUnitBlock();
if (unitblk != null)
{
if (unitblk instanceof MDBLOCK)
{
unitName = xmlParser.extractCommentText(((MDBLOCK) unitblk).getMdData());
unitMetadata = (MDBLOCK) unitblk;
}
else if (unitblk instanceof TXBLOCK)
{
unitName = ((TXBLOCK) unitblk).getTxData().trim();
}
}
}
}
if (unitName.length() > 0)
{
Long iid = existingUnts.get(unitName);
if (iid == null)
{
// create unit instance
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "unt");
ins.setStringVal("iname", unitName);
ins.setDoubleVal("factor", 1d);
ins.setDoubleVal("offset", 0d);
if (unitMetadata != null)
{
xmlParser.writeCNCommentToUnit(ins, unitMetadata.getMdData());
}
iid = ins.execute();
existingUnts.put(unitName, iid);
}
return iid;
}
return -1;
}
/**************************************************************************************
* helper methods
**************************************************************************************/
/**
* Get the conversion parameters of the conversion block for 1:1, linear or
* rational conversion.
*
* @param ccBlock
* The Conversion Block.
* @return The parameters as Double-Array.
* @throws AoException
* in case of errors
*/
private static double[] getGenerationParameters(CCBLOCK ccBlock) throws AoException
{
// CCBLOCK may be null, assume explicit
if (ccBlock == null)
{
return new double[0];
}
int formula = ccBlock.getType();
// '1:1'
if (formula == 0)
{
return new double[0];
}
// 'linear'
else if (formula == 1)
{
return ccBlock.getValues();
}
// 'rational'
else if (formula == 2)
{
double[] genParams = ccBlock.getValues();
if (isRatConv2RawLinExt(ccBlock))
{
// sequence rep.: 'raw_linear_external'
return new double[]
{ genParams[2], genParams[1] };
}
else if (isRatConv2ExtComp(ccBlock))
{
// sequence rep.: 'external_component'
return new double[0];
}
else
{
throw new AoException(ErrorCode.AO_IMPLEMENTATION_PROBLEM, SeverityFlag.ERROR, 0,
"unable write channel with rational conversion for ccBlock: " + ccBlock);
}
}
// No parameters otherwise
return new double[0];
}
/**
* Writes the content of an SIBLOCK to the session.
*
* @param ins
* The insert statement to use.
* @param siBlock
* The SIBLOCK.
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
private void writeSiBlock(ODSInsertStatement ins, SIBLOCK siBlock) throws AoException, IOException
{
// si_tx_name
TXBLOCK txName = siBlock.getTxNameBlock();
if (txName != null)
{
ins.setStringVal("src_name", txName.getTxData());
}
// si_tx_path
TXBLOCK txPath = siBlock.getTxPathBlock();
if (txPath != null)
{
ins.setStringVal("src_path", txPath.getTxData());
}
// si_md_comment
BLOCK block = siBlock.getMdCommentBlock();
if (block instanceof TXBLOCK)
{
ins.setStringVal("src_cmt", ((TXBLOCK) block).getTxData());
}
else if (block instanceof MDBLOCK)
{
xmlParser.writeSICommentToCg(ins, ((MDBLOCK) block).getMdData());
}
ins.setEnumVal("src_type", siBlock.getSourceType());
ins.setEnumVal("src_bus", siBlock.getBusType());
ins.setShortVal("src_sim", (short) (siBlock.getFlags() & 1));
}
/**
* Read the name of a Channel.
*
* @param cnBlock
* The Channel Block
* @return The name.
* @throws IOException
*/
private static String readMeqName(CNBLOCK cnBlock) throws IOException
{
TXBLOCK nameblk = cnBlock.getCnTxNameBlock();
String meqName = "default";
if (nameblk != null)
{
meqName = nameblk.getTxData().trim();
}
else
{
LOG.warn("No channel name found!");
}
return meqName;
}
/**
* Returns the target ASAM ODS sequence representation for the external
* component description.<br/>
* List of MDF4 formula types:
* <ul>
* <li>0 = 1:1 Conversion</li>
* <li>1 = parametric, linear</li>
* <li>2 = rational conversion</li>
* <li>3 = ASAM-MCD2 Text formula</li>
* <li>4 = tabular with interpolation</li>
* <li>5 = tabular</li>
* <li>6 = Value Range to Value tabular lookup</li>
* <li>7 = Value to Text/Scale tabular lookup</li>
* <li>8 = Value Range to Text/Scale tabular lookup</li>
* <li>9 = Text to Value tabular lookup</li>
* <li>10 = Text to Text tabular lookup</li>
* </ul>
*
* @author Tobias Leemann
* @return The ASAM ODS sequence representation enum value.
* @throws ConvertException
*/
private static int getSeqRep(CNBLOCK cnBlock, CCBLOCK ccBlock) throws AoException
{
// CCBLOCK may be null, assume explicit with external Component
if (ccBlock == null)
{
return 7;
}
if (cnBlock.getChannelType() == 3)
{
// virtual master channel -> implicit_linear
return 2;
}
int formula = ccBlock.getType();
// '1:1 conversion formula' => external_component
if (formula == 0)
{
return 7;
}
// 'linear' => 'raw_linear_external'
else if (formula == 1)
{
return 8;
}
// 'rational conversion' => 'external_component'
else if (formula == 2)
{
if (isRatConv2RawLinExt(ccBlock))
{
// redirect => 'raw_linear_external'
return 8;
}
else if (isRatConv2ExtComp(ccBlock))
{
// manual calculation => 'external_component'
return 7;
}
else
{
throw new AoException(ErrorCode.AO_IMPLEMENTATION_PROBLEM, SeverityFlag.ERROR, 0,
"unable write channel with rational conversion for ccBlock: " + ccBlock);
}
}
// 'Text formula' => 'external_component'
else if (formula == 3)
{
LOG.error("Text formulas are not supported at the moment.");
return 7;
}
// 'ALL lookup conversions' => 'external_component'
else if (formula <= 10)
{
return 7;
}
else
{
LOG.warn("Unsupported Formula type " + formula);
return 7;
}
}
/**
* Returns the target ASAM ODS external component type specification enum
* value for a MDF4 channel description.<br/>
* The data type is determined by the signal data type and the number of
* bits.<br/>
*
* @param cnBlock
* The MDF3 CNBLOCK.
* @return The ASAM ODS type specification enumeration value.
* @throws IOException
* Unsupported MDF3 data type.
*/
private static int getValueType(CNBLOCK cnBlock) throws IOException
{
int dt = cnBlock.getDataType();
int nb = (int) cnBlock.getBitCount();
int bitOffset = cnBlock.getBitOffset();
// 0 = unsigned integer LEO
if (dt == 0)
{
if (nb == 8 && bitOffset == 0)
{ // 8 bit: dt_byte
return 1;
}
else if (nb == 16 && bitOffset == 0)
{ // 16 bit: dt_ushort
return 21;
}
else if (nb == 32 && bitOffset == 0)
{ // 32 bit: dt_ulong
return 23;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: dt_longlong
return 4;
}
else
{ // variable bit length: dt_bit_uint
return 29;
}
}
// 1 = unsigned integer BEO
else if (dt == 1)
{
if (nb == 8 && bitOffset == 0)
{ // 8 bit: dt_byte
return 1;
}
else if (nb == 16 && bitOffset == 0)
{ // 16 bit: dt_ushort_beo
return 22;
}
else if (nb == 32 && bitOffset == 0)
{ // 32 bit: dt_ulong_beo
return 24;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: dt_longlong_beo
return 9;
}
else
{ // variable bit length: dt_bit_uint_beo
return 30;
}
}
// 2 = signed integer LEO
else if (dt == 2)
{
if (nb == 8 && bitOffset == 0)
{ // 8 bit: dt_sbyte
return 19;
}
else if (nb == 16 && bitOffset == 0)
{ // 16 bit: dt_short
return 2;
}
else if (nb == 32 && bitOffset == 0)
{ // 32 bit: dt_long
return 3;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: dt_longlong
return 4;
}
else
{ // variable bit length: dt_bit_int
return 27;
}
}
// 3 = signed integer BEO
else if (dt == 3)
{
if (nb == 8 && bitOffset == 0)
{ // 8 bit: dt_byte
return 1;
}
else if (nb == 16 && bitOffset == 0)
{ // 16 bit: dt_short_beo
return 7;
}
else if (nb == 32 && bitOffset == 0)
{ // 32 bit: dt_long_beo
return 8;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: dt_longlong_beo
return 9;
}
else
{ // variable bit length: dt_bit_int_beo
return 28;
}
}
// 4 IEEE 754 floating-point format LEO
else if (dt == 4)
{
if (nb == 32 && bitOffset == 0)
{ // 32 bit: ieeefloat4
return 5;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: ieeefloat8
return 6;
}
else
{ // variable bit length: dt_bit_float
return 31;
}
}
// 5 = IEEE 754 floating-point format BEO
else if (dt == 5)
{
if (nb == 32 && bitOffset == 0)
{ // 32 bit: ieeefloat4_beo
return 10;
}
else if (nb == 64 && bitOffset == 0)
{ // 64 bit: ieeefloat8_beo
return 11;
}
else
{ // variable bit length: dt_bit_float_beo
return 32;
}
}
// 6-9 = String (NULL terminated): dt_string
else if (dt == 6 || dt == 7 || dt == 8 || dt == 9)
{
return 12;
}
// 10 = Byte Array: dt_bytestr
else if (dt == 10)
{
return 13;
}
else
{
// TODO
LOG.warn("Data type " + dt + " is not yet supported.");
return 13;
}
}
/**
* Returns the raw dataType for given type specification.
*
* @param typeSpec
* The TypeSpec.
* @param cnBlock
* The CNBLOCK.
* @return The raw dataType.
* @throws AoException
* unable to obtain raw datatype
*/
private static int getRawDataTypeForValueType(int typeSpec, CNBLOCK cnBlock) throws AoException
{
if (cnBlock.getChannelType() == 3)
{
// virtual master channel (implicit linear), calculated with generation
// parameters
return 7;
}
int ret = 0;
if (typeSpec == 0)
{ // dt_boolean
ret = 4; // DT_BOOLEAN
}
else if (typeSpec == 1)
{ // dt_byte
ret = 5; // DT_BYTE
}
else if (typeSpec == 2)
{ // dt_short
ret = 2; // DT_SHORT
}
else if (typeSpec == 3)
{ // dt_long
ret = 6; // DT_LONG
}
else if (typeSpec == 4)
{ // dt_longlong
ret = 8; // DT_LONGLONG
}
else if (typeSpec == 5)
{ // ieeefloat4
ret = 3; // DT_FLOAT
}
else if (typeSpec == 6)
{ // ieeefloat8
ret = 7; // DT_DOUBLE
}
else if (typeSpec == 7)
{ // dt_short_beo
ret = 2; // DT_SHORT
}
else if (typeSpec == 8)
{ // dt_long_beo
ret = 6; // DT_LONG
}
else if (typeSpec == 9)
{ // dt_longlong_beo
ret = 8; // DT_LONGLONG
}
else if (typeSpec == 10)
{ // ieeefloat4_beo
ret = 3; // DT_FLOAT
}
else if (typeSpec == 11)
{ // ieeefloat8_beo
ret = 7; // DT_DOUBLE
}
else if (typeSpec == 12)
{ // dt_string
ret = 1; // DT_STRING
}
else if (typeSpec == 13)
{ // dt_bytestr
ret = 11; // DT_BYTESTR
}
else if (typeSpec == 14)
{ // dt_blob
ret = 12; // DT_BLOB
}
else if (typeSpec == 15)
{ // dt_boolean_flags_beo
ret = 4; // DT_BOOLEAN
}
else if (typeSpec == 16)
{ // dt_byte_flags_beo
ret = 5; // DT_BYTE
}
else if (typeSpec == 17)
{ // dt_string_flags_beo
ret = 1; // DT_STRING
}
else if (typeSpec == 18)
{ // dt_bytestr_beo
ret = 11; // DT_BYTESTR
}
else if (typeSpec == 19)
{ // dt_sbyte
ret = 2; // DT_SHORT
}
else if (typeSpec == 20)
{ // dt_sbyte_flags_beo
ret = 2; // DT_SHORT
}
else if (typeSpec == 21)
{ // dt_ushort
ret = 6; // DT_LONG
}
else if (typeSpec == 22)
{ // dt_ushort_beo
ret = 6; // DT_LONG
}
else if (typeSpec == 23)
{ // dt_ulong
ret = 8; // DT_LONGLONG
}
else if (typeSpec == 24)
{ // dt_ulong_beo
ret = 8; // DT_LONGLONG
}
else if (typeSpec == 25)
{ // dt_string_utf8
ret = 1; // DT_STRING
}
else if (typeSpec == 26)
{ // dt_string_utf8_beo
ret = 1; // DT_STRING
}
// dt_bit_int [27], dt_bit_int_beo [28], dt_bit_uint [29],
// dt_bit_uint_beo [30]
else if (typeSpec == 27 || typeSpec == 28 || typeSpec == 29 || typeSpec == 30)
{
int dt = cnBlock.getDataType();
int nb = (int) cnBlock.getBitCount();
if ((dt == 0 || dt == 1) && nb >= 1 && nb <= 8)
{ // unsigned byte
ret = 5; // DT_BYTE
}
else if ((dt == 2 || dt == 3) && nb >= 1 && nb <= 8)
{ // signed
// byte
ret = 5; // DT_BYTE
}
else if ((dt == 0 || dt == 1) && nb >= 9 && nb <= 16)
{ // unsigned
// short
ret = 6; // DT_LONG
}
else if ((dt == 2 || dt == 3) && nb >= 9 && nb <= 16)
{ // signed
// short
ret = 2; // DT_SHORT
}
else if ((dt == 0 || dt == 1) && nb >= 17 && nb <= 32)
{ // unsigned
// int
ret = 8; // DT_LONGLONG
}
else if ((dt == 2 || dt == 3) && nb >= 17 && nb <= 32)
{ // int
ret = 6; // DT_LONG
}
else if ((dt == 0 || dt == 1) && nb >= 33)
{ // unsigned int >32
// bit
ret = 8; // DT_LONGLONG
}
else if ((dt == 2 || dt == 3) && nb >= 33)
{ // signed int >32 bit
ret = 8; // DT_LONGLONG
}
}
// dt_bit_float [31], dt_bit_float_beo [32]
else if (typeSpec == 31 || typeSpec == 32)
{
int dt = cnBlock.getDataType();
int nb = (int) cnBlock.getBitCount();
if ((dt == 2 || dt == 3 || dt == 11 || dt == 12 || dt == 15 || dt == 16) && (nb == 16 || nb == 32))
{ // half precision floating point, ieeefloat4,
// ieeefloat4_beo
ret = 3; // DT_FLOAT
}
else if ((dt == 2 || dt == 3 || dt == 11 || dt == 12 || dt == 15 || dt == 16) && nb == 64)
{ // ieeefloat8,
// ieeefloat8_beo
ret = 7; // DT_DOUBLE
}
// TODO Non-IEEE floats
}
// not found!
else
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0, "Unsupported typeSpec: " + typeSpec);
}
return ret;
}
/**
* Returns the target ASAM ODS measurement quantity data type for a MDF4
* channel description.<br/>
* The data type is determined by the formula, the signal data type and the
* number of bits.
*
* @param expandDataType
* Indicates whether to expand data type or not.
* @param cnBlock
* The MDF CNBLOCK.
* @param ccBlock
* The MDF CCBLOCK.
* @return The ASAM ODS data type.
* @throws IOException
* Unable to determine data type.
* @throws AoException
* in case of errors
*/
private static int getDataType(boolean expandDataType, CNBLOCK cnBlock, CCBLOCK ccBlock)
throws IOException, AoException
{
// CCBLOCK may be null, assume 1:1
int formula = 0;
if (ccBlock != null)
{
formula = ccBlock.getType();
}
if (cnBlock.getChannelType() == 3)
{
// virtual master channel (implicit linear), calculated with generation
// parameters
return 7;
}
int dt = cnBlock.getDataType();
int nb = (int) cnBlock.getBitCount();
// STRING
if (dt >= 6 && dt <= 9)
{
return 1; // DT_STRING
}
// 1 = parametric, linear
if (formula == 1)
{
if (nb == 0)
{
// virtual data channel that specifies a linear conversion,
// chapter 4.16 specifies them to have cn_data_type=0, cn_bit_count=0
// and a linear conversion rule; it will be mapped to DT_LONGONG, the
// only ODS datatype capable of storing the unsigned integer value range
return 8; // DT_LONGLONG
}
else if (nb == 1)
{
// 1 bit should be DT_BOOLEAN, but most of tools do not support
// this
return 5; // DT_BYTE
}
else if (nb >= 2 && nb <= 32)
{
return 3; // DT_FLOAT
}
else if (nb >= 33 && nb <= 64)
{
return 7; // DT_DOUBLE
}
}
// 2 = rational conversion
else if (formula == 2)
{
// values are calculated either way with real (64 bit ieee-754)
// double values no matter how many bits are used for internal
// values, therefore it is safe to return DT_DOUBLE here
return 7; // DT_DOUBLE
}
// 0= 1:1 conversion formula (Int = Phys)
else if (formula == 0)
{
if (dt == 10)
{ // ByteArray data
return 11; // DT_BYTESTR
}
else if (dt <= 9 && dt >= 6)
{ // STRING (6-9)
return 1; // DT_STRING
}
else if ((dt == 0 || dt == 1) && nb >= 1 && nb <= 8)
{ // dt_byte
return expandDataType ? 3 : 5; // [DT_FLOAT] DT_BYTE
}
else if ((dt == 2 || dt == 3) && nb >= 1 && nb <= 8)
{ // dt_sbyte
return expandDataType ? 3 : 2; // [DT_FLOAT] DT_SHORT
}
else if ((dt == 0 || dt == 1) && nb >= 9 && nb <= 16)
{ // dt_ushort,
// dt_ushort_beo
return expandDataType ? 3 : 6; // [DT_FLOAT] DT_LONG
}
else if ((dt == 2 || dt == 3) && nb >= 9 && nb <= 16)
{ // dt_short,
// dt_short_beo
return expandDataType ? 3 : 2; // [DT_FLOAT] DT_SHORT
}
else if ((dt == 0 || dt == 1) && nb >= 17 && nb <= 32)
{ // dt_ulong,
// dt_ulong_beo
return expandDataType ? 7 : 8; // [DT_DOUBLE] DT_LONGLONG
}
else if ((dt == 2 || dt == 3) && nb >= 17 && nb <= 32)
{ // dt_long,
// dt_long_beo
return expandDataType ? 3 : 6; // [DT_FLOAT] DT_LONG
}
else if ((dt == 0 || dt == 1) && nb >= 33)
{ // unsigned int >32
// bit
return expandDataType ? 7 : 8; // [DT_DOUBLE] DT_LONGLONG
}
else if ((dt == 2 || dt == 3) && nb >= 33 && nb <= 64)
{ // dt_longlong,
// dt_longlong_beo
return expandDataType ? 7 : 8; // [DT_DOUBLE] DT_LONGLONG
}
else if ((dt == 4 || dt == 5) && nb == 16)
{ // half precision floating point
return 3; // DT_FLOAT
}
else if ((dt == 4 || dt == 5) && nb == 32)
{ // ieeefloat4,
// ieeefloat4_beo
return 3; // DT_FLOAT
}
else if ((dt == 4 || dt == 5) && nb == 64)
{ // ieeefloat8,
// ieeefloat8_beo
return 7; // DT_DOUBLE
}
}
else if (formula == 9 || formula == 10)
{ // X to text
return 1; // DT_STRING
}
else if (formula == 4 || formula == 5 || formula == 6)
{
return 7; // DT_DOUBLE
}
else if (formula == 7 || formula == 8)
{
return getRawDataTypeForValueType(getValueType(cnBlock), cnBlock);
}
throw new IOException("Unsupported MDF4 datatype: " + cnBlock + "\n " + ccBlock);
}
/**
* Checks whether channel with rational conversion may be stored as a
* 'raw_linear_external'.
* <p>
* NOTE: This is a workaround to provide channels with formula == 2 with a
* reduced set of generation parameters. Such channel values are described
* with a rational conversion, which can not be described in ODS. To do so the
* generation parameters p1, p3 and p4 must be ZERO and p5 must be ONE.
* <p>
* <b>ATTENTION: THIS IS JUST A WORKAROUND UND HAS TO BE REMOVED AS SOON AS IT
* IS POSSIBLE TO DESCRIBE SUCH CHANNELS IN ODS!</b>
*
* @param ccBlock
* the conversion block
* @return true if the values must be calculated and stored in an external
* file
*/
@Deprecated
private static boolean isRatConv2RawLinExt(CCBLOCK ccBlock)
{
if (ccBlock == null)
{
return false;
}
else if (ccBlock.getType() == 2)
{
// rational conversion
double[] genParams = ccBlock.getValues();
if (genParams == null || genParams.length != 6)
{
return false;
}
else
{
boolean p1Zero = Double.compare(genParams[0], 0D) == 0;
boolean p4Zero = Double.compare(genParams[3], 0D) == 0;
boolean p5Zero = Double.compare(genParams[4], 0D) == 0;
boolean p6One = Double.compare(genParams[5], 1D) == 0;
return p1Zero && p4Zero && p5Zero && p6One;
}
}
return false;
}
/**
* Checks whether channel with rational conversion may be stored as a
* 'external_component'.
* <p>
* NOTE: This is a workaround to provide channels with formula == 2. Such
* channel values are described with a rational conversion, which can not be
* described in ODS. Therefore the whole value sequence is calculated and
* referenced as an 'external_component'.
* <p>
* <b>ATTENTION: THIS IS JUST A WORKAROUND UND HAS TO BE REMOVED AS SOON AS IT
* IS POSSIBLE TO DESCRIBE SUCH CHANNELS IN ODS!</b>
*
* @param ccBlock
* the conversion block
* @return true if the values must be calculated and stored in an external
* file
*/
@Deprecated
private static boolean isRatConv2ExtComp(CCBLOCK ccBlock)
{
if (ccBlock == null)
{
return false;
}
else if (ccBlock.getType() == 2)
{
// rational conversion
double[] genParams = ccBlock.getValues();
if (genParams == null || genParams.length != 6)
{
return false;
}
else
{
boolean p1Zero = Double.compare(genParams[0], 0D) != 0;
boolean p4Zero = Double.compare(genParams[3], 0D) != 0;
boolean p5Zero = Double.compare(genParams[4], 0D) != 0;
boolean p6One = Double.compare(genParams[5], 1D) != 0;
return p1Zero || p4Zero || p5Zero || p6One;
}
}
return false;
}
/**
* Write the instances of 'AoExternalComponent'.
* <p>
* One instance is written for all calculated vaues.
* <p>
* NOTE: This is a workaround to provide channels with formula == 2. Such
* channel values are described with a rational conversion, which can not be
* described in ODS. Therefore the whole value sequence is calculated and
* referenced as an 'external_component'.
* <p>
* <b>ATTENTION: THIS IS JUST A WORKAROUND UND HAS TO BE REMOVED AS SOON AS IT
* IS POSSIBLE TO DESCRIBE SUCH CHANNELS IN ODS!</b>
*
* @param modelCache
* The application model cache.
* @param iidLc
* The instance id of the 'AoLocalColumn' instance.
* @param idBlock
* The IDBLOCK.
* @param dgBlock
* The DGBLOCK.
* @param cgBlock
* The CGBLOCK.
* @param cnBlock
* The CNBLOCK.
* @param ccBlock
* The CCBLOCK.
* @param sectionstart
* The link to data, can be a DL, RD or DT Block or HL, DZ, but these
* are not supported.
* @param parity
* Used for previews, where only each third record has to be addressed.
* If every record will be addressed, parity has to be set to 0.
* Otherwise valid values are 1, 2 and 3. (e.g. if parity==1 the first
* of each three records will be addressed)
*
* @return true, if the global flag of the channel has to be set to valid, false for invalid, null otherwise
* @throws AoException
* Error writing to session.
* @throws IOException
* Error reading from MDF file.
*/
@Deprecated
private Boolean createCustomRatConvEC(ODSModelCache modelCache, long iidLc, IDBLOCK idBlock, DGBLOCK dgBlock,
CGBLOCK cgBlock, CNBLOCK cnBlock, CCBLOCK ccBlock, long sectionstart, int parity) throws AoException, IOException
{
if (parity != 0)
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"channel preview with formula == 2 (rational conversion) is currently not supported");
}
DLBLOCK currdl = null; // current list block
int dlindex = 0; // index in data list block.
long currblock = -1;
switch (BLOCK.getBlockType(idBlock.sbc, sectionstart))
{
case "##DT":
case "##RD":
currblock = sectionstart;
break;
case "##DL":
currdl = DLBLOCK.read(idBlock.sbc, sectionstart);
currblock = currdl.getLnkDlData()[0];
// perform check for records over blockborders.
if (currdl.breaksRecords(cgBlock.getDataBytes() + dgBlock.getRecIdSize() + cgBlock.getInvalBytes()))
{
throw new IOException("This data list cannot be read because records are splitted.");
}
break;
case "##DZ":
case "##HL":
throw new IOException("Zipped blocks cannot be parsed into the ODS-Format.");
}
if (currblock < 0)
{
return null;
}
if (customRatConfPath == null)
{
customRatConfPath = idBlock.getMdfFilePath().resolveSibling("rational_conversion.calc");
if (!Files.exists(customRatConfPath))
{
Files.createFile(customRatConfPath);
}
}
Boolean setGlobalFlag = null;
try (SeekableByteChannel channel = Files.newByteChannel(customRatConfPath, StandardOpenOption.APPEND))
{
ByteBuffer writeBuffer = ByteBuffer.wrap(new byte[8]);
writeBuffer.order(ByteOrder.LITTLE_ENDIAN);
long startOffset = channel.position();
long count = 0;
while (currblock != -1)
{
long vo = dgBlock.getRecIdSize() + cnBlock.getByteOffset() + cnBlock.getBitOffset() / 8;
long recSizeWoInval = cgBlock.getDataBytes() + dgBlock.getRecIdSize();
long bs = recSizeWoInval + cgBlock.getInvalBytes();
long so = currblock + 24L;
int cl;
if (BLOCK.getBlockType(idBlock.sbc, currblock).equals(DTBLOCK.BLOCK_ID))
{
cl = (int) ((DTBLOCK.read(idBlock.sbc, currblock).getLength() - 24) / bs);
}
else if (BLOCK.getBlockType(idBlock.sbc, currblock).equals(RDBLOCK.BLOCK_ID))
{
cl = (int) ((RDBLOCK.read(idBlock.sbc, currblock).getLength() - 24) / bs);
}
else
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"component length 'cl' is not allowed to be unknown");
}
short bo = cnBlock.getBitOffset();
short bc = bo != 0 && cnBlock.getBitCount() % 8 != 0 ? (short) cnBlock.getBitCount() : 0;
double[] p = ccBlock.getValues();
if (bo != 0 || bc != 0 || cnBlock.getBitCount() % 8 != 0)
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"bit count '" + bc + "' and bit offset '" + bo + "' is not supported for custom ration conversion");
}
int bits = (int) cnBlock.getBitCount();
int dt = cnBlock.getDataType();
boolean isInteger = dt > -1 && dt < 4;
boolean isReal = dt > 3 && dt < 6;
boolean isUnsigned = dt == 0 || dt == 1;
ByteOrder byteOrder = dt == 1 || dt == 3 || dt == 5 ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
for (int i = 0; i < cl; i++)
{
idBlock.sbc.position(so + i * bs + vo);
ByteBuffer byteBuffer = ByteBuffer.allocate(bits / 8);
byteBuffer.order(byteOrder);
idBlock.sbc.read(byteBuffer);
byteBuffer.rewind();
double internal = 0;
if (isInteger)
{
if (bits == 8)
{
// isUnsigned ? short : byte
internal = isUnsigned ? byteBuffer.get() & 0xFF : byteBuffer.get();
}
else if (bits == 16)
{
// isUnsigned ? int : short
internal = isUnsigned ? byteBuffer.getShort() & 0xFFFF : byteBuffer.getShort();
}
else if (bits == 32)
{
// isUnsigned ? long : int
internal = isUnsigned ? byteBuffer.getInt() & 0xFFFFFFFF : byteBuffer.getInt();
}
else if (bits == 64)
{
if (isUnsigned)
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"reading unsigned 64 bit integeres is not implemented");
}
// to support unsigned 64 bit, BigInteger has to be
// used -> performance costs
internal = byteBuffer.getLong();
}
else
{
String unsigned = isUnsigned ? "unsigned" : "signed";
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"customized reading of '" + bits + "' bit '" + unsigned + "' intergers is not implemented");
}
}
else if (isReal)
{
if (bits == 32)
{
// ieee754 floating point
internal = byteBuffer.getFloat();
}
else if (bits == 64)
{
// ieee754 floating point
internal = byteBuffer.getDouble();
}
else
{
String unsigned = isUnsigned ? "unsigned" : "signed";
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"customized reading of '" + bits + "' bit '" + unsigned + "' real is not implemented");
}
}
else
{
throw new AoException(ErrorCode.AO_BAD_PARAMETER, SeverityFlag.ERROR, 0,
"given value is neither an integer nor a real number");
}
double phys = (p[0] * Math.pow(internal, 2) + p[1] * internal + p[2])
/ (p[3] * Math.pow(internal, 2) + p[4] * internal + p[5]);
writeBuffer.putDouble(phys);
writeBuffer.rewind();
channel.write(writeBuffer);
writeBuffer.rewind();
count++;
}
// switch to next block or escape
if (currdl == null)
{
currblock = -1; // just a single data block
}
else
{
// search next block in list.
dlindex++;
if (dlindex < currdl.getCount())
{ // more blocks in list
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
// switch to next list, and start with first block?
if (currdl.getLnkDlNext() > 0)
{
currdl = currdl.getDlNextBlock();
dlindex = 0;
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
currblock = -1;
}
}
}
// switch to next block or escape
if (currdl == null)
{
currblock = -1; // just a single data block
}
else
{
// search next block in list.
dlindex++;
if (dlindex < currdl.getCount())
{ // more blocks in list
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
// switch to next list, and start with first block?
if (currdl.getLnkDlNext() > 0)
{
currdl = currdl.getDlNextBlock();
dlindex = 0;
currblock = currdl.getLnkDlData()[dlindex];
}
else
{
currblock = -1;
}
}
}
}
if (count < 0 || count != (int) count || count * 8 > Integer.MAX_VALUE)
{
throw new AoException(ErrorCode.AO_IMPLEMENTATION_PROBLEM, SeverityFlag.ERROR, 0,
"value count exceeded max supported block size supported by ODS");
}
// values have been calculated and written to an external file
ODSInsertStatement ins = new ODSInsertStatement(modelCache, "ec");
ins.setStringVal("iname", "ec_custom_rat_conv");
ins.setLongVal("cl", (int) count);
ins.setEnumVal("vt", 6); // ieeefloat8 (little endian; 11 would be
// big endian)
ins.setLongLongVal("so", startOffset);
ins.setLongVal("bs", (int) count * 8);
ins.setLongVal("vb", (int) count);
ins.setLongVal("vo", 0);
ins.setStringVal("fl", customRatConfPath.getFileName().toString());
ins.setLongLongVal("lc", iidLc);
// export flags
setGlobalFlag = exportFlags(ins, idBlock, dgBlock, cgBlock, cnBlock, startOffset, count);
ins.execute();
}
return setGlobalFlag;
}
/**
* Exports ODS compliant flags into a separate file if needed. If a global
* flag can be set it is returned (true=valid, false=invalid) rather than
* written to the file. That flag needs to be set to the parent local column
* afterwards!
*
* @param ins
* flag file an start offset will be added if flags were written to
* file, not null
* @param idBlock
* the {@link IDBLOCK}, not null
* @param dgBlock
* the {@link DGBLOCK}, not null
* @param cgBlock
* the {@link CGBLOCK}, not null
* @param cnBlock
* the {@link CNBLOCK}, not null
* @param startOffset
* start of the next record
* @param count
* how many records to read
*
* @return true, if all flags are valid, so the global flag can be set instead
* of writing flag values to the flags file. False if all flags are
* invalid. Null is returned, if flags were written to file.
* @throws IOException
* in case of errors
*/
private Boolean exportFlags(ODSInsertStatement ins, IDBLOCK idBlock, DGBLOCK dgBlock, CGBLOCK cgBlock,
CNBLOCK cnBlock, long startOffset, long count) throws IOException
{
Boolean retVal = null;
if ((cnBlock.getFlags() & 0x02) == 0 || cnBlock.getInvalBitPos() < 1 || count < 1)
{
// either invalidation bit not set or deactivated or no values available
// -> nothing to do
return retVal;
}
try (SeekableByteChannel flagsChannel = loadFlagsFileChannel(idBlock))
{
// to check whether a global flag can be set instead of writing the flags
// all to the file, all of them need to be analyzed first
/**
* For long channels the flags are cached in bulks until it is clear
* whether to write them to file or set a global flag
*/
List<ByteBuffer> flagBulks = new ArrayList<>();
/**
* To reduce memory footprint, the index of a bulk of identical flags will
* be mapped here to the common flag
*/
Map<Integer, Boolean> listIndex2Flag = new HashMap<>();
int dt = cnBlock.getDataType();
ByteOrder byteOrder = dt == 1 || dt == 3 || dt == 5 ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
int recordSize = (int) (dgBlock.getRecIdSize() + cgBlock.getDataBytes() + cgBlock.getInvalBytes());
int flabBytesOffset = (int) (dgBlock.getRecIdSize() + cgBlock.getDataBytes() + (cnBlock.getInvalBitPos() >> 3));
int bitOffset = ((int) cnBlock.getInvalBitPos()) & 0x07;
idBlock.sbc.position(startOffset);
readAndStoreFlagsFromRecords(idBlock.sbc, byteOrder, count, recordSize, flabBytesOffset, bitOffset, flagBulks,
listIndex2Flag);
// check if a global flag could be set
Boolean globalFlagToSet = analyzeGlobalFlagToSet(flagBulks, listIndex2Flag);
if (globalFlagToSet != null)
{
retVal = globalFlagToSet;
}
// otherwise write flags to file and update insert statement
else
{
ins.setStringVal("ffl", flagFile.getFileName().toString());
ins.setLongLongVal("fso", flagsChannel.position());
writeFlagsToFile(flagsChannel, flagBulks, listIndex2Flag);
}
}
return retVal;
}
/**
* Reads records from the given channel, extracts the flags and writes them
* into the given storage structures. Reads bulks of records rather than
* single records or even single flags from the channel for performance
* reasons.
*
* Note: Bulk reading records and extracting their flags in code massively
* reduces the handling time of the flags compared to the (previously
* implemented) read of one flag after another.
*
* @param sourceChannel
* the channel of the source file set to the starting position for
* reading the records
* @param byteOrder
* the ByteOrder to use for the buffers
* @param overallNrOfRecords
* the overall number of records to read and extract their flags
* @param recordSize
* the size of a single record entry (id, data, invalBit)
* @param flabBytesOffset
* the byte offset in a record to read the ivalBit from
* @param bitOffset
* the bit offset of the channel
* @param flagBulks
* the list to fill with the ByteBuffers of each read bulk of flags
* @param listIndex2Flag
* the map to write the common flag to for each bulk list index
* @throws IOException
* on buffer read errors
*/
private void readAndStoreFlagsFromRecords(SeekableByteChannel sourceChannel, ByteOrder byteOrder,
long overallNrOfRecords, int recordSize, int flabBytesOffset, int bitOffset, List<ByteBuffer> flagBulks,
Map<Integer, Boolean> listIndex2Flag) throws IOException
{
ByteBuffer flagsBuffer = ByteBuffer.allocate(FLAGS_BUFFER_SIZE);
flagsBuffer.order(byteOrder);
ByteBuffer bb = ByteBuffer.allocate(recordSize * MAX_RECORDS_PER_FLAGS_READ);
bb.order(byteOrder);
Boolean validFlagForCurrentBulk = null;
boolean sameFlagForCurrentBulk = true;
int bulkIndex = 0;
long recordCounter = 0;
while (recordCounter < overallNrOfRecords)
{
// read MAX_RECORDS_PER_FLAGS_READ records
bb.rewind();
int bytesRead = sourceChannel.read(bb);
int recordsRead = (int) Math.min(overallNrOfRecords - recordCounter, bytesRead / recordSize);
recordCounter += recordsRead;
bb.rewind();
// read each record from the buffer and extract flag
for (int k = 0; k < recordsRead; k++)
{
// extract from record and convert flag to ODS flag and add to write
// buffer
byte[] recordBytes = new byte[recordSize];
bb.get(recordBytes);
byte flagByte = recordBytes[flabBytesOffset];
short flag = ((flagByte & 0xff) & (1 << bitOffset)) != 0 ? 0 : (short) 15;
flagsBuffer.putShort(flag);
// check each record's flag against the other flags of current bulk
if (validFlagForCurrentBulk == null)
{
validFlagForCurrentBulk = flag == (short) 15;
}
else if ((flag != (short) 15 && validFlagForCurrentBulk) || (flag == (short) 15 && !validFlagForCurrentBulk))
{
sameFlagForCurrentBulk = false;
}
// prepare flags bulk
if (flagsBuffer.position() % FLAGS_BUFFER_SIZE == 0)
{
if (sameFlagForCurrentBulk)
{
// if same flags in current bulk, save this flag for bulk index and
// set null in flagBulks
flagBulks.add(null);
listIndex2Flag.put(bulkIndex, validFlagForCurrentBulk);
flagsBuffer.rewind();
}
else
{
// if different flags read in current bulk, store the current
// bytebuffer to write flags to file at the end
flagBulks.add(flagsBuffer);
flagsBuffer = ByteBuffer.allocate(FLAGS_BUFFER_SIZE);
flagsBuffer.order(byteOrder);
}
bulkIndex++;
validFlagForCurrentBulk = null;
sameFlagForCurrentBulk = true;
}
}
}
// prepare remaining flags
if (flagsBuffer.position() > 0)
{
// since the last bulk of flags' length may likely differ from the
// MAX_RECORDS_PER_FLAGS_READ it always has to be stored, in order to have
// the correct number of flags written to the file at the end, in case
int pos = flagsBuffer.position();
flagsBuffer.rewind();
flagsBuffer.limit(pos);
flagBulks.add(flagsBuffer);
if (sameFlagForCurrentBulk)
{
// if same flags in current bulk, save this flag for bulk index as well
listIndex2Flag.put(bulkIndex, validFlagForCurrentBulk);
}
}
}
/**
* @param flagBulks
* @param listIndex2Flag
* @return
*/
private Boolean analyzeGlobalFlagToSet(List<ByteBuffer> flagBulks, Map<Integer, Boolean> listIndex2Flag)
{
boolean setGlobalFlag = true;
Boolean globalFlagToSet = null;
for (int i = 0; i < flagBulks.size(); i++)
{
ByteBuffer buffer = flagBulks.get(i);
if (buffer != null && listIndex2Flag.get(i) == null)
{
// All homogeneous flag bulks are set to null, so if there is at least
// one, that was not set to null (i.e. contains valid and invalid
// values), no global flag can be set to the whole channel.
// Important note: the last bulk's buffer is always written to the list,
// but its common flag in the map may be set to true/false, if this last
// bulk also contains only the same flag
setGlobalFlag = false;
break;
}
else
{
Boolean bulkFlag = listIndex2Flag.get(i);
if (bulkFlag != null)
{
if (globalFlagToSet == null)
{
globalFlagToSet = bulkFlag;
}
else if (!globalFlagToSet.equals(bulkFlag))
{
// if by coincidence there are bulks which all contain the same
// value, but some all valid and others all invalid, this is checked
// here and no global flag can be set for the channel, but this
// should be a very rare border case
setGlobalFlag = false;
break;
}
}
}
}
Boolean retVal = null;
if (setGlobalFlag)
{
retVal = globalFlagToSet;
}
return retVal;
}
/**
* @param flagsChannel
* @param flagBulks
* @param listIndex2Flag
* @throws IOException
*/
private void writeFlagsToFile(SeekableByteChannel flagsChannel, List<ByteBuffer> flagBulks,
Map<Integer, Boolean> listIndex2Flag) throws IOException
{
for (int i = 0; i < flagBulks.size(); i++)
{
ByteBuffer buffer = flagBulks.get(i);
if (buffer == null)
{
// one or more of the bulks could be homogeneously the same flag,
// while some may contain valid and invalid values at the same time,
// therefore the homogeneous bulks need to be handled here
Boolean bulkFlag = listIndex2Flag.get(i);
if (bulkFlag != null)
{
// Homogeneous bulks' buffers always have a length of
// MAX_RECORDS_PER_FLAGS_READ, since the last bulk (that may deviate
// from that length) is always written to the buffer list and 'buffer'
// here is not null in that case
short[] flags = new short[MAX_RECORDS_PER_FLAGS_READ];
Arrays.fill(flags, bulkFlag ? (short) 15 : (short) 0);
buffer = ByteBuffer.wrap(shortToByteArray(flags));
}
}
// write flags from buffer to the flags file channel
if (buffer != null)
{
buffer.rewind();
flagsChannel.write(buffer);
}
}
}
private byte[] shortToByteArray(short[] input)
{
int shortIndex = 0;
int byteIndex = 0;
int iterations = input.length;
byte[] buffer = new byte[input.length * 2];
while (shortIndex != iterations) {
buffer[byteIndex] = (byte) (input[shortIndex] & 0x00FF);
buffer[byteIndex + 1] = (byte) ((input[shortIndex] & 0xFF00) >> 8);
++shortIndex;
byteIndex += 2;
}
return buffer;
}
/**
* Creates a {@link SeekableByteChannel} for the flags file.
*
* @param idBlock
* used to resolve target flags file, not null
* @return the {@link SeekableByteChannel}, not null
* @throws IOException
* if unable to create a flags file
*/
private SeekableByteChannel loadFlagsFileChannel(IDBLOCK idBlock) throws IOException
{
if (flagFile == null)
{
flagFile = idBlock.getMdfFilePath().resolveSibling("flags.bin");
while (Files.exists(flagFile))
{
flagFile = flagFile.resolveSibling("flags_" + UUID.randomUUID().toString().split("-")[0] + ".bin");
}
Files.createFile(flagFile);
}
return Files.newByteChannel(flagFile, StandardOpenOption.APPEND);
}
}