ctf.core: Parse data stream of a trace Implement parsing of data streams for CTF2 Traces. Testing of this patch is postponed until after the follow-up patch [1]. [1]https://git.eclipse.org/r/c/tracecompass/org.eclipse.tracecompass/+/203295 Change-Id: I6ce9273e46b1e970a77354346fb5814b061d2c81 Signed-off-by: Sehr Moosabhoy <sehr.moosabhoy@ericsson.com> Reviewed-on: https://git.eclipse.org/r/c/tracecompass/org.eclipse.tracecompass/+/203294 Tested-by: Trace Compass Bot <tracecompass-bot@eclipse.org> Tested-by: Marco Miller <marco.miller@ericsson.com> Reviewed-by: Marco Miller <marco.miller@ericsson.com>

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/CTFStreamInput.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/CTFStreamInput.java
index 1aea251..5c4d80b 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/CTFStreamInput.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/CTFStreamInput.java

@@ -17,6 +17,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileChannel.MapMode;
 import java.nio.file.StandardOpenOption;
@@ -210,8 +211,9 @@
          * The BitBuffer to extract data from the StreamInput
          */
         BitBuffer bitBuffer = new BitBuffer();
-        bitBuffer.setByteOrder(getStream().getTrace().getByteOrder());
 
+        ByteOrder byteOrder = getStream().getTrace().getByteOrder();
+        bitBuffer.setByteOrder(byteOrder);
     }
 
     /**

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/Metadata.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/Metadata.java
index c50a939..eb11a90 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/Metadata.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/Metadata.java

@@ -258,6 +258,9 @@
                 fragment = Objects.requireNonNull(gson.fromJson(jsonBlocks[i], JsonEventRecordMetadataNode.class));
             } else if (type.equals(JsonMetadataStrings.FRAGMENT_DATA_STREAM)) {
                 fragment = Objects.requireNonNull(gson.fromJson(jsonBlocks[i], JsonDataStreamMetadataNode.class));
+                if (!jsonBlocks[i].contains("id:")) { //$NON-NLS-1$
+                    ((JsonDataStreamMetadataNode) fragment).setId(-1);
+                }
             } else if (type.equals(JsonMetadataStrings.FRAGMENT_FIELD_ALIAS)) {
                 fragment = Objects.requireNonNull(gson.fromJson(jsonBlocks[i], JsonFieldClassAliasMetadataNode.class));
             }
@@ -271,8 +274,8 @@
     }
 
     /**
-     * Checks the version of the CTF trace by reading the first JSON fragment
-     * if it is a CTF2 fragment it updates the major of the trace
+     * Checks the version of the CTF trace by reading the first JSON fragment if
+     * it is a CTF2 fragment it updates the major of the trace
      *
      * @throws CTFException
      *             throws exception if file is invalid

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/IOStructGen.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/IOStructGen.java
index 6a3d143..1063e7d 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/IOStructGen.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/IOStructGen.java

@@ -151,7 +151,7 @@
                 }
                 traceNode = child;
                 parseTrace(traceNode);
-            } else if (CTFParser.tokenNames[CTFParser.STREAM].equals(type)) {
+            } else if (CTFParser.tokenNames[CTFParser.STREAM].equals(type) || JsonMetadataStrings.FRAGMENT_DATA_STREAM.equals(type)) {
                 StreamParser.INSTANCE.parse(child, new StreamParser.Param(fTrace, fRoot));
                 hasStreams = true;
             } else if (CTFParser.tokenNames[CTFParser.EVENT].equals(type) || JsonMetadataStrings.FRAGMENT_EVENT_RECORD.equals(type)) {

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/JsonDataStreamMetadataNode.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/JsonDataStreamMetadataNode.java
index c3ae214..53b12ef 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/JsonDataStreamMetadataNode.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/JsonDataStreamMetadataNode.java

@@ -118,6 +118,16 @@
         return fEventRecordCommonContextClass;
     }
 
+    /**
+     * Set the id of this data stream
+     *
+     * @param id
+     *            id of the data stream
+     */
+    public void setId(int id) {
+        fId = id;
+    }
+
     @Override
     public void initialize() throws CTFException {
         super.initialize();

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamDeclarationParser.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamDeclarationParser.java
index a8382fe..f35cc7a 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamDeclarationParser.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamDeclarationParser.java

@@ -26,6 +26,9 @@
 import org.eclipse.tracecompass.ctf.parser.CTFParser;
 import org.eclipse.tracecompass.internal.ctf.core.Activator;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.AbstractScopedCommonTreeParser;
+import org.eclipse.tracecompass.internal.ctf.core.event.metadata.CTFJsonMetadataNode;
+import org.eclipse.tracecompass.internal.ctf.core.event.metadata.JsonDataStreamMetadataNode;
+import org.eclipse.tracecompass.internal.ctf.core.event.metadata.JsonStructureFieldMetadataNode;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.Messages;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.MetadataStrings;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.ParseException;
@@ -138,97 +141,142 @@
         CTFStream stream = ((Param) param).fStream;
         CTFTrace fTrace = ((Param) param).fTrace;
 
-        /* There should be a left and right */
+        if (streamDecl instanceof CTFJsonMetadataNode) {
+            JsonDataStreamMetadataNode decl = (JsonDataStreamMetadataNode) streamDecl;
 
-        ICTFMetadataNode leftNode = streamDecl.getChild(0);
-        ICTFMetadataNode rightNode = streamDecl.getChild(1);
-
-        List<ICTFMetadataNode> leftStrings = leftNode.getChildren();
-
-        if (!isAnyUnaryString(leftStrings.get(0))) {
-            throw new ParseException(IDENTIFIER_MUST_BE_A_STRING);
-        }
-
-        String left = concatenateUnaryStrings(leftStrings);
-
-        if (left.equals(MetadataStrings.ID)) {
-            if (stream.isIdSet()) {
-                throw new ParseException(STREAM_ID + ALREADY_DEFINED);
+            if (decl.getId() >= 0) {
+                long streamId = decl.getId();
+                stream.setId(streamId);
             }
 
-            long streamID = StreamIdParser.INSTANCE.parse(rightNode, null);
-
-            stream.setId(streamID);
-        } else if (left.equals(MetadataStrings.EVENT_HEADER)) {
-            if (stream.isEventHeaderSet()) {
-                throw new ParseException(EVENT_HEADER + ALREADY_DEFINED);
+            JsonStructureFieldMetadataNode eventHeader = decl.getEventRecordHeaderClass();
+            if (eventHeader != null) {
+                IDeclaration eventHeaderDecl = TypeSpecifierListParser.INSTANCE.parse(eventHeader, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+                DeclarationScope eventHeaderScope = lookupStructName(eventHeader, scope);
+                verifyEventHeaderScope(eventHeaderScope);
+                setEventHeader(stream, eventHeaderDecl);
             }
 
-            ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
-
-            if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
-                throw new ParseException(EVENT_HEADER + EXPECTS_A_TYPE_SPECIFIER);
+            JsonStructureFieldMetadataNode eventContext = decl.getEventRecordCommonContextClass();
+            if (eventContext != null) {
+                IDeclaration eventContextDecl = TypeSpecifierListParser.INSTANCE.parse(eventContext, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+                verifyEventContext(eventContextDecl);
+                stream.setEventContext((StructDeclaration) eventContextDecl);
             }
 
-            IDeclaration eventHeaderDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
-            DeclarationScope eventHeaderScope = lookupStructName(typeSpecifier, scope);
-            if (eventHeaderScope == null) {
-                throw new ParseException(EVENT_HEADER + SCOPE_NOT_FOUND);
+            JsonStructureFieldMetadataNode packetContext = decl.getPacketContextFieldClass();
+            if (packetContext != null) {
+                IDeclaration packetContextDecl = TypeSpecifierListParser.INSTANCE.parse(packetContext, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+                verifyPacketContext(packetContextDecl);
+                stream.setPacketContext((StructDeclaration) packetContextDecl);
             }
-            DeclarationScope eventScope = new DeclarationScope(scope, MetadataStrings.EVENT);
-            eventHeaderScope.setName(CTFStrings.HEADER);
-            eventScope.addChild(eventHeaderScope);
-
-            if (eventHeaderDecl instanceof StructDeclaration) {
-                stream.setEventHeader((StructDeclaration) eventHeaderDecl);
-            } else if (eventHeaderDecl instanceof IEventHeaderDeclaration) {
-                stream.setEventHeader((IEventHeaderDeclaration) eventHeaderDecl);
-            } else {
-                throw new ParseException(EVENT_HEADER + EXPECTS_A_STRUCT);
-            }
-
-        } else if (left.equals(MetadataStrings.EVENT_CONTEXT)) {
-            if (stream.isEventContextSet()) {
-                throw new ParseException(EVENT_CONTEXT + ALREADY_DEFINED);
-            }
-
-            ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
-
-            if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
-                throw new ParseException(EVENT_CONTEXT + EXPECTS_A_TYPE_SPECIFIER);
-            }
-
-            IDeclaration eventContextDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
-
-            if (!(eventContextDecl instanceof StructDeclaration)) {
-                throw new ParseException(EVENT_CONTEXT + EXPECTS_A_STRUCT);
-            }
-
-            stream.setEventContext((StructDeclaration) eventContextDecl);
-        } else if (left.equals(MetadataStrings.PACKET_CONTEXT)) {
-            if (stream.isPacketContextSet()) {
-                throw new ParseException(PACKET_CONTEXT + ALREADY_DEFINED);
-            }
-
-            ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
-
-            if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
-                throw new ParseException(PACKET_CONTEXT + EXPECTS_A_TYPE_SPECIFIER);
-            }
-
-            IDeclaration packetContextDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
-
-            if (!(packetContextDecl instanceof StructDeclaration)) {
-                throw new ParseException(PACKET_CONTEXT + EXPECTS_A_STRUCT);
-            }
-
-            stream.setPacketContext((StructDeclaration) packetContextDecl);
         } else {
-            Activator.log(IStatus.WARNING, Messages.IOStructGen_UnknownStreamAttributeWarning + ' ' + left);
+            /* There should be a left and right */
+
+            ICTFMetadataNode leftNode = streamDecl.getChild(0);
+            ICTFMetadataNode rightNode = streamDecl.getChild(1);
+
+            List<ICTFMetadataNode> leftStrings = leftNode.getChildren();
+
+            if (!isAnyUnaryString(leftStrings.get(0))) {
+                throw new ParseException(IDENTIFIER_MUST_BE_A_STRING);
+            }
+
+            String left = concatenateUnaryStrings(leftStrings);
+
+            if (left.equals(MetadataStrings.ID)) {
+                if (stream.isIdSet()) {
+                    throw new ParseException(STREAM_ID + ALREADY_DEFINED);
+                }
+
+                long streamID = StreamIdParser.INSTANCE.parse(rightNode, null);
+
+                stream.setId(streamID);
+            } else if (left.equals(MetadataStrings.EVENT_HEADER)) {
+                if (stream.isEventHeaderSet()) {
+                    throw new ParseException(EVENT_HEADER + ALREADY_DEFINED);
+                }
+
+                ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
+
+                if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
+                    throw new ParseException(EVENT_HEADER + EXPECTS_A_TYPE_SPECIFIER);
+                }
+
+                IDeclaration eventHeaderDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+                DeclarationScope eventHeaderScope = lookupStructName(typeSpecifier, scope);
+                DeclarationScope eventScope = new DeclarationScope(scope, MetadataStrings.EVENT);
+                verifyEventHeaderScope(eventHeaderScope);
+                eventHeaderScope.setName(CTFStrings.HEADER);
+                eventScope.addChild(eventHeaderScope);
+                setEventHeader(stream, eventHeaderDecl);
+            } else if (left.equals(MetadataStrings.EVENT_CONTEXT)) {
+                if (stream.isEventContextSet()) {
+                    throw new ParseException(EVENT_CONTEXT + ALREADY_DEFINED);
+                }
+
+                ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
+
+                if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
+                    throw new ParseException(EVENT_CONTEXT + EXPECTS_A_TYPE_SPECIFIER);
+                }
+
+                IDeclaration eventContextDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+
+                verifyEventContext(eventContextDecl);
+
+                stream.setEventContext((StructDeclaration) eventContextDecl);
+            } else if (left.equals(MetadataStrings.PACKET_CONTEXT)) {
+                if (stream.isPacketContextSet()) {
+                    throw new ParseException(PACKET_CONTEXT + ALREADY_DEFINED);
+                }
+
+                ICTFMetadataNode typeSpecifier = rightNode.getChild(0);
+
+                if (!(CTFParser.tokenNames[CTFParser.TYPE_SPECIFIER_LIST].equals(typeSpecifier.getType()))) {
+                    throw new ParseException(PACKET_CONTEXT + EXPECTS_A_TYPE_SPECIFIER);
+                }
+
+                IDeclaration packetContextDecl = TypeSpecifierListParser.INSTANCE.parse(typeSpecifier, new TypeSpecifierListParser.Param(fTrace, null, null, scope));
+
+                verifyPacketContext(packetContextDecl);
+
+                stream.setPacketContext((StructDeclaration) packetContextDecl);
+            } else {
+                Activator.log(IStatus.WARNING, Messages.IOStructGen_UnknownStreamAttributeWarning + ' ' + left);
+            }
         }
         return stream;
     }
 
+    private static void verifyPacketContext(IDeclaration packetContextDecl) throws ParseException {
+        if (!(packetContextDecl instanceof StructDeclaration)) {
+            throw new ParseException(PACKET_CONTEXT + EXPECTS_A_STRUCT);
+        }
+    }
+
+    private static void verifyEventContext(IDeclaration eventContextDecl) throws ParseException {
+        if (!(eventContextDecl instanceof StructDeclaration)) {
+            throw new ParseException(EVENT_CONTEXT + EXPECTS_A_STRUCT);
+        }
+    }
+
+    private static void verifyEventHeaderScope(DeclarationScope eventHeaderScope) throws ParseException {
+        if (eventHeaderScope == null) {
+            throw new ParseException(EVENT_HEADER + SCOPE_NOT_FOUND);
+        }
+    }
+
+    private static void setEventHeader(CTFStream stream, IDeclaration eventHeaderDecl) throws ParseException {
+        if (eventHeaderDecl instanceof StructDeclaration) {
+            stream.setEventHeader((StructDeclaration) eventHeaderDecl);
+        } else if (eventHeaderDecl instanceof IEventHeaderDeclaration) {
+            stream.setEventHeader((IEventHeaderDeclaration) eventHeaderDecl);
+        } else {
+            throw new ParseException(EVENT_HEADER + EXPECTS_A_STRUCT);
+        }
+    }
+
     private static DeclarationScope lookupStructName(ICTFMetadataNode typeSpecifier, DeclarationScope scope) {
         /*
          * This needs a struct.struct_name.name to work, luckily, that is 99.99%

diff --git a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamParser.java b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamParser.java
index 44f5272..3b324df 100644
--- a/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamParser.java
+++ b/ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/internal/ctf/core/event/metadata/tsdl/stream/StreamParser.java

@@ -20,6 +20,7 @@
 import org.eclipse.tracecompass.ctf.core.trace.CTFTrace;
 import org.eclipse.tracecompass.ctf.parser.CTFParser;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.AbstractScopedCommonTreeParser;
+import org.eclipse.tracecompass.internal.ctf.core.event.metadata.CTFJsonMetadataNode;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.MetadataStrings;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.ParseException;
 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.tsdl.TypeAliasParser;
@@ -92,23 +93,28 @@
         CTFTrace trace = ((Param) param).fTrace;
         CTFStream stream = new CTFStream(trace);
 
-        List<ICTFMetadataNode> children = streamNode.getChildren();
-        if (children == null) {
-            throw new ParseException("Empty stream block"); //$NON-NLS-1$
-        }
-
         DeclarationScope scope = new DeclarationScope(parameter.fCurrentScope, MetadataStrings.STREAM);
 
-        for (ICTFMetadataNode child : children) {
-            String type = child.getType();
-            if (CTFParser.tokenNames[CTFParser.TYPEALIAS].equals(type)) {
-                TypeAliasParser.INSTANCE.parse(child, new TypeAliasParser.Param(trace, scope));
-            } else if (CTFParser.tokenNames[CTFParser.TYPEDEF].equals(type)) {
-                TypedefParser.INSTANCE.parse(child, new TypedefParser.Param(trace, scope));
-            } else if (CTFParser.tokenNames[CTFParser.CTF_EXPRESSION_TYPE].equals(type) || CTFParser.tokenNames[CTFParser.CTF_EXPRESSION_VAL].equals(type)) {
-                StreamDeclarationParser.INSTANCE.parse(child, new StreamDeclarationParser.Param(trace, stream, scope));
-            } else {
-                throw childTypeError(child);
+        if (streamNode instanceof CTFJsonMetadataNode) {
+            StreamDeclarationParser.INSTANCE.parse(streamNode, new StreamDeclarationParser.Param(trace, stream, scope));
+        } else {
+            List<ICTFMetadataNode> children = streamNode.getChildren();
+
+            if (children == null) {
+                throw new ParseException("Empty stream block"); //$NON-NLS-1$
+            }
+
+            for (ICTFMetadataNode child : children) {
+                String type = child.getType();
+                if (CTFParser.tokenNames[CTFParser.TYPEALIAS].equals(type)) {
+                    TypeAliasParser.INSTANCE.parse(child, new TypeAliasParser.Param(trace, scope));
+                } else if (CTFParser.tokenNames[CTFParser.TYPEDEF].equals(type)) {
+                    TypedefParser.INSTANCE.parse(child, new TypedefParser.Param(trace, scope));
+                } else if (CTFParser.tokenNames[CTFParser.CTF_EXPRESSION_TYPE].equals(type) || CTFParser.tokenNames[CTFParser.CTF_EXPRESSION_VAL].equals(type)) {
+                    StreamDeclarationParser.INSTANCE.parse(child, new StreamDeclarationParser.Param(trace, stream, scope));
+                } else {
+                    throw childTypeError(child);
+                }
             }
         }