OpenApi.py: added support for allOf

Signed-off-by: Gabor Szalai <gabor.szalai@ericsson.com>
diff --git a/tools/OpenApi.py b/tools/OpenApi.py
index a3dfa48..4e53717 100755
--- a/tools/OpenApi.py
+++ b/tools/OpenApi.py
@@ -4,10 +4,36 @@
 import pprint
 import re
 
-ttcn_keywords=["action","activate","address","alive","all","alt","altstep","and","and4b","any","anytype","bitstring","boolean","break","case","call","catch","char","charstring","check","clear","complement","component","connect","const","continue","control","create","deactivate","decmatch","default","disconnect","display","do","done","else","encode","enumerated","error","except","exception","execute","extends","extension","external","fail","false","float","for","friend","from","function","getverdict","getcall","getreply","goto","group","halt","hexstring","if","ifpresent","import","in","inconc","infinity","inout","integer","interleave","kill","killed","label","language","length","log","map","match","message","mixed","mod","modifies","module","modulepar","mtc","noblock","none","not","not_a_number","not4b","nowait","null","octetstring","of","omit","on","optional","or","or4b","out","override","param","pass","pattern","permutation","port","present","private","procedure","public","raise","read","receive","record","recursive","rem","repeat","reply","return","running","runs","select","self","send","sender","set","setencode","setverdict","signature","start","stop","subset","superset","system","template","testcase","timeout","timer","to","trigger","true","type","union","universal","unmap","value","valueof","var","variant","verdicttype","while","with","xor","xor4b","bit2hex","bit2int","bit2oct","bit2str","char2int","float2int","hex2bit","hex2int","hex2oct","hex2str","int2bit","int2char","int2float","int2hex","int2oct","int2str","int2unichar","ischosen","ispresent","lengthof","oct2bit","oct2hex","oct2int","oct2str","regexp","rnd","sixeof","str2int","str2oct","substr","unichar2int","replace"]
+# Reserved word list. Used to sanitaze the type and field names.
+ttcn_keywords=[ # Reserved words
+               "action","activate","address","alive","all","alt","altstep","and","and4b","any","anytype",
+               "bitstring","boolean","break","case","call","catch","char","charstring","check","clear","complement",
+               "component","connect","const","continue","control","create","deactivate","decmatch","default","disconnect",
+               "display","do","done","else","encode","enumerated","error","except","exception","execute","extends",
+               "extension","external","fail","false","float","for","friend","from","function","getverdict","getcall",
+               "getreply","goto","group","halt","hexstring","if","ifpresent","import","in","inconc","infinity",
+               "inout","integer","interleave","kill","killed","label","language","length","log","map","match","message",
+               "mixed","mod","modifies","module","modulepar","mtc","noblock","none","not","not_a_number","not4b","nowait",
+               "null","octetstring","of","omit","on","optional","or","or4b","out","override","param","pass","pattern","permutation",
+               "port","present","private","procedure","public","raise","read","receive","record","recursive","rem","repeat",
+               "reply","return","running","runs","select","self","send","sender","set","setencode","setverdict","signature","start",
+               "stop","subset","superset","system","template","testcase","timeout","timer","to","trigger","true","type","union","universal",
+               "unmap","value","valueof","var","variant","verdicttype","while","with","xor","xor4b","bit2hex","bit2int","bit2oct","bit2str",
+               "char2int","float2int","hex2bit","hex2int","hex2oct","hex2str","int2bit","int2char","int2float","int2hex","int2oct","int2str",
+               "int2unichar","ischosen","ispresent","lengthof","oct2bit","oct2hex","oct2int","oct2str","regexp","rnd","sixeof","str2int",
+               "str2oct","substr","unichar2int","replace",
+               # Reserved by extension packages
+               "apply", "assert", "at", "configuration", "conjunct", "cont",", ""delta", "disjunct", "duration", "finished", "history", "implies",
+               "inv", "mode", "notinv", "now", "onentry", "onexit", "par", "prev", "realtime", "seq", "setstate", "static", "stepsize", "stream",
+               "timestamp", "until", "values", "wait",
+               # OO reserved
+               "class", "finally", "object","this"
+               ]
 
+# Converts a number to the word representation.
+# Used to replace the leading numbers to letters in the names
+# Example: 5qi -> this function is used to replace the 5 to five 
 def numToWords(num):
-    '''words = {} convert an integer number into words'''
     units = ['','one','two','three','four','five','six','seven','eight','nine']
     teens = ['','eleven','twelve','thirteen','fourteen','fifteen','sixteen', \
              'seventeen','eighteen','nineteen']
@@ -40,29 +66,52 @@
                 else: words.append(tens[t])
             else:
                 if u>=1: words.append(units[u])
-            if (g>=1) and ((h+t+u)>0): words.append(thousands[g]+',')
+            if (g>=1) and ((h+t+u)>0): words.append(thousands[g])
     return ''.join(words)
 
+# Adds the module 'new_module' to the import list of the module if it is not in the list
+def addImport(new_module):
+  if (new_module != module_name) and (not new_module in module_data['import']):
+    module_data['import'].append(new_module)
 
-def clean_name(instr, typename=False):
+# Converts the identifiers to a valid TTCN-3 identifiers
+#  - Ensure that the identifier starts with letters
+#     * The whole identifier is a number -> Adds "Num_" prefix
+#     * Starts with a number -> convert the initial number part to word -> 5qi -> fiveqi
+#  - Contains only letters, numbers and underscore
+#     * Replaces every other charaters with underscore
+#     * The caller can add extra charatesr to the list of the valid charcters in the 'extrachr' parameter
+#  - Postfix the reserved words with underscore
+#  - Capitalize the first character for type names
+def clean_name(instr, typename=False, extrachr=""):
   #print(instr)
   if instr in ttcn_keywords:
     return instr + "_"
   
+  if not isinstance(instr, str):
+    instr=str(instr)
+  
+  if instr.isdigit():
+    instr = "Num_" + instr
+  
   m = re.search('(^\d+)(.*)',instr)
   if m:
     instr = numToWords(int(m.group(1))) + m.group(2)
-    # if m.group(2).isupper():
-      # a = instr.upper()
-      # instr = a
     if typename:
       a = instr[:1].upper()+instr[1:]
       instr = a
   elif instr[:1] =="_":
     a=instr[1:]
     instr =a
-  return instr.replace("-","_")
+  rx='[^a-zA-Z0-9_' + extrachr + ']'
+  return re.sub(rx,'_',instr)
 
+
+# Returns the module and the referenced type name as tuplefrom the $ref
+# 
+#  local reference:  '#/components/schemas/SnssaiExtension' ->  '', 'SnssaiExtension'
+#  remote reference: 'TS29571_CommonData.yaml#/components/schemas/Ncgi' -> 'TS29571_CommonData', 'Ncgi'
+#
 def get_module(ref):
   if ref[0]=='#':
     return '', ref.split('/')[-1]
@@ -70,6 +119,15 @@
     return ref.split("#")[0].replace(".yaml",""),ref.split('/')[-1]
   
 
+# The process_path, process_schema, and process_used_schem_name functions
+# parses the paths: parts of the yaml definitions and collects the type refences
+# Encoder and decoder functions are needed for the referenced types as they are used
+# in API messages directly.
+# Also creates prototype for possible arrays
+#
+# If the generation of encoder function or array definiton is missed these functions
+# should be updated to find the missing references.
+#
 def process_path(data,module_data):
   for m in data:
     #print("process_path ", m, " " , data[m])
@@ -95,15 +153,14 @@
   if "$ref" in schema:
     refmodule,refstr=get_module(schema["$ref"])
     if refmodule!= '':
-      if not refmodule in module_data['import']:
-        module_data['import'].append(refmodule)
+      addImport(refmodule)
       refstr=refmodule+"."+refstr
     if not refstr in module_data['functions']:
       module_data['functions'].append(refstr)
   elif "type" in schema:
     if schema["type"] == "array" :
       if "$ref" in schema["items"] :
-        refmodule,refstr=get_module(schema["items"]["$ref"])
+        refmodule,refstr=get_module(schema["items"]["$ref"])  # This direct code generation should be moved out from here.
         print("// " + refmodule + " type record of " + refstr + " " +refstr + "_list")
         print(f'// external function f_enc_{refstr}_list(in {refstr}_list pdu) return octetstring ')
         print('// with { extension "prototype(convert) encode(JSON)" }')
@@ -128,145 +185,10 @@
   if "schema" in data:
     process_schema(data["schema"],module_data)   
 
-def add_buff(buff, txt, end='\n'):
-  return buff+txt+end
-
-def typewriter(buff, indent, pr_type ,name, data ,module_data):
-  #print("typewriter ", indent, pr_type ,name, data)
-  if "type" in data:
-    if data["type"] == "string":
-      if "enum" in data:
-        enum_data=data["enum"]
-        outstr=f'{indent}{pr_type}enumerated {name} {{'
-        sep=" "
-        for i in enum_data:
-          outstr+=sep+i
-          sep=", "
-        outstr+='}'
-        buff=add_buff(buff,outstr)
-       
-      else:
-        buff=add_buff(buff,f'{indent}{pr_type}charstring {name}',end='')
-        if "pattern" in data:
-          buff=add_buff(buff,f' // (pattern \"{data["pattern"]}\")',end='')
-    elif data["type"] == "integer":
-      restrition=False
-      restritionstr=""
-      minstr=" (-infinity.."
-      maxstr="infinity)"
-      if "minimum" in data:
-        restrition=True
-        minstr=f' ({data["minimum"]}..'
-      if "maximum" in data:
-        restrition=True
-        maxstr=f'{data["maximum"]})'
-      if restrition:
-        restritionstr=minstr+maxstr
-      buff=add_buff(buff,f'{indent}{pr_type}integer {name}{restritionstr}',end='')
-    elif data["type"] == "number":
-      buff=add_buff(buff,f'{indent}{pr_type}float {name}',end='')
-    elif data["type"] == "boolean":
-      buff=add_buff(buff,f'{indent}{pr_type}boolean {name}',end='')
-    elif data["type"] == "array":
-      buff=add_buff(buff,f'{indent}{pr_type}set of ',end='')
-      buff=typewriter(buff,indent+"  ","","",data["items"],module_data)
-      buff=add_buff(buff,name,end='')
-    elif data["type"] == "object":
-      if "required" in data:
-        required_data=data["required"]
-      else:
-        required_data=[]
-      if "properties" in data:
-        if pr_type == "":
-          buff=add_buff(buff,f'{indent}{pr_type}set {{',end='')
-        else:
-          buff=add_buff(buff,f'{indent}{pr_type}set {name} {{',end='')
-        prop_data=data["properties"]
-        sep=""
-        for prop in prop_data:
-          buff=add_buff(buff,sep)
-          sep=","
-          opt_str=""
-          if not prop in required_data:
-            opt_str=" optional"
-          buff=typewriter(buff,indent+"  ","",prop,prop_data[prop],module_data)
-          buff=add_buff(buff,opt_str,end='')
-        buff=add_buff(buff,"")
-        buff=add_buff(buff,"  }",end='')
-        if pr_type == "":
-          buff=add_buff(buff,name,end='')
-      elif "additionalProperties" in data:
-        buff=add_buff(buff,f'{indent}{pr_type}set of record {{')
-        buff=add_buff(buff,f'{indent}  universal charstring key,')
-        buff=typewriter(buff,indent+"  ","","additionalProperties",data["additionalProperties"],module_data)
-        buff=add_buff(buff,"")
-        buff=add_buff(buff,f'{indent}}} {name}',end='')
-      else:
-        buff=add_buff(buff,f'{indent}{pr_type}object2 {name}',end='')
-      
-    else:
-      print (f' // skiped {name}',end='')
-  elif "$ref" in data:
-    refmodule,refstr=get_module(data["$ref"])
-    if refmodule!= '':
-      if not refmodule in module_data['import']:
-        module_data['import'].append(refmodule)
-      refstr=refmodule+"."+refstr
-    buff=add_buff(buff,f'{indent}{pr_type}{refstr}    {name}',end='')
-  elif "allOf" in data:
-    #print(name,data["allOf"])
-    buff=typewriter(buff,indent+"  ","type /* allOf */",name,data["allOf"][1],module_data)     
-  elif "anyOf" in data:
-    enum_found=False
-    enum_data=''
-    for e in data["anyOf"]:
-      if "enum" in e:
-        enum_data=e["enum"]
-        enum_found=True
-        break
-    
-    if enum_found:
-      outstr=f'  type enumerated {name}_enum {{'
-      sep=" "
-      for i in enum_data:
-        outstr+=sep+i
-        sep=", "
-      outstr+='}'
-      buff=add_buff(buff,outstr)
-      buff=add_buff(buff,'')
-      buff=add_buff(buff,f'  type union {name} {{')
-      buff=add_buff(buff,f'    {name}_enum  enum_val,')
-      buff=add_buff(buff,'    charstring           other_val') 
-      buff=add_buff(buff,'  } with {')
-      buff=add_buff(buff,'    variant "JSON: as value"')
-      buff=add_buff(buff,'  }')
-      
-
-  elif "oneOf" in data:
-    if pr_type == "":
-      buff=add_buff(buff,f'{indent}{pr_type}union {{',end='')
-    else:
-      buff=add_buff(buff,f'{indent}{pr_type}union {name} {{',end='')
-    prop_data=data["oneOf"]
-    sep=""
-    i=1
-    for prop in prop_data:
-      buff=add_buff(buff,sep)
-      sep=","
-      opt_str=""
-      buff=typewriter(buff,indent+"  ","",f'field{i}',prop,module_data)
-      i+=1
-    buff=add_buff(buff,"")
-    buff=add_buff(buff,'  } with {')
-    buff=add_buff(buff,'    variant "JSON: as value"')
-    buff=add_buff(buff,'  }')
-    if pr_type == "":
-      buff=add_buff(buff,name,end='')
-    
-  else:
-      print (f' // skiped {name}')
-  return buff
-
+# Processes one schema definition and build the data structure needed for code generation.
+# The processed data is appended to the type_tree list.
+# All processing of the schema definition is done here, except the resolution of the allOf
+# The allOf should be reprocessed after the schema processing, because the fields needs to be collected from the processed data.
 def type_builder(name,data,tree):
   global type_tree
   #print("type:", name)
@@ -289,7 +211,7 @@
           cename=clean_name(ev,True)
           element_data["values"].append(cename)
           if cename != ev:
-            element_data["variant"].append("text '"+ cename +"' as '"+ev+"'")
+            element_data["variant"].append("text '"+ cename +"' as '"+str(ev)+"'")
       else:
         element_data["type"]="charstring"
         if "pattern" in data:
@@ -352,8 +274,7 @@
         element_data["variant"].append("as map")
       else:
         element_data["type"]="JSON_Generic.JSON_generic_val"
-        if not "JSON_Generic" in module_data['import']:
-          module_data['import'].append("JSON_Generic")
+        addImport("JSON_Generic")
     else:
       element_data["type"]=data["type"]
       #print('!!!!!!unsupported' + data["type"])
@@ -361,8 +282,7 @@
   elif "$ref" in data:
     refmodule,refstr=get_module(data["$ref"])
     if refmodule!= '':
-      if not refmodule in module_data['import']:
-        module_data['import'].append(refmodule)
+      addImport(refmodule)
       element_data["type"]=refmodule+"."+clean_name(refstr,True)
     else:
       element_data["type"]=clean_name(refstr,True)
@@ -408,21 +328,30 @@
       element_data["fields"].append(field[0])
 
   elif "allOf" in data:
+    element_data["allOf"]=True
+    element_data["fields"]=[]
+    i=0
+    for e in data["allOf"]:
+      if "$ref" in e:
+        refmodule,refstr=get_module(e["$ref"])
+        element_data["fields"].append({"ref":True,"refmodule":refmodule,"refstr":clean_name(refstr,True)})
+      else:
+        field=[]
+        type_builder(f'field{i}',e,field)
+        i+=1
+        element_data["fields"].append(field[0])
     element_data["type"]="JSON_Generic.JSON_generic_val"
-    if not "JSON_Generic" in module_data['import']:
-      module_data['import'].append("JSON_Generic")
+    addImport("JSON_Generic")
 
     
   else:
     element_data["type"]="JSON_Generic.JSON_generic_val"
-    if not "JSON_Generic" in module_data['import']:
-      module_data['import'].append("JSON_Generic")
+    addImport("JSON_Generic")
     #print('!!!!!!unsupported ',name)  
     #pprint.pprint(data)
   
   if "nullable" in data:
-    if not "JSON_Generic" in module_data['import']:
-      module_data['import'].append("JSON_Generic")
+    addImport("JSON_Generic")
     element_data2={}
     element_data2["type"]="union"
     element_data2["name"]=element_data["name"]
@@ -438,43 +367,87 @@
   else:
     tree.append(element_data)
 
+# Finds the data of the type or return None
+def find_type(tname):
+  for t in type_tree:
+    if t["name"] == tname:
+      return t
+  return None
+
+# Generates the TTCN-3 type definition from the processed type data
 def print_type(t, lend="\n", top=False):
   global ident_level
   global ident_c
-  print(t["type"], " ",end="",sep="")
-  if top and (t["type"] !="record of"):
-    print(t["name"]," ", end="",sep="")
-  if "fields" in t:
-    print(" {",sep="")
-    ident_level+=1
-    separatot=ident_c*ident_level
-    for f in t["fields"]:
-      print(separatot, end="",sep="")
-      separatot=",\n" + ident_c*ident_level
-      print_type(f,"")
-      if "mandatory" in t:
-        if f["name"] not in t["mandatory"]:
-          print(" optional", end="")
-    ident_level-=1
-    print("",sep="")
-    print(ident_c*ident_level,"}",sep="",end=lend)
-  elif "inner_type" in t:
-    print_type(t["inner_type"],"")
-  elif "values" in t:
-    print("{ ",sep="",end="")
-    separatot=""
-    for e in t["values"]:
-      print(separatot, end="",sep="")
-      separatot=", "
-      print(e, end="",sep="")
-    print("}",sep="",end="")
-  
-  if (not top) or (t["type"] =="record of"):
-    print(" ", t["name"],end="",sep="")
-  if "restriction" in t:
-    print(" ",t["restriction"],end="",sep="")
-  print(lend,end="")
+  if "allOf" in t:  # The allOf should be reprocessed to collect the actual fields.
+    element_data = {}
+    element_data["name"]=t["name"]
+    element_data["type"]="set"
+    element_data["fields"]=[]
+    element_data["variant"]=[]
+    element_data["mandatory"]=[]
+    is_ok=True
+    for f in t["fields"]:   # the "fields" contains the reference or the data of the to be merged objects
+      if "ref" in f:  # Collect fields from the referenced type
+        if f["refmodule"] == "" :  
+          it=find_type(f["refstr"])
+          if it != None:
+            element_data["fields"]+= it["fields"]
+            element_data["variant"]+= it["variant"]
+            element_data["mandatory"]+= it["mandatory"]
+          else:
+            is_ok=False
+        else:
+          print("// Please add the fields from " + f["refmodule"] + "." + f["refstr"])
+      elif f["type"] == "set":  # Collect fields from the directly defined object
+        element_data["fields"]+= f["fields"]
+        element_data["variant"]+= f["variant"]
+        element_data["mandatory"]+= f["mandatory"]
+      else:
+        is_ok=False
+    
+    if is_ok:  # The field collection was successfull, generate the code
+      print_type(element_data,lend,top)
+      t["fields"]=element_data["fields"]
+      
+    else: # the allOf can't be resolved, use generic JSON type
+      del t["fields"]  # remove the "fields" list, teh type_builder already filled the type data for the generic JSON type, just use it
+      print_type(t,lend,top)
+  else:
+    print(t["type"], " ",end="",sep="")
+    if top and (t["type"] !="record of"):
+      print(t["name"]," ", end="",sep="")
+    if "fields" in t:
+      print(" {",sep="")
+      ident_level+=1
+      separatot=ident_c*ident_level
+      for f in t["fields"]:
+        print(separatot, end="",sep="")
+        separatot=",\n" + ident_c*ident_level
+        print_type(f,"")
+        if "mandatory" in t:
+          if f["name"] not in t["mandatory"]:
+            print(" optional", end="")
+      ident_level-=1
+      print("",sep="")
+      print(ident_c*ident_level,"}",sep="",end=lend)
+    elif "inner_type" in t:
+      print_type(t["inner_type"],"")
+    elif "values" in t:
+      print("{ ",sep="",end="")
+      separatot=""
+      for e in t["values"]:
+        print(separatot, end="",sep="")
+        separatot=", "
+        print(e, end="",sep="")
+      print("}",sep="",end="")
+    
+    if (not top) or (t["type"] =="record of"):
+      print(" ", t["name"],end="",sep="")
+    if "restriction" in t:
+      print(" ",t["restriction"],end="",sep="")
+    print(lend,end="")
 
+# Gather the variants for the type, builds the variant references
 def gather_variants(t,spec,variants):
   if "variant" in t:
     for v in t["variant"]:
@@ -486,12 +459,15 @@
          spec2+="."
        gather_variants(f,spec2+f["name"],variants)
 
+# By default the yaml loader parses the Yes, No as bool value instead of string
+# The bool constructor is overridden to return them as string
 from yaml.constructor import Constructor
 
 def add_bool(self, node):
     return self.construct_scalar(node)
 
 Constructor.add_constructor(u'tag:yaml.org,2002:bool', add_bool)
+
   
 f=open(sys.argv[1])
 
@@ -518,6 +494,8 @@
   if "schemas" in doc["components"]:
     schemas=doc["components"]["schemas"]
     for name in schemas:
+      # The type name suffix 'Rm' is used for a nullable type alias 
+      # Generate the nullable structure but use the reference to the original type
       if (name[-2:] == "Rm" ) and (name[:-2] in schemas ):
         type_tree.append({'fields': [{'name': 'null_val',
               'nullable': False,
@@ -526,18 +504,17 @@
               'nullable': False,
               'type': clean_name(name[:-2],True),
               'variant': []}],
-  'name': clean_name(name,True),
-  'nullable': True,
-  'type': 'union',
-  'variant': ['JSON: as value']})
-      elif name == "NullValue":
+              'name': clean_name(name,True),
+              'nullable': True,
+              'type': 'union',
+              'variant': ['JSON: as value']})
+        addImport("JSON_Generic")
+      elif name == "NullValue": # Special type, used for nullable enums
         type_tree.append({'type': 'JSON_generic_val.JSON_null_val','name':'NullValue','nullable': True})
       else:
+        # Normal type schema processing.
         data=schemas[name]
         type_builder(clean_name(name,True),data,type_tree)
-      #buff=typewriter(buff,"  ", "type ",name,data,module_data)
-      #buff=add_buff(buff,'')
-      #buff=add_buff(buff,'')
   if "responses" in doc["components"]:
     #print(doc["components"]["responses"])
     for r in doc["components"]["responses"]:
@@ -555,12 +532,16 @@
 print("")
 
 for fs in module_data['functions']:
-  f=clean_name(fs,True)
-  print(f'external function f_enc_{f}(in {f} pdu) return octetstring ')
-  print('with { extension "prototype(convert) encode(JSON)" }')
+  f=clean_name(fs,True,".")
+  if "." in f:
+    pre="// "
+  else:
+    pre=""
+  print(pre+f'external function f_enc_{f}(in {f} pdu) return octetstring ')
+  print(pre+'with { extension "prototype(convert) encode(JSON)" }')
   print("")
-  print(f'external function f_dec_{f}(in octetstring stream, out {f} pdu) return integer ')
-  print('with { extension "prototype(backtrack) decode(JSON)" }')
+  print(pre+f'external function f_dec_{f}(in octetstring stream, out {f} pdu) return integer ')
+  print(pre+'with { extension "prototype(backtrack) decode(JSON)" }')
   print("")
 
 print("")