OpenApi.py: added support for allOf
Signed-off-by: Gabor Szalai <gabor.szalai@ericsson.com>
diff --git a/tools/OpenApi.py b/tools/OpenApi.py
index a3dfa48..4e53717 100755
--- a/tools/OpenApi.py
+++ b/tools/OpenApi.py
@@ -4,10 +4,36 @@
import pprint
import re
-ttcn_keywords=["action","activate","address","alive","all","alt","altstep","and","and4b","any","anytype","bitstring","boolean","break","case","call","catch","char","charstring","check","clear","complement","component","connect","const","continue","control","create","deactivate","decmatch","default","disconnect","display","do","done","else","encode","enumerated","error","except","exception","execute","extends","extension","external","fail","false","float","for","friend","from","function","getverdict","getcall","getreply","goto","group","halt","hexstring","if","ifpresent","import","in","inconc","infinity","inout","integer","interleave","kill","killed","label","language","length","log","map","match","message","mixed","mod","modifies","module","modulepar","mtc","noblock","none","not","not_a_number","not4b","nowait","null","octetstring","of","omit","on","optional","or","or4b","out","override","param","pass","pattern","permutation","port","present","private","procedure","public","raise","read","receive","record","recursive","rem","repeat","reply","return","running","runs","select","self","send","sender","set","setencode","setverdict","signature","start","stop","subset","superset","system","template","testcase","timeout","timer","to","trigger","true","type","union","universal","unmap","value","valueof","var","variant","verdicttype","while","with","xor","xor4b","bit2hex","bit2int","bit2oct","bit2str","char2int","float2int","hex2bit","hex2int","hex2oct","hex2str","int2bit","int2char","int2float","int2hex","int2oct","int2str","int2unichar","ischosen","ispresent","lengthof","oct2bit","oct2hex","oct2int","oct2str","regexp","rnd","sixeof","str2int","str2oct","substr","unichar2int","replace"]
+# Reserved word list. Used to sanitaze the type and field names.
+ttcn_keywords=[ # Reserved words
+ "action","activate","address","alive","all","alt","altstep","and","and4b","any","anytype",
+ "bitstring","boolean","break","case","call","catch","char","charstring","check","clear","complement",
+ "component","connect","const","continue","control","create","deactivate","decmatch","default","disconnect",
+ "display","do","done","else","encode","enumerated","error","except","exception","execute","extends",
+ "extension","external","fail","false","float","for","friend","from","function","getverdict","getcall",
+ "getreply","goto","group","halt","hexstring","if","ifpresent","import","in","inconc","infinity",
+ "inout","integer","interleave","kill","killed","label","language","length","log","map","match","message",
+ "mixed","mod","modifies","module","modulepar","mtc","noblock","none","not","not_a_number","not4b","nowait",
+ "null","octetstring","of","omit","on","optional","or","or4b","out","override","param","pass","pattern","permutation",
+ "port","present","private","procedure","public","raise","read","receive","record","recursive","rem","repeat",
+ "reply","return","running","runs","select","self","send","sender","set","setencode","setverdict","signature","start",
+ "stop","subset","superset","system","template","testcase","timeout","timer","to","trigger","true","type","union","universal",
+ "unmap","value","valueof","var","variant","verdicttype","while","with","xor","xor4b","bit2hex","bit2int","bit2oct","bit2str",
+ "char2int","float2int","hex2bit","hex2int","hex2oct","hex2str","int2bit","int2char","int2float","int2hex","int2oct","int2str",
+ "int2unichar","ischosen","ispresent","lengthof","oct2bit","oct2hex","oct2int","oct2str","regexp","rnd","sixeof","str2int",
+ "str2oct","substr","unichar2int","replace",
+ # Reserved by extension packages
+ "apply", "assert", "at", "configuration", "conjunct", "cont",", ""delta", "disjunct", "duration", "finished", "history", "implies",
+ "inv", "mode", "notinv", "now", "onentry", "onexit", "par", "prev", "realtime", "seq", "setstate", "static", "stepsize", "stream",
+ "timestamp", "until", "values", "wait",
+ # OO reserved
+ "class", "finally", "object","this"
+ ]
+# Converts a number to the word representation.
+# Used to replace the leading numbers to letters in the names
+# Example: 5qi -> this function is used to replace the 5 to five
def numToWords(num):
- '''words = {} convert an integer number into words'''
units = ['','one','two','three','four','five','six','seven','eight','nine']
teens = ['','eleven','twelve','thirteen','fourteen','fifteen','sixteen', \
'seventeen','eighteen','nineteen']
@@ -40,29 +66,52 @@
else: words.append(tens[t])
else:
if u>=1: words.append(units[u])
- if (g>=1) and ((h+t+u)>0): words.append(thousands[g]+',')
+ if (g>=1) and ((h+t+u)>0): words.append(thousands[g])
return ''.join(words)
+# Adds the module 'new_module' to the import list of the module if it is not in the list
+def addImport(new_module):
+ if (new_module != module_name) and (not new_module in module_data['import']):
+ module_data['import'].append(new_module)
-def clean_name(instr, typename=False):
+# Converts the identifiers to a valid TTCN-3 identifiers
+# - Ensure that the identifier starts with letters
+# * The whole identifier is a number -> Adds "Num_" prefix
+# * Starts with a number -> convert the initial number part to word -> 5qi -> fiveqi
+# - Contains only letters, numbers and underscore
+# * Replaces every other charaters with underscore
+# * The caller can add extra charatesr to the list of the valid charcters in the 'extrachr' parameter
+# - Postfix the reserved words with underscore
+# - Capitalize the first character for type names
+def clean_name(instr, typename=False, extrachr=""):
#print(instr)
if instr in ttcn_keywords:
return instr + "_"
+ if not isinstance(instr, str):
+ instr=str(instr)
+
+ if instr.isdigit():
+ instr = "Num_" + instr
+
m = re.search('(^\d+)(.*)',instr)
if m:
instr = numToWords(int(m.group(1))) + m.group(2)
- # if m.group(2).isupper():
- # a = instr.upper()
- # instr = a
if typename:
a = instr[:1].upper()+instr[1:]
instr = a
elif instr[:1] =="_":
a=instr[1:]
instr =a
- return instr.replace("-","_")
+ rx='[^a-zA-Z0-9_' + extrachr + ']'
+ return re.sub(rx,'_',instr)
+
+# Returns the module and the referenced type name as tuplefrom the $ref
+#
+# local reference: '#/components/schemas/SnssaiExtension' -> '', 'SnssaiExtension'
+# remote reference: 'TS29571_CommonData.yaml#/components/schemas/Ncgi' -> 'TS29571_CommonData', 'Ncgi'
+#
def get_module(ref):
if ref[0]=='#':
return '', ref.split('/')[-1]
@@ -70,6 +119,15 @@
return ref.split("#")[0].replace(".yaml",""),ref.split('/')[-1]
+# The process_path, process_schema, and process_used_schem_name functions
+# parses the paths: parts of the yaml definitions and collects the type refences
+# Encoder and decoder functions are needed for the referenced types as they are used
+# in API messages directly.
+# Also creates prototype for possible arrays
+#
+# If the generation of encoder function or array definiton is missed these functions
+# should be updated to find the missing references.
+#
def process_path(data,module_data):
for m in data:
#print("process_path ", m, " " , data[m])
@@ -95,15 +153,14 @@
if "$ref" in schema:
refmodule,refstr=get_module(schema["$ref"])
if refmodule!= '':
- if not refmodule in module_data['import']:
- module_data['import'].append(refmodule)
+ addImport(refmodule)
refstr=refmodule+"."+refstr
if not refstr in module_data['functions']:
module_data['functions'].append(refstr)
elif "type" in schema:
if schema["type"] == "array" :
if "$ref" in schema["items"] :
- refmodule,refstr=get_module(schema["items"]["$ref"])
+ refmodule,refstr=get_module(schema["items"]["$ref"]) # This direct code generation should be moved out from here.
print("// " + refmodule + " type record of " + refstr + " " +refstr + "_list")
print(f'// external function f_enc_{refstr}_list(in {refstr}_list pdu) return octetstring ')
print('// with { extension "prototype(convert) encode(JSON)" }')
@@ -128,145 +185,10 @@
if "schema" in data:
process_schema(data["schema"],module_data)
-def add_buff(buff, txt, end='\n'):
- return buff+txt+end
-
-def typewriter(buff, indent, pr_type ,name, data ,module_data):
- #print("typewriter ", indent, pr_type ,name, data)
- if "type" in data:
- if data["type"] == "string":
- if "enum" in data:
- enum_data=data["enum"]
- outstr=f'{indent}{pr_type}enumerated {name} {{'
- sep=" "
- for i in enum_data:
- outstr+=sep+i
- sep=", "
- outstr+='}'
- buff=add_buff(buff,outstr)
-
- else:
- buff=add_buff(buff,f'{indent}{pr_type}charstring {name}',end='')
- if "pattern" in data:
- buff=add_buff(buff,f' // (pattern \"{data["pattern"]}\")',end='')
- elif data["type"] == "integer":
- restrition=False
- restritionstr=""
- minstr=" (-infinity.."
- maxstr="infinity)"
- if "minimum" in data:
- restrition=True
- minstr=f' ({data["minimum"]}..'
- if "maximum" in data:
- restrition=True
- maxstr=f'{data["maximum"]})'
- if restrition:
- restritionstr=minstr+maxstr
- buff=add_buff(buff,f'{indent}{pr_type}integer {name}{restritionstr}',end='')
- elif data["type"] == "number":
- buff=add_buff(buff,f'{indent}{pr_type}float {name}',end='')
- elif data["type"] == "boolean":
- buff=add_buff(buff,f'{indent}{pr_type}boolean {name}',end='')
- elif data["type"] == "array":
- buff=add_buff(buff,f'{indent}{pr_type}set of ',end='')
- buff=typewriter(buff,indent+" ","","",data["items"],module_data)
- buff=add_buff(buff,name,end='')
- elif data["type"] == "object":
- if "required" in data:
- required_data=data["required"]
- else:
- required_data=[]
- if "properties" in data:
- if pr_type == "":
- buff=add_buff(buff,f'{indent}{pr_type}set {{',end='')
- else:
- buff=add_buff(buff,f'{indent}{pr_type}set {name} {{',end='')
- prop_data=data["properties"]
- sep=""
- for prop in prop_data:
- buff=add_buff(buff,sep)
- sep=","
- opt_str=""
- if not prop in required_data:
- opt_str=" optional"
- buff=typewriter(buff,indent+" ","",prop,prop_data[prop],module_data)
- buff=add_buff(buff,opt_str,end='')
- buff=add_buff(buff,"")
- buff=add_buff(buff," }",end='')
- if pr_type == "":
- buff=add_buff(buff,name,end='')
- elif "additionalProperties" in data:
- buff=add_buff(buff,f'{indent}{pr_type}set of record {{')
- buff=add_buff(buff,f'{indent} universal charstring key,')
- buff=typewriter(buff,indent+" ","","additionalProperties",data["additionalProperties"],module_data)
- buff=add_buff(buff,"")
- buff=add_buff(buff,f'{indent}}} {name}',end='')
- else:
- buff=add_buff(buff,f'{indent}{pr_type}object2 {name}',end='')
-
- else:
- print (f' // skiped {name}',end='')
- elif "$ref" in data:
- refmodule,refstr=get_module(data["$ref"])
- if refmodule!= '':
- if not refmodule in module_data['import']:
- module_data['import'].append(refmodule)
- refstr=refmodule+"."+refstr
- buff=add_buff(buff,f'{indent}{pr_type}{refstr} {name}',end='')
- elif "allOf" in data:
- #print(name,data["allOf"])
- buff=typewriter(buff,indent+" ","type /* allOf */",name,data["allOf"][1],module_data)
- elif "anyOf" in data:
- enum_found=False
- enum_data=''
- for e in data["anyOf"]:
- if "enum" in e:
- enum_data=e["enum"]
- enum_found=True
- break
-
- if enum_found:
- outstr=f' type enumerated {name}_enum {{'
- sep=" "
- for i in enum_data:
- outstr+=sep+i
- sep=", "
- outstr+='}'
- buff=add_buff(buff,outstr)
- buff=add_buff(buff,'')
- buff=add_buff(buff,f' type union {name} {{')
- buff=add_buff(buff,f' {name}_enum enum_val,')
- buff=add_buff(buff,' charstring other_val')
- buff=add_buff(buff,' } with {')
- buff=add_buff(buff,' variant "JSON: as value"')
- buff=add_buff(buff,' }')
-
-
- elif "oneOf" in data:
- if pr_type == "":
- buff=add_buff(buff,f'{indent}{pr_type}union {{',end='')
- else:
- buff=add_buff(buff,f'{indent}{pr_type}union {name} {{',end='')
- prop_data=data["oneOf"]
- sep=""
- i=1
- for prop in prop_data:
- buff=add_buff(buff,sep)
- sep=","
- opt_str=""
- buff=typewriter(buff,indent+" ","",f'field{i}',prop,module_data)
- i+=1
- buff=add_buff(buff,"")
- buff=add_buff(buff,' } with {')
- buff=add_buff(buff,' variant "JSON: as value"')
- buff=add_buff(buff,' }')
- if pr_type == "":
- buff=add_buff(buff,name,end='')
-
- else:
- print (f' // skiped {name}')
- return buff
-
+# Processes one schema definition and build the data structure needed for code generation.
+# The processed data is appended to the type_tree list.
+# All processing of the schema definition is done here, except the resolution of the allOf
+# The allOf should be reprocessed after the schema processing, because the fields needs to be collected from the processed data.
def type_builder(name,data,tree):
global type_tree
#print("type:", name)
@@ -289,7 +211,7 @@
cename=clean_name(ev,True)
element_data["values"].append(cename)
if cename != ev:
- element_data["variant"].append("text '"+ cename +"' as '"+ev+"'")
+ element_data["variant"].append("text '"+ cename +"' as '"+str(ev)+"'")
else:
element_data["type"]="charstring"
if "pattern" in data:
@@ -352,8 +274,7 @@
element_data["variant"].append("as map")
else:
element_data["type"]="JSON_Generic.JSON_generic_val"
- if not "JSON_Generic" in module_data['import']:
- module_data['import'].append("JSON_Generic")
+ addImport("JSON_Generic")
else:
element_data["type"]=data["type"]
#print('!!!!!!unsupported' + data["type"])
@@ -361,8 +282,7 @@
elif "$ref" in data:
refmodule,refstr=get_module(data["$ref"])
if refmodule!= '':
- if not refmodule in module_data['import']:
- module_data['import'].append(refmodule)
+ addImport(refmodule)
element_data["type"]=refmodule+"."+clean_name(refstr,True)
else:
element_data["type"]=clean_name(refstr,True)
@@ -408,21 +328,30 @@
element_data["fields"].append(field[0])
elif "allOf" in data:
+ element_data["allOf"]=True
+ element_data["fields"]=[]
+ i=0
+ for e in data["allOf"]:
+ if "$ref" in e:
+ refmodule,refstr=get_module(e["$ref"])
+ element_data["fields"].append({"ref":True,"refmodule":refmodule,"refstr":clean_name(refstr,True)})
+ else:
+ field=[]
+ type_builder(f'field{i}',e,field)
+ i+=1
+ element_data["fields"].append(field[0])
element_data["type"]="JSON_Generic.JSON_generic_val"
- if not "JSON_Generic" in module_data['import']:
- module_data['import'].append("JSON_Generic")
+ addImport("JSON_Generic")
else:
element_data["type"]="JSON_Generic.JSON_generic_val"
- if not "JSON_Generic" in module_data['import']:
- module_data['import'].append("JSON_Generic")
+ addImport("JSON_Generic")
#print('!!!!!!unsupported ',name)
#pprint.pprint(data)
if "nullable" in data:
- if not "JSON_Generic" in module_data['import']:
- module_data['import'].append("JSON_Generic")
+ addImport("JSON_Generic")
element_data2={}
element_data2["type"]="union"
element_data2["name"]=element_data["name"]
@@ -438,43 +367,87 @@
else:
tree.append(element_data)
+# Finds the data of the type or return None
+def find_type(tname):
+ for t in type_tree:
+ if t["name"] == tname:
+ return t
+ return None
+
+# Generates the TTCN-3 type definition from the processed type data
def print_type(t, lend="\n", top=False):
global ident_level
global ident_c
- print(t["type"], " ",end="",sep="")
- if top and (t["type"] !="record of"):
- print(t["name"]," ", end="",sep="")
- if "fields" in t:
- print(" {",sep="")
- ident_level+=1
- separatot=ident_c*ident_level
- for f in t["fields"]:
- print(separatot, end="",sep="")
- separatot=",\n" + ident_c*ident_level
- print_type(f,"")
- if "mandatory" in t:
- if f["name"] not in t["mandatory"]:
- print(" optional", end="")
- ident_level-=1
- print("",sep="")
- print(ident_c*ident_level,"}",sep="",end=lend)
- elif "inner_type" in t:
- print_type(t["inner_type"],"")
- elif "values" in t:
- print("{ ",sep="",end="")
- separatot=""
- for e in t["values"]:
- print(separatot, end="",sep="")
- separatot=", "
- print(e, end="",sep="")
- print("}",sep="",end="")
-
- if (not top) or (t["type"] =="record of"):
- print(" ", t["name"],end="",sep="")
- if "restriction" in t:
- print(" ",t["restriction"],end="",sep="")
- print(lend,end="")
+ if "allOf" in t: # The allOf should be reprocessed to collect the actual fields.
+ element_data = {}
+ element_data["name"]=t["name"]
+ element_data["type"]="set"
+ element_data["fields"]=[]
+ element_data["variant"]=[]
+ element_data["mandatory"]=[]
+ is_ok=True
+ for f in t["fields"]: # the "fields" contains the reference or the data of the to be merged objects
+ if "ref" in f: # Collect fields from the referenced type
+ if f["refmodule"] == "" :
+ it=find_type(f["refstr"])
+ if it != None:
+ element_data["fields"]+= it["fields"]
+ element_data["variant"]+= it["variant"]
+ element_data["mandatory"]+= it["mandatory"]
+ else:
+ is_ok=False
+ else:
+ print("// Please add the fields from " + f["refmodule"] + "." + f["refstr"])
+ elif f["type"] == "set": # Collect fields from the directly defined object
+ element_data["fields"]+= f["fields"]
+ element_data["variant"]+= f["variant"]
+ element_data["mandatory"]+= f["mandatory"]
+ else:
+ is_ok=False
+
+ if is_ok: # The field collection was successfull, generate the code
+ print_type(element_data,lend,top)
+ t["fields"]=element_data["fields"]
+
+ else: # the allOf can't be resolved, use generic JSON type
+ del t["fields"] # remove the "fields" list, teh type_builder already filled the type data for the generic JSON type, just use it
+ print_type(t,lend,top)
+ else:
+ print(t["type"], " ",end="",sep="")
+ if top and (t["type"] !="record of"):
+ print(t["name"]," ", end="",sep="")
+ if "fields" in t:
+ print(" {",sep="")
+ ident_level+=1
+ separatot=ident_c*ident_level
+ for f in t["fields"]:
+ print(separatot, end="",sep="")
+ separatot=",\n" + ident_c*ident_level
+ print_type(f,"")
+ if "mandatory" in t:
+ if f["name"] not in t["mandatory"]:
+ print(" optional", end="")
+ ident_level-=1
+ print("",sep="")
+ print(ident_c*ident_level,"}",sep="",end=lend)
+ elif "inner_type" in t:
+ print_type(t["inner_type"],"")
+ elif "values" in t:
+ print("{ ",sep="",end="")
+ separatot=""
+ for e in t["values"]:
+ print(separatot, end="",sep="")
+ separatot=", "
+ print(e, end="",sep="")
+ print("}",sep="",end="")
+
+ if (not top) or (t["type"] =="record of"):
+ print(" ", t["name"],end="",sep="")
+ if "restriction" in t:
+ print(" ",t["restriction"],end="",sep="")
+ print(lend,end="")
+# Gather the variants for the type, builds the variant references
def gather_variants(t,spec,variants):
if "variant" in t:
for v in t["variant"]:
@@ -486,12 +459,15 @@
spec2+="."
gather_variants(f,spec2+f["name"],variants)
+# By default the yaml loader parses the Yes, No as bool value instead of string
+# The bool constructor is overridden to return them as string
from yaml.constructor import Constructor
def add_bool(self, node):
return self.construct_scalar(node)
Constructor.add_constructor(u'tag:yaml.org,2002:bool', add_bool)
+
f=open(sys.argv[1])
@@ -518,6 +494,8 @@
if "schemas" in doc["components"]:
schemas=doc["components"]["schemas"]
for name in schemas:
+ # The type name suffix 'Rm' is used for a nullable type alias
+ # Generate the nullable structure but use the reference to the original type
if (name[-2:] == "Rm" ) and (name[:-2] in schemas ):
type_tree.append({'fields': [{'name': 'null_val',
'nullable': False,
@@ -526,18 +504,17 @@
'nullable': False,
'type': clean_name(name[:-2],True),
'variant': []}],
- 'name': clean_name(name,True),
- 'nullable': True,
- 'type': 'union',
- 'variant': ['JSON: as value']})
- elif name == "NullValue":
+ 'name': clean_name(name,True),
+ 'nullable': True,
+ 'type': 'union',
+ 'variant': ['JSON: as value']})
+ addImport("JSON_Generic")
+ elif name == "NullValue": # Special type, used for nullable enums
type_tree.append({'type': 'JSON_generic_val.JSON_null_val','name':'NullValue','nullable': True})
else:
+ # Normal type schema processing.
data=schemas[name]
type_builder(clean_name(name,True),data,type_tree)
- #buff=typewriter(buff," ", "type ",name,data,module_data)
- #buff=add_buff(buff,'')
- #buff=add_buff(buff,'')
if "responses" in doc["components"]:
#print(doc["components"]["responses"])
for r in doc["components"]["responses"]:
@@ -555,12 +532,16 @@
print("")
for fs in module_data['functions']:
- f=clean_name(fs,True)
- print(f'external function f_enc_{f}(in {f} pdu) return octetstring ')
- print('with { extension "prototype(convert) encode(JSON)" }')
+ f=clean_name(fs,True,".")
+ if "." in f:
+ pre="// "
+ else:
+ pre=""
+ print(pre+f'external function f_enc_{f}(in {f} pdu) return octetstring ')
+ print(pre+'with { extension "prototype(convert) encode(JSON)" }')
print("")
- print(f'external function f_dec_{f}(in octetstring stream, out {f} pdu) return integer ')
- print('with { extension "prototype(backtrack) decode(JSON)" }')
+ print(pre+f'external function f_dec_{f}(in octetstring stream, out {f} pdu) return integer ')
+ print(pre+'with { extension "prototype(backtrack) decode(JSON)" }')
print("")
print("")