From 1a86bd981fd697d52565d8d62748d0d258d92472 Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Mon, 24 Oct 2022 21:57:14 +0200 Subject: [PATCH 1/2] add top level ref types --- tests/builder/test_json_builder.py | 50 +++++++++-------- tests/model/test_modelFunc.py | 2 +- .../models/json/examples/simple_allof.json | 4 ++ version.txt | 2 +- yacg/builder/impl/dictionaryBuilder.py | 56 +++++++++++++++---- yacg/model/model.py | 3 +- 6 files changed, 80 insertions(+), 37 deletions(-) diff --git a/tests/builder/test_json_builder.py b/tests/builder/test_json_builder.py index 8db86ad1..155eab1a 100644 --- a/tests/builder/test_json_builder.py +++ b/tests/builder/test_json_builder.py @@ -160,9 +160,11 @@ def testGetTypeAndAllRelatedTypes(self): model = config.Model() model.schema = modelFile modelTypes = getModelFromJson(model, []) - t1 = getTypeAndAllRelatedTypes(modelTypes[1]) + # 'OpenApiInfo' + t1 = getTypeAndAllRelatedTypes(modelTypes[2]) self.assertEqual(len(t1), 2) - t3 = getTypeAndAllRelatedTypes(modelTypes[3]) + # 'PathType' + t3 = getTypeAndAllRelatedTypes(modelTypes[27]) self.assertEqual(len(t3), 10) def testSingleTypeSchema3(self): @@ -255,12 +257,12 @@ def testSimpleAllOf(self): model.schema = modelFile modelTypes = getModelFromJson(model, []) self.assertIsNotNone(modelTypes) - self.assertEqual(3, len(modelTypes)) - self._checkUpType(0, 'SimpleAllOfSchema', 1, modelTypes, []) - self._checkUpType(1, 'Address', 3, modelTypes, []) - self._checkUpType(2, 'SimpleAllOfSchemaTypeEnum', 0, modelTypes, []) + self.assertEqual(4, len(modelTypes)) + self._checkUpType(2, 'SimpleAllOfSchema', 1, modelTypes, []) + self._checkUpType(0, 'Address', 3, modelTypes, []) + self._checkUpType(1, 'SimpleAllOfSchemaTypeEnum', 0, modelTypes, []) - addressType = modelTypes[1] + addressType = modelTypes[0] self.assertEqual(4, addressType.properties[0].ordinal) self.assertEqual(5, addressType.properties[1].ordinal) self.assertEqual(6, addressType.properties[2].ordinal) @@ -273,14 +275,16 @@ def testSophisticatedAllOf(self): model.schema = modelFile modelTypes = getModelFromJson(model, []) self.assertIsNotNone(modelTypes) - self.assertEqual(7, len(modelTypes)) - type = self._checkUpType(0, 'MoreSophisticatedAllOf', 1, modelTypes, []) + self.assertEqual(8, len(modelTypes)) + type = self._checkUpType(3, 'MoreSophisticatedAllOf', 1, modelTypes, []) self.assertIsNotNone(type.extendsType) - address = self._checkUpType(1, 'Address', 3, modelTypes, []) + address = self._checkUpType(0, 'Address', 3, modelTypes, []) self.assertEqual(type.extendsType, address) self._checkUpType(2, 'MoreSophisticatedAllOfTypeEnum', 0, modelTypes, []) - self._checkUpType(3, 'MainAddress', 2, modelTypes, []) + self._checkUpType(7, 'MainAddress', 2, modelTypes, []) self._checkUpType(6, 'MainAddressComplex', 3, modelTypes, []) + self.assertEqual(len(modelTypes[1].properties), 3) + self.assertEqual(len(modelTypes[1].properties), len(modelTypes[0].properties)) def testTags(self): modelFile = 'resources/models/json/yacg_model_schema.json' @@ -470,20 +474,20 @@ def testDictionary4(self): self.assertIsNotNone(modelTypes) self.assertEqual(5, len(modelTypes)) self.assertTrue(isinstance(modelTypes[0], ComplexType)) - self.assertTrue(isinstance(modelTypes[1], ComplexType)) - self.assertTrue(isinstance(modelTypes[2], EnumType)) + self.assertTrue(isinstance(modelTypes[1], EnumType)) + self.assertTrue(isinstance(modelTypes[2], ComplexType)) self.assertTrue(isinstance(modelTypes[3], DictionaryType)) self.assertTrue(isinstance(modelTypes[4], DictionaryType)) self.assertTrue(isinstance(modelTypes[3].valueType, IntegerType)) self.assertTrue(isinstance(modelTypes[4].valueType, StringType)) - self.assertTrue(isinstance(modelTypes[1].properties[2].type, StringType)) - self.assertTrue(isinstance(modelTypes[1].properties[3].type, ObjectType)) - self.assertTrue(isinstance(modelTypes[1].properties[4].type, DictionaryType)) - self.assertTrue(isinstance(modelTypes[1].properties[4].type.valueType, IntegerType)) - self.assertTrue(isinstance(modelTypes[1].properties[5].type, DictionaryType)) - self.assertTrue(isinstance(modelTypes[1].properties[5].type.valueType, StringType)) + self.assertTrue(isinstance(modelTypes[0].properties[2].type, StringType)) + self.assertTrue(isinstance(modelTypes[0].properties[3].type, ObjectType)) + self.assertTrue(isinstance(modelTypes[0].properties[4].type, DictionaryType)) + self.assertTrue(isinstance(modelTypes[0].properties[4].type.valueType, IntegerType)) + self.assertTrue(isinstance(modelTypes[0].properties[5].type, DictionaryType)) + self.assertTrue(isinstance(modelTypes[0].properties[5].type.valueType, StringType)) def testEndlessRecursion(self): modelFile = 'resources/models/json/yacg_asyncapi_types.json' @@ -506,13 +510,13 @@ def testForDoubleTypeNames(self): self.assertEqual([], (getNotUniqueTypeNames(modelTypes))) modelTypes.append(modelTypes[0]) self.assertEqual(6, len(modelTypes)) - self.assertEqual(['SimpleAllOfSchema'], (getNotUniqueTypeNames(modelTypes))) + self.assertEqual(['Address'], (getNotUniqueTypeNames(modelTypes))) modelTypes.append(modelTypes[0]) self.assertEqual(7, len(modelTypes)) - self.assertEqual(['SimpleAllOfSchema'], (getNotUniqueTypeNames(modelTypes))) - modelTypes.append(modelTypes[1]) + self.assertEqual(['Address'], (getNotUniqueTypeNames(modelTypes))) + modelTypes.append(modelTypes[2]) self.assertEqual(8, len(modelTypes)) - self.assertEqual(['SimpleAllOfSchema', 'Address'], (getNotUniqueTypeNames(modelTypes))) + self.assertEqual(['Address', 'SimpleAllOfSchema'], (getNotUniqueTypeNames(modelTypes))) def testFixDoubleTypeNames(self): modelFile = 'tests/resources/models/json/examples/schema_with_external_ref_2.json' diff --git a/tests/model/test_modelFunc.py b/tests/model/test_modelFunc.py index 8116d943..5ac01205 100644 --- a/tests/model/test_modelFunc.py +++ b/tests/model/test_modelFunc.py @@ -412,7 +412,7 @@ def testTypeToJSONDict_1(self): self.assertEqual(len(enumDict1), 2) self.assertEqual(enumDict1.get("type", None), "string") self.assertEqual(enumDict1.get("enum", None), ["topic", "direct", "fanout", "default", "headers"]) - dictType1 = extractedTypes[27] + dictType1 = extractedTypes[26] dictDict1 = modelFuncs.typeToJSONDict(dictType1, localTypePrefix) self.assertEqual(len(dictDict1), 3) self.assertEqual(dictDict1.get("type", None), "object") diff --git a/tests/resources/models/json/examples/simple_allof.json b/tests/resources/models/json/examples/simple_allof.json index a5956916..1c99a543 100644 --- a/tests/resources/models/json/examples/simple_allof.json +++ b/tests/resources/models/json/examples/simple_allof.json @@ -5,6 +5,7 @@ "type": "object", "definitions": { "Address": { + "description": "this is a test type", "type": "object", "properties": { "street_addr": { @@ -21,6 +22,9 @@ } }, "required": ["street_address", "city", "state"] + }, + "EvilAddress": { + "$ref": "#/definitions/Address" } }, "allOf": [ diff --git a/version.txt b/version.txt index d6a86bf4..42cdd0b5 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -5.6.2 +5.7.0 diff --git a/yacg/builder/impl/dictionaryBuilder.py b/yacg/builder/impl/dictionaryBuilder.py index 6535f25a..ccaad9ea 100644 --- a/yacg/builder/impl/dictionaryBuilder.py +++ b/yacg/builder/impl/dictionaryBuilder.py @@ -89,6 +89,7 @@ def __initEnumValues(mainType, parsedSchema): def __extractTopLevelObjectType(parsedSchema, modelTypes, modelFileContainer): schemaProperties = parsedSchema.get('properties', None) allOfEntry = parsedSchema.get('allOf', None) + refEntry = parsedSchema.get('$ref', None) additionalProperties = __getAdditionalPropertiesForDictionaryType(parsedSchema) if (schemaProperties is not None) or (allOfEntry is not None) or (additionalProperties is not None): # extract top level type @@ -97,7 +98,7 @@ def __extractTopLevelObjectType(parsedSchema, modelTypes, modelFileContainer): description = parsedSchema.get('description', None) mainType = _extractObjectType( typeNameStr, schemaProperties, additionalProperties, - allOfEntry, description, modelTypes, modelFileContainer) + allOfEntry, refEntry, description, modelTypes, modelFileContainer) __initTags(mainType, parsedSchema) _markRequiredAttributes(mainType, parsedSchema.get('required', [])) mainType.topLevelType = True @@ -118,6 +119,19 @@ def __extractTopLevelEnumType(parsedSchema, modelTypes, modelFileContainer): return False +def __extractTopLevelEnumType(parsedSchema, modelTypes, modelFileContainer): + enumEntry = parsedSchema.get('enum', None) + if enumEntry is not None: + titleStr = parsedSchema.get('title', None) + typeNameStr = toUpperCamelCase(titleStr) + mainType = _extractEnumType(typeNameStr, None, enumEntry, modelTypes, modelFileContainer) + __initTags(mainType, parsedSchema) + __initEnumValues(mainType, parsedSchema) + mainType.topLevelType = True + return True + return False + + def __extractPureArrayType(typeName, parsedSchema, modelTypes, modelFileContainer): if parsedSchema.get('items', None) is not None: typeNameStr = typeName @@ -253,6 +267,7 @@ def _extractTypeAndRelatedTypes(modelFileContainer, desiredTypeName, modelTypes) schemaProperties = modelFileContainer.parsedSchema.get('properties', None) allOfEntry = modelFileContainer.parsedSchema.get('allOf', None) + refEntry = modelFileContainer.parsedSchema.get('$ref', None) additionalProperties = __getAdditionalPropertiesForDictionaryType(modelFileContainer.parsedSchema) if (schemaProperties is not None) or (allOfEntry is not None) or (additionalProperties is not None): # extract top level type @@ -267,7 +282,7 @@ def _extractTypeAndRelatedTypes(modelFileContainer, desiredTypeName, modelTypes) description = modelFileContainer.parsedSchema.get('description', None) type = _extractObjectType( typeNameStr, schemaProperties, additionalProperties, - allOfEntry, description, modelTypes, modelFileContainer) + allOfEntry, refEntry, description, modelTypes, modelFileContainer) type.topLevelType = True if len(type.tags) == 0: tags = modelFileContainer.parsedSchema.get('x-tags', None) @@ -323,8 +338,9 @@ def _extractDefinitionsTypes(definitions, modelTypes, modelFileContainer, desire if itemsEntry is not None: __extractPureArrayType(key, object, modelTypes, modelFileContainer) else: + refEntry = object.get('$ref', None) type = _extractObjectType( - key, properties, additionalProperties, allOfEntry, + key, properties, additionalProperties, allOfEntry, refEntry, description, modelTypes, modelFileContainer) if len(type.tags) == 0: tags = object.get('x-tags', None) @@ -334,7 +350,7 @@ def _extractDefinitionsTypes(definitions, modelTypes, modelFileContainer, desire def _extractObjectType( - typeNameStr, properties, additionalProperties, allOfEntries, + typeNameStr, properties, additionalProperties, allOfEntries, refEntry, description, modelTypes, modelFileContainer): """build a type object @@ -343,6 +359,7 @@ def _extractObjectType( properties -- dict of a schema properties-block additionalProperties -- dict of a schema additionalProperties-block allOfEntries -- dict of allOf block + refEntryDict -- dict of '$ref' key description -- optional description of that type modelTypes -- list of already loaded models modelFileContainer -- file name and path to the model to load, instance of ModelFileContainer @@ -365,17 +382,34 @@ def _extractObjectType( newType.source = modelFileContainer.fileName if description is not None: newType.description = description - if alreadyCreatedType is None: - modelTypes.append(newType) + if allOfEntries is not None: for allOfEntry in allOfEntries: - refEntry = allOfEntry.get('$ref', None) + tmpRefEntry = allOfEntry.get('$ref', None) propertiesEntry = allOfEntry.get('properties', None) if (propertiesEntry is not None): _extractAttributes(newType, propertiesEntry, modelTypes, modelFileContainer) _markRequiredAttributes(type, allOfEntry.get('required', [])) - elif refEntry is not None: - newType.extendsType = _extractReferenceType(refEntry, modelTypes, modelFileContainer) + elif tmpRefEntry is not None: + newType.extendsType = _extractReferenceType(tmpRefEntry, modelTypes, modelFileContainer) + else: + if refEntry is not None: + tmpType = _extractReferenceType(refEntry, modelTypes, modelFileContainer) + ret = None + if isinstance(tmpType, EnumType): + ret = EnumType(tmpType.__dict__) + elif isinstance(tmpType, ArrayType): + ret = ArrayType(tmpType.__dict__) + elif isinstance(tmpType, DictionaryType): + ret = DictionaryType(tmpType.__dict__) + elif isinstance(tmpType, ComplexType): + ret = ComplexType(vars(tmpType)) + if ret is not None: + ret.name = newType.name + _appendToAlreadyLoadedTypes(ret, modelTypes) + return ret + + _appendToAlreadyLoadedTypes(newType, modelTypes) if (hasattr(newType, 'properties')) and (len(newType.properties) == 0): _extractAttributes(newType, properties, modelTypes, modelFileContainer) elif additionalProperties is not None: @@ -1109,7 +1143,7 @@ def __getTypeFromSchemaDictAndAsignId(schema, typeHost, modelTypes, modelFileCon propertiesDict = dictToUse.get('properties', None) if propertiesDict is not None: newInnerTypeName = innerTypeName - typeHost.type = _extractObjectType(newInnerTypeName, propertiesDict, None, None, None, modelTypes, modelFileContainer) + typeHost.type = _extractObjectType(newInnerTypeName, propertiesDict, None, None, None, None, modelTypes, modelFileContainer) else: errorMsg = 'Attention, inner type declarations are currently not implemented for some additional model scenarios.' logging.error(errorMsg) @@ -1299,7 +1333,7 @@ def _initAsyncApiMessageHeaders(messageDict, modelTypes, modelFileContainer, inn propertiesDict = headersDict.get('properties', None) if propertiesDict is not None: - return _extractObjectType(innerTypeName, propertiesDict, None, None, None, modelTypes, modelFileContainer) + return _extractObjectType(innerTypeName, propertiesDict, None, None, None, None, modelTypes, modelFileContainer) else: errorMsg = 'Wrong message headers inner type' logging.error(errorMsg) diff --git a/yacg/model/model.py b/yacg/model/model.py index 99ef21c1..b6cc81d0 100644 --- a/yacg/model/model.py +++ b/yacg/model/model.py @@ -597,7 +597,8 @@ def __init__(self, dictObj=None): self.format = None if dictObj is not None: - self.initFromDict(dictObj) + d = vars(dictObj) if not isinstance(dictObj, dict) else dictObj + self.initFromDict(d) def initFromDict(self, dictObj): if dictObj is None: From 62584360bc1e5ec58d31f72ef75afc0c83a1581f Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Mon, 24 Oct 2022 23:54:10 +0200 Subject: [PATCH 2/2] fix tests --- CHANGELOG.md | 4 +++ .../examples/normalizedOpenApiJson.mako | 4 +-- tests/builder/test_json_builder.py | 12 +++---- .../models/json/examples/simple_allof.json | 2 +- yacg/builder/impl/dictionaryBuilder.py | 33 ++++++++++++------- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd53445..247b8fc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 5.7.0 +* add types with top-level '$ref' entry +* allow allOf for inner types + # 5.6.3 * removed AsyncApi PublishOperation type * fix bugs in normalizeSchema diff --git a/resources/templates/examples/normalizedOpenApiJson.mako b/resources/templates/examples/normalizedOpenApiJson.mako index fe9df99f..90e73fc8 100644 --- a/resources/templates/examples/normalizedOpenApiJson.mako +++ b/resources/templates/examples/normalizedOpenApiJson.mako @@ -156,7 +156,7 @@ "description": "${type.description}", % endif "type": "object" - % if type.extendsType is not None: + % if hasattr(type, 'extendsType') and type.extendsType is not None: ,"allOf": [{ "$ref": "#/components/schemas/${type.extendsType.name}" } @@ -224,7 +224,7 @@ } % endif ] - % else: + % elif hasattr(type, "properties"): % if len(type.properties) > 0: ,"properties": { % for property in type.properties: diff --git a/tests/builder/test_json_builder.py b/tests/builder/test_json_builder.py index 155eab1a..a29d88b3 100644 --- a/tests/builder/test_json_builder.py +++ b/tests/builder/test_json_builder.py @@ -257,7 +257,7 @@ def testSimpleAllOf(self): model.schema = modelFile modelTypes = getModelFromJson(model, []) self.assertIsNotNone(modelTypes) - self.assertEqual(4, len(modelTypes)) + self.assertEqual(6, len(modelTypes)) self._checkUpType(2, 'SimpleAllOfSchema', 1, modelTypes, []) self._checkUpType(0, 'Address', 3, modelTypes, []) self._checkUpType(1, 'SimpleAllOfSchemaTypeEnum', 0, modelTypes, []) @@ -275,14 +275,14 @@ def testSophisticatedAllOf(self): model.schema = modelFile modelTypes = getModelFromJson(model, []) self.assertIsNotNone(modelTypes) - self.assertEqual(8, len(modelTypes)) - type = self._checkUpType(3, 'MoreSophisticatedAllOf', 1, modelTypes, []) + self.assertEqual(12, len(modelTypes)) + type = self._checkUpType(5, 'MoreSophisticatedAllOf', 1, modelTypes, []) self.assertIsNotNone(type.extendsType) address = self._checkUpType(0, 'Address', 3, modelTypes, []) self.assertEqual(type.extendsType, address) - self._checkUpType(2, 'MoreSophisticatedAllOfTypeEnum', 0, modelTypes, []) - self._checkUpType(7, 'MainAddress', 2, modelTypes, []) - self._checkUpType(6, 'MainAddressComplex', 3, modelTypes, []) + self._checkUpType(4, 'MoreSophisticatedAllOfTypeEnum', 0, modelTypes, []) + self._checkUpType(11, 'MainAddress', 2, modelTypes, []) + self._checkUpType(10, 'MainAddressComplex', 3, modelTypes, []) self.assertEqual(len(modelTypes[1].properties), 3) self.assertEqual(len(modelTypes[1].properties), len(modelTypes[0].properties)) diff --git a/tests/resources/models/json/examples/simple_allof.json b/tests/resources/models/json/examples/simple_allof.json index e1026725..238a3eae 100644 --- a/tests/resources/models/json/examples/simple_allof.json +++ b/tests/resources/models/json/examples/simple_allof.json @@ -32,7 +32,7 @@ "type": "object", "allOf": [ { - "$ref": "#/definitions/Addres" + "$ref": "#/definitions/Address" }, { "properties": { diff --git a/yacg/builder/impl/dictionaryBuilder.py b/yacg/builder/impl/dictionaryBuilder.py index 22854231..ee5afefd 100644 --- a/yacg/builder/impl/dictionaryBuilder.py +++ b/yacg/builder/impl/dictionaryBuilder.py @@ -384,14 +384,7 @@ def _extractObjectType( newType.description = description if allOfEntries is not None: - for allOfEntry in allOfEntries: - tmpRefEntry = allOfEntry.get('$ref', None) - propertiesEntry = allOfEntry.get('properties', None) - if (propertiesEntry is not None): - _extractAttributes(newType, propertiesEntry, modelTypes, modelFileContainer) - _markRequiredAttributes(type, allOfEntry.get('required', [])) - elif tmpRefEntry is not None: - newType.extendsType = _extractReferenceType(tmpRefEntry, modelTypes, modelFileContainer) + __handleAllOf(newType, allOfEntries, modelTypes, modelFileContainer) else: if refEntry is not None: tmpType = _extractReferenceType(refEntry, modelTypes, modelFileContainer) @@ -908,7 +901,8 @@ def _extractComplexType(newTypeName, newProperty, propDict, modelTypes, modelFil modelFileContainer -- file name and path to the model to load """ - innerTypeName = toUpperCamelCase(newTypeName + ' ' + newProperty.name) + propName = newProperty.name if newProperty.name is not None else "Inner" + innerTypeName = toUpperCamelCase(newTypeName + ' ' + propName) properties = propDict.get('properties', None) additionalProperties = __getAdditionalPropertiesForDictionaryType(propDict) newInnerType = ComplexType() if additionalProperties is None else DictionaryType() @@ -930,12 +924,27 @@ def _extractComplexType(newTypeName, newProperty, propDict, modelTypes, modelFil if additionalProperties is not None: _extractDictionaryValueType(newInnerType, additionalProperties, modelTypes, modelFileContainer) else: - logging.error( - "modelFile: %s, type=%s, property=%s: inner complex type without properties" - % (modelFileContainer.fileName, newTypeName, newProperty.name)) + allOfEntries = propDict.get("allOf", None) + if allOfEntries is not None: + __handleAllOf(newInnerType, allOfEntries, modelTypes, modelFileContainer) + else: + logging.error( + "modelFile: %s, type=%s, property=%s: inner complex type without properties" + % (modelFileContainer.fileName, newTypeName, newProperty.name)) return newInnerType +def __handleAllOf(newType, allOfEntries, modelTypes, modelFileContainer): + for allOfEntry in allOfEntries: + tmpRefEntry = allOfEntry.get('$ref', None) + propertiesEntry = allOfEntry.get('properties', None) + if (propertiesEntry is not None): + _extractAttributes(newType, propertiesEntry, modelTypes, modelFileContainer) + _markRequiredAttributes(type, allOfEntry.get('required', [])) + elif tmpRefEntry is not None: + newType.extendsType = _extractReferenceType(tmpRefEntry, modelTypes, modelFileContainer) + + def _markRequiredAttributes(type, requiredArray): """interate over the requiredArray and mark all matching attributes in the properties type.