Merge pull request #77 from OkieOth/codegen_log

Codegen log
OkieOth · Mar 20, 2022 · a13a09a · a13a09a
2 parents 60aa382 + 32e6952
commit a13a09a
Show file tree

Hide file tree

Showing 11 changed files with 283 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 4.3.0
+* yacg commandline switch to protocol the codegen
+* yacg commandline switch to skip codegen when model version hasn't change to the last protocolled run
+
 # 4.2.0
 * add typescript helper funcs
 * enable plantuml template to visualize type and property tags

diff --git a/bin/generateCodeForYacg.sh b/bin/generateCodeForYacg.sh
@@ -5,14 +5,19 @@ scriptPos=${0%/*}
 # script takes the yacg models and generate the program code
 # based on them
 
+# to see what jobs are executed run:
+#  `bin/generateCodeForYacg.sh --skipCodeGenDryRun`
+
 pushd $scriptPos/.. > /dev/null
 
-echo "create meta model cconfiglasses ..."
+echo "create meta model classes ..."
 if ! pipenv run python3 yacg.py --models \
     resources/models/json/yacg_model_schema.json \
     --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/model.py \
                 pythonBeansTests=${scriptPos}/../tests/model/test_model.py \
                 plantUml=${scriptPos}/../docs/puml/yacg_model.puml \
+    --protocolFile logs/gen_yacg_model.log \
+    --skipCodeGenIfVersionUnchanged $*\
     --templateParameters baseModelDomain=yacg.model.model \
                          title="yacg model"; then
     echo "    ERROR while create meta model classes"
@@ -25,6 +30,8 @@ if ! pipenv run python3 yacg.py --models \
     --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/config.py \
                 pythonBeansTests=${scriptPos}/../tests/model/test_config.py \
                 plantUml=${scriptPos}/../docs/puml/yacg_config_schema.puml \
+    --protocolFile logs/gen_config_model.log \
+    --skipCodeGenIfVersionUnchanged $*\
     --templateParameters baseModelDomain=yacg.model.config \
                          title="yacg configuration model"; then
     echo "    ERROR while create config model classes"
@@ -38,6 +45,8 @@ if ! pipenv run python3 yacg.py --models \
                 pythonBeansTests=${scriptPos}/../tests/model/test_openapi.py \
                 plantUml=${scriptPos}/../docs/puml/yacg_openapi.puml \
     --blackListed yacg.model.model=domain \
+    --protocolFile logs/gen_openapi_model.log \
+    --skipCodeGenIfVersionUnchanged $* \
     --templateParameters baseModelDomain=yacg.model.openapi \
                          title="yacg openapi model"; then
     echo "    ERROR while create openapi model classes"

diff --git a/logs/gen_config_model.log b/logs/gen_config_model.log
@@ -0,0 +1,11 @@
+{
+    "date": "20-03-2022 15:20:17.499061",
+    "jobs": {
+        "default": {
+            "yacg_config_schema.json": {
+                "version": "0.0.1",
+                "md5": "e69a956af165c7beec17c4baed8078f6"
+            }
+        }
+    }
+}
diff --git a/logs/gen_openapi_model.log b/logs/gen_openapi_model.log
@@ -0,0 +1,15 @@
+{
+    "date": "20-03-2022 17:32:31.253440",
+    "jobs": {
+        "default": {
+            "yacg_openapi_paths.json": {
+                "version": "0.0.1",
+                "md5": "a1894122ff8326f4e6298ce7b195eefd"
+            },
+            "yacg_model_schema.json": {
+                "version": "0.0.1",
+                "md5": "72d627008a787dbdcd88135ee5b35f38"
+            }
+        }
+    }
+}
diff --git a/logs/gen_yacg_model.log b/logs/gen_yacg_model.log
@@ -0,0 +1,11 @@
+{
+    "date": "20-03-2022 15:20:16.898379",
+    "jobs": {
+        "default": {
+            "yacg_model_schema.json": {
+                "version": "0.0.1",
+                "md5": "72d627008a787dbdcd88135ee5b35f38"
+            }
+        }
+    }
+}
diff --git a/resources/models/json/yacg_config_schema.json b/resources/models/json/yacg_config_schema.json
@@ -21,14 +21,14 @@
           "description": "list of models used for that job",
           "type": "array",
           "items": {
-            "$ref": "#/definitions/Model"           
+            "$ref": "#/definitions/Model"
           }
         },
         "tasks": {
           "description": "list of tasks that should run",
           "type": "array",
           "items": {
-            "$ref": "#/definitions/Task"           
+            "$ref": "#/definitions/Task"
           }
         }
       },

diff --git a/tests/util/test_yacgUtils.py b/tests/util/test_yacgUtils.py
@@ -1,5 +1,6 @@
 import unittest
 import yacg.util.yacg_utils as yacg_utils
+import yacg.util.protocol_funcs as protocol_funcs
 
 
 class TestYacgUtils (unittest.TestCase):
@@ -108,3 +109,40 @@ def testGetConfigTasksCmdLineSwitch1(self):
         self.assertEqual("puml", jobArray[0].tasks[1].name)
         self.assertEqual("puml", jobArray[1].tasks[1].name)
         self.assertEqual("puml", jobArray[2].tasks[1].name)
+
+    def testGetModelMetaData(self):
+        s = []
+        s.append('/home/eiko/prog/git/yacg/resources/models/json/yacg_openapi_paths.json')
+        s.append('/home/eiko/prog/git/yacg/resources/models/json/yacg_model_schema.json')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, '/home/eiko/prog/git/yacg/resources/models/json/')
+        s = []
+        s.append('yacg_openapi_paths.json')
+        s.append('/home/eiko/prog/git/yacg/resources/models/json/yacg_model_schema.json')
+        s.append('/home/eiko/prog/git/yacg/resources/models/json/yacg_model_schema.json')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, '')
+        s = []
+        s.append('')
+        s.append('')
+        s.append('')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, '')
+        s = []
+        s.append('yacg_openapi_paths.json')
+        s.append('yacg_model_schema.json')
+        s.append('yacg_model_schema.json')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, '')
+        s = []
+        s.append('test/yacg_openapi_paths.json')
+        s.append('test/yacg_model_schema.json')
+        s.append('test/yacg_model_schema.json')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, 'test/')
+        s = []
+        s.append('/test/yacg_openapi_paths.json')
+        s.append('/test/yacg_model_schema.json')
+        s.append('/test/yacg_model_schema.json')
+        identicalPart = protocol_funcs.scanForIdenticalModelParts(s)
+        self.assertEqual(identicalPart, '/test/')
diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-4.2.0
+4.3.0
diff --git a/yacg.py b/yacg.py
@@ -1,7 +1,7 @@
 import argparse
 import sys
 import logging
-
+from datetime import datetime
 from yacg.util.fileUtils import doesFileExist
 from yacg.util.outputUtils import printError, getErrorTxt, getOkTxt
 from yacg.builder.jsonBuilder import getModelFromJson
@@ -13,6 +13,7 @@
 import yacg.util.yacg_utils as yacg_utils
 import yacg.model.config as config
 import yacg.model.modelFuncs as modelFuncs
+import yacg.util.protocol_funcs as protocolFuncs
 
 
 description = """Yet another code generation.
@@ -39,6 +40,10 @@
 parser.add_argument('--usedFilesOnly', help='import models but only print the used files to stdout', action='store_true')
 parser.add_argument('--flattenInheritance', help='flatten included types so that inheritance', action='store_true')
 parser.add_argument('--noLogs', help='do not print logs', action='store_true')
+parser.add_argument('--protocolFile', help='where the metadata of the used models for this specifig gen job are stored')
+parser.add_argument('--skipCodeGenIfVersionUnchanged', help='when the model versions are unchanged, then the codegen is skipped', action='store_true')  # noqa: E501
+parser.add_argument('--skipCodeGenIfMd5Unchanged', help='when the model file md5 is unchanged, then the codegen is skipped', action='store_true')  # noqa: E501
+parser.add_argument('--skipCodeGenDryRun', help='prints only the log messages if codegen should be skipped', action='store_true')
 
 
 def getFileExt(fileName):
@@ -294,11 +299,40 @@ def _isConfigurationValid(codeGenerationJobs):
 def __doCodeGen(codeGenerationJobs, args):
     """process the jobs to do the actual code generation
     """
-
+    previousCodeGenMetaData = protocolFuncs.getPreviousMetaData(args.protocolFile, args.noLogs)
+    previousJobsMetaData = previousCodeGenMetaData.get("jobs", {})
+    codeGenMetaData = {}
+    jobsMetaData = {}
+    codeGenMetaData["date"] = datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")
+    codeGenMetaData["jobs"] = jobsMetaData
+    allSkipped = True
+    jobIndex = 1
     for job in codeGenerationJobs:
         alloadedTypes = readModels(job, args.flattenInheritance)
+        modelMetaData = protocolFuncs.getModelMetaData(alloadedTypes, job.models[0].schema)
+        jobName = job.name if job.name else "UNKNOWN_JOB_{}".format(jobIndex)
+        jobsMetaData[jobName] = modelMetaData
+        jobIndex = jobIndex + 1
+        if protocolFuncs.shouldSkipCodeGen(
+                args.skipCodeGenIfVersionUnchanged,
+                args.skipCodeGenIfMd5Unchanged,
+                previousJobsMetaData,
+                modelMetaData,
+                jobName,
+                args.noLogs) is True:
+            if not args.noLogs:
+                logging.info(" SKIP CODEGEN: {}".format(jobName))
+            continue
+        if not args.noLogs:
+            logging.info(" do codeGen: {}".format(jobName))
+        if args.skipCodeGenDryRun is True:
+            if not args.noLogs:
+                logging.info(" 'skipCodeGenDryRun' is set, so no codeGen is executed': {}".format(jobName))
+            continue
+
         # dictionary types are not really useful as toplevel types ... so it's
         # better to remove them - TODO add a commandline switch for that
+        allSkipped = False
         loadedTypes = []
         for t in alloadedTypes:
             if not isinstance(t, DictionaryType):
@@ -322,6 +356,8 @@ def __doCodeGen(codeGenerationJobs, args):
                     task.blackListed,
                     task.whiteListed,
                     task.randomDataTask)
+    if (not allSkipped) and (args.skipCodeGenDryRun is not True):
+        protocolFuncs.writeProtocolFile(args.protocolFile, codeGenMetaData)
 
 
 def __printUsedFiles(codeGenerationJobs, args):

diff --git a/yacg/model/model.py b/yacg/model/model.py
@@ -103,6 +103,7 @@ def valueAsString(cls, enumValue):
             return ''
 
 
+
 class NumberType (Type):
     """ floating point values
     """
@@ -170,6 +171,7 @@ def valueAsString(cls, enumValue):
             return ''
 
 
+
 class BooleanType (Type):
     """ boolean values
     """
@@ -692,3 +694,5 @@ def initFromDict(self, dictObj):
         subDictObj = dictObj.get('property', None)
         if subDictObj is not None:
             self.property = Property(subDictObj)
+
+