Automatic loading of packages from zap files. (project-chip#806)

* Remove a bunch of useless logic for loading validator. * Clean up some mess of package loading, removing session where it is not needed. * Update negative case. * Fix the devserver execution for cypress. * Make packageMatch a non-boolean option. * Implement autoloading properly. * Add a package matching mode which simply ignores the packages in zap file, used in unit tests. * Clean up a unit test. * Fix the matter test file to no longer point to zigbee. * Simplify the analyze script. * Update the template loader to not try to reload package options. * Update docs. * Clean up a comment. * Remove unused argument.
emargolis · Nov 16, 2022 · 63cd640 · 63cd640
1 parent b5262d9
commit 63cd640
Show file tree

Hide file tree

Showing 20 changed files with 463 additions and 378 deletions.
diff --git a/docs/api.md b/docs/api.md
diff --git a/package.json b/package.json
@@ -66,17 +66,17 @@
     "zapmeta": "node src-script/zap-start.js --logToStdout --zcl ./test/resource/meta/zcl.json --gen ./test/resource/meta/gen-test.json ./test/resource/test-meta.zap",
     "zaphelp": "node src-script/zap-start.js --help",
     "zap-dotdot": "node src-script/zap-start.js --logToStdout --zcl ./zcl-builtin/dotdot/library.xml",
-    "zap-devserver": "node src-script/zap-start.js server --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --reuseZapInstance",
-    "matterzap-devserver": "node src-script/zap-start.js server --allowCors ---logToStdout --zcl ./zcl-builtin/matter/zcl.json --gen ./test/resource/meta/gen-test.json --reuseZapInstance",
+    "zigbeezap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/zigbee-server/ --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --reuseZapInstance",
+    "matterzap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/matter-server/ --allowCors --logToStdout --zcl ./zcl-builtin/matter/zcl.json --gen ./test/resource/meta/gen-test.json --reuseZapInstance",
     "server": "node src-script/zap-start.js server --logToStdout --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json --reuseZapInstance",
     "stop": "node src-script/zap-start.js stop --reuseZapInstance",
     "status": "node src-script/zap-start.js status --reuseZapInstance",
     "self-check": "node src-script/zap-start.js selfCheck -g ./test/gen-template/zigbee/gen-templates.json",
     "post-import": "node src-script/zap-start.js convert --postImportScript test/resource/test-script.js test/resource/three-endpoint-device.zap -o test.zap",
-    "analyze": "node src-script/zap-start.js analyze -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/zigbee/gen-templates.json ./test/resource/three-endpoint-device.zap -o ./tmp",
+    "analyze": "node src-script/zap-start.js analyze ./test/resource/three-endpoint-device.zap",
     "version": "node src-script/zap-start.js --version",
     "metafile-check": "node src-script/zap-validate-metafiles.js",
-    "test:gen": "npm run gen && npm run genmatter && npm run gendotdot && npm run genmeta",
+    "test:gen": "npm run gen && npm run genmatter && npm run gendotdot && npm run genmeta && npm run genmeta2",
     "gen": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/gen -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/zigbee/gen-templates.json -i ./test/resource/three-endpoint-device.zap -o ./tmp",
     "gen2": "node src-script/zap-generate.js --genResultFile -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/zigbee/gen-templates.json -i ./test/resource/generation-test-file-1.zap -o ./tmp",
     "gen3": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/gen3 -z ./zcl-builtin/dotdot/library.xml -g ./test/gen-template/zigbee/gen-templates.json -i ./test/resource/generation-test-file-1.zap -o ./tmp",

diff --git a/src-electron/db/db-api.js b/src-electron/db/db-api.js
@@ -113,6 +113,16 @@ async function dbCommit(db) {
   })
 }
 
+/**
+ * Not an async function, simply returns a boolean value whether
+ * there is a currently active transaction.
+ *
+ * @returns true if transaction is active, false if not.
+ */
+function isTransactionActive() {
+  return inTransaction
+}
+
 /**
  * Returns a promise to execute a rollback of a transaction.
  *
@@ -299,12 +309,13 @@ async function dbMultiInsert(db, sql, arrayOfArrays) {
     env.logSql('Preparing insert', sql, arrayOfArrays.length)
     let lastIds = []
     let statement = db.prepare(sql, function (err) {
-      if (err) reject('Error While preparing sql command: ' + sql + ', ' + err)
+      if (err)
+        reject('Error while preparing sql statement: ' + sql + ', ' + err)
       for (const singleArray of arrayOfArrays) {
         statement.run(singleArray, (err2) => {
           if (err2)
             reject(
-              'Error While running sql command: ' +
+              'Error while running sql statement: ' +
                 sql +
                 ', values: ' +
                 singleArray +
@@ -316,7 +327,7 @@ async function dbMultiInsert(db, sql, arrayOfArrays) {
       }
       statement.finalize((err3) => {
         if (err3)
-          reject('Error While finalizing sql command: ' + sql + ', ' + err3)
+          reject('Error while finalizing sql statement: ' + sql + ', ' + err3)
         resolve(lastIds)
       })
     })
@@ -578,6 +589,7 @@ function toInClause(value) {
 
 exports.dbBeginTransaction = dbBeginTransaction
 exports.dbCommit = dbCommit
+exports.isTransactionActive = isTransactionActive
 exports.dbRollback = dbRollback
 exports.dbRemove = dbRemove
 exports.dbUpdate = dbUpdate

diff --git a/src-electron/db/db-mapping.js b/src-electron/db/db-mapping.js
@@ -505,6 +505,7 @@ exports.map = {
       packageRef: x.PACKAGE_REF,
       sessionRef: x.SESSION_REF,
       required: x.REQUIRED,
+      type: x.TYPE,
     }
   },
   sessionLog: (x) => {

diff --git a/src-electron/db/query-package.js b/src-electron/db/query-package.js
@@ -240,7 +240,7 @@ async function insertPathCrc(
  * @param {*} crc
  * @param {*} type
  * @param {*} [parentId=null]
- * @returns Promise of an insert or update.
+ * @returns An object containing: id, existedPreviously
  */
 async function registerTopLevelPackage(
   db,
@@ -251,20 +251,28 @@ async function registerTopLevelPackage(
   category = null,
   description = null
 ) {
-  return getPackageByPathAndType(db, path, type).then((row) => {
-    if (row == null) {
-      return dbApi.dbInsert(
-        db,
-        `
+  let row = await getPackageByPathAndType(db, path, type)
+  if (row == null) {
+    // Doesn't exist. We have to add it.
+    let id = await dbApi.dbInsert(
+      db,
+      `
 INSERT INTO PACKAGE ( 
   PATH, CRC, TYPE, PARENT_PACKAGE_REF, VERSION, CATEGORY, DESCRIPTION 
 ) VALUES (?,?,?,?,?,?,?)`,
-        [path, crc, type, null, version, category, description]
-      )
-    } else {
-      return Promise.resolve(row.id)
+      [path, crc, type, null, version, category, description]
+    )
+    return {
+      id: id,
+      existedPreviously: false,
     }
-  })
+  } else {
+    // Exists. We just return it.
+    return {
+      id: row.id,
+      existedPreviously: true,
+    }
+  }
 }
 
 /**
@@ -448,6 +456,35 @@ async function getSessionPackages(db, sessionId) {
     .then((rows) => rows.map(dbMapping.map.sessionPackage))
 }
 
+/**
+ * Returns the session package IDs with types.
+ * @param {*} db
+ * @param {*} sessionId
+ * @returns The promise that resolves into an array of package IDs.
+ */
+async function getSessionPackagesWithTypes(db, sessionId) {
+  return dbApi
+    .dbAll(
+      db,
+      `
+SELECT 
+  SP.PACKAGE_REF,
+  SP.SESSION_REF,
+  SP.REQUIRED,
+  P.TYPE
+FROM 
+  SESSION_PACKAGE AS SP
+INNER JOIN 
+  PACKAGE AS P
+ON 
+  SP.PACKAGE_REF = P.PACKAGE_ID
+WHERE 
+  SP.SESSION_REF = ? AND SP.ENABLED = 1`,
+      [sessionId]
+    )
+    .then((rows) => rows.map(dbMapping.map.sessionPackage))
+}
+
 /**
  * Returns all packages associated w/ a given sessionId
  * @param {*} db
@@ -669,10 +706,14 @@ async function insertPackageExtensionDefault(
   return dbApi.dbMultiInsert(
     db,
     `
-INSERT INTO PACKAGE_EXTENSION_DEFAULT
-  (PACKAGE_EXTENSION_REF, ENTITY_CODE, ENTITY_QUALIFIER, PARENT_CODE, MANUFACTURER_CODE, VALUE)
-VALUES
-  ( ?, ?, ?, ?, ?, ?)
+INSERT INTO PACKAGE_EXTENSION_DEFAULT (
+  PACKAGE_EXTENSION_REF, 
+  ENTITY_CODE, 
+  ENTITY_QUALIFIER, 
+  PARENT_CODE, 
+  MANUFACTURER_CODE, 
+  VALUE 
+) VALUES ( ?, ?, ?, ?, ?, ? )
 ON CONFLICT DO NOTHING
     `,
     defaultArray.map((d) => {
@@ -707,10 +748,15 @@ async function insertPackageExtension(
     .dbMultiInsert(
       db,
       `
-INSERT INTO PACKAGE_EXTENSION 
-  (PACKAGE_REF, ENTITY, PROPERTY, TYPE, CONFIGURABILITY, LABEL, GLOBAL_DEFAULT) 
-VALUES 
-  (?, ?, ?, ?, ?, ?, ?)
+INSERT INTO PACKAGE_EXTENSION ( 
+  PACKAGE_REF, 
+  ENTITY, 
+  PROPERTY, 
+  TYPE, 
+  CONFIGURABILITY, 
+  LABEL, 
+  GLOBAL_DEFAULT
+) VALUES (?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT DO NOTHING`,
       propertyArray.map((p) => {
         return [
@@ -920,6 +966,7 @@ exports.registerTopLevelPackage = registerTopLevelPackage
 exports.updateVersion = updateVersion
 exports.insertSessionPackage = insertSessionPackage
 exports.getSessionPackages = getSessionPackages
+exports.getSessionPackagesWithTypes = getSessionPackagesWithTypes
 exports.insertOptionsKeyValues = insertOptionsKeyValues
 exports.selectAllOptionsValues = selectAllOptionsValues
 exports.selectSpecificOptionValue = selectSpecificOptionValue

diff --git a/src-electron/db/zap-schema.sql b/src-electron/db/zap-schema.sql
@@ -1118,16 +1118,15 @@ CREATE TABLE IF NOT EXISTS "SETTING" (
   UNIQUE(CATEGORY, KEY)
 );
 /*
-  Session Notification table
-*/
-
- CREATE TABLE IF NOT EXISTS "SESSION_NOTICE" (
-   "SESSION_REF" integer,
-   "NOTICE_TYPE" text,
-   "NOTICE_MESSAGE" text,
-   "NOTICE_SEVERITY" integer,
-   "NOTICE_ORDER"  integer primary key autoincrement,
-   "DISPLAY" integer,
-   foreign key (SESSION_REF) references SESSION(SESSION_ID) on delete cascade
- );
-/* EO SCHEMA */
+ Session Notification table
+ */
+CREATE TABLE IF NOT EXISTS "SESSION_NOTICE" (
+  "SESSION_REF" integer,
+  "NOTICE_TYPE" text,
+  "NOTICE_MESSAGE" text,
+  "NOTICE_SEVERITY" integer,
+  "NOTICE_ORDER" integer primary key autoincrement,
+  "DISPLAY" integer,
+  foreign key (SESSION_REF) references SESSION(SESSION_ID) on delete cascade
+);
+/* EO SCHEMA */
diff --git a/src-electron/generator/generation-engine.js b/src-electron/generator/generation-engine.js
@@ -33,21 +33,22 @@ const e = require('express')
 /**
  * Given a path, it will read generation template object into memory.
  *
- * @param {*} context.path
- * @returns context.templates, context.crc
+ * @param {*} path
+ * @returns Object that contains: data, crc, templateData
  */
-async function loadGenTemplate(context) {
-  context.data = await fsPromise.readFile(context.path, 'utf8')
-  context.crc = util.checksum(context.data)
-  context.templateData = JSON.parse(context.data)
+async function loadGenTemplateFromFile(path) {
+  let ret = {}
+  ret.data = await fsPromise.readFile(path, 'utf8')
+  ret.crc = util.checksum(ret.data)
+  ret.templateData = JSON.parse(ret.data)
 
   let requiredFeatureLevel = 0
-  if ('requiredFeatureLevel' in context.templateData) {
-    requiredFeatureLevel = context.templateData.requiredFeatureLevel
+  if ('requiredFeatureLevel' in ret.templateData) {
+    requiredFeatureLevel = ret.templateData.requiredFeatureLevel
   }
-  let status = util.matchFeatureLevel(requiredFeatureLevel, context.path)
+  let status = util.matchFeatureLevel(requiredFeatureLevel, path)
   if (status.match) {
-    return context
+    return ret
   } else {
     throw status.message
   }
@@ -114,7 +115,7 @@ async function loadTemplateOptionsFromJsonFile(
  * @returns promise that resolves with the same context passed in, except packageId added to it
  */
 async function recordTemplatesPackage(context) {
-  context.packageId = await queryPackage.registerTopLevelPackage(
+  let topLevel = await queryPackage.registerTopLevelPackage(
     context.db,
     context.path,
     context.crc,
@@ -124,6 +125,9 @@ async function recordTemplatesPackage(context) {
     context.templateData.description
   )
 
+  context.packageId = topLevel.id
+  if (topLevel.existedPreviously) return context
+
   let promises = []
   env.logDebug(`Loading ${context.templateData.templates.length} templates.`)
 
@@ -484,6 +488,7 @@ async function loadTemplates(
     }
     if (genTemplatesJsonArray != null && genTemplatesJsonArray.length > 0) {
       for (let jsonFile of genTemplatesJsonArray) {
+        if (jsonFile == null || jsonFile == '') continue
         let ctx = await loadSingleTemplate(db, jsonFile)
         if (ctx.error) {
           if (options.failOnLoadingError) globalCtx.error = ctx.error
@@ -496,10 +501,15 @@ async function loadTemplates(
       }
     }
     return globalCtx
-  } else {
+  } else if (genTemplatesJsonArray != null) {
     let ctx = await loadSingleTemplate(db, genTemplatesJsonArray)
     ctx.packageIds = [ctx.packageId]
     return ctx
+  } else {
+    // We didn't load anything, we don't return anything.
+    return {
+      nop: true,
+    }
   }
 }
 
@@ -519,6 +529,7 @@ async function loadSingleTemplate(db, genTemplatesJson) {
     env.logWarning(context.error)
     return Promise.resolve(context)
   }
+  let isTransactionAlreadyExisting = dbApi.isTransactionActive()
 
   let file = path.resolve(genTemplatesJson)
   if (!fs.existsSync(file)) {
@@ -527,16 +538,16 @@ async function loadSingleTemplate(db, genTemplatesJson) {
     return context
   }
   context.path = file
-  await dbApi.dbBeginTransaction(db)
+  if (!isTransactionAlreadyExisting) await dbApi.dbBeginTransaction(db)
   try {
-    context = await loadGenTemplate(context)
+    Object.assign(context, await loadGenTemplateFromFile(file))
     context = await recordTemplatesPackage(context)
     return context
   } catch (err) {
-    env.logInfo(`Can not read templates from: ${context.file}`)
+    env.logInfo(`Can not read templates from: ${file}`)
     throw err
   } finally {
-    dbApi.dbCommit(db)
+    if (!isTransactionAlreadyExisting) await dbApi.dbCommit(db)
   }
 }
 
@@ -846,7 +857,6 @@ async function generateAndWriteFiles(
     appendedPath = templateGeneratorOptions.appendDirectory
   }
 
-  console.log(`Appended path: ${appendedPath}`)
   if (appendedPath != null) {
     outputDirectory = path.join(outputDirectory, appendedPath)
   }