Skip to content

Commit

Permalink
Add the new matter XML files and start developing mechanism to select…
Browse files Browse the repository at this point in the history
…ively load them in. (project-chip#1211)

At this point, the new data is not yet used, this simply adds the first part of the loading mechanism to deal with it.
  • Loading branch information
tecimovic authored Nov 30, 2023
1 parent 3461293 commit 415859f
Show file tree
Hide file tree
Showing 120 changed files with 24,300 additions and 17 deletions.
15 changes: 10 additions & 5 deletions src-electron/zcl/zcl-loader-dotdot.js
Original file line number Diff line number Diff line change
Expand Up @@ -767,8 +767,13 @@ async function processDataType(db, filePath, packageId, data, dataType) {
env.logError(
'Could not find the discriminator for the data type: ' + dataType
)
queryNotification.setNotification(db, "ERROR",
'Could not find the discriminator for the data type: ' + dataType, packageId, 1)
queryNotification.setNotification(
db,
'ERROR',
'Could not find the discriminator for the data type: ' + dataType,
packageId,
1
)
}
}

Expand Down Expand Up @@ -1255,7 +1260,7 @@ async function loadIndividualDotDotFile(db, filePath) {
* @param {*} ctx Context of loading.
* @returns a Promise that resolves with the db.
*/
async function loadDotdotZcl(db, metafile) {
async function loadToplevelXmlFile(db, metafile) {
let ctx = {
metadataFile: metafile,
db: db,
Expand Down Expand Up @@ -1287,13 +1292,13 @@ async function loadDotdotZcl(db, metafile) {
await zclLoader.processZclPostLoading(db, ctx.packageId)
} catch (err) {
env.logError(err)
queryNotification.setNotification(db, "ERROR", err, ctx.packageId, 1)
queryNotification.setNotification(db, 'ERROR', err, ctx.packageId, 1)
throw err
} finally {
await dbApi.dbCommit(db)
}
return ctx
}

exports.loadDotdotZcl = loadDotdotZcl
exports.loadToplevelXmlFile = loadToplevelXmlFile
exports.loadIndividualDotDotFile = loadIndividualDotDotFile
129 changes: 129 additions & 0 deletions src-electron/zcl/zcl-loader-new-data-model.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/**
*
* Copyright (c) 2023 Silicon Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const util = require('../util/util')
const fs = require('fs')
const fsp = fs.promises
const types = require('../util/types')
const env = require('../util/env')
const queryLoader = require('../db/query-loader')

/**
* Parses the new XML files. Returns an object containing
* loaded data:
* clusterIdsLoaded: array of cluster ids that were loaded
*
* @param {*} db
* @param {*} packageId
* @param {*} files
* @param {*} context
* @returns Promise that resolves when all the new XML data is loaded.
*/
async function parseNewXmlFiles(db, packageId, files, context) {
let ret = {
clusterIdsLoaded: [],
}
if (files == null || files.length == 0) {
return ret
}
let clusters = []
for (let f of files) {
// First parse the XML file into a normalized object.
let cluster = await parseSingleNewXmlFile(f)
if (cluster == null) {
env.logWarning(
`Invalid XML file: ${f}, missing toplevel 'cluster' element.`
)
} else {
clusters.push(cluster)
}
}

// We collected the data back. Now we have to write it into the database.
await queryLoader.insertClusters(db, packageId, clusters)

ret.clusterIdsLoaded.push(...clusters.map((cluster) => cluster.code))
return ret
}

function prepXmlFeature(f) {
let feature = {}
feature.bit = parseInt(f.$.bit)
feature.code = f.$.code
feature.name = f.$.name
feature.summary = f.$.summary
return feature
}
function prepXmlAttribute(a) {
let attribute = {}
attribute.id = types.hexStringToInt(a.$.id)
attribute.name = a.$.name
attribute.type = a.$.type
attribute.default = a.$.default
return attribute
}
function prepXmlCommand(c) {
let command = {}
command.id = types.hexStringToInt(c.$.id)
command.name = c.$.name
return command
}
function prepXmlEvent(e) {
let event = {}
event.id = types.hexStringToInt(e.$.id)
event.name = e.$.name
return event
}

async function parseSingleNewXmlFile(f) {
let content = await fsp.readFile(f)
let xmlObject = await util.parseXml(content)
if (xmlObject.cluster == null) {
return null
}

// Ok, now we need to parse the thing properly....
let data = { features: [], attributes: [], commands: [], events: [] }

data.code = types.hexStringToInt(xmlObject.cluster.$.id)
data.name = xmlObject.cluster.$.name
data.revision = xmlObject.cluster.$.revision

if (xmlObject.cluster.features) {
for (let feature of xmlObject.cluster.features[0].feature) {
data.features.push(prepXmlFeature(feature))
}
}
if (xmlObject.cluster.attributes) {
for (let attribute of xmlObject.cluster.attributes[0].attribute) {
data.attributes.push(prepXmlAttribute(attribute))
}
}
if (xmlObject.cluster.commands) {
for (let command of xmlObject.cluster.commands[0].command) {
data.commands.push(prepXmlCommand(command))
}
}
if (xmlObject.cluster.events) {
for (let event of xmlObject.cluster.events[0].event) {
data.events.push(prepXmlEvent(event))
}
}

return data
}

exports.parseNewXmlFiles = parseNewXmlFiles
59 changes: 51 additions & 8 deletions src-electron/zcl/zcl-loader-silabs.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const zclLoader = require('./zcl-loader')
const _ = require('lodash')
const querySessionNotification = require('../db/query-session-notification')
const queryPackageNotification = require('../db/query-package-notification')
const newDataModel = require('./zcl-loader-new-data-model')

/**
* Promises to read the JSON file and resolve all the data.
Expand Down Expand Up @@ -158,6 +159,15 @@ async function collectDataFromJsonFile(metadataFile, data) {
automaticallyCreateFields: false,
}
}

if ('newXmlFile' in obj) {
returnObject.newXmlFile = obj.newXmlFile.map((f) =>
path.join(path.dirname(metadataFile), f)
)
} else {
returnObject.newXmlFile = []
}

env.logDebug(
`Resolving: ${returnObject.zclFiles}, version: ${returnObject.version}`
)
Expand Down Expand Up @@ -665,11 +675,26 @@ function prepareCluster(cluster, context, isExtension = false) {
*/
async function processClusters(db, filePath, packageId, data, context) {
env.logDebug(`${filePath}, ${packageId}: ${data.length} clusters.`)
return queryLoader.insertClusters(
db,
packageId,
data.map((x) => prepareCluster(x, context))
)

// We prepare clusters, but we ignore the ones that have already been loaded.
let preparedClusters = data
.map((x) => prepareCluster(x, context))
.filter((cluster) => {
if (
context.clustersLoadedFromNewFiles &&
context.clustersLoadedFromNewFiles.includes(cluster.code)
) {
env.logDebug(
`Bypassing loading of cluster ${cluster.code} from old files.`
)
return false
} else {
return true
}
})

// and then run the DB process.
return queryLoader.insertClusters(db, packageId, preparedClusters)
}

/**
Expand Down Expand Up @@ -1867,7 +1892,7 @@ async function parseZclFiles(db, packageId, zclFiles, context) {
await processDataTypeDiscriminator(db, packageId, context.ZCLDataTypes)

// Load the Types File first such the atomic types are loaded and can be
//referenced by other types
// referenced by other types
let typesFiles = zclFiles.filter((file) => file.includes('types.xml'))
let typeFilePromise = typesFiles.map((file) =>
parseSingleZclFile(db, packageId, file, context)
Expand Down Expand Up @@ -2231,6 +2256,14 @@ async function processCustomZclDeviceType(db, packageId) {
await queryLoader.insertDeviceTypes(db, packageId, customDeviceTypes)
}

async function loadZclJson(db, metafile) {
return loadZclJsonOrProperties(db, metafile, true)
}

async function loadZclProperties(db, metafile) {
return loadZclJsonOrProperties(db, metafile, false)
}

/**
* Toplevel function that loads the toplevel metafile
* and orchestrates the promise chain.
Expand All @@ -2240,7 +2273,7 @@ async function processCustomZclDeviceType(db, packageId) {
* @param {*} ctx The context of loading.
* @returns a Promise that resolves with the db.
*/
async function loadSilabsZcl(db, metafile, isJson = false) {
async function loadZclJsonOrProperties(db, metafile, isJson = false) {
let ctx = {
metadataFile: metafile,
db: db,
Expand Down Expand Up @@ -2280,6 +2313,15 @@ async function loadSilabsZcl(db, metafile, isJson = false) {
ctx.description
)
}

// Load the new XML files and collect which clusters were already loaded,
// so that they can be ommited while loading the old files.
let newFileResult = await newDataModel.parseNewXmlFiles(
db,
ctx.packageId,
ctx.newXmlFile
)
ctx.clustersLoadedFromNewFiles = newFileResult.clusterIdsLoaded
await parseZclFiles(db, ctx.packageId, ctx.zclFiles, ctx)
// Validate that our attributeAccessInterfaceAttributes, if present, is
// sane.
Expand Down Expand Up @@ -2349,5 +2391,6 @@ async function loadSilabsZcl(db, metafile, isJson = false) {
return ctx
}

exports.loadSilabsZcl = loadSilabsZcl
exports.loadIndividualSilabsFile = loadIndividualSilabsFile
exports.loadZclJson = loadZclJson
exports.loadZclProperties = loadZclProperties
8 changes: 4 additions & 4 deletions src-electron/zcl/zcl-loader.js
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,11 @@ async function loadZcl(db, metadataFile) {
let resolvedMetafile = path.resolve(metadataFile)

if (ext == '.xml') {
return dLoad.loadDotdotZcl(db, resolvedMetafile)
return dLoad.loadToplevelXmlFile(db, resolvedMetafile)
} else if (ext == '.properties') {
return sLoad.loadSilabsZcl(db, resolvedMetafile, false)
return sLoad.loadZclProperties(db, resolvedMetafile)
} else if (ext == '.json') {
return sLoad.loadSilabsZcl(db, resolvedMetafile, true)
return sLoad.loadZclJson(db, resolvedMetafile)
} else {
throw new Error(`Unknown zcl metafile type: ${metadataFile}`)
}
Expand All @@ -135,7 +135,7 @@ async function loadIndividualFile(db, filePath, sessionId) {
`Unable to read file: ${filePath}. Expecting an XML file with ZCL clusters.`
)
env.logWarning(err)
queryNotification.setNotification(db, "WARNING", err, sessionId, 2, 0)
queryNotification.setNotification(db, 'WARNING', err, sessionId, 2, 0)
return { succeeded: false, err }
}
}
Expand Down
Loading

0 comments on commit 415859f

Please sign in to comment.