[projmgr] External Generator PoC - Initial commit

tools/projmgr/CMakeLists.txt

@@ -13,12 +13,12 @@ set_property(DIRECTORY PROPERTY VS_STARTUP_PROJECT projmgr)
 SET(PROJMGR_SOURCE_FILES ProjMgr.cpp ProjMgrKernel.cpp ProjMgrCallback.cpp
  ProjMgrParser.cpp ProjMgrWorker.cpp ProjMgrGenerator.cpp ProjMgrXmlParser.cpp
  ProjMgrYamlParser.cpp ProjMgrLogger.cpp ProjMgrYamlSchemaChecker.cpp
- ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp
+ ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp ProjMgrExtGenerator.cpp
 )
 SET(PROJMGR_HEADER_FILES ProjMgr.h ProjMgrKernel.h ProjMgrCallback.h
  ProjMgrParser.h ProjMgrWorker.h ProjMgrGenerator.h ProjMgrXmlParser.h
  ProjMgrYamlParser.h ProjMgrLogger.h ProjMgrYamlSchemaChecker.h
- ProjMgrYamlEmitter.h ProjMgrUtils.h
+ ProjMgrYamlEmitter.h ProjMgrUtils.h ProjMgrExtGenerator.h
 )
 
 list(TRANSFORM PROJMGR_SOURCE_FILES PREPEND src/)

tools/projmgr/include/ProjMgr.h

@@ -88,6 +88,7 @@ class ProjMgr {
  ProjMgrWorker m_worker;
  ProjMgrGenerator m_generator;
  ProjMgrYamlEmitter m_emitter;
+ ProjMgrExtGenerator m_extGenerator;
 
  std::string m_csolutionFile;
  std::string m_cdefaultFile;

tools/projmgr/include/ProjMgrExtGenerator.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2020-2023 Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef PROJMGREXTGENERATOR_H
+#define PROJMGREXTGENERATOR_H
+
+#include "ProjMgrParser.h"
+#include "ProjMgrUtils.h"
+
+ /**
+ * @brief external generator item containing
+ * component identifier
+ * directory for generated files
+ * project type
+ */
+struct ExtGeneratorItem {
+ std::string componentId;
+ std::string genDir;
+ std::string projectType;
+};
+
+
+struct CbuildGenItem {
+ StrVec forContext;
+ std::string device;
+ std::string board;
+ std::string projectPart;
+};
+
+/**
+ * @brief map of used generators, directories and contexts
+*/
+typedef std::map<std::string, StrVecMap> GeneratorContextVecMap;
+
+/**
+ * @brief vector of external generator items
+*/
+typedef std::vector<ExtGeneratorItem> ExtGeneratorVec;
+
+/**
+ * @brief map of vector of external generator items
+*/
+typedef std::map<std::string, ExtGeneratorVec> ExtGeneratorVecMap;
+
+/**
+ * @brief solution/project types
+*/
+static constexpr const char* TYPE_SINGLE_CORE = "single-core";
+static constexpr const char* TYPE_MULTI_CORE = "multi-core";
+static constexpr const char* TYPE_TRUSTZONE = "trustzone";
+static constexpr const char* PROCESSOR_CORE = "processor-core";
+
+/**
+ * @brief projmgr external generator class responsible for handling global generators
+*/
+class ProjMgrExtGenerator {
+public:
+ /**
+ * @brief class constructor
+ */
+ ProjMgrExtGenerator(ProjMgrParser* parser);
+
+
+ /**
+ * @brief class destructor
+ */
+ ~ProjMgrExtGenerator(void);
+
+
+ bool RetrieveGlobalGenerators(void);
+
+ bool IsGlobalGenerator(const std::string& generatorId);
+ bool CheckGeneratorId(const std::string& generatorId, const std::string& componentId);
+ const std::string& GetGlobalGenDir(const std::string& generatorId);
+ const std::string& GetGlobalGenRunCmd(const std::string& generatorId);
+ void AddUsedGenerator(const std::string& generatorId, const std::string& genDir, const std::string& contextId);
+ const GeneratorContextVecMap& GetUsedGenerators(void);
+ bool GetCgen(const std::string& contextId, ClayerItem& cgen);
+
+protected:
+ ProjMgrParser* m_parser = nullptr;
+ StrVec m_globalGeneratorFiles;
+ std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
+ GeneratorContextVecMap m_usedGenerators;
+
+ bool m_checkSchema;
+ std::string m_compilerRoot;
+
+
+
+
+};
+
+#endif // PROJMGREXTGENERATOR_H

tools/projmgr/include/ProjMgrParser.h

@@ -140,12 +140,14 @@ struct TargetType {
  * @brief directories item containing
  * intdir directory,
  * outdir directory,
+ * cbuild directory,
  * cprj directory,
  * rte directory,
 */
 struct DirectoriesItem {
  std::string intdir;
  std::string outdir;
+ std::string cbuild;
  std::string cprj;
  std::string rte;
 };
@@ -402,6 +404,20 @@ struct ClayerItem {
  GeneratorsItem generators;
 };
 
+/**
+ * @brief global generator item containing
+ * generator id,
+ * download url,
+ * bridge program,
+ * output
+*/
+struct GlobalGeneratorItem {
+ std::string id;
+ std::string downloadUrl;
+ std::string run;
+ std::string output;
+};
+
 /**
  * @brief projmgr parser class for public interfacing
 */
@@ -448,6 +464,12 @@ class ProjMgrParser {
  */
  bool ParseGenericClayer(const std::string& input, bool checkSchema);
 
+ /**
+ * @brief parse global generator
+ * @param input generator.yml file
+ */
+ bool ParseGlobalGenerator(const std::string& input, bool checkSchema);
+
  /**
  * @brief get cdefault
  * @return cdefault item
@@ -478,12 +500,19 @@ class ProjMgrParser {
  */
  std::map<std::string, ClayerItem>& GetGenericClayers(void);
 
+ /**
+ * @brief get global generators
+ * @return global generators map
+ */
+ std::map<std::string, GlobalGeneratorItem>& GetGlobalGenerators(void);
+
 protected:
  CdefaultItem m_cdefault;
  CsolutionItem m_csolution;
  std::map<std::string, CprojectItem> m_cprojects;
  std::map<std::string, ClayerItem> m_clayers;
  std::map<std::string, ClayerItem> m_genericClayers;
+ std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
 };
 
 #endif // PROJMGRPARSER_H

tools/projmgr/include/ProjMgrWorker.h

@@ -7,6 +7,7 @@
 #ifndef PROJMGRWORKER_H
 #define PROJMGRWORKER_H
 
+#include "ProjMgrExtGenerator.h"
 #include "ProjMgrKernel.h"
 #include "ProjMgrParser.h"
 #include "ProjMgrUtils.h"
@@ -252,6 +253,7 @@ struct ContextTypesItem {
  * output type,
  * device selection,
  * board selection,
+ * device item struct,
  * list of package requirements,
  * map of required pdsc files and optionally its local path
  * list of component requirements,
@@ -269,6 +271,7 @@ struct ContextTypesItem {
  * valid connections,
  * linker options,
  * map of variables,
+ * external generator directory,
  * boolean processed precedences
 */
 struct ContextItem {
@@ -289,6 +292,7 @@ struct ContextItem {
  OutputTypes outputTypes;
  std::string device;
  std::string board;
+ DeviceItem deviceItem;
  std::vector<PackageItem> packRequirements;
  std::map<std::string, std::pair<std::string, std::string>> pdscFiles;
  std::vector<PackInfo>missingPacks;
@@ -310,6 +314,7 @@ struct ContextItem {
  std::vector<ConnectionsCollectionVec> validConnections;
  LinkerContextItem linker;
  std::map<std::string, std::string> variables;
+ StrMap extGenDir;
  bool precedences;
 };
 
@@ -362,28 +367,22 @@ class ProjMgrWorker {
  /**
  * @brief class constructor
  */
- ProjMgrWorker(void);
+ ProjMgrWorker(ProjMgrParser* parser, ProjMgrExtGenerator* extGenerator);
 
  /**
  * @brief class destructor
  */
  ~ProjMgrWorker(void);
 
- /**
- * @brief set parser
- * @param pointer to parser
- */
- void SetParser(ProjMgrParser* parser);
-
  /**
  * @brief process context
  * @param reference to context
- * @param loadGpdsc boolean automatically load gpdsc, default true
+ * @param loadGenFiles boolean automatically load generated files, default true
  * @param resolveDependencies boolean automatically resolve dependencies, default true
  * @param updateRteFiles boolean update RTE files, default true
  * @return true if executed successfully
  */
- bool ProcessContext(ContextItem& context, bool loadGpdsc = true, bool resolveDependencies = true, bool updateRteFiles = true);
+ bool ProcessContext(ContextItem& context, bool loadGenFiles = true, bool resolveDependencies = true, bool updateRteFiles = true);
 
  /**
  * @brief list available packs
@@ -554,6 +553,8 @@ class ProjMgrWorker {
  */
  bool ExecuteGenerator(std::string& generatorId);
 
+ bool ExecuteExtGenerator(std::string& generatorId);
+
  /**
  * @brief initialize model
  * @return true if executed successfully
@@ -605,10 +606,14 @@ class ProjMgrWorker {
  */
  void PrintMissingFilters(void);
 
+ bool ProcessGlobalGenerators(ContextItem* selectedContext, const std::string& generatorId,
+ std::string& projectType, StrVec& siblings);
+
 protected:
  ProjMgrParser* m_parser = nullptr;
  ProjMgrKernel* m_kernel = nullptr;
  RteGlobalModel* m_model = nullptr;
+ ProjMgrExtGenerator* m_extGenerator = nullptr;
  std::list<RtePackage*> m_loadedPacks;
  std::vector<ToolchainItem> m_toolchains;
  StrVec m_toolchainConfigFiles;
@@ -640,7 +645,7 @@ class ProjMgrWorker {
  bool GetProjectSetup(ContextItem& context);
  bool InitializeTarget(ContextItem& context);
  bool SetTargetAttributes(ContextItem& context, std::map<std::string, std::string>& attributes);
- bool ProcessPrecedences(ContextItem& context);
+ bool ProcessPrecedences(ContextItem& context, bool rerun = false);
  bool ProcessPrecedence(StringCollection& item);
  bool ProcessCompilerPrecedence(StringCollection& item, bool acceptRedefinition = false);
  bool ProcessDevice(ContextItem& context);
@@ -722,10 +727,13 @@ class ProjMgrWorker {
  void UpdatePartialReferencedContext(ContextItem& context, std::string& contextName);
  void ExpandAccessSequence(const ContextItem& context, const ContextItem& refContext, const std::string& sequence, std::string& item, bool withHeadingDot);
  bool GetGeneratorDir(const RteGenerator* generator, ContextItem& context, const std::string& layer, std::string& genDir);
+ bool GetGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
+ bool GetExtGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
  bool ParseContextLayers(ContextItem& context);
  bool AddPackRequirements(ContextItem& context, const std::vector<PackItem> packRequirements);
  void CheckTypeFilterSpelling(const TypeFilter& typeFilter);
  void CheckCompilerFilterSpelling(const std::string& compiler);
+ bool ProcessGeneratedLayers(ContextItem& context);
 };
 
 #endif // PROJMGRWORKER_H

tools/projmgr/include/ProjMgrYamlEmitter.h

@@ -33,12 +33,17 @@ class ProjMgrYamlEmitter {
  */
  static bool GenerateCbuildIndex(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, const std::string& outputDir);
 
+ static bool GenerateCbuildGenIndex(ProjMgrParser& parser, const std::vector<ContextItem*> siblings,
+ const std::string& type, const std::string& outputDir);
+
  /**
  * @brief generate cbuild.yml file
  * @param context pointer to the context
  * @return true if executed successfully
  */
  static bool GenerateCbuild(ContextItem* context);
+
+ static bool GenerateCbuildGen(ContextItem* context, const std::string& output);
 };
 
 #endif // PROJMGRYAMLEMITTER_H

tools/projmgr/include/ProjMgrYamlParser.h

@@ -21,11 +21,15 @@ static constexpr const char* YAML_BASE_DIR = "base-dir";
 static constexpr const char* YAML_BASE_NAME = "base-name";
 static constexpr const char* YAML_BOARD = "board";
 static constexpr const char* YAML_BUILD = "build";
+static constexpr const char* YAML_BUILD_GEN = "build-gen";
 static constexpr const char* YAML_BUILD_IDX = "build-idx";
+static constexpr const char* YAML_BUILD_GEN_IDX = "build-gen-idx";
 static constexpr const char* YAML_BUILDTYPES = "build-types";
 static constexpr const char* YAML_CATEGORY = "category";
 static constexpr const char* YAML_CBUILDS = "cbuilds";
 static constexpr const char* YAML_CBUILD = "cbuild";
+static constexpr const char* YAML_CBUILD_GENS = "cbuild-gens";
+static constexpr const char* YAML_CBUILD_GEN = "cbuild-gen";
 static constexpr const char* YAML_CDEFAULT = "cdefault";
 static constexpr const char* YAML_CLAYERS = "clayers";
 static constexpr const char* YAML_CLAYER = "clayer";
@@ -53,6 +57,7 @@ static constexpr const char* YAML_DEFINE = "define";
 static constexpr const char* YAML_DELPATH = "del-path";
 static constexpr const char* YAML_DESCRIPTION = "description";
 static constexpr const char* YAML_DEVICE = "device";
+static constexpr const char* YAML_DOWNLOAD_URL = "download-url";
 static constexpr const char* YAML_ENDIAN = "endian";
 static constexpr const char* YAML_FILE = "file";
 static constexpr const char* YAML_FILES = "files";
@@ -61,6 +66,7 @@ static constexpr const char* YAML_FORBOARD = "for-board";
 static constexpr const char* YAML_FORCOMPILER = "for-compiler";
 static constexpr const char* YAML_FORCONTEXT = "for-context";
 static constexpr const char* YAML_FORDEVICE = "for-device";
+static constexpr const char* YAML_FORPROJECTPART = "for-project-part";
 static constexpr const char* YAML_FPU = "fpu";
 static constexpr const char* YAML_GENERATED_BY = "generated-by";
 static constexpr const char* YAML_GENERATOR = "generator";
@@ -106,16 +112,19 @@ static constexpr const char* YAML_PATH = "path";
 static constexpr const char* YAML_PROCESSOR = "processor";
 static constexpr const char* YAML_PROJECT = "project";
 static constexpr const char* YAML_PROJECTS = "projects";
+static constexpr const char* YAML_PROJECT_TYPE = "project-type";
 static constexpr const char* YAML_PROVIDES = "provides";
 static constexpr const char* YAML_REGIONS = "regions";
 static constexpr const char* YAML_RTE = "rte";
+static constexpr const char* YAML_RUN = "run";
 static constexpr const char* YAML_SCOPE = "scope";
 static constexpr const char* YAML_SCRIPT = "script";
 static constexpr const char* YAML_SOLUTION = "solution";
 static constexpr const char* YAML_SELECTED_BY = "selected-by";
 static constexpr const char* YAML_SETUPS = "setups";
 static constexpr const char* YAML_SETUP = "setup";
 static constexpr const char* YAML_SET = "set";
+static constexpr const char* YAML_SUB_DIR = "sub-dir";
 static constexpr const char* YAML_SWITCH = "switch";
 static constexpr const char* YAML_TARGETTYPES = "target-types";
 static constexpr const char* YAML_TRUSTZONE = "trustzone";
@@ -177,6 +186,15 @@ class ProjMgrYamlParser {
  bool ParseClayer(const std::string& input, std::map<std::string,
  ClayerItem>& clayers, bool checkSchema);
 
+ /**
+ * @brief parse global generator
+ * @param input generator.yml file
+ * @param reference to store parsed generator item
+ * @param checkSchema false to skip schema validation
+ */
+ bool ParseGlobalGenerator(const std::string& input,
+ std::map<std::string, GlobalGeneratorItem>& generators, bool checkSchema);
+
 protected:
  void ParseMisc(const YAML::Node& parent, std::vector<MiscItem>& misc);
  void ParseDefine(const YAML::Node& parent, std::vector<std::string>& define);