Skip to content

Commit

Permalink
Implement ci target for Makefile to reproduce github CI actions & art…
Browse files Browse the repository at this point in the history 
…ifacts (#1769)

* Implement CI target in Makefile to emulate github CI actions & artifacts

* Improve filter for metadata

* metadata.py: update usage output for wrong number of input arguments / code review

* metadata.py: remove excessive checks for the second input argument / code review

* metadata.py: remove hard-coded model for multi-lang builds in ModelName argument processing / code review

* metadata.py: remove hard-coded models for multi-lang builds in file name pattern processing / code review

* metadata.py: update usage output to remove ambiguity about json extension for output file

* metadata.py: unify new lines style formatting

* metadata.py: sort the list of processing files in alphanumeric order before looping through them to get the same lang order on every generation in every json output file
  • Loading branch information
@ia
ia authored Aug 4, 2023
1 parent 4533c2f commit 3f880d9
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 15 deletions.
53 changes: 48 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ DOCKER_CMD=$(DOCKER_BIN) -f $(DOCKER_YML) run --rm builder
# MkDocs config
MKDOCS_YML=$(CURDIR)/scripts/IronOS-mkdocs.yml

# supported models
MODELS=TS100 TS80 TS80P Pinecil MHP30 Pinecilv2 S60 TS101 # target names & dir names
MODELS_ML=Pinecil Pinecilv2 # target names
MODELS_MULTILANG=Pinecil_multi-lang Pinecilv2_multi-lang # dir names

# zip command (to pack artifacts)
ZIP=zip -q -j -r


### targets

Expand Down Expand Up @@ -91,7 +99,7 @@ list:
@echo
@echo "Full list of current supported IDs:"
@echo " * LANG_ID: $(shell echo "`ls Translations/ | grep -e "^translation_.*.json$$" | sed -e 's,^translation_,,g; s,\.json$$,,g; ' | tr '\n' ' '`")"
@echo " * MODEL_ID: TS100 TS101 TS80 TS80P MHP30 Pinecil Pinecilv2 S60"
@echo " * MODEL_ID: $(MODELS)"
@echo
@echo "For example, to make a local build of IronOS firmware for TS100 with English language, just type:"
@echo
Expand Down Expand Up @@ -171,11 +179,41 @@ build-all:
@chmod 0777 $(OUT_DIR)
cd source && bash ./build.sh
@echo "All Firmware built"
@cp -r $(OUT_HEX)/*.bin $(OUT_DIR)
@cp -r $(OUT_HEX)/*.hex $(OUT_DIR)
@cp -r $(OUT_HEX)/*.dfu $(OUT_DIR)
@for model in $(MODELS); do \
mkdir -p $(OUT_DIR)/$${model} ; \
cp -r $(OUT_HEX)/$${model}_*.bin $(OUT_DIR)/$${model}/ ; \
cp -r $(OUT_HEX)/$${model}_*.hex $(OUT_DIR)/$${model}/ ; \
cp -r $(OUT_HEX)/$${model}_*.dfu $(OUT_DIR)/$${model}/ ; \
done;
@echo "Resulting output directory: $(OUT_DIR)"

# target to build multilang supported builds for Pinecil & PinecilV2
build-multilang:
@for modelml in $(MODELS_ML); do \
$(MAKE) -C source/ -j2 model=$${modelml} firmware-multi_compressed_European firmware-multi_compressed_Bulgarian+Russian+Serbian+Ukrainian firmware-multi_Chinese+Japanese ; \
mkdir -p $(OUT_DIR)/$${modelml}_multi-lang ; \
cp $(OUT_HEX)/$${modelml}_multi_*.bin $(OUT_DIR)/$${modelml}_multi-lang ; \
cp $(OUT_HEX)/$${modelml}_multi_*.hex $(OUT_DIR)/$${modelml}_multi-lang ; \
cp $(OUT_HEX)/$${modelml}_multi_*.dfu $(OUT_DIR)/$${modelml}_multi-lang ; \
done;
@echo "Resulting output directory: $(OUT_DIR)"

# target to reproduce zips according to github CI settings; artifacts will be in $(OUT_DIR)/CI/*.zip
ci: tests build-all build-multilang
@mkdir -p $(OUT_DIR)/metadata;
@for m in $(MODELS) $(MODELS_MULTILANG); do \
cp LICENSE scripts/LICENSE_RELEASE.md $(OUT_DIR)/$${m}/ ; \
$(ZIP) $(OUT_DIR)/$${m}.zip $(OUT_DIR)/$${m} ; \
./source/metadata.py $${m}.json $${m}; \
cp $(OUT_HEX)/$${m}.json $(OUT_DIR)/metadata; \
done;
@$(ZIP) $(OUT_DIR)/metadata.zip $(OUT_DIR)/metadata
@mkdir -p $(OUT_DIR)/CI
@mv $(OUT_DIR)/*.zip $(OUT_DIR)/CI
@chmod 0777 $(OUT_DIR)/CI
@chmod 0666 $(OUT_DIR)/CI/*.zip
@echo "Resulting artifacts directory: $(OUT_DIR)/CI"

# pass-through target for Makefile inside source/ dir
%:
$(MAKE) -C source/ $@
Expand All @@ -191,4 +229,9 @@ clean-build:
clean-full: clean-build docker-clean

# phony targets
.PHONY: help list docker-check docker-shell docker-build docker-clean docs docs-deploy test-md test-sh test-py test-ccpp tests clean-build clean-full
.PHONY: help list
.PHONY: docker-check docker-shell docker-build docker-clean
.PHONY: docs docs-deploy
.PHONY: test-md test-sh test-py test-ccpp tests
.PHONY: build-all build-multilang ci
.PHONY: clean-build clean-full
4 changes: 2 additions & 2 deletions scripts/IronOS.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ WORKDIR /build/ironos
# Installing the two compilers (ARM & RISCV), python3 & pip, clang tools, etc.:
## - compilers: gcc-*, newlib-*
## - python3: py*, black (required to check Python code formatting)
## - misc: findutils, make, git, diffutils
## - misc: findutils, make, git, diffutils, zip
## - musl-dev (required for the multi lang firmwares)
## - clang (required for clang-format to check C++ code formatting)
## - shellcheck (to check sh scripts)

ARG APK_COMPS="gcc-riscv-none-elf gcc-arm-none-eabi newlib-riscv-none-elf \
newlib-arm-none-eabi"
ARG APK_PYTHON="python3 py3-pip black"
ARG APK_MISC="findutils make git diffutils"
ARG APK_MISC="findutils make git diffutils zip"
ARG APK_DEV="musl-dev clang bash clang-extra-tools shellcheck"

# PIP packages to check & test Python code
Expand Down
28 changes: 20 additions & 8 deletions source/metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,33 @@
# Creates an index metadata json file of the hexfiles folder
# This is used by automation like the Pinecil updater


if len(sys.argv) != 2:
print("Requires the output json name as an arg")
if len(sys.argv) < 2 or len(sys.argv) > 3:
print("Usage: metadata.py OUTPUT_FILE [model]")
print(" OUTPUT_FILE - the name of output file in json format with meta info about binary files")
print(" model [optional] - name of the model (as for `make model=NAME`) to scan files for explicitly (all files in source/Hexfile by default otherwise)")
exit(1)

# If model is provided explicitly to scan related files only for json output, then process the argument
ModelName = None
if len(sys.argv) == 3:
ModelName = sys.argv[2]
if ModelName.endswith("_multi-lang"):
# rename on-the-fly for direct compatibility with make targets like PINECILMODEL_multi-lang
ModelName = ModelName.rstrip("-lang")

HERE = Path(__file__).resolve().parent

HexFileFolder = os.path.join(HERE, "Hexfile")
OutputJSONPath = os.path.join(HexFileFolder, sys.argv[1])
TranslationsFilesPath = os.path.join(HERE.parent, "Translations")


def load_json(filename: str):
with open(filename) as f:
return json.loads(f.read())


def read_git_tag():
return f"{subprocess.check_output(['git', 'rev-parse', '--short=7', 'HEAD']).strip().decode('ascii').upper()}"


def read_version():
with open(HERE / "version.h") as version_file:
for line in version_file:
Expand All @@ -40,10 +46,9 @@ def read_version():
return matches[0]
raise Exception("Could not parse version")


# Fetch our file listings
translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")]
output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))]
output_files = [os.path.join(HexFileFolder, f) for f in sorted(os.listdir(HexFileFolder)) if os.path.isfile(os.path.join(HexFileFolder, f))]

parsed_languages = {}
for path in translation_files:
Expand All @@ -61,6 +66,13 @@ def read_version():
if file_path.endswith(".hex") or file_path.endswith(".dfu"):
# Find out what language this file is
name: str = os.path.basename(file_path)
if ModelName is not None:
# If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it
if not name.startswith(ModelName + "_"):
continue
# If build of interest is not multi-lang one but scanning one is not MODEL_LANG-ID here, then skip it to avoid mess in json between MODEL_LANG-ID & MODEL_multi'
if not ModelName.endswith("_multi") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name):
continue
matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name)
if matches:
matches = matches[0]
Expand Down

0 comments on commit 3f880d9

Please sign in to comment.