diff --git a/.circleci/config.yml b/.circleci/config.yml index a0e4aabb..9616e69b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,6 @@ orbs: jobs: install-test-coverage: - # executor: python/default resource_class: medium docker: - image: ubuntu:rolling @@ -19,42 +18,30 @@ jobs: export DEBIAN_FRONTEND=noninteractive export TZ=America/USA apt update - apt install -y build-essential libantlr4-runtime-dev antlr4 cmake ninja-build python3-pip python3-dev python3 python3-pip python3-venv git lcov rubygems + apt install -y build-essential libantlr4-runtime-dev antlr4 cmake ninja-build python3-pip python3-dev python3 python3-pip python3-venv python3-build git lcov rubygems python3 -m venv venv echo "source $PWD/venv/bin/activate" >> $BASH_ENV - echo 'export SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_BUILD_TYPE:STRING=Debug -DCODE_COVERAGE=ON"' >> $BASH_ENV - # because we do not have enough RAM - echo 'export SKBUILD_BUILD_OPTIONS="-j1"' >> $BASH_ENV source $BASH_ENV - - - run: - command: | - update-alternatives --install /usr/bin/python python /usr/bin/python3 10; - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10; - - run: git submodule sync - run: git submodule update --init - # - python/load-cache - - run: command: | gcc -v; python --version; cmake --version; + meson --version; free; - run: pip install jupyter - run: gem install coveralls-lcov - #- run: - # command: | - # echo export CXXFLAGS=\"$CXXFLAGS --param ggc-min-expand=1 --param ggc-min-heapsize=32768\" >> "$BASH_ENV" - - py/install-setup-py: - rm-package-dir: true - build-args: "--verbose" - # - python/save-cache + - run: + name: build + command: | + python3 -m build -Csetup-args="-Db_coverage=true" -Ccompile-args="-j 1 --verbose" . # -j1 because we do not have enough RAM + mv $MODULE_NAME "removed_because_of_coverage_$MODULE_NAME" - py/test-and-coverage: - coverage: false + coverage: false # dissable python coverage, we are collecting only C++ one - run: name: Filter lcov outputs for scikit-build and upload it to coveralls.io command: | @@ -63,7 +50,7 @@ jobs: fi if [ -z "$CIRCLE_PULL_REQUEST" ]; then lcov --compat-libtool --directory . --capture --output-file coverage.info.raw; - lcov --remove coverage.info.raw -o coverage.info '/usr/*' '*_skbuild/*'; + lcov --remove coverage.info.raw -o coverage.info '/usr/*'; sed -i 's/\/root\/project\///g' coverage.info; sed -i 's/\/root\///g' coverage.info; coveralls-lcov coverage.info --repo-token ${COVERALLS_REPO_TOKEN}; @@ -72,7 +59,6 @@ jobs: fi install-test-deploy: - # executor: python/default resource_class: medium docker: - image: ubuntu:rolling @@ -85,16 +71,11 @@ jobs: export DEBIAN_FRONTEND=noninteractive export TZ=America/USA apt update - apt install -y build-essential libantlr4-runtime-dev antlr4 cmake ninja-build python3-pip python3-dev python3 python3-pip python3-venv git lcov rubygems + # libantlr4-runtime-dev antlr4 are ommited in order that meson downloads them using wrap + apt install -y build-essential cmake ninja-build python3-pip python3-dev python3 python3-pip python3-venv python3-build git lcov rubygems python3 -m venv venv echo "source $PWD/venv/bin/activate" >> $BASH_ENV - # because we do not have enough RAM - echo 'export SKBUILD_BUILD_OPTIONS="-j1"' >> $BASH_ENV source $BASH_ENV - - run: - command: | - update-alternatives --install /usr/bin/python python /usr/bin/python3 10; - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10; - run: git submodule sync - run: git submodule update --init @@ -109,13 +90,12 @@ jobs: - run: pip install jupyter - run: gem install coveralls-lcov - #- run: - # command: | - # echo export CXXFLAGS=\"$CXXFLAGS --param ggc-min-expand=1 --param ggc-min-heapsize=32768\" >> "$BASH_ENV" - - py/install-setup-py: - rm-package-dir: true - build-args: "--verbose" - # - python/save-cache + - run: + name: build + command: | + python3 -m build -Ccompile-args="-j 1 --verbose" . # -j1 because we do not have enough RAM + rm -rf $MODULE_NAME + - py/test-and-coverage: coverage: false - py/deploy-pypi-on-tag: diff --git a/.gitignore b/.gitignore index fc711135..cb481f08 100644 --- a/.gitignore +++ b/.gitignore @@ -36,15 +36,8 @@ dist/ _skbuild/ .eggs/ hdlConvertor.egg-info - -# parser generated source dirs/files -src/svConvertor/sv2017Parser/ -src/verilogPreproc/verilogPreprocParser/ -src/vhdlConvertor/vhdlParser/ - -include/hdlConvertor/svConvertor/sv2017Parser/*.h -include/hdlConvertor/verilogPreproc/verilogPreprocParser/*.h -include/hdlConvertor/vhdlConvertor/vhdlParser/*.h +venv/ +subprojects/antlr4-runtime # devel archives @@ -66,4 +59,4 @@ coverage.info coverage.info.raw # Ignore .vscode files -.vscode/ \ No newline at end of file +.vscode/ diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 580cb8cf..00000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Defines the CMake commands/policies -cmake_minimum_required(VERSION 3.10.0) - -# Set the project name -project(libhdlConvertor) - -# Java is used to generate source files for parsers, it is required only for installation -find_package(Java COMPONENTS Runtime REQUIRED) -include(UseJava) - -if(SKBUILD) - set(HDLCONVERTOR_PYTHON ON) -endif() - -if(HDLCONVERTOR_PYTHON) - message(STATUS "Building python hdlConvertor module with libhdlConvertor linked in") - find_package(PythonExtensions REQUIRED) - find_package(PythonLibs REQUIRED) - find_package(Cython REQUIRED) -else() - message(STATUS "Building libhdlConvertor (C++ lib) only, to build python module use setup.py.") -endif() - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj") - # using /MD flag for antlr4_runtime (for Visual C++ compilers only) - #set(ANTLR4_WITH_STATIC_CRT OFF) - #set(WITH_STATIC_CRT) -elseif(CMAKE_SYSTEM_NAME MATCHES "CYGWIN") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj") -elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pedantic -W -Wno-attributes") -endif() - -set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "ON") -option(CODE_COVERAGE "Enable coverage reporting" OFF) - -add_subdirectory(src) - -if(HDLCONVERTOR_PYTHON) - add_subdirectory(hdlConvertor) -else() - SET(CPACK_GENERATOR "DEB") - SET(CPACK_DEBIAN_PACKAGE_MAINTAINER "Michal Orsak ") - SET(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/Nic30/hdlConvertor") - SET(CPACK_DEBIAN_PACKAGE_DEPENDS "uuid-dev, libantlr4-runtime-dev, antlr4") - INCLUDE(CPack) -endif() - - diff --git a/README.md b/README.md index 20d7d7dd..b33fe3df 100644 --- a/README.md +++ b/README.md @@ -29,48 +29,37 @@ It contains: Linux: Installing dependencies (Ubuntu 20.10) ``` -# we recommend to use gcc>=9, Visual Studio 2017 or equivalent +# gcc>=9, Visual Studio 2017 or equivalent recommended sudo apt install build-essential uuid-dev cmake default-jre python3 python3-dev python3-pip libantlr4-runtime-dev antlr4 ninja-build ``` Installing this library ``` +# python recommends to use venv (virtualenv) https://docs.python.org/3/library/venv.html +# sudo pip3 install venv +# python3 -m venv myvenv +# source myvenv/bin/activate + # note this may be older version than you see in repo -sudo pip3 install hdlConvertor +pip3 install hdlConvertor # or download repository and run -sudo pip3 install --upgrade --force-reinstall --no-cache-dir git+https://github.com/Nic30/hdlConvertorAst.git -sudo pip3 install -r requirements.txt -sudo python3 setup.py install - -# if you are using version from git rather uninstall -# old library first if required -# sudo pip3 uninstall hdlConvertor -``` - -Installer also supports other options which may be usefull -``` -python setup.py install --prefix /tmp/python_install/ --build-type Debug -- -DANTLR_JAR_LOCATION=/antlr-4.7.1-complete.jar -- VERBOSE=1 +pip3 install --upgrade --force-reinstall --no-cache-dir git+https://github.com/Nic30/hdlConvertorAst.git # optionally +pip3 install . ``` You can also install only C++ library/generate .deb package (nothing specific, just normal cmake-based library) ``` -mkdir build && cd build -cmake .. && cmake . --build -cpack # to generate .deb package +meson setup build -Dpython_package=false +ninja -C build ``` - -Windows: - -Take a look at appveyor.yml and [doc/windows_installation_and_build.rst](doc/windows_installation_and_build.rst). It is required to download antlr4 first and have visual studio or other c++ compiler installed. - +For dev purposes also this link could be usefull https://meson-python.readthedocs.io/en/latest/how-to-guides/meson-args.html#how-to-guides-meson-args ## Usage The HDL AST (the parsed code) is represented by objects from `hdlConvertor.hdlAst`. Parsing and code modification is straightforward, as you can see in following example [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Nic30/hdlConvertor/master?filepath=notebooks%2F01_parse_and_dump.ipynb) - ### Similar projects: (Tell us If you know about some other project!) diff --git a/hdlConvertor/CMakeLists.txt b/hdlConvertor/CMakeLists.txt deleted file mode 100644 index b0cbbb17..00000000 --- a/hdlConvertor/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ - -#include(${CMAKE_CURRENT_SOURCE_DIR}../src/CMake_antlr4.txt) - - -file(GLOB hdlConvertor_py_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" -) -include_directories( - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/../include/" - "${ANTLR4CPP_INCLUDE_DIRS}" - ${PYTHON_INCLUDE_DIRS} -) - -add_cython_target(_hdlConvertor CXX) -add_library(_hdlConvertor MODULE ${_hdlConvertor} - ${hdlConvertor_py_SRC}) -python_extension_module(_hdlConvertor) - -target_link_libraries(_hdlConvertor - hdlConvertor_cpp_static -) -if(PYTHON_VERSION_MAJOR STREQUAL "2") - target_link_libraries(_hdlConvertor - stdc++fs - ) -endif() -if(CODE_COVERAGE) - target_link_libraries(_hdlConvertor - gcov - ) -endif() - -set_coverage_if_enabled(_hdlConvertor OFF) - - -#set_target_properties(_hdlConvertor -# PROPERTIES VERSION "1.0.0" -# OUTPUT_NAME _hdlConvertor -# LIBRARY_OUTPUT_DIRECTORY "${HDLCONVERTOR_SOURCE_DIR}/dist" -# RUNTIME_OUTPUT_DIRECTORY "${HDLCONVERTOR_SOURCE_DIR}/dist" -#) - -install(TARGETS _hdlConvertor LIBRARY DESTINATION - hdlConvertor) - diff --git a/hdlConvertor/meson.build b/hdlConvertor/meson.build new file mode 100644 index 00000000..0243b8a1 --- /dev/null +++ b/hdlConvertor/meson.build @@ -0,0 +1,41 @@ + +# install all python sources +py_sources = run_command(py3, '../utils/rec_glob.py', '.', '*.py', check:true)\ + .stdout().strip().split('\n') + +# mark sources of a single module for an installation +py3.install_sources( + py_sources, + pure: false, # Will be installed next to binaries + subdir: 'hdlConvertor', # Folder relative to site-packages to install to +) + +_hdlConvertor_cpp_sources = [ + 'toPy_expr.cpp', + 'toPy_statements.cpp', + 'toPy_type.cpp', + 'toPy.cpp', + 'toPy.h', +] + +py3.extension_module( + '_hdlConvertor', + _hdlConvertor_cpp_sources + ['_hdlConvertor.pyx', 'python_ver_independent_str.pyx', 'verilogPreproc.pyx'], + include_directories: [mainIncludeDir_inc], + dependencies : [py3_dep, antlr4_cpp_dep, hdlConvertor_cpp_dep], + install: true, + cpp_args: EXTRA_CXX_FLAGS, + override_options : ['cython_language=cpp'], + subdir: 'hdlConvertor/', +) + +#file(GLOB hdlConvertor_py_SRC +# "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" +#) +#include_directories( +# "${CMAKE_CURRENT_SOURCE_DIR}" +# "${CMAKE_CURRENT_SOURCE_DIR}/../include/" +#) + + + diff --git a/include/hdlConvertor/svConvertor/sv2017Parser/meson.build b/include/hdlConvertor/svConvertor/sv2017Parser/meson.build new file mode 100644 index 00000000..4cc40a30 --- /dev/null +++ b/include/hdlConvertor/svConvertor/sv2017Parser/meson.build @@ -0,0 +1,4 @@ +antrl_parser_headers = custom_target('antrl_parser_sv2017_headers', + input : antlr_parser_parser_src_gen, + output : generated_headers, + command : [cp, ] + generated_headers_abs + [meson.current_build_dir()]) diff --git a/include/hdlConvertor/verilogPreproc/verilogPreprocParser/meson.build b/include/hdlConvertor/verilogPreproc/verilogPreprocParser/meson.build new file mode 100644 index 00000000..f2ec1e72 --- /dev/null +++ b/include/hdlConvertor/verilogPreproc/verilogPreprocParser/meson.build @@ -0,0 +1,4 @@ +antrl_parser_headers = custom_target('antrl_parser_verilogPreproc_headers', + input : antlr_parser_parser_src_gen, + output : generated_headers, + command : [cp, ] + generated_headers_abs + [meson.current_build_dir()]) diff --git a/include/hdlConvertor/vhdlConvertor/vhdlParser/meson.build b/include/hdlConvertor/vhdlConvertor/vhdlParser/meson.build new file mode 100644 index 00000000..f655ca32 --- /dev/null +++ b/include/hdlConvertor/vhdlConvertor/vhdlParser/meson.build @@ -0,0 +1,4 @@ +antrl_parser_headers = custom_target('antrl_parser_vhdl_headers', + input : antlr_parser_parser_src_gen, + output : generated_headers, + command : [cp, ] + generated_headers_abs + [meson.current_build_dir()]) diff --git a/meson.build b/meson.build new file mode 100644 index 00000000..d22acfc5 --- /dev/null +++ b/meson.build @@ -0,0 +1,164 @@ +project( + 'hdlConvertor', + ['cpp', 'cython'], + version: '2.3', + license: 'MIT', + meson_version: '>=1.1.0', + default_options: ['cpp_std=c++17'], +) + + +############################################################################################## +# Lookup all dependencies and prepare common variables +############################################################################################## +# https://meson-python.readthedocs.io/en/latest/reference/config-settings.html +# https://github.com/shenxianpeng/gcov-example +py3_mod = import('python') +py3 = py3_mod.find_installation(required: true) +py3_dep = py3.dependency() +# Java is used to generate source files for parsers, it is required only for installation +cc = meson.get_compiler('cpp') + +EXTRA_CXX_FLAGS = [] +EXTRA_LD_FLAGS = [] +if meson.get_compiler('cpp').get_id() == 'msvc' + EXTRA_CXX_FLAGS.append('/bigobj') + # using /MD flag for antlr4_runtime (for Visual C++ compilers only) +elif host_machine.system() == 'cygwin' + EXTRA_CXX_FLAGS += ['-Wa,-mbig-obj'] +elif host_machine.system() == 'linux' + EXTRA_CXX_FLAGS += ['-Wall', '-pedantic', '-W', '-Wno-attributes'] +endif + +antlr4_cpp_dep = cc.find_library('antlr4-runtime', required : false) +if not antlr4_cpp_dep.found() + message('atempting to download antrl4-complete.jar and antlr4-runtime for C++ because installation was not found') + cmake = import('cmake') + + antlr4_runtime_opt = cmake.subproject_options() + antlr4_runtime_opt.add_cmake_defines({ + 'ANTLR_BUILD_CPP_TESTS': false, + 'ANTLR_BUILD_SHARED': false, + 'ANTLR4_INSTALL': false, + 'CMAKE_CXX_FLAGS': ' '.join(EXTRA_CXX_FLAGS), + 'CMAKE_POSITION_INDEPENDENT_CODE': true, + 'CMAKE_BUILD_TYPE': 'Release', + }) + antlr4_runtime_opt.set_install(false) # because meson-python does not know where to install it + antlr4_runtime_proj = cmake.subproject('antlr4-runtime', options: antlr4_runtime_opt) + antlr4_cpp_dep = antlr4_runtime_proj.dependency('antlr4_static') + antlr4_cpp_version = '4.13.0' + import('java') # to check that java is installed + antlr4_jar = join_paths(meson.current_build_dir() , 'antlr4-complete.jar') + run_command('python3', 'utils/download.py', + f'https://www.antlr.org/download/antlr-@antlr4_cpp_version@-complete.jar', antlr4_jar, + '--hash', 'md5', 'c9c8b566a7fcf8ef80e229f14b154a26', check: true) + antlr4_java = find_program('utils/antlr4-complete.sh') # an extra script so we do not need to write all arguments everytime +else + antlr4_cpp_inc = include_directories('/usr/include/antlr4-runtime/') + antlr4_cpp_dep = cc.find_library('antlr4-runtime', header_include_directories: antlr4_cpp_inc, has_headers: 'antlr4-runtime.h', required : false) + # redeclare because include_dir somehow is not in dependency + antlr4_cpp_dep = declare_dependency('antlr4-runtime', dependencies: [antlr4_cpp_dep], include_directories: antlr4_cpp_inc) + antlr4_java = find_program('antlr4', required : true) +endif + +cp = find_program('cp', required : true) + + +############################################################################################## +# build of antlr4 generated parsers +############################################################################################## + +#name folder_name generate_visitor generate_listener lexer_parser_separate +# * name the name of the parser grammar and parser itself +# * folder_name name of folder where convertor is stored +# (which is using this parser) +ANTLR_PARSERS = [ + ['sv2017', 'svConvertor', false, false], + ['verilogPreproc', 'verilogPreproc', true, false], + ['vhdl', 'vhdlConvertor', false, false], +] +curSrcDir = meson.current_source_dir() +mainIncludeDir = 'include/' +mainIncludeDir_inc = include_directories(mainIncludeDir) +all_parser_libs = [] +foreach parserTuple : ANTLR_PARSERS + name = parserTuple[0] + folder_name = parserTuple[1] + generate_visitor = parserTuple[2] + generate_listener = parserTuple[3] + message(f'Preparing build of antlr4 parser @name@') + antlr_gen_outputs = [ + f'@name@Lexer.cpp', + f'@name@Lexer.h', + f'@name@Parser.cpp', + f'@name@Parser.h', + ] + + # explicitely named all header files which are generated by ANTLR + + antlr_gen_args = ['-Dlanguage=Cpp',] + if generate_visitor + antlr_gen_args += '-visitor' + antlr_gen_outputs += [ + f'@name@ParserBaseVisitor.h', + f'@name@ParserVisitor.h', + ] + else + antlr_gen_args += '-no-visitor' + endif + if generate_listener + antlr_gen_args += '-listener' + antlr_gen_outputs += [ + f'@name@ParserBaseListener.h', + f'@name@ParserListener.h', + ] + else + antlr_gen_args += '-no-listener' + endif + + # create source directory + # create include directory + # run antlr to generate parser + # mv headers to include directory + # [todo] does not supports spaces in filenames + GRAMMAR_FILES = [join_paths(curSrcDir, 'grammars', f'@name@Lexer.g4'), + join_paths(curSrcDir, 'grammars', f'@name@Parser.g4')] + antlr_parser_parser_src_gen = custom_target(f'@name@_src_gen', + input : GRAMMAR_FILES, + output : antlr_gen_outputs, + command : [antlr4_java, ] + antlr_gen_args + + ['-package', f'@name@_antlr', '-encoding', 'utf-8', '-o', meson.current_build_dir()] + GRAMMAR_FILES) + + # run copy of header files + generated_headers_abs = [] + generated_headers = [] + foreach f: antlr_gen_outputs + if f.endswith('.h') + generated_headers_abs += join_paths(meson.current_build_dir(), f) + generated_headers += f + endif + endforeach + genIncludeDir = join_paths('include', 'hdlConvertor', folder_name, f'@name@Parser') + subdir(genIncludeDir) + parser_cpp_src = run_command(py3, 'utils/rec_glob.py', join_paths('src', folder_name), '*.cpp', check:true)\ + .stdout().strip().split('\n') + + parser_lib = static_library(name, + parser_cpp_src + antlr_parser_parser_src_gen + antrl_parser_headers, + cpp_args: ['-Wno-unused-parameter'] + EXTRA_CXX_FLAGS, + dependencies: [antlr4_cpp_dep], + include_directories: [mainIncludeDir_inc, include_directories(genIncludeDir)], + ) + all_parser_libs += parser_lib +endforeach + +subdir('src') + +if get_option('python_package') + subdir('hdlConvertor') +endif + + + + diff --git a/meson.options b/meson.options new file mode 100644 index 00000000..9833b3df --- /dev/null +++ b/meson.options @@ -0,0 +1,2 @@ +# use meson configure -Doption=newvalue to set +option('python_package', type : 'boolean', value : true, description : 'Specifies if python package should be build or just C++ one.') diff --git a/pyproject.toml b/pyproject.toml index cc13dff9..f6937d19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,46 @@ [build-system] -requires = ["setuptools", "wheel", "scikit-build", "Cython", "cmake", "ninja"] -build-backend = "setuptools.build_meta" +requires = [ + "wheel", # python package distribution lib. + "Cython", + "cmake>=3.26.3", # cmake buildsystem for antrl + "meson>=1.1.0", # meson buildsystem + 'meson-python>=0.12.1', # meson to pip adapter + "pybind11>=2.10.4", # c++ to Python binding + "ninja>=1.11.1", # build tool for meson to use +] +build-backend = 'mesonpy' + + +[project] +name = 'hdlConvertor' +description = 'VHDL and System Verilog parser written in c++' +dynamic = ["version", ] # specified in meson.build +# dynamic = ["readme"] +# readme = {file = ["README.md"]} +readme = "README.md" +license = { file = 'LICENSE' } +keyword = ['FPGA', 'ASIC', 'compiler', 'RTL', 'synthesis', 'logicsynthesis', + 'hdl', 'vhdl', 'parser', 'preprocessor', 'antlr4', 'verification', + 'Verilog', 'SystemVerilog', 'system verilog'] +authors = [ + { name = 'Michal Orsak', email = 'Nic30original@gmail.com' }, +] +classifiers = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Operating System :: OS Independent', + 'Topic :: Software Development :: Build Tools', + 'Programming Language :: C++', + 'Programming Language :: Cython', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: 3', + 'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', +] +dependencies = [ + 'hdlConvertorAst>=1.0', +] + +[project.urls] +homepage = 'https://github.com/Nic30/hdlConvertor' +repository = 'https://github.com/Nic30/hdlConvertor' diff --git a/setup.py b/setup.py deleted file mode 100755 index 20b0cdc5..00000000 --- a/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- - -import os -from setuptools import find_packages -try: - from skbuild import setup -except ImportError: - raise ImportError("Missing scikit-build, (should be automatically installed by pip)") -import sys - - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: - long_description = f.read() - -deps = ["typing", "future"] if sys.version_info[0] == 2 else [] - - -setup( - cmake_args=[ - # '-DCMAKE_BUILD_TYPE=Debug' - ], - name='hdlConvertor', - version='2.3', - description='VHDL and System Verilog parser written in c++', - long_description=long_description, - long_description_content_type="text/markdown", - url='https://github.com/Nic30/hdlConvertor', - author='Michal Orsak', - author_email='Nic30original@gmail.com', - keywords=['hdl', 'vhdl', 'verilog', 'system verilog', - 'parser', 'preprocessor', 'antlr4'], - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Operating System :: OS Independent', - 'Topic :: Software Development :: Build Tools', - 'Programming Language :: C++', - 'Programming Language :: Cython', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', - ], - install_requires=[ - 'hdlConvertorAst>=1.0', - ] + deps, - license="MIT", - packages=find_packages(exclude=["tests", ]), - test_suite="tests.main_test_suite", - test_runner="tests:TimeLoggingTestRunner", - tests_require=deps, -) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index 1a0db05b..00000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,124 +0,0 @@ -############################################################################################## -# build of the main library -############################################################################################## - -include(${CMAKE_CURRENT_SOURCE_DIR}/CMake_antlr4.txt) - -include_directories( - "${CMAKE_CURRENT_SOURCE_DIR}/../include/") - -add_subdirectory(svConvertor) -add_subdirectory(verilogPreproc) -add_subdirectory(vhdlConvertor) - -file(GLOB hdlConvertor_core_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/hdlAst/*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/baseHdlParser/*.cpp" -) - -list(APPEND hdlConvertor_core_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/notImplementedLogger.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/syntaxErrorLogger.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/conversion_exception.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/universal_fs.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/encodingConversions.cpp" -) -set(hdlConvertor_cpp_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/hdlConvertor.cpp" -) -add_library(hdlConvertor_core_static STATIC ${hdlConvertor_core_SRC}) -target_include_directories(hdlConvertor_core_static - INTERFACE - "${CMAKE_CURRENT_SOURCE_DIR}/../include/" -) -set_target_properties(hdlConvertor_core_static - PROPERTIES VERSION "1.0.0" - OUTPUT_NAME hdlConvertor_core - POSITION_INDEPENDENT_CODE ON -) -set_coverage_if_enabled(hdlConvertor_core_static ON) - -set(HDL_CONVERTOR_INTERNAL_LIBS - svConvertor_static - verilogPreproc_static - vhdlConvertor_static - hdlConvertor_core_static -) - -if(HDLCONVERTOR_PYTHON) - # [note] hdlConvertor_core_static and hdlConvertor_cpp_static has to be separated - # otherwise it is not possible to link this library - add_library(hdlConvertor_cpp_static STATIC ${hdlConvertor_cpp_SRC}) - set_coverage_if_enabled(hdlConvertor_cpp_static ON) - - target_link_libraries(hdlConvertor_cpp_static PUBLIC - ${HDL_CONVERTOR_INTERNAL_LIBS} - ${ANTLR4CPP_LIBRARIES} - ) - target_compile_features(hdlConvertor_cpp_static PUBLIC cxx_std_17) - - set_target_properties(hdlConvertor_cpp_static - PROPERTIES VERSION "1.0.0" - OUTPUT_NAME "hdlConvertor_cpp" - POSITION_INDEPENDENT_CODE ON - ) -else() - # add dummy file to make cmake happy - add_library(hdlConvertor_cpp_shared SHARED ${hdlConvertor_cpp_SRC}) - set_target_properties(hdlConvertor_cpp_shared - PROPERTIES VERSION "1.0.0" - OUTPUT_NAME "hdlConvertor" - # POSITION_INDEPENDENT_CODE ON - ) - target_link_libraries(hdlConvertor_cpp_shared - PRIVATE - "-Wl,--whole-archive" - ${HDL_CONVERTOR_INTERNAL_LIBS} - "-Wl,--no-whole-archive" - ) - target_link_libraries(hdlConvertor_cpp_shared - INTERFACE - ${ANTLR4CPP_LIBRARIES} - ) - target_compile_features(hdlConvertor_cpp_shared PUBLIC cxx_std_17) - set_coverage_if_enabled(hdlConvertor_cpp_shared ON) - if(CODE_COVERAGE) - target_link_libraries(hdlConvertor_cpp_shared PRIVATE gcov) - endif() - install(TARGETS hdlConvertor_cpp_shared - EXPORT hdlConvertorConfig - LIBRARY DESTINATION lib - # ARCHIVE DESTINATION lib - # FILES DESTINATION include ${CMAKE_INSTALL_INCLUDEDIR} - # PUBLIC_HEADER DESTINATION include - ) - install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../include/" - DESTINATION include ${CMAKE_INSTALL_INCLUDEDIR}) - # This makes the project importable from the install directory - # Put config file in per-project dir (name MUST match), can also - # just go into 'cmake'. - install(EXPORT hdlConvertorConfig DESTINATION share/hdlConvertor/cmake) - - # This makes the project importable from the build directory - export(TARGETS hdlConvertor_cpp_shared FILE hdlConvertorConfig.cmake) - set(MAIN_HDLCONVERTOR_LIB hdlConvertor_cpp_shared) - if(CODE_COVERAGE) - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # Add required flags (GCC & LLVM/Clang) - target_compile_options(${MAIN_HDLCONVERTOR_LIB} PUBLIC - -O0 # no optimization - -g # generate debug info - --coverage # sets all required flags - ) - target_link_libraries(${MAIN_HDLCONVERTOR_LIB} gcov) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) - target_link_options(${MAIN_HDLCONVERTOR_LIB} INTERFACE --coverage) - else() - target_link_libraries(${MAIN_HDLCONVERTOR_LIB} INTERFACE --coverage) - endif() - else() - message(ERROR "Code coverage supported only for gcc/clang compiler (current ${CMAKE_CXX_COMPILER_ID})") - endif() - endif() -endif() - diff --git a/src/CMake_antlr4.txt b/src/CMake_antlr4.txt deleted file mode 100644 index 045c1d5c..00000000 --- a/src/CMake_antlr4.txt +++ /dev/null @@ -1,191 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/CMake_coverage.txt) - -############################################################################################## -# antlr4 settings -############################################################################################## -# there are two types of distribution the all in one antlr4-complete.jar or group of antrl4 jars -find_jar(_ANTLR_JAR_LOCATION_antlr4_complete "antlr4-complete" - PATHS "/usr/share/java/antlr4-complete.jar" - "antlr4-complete.jar" ENV ANTLR_COMPLETE_PATH) -if(_ANTLR_JAR_LOCATION_antlr4_complete) - MESSAGE(STATUS "using antlr4-complete.jar found in ${_ANTLR_JAR_LOCATION_antlr4_complete}") - set(ANTLR_CLASSPATH "${_ANTLR_JAR_LOCATION_antlr4_complete}") -else() - set(ANTLR_CLASSPATH "") - set(ANTLR_JARS "stringtemplate4;antlr4;antlr4-runtime;antlr3-runtime;treelayout") - FOREACH(antlr_jar ${ANTLR_JARS}) - find_jar(_ANTLR_JAR_LOCATION_${antlr_jar} "${antlr_jar}" - PATHS "/usr/share/java/${antlr_jar}.jar") - set(_ANTLR_JAR_LOCATION ${_ANTLR_JAR_LOCATION_${antlr_jar}}) - if (NOT _ANTLR_JAR_LOCATION) - MESSAGE(FATAL_ERROR "ANTLR: Can not find ${antlr_jar} jar from ANTLR installation") - else() - MESSAGE(STATUS "ANTLR: using ${antlr_jar} jar found in ${_ANTLR_JAR_LOCATION}") - endif() - if(ANTLR_CLASSPATH) - set(ANTLR_CLASSPATH "${ANTLR_CLASSPATH}:${_ANTLR_JAR_LOCATION}") - else() - set(ANTLR_CLASSPATH "${_ANTLR_JAR_LOCATION}") - endif() - ENDFOREACH() -endif() - -# search for antlr4 include dir and library -if(ANTLR4CPP_ROOT) - MESSAGE(STATUS "Antlr4 cpp runtime root set to: ${ANTLR4CPP_ROOT}") -endif() -find_path(ANTLR4CPP_INCLUDE_DIRS antlr4-runtime.h - HINTS "${ANTLR4CPP_ROOT}/usr/include/antlr4-runtime/" - "${ANTLR4CPP_ROOT}/include/antlr4-runtime/" - PATH_SUFFIXES antlr4-runtime) -if(NOT ANTLR4CPP_INCLUDE_DIRS) - MESSAGE(FATAL_ERROR "Can not find ANTLR4CPP_INCLUDE_DIRS") -else() - MESSAGE(STATUS "ANTLR4CPP_INCLUDE_DIRS=${ANTLR4CPP_INCLUDE_DIRS}") -endif() -include_directories(${ANTLR4CPP_INCLUDE_DIRS}) - -# [todo] rather explicitly specify static/dynamic antlr linking -find_library(ANTLR4CPP_LIBRARIES - libantlr4-runtime.dll libantlr4-runtime.so - libantlr4-runtime.a libantlr4-runtime.dylib - antlr4-runtime-static.lib - HINTS "${ANTLR4CPP_ROOT}/usr/lib" - "${ANTLR4CPP_ROOT}/lib") - -if(NOT ANTLR4CPP_LIBRARIES) - MESSAGE(FATAL_ERROR "Can not find ANTLR4CPP_LIBRARIES") -else() - MESSAGE(STATUS "ANTLR4CPP_LIBRARIES=${ANTLR4CPP_LIBRARIES}") -endif() - -if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - add_compile_definitions(ANTLR4CPP_STATIC) -endif() - -############################################################################################## -# antlr4 utils -############################################################################################## - -set(PARSER_GEN_ONLY_VISITOR "-visitor" "-no-listener") -set(PARSER_GEN_ONLY_LISTENER "-listener" "-no-visitor") -set(PARSER_GEN_VISITOR "-visitor") - -# @attention this project dependent constants -set(ANTLR_GRAMMAR_ROOT "${PROJECT_SOURCE_DIR}/grammars") -set(PROJECT_INCLUDE_ROOT "${PROJECT_SOURCE_DIR}/include/hdlConvertor") - - -# https://samthursfield.wordpress.com/2015/11/21/cmake-dependencies-between-targets-and-files-and-custom-commands/ -#https://stackoverflow.com/questions/7862300/how-to-make-cmake-target-executed-whether-specified-file-was-changed -macro(AddParserToBuild name folder_name - generate_visitor generate_listener lexer_parser_separate) - # @param name the name of the parser grammar and parser itself - # @param folder_name name of folder where convertor is stored - # (which is using this parser) - # @param lexer_parser_separate if ON the ${name}Lexer.g4 ${name}Parser.g4 is used otherwise just ${name}.g4 - set(GENERATED_SRC - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}Lexer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}Parser.cpp - ) - if(CMAKE_SYSTEM_NAME MATCHES "Linux") - #Suppress warnings on generated code. - set_property(SOURCE ${GENERATED_SRC} PROPERTY - COMPILE_FLAGS -Wno-unused-parameter) - endif() - - # @attention PROJECT_INCLUDE_ROOT depends on project name and has to be updated for other projects - set(GEN_INC_DIR - ${PROJECT_INCLUDE_ROOT}/${folder_name}/${name}Parser) - set(GENERATED_INC - ${GEN_INC_DIR}/${name}Lexer.h - ${GEN_INC_DIR}/${name}Parser.h - ) - - # explicitely named all header files which are generated by ANTLR - set(ORIG_GENERATED_INC - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}Lexer.h - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}Parser.h - ) - - set(ANTLR_GEN_ARGS) - if (${generate_visitor}) - list(APPEND ANTLR_GEN_ARGS "-visitor") - list(APPEND ORIG_GENERATED_INC - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}ParserBaseVisitor.h - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}ParserVisitor.h - ) - else() - list(APPEND ANTLR_GEN_ARGS "-no-visitor") - endif() - if (${generate_listener}) - list(APPEND ANTLR_GEN_ARGS "-listener") - list(APPEND ORIG_GENERATED_INC - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}ParserBaseListener.h - ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser/${name}ParserListener.h - ) - else() - list(APPEND ANTLR_GEN_ARGS "-no-listener") - endif() - - set(PARSER_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser) - - # delete generated files on clean - set_property(DIRECTORY PROPERTY ADDITIONAL_MAKE_CLEAN_FILES - "${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser" - "${GEN_INC_DIR}") - - if(${lexer_parser_separate}) - set(GRAMMARS - ${ANTLR_GRAMMAR_ROOT}/${name}Lexer.g4 - ${ANTLR_GRAMMAR_ROOT}/${name}Parser.g4) - else() - set(GRAMMARS - ${ANTLR_GRAMMAR_ROOT}/${name}.g4) - endif() - - # create source directory - # create include directory - # run antlr to generate parser - # mv headers to include directory - # [todo] does not supports spaces in filenames - add_custom_command( - OUTPUT - ${GENERATED_INC} ${GENERATED_SRC} - COMMAND - ${CMAKE_COMMAND} -E make_directory ${PARSER_SRC_DIR} - COMMAND - ${CMAKE_COMMAND} -E make_directory ${GEN_INC_DIR} - COMMAND - "${Java_JAVA_EXECUTABLE}" -cp "${ANTLR_CLASSPATH}" "org.antlr.v4.Tool" - -Dlanguage=Cpp ${ANTLR_GEN_ARGS} -package ${name}_antlr -encoding utf-8 - -o ${CMAKE_CURRENT_SOURCE_DIR}/${name}Parser - ${GRAMMARS} - COMMAND - ${CMAKE_COMMAND} -E copy ${ORIG_GENERATED_INC} ${GEN_INC_DIR} - DEPENDS ${GRAMMARS} - COMMENT "generating sources for ${name}Parser" - ) - - file(GLOB SRC - "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" - ) - - add_library(${folder_name}_static STATIC - ${SRC} - ${GENERATED_SRC} - ${GENERATED_INC}) - set_coverage_if_enabled(${folder_name}_static ON) - - if(NOT HDLCONVERTOR_PYTHON) - # we do not want to install headers if we are building python module - set_target_properties(${folder_name}_static - PROPERTIES PUBLIC_HEADER "${GENERATED_INC}") - endif() - - set_target_properties(${folder_name}_static - PROPERTIES VERSION "1.0.0" - OUTPUT_NAME ${folder_name} - POSITION_INDEPENDENT_CODE ON - ) -endmacro() diff --git a/src/CMake_coverage.txt b/src/CMake_coverage.txt deleted file mode 100644 index ee2cbecf..00000000 --- a/src/CMake_coverage.txt +++ /dev/null @@ -1,23 +0,0 @@ - -function(set_coverage_if_enabled lib_name specify_link_interface) - if(CODE_COVERAGE) - message(STATUS "set_coverage_if_enabled: ${lib_name}") - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # Add required flags (GCC & LLVM/Clang) - target_compile_options(${lib_name} PUBLIC - -O0 # no optimization - -g # generate debug info - --coverage # sets all required flags - ) - if(specify_link_interface) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) - target_link_options(${lib_name} INTERFACE --coverage) - else() - target_link_libraries(${lib_name} INTERFACE --coverage) - endif() - endif() - else() - message(ERROR "Code coverage supported only for gcc/clang compiler (current ${CMAKE_CXX_COMPILER_ID})") - endif() - endif() -endfunction() \ No newline at end of file diff --git a/src/meson.build b/src/meson.build new file mode 100644 index 00000000..a139be7b --- /dev/null +++ b/src/meson.build @@ -0,0 +1,53 @@ + +############################################################################################## +# build of main library +############################################################################################## +hdlConvertor_core_src = [ + 'notImplementedLogger.cpp', + 'syntaxErrorLogger.cpp', + 'conversion_exception.cpp', + 'universal_fs.cpp', + 'encodingConversions.cpp', +] +hdlConvertor_core_src += run_command(py3, '../utils/rec_glob.py', './hdlAst', '*.cpp', check:true)\ + .stdout().strip().split('\n') +hdlConvertor_core_src += run_command(py3, '../utils/rec_glob.py', './baseHdlParser', '*.cpp', check:true)\ + .stdout().strip().split('\n') + +hdlConvertor_core_static_lib = static_library('hdlConvertor_core_static', + hdlConvertor_core_src, + cpp_args: EXTRA_CXX_FLAGS, + dependencies: [antlr4_cpp_dep], + include_directories: [mainIncludeDir_inc], + #link_with: [libtracker_common], +) +# my_inc = include_directories(...) +# my_lib = static_library(...) +# my_dep = declare_dependency(link_with : my_lib, include_directories : my_inc) + +hdlConvertor_cpp_src = [ + 'hdlConvertor.cpp' +] +if get_option('python_package') + # [note] hdlConvertor_core_static and hdlConvertor_cpp_static has to be separated + # otherwise it is not possible to link this library + hdlConvertor_cpp_static = static_library('hdlConvertor_cpp_static', + hdlConvertor_cpp_src, + cpp_args: EXTRA_CXX_FLAGS, + dependencies: [antlr4_cpp_dep], + include_directories: [mainIncludeDir_inc], + link_with: [hdlConvertor_core_static_lib, all_parser_libs], + ) + hdlConvertor_cpp_dep = declare_dependency(link_with : hdlConvertor_cpp_static) +else + # add dummy file to make cmake happy + hdlConvertor_cpp_shared = shared_library('hdlConvertor_cpp_shared', + hdlConvertor_cpp_src, + cpp_args: EXTRA_CXX_FLAGS, + dependencies: [antlr4_cpp_dep], + include_directories: [mainIncludeDir_inc], + link_with: [hdlConvertor_core_static_lib, all_parser_libs], + ) + hdlConvertor_cpp_dep = declare_dependency(link_with : hdlConvertor_cpp_shared) +endif + diff --git a/src/svConvertor/CMakeLists.txt b/src/svConvertor/CMakeLists.txt deleted file mode 100644 index 40d52f7e..00000000 --- a/src/svConvertor/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -AddParserToBuild("sv2017" "svConvertor" OFF OFF ON) \ No newline at end of file diff --git a/src/verilogPreproc/CMakeLists.txt b/src/verilogPreproc/CMakeLists.txt deleted file mode 100644 index 174a01c2..00000000 --- a/src/verilogPreproc/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -AddParserToBuild("verilogPreproc" "verilogPreproc" ON OFF ON) diff --git a/src/vhdlConvertor/CMakeLists.txt b/src/vhdlConvertor/CMakeLists.txt deleted file mode 100644 index acb15764..00000000 --- a/src/vhdlConvertor/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -AddParserToBuild("vhdl" "vhdlConvertor" OFF OFF ON) diff --git a/subprojects/antlr4-runtime.wrap b/subprojects/antlr4-runtime.wrap new file mode 100644 index 00000000..6c787fc5 --- /dev/null +++ b/subprojects/antlr4-runtime.wrap @@ -0,0 +1,5 @@ +[wrap-git] +url = https://github.com/antlr/antlr4.git +revision = 4.13.0 +diff_files=0001-add-CMakeLists.txt-to-root.patch +depth=1 \ No newline at end of file diff --git a/subprojects/packagefiles/0001-add-CMakeLists.txt-to-root.patch b/subprojects/packagefiles/0001-add-CMakeLists.txt-to-root.patch new file mode 100644 index 00000000..0a210a62 --- /dev/null +++ b/subprojects/packagefiles/0001-add-CMakeLists.txt-to-root.patch @@ -0,0 +1,23 @@ +From 9abfddeb99e8293b0279f0b66c03e05caf808960 Mon Sep 17 00:00:00 2001 +From: Nic30 +Date: Mon, 29 May 2023 12:14:29 +0200 +Subject: [PATCH] add CMakeLists.txt to root + +--- + CMakeLists.txt | 4 ++++ + 1 file changed, 4 insertions(+) + create mode 100644 CMakeLists.txt + +diff --git a/CMakeLists.txt b/CMakeLists.txt +new file mode 100644 +index 000000000..608f61056 +--- /dev/null ++++ b/CMakeLists.txt +@@ -0,0 +1,4 @@ ++# -*- mode:cmake -*- ++cmake_minimum_required (VERSION 3.15) ++project(antlr4runtime) ++add_subdirectory(runtime/Cpp) +-- +2.39.2 + diff --git a/utils/antlr4-complete.sh b/utils/antlr4-complete.sh new file mode 100755 index 00000000..f6bf3057 --- /dev/null +++ b/utils/antlr4-complete.sh @@ -0,0 +1,5 @@ +#! /bin/sh + +# :note: this script should be executed only in directory where antlr4-complete.jar is +CLASSPATH=antlr4-complete.jar +exec java -cp $CLASSPATH org.antlr.v4.Tool "$@" \ No newline at end of file diff --git a/utils/download.py b/utils/download.py new file mode 100644 index 00000000..d6e927b1 --- /dev/null +++ b/utils/download.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +We use SystemExit as this will not blast the whole traceback to Meson. +Usually just a terse stderr will suffice and not overwhelm the Meson user. + +https://www.scivision.dev/meson-file-download/ +""" + +from pathlib import Path +import urllib.request +import urllib.error +import hashlib +import argparse +import socket + + +def url_retrieve( + url: str, + outfile: Path, + filehash: tuple[str, str]=None, + overwrite: bool=False, +): + """ + Parameters + ---------- + url: str + URL to download from + outfile: pathlib.Path + output filepath (including name) + filehash: tuple of str, str + hash type (md5, sha1, etc.) and hash + overwrite: bool + overwrite if file exists + """ + outfile = Path(outfile).expanduser().resolve() + if outfile.is_dir(): + raise ValueError("Please specify full filepath, including filename") + # need .resolve() in case intermediate relative dir doesn't exist + if overwrite or not outfile.is_file(): + outfile.parent.mkdir(parents=True, exist_ok=True) + try: + urllib.request.urlretrieve(url, str(outfile)) + except (socket.gaierror, urllib.error.URLError) as err: + raise SystemExit( + "ConnectionError: could not download {} due to {}".format(url, err) + ) + + if filehash: + hashMode, expectedHash = filehash + h = hashlib.new(hashMode) + h.update(outfile.read_bytes()) + hashRes = h.hexdigest() + if hashRes != expectedHash: + raise SystemExit("HashError: {} got: {}, expected: {}".format(outfile, hashRes, expectedHash)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("url", help="URL to file download") + p.add_argument("outfile", help="filename to download to") + p.add_argument("--hash", help="expected hash", nargs=2) + P = p.parse_args() + + url_retrieve(P.url, P.outfile, P.hash) diff --git a/utils/rec_glob.py b/utils/rec_glob.py new file mode 100644 index 00000000..5e15f65c --- /dev/null +++ b/utils/rec_glob.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pathlib import Path +import sys + + +def call_glob(): + if len(sys.argv) != 3: + print("[Error] Usage `glob.py path 'module'|pattern`, but provided `%r`" % sys.argv, file=sys.stderr) + sys.exit(1) + + root, mode = sys.argv[1:] + if mode == 'module': + # list all modules (the directories wich do contain .py files) + seen_module_directories = set() + for path in Path(root).rglob("*.py"): + path: Path + d = path.parent.as_posix() + if d in seen_module_directories: + continue + else: + print(d) + seen_module_directories.add(d) + + else: + # list files by extension in a folder + for path in Path(root).glob(mode): + print(path.as_posix()) + + +if __name__ == "__main__": + call_glob()