diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4fc1f6e7..ffd807c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,125 +52,21 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade --upgrade-strategy eager pip - pip install flake8 - pip install coverage pip install -r requirements.txt pip install -r docs/requirements.txt - - # Run flake8 only for Python 3.9 - - name: Lint with flake8 - if: matrix.python-version == '3.9' - run: | - pip install flake8 - flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - - # Run unittest with coverage for Python 3.9 - - name: Test with unittest and coverage (v3.9) + # Run unittest without coverage + - name: Test with unittest env: HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: matrix.python-version == '3.9' - run: | - pip install coverage - coverage run -m unittest discover tests - continue-on-error: true - - - - # Run unittest without coverage for non Python 3.9 - - name: Test with unittest (non-3.9) - env: - HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: matrix.python-version != '3.9' run: | python -m unittest discover tests continue-on-error: true - - - # Run spec tests with coverage for Python 3.9 - - name: Run spec_test coverage - env: - HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: matrix.python-version == '3.9' - run: | - coverage run --append -m unittest discover tests/spec_tests - ls -a - continue-on-error: true - - # Run spec tests without coverage for non Python 3.9 - name: Run spec_test env: HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: matrix.python-version != '3.9' run: | python -m unittest discover tests/spec_tests continue-on-error: true - - # Archive code coverage results for Python 3.9 - - name: Archive code coverage results for Python 3.9 - if: ${{matrix.python-version == '3.9'}} - uses: actions/upload-artifact@v4 - with: - name: code-coverage-report - path: .coverage - if-no-files-found: error - - check-secret: - runs-on: ubuntu-latest - outputs: - secrets-exist: ${{ steps.check-for-secrets.outputs.defined }} - steps: - - name: Check for Secret availability - id: check-for-secrets - # perform secret check & put boolean result as an output - shell: bash - run: | - if [ "${{ secrets.CC_TEST_REPORTER_ID }}" != '' ]; then - echo "defined=true" >> $GITHUB_OUTPUT; - else - echo "defined=false" >> $GITHUB_OUTPUT; - fi - - coverage: - name: Publish coverage - needs: [build, check-secret] - runs-on: ubuntu-latest - if: needs.check-secret.outputs.secrets-exist == 'true' - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }} - - - name: Install dependencies - run: | - python -m pip install --upgrade --upgrade-strategy eager pip - pip install flake8 - pip install coverage - pip install -r requirements.txt - pip install -r docs/requirements.txt - - - name: Download a single artifact - uses: actions/download-artifact@v4 - with: - name: code-coverage-report - - - name: publish-coverages - with: - coverageCommand: coverage xml - debug: true - uses: paambaati/codeclimate-action@v9.0.0 - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - diff --git a/.github/workflows/ci_cov.yaml b/.github/workflows/ci_cov.yaml new file mode 100644 index 00000000..e0d2a937 --- /dev/null +++ b/.github/workflows/ci_cov.yaml @@ -0,0 +1,89 @@ +name: CI + +on: + push: + branches: ["*"] + pull_request: + branches: ["*"] + +jobs: + + check-secret: + runs-on: ubuntu-latest + outputs: + secrets-exist: ${{ steps.check-for-secrets.outputs.defined }} + steps: + - name: Check for Secret availability + id: check-for-secrets + # perform secret check & put boolean result as an output + shell: bash + run: | + if [ "${{ secrets.CC_TEST_REPORTER_ID }}" != '' ]; then + echo "defined=true" >> $GITHUB_OUTPUT; + else + echo "defined=false" >> $GITHUB_OUTPUT; + fi + + build: + needs: check-secret + strategy: + matrix: + platform: [ubuntu-latest] + python-version: [ "3.9" ] + + runs-on: ${{ matrix.platform }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }} + + # Install dependencies + - name: Install dependencies + run: | + python -m pip install --upgrade --upgrade-strategy eager pip + pip install flake8 + pip install coverage + pip install -r requirements.txt + pip install -r docs/requirements.txt + + # Run flake8 only for Python 3.9 + - name: Lint with flake8 + run: | + pip install flake8 + flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + # Run unittest with coverage + - name: Test with unittest and coverage + env: + HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coverage run -m unittest discover tests + continue-on-error: true + + # Run spec tests with coverage + - name: Run spec_test coverage + env: + HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coverage run --append -m unittest discover tests/spec_tests + continue-on-error: true + + - name: publish-coverages + with: + coverageCommand: coverage xml + debug: true + uses: paambaati/codeclimate-action@v9.0.0 + env: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + diff --git a/hed/errors/error_messages.py b/hed/errors/error_messages.py index 5a801293..a84e9304 100644 --- a/hed/errors/error_messages.py +++ b/hed/errors/error_messages.py @@ -51,16 +51,19 @@ def val_error_element_deprecatedr(tag): def val_error_invalid_tag_character(tag, problem_tag): return f"Invalid character '{problem_tag}' in tag '{tag}'" + @hed_tag_error(ValidationErrors.INVALID_VALUE_CLASS_CHARACTER, has_sub_tag=True, actual_code=ValidationErrors.CHARACTER_INVALID) def val_error_INVALID_VALUE_CLASS_CHARACTER(tag, problem_tag, value_class): return f"Invalid character '{problem_tag}' in tag '{tag}' for value class '{value_class}'" + @hed_tag_error(ValidationErrors.INVALID_VALUE_CLASS_VALUE, has_sub_tag=True, actual_code=ValidationErrors.VALUE_INVALID) def val_error_INVALID_VALUE_CLASS_VALUE(tag, problem_tag, value_class): return f"'{tag}' has an invalid value portion for value class '{value_class}'" + @hed_error(ValidationErrors.TILDES_UNSUPPORTED) def val_error_tildes_not_supported(source_string, char_index): character = source_string[char_index] diff --git a/hed/errors/schema_error_messages.py b/hed/errors/schema_error_messages.py index d9446602..004601ae 100644 --- a/hed/errors/schema_error_messages.py +++ b/hed/errors/schema_error_messages.py @@ -33,19 +33,22 @@ def schema_error_SCHEMA_INVALID_CHILD(tag, child_tag_list): @hed_error(SchemaAttributeErrors.SCHEMA_ATTRIBUTE_INVALID) def schema_error_unknown_attribute(attribute_name, source_tag): - return f"Attribute '{attribute_name}' used by '{source_tag}' was not defined in the schema, or was used outside of it's defined class." + return f"Attribute '{attribute_name}' used by '{source_tag}' " + f"was not defined in the schema, or was used outside of it's defined class." @hed_error(SchemaWarnings.SCHEMA_PRERELEASE_VERSION_USED, default_severity=ErrorSeverity.WARNING) def schema_error_SCHEMA_PRERELEASE_VERSION_USED(current_version, known_versions): - return f"Schema version {current_version} used, which is prerelease or unofficial. Known versions are: {', '.join(known_versions)}" + return f"Schema version {current_version} used, which is prerelease or unofficial. " + f"Known versions are: {', '.join(known_versions)}" @hed_error(SchemaWarnings.SCHEMA_PROLOGUE_CHARACTER_INVALID, default_severity=ErrorSeverity.WARNING, actual_code=SchemaWarnings.SCHEMA_CHARACTER_INVALID) def schema_error_invalid_character_prologue(char_index, source_string, section_name): invalid_char = source_string[char_index] - return f"'{section_name}' has invalid character '{invalid_char}' at position {char_index} of string: {source_string}" + return f"'{section_name}' has invalid character '{invalid_char}' at " + f"position {char_index} of string: {source_string}" @hed_error(SchemaWarnings.SCHEMA_INVALID_CHARACTERS_IN_DESC, default_severity=ErrorSeverity.WARNING, diff --git a/hed/models/definition_dict.py b/hed/models/definition_dict.py index a9cb1207..a013d444 100644 --- a/hed/models/definition_dict.py +++ b/hed/models/definition_dict.py @@ -209,7 +209,7 @@ def _find_group(self, definition_tag, group, error_handler): # initial validation groups = group.groups() issues = [] - tags = group.tags() + # tags = group.tags() # if len(tags) != 1: # issues += \ # ErrorHandler.format_error_with_context(error_handler, diff --git a/hed/schema/hed_schema_df_constants.py b/hed/schema/hed_schema_df_constants.py index d48c81ca..cdee9429 100644 --- a/hed/schema/hed_schema_df_constants.py +++ b/hed/schema/hed_schema_df_constants.py @@ -26,7 +26,7 @@ DF_EXTRA_SUFFIXES = {PREFIXES_KEY, EXTERNAL_ANNOTATION_KEY} -DF_SUFFIXES_OMN = { *DF_SUFFIXES, *DF_EXTRA_SUFFIXES} +DF_SUFFIXES_OMN = {*DF_SUFFIXES, *DF_EXTRA_SUFFIXES} section_mapping_hed_id = { diff --git a/hed/schema/schema_io/df2schema.py b/hed/schema/schema_io/df2schema.py index 3d44abcc..db43ec6f 100644 --- a/hed/schema/schema_io/df2schema.py +++ b/hed/schema/schema_io/df2schema.py @@ -104,7 +104,7 @@ def _read_schema(self, dataframe): dataframe (pd.DataFrame): The dataframe for the main tags section """ self._schema._initialize_attributes(HedSectionKey.Tags) - known_parent_tags = {"HedTag": []} + known_parent_tags = {"HedTag": []} iterations = 0 # Handle this over multiple iterations in case tags have parent tags listed later in the file. # A properly formatted .tsv file will never have parents after the child. @@ -179,7 +179,7 @@ def _create_tag_entry(self, parent_tags, row_number, row): long_tag_name = tag_name return self._create_entry(row_number, row, HedSectionKey.Tags, long_tag_name) - self._add_fatal_error(row_number, row, f"No tag name found in row.", + self._add_fatal_error(row_number, row, "No tag name found in row.", error_code=HedExceptions.GENERIC_ERROR) def _read_section(self, df, section_key): @@ -276,5 +276,3 @@ def load_dataframes_from_strings(schema_data): return {key: value if isinstance(value, pd.DataFrame) else pd.read_csv(io.StringIO(value), sep="\t", dtype=str, na_filter=False) for key, value in schema_data.items()} - -