From 727f89025e7b8d573938282bcb7306696252fc44 Mon Sep 17 00:00:00 2001 From: Panagiotis Paralakis Date: Mon, 4 Mar 2024 10:59:28 +0000 Subject: [PATCH 1/6] fix all files using pre-commit --- .nvmrc | 1 + Dockerfile | 4 +- Makefile | 44 +++++------ checks/tests/test_tasks.py | 2 +- commodities/import_handlers.py | 4 +- commodities/models/dc.py | 9 ++- commodities/tests/conftest.py | 12 ++- common/business_rules.py | 21 ++--- common/fields.py | 5 +- .../0002_transaction_partition_1_of_3.py | 23 +++--- .../0003_transaction_partition_2_of_3.py | 23 +++--- .../0004_transaction_partition_3_of_3.py | 23 +++--- common/models/mixins/description.py | 2 +- common/models/tracked_qs.py | 14 +++- common/models/transactions.py | 24 +++--- common/querysets.py | 20 ++--- common/tests/models.py | 8 +- common/validators.py | 6 +- docker-compose.yml | 20 ++--- .../commands/upload_transactions.py | 17 ++-- exporter/serializers.py | 5 +- importer/cache/cache.py | 16 ++-- importer/nursery.py | 15 ++-- importer/taric.py | 10 ++- importer/tasks.py | 4 +- measures/business_rules.py | 2 +- .../0014_action_pair_data_migration.py | 2 +- measures/tests/conftest.py | 4 +- measures/tests/test_patterns.py | 2 +- measures/tests/test_querysets.py | 3 +- measures/tests/test_views.py | 4 +- package-lock.json | 6 +- .../models/crown_dependencies_envelope.py | 4 +- quotas/business_rules.py | 5 +- quotas/tests/test_business_rules.py | 7 +- quotas/tests/test_business_rules/test_ON10.py | 13 ++-- regulations/models.py | 28 +++---- reports/migrations/0001_report.py | 3 +- .../0002_create_custom_permissions.py | 5 +- reports/reports/base_table.py | 3 +- ...piring_quotas_with_no_definition_period.py | 14 ++-- .../test_quotas_cannot_be_used_report.py | 78 +++++++++++++------ reports/tests/test_report_utils.py | 4 +- reports/tests/test_report_views.py | 12 +-- reports/views.py | 6 +- settings/dev.py | 2 +- workbaskets/management/util.py | 13 +++- 47 files changed, 317 insertions(+), 235 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..4c89d655c --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +16.* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 98e41454c..575f3675c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Production static resource building +# Production static resource building # Currently moved to makefile for development # FROM node:lts-buster-slim AS jsdeps @@ -20,7 +20,7 @@ ARG ENV="prod" ENV ENV="${ENV}" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1\ - PATH="${PATH}:/home/tamato/.local/bin" + PATH="${PATH}:/home/tamato/.local/bin" # don't run as root RUN groupadd -g 1000 tamato && \ diff --git a/Makefile b/Makefile index 19647b75a..cf7c80681 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ DB_DUMP?=${PROJECT}_db.sql DB_NAME?=${PROJECT} DB_USER?=postgres DATE=$(shell date '+%Y_%m_%d') -TEMPLATE_NAME?="${DB_NAME}_${DATE}" +TEMPLATE_NAME?="${DB_NAME}_${DATE}" -include .env export @@ -20,7 +20,7 @@ SPHINXOPTS ?= .PHONY: help clean clean-bytecode clean-static collectstatic compilescss dependencies \ docker-clean docker-deep-clean docker-down docker-up-db docker-down docker-image \ docker-db-dump docker-test node_modules run test test-fast docker-makemigrations \ - docker-checkmigrations docker-migrate build-docs docker-restore-db docker-import-new-db + docker-checkmigrations docker-migrate build-docs docker-restore-db docker-import-new-db @@ -118,7 +118,7 @@ docker-test-fast: @echo @echo "> Running tests in docker..." @${COMPOSE_LOCAL} ${DOCKER_RUN} \ - ${PROJECT} ${PYTHON} -m pytest -x -n=auto --dist=loadfile + ${PROJECT} ${PYTHON} -m pytest -x -n=auto --dist=loadfile ## clean-docs: Clean the generated documentation files clean-docs: @@ -134,15 +134,15 @@ build-docs html: ## docker-clean: clean unused images and volumes docker-clean: @echo - @echo "> Cleaning unused images in docker..." + @echo "> Cleaning unused images in docker..." @${DOCKER} image prune -a -f - @echo "> Cleaning unused volumes in docker..." - @${DOCKER} volume prune -f + @echo "> Cleaning unused volumes in docker..." + @${DOCKER} volume prune -f ## docker-deep-clean: deep clean all unused systems (containers, networks, images, cache) docker-deep-clean: @echo - @echo "> Cleaning unused systems in docker..." + @echo "> Cleaning unused systems in docker..." @${DOCKER} system prune -a ## docker-down: shut down services in Docker @@ -154,38 +154,38 @@ docker-down: ## docker-up-db: shut down services in Docker docker-up-db: @echo - @echo "> Running db in docker..." + @echo "> Running db in docker..." @${COMPOSE_LOCAL} up -d db - @echo + @echo @echo "Waiting for database \"ready for connections\"" - @sleep 15; + @sleep 15; @echo "Database Ready for connections!" -## docker-import-new-db: Import new db DB_DUMP into new TEMPLATE_NAME in Docker container db must be running +## docker-import-new-db: Import new db DB_DUMP into new TEMPLATE_NAME in Docker container db must be running docker-import-new-db: docker-up-db - @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "DROP DATABASE ${TEMPLATE_NAME}" || true - @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "CREATE DATABASE ${TEMPLATE_NAME} TEMPLATE template0" + @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "DROP DATABASE ${TEMPLATE_NAME}" || true + @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "CREATE DATABASE ${TEMPLATE_NAME} TEMPLATE template0" @echo "> Running db dump: ${DB_DUMP} in docker..." - @cat ${DB_DUMP} | ${COMPOSE_LOCAL} exec -T db psql -U ${DB_USER} -d ${TEMPLATE_NAME} - @sleep 5; + @cat ${DB_DUMP} | ${COMPOSE_LOCAL} exec -T db psql -U ${DB_USER} -d ${TEMPLATE_NAME} + @sleep 5; -## docker-restore-db: Resotre db in Docker container set DB_NAME to rename db must be running +## docker-restore-db: Resotre db in Docker container set DB_NAME to rename db must be running docker-restore-db: docker-down docker-up-db - @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "DROP DATABASE ${DB_NAME}" || true - @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "CREATE DATABASE ${DB_NAME} TEMPLATE ${TEMPLATE_NAME}" + @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "DROP DATABASE ${DB_NAME}" || true + @${COMPOSE_LOCAL} exec -u ${DB_USER} db psql -c "CREATE DATABASE ${DB_NAME} TEMPLATE ${TEMPLATE_NAME}" @sleep 5; ## docker-db-dump: Run db dump to import data into Docker docker-db-dump: docker-up-db @echo "> Running db dump in docker..." - @cat ${DB_DUMP} | ${COMPOSE_LOCAL} exec -T db psql -U ${DB_USER} -d ${DB_NAME} + @cat ${DB_DUMP} | ${COMPOSE_LOCAL} exec -T db psql -U ${DB_USER} -d ${DB_NAME} -## docker-first-use: Run application for first time in Docker +## docker-first-use: Run application for first time in Docker docker-first-use: docker-down docker-clean node_modules compilescss docker-build docker-import-new-db \ - docker-restore-db docker-migrate docker-superuser docker-up + docker-restore-db docker-migrate docker-superuser docker-up ## docker-makemigrations: Run django makemigrations in Docker -docker-makemigrations: +docker-makemigrations: @echo @echo "> Running makemigrations in docker..." @${COMPOSE_LOCAL} ${DOCKER_RUN} \ diff --git a/checks/tests/test_tasks.py b/checks/tests/test_tasks.py index adadf9421..e60ddfe8f 100644 --- a/checks/tests/test_tasks.py +++ b/checks/tests/test_tasks.py @@ -56,7 +56,7 @@ def check(request): assert num_completed >= num_successful check = factories.TransactionCheckFactory.create( - **{trait: True for trait in traits} + **{trait: True for trait in traits}, ) check_names = [str(i) for i in range(num_checks)] completes = repeat(True, num_completed) diff --git a/commodities/import_handlers.py b/commodities/import_handlers.py index afd6a212f..26589127b 100644 --- a/commodities/import_handlers.py +++ b/commodities/import_handlers.py @@ -106,7 +106,7 @@ def get_absorbed_into_goods_nomenclature_link(self, model, kwargs): ): previous = ( models.GoodsNomenclatureSuccessor.objects.filter( - **{key: self.data[key] for key in self.identifying_fields} + **{key: self.data[key] for key in self.identifying_fields}, ) .latest_approved() .get() @@ -144,7 +144,7 @@ def create_missing_goods_nomenclature_description_period( goods_nomenclature_description_handler, ): """ - in some circumstances, we will receive an EU update that will reference + In some circumstances, we will receive an EU update that will reference a historic description period, and since TAP does not track SIDs currently for this data we cant resolve the reference. diff --git a/commodities/models/dc.py b/commodities/models/dc.py index c803ce06b..3f00c01d5 100644 --- a/commodities/models/dc.py +++ b/commodities/models/dc.py @@ -911,7 +911,10 @@ def to_transaction(self, workbasket: WorkBasket) -> TrackedModel: with workbasket.new_transaction(order=order) as transaction: return self.obj.new_version( - workbasket, transaction, update_type=self.update_type, **attrs + workbasket, + transaction, + update_type=self.update_type, + **attrs, ) def _get_preemptive_transaction_order(self, workbasket: WorkBasket) -> int: @@ -1693,7 +1696,9 @@ def __init__(self, **kwargs): def get_tracked_model_reflection( - obj: TrackedModel, transaction: Transaction = None, **overrides + obj: TrackedModel, + transaction: Transaction = None, + **overrides, ): """ Returns a reflection of a TrackedModel object. diff --git a/commodities/tests/conftest.py b/commodities/tests/conftest.py index 5b761d873..123110c26 100644 --- a/commodities/tests/conftest.py +++ b/commodities/tests/conftest.py @@ -166,7 +166,9 @@ def create_collection( def create_record( - transaction_pool: Iterator[Transaction], factory, **kwargs + transaction_pool: Iterator[Transaction], + factory, + **kwargs, ) -> TrackedModel: """ Returns a new TrackedModel instance. @@ -179,7 +181,9 @@ def create_record( def create_dependent_measure( - commodity: Commodity, transaction_pool: Iterator[Transaction], **kwargs + commodity: Commodity, + transaction_pool: Iterator[Transaction], + **kwargs, ) -> Measure: """Returns a new measure linked to a given good.""" factory = factories.MeasureFactory @@ -190,7 +194,9 @@ def create_dependent_measure( def create_footnote_association( - commodity: Commodity, transaction_pool: Iterator[Transaction], **kwargs + commodity: Commodity, + transaction_pool: Iterator[Transaction], + **kwargs, ) -> FootnoteAssociationGoodsNomenclature: """Returns a new footnote association linked to a given good.""" factory = factories.FootnoteAssociationGoodsNomenclatureFactory diff --git a/common/business_rules.py b/common/business_rules.py index 79b7a6283..03bc65b31 100644 --- a/common/business_rules.py +++ b/common/business_rules.py @@ -132,8 +132,8 @@ def get_linked_models( :param model TrackedModel: Get models linked to this model instance :param transaction Transaction: Get latest approved versions of linked - models as of this transaction - :rtype Iterator[TrackedModel]: The linked models + models as of this transaction :rtype Iterator[TrackedModel]: The + linked models """ for field, related_model in get_relations(type(model)).items(): business_rules = getattr(related_model, "business_rules", []) @@ -169,9 +169,11 @@ def violation( """ Create a violation exception object. - :param model Optional[TrackedModel]: The model that violates this business rule - :param message Optional[str]: A message explaining the violation - :rtype BusinessRuleViolation: An exception indicating a business rule violation + :param model Optional[TrackedModel]: The model that violates this + business rule :param message Optional[str]: A message explaining the + violation + :rtype BusinessRuleViolation: An exception indicating a business rule + violation """ return getattr(self.__class__, "Violation", BusinessRuleViolation)( @@ -185,7 +187,7 @@ def only_applicable_after(cutoff: Union[date, datetime, str]): Decorate BusinessRules to make them only applicable after a given date. :param cutoff Union[date, datetime, str]: The date, datetime or isoformat - date string of the time before which the rule should not apply + date string of the time before which the rule should not apply """ if isinstance(cutoff, str): @@ -226,8 +228,8 @@ def skip_when_update_type(cls: Type[BusinessRule], update_types: Iterable[Update """ Skip business rule validation for given update types. - :param cls Type[BusinessRule]: The BusinessRule to decorate - :param update_types Iterable[int]: The UpdateTypes to skip + :param cls Type[BusinessRule]: The BusinessRule to decorate :param + update_types Iterable[int]: The UpdateTypes to skip """ _original_validate = cls.validate @@ -336,7 +338,8 @@ def validate(self, model): Check whether the specified model violates this business rule. :param model TrackedModel: The model to check - :raises BusinessRuleViolation: Raised if the passed model violates this business rule. + :raises BusinessRuleViolation: Raised if the passed model violates this + business rule. """ if self.has_violation(model): raise self.violation(model) diff --git a/common/fields.py b/common/fields.py index 4c4c58f9f..1413e176b 100644 --- a/common/fields.py +++ b/common/fields.py @@ -100,7 +100,10 @@ class TaricDateRangeField(DateRangeField): range_type = TaricDateRange def from_db_value( - self, value: Union[DateRange, TaricDateRange], *_args, **_kwargs + self, + value: Union[DateRange, TaricDateRange], + *_args, + **_kwargs, ) -> TaricDateRange: """ By default Django ignores the range_type and just returns a Psycopg2 diff --git a/common/migrations/0002_transaction_partition_1_of_3.py b/common/migrations/0002_transaction_partition_1_of_3.py index 61dae9f37..7d4b7010f 100644 --- a/common/migrations/0002_transaction_partition_1_of_3.py +++ b/common/migrations/0002_transaction_partition_1_of_3.py @@ -8,17 +8,18 @@ class Migration(migrations.Migration): """ Transaction partition field 1 of 3. - 1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of - Transactions are from the seed file. - - 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION: Before this - migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved - workbasket status. After this migration was written data schemas allow more control over SEED / REVISION - transactions. DRAFT: Draft Transactions are inferred by checking for transactions not in the first workbasket - that lack an approved workbasket status.. - - 3/3 - Set the default value to (3, DRAFT) + 1/3 Create Partition field, with partition defaulting to (1, SEED), as at + the time of writing most of Transactions are from the seed file. + + 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, + DRAFT). REVISION: Before this migration was written REVISION transactions + are contained in workbaskets after the first workbasket with approved + workbasket status. After this migration was written data schemas allow more + control over SEED / REVISION transactions. DRAFT: Draft Transactions are + inferred by checking for transactions not in the first workbasket that lack + an approved workbasket status.. + + 3/3 Set the default value to (3, DRAFT) """ dependencies = [ diff --git a/common/migrations/0003_transaction_partition_2_of_3.py b/common/migrations/0003_transaction_partition_2_of_3.py index c784f5af9..669ed56f7 100644 --- a/common/migrations/0003_transaction_partition_2_of_3.py +++ b/common/migrations/0003_transaction_partition_2_of_3.py @@ -42,17 +42,18 @@ class Migration(migrations.Migration): """ Transaction partition field 2 of 3. - 1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of - Transactions are from the seed file. - - 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION: Before this - migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved - workbasket status. After this migration was written data schemas allow more control over SEED / REVISION - transactions. DRAFT: Draft Transactions are inferred by checking for transactions not in the first workbasket - that lack an approved workbasket status.. - - 3/3 - Set the default value to (3, DRAFT) + 1/3 Create Partition field, with partition defaulting to (1, SEED), as at + the time of writing most of Transactions are from the seed file. + + 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, + DRAFT). REVISION: Before this migration was written REVISION transactions + are contained in workbaskets after the first workbasket with approved + workbasket status. After this migration was written data schemas allow more + control over SEED / REVISION transactions. DRAFT: Draft Transactions are + inferred by checking for transactions not in the first workbasket that lack + an approved workbasket status.. + + 3/3 Set the default value to (3, DRAFT) """ dependencies = [ diff --git a/common/migrations/0004_transaction_partition_3_of_3.py b/common/migrations/0004_transaction_partition_3_of_3.py index 56fc0b938..287089161 100644 --- a/common/migrations/0004_transaction_partition_3_of_3.py +++ b/common/migrations/0004_transaction_partition_3_of_3.py @@ -8,17 +8,18 @@ class Migration(migrations.Migration): """ Transaction partition field 3 of 3. - 1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of - Transactions are from the seed file. - - 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION: Before this - migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved - workbasket status. After this migration was written data schemas allow more control over SEED / REVISION - transactions. DRAFT: Draft Transactions are inferred by checking for transactions not in the first workbasket - that lack an approved workbasket status.. - - 3/3 - Set the default value to (3, DRAFT) + 1/3 Create Partition field, with partition defaulting to (1, SEED), as at + the time of writing most of Transactions are from the seed file. + + 2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, + DRAFT). REVISION: Before this migration was written REVISION transactions + are contained in workbaskets after the first workbasket with approved + workbasket status. After this migration was written data schemas allow more + control over SEED / REVISION transactions. DRAFT: Draft Transactions are + inferred by checking for transactions not in the first workbasket that lack + an approved workbasket status.. + + 3/3 Set the default value to (3, DRAFT) """ dependencies = [ diff --git a/common/models/mixins/description.py b/common/models/mixins/description.py index 36104e6c2..4972ef3c4 100644 --- a/common/models/mixins/description.py +++ b/common/models/mixins/description.py @@ -147,7 +147,7 @@ def get_descriptions(self) -> TrackedModelQuerySet: } query = descriptions_model.objects.filter(**filter_kwargs).order_by( - *descriptions_model._meta.ordering + *descriptions_model._meta.ordering, ) return query.current() diff --git a/common/models/tracked_qs.py b/common/models/tracked_qs.py index 10d672aff..73e363687 100644 --- a/common/models/tracked_qs.py +++ b/common/models/tracked_qs.py @@ -195,7 +195,11 @@ def version_ordering(self) -> TrackedModelQuerySet: return self.order_by("transaction__partition", "transaction__order") def _get_current_related_lookups( - self, model, *lookups, prefix="", recurse_level=0 + self, + model, + *lookups, + prefix="", + recurse_level=0, ) -> List[str]: """ Build a list of lookups for the current versions of related objects. @@ -241,10 +245,14 @@ def with_latest_links(self, *lookups, recurse_level=0) -> TrackedModelQuerySet: run multiple queries for every current relation. """ related_lookups = self._get_current_related_lookups( - self.model, *lookups, recurse_level=recurse_level + self.model, + *lookups, + recurse_level=recurse_level, ) return self.select_related( - "version_group", "version_group__current_version", *related_lookups + "version_group", + "version_group__current_version", + *related_lookups, ) def with_transactions_and_models(self) -> TrackedModelQuerySet: diff --git a/common/models/transactions.py b/common/models/transactions.py index 3038236a6..597179824 100644 --- a/common/models/transactions.py +++ b/common/models/transactions.py @@ -25,13 +25,12 @@ class TransactionPartition(models.IntegerChoices): Within a partition, a transactions order applies, to obtain a global ordering call .order_by("partition", "order"). - The numbers chosen increment by "era" of transaction, starting - from approved transactions SEED_FILE, REVISION - then ending with DRAFT. + The numbers chosen increment by "era" of transaction, starting from approved + transactions SEED_FILE, REVISION then ending with DRAFT. This system enables a simple filter for "approved" transactions, by checking - for <= HIGHEST_APPROVED_PARTITION, this is currently set to REVISION, - which is implemented in TransactionQuerySet.approved(). + for <= HIGHEST_APPROVED_PARTITION, this is currently set to REVISION, which + is implemented in TransactionQuerySet.approved(). """ SEED_FILE = 1, "Seed" @@ -329,7 +328,8 @@ def _get_summary(self): Attempts a balance between readability and enough information to debug issues, so contains the pk and status of the transaction and workbasket. - Stringification is lazily evaluated, so this property can be passed to loggers. + Stringification is lazily evaluated, so this property can be passed to + loggers. """ return ( f"transaction: {self.partition}, {self.pk} " @@ -345,8 +345,8 @@ def summary(self): issues, so contains the partion, order for transactions and pk, status for workbaskets. - Stringification happens lazily so this property is suitable for use - when logging. + Stringification happens lazily so this property is suitable for use when + logging. """ # This is not decorated with lazy_string because it doesn't work with properties return self._get_summary() @@ -356,11 +356,11 @@ class TransactionGroup(models.Model): """ An ordered group of Transactions. - Transactions often must be applied in a specific sequence, for example to ensure - a Measure exists before a Footnote can be associated with it. + Transactions often must be applied in a specific sequence, for example to + ensure a Measure exists before a Footnote can be associated with it. - A Transaction may belong to several groups, for example a group associated with an - imported Envelope, and with a WorkBasket. + A Transaction may belong to several groups, for example a group associated + with an imported Envelope, and with a WorkBasket. """ group_id = models.IntegerField() diff --git a/common/querysets.py b/common/querysets.py index 77dedac89..0f03f0e5d 100644 --- a/common/querysets.py +++ b/common/querysets.py @@ -25,7 +25,8 @@ def not_in_effect(self, at_date: date) -> QuerySet: Filter queryset to only those whose validity period does NOT include the given date. - :param at_date date: Exclude results whose validity period contains this date. + :param at_date date: Exclude results whose validity period contains this + date. :rtype QuerySet: """ return self.with_validity_field().exclude( @@ -37,8 +38,8 @@ def no_longer_in_effect(self, at_date: date) -> QuerySet: Filter queryset to only those whose validity period ends before the given date. - :param at_date date: Exclude results whose validity period contains or comes - after this date. + :param at_date date: Exclude results whose validity period contains or + comes after this date. :rtype QuerySet: """ return self.with_validity_field().filter( @@ -51,7 +52,8 @@ def not_yet_in_effect(self, at_date: date) -> QuerySet: Filter the queryset to only those whose validity period starts after the given date. - :param at_date date: Exclude results with validity starts before this date. + :param at_date date: Exclude results with validity starts before this + date. :rtype QuerySet: """ return self.with_validity_field().filter( @@ -78,8 +80,8 @@ def as_at(self, date: date) -> QuerySet: If done from the TrackedModel this will return all instances of all tracked models as represented at a particular date. - :param date date: Exclude results with validity periods that do not contain this - date. + :param date date: Exclude results with validity periods that do not + contain this date. :rtype QuerySet: """ return self.filter(valid_between__contains=date) @@ -120,7 +122,7 @@ def approved_query_filter(cls, prefix=""): return Q( **{ f"{prefix}transaction__partition__in": TransactionPartition.approved_partitions(), - } + }, ) @classmethod @@ -151,11 +153,11 @@ def as_at_transaction_filter(cls, transaction, prefix=""): **{ f"{prefix}transaction__partition": TransactionPartition.DRAFT, f"{prefix}transaction__workbasket__id": transaction.workbasket_id, - } + }, ) | Q( **{ f"{prefix}transaction__partition__in": TransactionPartition.approved_partitions(), - } + }, ) earlier_partition = { diff --git a/common/tests/models.py b/common/tests/models.py index 44f3b8791..f88a03dfc 100644 --- a/common/tests/models.py +++ b/common/tests/models.py @@ -91,11 +91,15 @@ class Models: """ all_models = generate_model_history( - factory, valid_between=date_ranges.earlier, **kwargs + factory, + valid_between=date_ranges.earlier, + **kwargs, ) active_model = factory.create( - valid_between=date_ranges.current, update_type=UpdateType.UPDATE, **kwargs + valid_between=date_ranges.current, + update_type=UpdateType.UPDATE, + **kwargs, ) all_models.append(active_model) diff --git a/common/validators.py b/common/validators.py index c488b40aa..bac4e1cb5 100644 --- a/common/validators.py +++ b/common/validators.py @@ -48,9 +48,9 @@ class NumericSIDValidator(NumberRangeValidator): """ Validates TARIC SID values. - A TARIC SID is a unique number generated by the system as an internal access key. - This will be used in areas where there is a need to change access key data - dynamically, or where the logical key is too long. + A TARIC SID is a unique number generated by the system as an internal access + key. This will be used in areas where there is a need to change access key + data dynamically, or where the logical key is too long. It is commonly a number between 1 and 99999999 (max 8 digits). """ diff --git a/docker-compose.yml b/docker-compose.yml index e59c67210..879ebed93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" shm_size: 32g stdin_open: true - tty: true + tty: true healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s @@ -22,7 +22,7 @@ services: image: redis ports: - "127.0.0.1:6379:6379" - restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" + restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" celery-redis: image: redis @@ -37,11 +37,11 @@ services: - "ENV=${ENV:-prod}" volumes: - ./:/app/ - command: + command: [ - "celery", "-A" , "common.celery" ,"worker", "-O", "fair", "-l", "info", "-Q", "standard" + "celery", "-A" , "common.celery" ,"worker", "-O", "fair", "-l", "info", "-Q", "standard" ] - env_file: + env_file: - .env - settings/envs/docker.env restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" @@ -58,7 +58,7 @@ services: volumes: - ./:/app/ command: ["celery", "-A" , "common.celery" ,"worker", "-O", "fair", "-l", "info", "-Q", "rule-check"] - env_file: + env_file: - .env - settings/envs/docker.env restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" @@ -75,7 +75,7 @@ services: volumes: - ./:/app/ command: ["celery", "-A" , "common.celery" ,"worker", "-O", "fair", "-l", "info", "-Q", "importer", "--concurrency", "1"] - env_file: + env_file: - .env - settings/envs/docker.env restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" @@ -90,7 +90,7 @@ services: volumes: - ./:/app/ command: ["celery", "-A" , "common.celery" ,"beat", "-l", "info"] - env_file: + env_file: - .env - settings/envs/docker.env restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" @@ -117,7 +117,7 @@ services: retries: 3 tamato: - build: + build: context: . args: - "ENV=${ENV:-prod}" @@ -137,7 +137,7 @@ services: command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" stdin_open: true - tty: true + tty: true volumes: tamato-data-volume: {} diff --git a/exporter/management/commands/upload_transactions.py b/exporter/management/commands/upload_transactions.py index 081229a86..d6203a672 100644 --- a/exporter/management/commands/upload_transactions.py +++ b/exporter/management/commands/upload_transactions.py @@ -14,15 +14,12 @@ def run_task_or_exit(task, local=False, *args, **kwargs): """ Run celery task, block then return the return value of the task, or exit.. - :param local: Run locally (bypass celery) - :param task: Celery task to run. - :return: Result of task. - - Utility function for management commands that require the ability - to start a celery task, then wait for the result. - - Tasks may optionally be run locally, which can be useful if the - user needs to run a task, but celery infrastructure is unavailable. + :param local: Run locally (bypass celery) + :param task: Celery task to run. + :return: Result of task. Utility function for management commands that + require the ability to start a celery task, then wait for the result. + Tasks may optionally be run locally, which can be useful if the user + needs to run a task, but celery infrastructure is unavailable. """ try: if local: @@ -57,7 +54,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): local = options["local"] upload_result = UploadTaskResultData( - **run_task_or_exit(upload_workbaskets, local=local) + **run_task_or_exit(upload_workbaskets, local=local), ) upload_result.output(self.stdout) diff --git a/exporter/serializers.py b/exporter/serializers.py index bb3c9cdc1..69c132dec 100644 --- a/exporter/serializers.py +++ b/exporter/serializers.py @@ -57,7 +57,10 @@ def __init__( """ self.output_constructor = output_constructor EnvelopeSerializer.__init__( - self, self.output_constructor(), envelope_id=envelope_id, **kwargs + self, + self.output_constructor(), + envelope_id=envelope_id, + **kwargs, ) def start_next_envelope(self): diff --git a/importer/cache/cache.py b/importer/cache/cache.py index f7ff1581c..7290f5ee6 100644 --- a/importer/cache/cache.py +++ b/importer/cache/cache.py @@ -8,14 +8,14 @@ class ObjectCacheFacade(BaseEngine): """ Stores objects in a cache to be fetched for later use. - To avoid committing to a specific medium of storage this facade is being implemented - to provide a neutral interface for getting, putting and popping object data into and - out of storage. The only requirement is a unique hashable key for which to fetch the - data with. - - Currently the implementation relies simply on a process level mutable dictionary. - However in future this may be updated to use something more persistent such as - Redis or similar. + To avoid committing to a specific medium of storage this facade is being + implemented to provide a neutral interface for getting, putting and popping + object data into and out of storage. The only requirement is a unique + hashable key for which to fetch the data with. + + Currently the implementation relies simply on a process level mutable + dictionary. However in future this may be updated to use something more + persistent such as Redis or similar. """ DEFAULT_ENGINE = "importer.cache.memory.MemoryCacheEngine" diff --git a/importer/nursery.py b/importer/nursery.py index 6fb8e1bd9..e557734f9 100644 --- a/importer/nursery.py +++ b/importer/nursery.py @@ -27,13 +27,14 @@ class TariffObjectNursery: Provides an interface between raw data and the Django modelling system for the tariff. - The primary function is to take a raw python object (generally a dictionary) and convert - it into a row in the database via the Django models. - - This layer of separation comes from the fact that often, when receiving data, the system - will receive incomplete data objects which depend on data received later. As a result - this "Nursery" is designed to store and look after incomplete data objects until it can - be matched with the rest of the data and then dispatched to the database. + The primary function is to take a raw python object (generally a dictionary) + and convert it into a row in the database via the Django models. + + This layer of separation comes from the fact that often, when receiving + data, the system will receive incomplete data objects which depend on data + received later. As a result this "Nursery" is designed to store and look + after incomplete data objects until it can be matched with the rest of the + data and then dispatched to the database. """ handlers: Dict[str, Type[BaseHandler]] = {} diff --git a/importer/taric.py b/importer/taric.py index cf62f2dab..6762eeedf 100644 --- a/importer/taric.py +++ b/importer/taric.py @@ -58,7 +58,8 @@ def save(self, data: Mapping[str, Any], transaction_id: int): Save the Record to the database. :param data: A dict of the parsed element, mapping field names to values - :param transaction_id: The primary key of the transaction to add the record to + :param transaction_id: The primary key of the transaction to add the + record to """ method_name = { str(UpdateType.UPDATE): "update", @@ -83,7 +84,8 @@ def save(self, data: Mapping[str, Any], transaction_id: int): Save the contained records to the database. :param data: A dict of parsed element, mapping field names to values - :param transaction_id: The primary key of the transaction to add records to + :param transaction_id: The primary key of the transaction to add records + to """ logger.debug("Saving message id: %s", data.get("id")) for record_data in data.get("record", []): @@ -107,8 +109,8 @@ def save( """ Save the transaction and the contained records to the database. - :param data: A dict of the parsed element, containing at least an "id" and list - of "message" dicts + :param data: A dict of the parsed element, containing at least an "id" + and list of "message" dicts :param envelope: Containing Envelope """ logging.debug(f"Saving transaction {self.data['id']}") diff --git a/importer/tasks.py b/importer/tasks.py index 4e1c10a43..d41290b03 100644 --- a/importer/tasks.py +++ b/importer/tasks.py @@ -118,7 +118,9 @@ def setup_chunk_task( get_partition_scheme(partition_scheme_setting) if batch.ready_chunks.filter( - record_code=record_code, status=ImporterChunkStatus.RUNNING, **kwargs + record_code=record_code, + status=ImporterChunkStatus.RUNNING, + **kwargs, ).exists(): return diff --git a/measures/business_rules.py b/measures/business_rules.py index 1c317f338..d7a5cb82b 100644 --- a/measures/business_rules.py +++ b/measures/business_rules.py @@ -887,7 +887,7 @@ def validate(self, measure): **{ f"{self.component_field}__isnull": code == ApplicabilityCode.MANDATORY, - } + }, ) if ( components.filter(inapplicable) diff --git a/measures/migrations/0014_action_pair_data_migration.py b/measures/migrations/0014_action_pair_data_migration.py index a816d2e66..1d9035dab 100644 --- a/measures/migrations/0014_action_pair_data_migration.py +++ b/measures/migrations/0014_action_pair_data_migration.py @@ -14,7 +14,7 @@ ("34", "14"), ("36", "16"), ] -"""positive, negative action code tuples.""" +"""Positive, negative action code tuples.""" @atomic diff --git a/measures/tests/conftest.py b/measures/tests/conftest.py index 107809d7c..935663ac0 100644 --- a/measures/tests/conftest.py +++ b/measures/tests/conftest.py @@ -34,7 +34,7 @@ def check(field_name, value, factory=None, applicability_field=None): **{ applicability_field: ApplicabilityCode.MANDATORY, field_name: None, - } + }, ) with pytest.raises(ValidationError): @@ -42,7 +42,7 @@ def check(field_name, value, factory=None, applicability_field=None): **{ applicability_field: ApplicabilityCode.NOT_PERMITTED, field_name: value, - } + }, ) return True diff --git a/measures/tests/test_patterns.py b/measures/tests/test_patterns.py index 1f1119024..5d013259d 100644 --- a/measures/tests/test_patterns.py +++ b/measures/tests/test_patterns.py @@ -126,7 +126,7 @@ def test_all_records_in_same_transaction( measure_creation_pattern: MeasureCreationPattern, ): tracked_models = measure_creation_pattern.create_measure_tracked_models( - **measure_data + **measure_data, ) assert len(set(m.transaction for m in tracked_models)) == 1 diff --git a/measures/tests/test_querysets.py b/measures/tests/test_querysets.py index c8dd0fbdf..34dec9064 100644 --- a/measures/tests/test_querysets.py +++ b/measures/tests/test_querysets.py @@ -296,7 +296,8 @@ def test_with_reference_price_string_measurement( """ condition_measurement = factories.MeasurementFactory.create(**measurement_kwargs) condition = factories.MeasureConditionFactory.create( - condition_measurement=condition_measurement, **condition_kwargs + condition_measurement=condition_measurement, + **condition_kwargs, ) qs = MeasureCondition.objects.with_reference_price_string() price_condition = qs.first() diff --git a/measures/tests/test_views.py b/measures/tests/test_views.py index 619fb8798..98b13d9c4 100644 --- a/measures/tests/test_views.py +++ b/measures/tests/test_views.py @@ -721,7 +721,7 @@ def test_measure_update_create_conditions( assert condition.update_type == UpdateType.CREATE components = condition.components.approved_up_to_transaction(tx).order_by( - *MeasureConditionComponent._meta.ordering + *MeasureConditionComponent._meta.ordering, ) assert components.count() == 2 @@ -913,7 +913,7 @@ def test_measure_update_negative_condition( condition = updated_measure.conditions.approved_up_to_transaction(tx).last() components = condition.components.approved_up_to_transaction(tx).order_by( - *MeasureConditionComponent._meta.ordering + *MeasureConditionComponent._meta.ordering, ) assert components.count() == 1 diff --git a/package-lock.json b/package-lock.json index 9f913145e..edbe5738c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15322,9 +15322,9 @@ "integrity": "sha512-EcwnP9PsWubmTcsJtLF8w0D5b69j43LwYtSqGyPtO3903J0bbwB2t5ujWZ9UhsbLamuUmigBkNpLItU3/KuFqw==" }, "govuk-frontend": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/govuk-frontend/-/govuk-frontend-3.14.0.tgz", - "integrity": "sha512-y7FTuihCSA8Hty+e9h0uPhCoNanCAN+CLioNFlPmlbeHXpbi09VMyxTcH+XfnMPY4Cp++7096v0rLwwdapTXnA==" + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/govuk-frontend/-/govuk-frontend-3.15.0.tgz", + "integrity": "sha512-kInDei8hrkMcrW7yC2EwhbSNBOBBPTdLxIDAye2G7KNrD9cyvNAfd0KchrfP/nWBOzu67ANoz2rtoRDLWiBN2w==" }, "govuk-react": { "version": "0.10.6", diff --git a/publishing/models/crown_dependencies_envelope.py b/publishing/models/crown_dependencies_envelope.py index aab4640c7..043acfcd0 100644 --- a/publishing/models/crown_dependencies_envelope.py +++ b/publishing/models/crown_dependencies_envelope.py @@ -30,7 +30,9 @@ class ApiEnvelopeInvalidWorkBasketStatus(Exception): class CrownDependenciesEnvelopeManager(Manager): @atomic def create( - self, packaged_work_basket: PackagedWorkBasket, **kwargs + self, + packaged_work_basket: PackagedWorkBasket, + **kwargs, ) -> "CrownDependenciesEnvelope": """ Create a new instance, from the packaged workbasket successfully diff --git a/quotas/business_rules.py b/quotas/business_rules.py index 5d625626c..24325f70c 100644 --- a/quotas/business_rules.py +++ b/quotas/business_rules.py @@ -210,9 +210,8 @@ def validate(self, order_number_origin): class ON13(BusinessRule): - """An exclusion can only be entered if the order number origin is a geographical - area group (area code = 1). - """ + """An exclusion can only be entered if the order number origin is a + geographical area group (area code = 1).""" def validate(self, exclusion): if exclusion.origin.geographical_area.area_code != AreaCode.GROUP: diff --git a/quotas/tests/test_business_rules.py b/quotas/tests/test_business_rules.py index 753e8a5c2..baf058ed5 100644 --- a/quotas/tests/test_business_rules.py +++ b/quotas/tests/test_business_rules.py @@ -1,5 +1,4 @@ from collections import defaultdict -from datetime import date from decimal import Decimal import pytest @@ -9,7 +8,6 @@ from common.tests import factories from common.tests.util import only_applicable_after from common.tests.util import raises_if -from common.util import TaricDateRange from common.validators import UpdateType from geo_areas.validators import AreaCode from quotas import business_rules @@ -205,9 +203,8 @@ def test_ON12_does_not_raise(delete_record): ], ) def test_ON13(area_code, expect_error): - """An exclusion can only be entered if the order number origin is a geographical - area group (area code = 1). - """ + """An exclusion can only be entered if the order number origin is a + geographical area group (area code = 1).""" origin = factories.QuotaOrderNumberOriginFactory.create( geographical_area__area_code=area_code, diff --git a/quotas/tests/test_business_rules/test_ON10.py b/quotas/tests/test_business_rules/test_ON10.py index 7db05ec70..e755fe385 100644 --- a/quotas/tests/test_business_rules/test_ON10.py +++ b/quotas/tests/test_business_rules/test_ON10.py @@ -5,11 +5,8 @@ from common.business_rules import BusinessRuleViolation from common.tests import factories from common.tests.util import only_applicable_after -from common.util import TaricDateRange, validity_range_contains_range +from common.util import TaricDateRange from quotas import business_rules -from quotas.models import QuotaOrderNumber -from quotas.models import QuotaOrderNumberOrigin - pytestmark = pytest.mark.django_db @@ -58,7 +55,8 @@ def test_ON10_not_effective_valid_between(): measure = factories.MeasureWithQuotaFactory.create( order_number__origin__valid_between=TaricDateRange( - date(2021, 1, 1), date(2021, 1, 31) + date(2021, 1, 1), + date(2021, 1, 31), ), valid_between=valid_between, transaction=unapproved_transaction, @@ -79,14 +77,15 @@ def test_ON10_is_effective_valid_between(): unapproved_transaction = factories.UnapprovedTransactionFactory.create() regulation = factories.RegulationFactory.create( - effective_end_date=date(2021, 1, 31) + effective_end_date=date(2021, 1, 31), ) valid_between = TaricDateRange(date(2020, 1, 1), None) measure = factories.MeasureWithQuotaFactory.create( order_number__origin__valid_between=TaricDateRange( - date(2020, 1, 1), date(2021, 1, 31) + date(2020, 1, 1), + date(2021, 1, 31), ), valid_between=valid_between, generating_regulation=regulation, diff --git a/regulations/models.py b/regulations/models.py index 01880e41e..9c7ccdfb1 100644 --- a/regulations/models.py +++ b/regulations/models.py @@ -133,8 +133,8 @@ class Regulation(TrackedModel, ValidityMixin): approved = models.BooleanField( default=False, ) - - """The code which indicates whether or not a regulation has been replaced.""" + """The code which indicates whether or not a regulation has been + replaced.""" replacement_indicator = models.PositiveIntegerField( choices=validators.ReplacementIndicator.choices, default=validators.ReplacementIndicator.NOT_REPLACED, @@ -265,11 +265,11 @@ class Amendment(TrackedModel): """ This regulation amends a base regulation or an antidumping regulation. - It can affect the tariff and commercial aspects of the updated regulation but it - does not extend nor close the regulation itself. + It can affect the tariff and commercial aspects of the updated regulation + but it does not extend nor close the regulation itself. - If the modification regulation has no end date of validity, its end date is defined - by the end date of the regulations it modifies. + If the modification regulation has no end date of validity, its end date is + defined by the end date of the regulations it modifies. """ record_code = "290" @@ -300,11 +300,11 @@ class Extension(TrackedModel): validity end date of a base or modification regulation. This means that the measures falling under the prorogued regulation are prorogued as well. - A prorogation regulation can extend several regulations at different dates and a - regulation can be extended several times. + A prorogation regulation can extend several regulations at different dates + and a regulation can be extended several times. - If a regulation has been prorogued, its published end date does not take into - account the prorogation and is different from its effective end date. + If a regulation has been prorogued, its published end date does not take + into account the prorogation and is different from its effective end date. Prorogation regulations are also called "extension regulations". """ @@ -373,11 +373,11 @@ class Termination(TrackedModel): applies by definition to all measures under the abrogated regulation; unless these measures have a specific end dates. - If a regulation has been abrogated, its published end date does not take into - account the abrogation and is different from its effective end date. + If a regulation has been abrogated, its published end date does not take + into account the abrogation and is different from its effective end date. - An explicit abrogation regulation has no validity period; it ends the validity - period of a measure generating regulation or an FTS regulation. + An explicit abrogation regulation has no validity period; it ends the + validity period of a measure generating regulation or an FTS regulation. """ record_code = "280" diff --git a/reports/migrations/0001_report.py b/reports/migrations/0001_report.py index 187809243..e4ca24966 100644 --- a/reports/migrations/0001_report.py +++ b/reports/migrations/0001_report.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.20 on 2023-09-11 14:27 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): diff --git a/reports/migrations/0002_create_custom_permissions.py b/reports/migrations/0002_create_custom_permissions.py index 187b9fa12..8437f38e3 100644 --- a/reports/migrations/0002_create_custom_permissions.py +++ b/reports/migrations/0002_create_custom_permissions.py @@ -1,8 +1,9 @@ -from django.contrib.auth.models import Permission, Group +from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType -from reports.models import Report from django.db import migrations +from reports.models import Report + def create_custom_permissions(apps, schema_editor): content_type = ContentType.objects.get_for_model(Report) diff --git a/reports/reports/base_table.py b/reports/reports/base_table.py index b638c9353..161020611 100644 --- a/reports/reports/base_table.py +++ b/reports/reports/base_table.py @@ -1,4 +1,5 @@ from abc import abstractmethod + from django.urls import reverse from django.utils.safestring import mark_safe @@ -17,7 +18,7 @@ def link_renderer_for_quotas(self, order_number, text, fragment=None): url = reverse("quota-ui-detail", args=[order_number.sid]) href = url + fragment if fragment else url return mark_safe( - f"{text}" + f"{text}", ) @abstractmethod diff --git a/reports/tests/test_expiring_quotas_with_no_definition_period.py b/reports/tests/test_expiring_quotas_with_no_definition_period.py index 1965e20e1..20ee11de0 100644 --- a/reports/tests/test_expiring_quotas_with_no_definition_period.py +++ b/reports/tests/test_expiring_quotas_with_no_definition_period.py @@ -1,6 +1,8 @@ -import pytest import datetime + +import pytest from dateutil.relativedelta import relativedelta + from common.tests import factories from common.util import TaricDateRange from reports.reports.expiring_quotas_with_no_definition_period import Report @@ -33,7 +35,7 @@ def test_find_quotas_without_future_definition(self, quota_order_number): ) result = report.find_quotas_without_future_definition( - [expiring_quota_definition] + [expiring_quota_definition], ) assert expiring_quota_definition in result @@ -48,11 +50,11 @@ def test_find_quota_blocking_without_future_definition(self, quota_order_number) ), ) blocking = factories.QuotaBlockingFactory.create( - quota_definition=expiring_quota_definition + quota_definition=expiring_quota_definition, ) result = report.find_quota_blocking_without_future_definition( - [expiring_quota_definition] + [expiring_quota_definition], ) assert expiring_quota_definition in result @@ -68,11 +70,11 @@ def test_find_quota_suspension_without_future_definition(self, quota_order_numbe ), ) suspension = factories.QuotaSuspensionFactory.create( - quota_definition=expiring_quota_definition + quota_definition=expiring_quota_definition, ) result = report.find_quota_suspension_without_future_definition( - [expiring_quota_definition] + [expiring_quota_definition], ) assert expiring_quota_definition in result diff --git a/reports/tests/test_quotas_cannot_be_used_report.py b/reports/tests/test_quotas_cannot_be_used_report.py index 3e904a43b..722c97a78 100644 --- a/reports/tests/test_quotas_cannot_be_used_report.py +++ b/reports/tests/test_quotas_cannot_be_used_report.py @@ -1,14 +1,14 @@ -import pytest -from common.tests import factories -from reports.reports.quotas_cannot_be_used import Report -from common.models.utils import override_current_transaction -from geo_areas.validators import AreaCode import datetime -from common.util import TaricDateRange +import pytest -from quotas.models import QuotaOrderNumberOriginExclusion +from common.models.utils import override_current_transaction +from common.tests import factories +from common.util import TaricDateRange +from geo_areas.validators import AreaCode from measures.models import Measure +from quotas.models import QuotaOrderNumberOriginExclusion +from reports.reports.quotas_cannot_be_used import Report @pytest.fixture @@ -47,7 +47,11 @@ def test_quotas_report_logic( test_function(quota_order_number, approved_transaction, country1, date_ranges) def test_quotas_without_definitions_appear_in_report( - self, quota_order_number, approved_transaction, country1, date_ranges + self, + quota_order_number, + approved_transaction, + country1, + date_ranges, ): with override_current_transaction(approved_transaction): assert quota_order_number.definitions.current().count() == 0 @@ -56,7 +60,11 @@ def test_quotas_without_definitions_appear_in_report( assert Report().query()[0].reason == "Definition period has not been set" def test_quota_definitions_with_no_end_date_appear_in_report( - self, quota_order_number, approved_transaction, country1, date_ranges + self, + quota_order_number, + approved_transaction, + country1, + date_ranges, ): quota_definition = factories.QuotaDefinitionFactory.create( order_number=quota_order_number, @@ -70,10 +78,15 @@ def test_quota_definitions_with_no_end_date_appear_in_report( assert len(Report().query()) == 1 def test_quotas_without_measures_appear_in_report( - self, quota_order_number, approved_transaction, country1, date_ranges + self, + quota_order_number, + approved_transaction, + country1, + date_ranges, ): quota_order_number.valid_between = TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ) assert quota_order_number.valid_between.upper == datetime.date.today() @@ -82,7 +95,8 @@ def test_quotas_without_measures_appear_in_report( transaction=approved_transaction, ) quota_definition.valid_between = TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ) assert quota_definition.valid_between.upper == datetime.date.today() @@ -97,7 +111,11 @@ def test_quotas_without_measures_appear_in_report( assert len(Report().query()) == 1 def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( - self, quota_order_number, approved_transaction, country1, date_ranges + self, + quota_order_number, + approved_transaction, + country1, + date_ranges, ): with override_current_transaction(approved_transaction): quota_order_number.valid_between = date_ranges.normal @@ -113,7 +131,8 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( transaction=approved_transaction, geographical_area=geo_group, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), ) factories.QuotaOrderNumberOriginExclusionFactory.create( @@ -124,7 +143,8 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( geo_membership_no_end_date = factories.GeographicalMembershipFactory.create( valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), geo_group=geo_group, member=country1, @@ -133,7 +153,8 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( measure = factories.MeasureFactory.create( order_number=quota_order_number, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), geographical_area=geo_group, ) @@ -147,7 +168,8 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( factories.QuotaOrderNumberOriginFactory.create( order_number=measure.order_number, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), ) assert quota_order_number.definitions.current().count() == 1 @@ -161,7 +183,7 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( retrieved_geo_exclusion = ( QuotaOrderNumberOriginExclusion.objects.latest_approved().get( - origin_id=origin.pk + origin_id=origin.pk, ) ) assert ( @@ -172,7 +194,11 @@ def test_quotas_with_measure_with_matching_exclusion_do_not_appear_in_report( assert len(Report().query()) == 0 def test_quotas_with_measures_without_matching_exclusion_appear_in_report( - self, quota_order_number, approved_transaction, country1, date_ranges + self, + quota_order_number, + approved_transaction, + country1, + date_ranges, ): with override_current_transaction(approved_transaction): quota_order_number.valid_between = date_ranges.normal @@ -188,7 +214,8 @@ def test_quotas_with_measures_without_matching_exclusion_appear_in_report( transaction=approved_transaction, geographical_area=geo_group, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), ) factories.QuotaOrderNumberOriginExclusionFactory.create( @@ -198,7 +225,8 @@ def test_quotas_with_measures_without_matching_exclusion_appear_in_report( geo_membership_no_end_date = factories.GeographicalMembershipFactory.create( valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), geo_group=geo_group, member=country1, @@ -207,7 +235,8 @@ def test_quotas_with_measures_without_matching_exclusion_appear_in_report( measure = factories.MeasureFactory.create( order_number=quota_order_number, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), geographical_area=geo_group, ) @@ -221,7 +250,8 @@ def test_quotas_with_measures_without_matching_exclusion_appear_in_report( factories.QuotaOrderNumberOriginFactory.create( order_number=measure.order_number, valid_between=TaricDateRange( - datetime.date.today(), datetime.date.today() + datetime.date.today(), + datetime.date.today(), ), ) assert quota_order_number.definitions.current().count() == 1 @@ -235,7 +265,7 @@ def test_quotas_with_measures_without_matching_exclusion_appear_in_report( retrieved_geo_exclusion = ( QuotaOrderNumberOriginExclusion.objects.latest_approved().get( - origin_id=origin.pk + origin_id=origin.pk, ) ) assert ( diff --git a/reports/tests/test_report_utils.py b/reports/tests/test_report_utils.py index 80409a469..27b859a5a 100644 --- a/reports/tests/test_report_utils.py +++ b/reports/tests/test_report_utils.py @@ -63,7 +63,9 @@ def test_link_renderer_for_quotas(self, db): # Test with fragment result = report_instance.link_renderer_for_quotas( - order_number_obj, "Test Text", fragment="#blocking-periods" + order_number_obj, + "Test Text", + fragment="#blocking-periods", ) expected_url = ( reverse("quota-ui-detail", args=[order_number_obj.sid]) diff --git a/reports/tests/test_report_views.py b/reports/tests/test_report_views.py index bbbabb3f1..aa99a0906 100644 --- a/reports/tests/test_report_views.py +++ b/reports/tests/test_report_views.py @@ -1,13 +1,12 @@ # Create your tests here. import pytest from django.urls import reverse -from django.test import RequestFactory -from reports.utils import get_reports -from reports.views import export_report_to_csv, export_report_to_excel -from reports.reports.expiring_quotas_with_no_definition_period import Report from reports.reports.cds_approved import Report as ChartReport - +from reports.reports.expiring_quotas_with_no_definition_period import Report +from reports.utils import get_reports +from reports.views import export_report_to_csv +from reports.views import export_report_to_excel pytestmark = pytest.mark.django_db @@ -59,7 +58,8 @@ def test_export_report_invalid_tab(self, request): invalid_tab = "Invalid tab" with pytest.raises( - ValueError, match=f"Invalid current_tab value: {invalid_tab}" + ValueError, + match=f"Invalid current_tab value: {invalid_tab}", ): export_report_to_csv(request, report_slug, current_tab=invalid_tab) diff --git a/reports/views.py b/reports/views.py index 608a4cd76..3c55119e1 100644 --- a/reports/views.py +++ b/reports/views.py @@ -4,10 +4,10 @@ from django.http import HttpResponse from django.shortcuts import render from openpyxl import Workbook -from openpyxl.chart import BarChart, Reference +from openpyxl.chart import BarChart +from openpyxl.chart import Reference import reports.reports.index as index_model - import reports.utils as utils @@ -110,7 +110,7 @@ def export_report_to_excel(request, report_slug): report_instance = report_class() response = HttpResponse( - content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) response[ diff --git a/settings/dev.py b/settings/dev.py index eaa90893d..5d926465c 100644 --- a/settings/dev.py +++ b/settings/dev.py @@ -51,6 +51,6 @@ ) try: - from settings.dev_override import * + pass except ImportError: pass diff --git a/workbaskets/management/util.py b/workbaskets/management/util.py index ba0e708e6..84f459276 100644 --- a/workbaskets/management/util.py +++ b/workbaskets/management/util.py @@ -15,7 +15,11 @@ def first_line_of(s: str) -> str: class WorkBasketCommandMixin: def _output_workbasket_readable( - self, workbasket, show_transaction_info, indent=4, **kwargs + self, + workbasket, + show_transaction_info, + indent=4, + **kwargs, ): spaces = " " * indent self.stdout.write(f"WorkBasket {workbasket}:") @@ -58,7 +62,9 @@ def output_workbasket( self._output_workbasket_compact(workbasket, show_transaction_info, **kwargs) else: self._output_workbasket_readable( - workbasket, show_transaction_info, **kwargs + workbasket, + show_transaction_info, + **kwargs, ) def output_workbaskets(self, workbaskets, show_transaction_info, output_format): @@ -66,7 +72,8 @@ def output_workbaskets(self, workbaskets, show_transaction_info, output_format): Output a list of workbaskets. :param workbaskets: Sequence of workbaskets to output. - :param output_format: Output format, value of WorkBasketOutputFormat Enum. + :param output_format: Output format, value of WorkBasketOutputFormat + Enum. :param show_transaction_info: Whether to show first / this is slower. """ if output_format == WorkBasketOutputFormat.COMPACT: From 7835b39d1e5ede01d46d3c160fdcb1f7dfaa8548 Mon Sep 17 00:00:00 2001 From: Panagiotis Paralakis Date: Mon, 4 Mar 2024 13:30:55 +0000 Subject: [PATCH 2/6] fix Sphinx docstring format --- common/business_rules.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/business_rules.py b/common/business_rules.py index 03bc65b31..e291aea28 100644 --- a/common/business_rules.py +++ b/common/business_rules.py @@ -132,8 +132,9 @@ def get_linked_models( :param model TrackedModel: Get models linked to this model instance :param transaction Transaction: Get latest approved versions of linked - models as of this transaction :rtype Iterator[TrackedModel]: The - linked models + models as of this transaction + :return: The linked models + :rtype: Iterator[TrackedModel] """ for field, related_model in get_relations(type(model)).items(): business_rules = getattr(related_model, "business_rules", []) @@ -228,8 +229,10 @@ def skip_when_update_type(cls: Type[BusinessRule], update_types: Iterable[Update """ Skip business rule validation for given update types. - :param cls Type[BusinessRule]: The BusinessRule to decorate :param - update_types Iterable[int]: The UpdateTypes to skip + :param cls: The BusinessRule to decorate :param + :type cls: Type[BusinessRule] + :param update_types: The UpdateTypes to skip + :type update_types: Iterable[int] """ _original_validate = cls.validate From 6a7f69e53ac15615f0eb5302e8d163d893b38f61 Mon Sep 17 00:00:00 2001 From: Panagiotis Paralakis Date: Mon, 4 Mar 2024 13:40:25 +0000 Subject: [PATCH 3/6] remove forgotten :param word --- common/business_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/business_rules.py b/common/business_rules.py index e291aea28..e7984008c 100644 --- a/common/business_rules.py +++ b/common/business_rules.py @@ -229,7 +229,7 @@ def skip_when_update_type(cls: Type[BusinessRule], update_types: Iterable[Update """ Skip business rule validation for given update types. - :param cls: The BusinessRule to decorate :param + :param cls: The BusinessRule to decorate :type cls: Type[BusinessRule] :param update_types: The UpdateTypes to skip :type update_types: Iterable[int] From ce2091d94ae88dec91b480e95f3db7891d79c1e0 Mon Sep 17 00:00:00 2001 From: Panagiotis Paralakis Date: Mon, 4 Mar 2024 15:25:22 +0000 Subject: [PATCH 4/6] update node lts version references --- .nvmrc | 2 +- README.rst | 36 ++++++++++++++++++------------------ package-lock.json | 4 ++-- package.json | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.nvmrc b/.nvmrc index 4c89d655c..a3d2332c1 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -16.* \ No newline at end of file +20.11 \ No newline at end of file diff --git a/README.rst b/README.rst index ee16932d0..967ba1508 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,7 @@ Prerequisites The following dependencies are required to run this app: - Python_ 3.8.x -- Node.js_ 16.8.x +- Node.js_ 20.11.* (LTS) Iron - PostgreSQL_ 12 - Redis_ 5.x @@ -71,10 +71,10 @@ Installing $ npm install $ npm run build -Those using Mac m1 laptops may have problems installing certain packages (e.g. -psycopg2 and lxml) via requirements-dev.txt. In this scenario you should run the -following from a rosetta terminal (see `this article -`_ ), +Those using Mac m1 laptops may have problems installing certain packages (e.g. +psycopg2 and lxml) via requirements-dev.txt. In this scenario you should run the +following from a rosetta terminal (see `this article +`_ ), substituting your own python version as appropriate: .. code:: sh @@ -85,7 +85,7 @@ substituting your own python version as appropriate: $ export LDFLAGS="-L/opt/homebrew/opt/openssl@1.1/lib -L${HOME}/.pyenv/versions/3.8.10/lib" $ arch -arm64 pip install psycopg2 --no-binary :all: -Credit due to armenzg and his `answer here +Credit due to armenzg and his `answer here `_ . Running @@ -248,7 +248,7 @@ Build and Run for the first time: # to overwrite default db dump name pass in DB_DUMP=tamato_db.sql $ make docker-first-use # take a tea break to import the db dump then - # enter super user details when prompted + # enter super user details when prompted # and visit localhost:8000/ when the containers are up Run the tamato app every other time: @@ -274,7 +274,7 @@ Import from a dump of the database: # can override the name of the template at TEMPLATE_NAME $ make docker-import-new-db - # Will restore the db DB_NAME with the provided TEMPLATE_NAME + # Will restore the db DB_NAME with the provided TEMPLATE_NAME $ make docker-restore-db Sometimes docker gets clogged up and we need to clean it: @@ -332,13 +332,13 @@ loading environment settings that are specific to this configuration: services: celery: - env_file: + env_file: - .env - settings/envs/docker.env - settings/envs/docker.override.env rule-check-celery: - env_file: + env_file: - .env - settings/envs/docker.env - settings/envs/docker.override.env @@ -424,7 +424,7 @@ Open another terminal and start a Celery worker: .. code:: sh celery -A common.celery worker --loglevel=info -Q standard,rule-check - # The celery worker can be run as two workers for each queue + # The celery worker can be run as two workers for each queue celery -A common.celery worker --loglevel=info -Q standard celery -A common.celery worker --loglevel=info -Q rule-check @@ -467,19 +467,19 @@ Output defaults to stdout if filename is ``-`` or is not supplied. Mocking s3 upload with minio ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -1. Follow `instructions `_ to install minio server +1. Follow `instructions `_ to install minio server 2. Export MINIO_ROOT_USER and MINIO_ROOT_PASSWORD variables of your choice -3. Run server with: +3. Run server with: .. code:: sh - + minio server --quiet --address 0.0.0.0:9003 ~/data -4. Navigate to http://localhost:9003/ and login using root user and password credentials just +4. Navigate to http://localhost:9003/ and login using root user and password credentials just created. Create a bucket and an access key via the console. 5. Export environment variables for any storages you wish to dummy (e.g. for sqlite dump export this will be SQLITE_STORAGE_BUCKET_NAME, SQLITE_S3_ACCESS_KEY_ID, SQLITE_S3_SECRET_ACCESS_KEY, - SQLITE_S3_ENDPOINT_URL, and SQLITE_STORAGE_DIRECTORY), setting s3 endpoint url to + SQLITE_S3_ENDPOINT_URL, and SQLITE_STORAGE_DIRECTORY), setting s3 endpoint url to http://localhost:9003/ 6. Alternatively, export all environment variables temporarily to an environment such as Bash (useful when running a local development instance of a Celery worker): @@ -494,7 +494,7 @@ Virus Scan and running locally We use a shared service accross the department for virus scanning to run locally set up the following: 1. Follow set up `instructions `_ and run it 2. set SKIP_CLAM_AV_FILE_UPLOAD to False and CLAM_USE_HTTP True -3. add CLAM_AV_DOMAIN without http(s):// +3. add CLAM_AV_DOMAIN without http(s):// 4. set CLAM_AV_USERNAME,CLAM_AV_PASSWORD as the username and password found in the config.py in the dit-clamav-rest project @@ -502,7 +502,7 @@ Application maintenance mode ---------------------------- The application can be put into a "maintenance mode" type of operation. By doing -so, all user web access is routed to a maintenance view and the default database +so, all user web access is routed to a maintenance view and the default database route removes the application's access to the database. This prevents inadvertent changes by users, via the application UI, to application data while in maintenance mode. Note, however, that this would not restrict other forms of diff --git a/package-lock.json b/package-lock.json index edbe5738c..4fdec219a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,8 +42,8 @@ "react-test-renderer": "^18.2.0" }, "engines": { - "node": "^20.10.0", - "npm": "^10.3.0" + "node": "^20.11.*", + "npm": "^10.2.*" } }, "node_modules/@adobe/css-tools": { diff --git a/package.json b/package.json index 01a3be7b6..8f83ac0a1 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "repository": "https://github.com/uktrade/tamato.git", "licence": "MIT", "engines": { - "node": "^20.10.0", - "npm": "^10.3.0" + "node": "^20.11.*", + "npm": "^10.2.*" }, "dependencies": { "@babel/core": "^7.23.2", From a5396867e4fa36282e29eb1dd9df4b91563dbbf1 Mon Sep 17 00:00:00 2001 From: Panos Paralakis Date: Tue, 5 Mar 2024 12:04:21 +0000 Subject: [PATCH 5/6] disable autoflake unused import when override dev --- settings/dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/dev.py b/settings/dev.py index 5d926465c..8c275eda1 100644 --- a/settings/dev.py +++ b/settings/dev.py @@ -51,6 +51,6 @@ ) try: - pass + from settings.dev_override import * # pylint: disable=unused-import except ImportError: pass From 3744aecb86b18d4073952b2a349fcd98ebc6937e Mon Sep 17 00:00:00 2001 From: Panos Paralakis Date: Tue, 5 Mar 2024 12:49:57 +0000 Subject: [PATCH 6/6] fix Sphinx docstrings format --- common/business_rules.py | 7 ++++--- exporter/management/commands/upload_transactions.py | 12 +++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/common/business_rules.py b/common/business_rules.py index e7984008c..7b9b0363d 100644 --- a/common/business_rules.py +++ b/common/business_rules.py @@ -170,9 +170,10 @@ def violation( """ Create a violation exception object. - :param model Optional[TrackedModel]: The model that violates this - business rule :param message Optional[str]: A message explaining the - violation + :param model: The model that violates this business rule + :type model: Optional[TrackedModel] + :param message: A message explaining the violation + :type message: Optional[str] :rtype BusinessRuleViolation: An exception indicating a business rule violation """ diff --git a/exporter/management/commands/upload_transactions.py b/exporter/management/commands/upload_transactions.py index d6203a672..14cacc3c4 100644 --- a/exporter/management/commands/upload_transactions.py +++ b/exporter/management/commands/upload_transactions.py @@ -12,14 +12,16 @@ def run_task_or_exit(task, local=False, *args, **kwargs): """ - Run celery task, block then return the return value of the task, or exit.. + Run celery task, block then return the return value of the task, or exit. + + Utility function for management commands that require the ability to start + a celery task, then wait for the result. Tasks may optionally be run + locally, which can be useful if the user needs to run a task, but celery + infrastructure is unavailable. :param local: Run locally (bypass celery) :param task: Celery task to run. - :return: Result of task. Utility function for management commands that - require the ability to start a celery task, then wait for the result. - Tasks may optionally be run locally, which can be useful if the user - needs to run a task, but celery infrastructure is unavailable. + :return: Result of task. """ try: if local: