Skip to content

Commit

Permalink
mypy config to exclude, and test selection via markers
Browse files Browse the repository at this point in the history 
- mypy was configred via `files` which fails when specifying a target dir. Now turned into an exclude in the config, and adding explicit files to the Makefile when a service is selected.
- added SKIP_MYPY to be able to skip mypy check (as long as the mypy chnage is not complete)
- using conftest.py to give all the tests default markers
- the include/exclude from the makefile is now in the conftest.py marking as well
- for services, also picking up test that reference that service in its name (specific pattern)
  • Loading branch information
@hans-d
hans-d committed Mar 7, 2023
1 parent 4b09e3c commit 2ec997c
Show file tree
Hide file tree
Showing 3 changed files with 299 additions and 18 deletions.
36 changes: 19 additions & 17 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
SHELL := /bin/bash

ifdef SERVICE_NAME
TEST_SERVICE = test_${SERVICE_NAME}
TEST_SERVICE = and ${SERVICE_NAME}
TEST_SERVICE_DIR = test_${SERVICE_NAME}
TF_SERVICE_NAME = ${SERVICE_NAME}
MYPY_EXPLICIT_FILES ?= moto/${SERVICE_NAME}/*.py*
endif

TF_SERVICE_NAME ?= "default"
TF_SERVICE_NAME ?= default
TEST_NAMES ?= "*"

# Parallel tests will be run separate
# -m tags are specified in tests/conftest.py, or on individual files/tests
ifeq ($(TEST_SERVER_MODE), true)
# exclude test_kinesisvideoarchivedmedia
# because testing with moto_server is difficult with data-endpoint
TEST_EXCLUDE := --ignore tests/test_kinesisvideoarchivedmedia --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
# Parallel tests will be run separate
PARALLEL_TESTS := ./tests/test_acm/ ./tests/test_acmpca/ ./tests/test_amp/ ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs
TEST_EXCLUDE := -m "not parallel and not parallel_server_only and not skip_server ${TEST_SERVICE}"
PARALLEL_TESTS := -m "parallel or parallel_server_only ${TEST_SERVICE}"
else
TEST_EXCLUDE := --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
PARALLEL_TESTS := ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs
TEST_EXCLUDE := -m "not parallel ${TEST_SERVICE}"
PARALLEL_TESTS := -m "parallel ${TEST_SERVICE}"
endif

init:
Expand All @@ -25,29 +26,30 @@ init:

lint:
@echo "Running flake8..."
flake8 moto/${SERVICE_NAME} tests/${TEST_SERVICE}
flake8 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
@echo "Running black... "
$(eval black_version := $(shell grep "^black==" requirements-dev.txt | sed "s/black==//"))
@echo "(Make sure you have black-$(black_version) installed, as other versions will produce different results)"
black --check moto/${SERVICE_NAME} tests/${TEST_SERVICE}
black --check moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
@echo "Running pylint..."
pylint -j 0 moto/${SERVICE_NAME} tests/${TEST_SERVICE}
pylint -j 0 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
@echo "Running MyPy..."
mypy --install-types --non-interactive moto/${SERVICE_NAME}
if [[ "${SKIP_MYPY}" == "" ]]; then mypy --install-types --non-interactive ${MYPY_EXPLICIT_FILES}; else echo "Skipping"; fi

format:
black moto/ tests/

test-only:
rm -f .coverage
rm -rf cover
pytest -sv --cov=moto --cov-report xml ./tests/${TEST_SERVICE} $(TEST_EXCLUDE)
pytest -sv --cov=moto --cov-report xml ./tests $(TEST_EXCLUDE) ${PYTEST_ARGS}
# https://github.com/aws/aws-xray-sdk-python/issues/196 - Run these tests separately without Coverage enabled
if [[ "${SERVICE_NAME}" == "" || "${SERVICE_NAME}" == "xray" ]]; then pytest -sv ./tests/test_xray; else echo "Skipping"; fi
if [[ "$(filter %${SERVICE_NAME},$(PARALLEL_TESTS))" != "" ]]; then MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(filter %${SERVICE_NAME},$(PARALLEL_TESTS)); else echo "Skipping"; fi
if [[ "${SERVICE_NAME}" == "" || "${SERVICE_NAME}" == "xray" ]]; then pytest -sv ./tests/test_xray ${PYTEST_ARGS}; else echo "Skipping"; fi
MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(PARALLEL_TESTS) ${PYTEST_ARGS}

test: lint test-only
@echo "USAGE: make test [SERVICE_NAME=s3] (defaults to all)"
@echo "USAGE: make test [SERVICE_NAME=s3] [SKIP_MYPY=true]"
@echo " defaults to all services and running mypy (not all services are mypy proof, see also setup.cfg)"

terraformtests:
@echo "Make sure that the MotoServer is already running on port 4566 (moto_server -p 4566)"
Expand Down
132 changes: 131 additions & 1 deletion setup.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,130 @@ universal=1

[tool:pytest]
markers =
# special cases
network: marks tests which require network connection
parallel: can be run in parallel
parallel_server_only: can be run in parallel (in server mode only)
skip_server: should not be run in server mode
# services
acm: acm service
acmpca: acmpca service
amp: amp service
apigateway: apigateway service
apigatewayv2: apigatewayv2 service
applicationautoscaling: applicationautoscaling service
appsync: appsync service
athena: athena service
autoscaling: autoscaling service
awslambda: awslambda service
batch: batch service
batch_simple: batch_simple service
budgets: budgets service
ce: ce service
cloudformation: cloudformation service
cloudfront: cloudfront service
cloudtrail: cloudtrail service
cloudwatch: cloudwatch service
codebuild: codebuild service
codecommit: codecommit service
codepipeline: codepipeline service
cognitoidentity: cognitoidentity service
cognitoidp: cognitoidp service
comprehend: comprehend service
config: config service
databrew: databrew service
datapipeline: datapipeline service
datasync: datasync service
dax: dax service
dms: dms service
ds: ds service
dynamodb: dynamodb service
dynamodb_v20111205: dynamodb_v20111205 service
dynamodbstreams: dynamodbstreams service
ebs: ebs service
ec2: ec2 service
ec2instanceconnect: ec2instanceconnect service
ecr: ecr service
ecs: ecs service
efs: efs service
eks: eks service
elasticache: elasticache service
elasticbeanstalk: elasticbeanstalk service
elastictranscoder: elastictranscoder service
elb: elb service
elbv2: elbv2 service
emr: emr service
emrcontainers: emrcontainers service
emrserverless: emrserverless service
es: es service
events: events service
firehose: firehose service
forecast: forecast service
glacier: glacier service
glue: glue service
greengrass: greengrass service
guardduty: guardduty service
iam: iam service
identitystore: identitystore service
instance_metadata: instance_metadata service
iot: iot service
iotdata: iotdata service
kinesis: kinesis service
kinesisvideo: kinesisvideo service
kinesisvideoarchivedmedia: kinesisvideoarchivedmedia service
kms: kms service
logs: logs service
managedblockchain: managedblockchain service
mediaconnect: mediaconnect service
medialive: medialive service
mediapackage: mediapackage service
mediastore: mediastore service
mediastoredata: mediastoredata service
meteringmarketplace: meteringmarketplace service
moto_api: moto_api service
moto_server: moto_server service
mq: mq service
neptune: neptune service
opsworks: opsworks service
organizations: organizations service
packages: packages service
personalize: personalize service
pinpoint: pinpoint service
polly: polly service
quicksight: quicksight service
ram: ram service
rds: rds service
redshift: redshift service
redshiftdata: redshiftdata service
rekognition: rekognition service
resourcegroups: resourcegroups service
resourcegroupstaggingapi: resourcegroupstaggingapi service
route53: route53 service
route53resolver: route53resolver service
s3: s3 service
s3bucket_path: s3bucket_path service
s3control: s3control service
sagemaker: sagemaker service
sdb: sdb service
secretsmanager: secretsmanager service
servicediscovery: servicediscovery service
servicequotas: servicequotas service
ses: ses service
signer: signer service
sns: sns service
sqs: sqs service
ssm: ssm service
ssoadmin: ssoadmin service
stepfunctions: stepfunctions service
sts: sts service
support: support service
swf: swf service
textract: textract service
timestreamwrite: timestreamwrite service
transcribe: transcribe service
utilities: utilities service
wafv2: wafv2 service
xray: xray service

[coverage:run]
relative_files = True
Expand All @@ -229,7 +352,14 @@ disable = W,C,R,E
enable = anomalous-backslash-in-string, arguments-renamed, dangerous-default-value, deprecated-module, function-redefined, import-self, redefined-builtin, redefined-outer-name, reimported, pointless-statement, super-with-arguments, unused-argument, unused-import, unused-variable, useless-else-on-loop, wildcard-import

[mypy]
files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/f*,moto/moto_api,moto/neptune
files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/moto_api,moto/neptune
exclude = (?x)(
^moto/[g-l]
| ^moto/m(?!oto_api) # all m*, except moto_api
| ^moto/n(?!eptune) # all n*, except neptune
| ^moto/[o-z]
)

show_column_numbers=True
show_error_codes = True
disable_error_code=abstract
Expand Down
149 changes: 149 additions & 0 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import pytest

SERVICES = (
"acm",
"acmpca",
"amp",
"apigateway",
"apigatewayv2",
"applicationautoscaling",
"appsync",
"athena",
"autoscaling",
"awslambda",
"batch",
"batch_simple",
"budgets",
"ce",
"cloudformation",
"cloudfront",
"cloudtrail",
"cloudwatch",
"codebuild",
"codecommit",
"codepipeline",
"cognitoidentity",
"cognitoidp",
"comprehend",
"config",
"databrew",
"datapipeline",
"datasync",
"dax",
"dms",
"ds",
"dynamodb",
"dynamodb_v20111205",
"dynamodbstreams",
"ebs",
"ec2",
"ec2instanceconnect",
"ecr",
"ecs",
"efs",
"eks",
"elasticache",
"elasticbeanstalk",
"elastictranscoder",
"elb",
"elbv2",
"emr",
"emrcontainers",
"emrserverless",
"es",
"events",
"firehose",
"forecast",
"glacier",
"glue",
"greengrass",
"guardduty",
"iam",
"identitystore",
"instance_metadata",
"iot",
"iotdata",
"kinesis",
"kinesisvideo",
"kinesisvideoarchivedmedia",
"kms",
"logs",
"managedblockchain",
"mediaconnect",
"medialive",
"mediapackage",
"mediastore",
"mediastoredata",
"meteringmarketplace",
"moto_api",
"moto_server",
"mq",
"neptune",
"opsworks",
"organizations",
"packages",
"personalize",
"pinpoint",
"polly",
"quicksight",
"ram",
"rds",
"redshift",
"redshiftdata",
"rekognition",
"resourcegroups",
"resourcegroupstaggingapi",
"route53",
"route53resolver",
"s3",
"s3bucket_path",
"s3control",
"sagemaker",
"sdb",
"secretsmanager",
"servicediscovery",
"servicequotas",
"ses",
"signer",
"sns",
"sqs",
"ssm",
"ssoadmin",
"stepfunctions",
"sts",
"support",
"swf",
"textract",
"timestreamwrite",
"transcribe",
"utilities",
"wafv2",
"xray",
)


EXTRA_MARKERS_TEST_DIRS = {
"acm": (pytest.mark.parallel_server_only,),
"acmpca": (pytest.mark.parallel_server_only,),
"amp": (pytest.mark.parallel_server_only,),
"awslambda": (pytest.mark.parallel_server_only,),
"batch": (pytest.mark.parallel,),
"ec2": (pytest.mark.parallel,),
# exclude test_kinesisvideoarchivedmedia
# because testing with moto_server is difficult with data-endpoint
"kinesisvideoarchivedmedia": (pytest.mark.skip_server,),
"sqs": (pytest.mark.parallel,),
}


def pytest_collection_modifyitems(items): # noqa: SC200
for item in items:
for service, markers in EXTRA_MARKERS_TEST_DIRS.items():
if f"tests/test_{service}/" in item.nodeid: # noqa: SC200
for marker in markers:
item.add_marker(marker)
for service in SERVICES:
if f"tests/test_{service}/" in item.nodeid: # noqa: SC200
item.add_marker(getattr(pytest.mark, service))
if f"_{service}.py" in item.nodeid:
item.add_marker(getattr(pytest.mark, service))

0 comments on commit 2ec997c

Please sign in to comment.