diff --git a/.gitignore b/.gitignore index 3425e8d..4c581c4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ myimage.yaml *.pyc .version .env -.vscode \ No newline at end of file +.vscode +.idea/* +.idea/inspectionProfiles/* diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d8772..773c3ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [3.16.0] - 2024-06-26 +### Added +- CASMCMS-8915: IMS API features for tagging built images. +### Dependencies +- CASMCMS-8022 - update python dependencies to most recent versions. +| Package | From | To | +|-----------------------|------------|----------| +| `aniso8601` | 3.0.2 | 9.0.1 | +| `boto3` | 1.12.49 | 1.34.114 | +| `botocore` | 1.15.49 | 1.34.114 | +| `cachetools` | 3.0.0 | 5.3.3 | +| `certifi` | 2019.11.28 | 2024.2.2 | +| `chardet` | 3.0.4 | 5.2.0 | +| `click` | 6.7 | 8.1.7 | +| `docutils` | 0.14 | 0.21.2 | +| `Flask` | 1.1.4 | 3.0.3 | +| `flask-marshmallow` | 0.9.0 | 1.2.1 | +| `google-auth` | 1.6.3 | 2.29.0 | +| `gunicorn` | 19.10.0 | 22.0.0 | +| `idna` | 2.8 | 3.7 | +| `itsdangerous` | 0.24 | 2.2.0 | +| `Jinja2` | 2.10.3 | 3.1.4 | +| `jmespath` | 0.9.5 | 1.0.1 | +| `MarkupSafe` | 1.1.1 | 2.1.5 | +| `marshmallow` | 3.0.0b16 | 3.21.2 | +| `oauthlib` | 2.1.0 | 3.2.2 | +| `pyasn1` | 0.4.8 | 0.6.0 | +| `pyasn1-modules` | 0.2.8 | 0.4.0 | +| `pytz` | 2018.4 | 2024.1 | +| `requests` | 2.23.0 | 2.31.0 | +| `requests-oauthlib` | 1.0.0 | 1.3.1 | +| `rsa` | 4.7.2 | 4.9 | +| `s3transfer` | 0.3.7 | 0.10.1 | +| `urllib3` | 1.25.11 | 1.26.18 | +| `websocket-client` | 0.54.0 | 1.8.0 | +| `Werkzeug` | 0.15.6 | 3.0.3 | ## [3.15.0] - 2024-05-20 ### Dependencies diff --git a/README.md b/README.md index cedef62..7c32743 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ before trying to build: ``` $ docker login artifactory.algol60.net ``` -See more information on authetication here: +See more information on authentication here: https://rndwiki-pro.its.hpecorp.net/display/CSMTemp/Client+Authentication#ClientAuthentication-SecurityConsiderations ## Running Locally @@ -206,6 +206,8 @@ This will start the IMS server on `http://localhost:9100`. An S3 instance is required for the IMS server to do anything meaningful. See the [Configuration Options](#Configuration-Options) section for more information and further configuration possibilities. +Fetch recipes and images: + ``` $ curl http://127.0.0.1:9100/images [] @@ -213,9 +215,38 @@ $ curl http://127.0.0.1:9100/recipes [] ``` +Add a public key: + +``` +$ curl http://127.0.0.1:9100/public-keys -X POST -H "Content-Type: application/json" \ + --data '{"name":"test","public_key":"TEST_KEY_DATA"}' +``` + +Get a recipe description: + +``` +$ curl http://127.0.0.1:9100/recipes/RECIPE_ID +``` + +Patch an image record: + +``` +curl http://127.0.0.1:9100/images/IMAGE_ID -X PATCH -H "Content-Type: application/json" \ + --data '{"platform":"aarch64"}' +``` + +Create a job: + +``` +curl http://127.0.0.1:9100/jobs -X POST -H "Content-Type: application/json" \ + --data '{ "job_type":"create","require_dkms":"False","image_root_archive_name":"Test","artifact_id":"RECIPE_ID", \ + "public_key_id":"PUBLIC_KEY_ID"}' +``` + **NOTE:** To successfully post jobs to the IMS Jobs endpoint, you must be running under kubernetes as the IMS Service tries to launch a new K8S job. This is not expected to -work when running the IMS Service locally under Docker. +work when running the IMS Service locally under Docker. However running locally you +can watch the logs and verify the request posts correctly. ## Testing @@ -228,6 +259,47 @@ $ ./runCodeStyleCheck.sh $ ./runLint.sh ``` +### Setting up PyTest + +PyTest runs the unit tests directly, not inside a container. This means that it needs +some additional changes to get the correct configuration to run locally. + +The following are required beyond the development system setup: +* The Python requirements in requirements-test.txt + +1. Activate the virtual environment created for development + + ``` + $ . .env/bin/activate + ``` + +1. Install required python modules + + ``` + $ pip3 install -r requirements-test.txt + ``` + +1. Create pytest.ini file with env vars + + In the parent directory that IMS is cloned into, create the file 'pytest.ini' + and put the following contents in it: + + ``` + [pytest] + env = + FLASK_ENV=development + ``` + +1. Create a directory for the IMS data files + + This is the directory that the unit tests will use to create the + data file for IMS object records. + + ``` + cd ~ + mkdir -p ims/data + ``` + ### CT Tests See cms-tools repo for details on running CT tests for this service. diff --git a/api/openapi.yaml b/api/openapi.yaml index 9809cc4..c9cf074 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -2245,6 +2245,55 @@ components: type: array items: $ref: '#/components/schemas/RecipeKeyValuePair' + ImageMetadataAnnotationKeyValuePair: + description: Key/value pair used to further define an Image + type: object + required: + - key + - value + properties: + key: + description: Template variable to associate with the IMS image + example: includes_additional_packages + type: string + value: + description: Value variable to associate with the IMS image + example: "foo,bar,baz" + type: string + ImagePatchRecord: + description: Values used to update an existing IMS Image Record + type: object + properties: + link: + $ref: '#/components/schemas/ArtifactLinkRecord' + arch: + description: Target architecture for the recipe. + example: aarch64 + enum: + - aarch64 + - x86_64 + type: string + metadata: + description: An object which indicates a number of annotation patch operations relating to image tags. + type: object + required: + - operation + - key + properties: + operation: + description: How to update a given key within the context of a patch operation + type: string + enum: + - set + - remove + key: + description: The key to update for a given image + type: string + example: includes_additional_packages + value: + description: The value to associate with a key during a patch operation + type: string + example: "vim,emacs,man" ImageRecord: description: An Image Record type: object @@ -2277,6 +2326,11 @@ components: - aarch64 - x86_64 type: string + metadata: + description: An object of key/value associated with an Image + type: object + properties: + $ref: '#/components/schemas/ImageMetadataAnnotationKeyValuePair' DeletedImageRecord: description: A Deleted Image Record type: object @@ -2316,19 +2370,11 @@ components: - aarch64 - x86_64 type: string - ImagePatchRecord: - description: Values to update an ImageRecord with - type: object - properties: - link: - $ref: '#/components/schemas/ArtifactLinkRecord' - arch: - description: Target architecture for the recipe. - example: aarch64 - enum: - - aarch64 - - x86_64 - type: string + metadata: + description: List of key/value pairs to associate with an Image + type: object + properties: + $ref: '#/components/schemas/ImageMetadataAnnotationKeyValuePair' JobRecord: description: A Job Record type: object diff --git a/constraints.txt b/constraints.txt index bb5ec03..8a0e880 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,37 +1,38 @@ -aniso8601==3.0.2 -boto3==1.12.49 -botocore==1.15.49 -cachetools==3.0.0 -certifi==2019.11.28 -chardet==3.0.4 -click==6.7 -docutils==0.14 +# Constraints updated 5/29/2024 +aniso8601==9.0.1 +boto3==1.34.114 +botocore==1.34.114 +cachetools==5.3.3 +certifi==2024.2.2 +chardet==5.2.0 +click==8.1.7 +docutils==0.21.2 fabric==3.2.2 -Flask==1.1.4 -flask-marshmallow==0.9.0 +Flask==3.0.3 +flask-marshmallow==1.2.1 Flask-RESTful==0.3.10 -google-auth==1.6.3 -gunicorn==19.10.0 +google-auth==2.29.0 +gunicorn==22.0.0 httpproblem==0.2.0 -idna==2.8 -itsdangerous==0.24 -Jinja2==2.10.3 -jmespath==0.9.5 +idna==3.7 +itsdangerous==2.2.0 +Jinja2==3.1.4 +jmespath==1.0.1 # CSM 1.6 uses Kubernetes 1.22, so use client v22.x to ensure compatability kubernetes==22.6.0 -MarkupSafe==1.1.1 -marshmallow==3.0.0b16 -oauthlib==2.1.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pytest==8.1.1 +MarkupSafe==2.1.5 +marshmallow==3.21.2 +oauthlib==3.2.2 +pyasn1==0.6.0 # most recent: 1.6.1, pyasn1-modules 0.4.0 requires <0.7.0 +pyasn1-modules==0.4.0 python-dateutil==2.8.2 -pytz==2018.4 +pytest==8.1.1 +pytz==2024.1 PyYAML==6.0.1 -requests==2.23.0 -requests-oauthlib==1.0.0 -rsa==4.7.2 -s3transfer==0.3.7 -urllib3==1.25.11 -websocket-client==0.54.0 -Werkzeug==0.15.6 +requests==2.31.0 +requests-oauthlib==1.3.1 +rsa==4.9 +s3transfer==0.10.1 +urllib3==1.26.18 # most recent 2.2.1, botocore 1.34.114 requires <1.27.0 +websocket-client==1.8.0 +Werkzeug==3.0.3 diff --git a/requirements-test.txt b/requirements-test.txt index ff41b36..98ced5f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -5,6 +5,7 @@ testtools fixtures pytest pytest-cov +pytest-env pycodestyle pylint mock diff --git a/src/server/__init__.py b/src/server/__init__.py index c1be95b..01ee5a5 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -29,6 +29,7 @@ # TODO CASMCMS-1154 Get a real data store import os import os.path +from marshmallow import EXCLUDE class DataStoreHACK(collections.abc.MutableMapping): """ A dictionary that reads/writes to a file """ @@ -51,7 +52,7 @@ def _read(self): # Setting 'unknown="Exclude" allows downgrades by just dropping any data # fields that are no longer part of the current schema. with open(self.store_file, 'r') as data_file: - obj_data = self.schema.loads(data_file.read(), many=True, unknown="EXCLUDE") + obj_data = self.schema.loads(data_file.read(), many=True, unknown=EXCLUDE) self.store = {str(getattr(obj, self.key_field)): obj for obj in obj_data} def _write(self): diff --git a/src/server/models/__init__.py b/src/server/models/__init__.py index 18b3572..f423023 100644 --- a/src/server/models/__init__.py +++ b/src/server/models/__init__.py @@ -29,9 +29,10 @@ class ArtifactLink(Schema): """ A schema specifically for validating artifact links """ - path = fields.Str(required=True, description="URL or path to the artifact", - validate=Length(min=1, error="name field must not be blank")) - etag = fields.Str(required=False, default="", description="Artifact entity tag") + path = fields.Str(required=True, validate=Length(min=1, error="name field must not be blank"), + metadata={"metadata": {"description": "URL or path to the artifact"}}) + etag = fields.Str(required=False, dump_default="", load_default="", + metadata={"metadata": {"description": "Artifact entity tag"}}) type = fields.Str(required=True, allow_none=False, - description="The type of artifact link", + metadata={"metadata": {"description": "The type of artifact link"}}, validate=OneOf(ARTIFACT_LINK_TYPES, error="Type must be one of: {choices}.")) diff --git a/src/server/models/images.py b/src/server/models/images.py index eaeaf39..3d8fc99 100644 --- a/src/server/models/images.py +++ b/src/server/models/images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2018-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2018-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -34,15 +34,17 @@ from src.server.models import ArtifactLink from src.server.helper import ARCH_X86_64, ARCH_ARM64 + class V2ImageRecord: """ The ImageRecord object """ # pylint: disable=W0622 - def __init__(self, name, link=None, id=None, created=None, arch=ARCH_X86_64): + def __init__(self, name, link=None, id=None, created=None, arch=ARCH_X86_64, metadata=None): # Supplied self.name = name self.link = link - + self.metadata = metadata if metadata else {} + # v2.1 self.arch = arch @@ -56,15 +58,20 @@ def __repr__(self): class V2ImageRecordInputSchema(Schema): """ A schema specifically for defining and validating user input """ - name = fields.Str(required=True, description="the name of the image", - validate=Length(min=1, error="name field must not be blank")) + name = fields.Str(required=True, validate=Length(min=1, error="name field must not be blank"), + metadata={"metadata": {"description": "Name of the image"}}) link = fields.Nested(ArtifactLink, required=False, allow_none=True, - description="the location of the image manifest") - arch = fields.Str(required=False, default = ARCH_X86_64, description="Architecture of the image", - validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=True, dump_default=True) + metadata={"metadata": {"description": "Location of the image manifest"}}) + arch = fields.Str(required=False, validate=OneOf([ARCH_ARM64, ARCH_X86_64]), + load_default=ARCH_X86_64, dump_default=ARCH_X86_64, + metadata={"metadata": {"description": "Architecture of the image"}}) + metadata = fields.Mapping(keys=fields.Str(required=True), + values=fields.Str(required=False, dump_default='', load_default=''), + metadata={"metadata": {"description": "User supplied additional information about an image"}}, + dump_default={}, load_default={}) @post_load - def make_image(self, data): + def make_image(self, data, many, partial): """ Marshall an object out of the individual data components """ return V2ImageRecord(**data) @@ -79,16 +86,28 @@ class V2ImageRecordSchema(V2ImageRecordInputSchema): read in from a database. Builds upon the basic input fields in ImageRecordInputSchema. """ - id = fields.UUID(description="the unique id of the image") - created = fields.DateTime(description="the time the image record was created") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the image"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the image record was created"}}) + + +class V2ImageRecordMetadataPatchSchema(Schema): + operation = fields.Str(required=True, + metadata={"metadata": {"description": "A method for how to change a metadata struct."}}, + validate=OneOf(['set', 'remove'])) + key = fields.Str(required=True, metadata={"metadata": {"description":"The metadata key that is to be affected."}}) + value = fields.Str(required=False, metadata={"metadata": {"description":"The value to store for the provided key."}}) class V2ImageRecordPatchSchema(Schema): """ - Schema for a updating an ImageRecord object. + Schema for updating an ImageRecord object. """ link = fields.Nested(ArtifactLink, required=False, allow_none=False, - description="the location of the image manifest") - arch = fields.Str(required=False, description="Architecture of the recipe", default=ARCH_X86_64, - validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=True, dump_default=True) + metadata={"metadata": {"description": "Location of the image manifest"}}) + arch = fields.Str(required=False, validate=OneOf([ARCH_ARM64, ARCH_X86_64]), + load_default=ARCH_X86_64, dump_default=ARCH_X86_64, + metadata={"metadata": {"description": "Architecture of the recipe"}}) + metadata = fields.Nested(V2ImageRecordMetadataPatchSchema(), + required=False, + metadata={"metadata": {"description": "An action to set or remove image metadata information."}}) diff --git a/src/server/models/jobs.py b/src/server/models/jobs.py index 53a38c3..8a0642f 100644 --- a/src/server/models/jobs.py +++ b/src/server/models/jobs.py @@ -128,56 +128,50 @@ def __repr__(self): class SshContainerInputSchema(Schema): """ A schema specifically for defining and validating user input of SSH Containers """ - name = fields.String(description="SSH Container name") - jail = fields.Boolean(description="Whether to use an SSH jail to restrict access to the image root. " - "Default = False", default=False) + name = fields.String(metadata={"metadata": {"description": "SSH Container name"}}) + jail = fields.Boolean(metadata={"metadata": {"description": "Whether to use an SSH jail to restrict access to the image root. Default = False"}}, + load_default=False, dump_default=False) class V2JobRecordInputSchema(Schema): """ A schema specifically for defining and validating user input of Job requests """ artifact_id = fields.UUID(required=True, - description="IMS record id (either recipe or image depending on job_type)" - "for the source artifact") - public_key_id = fields.UUID(required=True, - description="IMS record id for the public_key record to use.") - job_type = fields.Str(required=True, - description="The type of job, either 'create' or 'customize'", + metadata={"metadata": {"description": "IMS record id (either recipe or image depending on job_type) for the source artifact"}},) + public_key_id = fields.UUID(required=True,metadata={"metadata": {"description": "IMS record id for the public_key record to use."}}) + job_type = fields.Str(required=True,metadata={"metadata": {"description": "The type of job, either 'create' or 'customize'"}}, validate=OneOf(JOB_TYPES, error="Job type must be one of: {choices}.")) - image_root_archive_name = fields.Str(required=True, - description="Name to be given to the image root artifact", - validate=Length(min=1, - error="image_root_archive_name field must not be blank")) - enable_debug = fields.Boolean(default=False, - Description="Whether to enable debugging of the job") - build_env_size = fields.Integer(Default=15, - Description="approximate disk size in GiB to reserve for the image build" - "environment (usually 2x final image size)", + image_root_archive_name = fields.Str(required=True, metadata={"metadata": {"description": "Name to be given to the image root artifact"}}, + validate=Length(min=1, error="image_root_archive_name field must not be blank")) + enable_debug = fields.Boolean(load_default=False,dump_default=False, + metadata={"metadata": {"description": "Whether to enable debugging of the job"}}) + build_env_size = fields.Integer(load_default=15,dump_default=15, + metadata={"metadata": {"description": "Approximate disk size in GiB to reserve for the image build environment (usually 2x final image size)"}}, validate=Range(min=1, error="build_env_size must be greater than or equal to 1")) - kernel_file_name = fields.Str(description="Name of the kernel file to extract and upload") + kernel_file_name = fields.Str(metadata={"metadata": {"description": "Name of the kernel file to extract and upload"}}) - initrd_file_name = fields.Str(default="initrd", - description="Name of the initrd file to extract and upload", + initrd_file_name = fields.Str(load_default="initrd", dump_default="initrd", + metadata={"metadata": {"description": "Name of the initrd file to extract and upload"}}, validate=Length(min=1, error="initrd_file_name field must not be blank")) kernel_parameters_file_name = \ - fields.Str(default="kernel-parameters", - description="Name of the kernel parameters file to extract and upload", + fields.Str(load_default="kernel-parameters", dump_default="kernel-parameters", + metadata={"metadata": {"description": "Name of the kernel parameters file to extract and upload"}}, validate=Length(min=1, error="kernel_parameters_file_name field must not be blank")) ssh_containers = fields.List(fields.Nested(SshContainerInputSchema()), allow_none=True) # v2.1 - require_dkms = fields.Boolean(required=False, default=False, load_default=True, dump_default=True, - description="Job requires the use of dkms") + require_dkms = fields.Boolean(required=False, load_default=False, dump_default=False, + metadata={"metadata": {"description": "Job requires the use of dkms"}}) # v2.2 - job_mem_size = fields.Integer(Default=1, - Description="approximate working memory in GiB to reserve for the build job " - "environment (loosely proporational to the final image size)", - validate=Range(min=1, error="build_env_size must be greater than or equal to 1")) + job_mem_size = fields.Integer(load_default=1, dump_default=1, + validate=Range(min=1, error="build_env_size must be greater than or equal to 1"), + metadata={"metadata": {"description": "Approximate working memory in GiB to reserve for the build job " + "environment (loosely proportional to the final image size)"}}) @post_load - def make_job(self, data): + def make_job(self, data, many, partial): """ Marshall an object out of the individual data components """ return V2JobRecord(**data) @@ -188,16 +182,16 @@ class Meta: # pylint: disable=missing-docstring class SshConnectionInfo(Schema): """ A schema specifically for validating SSH Container Connection Info """ - host = fields.String(description="Host ip or name to use to connect to the SSH container") - port = fields.Integer(description="Port number to use to connect to the SSH container") + host = fields.String(metadata={"metadata": {"description": "Host ip or name to use to connect to the SSH container"}}) + port = fields.Integer(metadata={"metadata": {"description": "Port number to use to connect to the SSH container"}}) class SshContainerSchema(SshContainerInputSchema): """ A schema specifically for validating SSH Containers """ - status = fields.String(description="SSH Container Status") - host = fields.String(description="Host ip or name to use to connect to the SSH container") - port = fields.Integer(description="Port number to use to connect to the SSH container") - connection_info = fields.Dict(key=fields.Str(), values=fields.Nested(SshConnectionInfo)) + status = fields.String(metadata={"metadata": {"description": "SSH Container Status"}}) + host = fields.String(metadata={"metadata": {"description": "Host ip or name to use to connect to the SSH container"}}) + port = fields.Integer(metadata={"metadata": {"description": "Port number to use to connect to the SSH container"}}) + connection_info = fields.Dict(keys=fields.Str(), values=fields.Nested(SshConnectionInfo)) class Meta: # pylint: disable=missing-docstring # host and port have been deprecated and are no longer used. @@ -210,38 +204,38 @@ class V2JobRecordSchema(V2JobRecordInputSchema): read in from a database. Builds upon the basic input fields in JobRecordInputSchema. """ - id = fields.UUID(description="the unique id of the job") - created = fields.DateTime(description="the time the job record was created") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the job"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the job record was created"}}) kubernetes_job = fields.Str(allow_none=True, - description="Job name for the underlying Kubernetes job") + metadata={"metadata": {"description": "Job name for the underlying Kubernetes job"}}) kubernetes_service = fields.Str(allow_none=True, - description="Service name for the underlying Kubernetes service") + metadata={"metadata": {"description": "Service name for the underlying Kubernetes service"}}) kubernetes_configmap = fields.Str(allow_none=True, - description="ConfigMap name for the underlying Kubernetes configmap") - kubernetes_namespace = fields.Str(allow_none=True, default="default", - description="Kubernetes namespace where the IMS job resources were created") - status = fields.Str(allow_none=False, - description="state of the job request", + metadata={"metadata": {"description": "ConfigMap name for the underlying Kubernetes configmap"}}) + kubernetes_namespace = fields.Str(allow_none=True, load_default="default", dump_default="default", + metadata={"metadata": {"description": "Kubernetes namespace where the IMS job resources were created"}}) + status = fields.Str(allow_none=False,metadata={"metadata": {"description": "State of the job request"}}, validate=OneOf(STATUS_TYPES, error="Job state must be one of: {choices}.")) resultant_image_id = fields.UUID(allow_none=True, - description="the unique id of the resultant image record") + metadata={"metadata": {"description": "Unique id of the resultant image record"}}) ssh_containers = fields.List(fields.Nested(SshContainerSchema()), allow_none=True) # v2.1 - arch = fields.Str(description="Architecture of the job", default=ARCH_X86_64, - validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=True, dump_default=True) + arch = fields.Str(metadata={"metadata": {"description": "Architecture of the job"}}, + validate=OneOf([ARCH_ARM64,ARCH_X86_64]), + load_default=ARCH_X86_64, dump_default=ARCH_X86_64) # v2.2 kubernetes_pvc = fields.Str(allow_none=True, - description="PVC name for the underlying Kubernetes image pvc") + metadata={"metadata": {"description": "PVC name for the underlying Kubernetes image pvc"}}) # v2.3 - remote_build_node = fields.Str(allow_none=False, default="", - description="XName of remote job if running on a remote node") + remote_build_node = fields.Str(allow_none=False, load_default="", dump_default="", + metadata={"metadata": {"description": "XName of remote job if running on a remote node"}}) # after reading in the data, make sure there is an arch defined - default to x86 @post_load(pass_original=False) - def fill_arch(self, data, **kwargs): + def fill_arch(self, data, many, partial, **kwargs): if not "arch" in data or data["arch"] is None: data["arch"] = ARCH_X86_64 return data @@ -250,11 +244,10 @@ class V2JobRecordPatchSchema(Schema): """ Schema for a updating a JobRecord object. """ - status = fields.Str(required=False, - description="state of the job request", + status = fields.Str(required=False,metadata={"metadata": {"description": "State of the job request"}}, validate=OneOf(STATUS_TYPES, error="Job state must be one of: {choices}.")) resultant_image_id = fields.UUID(required=False, - description="the unique id of the resultant image record") + metadata={"metadata": {"description": "Unique id of the resultant image record"}}) #NOTE: this can't live in helper.py due to a circular dependency def find_remote_node_for_job(app, job: V2JobRecordSchema) -> str: diff --git a/src/server/models/publickeys.py b/src/server/models/publickeys.py index 27a6bad..73bd8c8 100644 --- a/src/server/models/publickeys.py +++ b/src/server/models/publickeys.py @@ -51,13 +51,13 @@ def __repr__(self): class V2PublicKeyRecordInputSchema(Schema): """ A schema specifically for defining and validating user input """ - name = fields.Str(required=True, description="A name to identify the public key", - validate=Length(min=1, error="name field must not be blank")) - public_key = fields.Str(required=True, description="The raw public key file contents", - validate=Length(min=1, error="public_key field must not be blank")) + name = fields.Str(required=True, validate=Length(min=1, error="name field must not be blank"), + metadata={"metadata": {"description": "Name to identify the public key"}}) + public_key = fields.Str(required=True, validate=Length(min=1, error="public_key field must not be blank"), + metadata={"metadata": {"description": "Raw public key file contents"}}) @post_load - def make_public_key(self, data): + def make_public_key(self, data, many, partial): """ Marshall an object out of the individual data components """ return V2PublicKeyRecord(**data) @@ -72,5 +72,5 @@ class V2PublicKeyRecordSchema(V2PublicKeyRecordInputSchema): read in from a database. Builds upon the basic input fields in PublicKeyRecordInputSchema. """ - id = fields.UUID(description="the unique id of the public key") - created = fields.DateTime(description="the time the public key9 record was created") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the public key"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the public key9 record was created"}}) diff --git a/src/server/models/recipes.py b/src/server/models/recipes.py index d5182bd..1d49c96 100644 --- a/src/server/models/recipes.py +++ b/src/server/models/recipes.py @@ -47,8 +47,8 @@ class RecipeKeyValuePair(Schema): """ A schema specifically for defining and validating user input of SSH Containers """ - key = fields.String(description="Template Key", required=True) - value = fields.String(description="Template Value", required=True) + key = fields.String(metadata={"metadata": {"description": "Template Key"}}, required=True) + value = fields.String(metadata={"metadata": {"description": "Template Value"}}, required=True) class V2RecipeRecord: @@ -77,31 +77,27 @@ def __repr__(self): class V2RecipeRecordInputSchema(Schema): """ A schema specifically for defining and validating user input """ # v2.0 - name = fields.Str(required=True, description="the name of the recipe", - validate=Length(min=1, error="name field must not be blank")) + name = fields.Str(required=True, validate=Length(min=1, error="name field must not be blank"), + metadata={"metadata": {"description": "Name of the recipe"}}) link = fields.Nested(ArtifactLink, required=False, allow_none=True, - description="the location of the recipe archive") - recipe_type = fields.Str(required=True, - description="The type of recipe, currently '%s' is the only valid value" - % RECIPE_TYPE_KIWI_NG, - validate=OneOf(RECIPE_TYPES, error="Recipe type must be one of: {choices}.")) - linux_distribution = fields.Str(required=True, - description="The linux distributiobn of the recipe, either '%s' or '%s' or '%s'" - % (LINUX_DISTRIBUTION_SLES12, LINUX_DISTRIBUTION_SLES15, - LINUX_DISTRIBUTION_CENTOS), + metadata={"metadata": {"description": "Location of the recipe archive"}}) + recipe_type = fields.Str(required=True, validate=OneOf(RECIPE_TYPES, error="Recipe type must be one of: {choices}."), + metadata={"metadata": {"description": f"The type of recipe, currently '{RECIPE_TYPE_KIWI_NG}' is the only valid value"}}) + linux_distribution = fields.Str(required=True,metadata={"metadata": {"description": f"The linux distribution of the recipe, either " + f"'{LINUX_DISTRIBUTION_SLES12}' or '{LINUX_DISTRIBUTION_SLES15}' or '{LINUX_DISTRIBUTION_CENTOS}'"}}, validate=OneOf(LINUX_DISTRIBUTIONS, error="Recipe type must be one of: {choices}.")) # v2.1 template_dictionary = fields.List(fields.Nested(RecipeKeyValuePair()), required=False, allow_none=True) # v2.2 - require_dkms = fields.Boolean(required=False, default=False, load_default=True, dump_default=True, - description="Recipe requires the use of dkms") - arch = fields.Str(required=False, description="Architecture of the recipe", default=ARCH_X86_64, - validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=True, dump_default=True) + require_dkms = fields.Boolean(load_default=False, dump_default=False, + metadata={"metadata": {"description": "Recipe requires the use of dkms"}}) + arch = fields.Str(required=False, metadata={"metadata": {"description": "Architecture of the recipe"}}, + validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=ARCH_X86_64, dump_default=ARCH_X86_64) @post_load - def make_recipe(self, data): + def make_recipe(self, data, many, partial): """ Marshall an object out of the individual data components """ data['template_dictionary'] = data.get('template_dictionary', []) return V2RecipeRecord(**data) @@ -117,8 +113,8 @@ class V2RecipeRecordSchema(V2RecipeRecordInputSchema): read in from a database. Builds upon the basic input fields in RecipeRecordInputSchema. """ - id = fields.UUID(description="the unique id of the recipe") - created = fields.DateTime(description="the time the recipe record was created") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the recipe"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the recipe record was created"}}) class V2RecipeRecordPatchSchema(Schema): @@ -126,9 +122,10 @@ class V2RecipeRecordPatchSchema(Schema): Schema for a updating a RecipeRecord object. """ link = fields.Nested(ArtifactLink, required=False, allow_none=False, - description="the location of the recipe archive") - arch = fields.Str(required=False, description="Architecture of the recipe", default=ARCH_X86_64, - validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=True, dump_default=True) - require_dkms = fields.Boolean(required=False, default=False, load_default=True, dump_default=True, - description="Recipe requires the use of dkms") + metadata={"metadata": {"description": "Location of the recipe archive"}}) + arch = fields.Str(required=False, validate=OneOf([ARCH_ARM64,ARCH_X86_64]), + load_default=ARCH_X86_64, dump_default=ARCH_X86_64, + metadata={"metadata": {"description": "Architecture of the recipe"}}) + require_dkms = fields.Boolean(required=False, load_default=False, dump_default=False, + metadata={"metadata": {"description": "Recipe requires the use of dkms"}}) template_dictionary = fields.List(fields.Nested(RecipeKeyValuePair()), required=False, allow_none=True) diff --git a/src/server/models/remote_build_nodes.py b/src/server/models/remote_build_nodes.py index 54d6cf6..7130250 100644 --- a/src/server/models/remote_build_nodes.py +++ b/src/server/models/remote_build_nodes.py @@ -144,11 +144,11 @@ def getStatus(self) -> (str, int): #(arch, current jobs) class V3RemoteBuildNodeRecordInputSchema(Schema): """ A schema specifically for defining and validating user input """ - xname = fields.Str(required=True, description="The XName of the remote build node", + xname = fields.Str(required=True, metadata={"metadata": {"description": "XName of the remote build node"}}, validate=Length(min=1, error="name field must not be blank")) @post_load - def make_remote_build_node(self, data): + def make_remote_build_node(self, data, many, partial): """ Marshall an object out of the individual data components """ return V3RemoteBuildNodeRecord(**data) diff --git a/src/server/v2/resources/images.py b/src/server/v2/resources/images.py index 9405ac7..9838040 100644 --- a/src/server/v2/resources/images.py +++ b/src/server/v2/resources/images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2018-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2018-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -83,6 +83,7 @@ def get(self): """ retrieve a list/collection of images """ log_id = get_log_id() current_app.logger.info("%s ++ images.v2.GET", log_id) + current_app.logger.info("%s ++ images.v2.GET RAW:%s" % (log_id, current_app.data["images"].values())) return_json = image_schema.dump(iter(current_app.data["images"].values()), many=True) current_app.logger.info("%s Returning json response: %s", log_id, return_json) return jsonify(return_json) @@ -166,6 +167,7 @@ def get(self, image_id): current_app.logger.info("%s no IMS image record matches image_id=%s", log_id, image_id) return generate_resource_not_found_response() + current_app.logger.info("%s ++ images.v2.GET Raw: %s" % (log_id, current_app.data['images'][image_id])) return_json = image_schema.dump(current_app.data['images'][image_id]) current_app.logger.info("%s Returning json response: %s", log_id, return_json) return jsonify(return_json) @@ -206,6 +208,7 @@ def patch(self, image_id): return generate_missing_input_response() # Validate input + current_app.logger.info("%s image patch json_value: %s" % (log_id, json_data)) errors = image_patch_input_schema.validate(json_data) if errors: current_app.logger.info("%s There was a problem validating the PATCH data: %s", log_id, errors) @@ -226,16 +229,34 @@ def patch(self, image_id): if problem: current_app.logger.info("%s Could not validate link artifact or artifact doesn't exist", log_id) return problem + setattr(image, key, value) elif key == "arch": current_app.logger.info(f"Patching architecture with {value}") image.arch = value + setattr(image, key, value) + elif key == 'metadata': + operation = value.get('operation') + annotation_key = value.get('key') + annotation_value = value.get('value', '') + current_app.logger.debug("Image Patch changeset: Current: %s -> %s %s %s" + % (image.metadata, operation, annotation_key, annotation_value)) + if operation not in ['set', 'remove']: + current_app.logger.info(f"Unknown requested operation change '{operation}'.") + return generate_data_validation_failure(errors=[]) + if operation == 'set': + image.metadata[annotation_key] = annotation_value + elif operation == 'remove': + try: + del image.metadata[annotation_key] + except KeyError: + current_app.logger.info("No-op when removing non-existent metadata from IMS record.") + current_app.logger.debug("Image metadata result: %s", image.metadata) else: current_app.logger.info(f"{log_id} Not able to patch record field {key} with value {value}") return generate_data_validation_failure(errors=[]) - - setattr(image, key, value) + current_app.logger.debug(f"{log_id} image metadata information dump: '%s'" % image.metadata) current_app.data['images'][image_id] = image return_json = image_schema.dump(current_app.data['images'][image_id]) - current_app.logger.info("%s Returning json response: %s", log_id, return_json) + current_app.logger.debug("%s Returning json response: %s", log_id, return_json) return jsonify(return_json) diff --git a/src/server/v3/models/images.py b/src/server/v3/models/images.py index fba5540..930e1f4 100644 --- a/src/server/v3/models/images.py +++ b/src/server/v3/models/images.py @@ -34,10 +34,10 @@ class V3DeletedImageRecord(V2ImageRecord): """ The ImageRecord object """ # pylint: disable=W0622 - def __init__(self, name, link=None, id=None, created=None, deleted=None, arch="x86_64"): + def __init__(self, name, link=None, id=None, created=None, deleted=None, arch="x86_64", metadata=None): # Supplied self.deleted = deleted or datetime.datetime.now() - super().__init__(name, link=link, id=id, created=created, arch=arch) + super().__init__(name, link=link, id=id, created=created, arch=arch, metadata=metadata) def __repr__(self): return ''.format(self=self) @@ -47,7 +47,7 @@ class V3DeletedImageRecordInputSchema(V2ImageRecordInputSchema): """ A schema specifically for defining and validating user input """ @post_load - def make_image(self, data): + def make_image(self, data, many, partial): """ Marshall an object out of the individual data components """ return V3DeletedImageRecord(**data) @@ -62,16 +62,16 @@ class V3DeletedImageRecordSchema(V3DeletedImageRecordInputSchema): read in from a database. Builds upon the basic input fields in ImageRecordInputSchema. """ - id = fields.UUID(description="the unique id of the image") - created = fields.DateTime(description="the time the image record was created") - deleted = fields.DateTime(description="the time the image record was deleted") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the image"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the image record was created"}}) + deleted = fields.DateTime(metadata={"metadata": {"description": "Time the image record was deleted"}}) class V3DeletedImageRecordPatchSchema(Schema): """ - Schema for a updating an ImageRecord object. + Schema for updating an ImageRecord object. """ operation = fields.Str(required=True, - description='The operation or action that should be taken on the image record. ' - f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }', + metadata={"metadata": {"description": "The operation or action that should be taken on the image record. " + f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }'}}, validate=OneOf(PATCH_OPERATIONS, error="Recipe type must be one of: {choices}.")) diff --git a/src/server/v3/models/public_keys.py b/src/server/v3/models/public_keys.py index 8ed8e65..1ca770f 100644 --- a/src/server/v3/models/public_keys.py +++ b/src/server/v3/models/public_keys.py @@ -47,7 +47,7 @@ class V3DeletedPublicKeyRecordInputSchema(V2PublicKeyRecordInputSchema): """ A schema specifically for defining and validating user input """ @post_load - def make_public_key(self, data): + def make_public_key(self, data, many, partial): """ Marshall an object out of the individual data components """ return V3DeletedPublicKeyRecord(**data) @@ -62,9 +62,9 @@ class V3DeletedPublicKeyRecordSchema(V3DeletedPublicKeyRecordInputSchema): read in from a database. Builds upon the basic input fields in DeletedRecipeRecordInputSchema. """ - id = fields.UUID(description="the unique id of the public_key") - created = fields.DateTime(description="the time the public_key record was created") - deleted = fields.DateTime(description="the time the public_key record was deleted") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the public_key"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the public_key record was created"}}) + deleted = fields.DateTime(metadata={"metadata": {"description": "Time the public_key record was deleted"}}) class V3DeletedPublicKeyRecordPatchSchema(Schema): @@ -72,6 +72,6 @@ class V3DeletedPublicKeyRecordPatchSchema(Schema): Schema for a updating an PublicKey object. """ operation = fields.Str(required=True, - description='The operation or action that should be taken on the recipe record. ' - f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }', + metadata={"metadata": {"description": "The operation or action that should be taken on the recipe record. " + f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }'}}, validate=OneOf(PATCH_OPERATIONS, error="Recipe type must be one of: {choices}.")) diff --git a/src/server/v3/models/recipes.py b/src/server/v3/models/recipes.py index 381eaba..87e481f 100644 --- a/src/server/v3/models/recipes.py +++ b/src/server/v3/models/recipes.py @@ -53,7 +53,7 @@ class V3DeletedRecipeRecordInputSchema(V2RecipeRecordInputSchema): """ A schema specifically for defining and validating user input """ @post_load - def make_recipe(self, data): + def make_recipe(self, data, many, partial): """ Marshall an object out of the individual data components """ return V3DeletedRecipeRecord(**data) @@ -68,9 +68,9 @@ class V3DeletedRecipeRecordSchema(V3DeletedRecipeRecordInputSchema): read in from a database. Builds upon the basic input fields in DeletedRecipeRecordInputSchema. """ - id = fields.UUID(description="the unique id of the recipe") - created = fields.DateTime(description="the time the recipe record was created") - deleted = fields.DateTime(description="the time the recipe record was deleted") + id = fields.UUID(metadata={"metadata": {"description": "Unique id of the recipe"}}) + created = fields.DateTime(metadata={"metadata": {"description": "Time the recipe record was created"}}) + deleted = fields.DateTime(metadata={"metadata": {"description": "Time the recipe record was deleted"}}) class V3DeletedRecipeRecordPatchSchema(Schema): @@ -78,6 +78,6 @@ class V3DeletedRecipeRecordPatchSchema(Schema): Schema for a updating an RecipeRecord object. """ operation = fields.Str(required=True, - description='The operation or action that should be taken on the recipe record. ' - f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }', + metadata={"metadata": {"description": "The operation or action that should be taken on the recipe record. " + f'Supported operations are: { ", ".join(PATCH_OPERATIONS) }'}}, validate=OneOf(PATCH_OPERATIONS, error="Recipe type must be one of: {choices}.")) diff --git a/src/server/v3/resources/images.py b/src/server/v3/resources/images.py index 244475b..b9895a7 100644 --- a/src/server/v3/resources/images.py +++ b/src/server/v3/resources/images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2020-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2020-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -28,6 +28,7 @@ from flask import jsonify, request, current_app from flask_restful import Resource +from copy import deepcopy from src.server.errors import problemify, generate_missing_input_response, generate_data_validation_failure, \ generate_resource_not_found_response, generate_patch_conflict @@ -373,18 +374,36 @@ def patch(self, image_id): current_app.logger.info(f"The artifact {value} is not in S3 and " f"was not soft-deleted. Ignoring") current_app.logger.info(str(exc)) + setattr(image, key, value) elif key == "arch": current_app.logger.info(f"Patching architecture with {value}") image.arch = value + setattr(image, key, value) + elif key == 'metadata': + operation = value.get('operation') + annotation_key = value.get('key') + annotation_value = value.get('value', '') + current_app.logger.debug("Image Patch changeset: Current: %s -> %s %s %s" + % (image.metadata, operation, annotation_key, annotation_value)) + if operation not in ['set', 'remove']: + current_app.logger.info(f"Unknown requested operation change '{operation}'.") + return generate_data_validation_failure(errors=[]) + if operation == 'set': + image.metadata[annotation_key] = annotation_value + elif operation == 'remove': + try: + del image.metadata[annotation_key] + except KeyError: + current_app.logger.info("No-op when removing non-existent metadata from IMS record.") + current_app.logger.debug("Image metadata result: %s", image.metadata) else: current_app.logger.info(f"{log_id} Not able to patch record field {key} with value {value}") return generate_data_validation_failure(errors=[]) + current_app.logger.info(f"{log_id} image metadata information dump: '%s'" % image.metadata) + current_app.data['images'][image_id] = image - setattr(image, key, value) - current_app.data[self.images_table][image_id] = image - - return_json = image_schema.dump(current_app.data[self.images_table][image_id]) - current_app.logger.info("%s Returning json response: %s", log_id, return_json) + return_json = image_schema.dump(current_app.data['images'][image_id]) + current_app.logger.debug("%s Returning json response: %s", log_id, return_json) return jsonify(return_json) diff --git a/tests/utils.py b/tests/utils.py index 43694b1..b37fa61 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,6 +25,8 @@ Test Utilities """ +# Format for read/write of test data datetime strings +DATETIME_STRING = '%Y-%m-%dT%H:%M:%S' def check_error_responses(testcase, response, status_code, fields): """ diff --git a/tests/v2/test_v2_images.py b/tests/v2/test_v2_images.py index be15acb..cc53b6b 100644 --- a/tests/v2/test_v2_images.py +++ b/tests/v2/test_v2_images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2018-2019, 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2018-2019, 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -37,7 +37,7 @@ from src.server import app from src.server.helper import S3Url -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v2.ims_fixtures import V2FlaskTestClientFixture, V2ImagesDataFixture @@ -52,11 +52,12 @@ def setUpClass(cls): def setUp(self): super(TestV2ImageEndpoint, self).setUp() - self.test_id = str(uuid.uuid4()) - self.test_id_link_none = str(uuid.uuid4()) - self.test_id_no_link = str(uuid.uuid4()) self.test_arch = "x86_64" self.app = self.useFixture(V2FlaskTestClientFixture()).client + + # Default fixture + self.test_id = str(uuid.uuid4()) + self.test_uri_with_link = '/images/{}'.format(self.test_id) self.data_record_with_link = { 'name': self.getUniqueString(), 'link': { @@ -67,32 +68,68 @@ def setUp(self): 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_id, 'arch': self.test_arch, + 'metadata': {} } + # Fixture without a link + self.test_id_link_none = str(uuid.uuid4()) + self.test_uri_link_none = '/images/{}'.format(self.test_id_link_none) self.data_record_link_none = { 'name': self.getUniqueString(), 'link': None, 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_id_link_none, 'arch': self.test_arch, + 'metadata': {} } + # Test Fixtures with no set link + self.test_id_no_link = str(uuid.uuid4()) + self.test_uri_no_link = '/images/{}'.format(self.test_id_no_link) self.data_record_no_link = { 'name': self.getUniqueString(), 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_id_no_link, 'arch': self.test_arch, + 'metadata': {} + } + # with_metadata; allows for testing records with stored metadata information + self.test_id_with_metadata = str(uuid.uuid4()) + self.test_uri_with_metadata = '/images/{}'.format(self.test_id_no_link) + self.data_record_with_metadata = { + 'name': self.getUniqueString(), + 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), + 'id': self.test_id_with_metadata, + 'arch': self.test_arch, + 'metadata': {'foo': 'bar'}, + 'link': { + 'path': 's3://boot-images/{}/manifest.json'.format(self.test_id_with_metadata), + 'etag': self.getUniqueString(), + 'type': 's3' + }, + } + # with_no_metadata; allows for testing existing data structures on system + self.test_id_with_no_metadata = str(uuid.uuid4()) + self.test_uri_with_no_metadata = '/images/{}'.format(self.test_id_with_no_metadata) + self.data_record_with_no_metadata = { + 'name': self.getUniqueString(), + 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), + 'id': self.test_id_with_no_metadata, + 'arch': self.test_arch, + 'link': { + 'path': 's3://boot-images/{}/manifest.json'.format(self.test_id_with_no_metadata), + 'etag': self.getUniqueString(), + 'type': 's3' + }, } + # Test Fixture for link cascades + self.test_uri_with_link_cascade_false = '/images/{}?cascade=False'.format(self.test_id) self.data = [ self.data_record_with_link, self.data_record_link_none, - self.data_record_no_link + self.data_record_no_link, + self.data_record_with_metadata, + self.data_record_with_no_metadata ] - self.useFixture(V2ImagesDataFixture(initial_data=self.data)) - self.test_uri_with_link = '/images/{}'.format(self.test_id) - self.test_uri_with_link_cascade_false = '/images/{}?cascade=False'.format(self.test_id) - self.test_uri_link_none = '/images/{}'.format(self.test_id_link_none) - self.test_uri_no_link = '/images/{}'.format(self.test_id_no_link) - self.s3_manifest_data = { "version": "1.0", "created": "2020-01-14 03:17:14", @@ -122,7 +159,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -142,7 +179,7 @@ def test_get_link_none(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_link_none[key], @@ -162,7 +199,7 @@ def test_get_no_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_no_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': self.assertEqual(response_data[key], None, @@ -261,8 +298,7 @@ def test_patch(self): self.stubber.activate() response = self.app.patch(self.test_uri_link_none, content_type='application/json', data=json.dumps(link_data)) self.stubber.deactivate() - - self.assertEqual(response.status_code, 200, 'status code was not 200') + self.assertEqual(response.status_code, 200, 'status code was not 200: data:%s response.data: %s' % (json.dumps(link_data), response.data)) response_data = json.loads(response.data) self.assertEqual(set(self.data_record_link_none.keys()).difference(response_data.keys()), set(), 'returned keys not the same') @@ -272,7 +308,7 @@ def test_patch(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': self.assertEqual(response_data[key], link_data['link'], @@ -331,7 +367,7 @@ def test_patch_same_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -340,10 +376,12 @@ def test_patch_same_link(self): def test_patch_change_arch(self): """ Test that we're able to patch a record with a new architecture""" - test_archs = ['x86_64','aarch64','x86_64'] + test_archs = ['x86_64', 'aarch64', 'x86_64'] for arch in test_archs: patch_data = {'arch': arch} - response = self.app.patch(self.test_uri_link_none, content_type='application/json', data=json.dumps(patch_data)) + response = self.app.patch(self.test_uri_link_none, + content_type='application/json', + data=json.dumps(patch_data)) self.assertEqual(response.status_code, 200, 'status code was not 200') response_data = json.loads(response.data) @@ -355,15 +393,40 @@ def test_patch_change_arch(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'arch': self.assertEqual(response_data[key], patch_data['arch'], 'resource field "{}" returned was not equal'.format(key)) else: - self.assertEqual(response_data[key], self.data_record_link_none[key], + self.assertEqual(response_data[key], + self.data_record_link_none[key], 'resource field "{}" returned was not equal'.format(key)) + def test_patch_set_metadata(self): + test_kv_pairs = [('foo', 'bar'), ('projected', 'image')] + for key_val, val_val in test_kv_pairs: + patch_data = {'metadata': {'operation': 'set', 'key': key_val, 'value': val_val}} + response = self.app.patch(self.test_uri_link_none, + content_type='application/json', + data=json.dumps(patch_data)) + self.assertEqual(response.status_code, 200, 'status code was not 200') + + def test_patch_remove_metadata(self): + patch_data = {'metadata': {'operation': 'remove', 'key': 'key'}} + response = self.app.patch(self.test_uri_with_metadata, + content_type='application/json', + data=json.dumps(patch_data)) + self.assertEqual(response.status_code, 200, 'status code was not 200') + + def test_patch_remove_metadata_idempotent(self): + patch_data = {'metadata': {'operation': 'remove', 'key': 'key'}} + response = self.app.patch(self.test_uri_with_no_metadata, + content_type='application/json', + data=json.dumps(patch_data)) + self.assertEqual(response.status_code, 200, 'status code was not 200') + + class TestV2ImagesCollectionEndpoint(TestCase): """ Test the images/ collection endpoint (ims.v2.resources.images.ImagesCollection) @@ -421,7 +484,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data[key]) @@ -469,7 +532,7 @@ def test_post(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata'], 'returned keys not the same') def test_post_link_none(self): @@ -488,7 +551,7 @@ def test_post_link_none(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata',], 'returned keys not the same') def test_post_no_link(self): @@ -506,7 +569,7 @@ def test_post_no_link(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata'], 'returned keys not the same') def test_post_400_no_input(self): diff --git a/tests/v2/test_v2_jobs.py b/tests/v2/test_v2_jobs.py index 0de1ffd..3e35cdd 100644 --- a/tests/v2/test_v2_jobs.py +++ b/tests/v2/test_v2_jobs.py @@ -43,7 +43,7 @@ from src.server.helper import S3Url from src.server.models.jobs import (KERNEL_FILE_NAME_ARM, KERNEL_FILE_NAME_X86, STATUS_TYPES) -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v2.ims_fixtures import (V2FlaskTestClientFixture, V2ImagesDataFixture, V2JobsDataFixture, V2PublicKeysDataFixture, @@ -94,7 +94,7 @@ def test_get(self, client_mock, config_mock, utils_mock): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key in self.data: self.assertEqual(response_data[key], self.data[key], @@ -376,7 +376,7 @@ def test_get(self, utils_mock, config_mock, client_mock): self.assertAlmostEqual(datetime.datetime.strptime(self.job_data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.job_data[key]) diff --git a/tests/v2/test_v2_public_keys.py b/tests/v2/test_v2_public_keys.py index 124e122..6fa877a 100644 --- a/tests/v2/test_v2_public_keys.py +++ b/tests/v2/test_v2_public_keys.py @@ -33,7 +33,7 @@ from testtools.matchers import HasLength from tests.v2.ims_fixtures import V2FlaskTestClientFixture, V2PublicKeysDataFixture -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING class TestV2PublicKeyEndpoint(TestCase): @@ -65,7 +65,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data[key], @@ -117,7 +117,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data[key]) diff --git a/tests/v2/test_v2_recipes.py b/tests/v2/test_v2_recipes.py index 6dfdd35..87a58b1 100644 --- a/tests/v2/test_v2_recipes.py +++ b/tests/v2/test_v2_recipes.py @@ -36,7 +36,7 @@ from src.server import app from src.server.helper import S3Url -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v2.ims_fixtures import V2FlaskTestClientFixture, V2RecipesDataFixture @@ -119,7 +119,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -190,7 +190,7 @@ def test_patch(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': self.assertEqual(response_data[key], link_data['link'], @@ -218,7 +218,7 @@ def test_patch_architecture(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'arch': self.assertEqual(response_data[key], arch_data['arch'], @@ -246,7 +246,7 @@ def test_patch_dkms(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'require_dkms': self.assertEqual(response_data[key], dkms_data['require_dkms'], @@ -305,7 +305,7 @@ def test_patch_same_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -356,7 +356,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data[key]) diff --git a/tests/v3/test_v3_deleted_images.py b/tests/v3/test_v3_deleted_images.py index 8473379..7e69474 100644 --- a/tests/v3/test_v3_deleted_images.py +++ b/tests/v3/test_v3_deleted_images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2020-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2020-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -33,7 +33,7 @@ from src.server import app from src.server.helper import S3Url, ARTIFACT_LINK_TYPE_S3 -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3ImagesDataFixture, V3DeletedImagesDataFixture @@ -79,6 +79,7 @@ def setUp(self): 'id': self.test_with_link_id, 'name': self.getUniqueString(), 'arch': "x86_64", + 'metadata': {}, 'link': { 'path': 's3://boot-images/{}/manifest.json'.format(self.test_with_link_id), 'etag': self.getUniqueString(), @@ -93,6 +94,7 @@ def setUp(self): self.test_no_link_record = { 'id': self.test_no_link_id, 'name': self.getUniqueString(), + 'metadata': {}, 'arch': "x86_64", 'link': None, 'created': (datetime.now() - timedelta(days=77)).replace(microsecond=0).isoformat(), @@ -171,7 +173,7 @@ def test_get(self): self.assertAlmostEqual(datetime.strptime(test_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_data[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(response_data[key], test_record[key], @@ -302,7 +304,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key], diff --git a/tests/v3/test_v3_deleted_public_keys.py b/tests/v3/test_v3_deleted_public_keys.py index b0b80c2..d6f1bea 100644 --- a/tests/v3/test_v3_deleted_public_keys.py +++ b/tests/v3/test_v3_deleted_public_keys.py @@ -31,7 +31,7 @@ from src.server import app from src.server.helper import S3Url from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3PublicKeysDataFixture, V3DeletedPublicKeysDataFixture -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING class TestV3DeletedPublicKeysBase(TestCase): @@ -81,7 +81,7 @@ def test_get(self): self.assertAlmostEqual(datetime.strptime(self.test_public_key_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_data[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(response_data[key], self.test_public_key_record[key], @@ -159,7 +159,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key], diff --git a/tests/v3/test_v3_deleted_recipes.py b/tests/v3/test_v3_deleted_recipes.py index 100a5c7..85acbf4 100644 --- a/tests/v3/test_v3_deleted_recipes.py +++ b/tests/v3/test_v3_deleted_recipes.py @@ -33,7 +33,7 @@ from src.server import app from src.server.helper import S3Url, ARTIFACT_LINK_TYPE_S3 from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3RecipesDataFixture, V3DeletedRecipesDataFixture -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING class TestV3DeletedRecipeBase(TestCase): @@ -174,7 +174,7 @@ def test_get(self): self.assertAlmostEqual(datetime.strptime(test_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_data[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(response_data[key], test_record[key], @@ -266,7 +266,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key], diff --git a/tests/v3/test_v3_images.py b/tests/v3/test_v3_images.py index 9792c71..dd61700 100644 --- a/tests/v3/test_v3_images.py +++ b/tests/v3/test_v3_images.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2020-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2020-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -36,7 +36,7 @@ from src.server import app from src.server.helper import S3Url, ARTIFACT_LINK_TYPE_S3 -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3ImagesDataFixture, V3DeletedImagesDataFixture @@ -61,6 +61,7 @@ def setUp(self): 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_with_link_id, 'arch': self.test_arch, + 'metadata': {} } self.test_link_none_id = str(uuid.uuid4()) @@ -71,6 +72,7 @@ def setUp(self): 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_link_none_id, 'arch': self.test_arch, + 'metadata': {} } self.test_no_link_id = str(uuid.uuid4()) @@ -80,12 +82,32 @@ def setUp(self): 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), 'id': self.test_no_link_id, 'arch': self.test_arch, + 'metadata': {} + } + self.test_data_record_with_metadata_id = str(uuid.uuid4()) + self.test_data_record_with_metadata_uri = '/v3/images/{}'.format(self.test_data_record_with_metadata_id) + self.test_data_record_with_metadata_record = { + 'name': self.getUniqueString(), + 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), + 'id': self.test_data_record_with_metadata_id, + 'arch': self.test_arch, + 'metadata': {'foo': 'bar'} } + self.data_record_with_no_metadata_id = str(uuid.uuid4()) + self.data_record_with_no_metadata_uri = '/v3/images/{}'.format(self.data_record_with_no_metadata_id) + self.data_record_with_no_metadata_record = { + 'name': self.getUniqueString(), + 'created': datetime.datetime.now().replace(microsecond=0).isoformat(), + 'id': self.data_record_with_no_metadata_id, + 'arch': self.test_arch + } self.data = [ self.test_with_link_record, self.test_link_none_record, - self.test_no_link_record + self.test_no_link_record, + self.test_data_record_with_metadata_record, + self.data_record_with_no_metadata_record ] self.app = self.useFixture(V3FlaskTestClientFixture()).client @@ -171,7 +193,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_with_link_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.test_with_link_record[key], @@ -191,7 +213,7 @@ def test_get_link_none(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_link_none_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.test_link_none_record[key], @@ -211,7 +233,7 @@ def test_get_no_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_no_link_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': self.assertEqual(response_data[key], None, @@ -349,7 +371,7 @@ def test_patch_change_architecture(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_link_none_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'arch': self.assertEqual(response_data[key], patch_data['arch'], @@ -403,11 +425,11 @@ def test_patch(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_link_none_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': - self.assertEqual(response_data[key], link_data['link'], - 'resource field "{}" returned was not equal'.format(key)) + self.assertEqual(response_data[key], link_data[key], + 'resource field "{}" returned was not equal: {} != {}'.format(key, response_data[key], link_data[key])) else: self.assertEqual(response_data[key], self.test_link_none_record[key], 'resource field "{}" returned was not equal'.format(key)) @@ -453,7 +475,7 @@ def test_patch_same_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_with_link_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.test_with_link_record[key], @@ -486,7 +508,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key]) @@ -536,7 +558,7 @@ def test_post(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata'], 'returned keys not the same') def test_post_link_none(self): @@ -555,7 +577,7 @@ def test_post_link_none(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata',], 'returned keys not the same') def test_post_no_link(self): @@ -573,7 +595,7 @@ def test_post_no_link(self): r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z') self.assertIsNotNone(response_data['created'], 'image creation date/time was not set properly') self.assertItemsEqual(response_data.keys(), - ['created', 'name', 'link', 'id', 'arch'], + ['created', 'name', 'link', 'id', 'arch', 'metadata'], 'returned keys not the same') def test_post_400_no_input(self): diff --git a/tests/v3/test_v3_jobs.py b/tests/v3/test_v3_jobs.py index 0ec8906..7d181d8 100644 --- a/tests/v3/test_v3_jobs.py +++ b/tests/v3/test_v3_jobs.py @@ -43,7 +43,7 @@ from src.server.helper import ARTIFACT_LINK_TYPE_S3, S3Url from src.server.models.jobs import (KERNEL_FILE_NAME_ARM, KERNEL_FILE_NAME_X86, STATUS_TYPES) -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING #from tests.v2.ims_fixtures import (V2FlaskTestClientFixture, # V2ImagesDataFixture, V2JobsDataFixture, # V2PublicKeysDataFixture, @@ -101,7 +101,7 @@ def test_get(self, client_mock, config_mock, utils_mock): self.assertAlmostEqual(datetime.datetime.strptime(self.data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key in self.data: self.assertEqual(response_data[key], self.data[key], @@ -382,7 +382,7 @@ def test_get(self, utils_mock, config_mock, client_mock): self.assertAlmostEqual(datetime.datetime.strptime(self.job_data[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.job_data[key]) diff --git a/tests/v3/test_v3_public_keys.py b/tests/v3/test_v3_public_keys.py index 748b310..0af3c45 100644 --- a/tests/v3/test_v3_public_keys.py +++ b/tests/v3/test_v3_public_keys.py @@ -33,7 +33,7 @@ from testtools.matchers import HasLength from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3PublicKeysDataFixture, V3DeletedPublicKeysDataFixture -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING class TestV3PublicKeyBase(TestCase): @@ -80,7 +80,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.test_public_key_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=1)) else: self.assertEqual(response_data[key], self.test_public_key_record[key], @@ -146,7 +146,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key]) diff --git a/tests/v3/test_v3_recipes.py b/tests/v3/test_v3_recipes.py index da4b9a3..556819e 100644 --- a/tests/v3/test_v3_recipes.py +++ b/tests/v3/test_v3_recipes.py @@ -36,7 +36,7 @@ from src.server import app from src.server.helper import S3Url, ARTIFACT_LINK_TYPE_S3 -from tests.utils import check_error_responses +from tests.utils import check_error_responses, DATETIME_STRING from tests.v3.ims_fixtures import V3FlaskTestClientFixture, V3RecipesDataFixture, V3DeletedRecipesDataFixture @@ -167,7 +167,7 @@ def test_get(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -251,7 +251,7 @@ def test_patch(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'link': self.assertEqual(response_data[key], link_data['link'], @@ -279,7 +279,7 @@ def test_patch_architecture(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'arch': self.assertEqual(response_data[key], arch_data['arch'], @@ -307,7 +307,7 @@ def test_patch_dkms(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_link_none[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) elif key == 'require_dkms': self.assertEqual(response_data[key], dkms_data['require_dkms'], @@ -357,7 +357,7 @@ def test_patch_same_link(self): self.assertAlmostEqual(datetime.datetime.strptime(self.data_record_with_link[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_data['created'], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=5)) else: self.assertEqual(response_data[key], self.data_record_with_link[key], @@ -387,7 +387,7 @@ def test_get_all(self): self.assertAlmostEqual(datetime.datetime.strptime(source_record[key], '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strptime(response_record[key], - '%Y-%m-%dT%H:%M:%S+00:00'), + DATETIME_STRING), delta=datetime.timedelta(seconds=1)) else: self.assertEqual(source_record[key], response_record[key])