diff --git a/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca b/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca index 9d8b5564..a1f13092 100644 --- a/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca +++ b/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca @@ -8,12 +8,16 @@ capability_types: final: "false" targetNamespace: "radon.capabilities.container" properties: + publish_ports: + type: list + required: false + status: supported port: type: integer required: true default: 2375 status: supported - publish_ports: + expose_ports: type: list required: false status: supported @@ -21,10 +25,6 @@ capability_types: type: list required: false status: supported - expose_ports: - type: list - required: false - status: supported version: type: list required: false diff --git a/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca index d2fe211d..3e2c0f30 100644 --- a/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca +++ b/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca @@ -36,17 +36,17 @@ capability_types: required: false default: "PRIVATE" status: supported + secure: + type: boolean + required: false + default: false + status: supported ports: type: map required: false status: supported constraints: - min_length: 1 - secure: - type: boolean - required: false - default: false - status: supported url_path: type: string required: false diff --git a/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca index b5a3f0df..30c7fa81 100644 --- a/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca +++ b/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca @@ -8,11 +8,11 @@ capability_types: final: "false" targetNamespace: "tosca.capabilities" properties: - distribution: + type: type: string required: false status: supported - type: + distribution: type: string required: false status: supported diff --git a/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca index 43edbd0c..7eae6943 100644 --- a/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca +++ b/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca @@ -8,12 +8,12 @@ capability_types: final: "false" targetNamespace: "tosca.capabilities" properties: - max_instances: + min_instances: type: integer required: true default: 1 status: supported - min_instances: + max_instances: type: integer required: true default: 1 diff --git a/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca b/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca index a58bcede..b3c4fc56 100644 --- a/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca +++ b/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca @@ -30,6 +30,11 @@ node_types: description: OpenStack image id (image names are not accepted) required: true status: supported + ssh_username: + type: string + description: ssh username of the base image + required: true + status: supported name: type: string description: Name that should be given to the VM in OpenStack @@ -58,6 +63,10 @@ node_types: type: string required: true default: { get_property: [ SELF, image ] } + ssh_username: + type: string + required: true + default: { get_property: [ SELF, ssh_username ] } vm_name: type: string required: true diff --git a/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml b/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml index f3d458a3..e8cd4eda 100644 --- a/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml +++ b/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml @@ -34,7 +34,7 @@ name: server groups: vms ansible_host: "{{ server_info.server.public_v4 }}" - ansible_user: centos + ansible_user: "{{ssh_username}}" ansible_ssh_common_args: > -o BatchMode=yes -o UserKnownHostsFile=/dev/null diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca index 455f6848..b936f4db 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca @@ -7,19 +7,3 @@ node_types: targetNamespace: "radon.nodes.datapipeline.destination" abstract: "false" final: "false" - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - - connectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.nifi.Pipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, UNBOUNDED ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, unbounded ] - valid_source_types: [ radon.nodes.datapipeline.MidwayPB, radon.nodes.datapipeline.SourcePB ] - type: radon.capabilities.datapipeline.ConnectToPipeline diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca index 84b9b6a3..aed08844 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca @@ -8,71 +8,36 @@ node_types: abstract: "false" final: "false" attributes: - id: + template_name: type: string + default: "PublishLocal_localConn" properties: create_if_missing: type: boolean required: false default: true status: supported - template_name: - type: string - required: true - status: supported directory: type: string required: true status: supported - template_file: - type: string - required: true - status: supported - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.nifi.Pipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, UNBOUNDED ] interfaces: Standard: type: tosca.interfaces.node.lifecycle.Standard operations: - stop: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 create: inputs: template_name: type: string required: true - default: { get_property: [ SELF, template_name ] } + default: { get_attribute: [ SELF, template_name ] } template_file: type: string required: true - default: { get_property: [ SELF, template_file ] } + default: "PublishLocal_localConn.xml" implementation: primary: create + dependencies: [ templateFile_localConn ] timeout: 0 configure: inputs: @@ -91,34 +56,13 @@ node_types: implementation: primary: configure timeout: 0 - delete: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: delete - timeout: 0 artifacts: - PublishLocal_localConn: + templateFile_localConn: type: tosca.artifacts.File file: PublishLocal_localConn.xml - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml create: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: create.yml configure: - type: tosca.artifacts.Implementation.Ansible - file: configure.yml - delete: - type: tosca.artifacts.Implementation.Ansible - file: delete.yml - PublishLocal_rmtConn: type: tosca.artifacts.File - file: PublishLocal_rmtConn.xml + file: configure.yml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/config/configure.yml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/configure/configure.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/config/configure.yml.mimetype rename to nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/configure/configure.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml index 4e7cee19..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml @@ -65,12 +65,12 @@ # dest: "/tmp/pipeline_info{{template_name}}.txt" # content: "pipeline_info are : {{ pipeline_info }}" - ## set PROCESS-GROUP for ConsumeLocal + ## set PROCESS-GROUP for S3BUCKET - name: Set pipeline id property value (process-groups) set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 # - name: Ansible create file if it doesn't exist example # copy: diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/delete/delete.yml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/delete/delete.yml.mimetype rename to nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/start/start.yml deleted file mode 100644 index c4097498..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/start/start.yml +++ /dev/null @@ -1,34 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - - tasks: - - - name: start the PROCESS GROUP. This will start everything with in. - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/PublishLocal_localConn/PublishLocal_localConn.xml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_localConn/PublishLocal_localConn.xml similarity index 84% rename from nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/PublishLocal_localConn/PublishLocal_localConn.xml rename to nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_localConn/PublishLocal_localConn.xml index b4eb7f66..473c9c00 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/PublishLocal_localConn/PublishLocal_localConn.xml +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_localConn/PublishLocal_localConn.xml @@ -1,13 +1,12 @@ diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/AWSlambda_localConn/AWSlambda_localConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_localConn/PublishLocal_localConn.xml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/AWSlambda_localConn/AWSlambda_localConn.xml.mimetype rename to nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_localConn/PublishLocal_localConn.xml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/PublishLocal_rmtConn/PublishLocal_rmtConn.xml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/PublishLocal_rmtConn/PublishLocal_rmtConn.xml rename to nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishRemote/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PublishRemote/NodeType.tosca index 0fbc5951..6447c0e6 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PublishRemote/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.destination/PublishRemote/NodeType.tosca @@ -7,13 +7,3 @@ node_types: targetNamespace: "radon.nodes.datapipeline.destination" abstract: "false" final: "false" - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, 1 ] - type: radon.capabilities.datapipeline.ConnectToPipeline diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/NodeType.tosca index 4ff5741f..767611c5 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/NodeType.tosca @@ -8,13 +8,10 @@ node_types: abstract: "false" final: "false" attributes: - id: - type: string - properties: template_name: type: string - required: true - status: supported + default: "S3Bucket_dest_PG_LocalConn" + properties: BucketName: type: string required: true @@ -31,32 +28,19 @@ node_types: Standard: type: tosca.interfaces.node.lifecycle.Standard operations: - stop: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: + create: inputs: - pipeline_id: + template_name: type: string required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 - create: - inputs: - pipeline_id: + default: { get_attribute: [ SELF, template_name ] } + template_file: type: string required: true - default: { get_attribute: [ SELF, id ] } + default: "S3Bucket_dest_PG_LocalConn.xml" implementation: primary: create + dependencies: [ templateFile_localConn ] timeout: 0 configure: inputs: @@ -77,30 +61,18 @@ node_types: required: true default: { get_property: [ SELF, Region ] } implementation: - primary: config - timeout: 0 - delete: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: delete + primary: configure timeout: 0 artifacts: - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml create: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: create.yml - delete: - type: tosca.artifacts.Implementation.Ansible - file: delete.yml - config: - type: tosca.artifacts.Implementation.Ansible + configure: + type: tosca.artifacts.File file: configure.yml + templateFile_LocalConn: + type: tosca.artifacts.File + file: S3Bucket_dest_PG_LocalConn.xml + templateFile_RmtConn: + type: tosca.artifacts.File + file: S3Bucket_dest_PG_RmtConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/configure/configure.yml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/configure/configure.yml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/configure/configure.yml rename to nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/configure/configure.yml diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/config/configure.yml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/configure/configure.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/config/configure.yml.mimetype rename to nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/configure/configure.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/create/create.yml index 31f18e5c..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/create/create.yml @@ -70,7 +70,7 @@ set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 # - name: Ansible create file if it doesn't exist example # copy: diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/delete/delete.yml deleted file mode 100644 index 8611c15a..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/delete/delete.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - tasks: - - name: Get process group info - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}" - method: GET - register: group_info - - - name: Delete the pipeline - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}?version={{group_info.json.revision.version}}&clientId={{group_info.json.revision.clientId }}" - method: DELETE - body_format: form-urlencoded - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml rename to nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_LocalConn/S3Bucket_dest_PG_LocalConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml rename to nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/templateFile_RmtConn/S3Bucket_dest_PG_RmtConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/LICENSE b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/NodeType.tosca deleted file mode 100644 index bfa6cbcc..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/NodeType.tosca +++ /dev/null @@ -1,129 +0,0 @@ -tosca_definitions_version: tosca_simple_yaml_1_3 - -node_types: - radon.nodes.datapipeline.destination.S3Bucket: - derived_from: radon.nodes.datapipeline.destination.PublishDataEndPoint - metadata: - targetNamespace: "radon.nodes.datapipeline.destination" - abstract: "false" - final: "false" - attributes: - id: - type: string - properties: - cred_file_path: - type: string - required: true - status: supported - BucketName: - type: string - required: true - status: supported - Region: - type: string - required: false - default: "eu-central-1" - status: supported - requirements: - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.nifi.Pipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, 1 ] - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, 1 ] - valid_source_types: [ radon.nodes.datapipeline.MidwayPB, radon.nodes.datapipeline.SourcePB ] - type: radon.capabilities.datapipeline.ConnectToPipeline - interfaces: - Standard: - type: tosca.interfaces.node.lifecycle.Standard - operations: - stop: - inputs: - stop: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 - create: - inputs: - template_name: - type: string - required: true - default: { get_property: [ SELF, template_name ] } - template_file: - type: string - required: true - default: { get_property: [ SELF, template_file ] } - implementation: - primary: create - timeout: 0 - configure: - inputs: - BucketName: - type: string - required: true - default: { get_property: [ SELF, BucketName ] } - cred_file_path: - type: string - required: true - default: { get_property: [ SELF, cred_file_path ] } - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - Region: - type: string - required: true - default: { get_property: [ SELF, Region ] } - implementation: - primary: configure - timeout: 0 - delete: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: delete - timeout: 0 - artifacts: - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - templateFile_LocalConn: - type: tosca.artifacts.File - file: S3Bucket_dest_PG_LocalConn.xml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml - create: - type: tosca.artifacts.Implementation.Ansible - file: create.yml - configure: - type: tosca.artifacts.Implementation.Ansible - file: configure.yml - templateFile_RmtConn: - type: tosca.artifacts.File - file: S3Bucket_dest_PG_RmtConn.xml - delete: - type: tosca.artifacts.Implementation.Ansible - file: delete.yml diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/README.md b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/README.md deleted file mode 100644 index 1e35350d..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/README.md +++ /dev/null @@ -1,9 +0,0 @@ -- receive data from both local and remote pipeline -- Publish data to S3bucket -- User Input: - - Bucket Name - - region - - credential file - - -This pipeline have at least one incoming connection. \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/create/create.yml deleted file mode 100644 index 31f18e5c..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/create/create.yml +++ /dev/null @@ -1,83 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - vars: - query: "resources[?name=='{{template_name}}' && starts_with(identifier, '/templates/')].identifier" - - tasks: - - - - name: Copy template file to remote system (shell command requires file to be in remote system) - copy: - src: "{{ template_file }}" - dest: template.xml - - - name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds ) - wait_for: - port: 8080 - delay: 4 - - - name: upload template file to NiFi template repository - shell: 'curl -X "POST" "http://localhost:8080/nifi-api/process-groups/root/templates/upload" \ - -H "Content-Type: multipart/form-data; charset=utf-8; boundary=__X_PAW_BOUNDARY__" \ - -k \ - --form template=@template.xml' - register: response - - - name: Get list of available NiFi resources for fetching the id of the template by its name - uri: - url: http://localhost:8080/nifi-api/resources - method: Get - register: nifi_resources - - - name: Initiate the template - vars: - template_id: "{{ nifi_resources.json | to_json | from_json | json_query(query) | first}}" - uri: - url: http://localhost:8080/nifi-api/process-groups/root/template-instance - method: POST - body: - "originX": 2.0 - "originY": 3.0 - "templateId": "{{template_id.split('/')[2]}}" - status_code: 201 - body_format: json - register: pipeline_info - - # - name: Ansible create file if it doesn't exist example - # copy: - # dest: "/tmp/pipeline_info{{template_name}}.txt" - # content: "pipeline_info are : {{ pipeline_info }}" - - ## set PROCESS-GROUP for S3BUCKET - - name: Set pipeline id property value (process-groups) - set_stats: - data: - id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" - when: pipeline_info.json.flow.processGroups|length > 0 - # - name: Ansible create file if it doesn't exist example - # copy: - # dest: "/tmp/id_processGroups.txt" - # content: "{{pipeline_info.json.flow.processGroups[0].id}}" - # when: pipeline_info.json.flow.processGroups|length > 0 - - - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/delete/delete.yml deleted file mode 100644 index 8611c15a..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/delete/delete.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - tasks: - - name: Get process group info - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}" - method: GET - register: group_info - - - name: Delete the pipeline - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}?version={{group_info.json.revision.version}}&clientId={{group_info.json.revision.clientId }}" - method: DELETE - body_format: form-urlencoded - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/start/start.yml deleted file mode 100644 index b1bd4e52..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/start/start.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - - tasks: - - name: start the PROCESS GROUP. This will start everything with in. - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/NodeType.tosca index d5b56fea..d65ecfff 100644 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/NodeType.tosca @@ -7,16 +7,7 @@ node_types: targetNamespace: "radon.nodes.datapipeline.process" abstract: "false" final: "false" - attributes: - id: - description: Unique ID of the pipeline - type: string properties: - template_name: - type: string - description: name of the pipeline - required: false - status: supported cred_file_path: type: string description: credential file for access and secret key @@ -33,60 +24,27 @@ node_types: required: true default: "EU (Frankfurt)" status: supported - template_file: + attributes: + template_name: type: string - description: XML implementation of the pipeline - required: false - status: supported - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.abstract.DataPipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, 1 ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, 1 ] - type: radon.capabilities.datapipeline.ConnectToPipeline + default: "AWSlambda_localConn" interfaces: Standard: type: tosca.interfaces.node.lifecycle.Standard operations: - stop: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 create: inputs: template_name: type: string required: true - default: { get_property: [ SELF, template_name ] } + default: { get_attribute: [ SELF, template_name ] } template_file: type: string required: true - default: { get_property: [ SELF, template_file ] } + default: "AWSlambda_localConn.xml" implementation: primary: create + dependencies: [ templateFile_localConn ] timeout: 0 configure: inputs: @@ -97,7 +55,7 @@ node_types: function_name: type: string required: true - default: { get_attribute: [ SELF, function_name ] } + default: { get_property: [ SELF, function_name ] } pipeline_id: type: string required: true @@ -105,35 +63,17 @@ node_types: region: type: string required: true - default: { get_attribute: [ SELF, region ] } + default: { get_property: [ SELF, region ] } implementation: primary: configure timeout: 0 - delete: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: delete - timeout: 0 artifacts: - AWSlambda_localConn: + templateFile_localConn: type: tosca.artifacts.File file: AWSlambda_localConn.xml - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml create: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: create.yml configure: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: configure.yml - delete: - type: tosca.artifacts.Implementation.Ansible - file: delete.yml diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/configure/configure.yml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/configure/configure.yml index 41a6c44d..e3470934 100644 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/configure/configure.yml +++ b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/configure/configure.yml @@ -18,9 +18,7 @@ --- - hosts: all vars: - get_id1: "processors[0].id" - # This will return the putlambda processor id (not the other processors) - # get_id2: "processors[1].id" + get_id1: "processors[?component.name=='PutLambda'].id" dest_cred_file_path: "/tmp/.opera/aws/cred" tasks: @@ -47,7 +45,7 @@ - name: Configure the first processor within the process group vars: - first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}" + first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) | first }}" uri: url: "http://localhost:8080/nifi-api/processors/{{first_prcsr_id}}" method: PUT @@ -64,25 +62,6 @@ status_code: 200 body_format: json - # - name: Configure the SECOND processor within the process group - # vars: - # second_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id2) }}" - # uri: - # url: "http://localhost:8080/nifi-api/processors/{{second_prcsr_id}}" - # method: PUT - # body: - # "revision": { "version": 0} - # "component": - # "id": "{{second_prcsr_id}}" - # "config": - # "properties": { - # "Credentials File": "{{dest_cred_file_path}}/credentials", - # "Bucket":"{{BucketName}}", - # "Region": "{{Region}}" - # } - # status_code: 200 - # body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/create/create.yml index 2c22798b..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/create/create.yml @@ -28,7 +28,7 @@ src: "{{ template_file }}" dest: template.xml - - name: Wait for port 8080 to become open on the host, don't start checking for 4 seconds + - name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds ) wait_for: port: 8080 delay: 4 @@ -60,38 +60,23 @@ body_format: json register: pipeline_info - - name: Ansible create file if it doesn't exist example - copy: - dest: "/tmp/pipeline_info{{template_name}}.txt" - content: "pipeline_info are : {{ pipeline_info }}" - - -## FOR REMOTE-PROCESS-GROUP - - name: Set pipeline id property value (remoteProcessGroups) - set_stats: - data: - id: "{{ pipeline_info.json.flow.remoteProcessGroups[0].id }}" - pipeline_type: "remoteProcessGroups" - when: pipeline_info.json.flow.remoteProcessGroups|length > 0 - register: id - - name: Ansible create file if it doesn't exist example - copy: - dest: "/tmp/id_remoteProcessGroups.txt" - content: "{{pipeline_info.json.flow.remoteProcessGroups[0].id}}" - when: pipeline_info.json.flow.remoteProcessGroups|length > 0 + # - name: Ansible create file if it doesn't exist example + # copy: + # dest: "/tmp/pipeline_info{{template_name}}.txt" + # content: "pipeline_info are : {{ pipeline_info }}" -## set PROCESS-GROUP + ## set PROCESS-GROUP for S3BUCKET - name: Set pipeline id property value (process-groups) set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" - when: pipeline_info.json.flow.processGroups|length > 0 - - name: Ansible create file if it doesn't exist example - copy: - dest: "/tmp/id_processGroups.txt" - content: "{{pipeline_info.json.flow.processGroups[0].id}}" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 + # - name: Ansible create file if it doesn't exist example + # copy: + # dest: "/tmp/id_processGroups.txt" + # content: "{{pipeline_info.json.flow.processGroups[0].id}}" + # when: pipeline_info.json.flow.processGroups|length > 0 diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/delete/delete.yml deleted file mode 100644 index 8611c15a..00000000 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/delete/delete.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - tasks: - - name: Get process group info - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}" - method: GET - register: group_info - - - name: Delete the pipeline - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}?version={{group_info.json.revision.version}}&clientId={{group_info.json.revision.clientId }}" - method: DELETE - body_format: form-urlencoded - - diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/start/start.yml deleted file mode 100644 index f4e91662..00000000 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/start/start.yml +++ /dev/null @@ -1,76 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - vars: - query_rootPG_id: "resources[?name=='NiFi Flow' && starts_with(identifier, '/process-groups/')].identifier" - query_RPG: "resources[?name=='NiFi Flow' && starts_with(identifier, '/remote-process-groups/')].identifier" - tasks: - - - name: Get list of available NiFi resources for fetching the id of the listS3, FetchS3Object, PutS3Object - uri: - url: http://localhost:8080/nifi-api/resources/ - method: Get - register: nifi_resources - - -## FOR ROOT PROCESS GROUP. start everything in the root process group - - name: start the ROOT PROCESS GROUP. This will start everything with in. - vars: - rootPG_ID: "{{ nifi_resources.json | to_json | from_json | json_query(query_rootPG_id) | first }}" - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{rootPG_ID.split('/')[2]}}" - method: PUT - body: - "id": "{{rootPG_ID.split('/')[2]}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - #get the VERSION info - - name: Check if RPG is present and fetch information if the pipeline is RPG (remoteProcessGroups) - vars: - RPG_id: "{{ nifi_resources.json | to_json | from_json | json_query(query_RPG) | first }}" - uri: - url: "http://localhost:8080/nifi-api/remote-process-groups/{{ RPG_id.split('/')[2] }}" - method: GET - register: rpg_version_info - when: nifi_resources.json | to_json | from_json | json_query(query_RPG) | length > 0 - - - name: write rpg_version_info to file - copy: - dest: /tmp/rpg_version_info.txt - content: "{{ rpg_version_info }}" - when: nifi_resources.json | to_json | from_json | json_query(query_RPG) | length > 0 - - #start the pipeline - - name: Start the pipeline (remoteProcessGroups) - vars: - rpg_id: "{{ rpg_version_info.json.id }}" - uri: - url: "http://localhost:8080/nifi-api/remote-process-groups/{{rpg_id}}/run-status" - method: PUT - body: - "revision": "{{ rpg_version_info.json.revision}}" - "state": "TRANSMITTING" - "disconnectedNodeAcknowledged": true - status_code: 200 - body_format: json - when: nifi_resources.json | to_json | from_json | json_query(query_RPG) | length > 0 - diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/AWSlambda_localConn/AWSlambda_localConn.xml b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/templateFile_localConn/AWSlambda_localConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/AWSlambda_localConn/AWSlambda_localConn.xml rename to nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/templateFile_localConn/AWSlambda_localConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/templateFile_localConn/AWSlambda_localConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/templateFile_localConn/AWSlambda_localConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/templateFile_localConn/AWSlambda_localConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/NodeType.tosca index 2bccf8b5..5f45ae4e 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/NodeType.tosca @@ -7,11 +7,11 @@ node_types: targetNamespace: "radon.nodes.datapipeline.source" abstract: "false" final: "false" - properties: + attributes: template_name: type: string - required: true - status: supported + default: "S3Bucket_SRC_PG_LocalConn" + properties: BucketName: type: string required: true @@ -24,43 +24,10 @@ node_types: type: string required: true status: supported - requirements: - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.abstract.DataPipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, 1 ] interfaces: Standard: type: tosca.interfaces.node.lifecycle.Standard operations: - stop: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 - create: - inputs: - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: create - timeout: 0 configure: inputs: cred_file_path: @@ -80,30 +47,35 @@ node_types: required: true default: { get_property: [ SELF, Region ] } implementation: - primary: config + primary: configure timeout: 0 - delete: + create: inputs: - pipeline_id: + template_name: type: string required: true - default: { get_attribute: [ SELF, id ] } + default: { get_attribute: [ SELF, template_name ] } + template_file: + type: string + required: true + default: "S3Bucket_SRC_PG_LocalConn.xml" implementation: - primary: delete + primary: create + dependencies: [ templateFile_localConn ] timeout: 0 artifacts: - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml + configure: + type: tosca.artifacts.File + file: configure.yml + templateFile_localConn: + type: tosca.artifacts.File + description: This Nifi template file is only for the local connection. + file: S3Bucket_SRC_PG_LocalConn.xml + templateFile_RmtConn: + type: tosca.artifacts.File + description: This nifi template file is for remote connect. + file: S3Bucket_SRC_PG_RmtConn.xml create: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: create.yml - delete: - type: tosca.artifacts.Implementation.Ansible - file: delete.yml - config: - type: tosca.artifacts.Implementation.Ansible - file: configure.yml + diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/config/configure.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/config/configure.yml deleted file mode 100644 index cc8d9493..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/config/configure.yml +++ /dev/null @@ -1,90 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - vars: - get_id1: "processors[0].id" - get_id2: "processors[1].id" - dest_cred_file_path: "/tmp/.opera/aws/cred" - - tasks: - - name: create directory for cred file - file: - path: "{{dest_cred_file_path}}" - state: directory - - - name: Upload the credentials file to the current directory. - copy: - src: "{{cred_file_path}}" - dest: "{{dest_cred_file_path}}/credentials" - when: cred_file_path != "none" or cred_file_path != "None" - - - - name: Get list of available NiFi processors within the process group - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}/processors" - method: Get - register: nifi_resources - when: cred_file_path != "none" or cred_file_path != "None" - - - - - name: Configure the first processor within the process group - vars: - first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}" - uri: - url: "http://localhost:8080/nifi-api/processors/{{first_prcsr_id}}" - method: PUT - body: - "revision": { "version": 0} - "component": - "id": "{{first_prcsr_id}}" - "config": - "properties": { - "Credentials File": "{{dest_cred_file_path}}/credentials", - "Bucket":"{{BucketName}}", - "Region": "{{Region}}" - } - status_code: 200 - body_format: json - - - name: Configure the SECOND processor within the process group - vars: - second_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id2) }}" - uri: - url: "http://localhost:8080/nifi-api/processors/{{second_prcsr_id}}" - method: PUT - body: - "revision": { "version": 0} - "component": - "id": "{{second_prcsr_id}}" - "config": - "properties": { - "Credentials File": "{{dest_cred_file_path}}/credentials", - "Bucket":"{{BucketName}}", - "Region": "{{Region}}" - } - status_code: 200 - body_format: json - - - - - - - diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/config/configure.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/configure/configure.yml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/config/configure.yml rename to nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/configure/configure.yml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/start/start.yml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/configure/configure.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/start/start.yml.mimetype rename to nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/configure/configure.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/create/create.yml index 31f18e5c..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/create/create.yml @@ -70,7 +70,7 @@ set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 # - name: Ansible create file if it doesn't exist example # copy: diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/delete/delete.yml deleted file mode 100644 index 8611c15a..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/delete/delete.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - tasks: - - name: Get process group info - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}" - method: GET - register: group_info - - - name: Delete the pipeline - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}?version={{group_info.json.revision.version}}&clientId={{group_info.json.revision.clientId }}" - method: DELETE - body_format: form-urlencoded - - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml deleted file mode 100644 index b1bd4e52..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - - tasks: - - name: start the PROCESS GROUP. This will start everything with in. - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml.mimetype deleted file mode 100644 index f1148cc2..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/start/start.yml.mimetype +++ /dev/null @@ -1 +0,0 @@ -text/plain \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml.mimetype deleted file mode 100644 index f1148cc2..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/stop/stop.yml.mimetype +++ /dev/null @@ -1 +0,0 @@ -text/plain \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml rename to nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_RmtConn/S3Bucket_SRC_PG_RmtConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml rename to nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/templateFile_localConn/S3Bucket_SRC_PG_LocalConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeDataEndPoint/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.source/ConsumeDataEndPoint/NodeType.tosca index f8a9ccfc..4f774dca 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeDataEndPoint/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeDataEndPoint/NodeType.tosca @@ -7,14 +7,3 @@ node_types: targetNamespace: "radon.nodes.datapipeline.source" abstract: "false" final: "false" - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Pipeline - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 1, UNBOUNDED ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, unbounded ] - valid_source_types: [ radon.nodes.datapipeline.MidwayPB, radon.nodes.datapipeline.DestinationPB ] - type: radon.capabilities.datapipeline.ConnectToPipeline diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca index e211dc58..a76d2df9 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca @@ -8,70 +8,32 @@ node_types: abstract: "false" final: "false" attributes: - id: - type: string - properties: template_name: type: string - required: true - status: supported + default: "ConsumeLocal_localConn" + properties: directory: type: string description: Path to the file/folder required: true status: supported - template_file: - type: string - required: true - status: supported - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.nifi.Nifi - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 0, UNBOUNDED ] interfaces: Standard: type: tosca.interfaces.node.lifecycle.Standard operations: - stop: - inputs: - pipeline_id: - type: string - description: ID of the pipeline. - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: false - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 create: inputs: template_name: type: string - description: Name of the template. required: true - default: { get_property: [ SELF, template_name ] } + default: { get_attribute: [ SELF, template_name ] } template_file: type: string - description: Path to the template file. required: true - default: { get_artifact: [ SELF, templateFile_localConn, file ] } + default: "ConsumeLocal_localConn.xml" implementation: - primary: Create + primary: create + dependencies: [ templateFile_localConn ] timeout: 0 configure: inputs: @@ -87,40 +49,18 @@ node_types: implementation: primary: configure timeout: 0 - delete: - inputs: - pipeline_id: - type: string - description: ID of the pipeline - required: true - default: { get_attribute: [ SELF, id ] } - implementation: - primary: delete - timeout: 0 artifacts: templateFile_LocalConn: type: tosca.artifacts.File description: Nifi template to consume local data and send to local pipelines. file: ConsumeLocal_localConn.xml - stop: - type: tosca.artifacts.Implementation.Ansible - description: Script to stop the pipeline. - file: stop.yml - Create: - type: tosca.artifacts.Implementation.Ansible - description: Script to create the pipeline + create: + type: tosca.artifacts.File file: create.yml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml configure: - type: tosca.artifacts.Implementation.Ansible + type: tosca.artifacts.File file: configure.yml templateFile_RmtConn: type: tosca.artifacts.File description: Nifi template to consume local data and send to remote pipelines. file: ConsumeLocal_RmtConn.xml - delete: - type: tosca.artifacts.Implementation.Ansible - description: Ansible script to delete the pipeline. - file: delete.yml diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca.save b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca.save new file mode 100644 index 00000000..48f4e4a7 --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/NodeType.tosca.save @@ -0,0 +1,70 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +node_types: + radon.nodes.datapipeline.source.ConsumeLocal: + derived_from: radon.nodes.datapipeline.source.ConsumeDataEndPoint + metadata: + targetNamespace: "radon.nodes.datapipeline.source" + abstract: "false" + final: "false" + attributes: + id: + type: string + properties: + template_name: + type: string + required: true + status: supported + directory: + type: string + description: Path to the file/folder + required: true + status: supported + interfaces: + Standard: + type: tosca.interfaces.node.lifecycle.Standard + operations: + configure: + inputs: + pipeline_id: + type: string + required: false + default: { get_attribute: [ SELF, id ] } + directory: + type: string + description: Directory on local machine from where the data will be read. + required: true + default: { get_property: [ SELF, directory ] } + implementation: + primary: configure + timeout: 0 + create: + inputs: + template_file: + type: string + required: true +# default: { get_artifact: [ SELF, templateFile_localConn, file ] } + + template_name: + type: string + required: true + default: { get_property: [SELF, template_name] } + implementation: + primary: create + dependencies: [templateFile_localConn] + timeout: 0 + artifacts: + templateFile_LocalConn: + type: tosca.artifacts.File + description: Nifi template to consume local data and send to local pipelines. + file: ConsumeLocal_localConn.xml + configure: + type: tosca.artifacts.File + file: configure.yml + templateFile_RmtConn: + type: tosca.artifacts.File + description: Nifi template to consume local data and send to remote pipelines. + file: ConsumeLocal_RmtConn.xml + create: + type: tosca.artifacts.File + file: create.yml diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/stop/stop.yml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/configure/configure.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/stop/stop.yml.mimetype rename to nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/configure/configure.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml index 4e7cee19..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml @@ -65,12 +65,12 @@ # dest: "/tmp/pipeline_info{{template_name}}.txt" # content: "pipeline_info are : {{ pipeline_info }}" - ## set PROCESS-GROUP for ConsumeLocal + ## set PROCESS-GROUP for S3BUCKET - name: Set pipeline id property value (process-groups) set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 # - name: Ansible create file if it doesn't exist example # copy: diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/LICENSE.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/S3Bucket/LICENSE.mimetype rename to nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/create/create.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/delete/delete.yml deleted file mode 100644 index 8611c15a..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/delete/delete.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - tasks: - - name: Get process group info - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}" - method: GET - register: group_info - - - name: Delete the pipeline - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}?version={{group_info.json.revision.version}}&clientId={{group_info.json.revision.clientId }}" - method: DELETE - body_format: form-urlencoded - - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/start/start.yml deleted file mode 100644 index c4097498..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/start/start.yml +++ /dev/null @@ -1,34 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - - tasks: - - - name: start the PROCESS GROUP. This will start everything with in. - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_LocalConn/ConsumeLocal_localConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_LocalConn/ConsumeLocal_localConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_LocalConn/ConsumeLocal_localConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_RmtConn/ConsumeLocal_RmtConn.xml.mimetype b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_RmtConn/ConsumeLocal_RmtConn.xml.mimetype new file mode 100644 index 00000000..407f9ace --- /dev/null +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/templateFile_RmtConn/ConsumeLocal_RmtConn.xml.mimetype @@ -0,0 +1 @@ +application/xml \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsumeRemote/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.source/ConsumeRemote/NodeType.tosca index 1aa400bb..42414175 100644 --- a/nodetypes/radon.nodes.datapipeline.source/ConsumeRemote/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline.source/ConsumeRemote/NodeType.tosca @@ -7,13 +7,3 @@ node_types: targetNamespace: "radon.nodes.datapipeline.source" abstract: "false" final: "false" - attributes: - id: - description: id of the pipeline - type: string - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE.mimetype b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE.mimetype deleted file mode 100644 index f1148cc2..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/LICENSE.mimetype +++ /dev/null @@ -1 +0,0 @@ -text/plain \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/NodeType.tosca deleted file mode 100644 index c65d41a4..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/NodeType.tosca +++ /dev/null @@ -1,139 +0,0 @@ -tosca_definitions_version: tosca_simple_yaml_1_3 - -node_types: - radon.nodes.datapipeline.source.S3Bucket: - derived_from: radon.nodes.datapipeline.source.ConsumeRemote - metadata: - targetNamespace: "radon.nodes.datapipeline.source" - abstract: "false" - final: "false" - attributes: - id: - type: string - properties: - cred_file_path: - type: string - required: true - status: supported - ContinuousFetch: - type: boolean - description: Indicates if new objects need to be fetched automatically. - required: false - default: true - status: supported - BucketName: - type: string - description: Name of the S3 bucket. - required: true - status: supported - Region: - type: string - description: Specify region, if it is different from eu-central-1 region. - required: false - default: "eu-central-1" - status: supported - ObjectName: - type: string - description: Name of the object to be fetched. Leave blank if ContinuousFetch is True. - required: false - default: "${filename}" - status: supported - requirements: - - host: - capability: tosca.capabilities.Container - node: radon.nodes.nifi.Nifi - relationship: tosca.relationships.HostedOn - occurrences: [ 1, 1 ] - - ConnectToPipeline: - capability: radon.capabilities.datapipeline.ConnectToPipeline - node: radon.nodes.datapipeline.MidwayPB - relationship: radon.relationships.datapipeline.ConnectNifiLocal - occurrences: [ 0, UNBOUNDED ] - capabilities: - ConnectToPipeline: - occurrences: [ 1, unbounded ] - valid_source_types: [ radon.nodes.datapipeline.DestinationPB, radon.nodes.datapipeline.MidwayPB ] - type: radon.capabilities.datapipeline.ConnectToPipeline - interfaces: - Standard: - type: tosca.interfaces.node.lifecycle.Standard - operations: - stop: - inputs: - id: - type: string - required: false - default: { get_attribute: [ SELF, id ] } - implementation: - primary: stop - timeout: 0 - start: - inputs: - pipeline_id: - type: string - required: false - default: { get_attribute: [ SELF, id ] } - implementation: - primary: start - timeout: 0 - create: - inputs: - template_name: - type: string - required: true - default: { get_property: [ SELF, template_name ] } - template_file: - type: string - description: path to the template file - required: true - default: { get_artifact: [ SELF, templateFile_localConn, file ] } - implementation: - primary: create - timeout: 0 - configure: - inputs: - BucketName: - type: string - required: true - default: { get_property: [ SELF, BucketName ] } - cred_file_path: - type: string - required: false - default: { get_property: [ SELF, cred_file_path ] } - pipeline_id: - type: string - required: true - default: { get_attribute: [ SELF, id ] } - Region: - type: string - required: false - default: { get_property: [ SELF, Region ] } - implementation: - primary: configure - timeout: 0 - delete: - artifacts: - stop: - type: tosca.artifacts.Implementation.Ansible - file: stop.yml - templateFile_localConn: - type: tosca.artifacts.File - description: This Nifi template file is only for the local connection. - file: S3Bucket_SRC_PG_LocalConn.xml - start: - type: tosca.artifacts.Implementation.Ansible - file: start.yml - create: - type: tosca.artifacts.Implementation.Ansible - file: create.yml - configure: - type: tosca.artifacts.Implementation.Ansible - file: configure.yml - templateFile_RmtConn: - type: tosca.artifacts.File - description: This nifi template file is for remote connect. - file: S3Bucket_SRC_PG_RmtConn.xml - credFile: - type: tosca.artifacts.File - deploy_path: /tmp/ - file: credentials diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md deleted file mode 100644 index 7392adc4..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Purpose: -To get the data/objects from the AWS S3Bucket. - -## Ansible Implementation: -### Default implementation: -Contains the Nifi processor to list the objects (ListS3) and fetch all the Objects based on the list using (FetchS3Object). -### Specific implementaion: -User need to provide the specific object name - - -@TODO: -- Update capability - - diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md.mimetype b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md.mimetype deleted file mode 100644 index f1148cc2..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/README.md.mimetype +++ /dev/null @@ -1 +0,0 @@ -text/plain \ No newline at end of file diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/configure/configure.yml b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/configure/configure.yml deleted file mode 100644 index cc8d9493..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/configure/configure.yml +++ /dev/null @@ -1,90 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - vars: - get_id1: "processors[0].id" - get_id2: "processors[1].id" - dest_cred_file_path: "/tmp/.opera/aws/cred" - - tasks: - - name: create directory for cred file - file: - path: "{{dest_cred_file_path}}" - state: directory - - - name: Upload the credentials file to the current directory. - copy: - src: "{{cred_file_path}}" - dest: "{{dest_cred_file_path}}/credentials" - when: cred_file_path != "none" or cred_file_path != "None" - - - - name: Get list of available NiFi processors within the process group - uri: - url: "http://localhost:8080/nifi-api/process-groups/{{pipeline_id}}/processors" - method: Get - register: nifi_resources - when: cred_file_path != "none" or cred_file_path != "None" - - - - - name: Configure the first processor within the process group - vars: - first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}" - uri: - url: "http://localhost:8080/nifi-api/processors/{{first_prcsr_id}}" - method: PUT - body: - "revision": { "version": 0} - "component": - "id": "{{first_prcsr_id}}" - "config": - "properties": { - "Credentials File": "{{dest_cred_file_path}}/credentials", - "Bucket":"{{BucketName}}", - "Region": "{{Region}}" - } - status_code: 200 - body_format: json - - - name: Configure the SECOND processor within the process group - vars: - second_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id2) }}" - uri: - url: "http://localhost:8080/nifi-api/processors/{{second_prcsr_id}}" - method: PUT - body: - "revision": { "version": 0} - "component": - "id": "{{second_prcsr_id}}" - "config": - "properties": { - "Credentials File": "{{dest_cred_file_path}}/credentials", - "Bucket":"{{BucketName}}", - "Region": "{{Region}}" - } - status_code: 200 - body_format: json - - - - - - - diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/start/start.yml deleted file mode 100644 index b1bd4e52..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/start/start.yml +++ /dev/null @@ -1,33 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - - ---- -- hosts: all - - tasks: - - name: start the PROCESS GROUP. This will start everything with in. - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "RUNNING" - status_code: 200 - body_format: json - - - diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/stop/stop.yml deleted file mode 100644 index c33c0a95..00000000 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/stop/stop.yml +++ /dev/null @@ -1,29 +0,0 @@ - ########################################################################## - # Copyright (c) 2019 Contributors to the RADON project - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ########################################################################## - ---- -- hosts: all - tasks: - - name: Stop the pipeline - uri: - url: "http://localhost:8080/nifi-api/flow/process-groups/{{pipeline_id}}" - method: PUT - body: - "id": "{{pipeline_id}}" - "state": "STOPPED" - status: 201 - body_format: json - diff --git a/nodetypes/radon.nodes.datapipeline/DestinationPB/NodeType.tosca b/nodetypes/radon.nodes.datapipeline/DestinationPB/NodeType.tosca index 265d92da..7478fc61 100644 --- a/nodetypes/radon.nodes.datapipeline/DestinationPB/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline/DestinationPB/NodeType.tosca @@ -5,5 +5,10 @@ node_types: derived_from: radon.nodes.datapipeline.PipelineBlock metadata: targetNamespace: "radon.nodes.datapipeline" - abstract: "false" + abstract: "true" final: "false" + capabilities: + ConnectToPipeline: + occurrences: [ 1, 1 ] + valid_source_types: [ radon.nodes.datapipeline.MidwayPB, radon.nodes.datapipeline.SourcePB ] + type: radon.capabilities.datapipeline.ConnectToPipeline diff --git a/nodetypes/radon.nodes.datapipeline/MidwayPB/NodeType.tosca b/nodetypes/radon.nodes.datapipeline/MidwayPB/NodeType.tosca index 6d3fbd01..b517f73a 100644 --- a/nodetypes/radon.nodes.datapipeline/MidwayPB/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline/MidwayPB/NodeType.tosca @@ -5,5 +5,16 @@ node_types: derived_from: radon.nodes.datapipeline.PipelineBlock metadata: targetNamespace: "radon.nodes.datapipeline" - abstract: "false" + abstract: "true" final: "false" + requirements: + - ConnectToPipeline: + capability: radon.capabilities.datapipeline.ConnectToPipeline + node: radon.nodes.datapipeline.PipelineBlock + relationship: radon.relationships.datapipeline.ConnectNifiLocal + occurrences: [ 1, 1 ] + capabilities: + ConnectToPipeline: + occurrences: [ 1, 1 ] + valid_source_types: [ radon.nodes.datapipeline.SourcePB, radon.nodes.datapipeline.MidwayPB ] + type: radon.capabilities.datapipeline.ConnectToPipeline diff --git a/nodetypes/radon.nodes.datapipeline/PipelineBlock/NodeType.tosca b/nodetypes/radon.nodes.datapipeline/PipelineBlock/NodeType.tosca index d8894da7..3b24e08f 100644 --- a/nodetypes/radon.nodes.datapipeline/PipelineBlock/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline/PipelineBlock/NodeType.tosca @@ -1,15 +1,65 @@ tosca_definitions_version: tosca_simple_yaml_1_3 node_types: - radon.nodes.PipelineBlock.PipelineBlock: + radon.nodes.datapipeline.PipelineBlock: derived_from: radon.nodes.abstract.DataPipeline metadata: targetNamespace: "radon.nodes.datapipeline" - abstract: "false" + abstract: "true" final: "false" + attributes: + id: + type: string properties: name: type: string description: Name of the pipeline node required: false status: supported + requirements: + - host: + capability: tosca.capabilities.Container + node: radon.nodes.nifi.Nifi + relationship: tosca.relationships.HostedOn + occurrences: [ 1, 1 ] + interfaces: + Standard: + type: tosca.interfaces.node.lifecycle.Standard + operations: + stop: + inputs: + pipeline_id: + type: string + required: true + default: { get_attribute: [ SELF, id ] } + implementation: + primary: stop + timeout: 0 + start: + inputs: + pipeline_id: + type: string + required: true + default: { get_attribute: [ SELF, id ] } + implementation: + primary: start + timeout: 0 + delete: + inputs: + pipeline_id: + type: string + required: true + default: { get_attribute: [ SELF, id ] } + implementation: + primary: delete + timeout: 0 + artifacts: + stop: + type: tosca.artifacts.File + file: stop.yml + start: + type: tosca.artifacts.File + file: start.yml + delete: + type: tosca.artifacts.File + file: delete.yml diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/create/create.yml b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/create/create.yml similarity index 98% rename from nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/create/create.yml rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/create/create.yml index 31f18e5c..6416f647 100644 --- a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/create/create.yml +++ b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/create/create.yml @@ -70,7 +70,7 @@ set_stats: data: id: "{{ pipeline_info.json.flow.processGroups[0].id }}" - pipeline_type: "processGroups" +# pipeline_type: "processGroups" when: pipeline_info.json.flow.processGroups|length > 0 # - name: Ansible create file if it doesn't exist example # copy: diff --git a/nodetypes/radon.nodes.datapipeline.destination/S3Bucket/README.md.mimetype b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/create/create.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/S3Bucket/README.md.mimetype rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/create/create.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/credFile/credentials b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/credFile/credentials similarity index 100% rename from nodetypes/radon.nodes.datapipeline.source/S3Bucket/files/credFile/credentials rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/credFile/credentials diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/delete/delete.yml b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/delete/delete.yml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/delete/delete.yml rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/delete/delete.yml diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/delete/delete.yml.mimetype b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/delete/delete.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/delete/delete.yml.mimetype rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/delete/delete.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/start/start.yml b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/start/start.yml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PubsS3Bucket/files/start/start.yml rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/start/start.yml diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/start/start.yml.mimetype b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/start/start.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/start/start.yml.mimetype rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/start/start.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/stop/stop.yml b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/stop/stop.yml similarity index 100% rename from nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/stop/stop.yml rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/stop/stop.yml diff --git a/nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/stop/stop.yml.mimetype b/nodetypes/radon.nodes.datapipeline/PipelineBlock/files/stop/stop.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.process/AWSLambda/files/stop/stop.yml.mimetype rename to nodetypes/radon.nodes.datapipeline/PipelineBlock/files/stop/stop.yml.mimetype diff --git a/nodetypes/radon.nodes.datapipeline/SourcePB/NodeType.tosca b/nodetypes/radon.nodes.datapipeline/SourcePB/NodeType.tosca index 36c44579..dc033196 100644 --- a/nodetypes/radon.nodes.datapipeline/SourcePB/NodeType.tosca +++ b/nodetypes/radon.nodes.datapipeline/SourcePB/NodeType.tosca @@ -5,5 +5,11 @@ node_types: derived_from: radon.nodes.datapipeline.PipelineBlock metadata: targetNamespace: "radon.nodes.datapipeline" - abstract: "false" + abstract: "true" final: "false" + requirements: + - connectToPipeline: + capability: radon.capabilities.datapipeline.ConnectToPipeline + node: radon.nodes.abstract.DataPipeline + relationship: radon.relationships.datapipeline.ConnectNifiLocal + occurrences: [ 1, 1 ] diff --git a/policytypes/radon.policies.testing/HttpEndpointTest/PolicyType.tosca b/policytypes/radon.policies.testing/HttpEndpointTest/PolicyType.tosca index 72007a7c..d2874a5e 100644 --- a/policytypes/radon.policies.testing/HttpEndpointTest/PolicyType.tosca +++ b/policytypes/radon.policies.testing/HttpEndpointTest/PolicyType.tosca @@ -22,15 +22,15 @@ policy_types: required: true default: "GET" status: supported - expected_status: + port: type: integer required: true - default: 200 + default: 80 status: supported - port: + expected_status: type: integer required: true - default: 80 + default: 200 status: supported expected_body: type: json @@ -41,11 +41,11 @@ policy_types: required: true default: false status: supported - test_header: - type: map - required: false - status: supported test_body: type: json required: false status: supported + test_header: + type: map + required: false + status: supported diff --git a/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/RelationshipType.tosca b/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/RelationshipType.tosca index 3188c368..a264822a 100644 --- a/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/RelationshipType.tosca +++ b/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/RelationshipType.tosca @@ -9,8 +9,26 @@ relationship_types: final: "false" interfaces: Configure: + type: tosca.interfaces.relationship.Configure operations: - postConfigureSource: - postConfigureTarget: - preConfigureTarget: - preConfigureSource: + post_configure_source: + inputs: + source_pipeline_server_ip: + type: string + required: true + default: { get_attribute: [ SOURCE, host, host, public_address ] } + target_pipeline_server_ip: + type: string + required: true + default: { get_attribute: [ TARGET, host, host, public_address ] } + source_pipeline_id: + type: string + required: true + default: { get_attribute: [ SOURCE, id ] } + target_pipeline_id: + type: string + required: true + default: { get_attribute: [ TARGET, id ] } + implementation: + primary: connect.yml + timeout: 0 diff --git a/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/files/connect.yml b/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/files/connect.yml new file mode 100644 index 00000000..af63af0c --- /dev/null +++ b/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/files/connect.yml @@ -0,0 +1,64 @@ +--- +- hosts: all + vars: + query_output_port_id: "outputPorts[0].id" + query_input_port_id: "inputPorts[0].id" + root_id_query: "resources[?name=='NiFi Flow' && starts_with(identifier, '/process-groups/')].identifier" + tasks: + + # ROOT INFO + - name: Get list of available NiFi resources for fetching the ids of root reource group, input port and output port + uri: + url: http://localhost:8080/nifi-api/resources + method: GET + register: nifi_resources + + + # OUTPUT PORT INFO + - name: Get the list of output port of source pipeline + uri: + url: http://localhost:8080/nifi-api/process-groups/{{source_pipeline_id}}/output-ports + method: GET + register: output_ports_list + + # INPUT PORT INFO + - name: Get the list of input port of target pipeline + uri: + url: http://localhost:8080/nifi-api/process-groups/{{target_pipeline_id}}/input-ports + method: GET + register: input_ports_list + + + - name: create NiFi connection + vars: + output_port_id: "{{ output_ports_list.json | to_json | from_json | json_query(query_output_port_id)}}" + input_port_id: "{{ input_ports_list.json | to_json | from_json | json_query(query_input_port_id)}}" + root_identifier: "{{ nifi_resources.json | to_json | from_json | json_query(root_id_query) | first}}" + uri: + url: http://localhost:8080/nifi-api/process-groups/{{ root_identifier.split('/')[2] }}/connections + method: POST + body: + "revision": + "version": 0 + "component": + "source": + "id": "{{ output_port_id }}" + "groupId": "{{ source_pipeline_id }}" + "type": "OUTPUT_PORT" + "destination": + "id": "{{ input_port_id }}" + "groupId": "{{ target_pipeline_id }}" + "type": "INPUT_PORT" + status_code: 201 + body_format: json + + + - name: start the PROCESS GROUP. This will start everything with in. + uri: + url: "http://localhost:8080/nifi-api/flow/process-groups/{{target_pipeline_id}}" + method: PUT + body: + "id": "{{target_pipeline_id}}" + "state": "RUNNING" + status_code: 200 + body_format: json diff --git a/nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/delete/delete.yml.mimetype b/relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/files/connect.yml.mimetype similarity index 100% rename from nodetypes/radon.nodes.datapipeline.source/ConsS3Bucket/files/delete/delete.yml.mimetype rename to relationshiptypes/radon.relationships.datapipeline/ConnectNifiLocal/files/connect.yml.mimetype diff --git a/servicetemplates/radon.blueprints.datapipeline/ToyExampleNifi/ServiceTemplate.tosca b/servicetemplates/radon.blueprints.datapipeline/ToyExampleNifi/ServiceTemplate.tosca index 491c0045..009a88ff 100644 --- a/servicetemplates/radon.blueprints.datapipeline/ToyExampleNifi/ServiceTemplate.tosca +++ b/servicetemplates/radon.blueprints.datapipeline/ToyExampleNifi/ServiceTemplate.tosca @@ -77,8 +77,8 @@ topology_template: y: "586" displayName: "VM Two" properties: - key_name: "rem-VM2" flavor: "m2.xsmall" + key_name: "rem-VM2" image: "13a94b11-98ee-43a4-ad29-00ae97e8f790" name: "vmtwo_centos2" network: "provider_64_net" @@ -94,8 +94,8 @@ topology_template: y: "588" displayName: "VM One" properties: - key_name: "rem-VM2" flavor: "m2.xsmall" + key_name: "rem-VM2" image: "13a94b11-98ee-43a4-ad29-00ae97e8f790" name: "vmone_centos1" network: "provider_64_net" diff --git a/servicetemplates/radon.blueprints.examples/DataPipelineExample/ServiceTemplate.tosca b/servicetemplates/radon.blueprints.examples/DataPipelineExample/ServiceTemplate.tosca index c2415e30..02e37600 100644 --- a/servicetemplates/radon.blueprints.examples/DataPipelineExample/ServiceTemplate.tosca +++ b/servicetemplates/radon.blueprints.examples/DataPipelineExample/ServiceTemplate.tosca @@ -7,37 +7,34 @@ topology_template: AWSLambda: type: radon.nodes.datapipeline.process.AWSLambda metadata: - x: "823" - y: "148" + x: "767" + y: "112" displayName: "AWSLambda" properties: - template_name: null - cred_file_path: null - function_name: null - name: null - region: "EU (Frankfurt)" - template_file: null + cred_file_path: "/home/jakovits/aws/credentials" + function_name: "radonthumbnail" + name: "callAWS" + region: "eu-central-1" requirements: + - ConnectToPipeline: + node: PubsS3Bucket + relationship: con_ConnectNifiLocal_0 + capability: ConnectToPipeline - host: node: Nifi relationship: con_HostedOn_2 capability: host - - ConnectToPipeline: - node: PubsS3Bucket - relationship: con_ConnectNifiLocal_1 - capability: ConnectToPipeline PubsS3Bucket: type: radon.nodes.datapipeline.destination.PubsS3Bucket metadata: - x: "1210" - y: "155" + x: "1322" + y: "149" displayName: "PubsS3Bucket" properties: - template_name: null - cred_file_path: null - BucketName: null - name: null - Region: null + BucketName: "radon-jakovits-output" + cred_file_path: "/home/jakovits/aws/credentials" + name: "sendToS3" + Region: "eu-central-1" requirements: - host: node: Nifi @@ -46,19 +43,18 @@ topology_template: ConsS3Bucket: type: radon.nodes.datapipeline.source.ConsS3Bucket metadata: - x: "350" - y: "155" + x: "178" + y: "148" displayName: "ConsS3Bucket" properties: - template_name: null - cred_file_path: null - BucketName: null - name: null - Region: null + BucketName: "radon-jakovits" + cred_file_path: "/home/jakovits/aws/credentials" + name: "getFromS3" + Region: "eu-central-1" requirements: - - ConnectToPipeline: + - connectToPipeline: node: AWSLambda - relationship: con_ConnectNifiLocal_0 + relationship: con_ConnectNifiLocal_1 capability: ConnectToPipeline - host: node: Nifi @@ -67,24 +63,25 @@ topology_template: OpenStack: type: radon.nodes.VM.OpenStack metadata: - x: "828" - y: "703" + x: "1146" + y: "710" displayName: "OpenStack" properties: - key_name: null - flavor: null - image: null - name: null - network: null + flavor: "6b254b9e-db1c-40de-994c-07d69dd732a6" + key_name: "jakovits_ldpc" + image: "13a94b11-98ee-43a4-ad29-00ae97e8f790" + ssh_username: "centos" + name: "nifihost2" + network: "provider_64_net" Nifi: type: radon.nodes.nifi.Nifi metadata: - x: "828" - y: "450" + x: "766" + y: "516" displayName: "Nifi" properties: port: 8080 - component_version: null + component_version: "1.11.4" requirements: - host: node: OpenStack diff --git a/servicetemplates/radon.blueprints.examples/EC2_on_AWS/ServiceTemplate.tosca b/servicetemplates/radon.blueprints.examples/EC2_on_AWS/ServiceTemplate.tosca index 713c4beb..36f24f4f 100644 --- a/servicetemplates/radon.blueprints.examples/EC2_on_AWS/ServiceTemplate.tosca +++ b/servicetemplates/radon.blueprints.examples/EC2_on_AWS/ServiceTemplate.tosca @@ -28,8 +28,8 @@ topology_template: ssh_key_name: { get_input: ssh_key_name } vpc_subnet_id: { get_input: vpc_subnet_id } instance_type: "t2.micro" - ssh_user: { get_input: ssh_user } ssh_key_file: { get_input: ssh_key_file } + ssh_user: { get_input: ssh_user } requirements: - host: node: AwsPlatform diff --git a/servicetemplates/radon.blueprints/SockShopTestingExample/ServiceTemplate.tosca b/servicetemplates/radon.blueprints/SockShopTestingExample/ServiceTemplate.tosca index 3724c4c7..3d17ffbb 100644 --- a/servicetemplates/radon.blueprints/SockShopTestingExample/ServiceTemplate.tosca +++ b/servicetemplates/radon.blueprints/SockShopTestingExample/ServiceTemplate.tosca @@ -40,20 +40,20 @@ topology_template: path: "/cart/" hostname: "localhost" method: "GET" - port: 80 expected_status: 200 + port: 80 expected_body: null ti_blueprint: null use_https: true - test_body: null - test_header: null test_id: null + test_header: null + test_body: null targets: [ SockShop ] SimpleJMeterLoadTest: type: radon.policies.testing.JMeterLoadTest properties: - jmx_file: "sockshop.jmx" hostname: "localhost" + jmx_file: "sockshop.jmx" port: 8080 ti_blueprint: "radon.blueprints.testing.JMeterMasterOnly" user.properties: null diff --git a/servicetemplates/radon.blueprints/ThumbnailGeneration/ServiceTemplate.tosca b/servicetemplates/radon.blueprints/ThumbnailGeneration/ServiceTemplate.tosca index 2729fef9..c6e7059e 100644 --- a/servicetemplates/radon.blueprints/ThumbnailGeneration/ServiceTemplate.tosca +++ b/servicetemplates/radon.blueprints/ThumbnailGeneration/ServiceTemplate.tosca @@ -34,8 +34,8 @@ topology_template: environment: "adasd" memory: 256 name: "radon-particle-test" - runtime: "java8" alias: "dev" + runtime: "java8" statement_id: "lambda_test_permission01" zip_file: "thumbnail-generator-dev.jar" timeout: 300 diff --git a/servicetemplates/radon.legacy.blueprints/ImageResize/ServiceTemplate.tosca b/servicetemplates/radon.legacy.blueprints/ImageResize/ServiceTemplate.tosca index 4874e9fc..3514f866 100644 --- a/servicetemplates/radon.legacy.blueprints/ImageResize/ServiceTemplate.tosca +++ b/servicetemplates/radon.legacy.blueprints/ImageResize/ServiceTemplate.tosca @@ -40,12 +40,12 @@ topology_template: aws_region: "eu-central-1" lambda_runtime: "python3.6" function_name: "radon-particle-test" - lambda_memory: 128 lambda_timeout: 3 - permission_id: "lambda_test_permission01" + lambda_memory: 128 function_alias: "test" - lambda_handler: "image_resize.lambda_handler" + permission_id: "lambda_test_permission01" zip_file: "X-test-ImageRes.zip" + lambda_handler: "image_resize.lambda_handler" requirements: - requires_role: node: AwsCreateRole