Updating the data pipeline node types (#36)

radon-h2020 · Jun 9, 2020 · 4c787f6 · 4c787f6
1 parent 30e48f7
commit 4c787f6
Show file tree

Hide file tree

Showing 99 changed files with 430 additions and 2,051 deletions.
diff --git a/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca b/capabilitytypes/radon.capabilities.container/DockerRuntime/CapabilityType.tosca
@@ -8,23 +8,23 @@ capability_types:
       final: "false"
       targetNamespace: "radon.capabilities.container"
     properties:
+      publish_ports:
+        type: list
+        required: false
+        status: supported
       port:
         type: integer
         required: true
         default: 2375
         status: supported
-      publish_ports:
+      expose_ports:
         type: list
         required: false
         status: supported
       volumes:
         type: list
         required: false
         status: supported
-      expose_ports:
-        type: list
-        required: false
-        status: supported
       version:
         type: list
         required: false

diff --git a/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/Endpoint/CapabilityType.tosca
@@ -36,17 +36,17 @@ capability_types:
         required: false
         default: "PRIVATE"
         status: supported
+      secure:
+        type: boolean
+        required: false
+        default: false
+        status: supported
       ports:
         type: map
         required: false
         status: supported
         constraints:
           - min_length: 1
-      secure:
-        type: boolean
-        required: false
-        default: false
-        status: supported
       url_path:
         type: string
         required: false

diff --git a/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/OperatingSystem/CapabilityType.tosca
@@ -8,11 +8,11 @@ capability_types:
       final: "false"
       targetNamespace: "tosca.capabilities"
     properties:
-      distribution:
+      type:
         type: string
         required: false
         status: supported
-      type:
+      distribution:
         type: string
         required: false
         status: supported

diff --git a/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca b/capabilitytypes/tosca.capabilities/Scalable/CapabilityType.tosca
@@ -8,12 +8,12 @@ capability_types:
       final: "false"
       targetNamespace: "tosca.capabilities"
     properties:
-      max_instances:
+      min_instances:
         type: integer
         required: true
         default: 1
         status: supported
-      min_instances:
+      max_instances:
         type: integer
         required: true
         default: 1

diff --git a/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca b/nodetypes/radon.nodes.VM/OpenStack/NodeType.tosca
@@ -30,6 +30,11 @@ node_types:
         description: OpenStack image id (image names are not accepted)
         required: true
         status: supported
+      ssh_username:
+        type: string
+        description: ssh username of the base image
+        required: true
+        status: supported
       name:
         type: string
         description: Name that should be given to the VM in OpenStack
@@ -58,6 +63,10 @@ node_types:
                 type: string
                 required: true
                 default: { get_property: [ SELF, image ] }
+              ssh_username:
+                type: string
+                required: true
+                default: { get_property: [ SELF, ssh_username ] }
               vm_name:
                 type: string
                 required: true

diff --git a/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml b/nodetypes/radon.nodes.VM/OpenStack/files/create/create.yml
@@ -34,7 +34,7 @@
         name: server
         groups: vms
         ansible_host: "{{ server_info.server.public_v4 }}"
-        ansible_user: centos
+        ansible_user: "{{ssh_username}}"
         ansible_ssh_common_args: > 
           -o BatchMode=yes
           -o UserKnownHostsFile=/dev/null

diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PublishDataEndPoint/NodeType.tosca
@@ -7,19 +7,3 @@ node_types:
       targetNamespace: "radon.nodes.datapipeline.destination"
       abstract: "false"
       final: "false"
-    requirements:
-      - host:
-          capability: tosca.capabilities.Container
-          node: radon.nodes.nifi.Nifi
-          relationship: tosca.relationships.HostedOn
-          occurrences: [ 1, 1 ]
-      - connectToPipeline:
-          capability: radon.capabilities.datapipeline.ConnectToPipeline
-          node: radon.nodes.nifi.Pipeline
-          relationship: radon.relationships.datapipeline.ConnectNifiLocal
-          occurrences: [ 1, UNBOUNDED ]
-    capabilities:
-      ConnectToPipeline:
-        occurrences: [ 1, unbounded ]
-        valid_source_types: [ radon.nodes.datapipeline.MidwayPB, radon.nodes.datapipeline.SourcePB ]
-        type: radon.capabilities.datapipeline.ConnectToPipeline
diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/NodeType.tosca
@@ -8,71 +8,36 @@ node_types:
       abstract: "false"
       final: "false"
     attributes:
-      id:
+      template_name:
         type: string
+        default: "PublishLocal_localConn"
     properties:
       create_if_missing:
         type: boolean
         required: false
         default: true
         status: supported
-      template_name:
-        type: string
-        required: true
-        status: supported
       directory:
         type: string
         required: true
         status: supported
-      template_file:
-        type: string
-        required: true
-        status: supported
-    requirements:
-      - host:
-          capability: tosca.capabilities.Container
-          node: radon.nodes.nifi.Nifi
-          relationship: tosca.relationships.HostedOn
-          occurrences: [ 1, 1 ]
-      - ConnectToPipeline:
-          capability: radon.capabilities.datapipeline.ConnectToPipeline
-          node: radon.nodes.nifi.Pipeline
-          relationship: radon.relationships.datapipeline.ConnectNifiLocal
-          occurrences: [ 1, UNBOUNDED ]
     interfaces:
       Standard:
         type: tosca.interfaces.node.lifecycle.Standard
         operations:
-          stop:
-            inputs:
-              pipeline_id:
-                type: string
-                required: true
-                default: { get_attribute: [ SELF, id ] }
-            implementation:
-              primary: stop
-              timeout: 0
-          start:
-            inputs:
-              pipeline_id:
-                type: string
-                required: true
-                default: { get_attribute: [ SELF, id ] }
-            implementation:
-              primary: start
-              timeout: 0
           create:
             inputs:
               template_name:
                 type: string
                 required: true
-                default: { get_property: [ SELF, template_name ] }
+                default: { get_attribute: [ SELF, template_name ] }
               template_file:
                 type: string
                 required: true
-                default: { get_property: [ SELF, template_file ] }
+                default: "PublishLocal_localConn.xml"
             implementation:
               primary: create
+              dependencies: [ templateFile_localConn ]
               timeout: 0
           configure:
             inputs:
@@ -91,34 +56,13 @@ node_types:
             implementation:
               primary: configure
               timeout: 0
-          delete:
-            inputs:
-              pipeline_id:
-                type: string
-                required: true
-                default: { get_attribute: [ SELF, id ] }
-            implementation:
-              primary: delete
-              timeout: 0
     artifacts:
-      PublishLocal_localConn:
+      templateFile_localConn:
         type: tosca.artifacts.File
         file: PublishLocal_localConn.xml
-      stop:
-        type: tosca.artifacts.Implementation.Ansible
-        file: stop.yml
-      start:
-        type: tosca.artifacts.Implementation.Ansible
-        file: start.yml
       create:
-        type: tosca.artifacts.Implementation.Ansible
+        type: tosca.artifacts.File
         file: create.yml
       configure:
-        type: tosca.artifacts.Implementation.Ansible
-        file: configure.yml
-      delete:
-        type: tosca.artifacts.Implementation.Ansible
-        file: delete.yml
-      PublishLocal_rmtConn:
         type: tosca.artifacts.File
-        file: PublishLocal_rmtConn.xml
+        file: configure.yml
diff --git a/...ucket/files/config/configure.yml.mimetype → ...al/files/configure/configure.yml.mimetype b/...ucket/files/config/configure.yml.mimetype → ...al/files/configure/configure.yml.mimetype
diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/create/create.yml
@@ -65,12 +65,12 @@
     #     dest: "/tmp/pipeline_info{{template_name}}.txt"
     #     content: "pipeline_info are : {{ pipeline_info }}"
 
-    ## set PROCESS-GROUP for ConsumeLocal
+    ## set PROCESS-GROUP for S3BUCKET
     - name: Set pipeline id property value (process-groups)
       set_stats:
         data:
           id: "{{ pipeline_info.json.flow.processGroups[0].id }}"        
-          pipeline_type: "processGroups"
+#          pipeline_type: "processGroups"
       when: pipeline_info.json.flow.processGroups|length > 0
     # - name: Ansible create file if it doesn't exist example
     #   copy:

diff --git a/...S3Bucket/files/delete/delete.yml.mimetype → ...ishLocal/files/create/create.yml.mimetype b/...S3Bucket/files/delete/delete.yml.mimetype → ...ishLocal/files/create/create.yml.mimetype
diff --git a/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/start/start.yml b/nodetypes/radon.nodes.datapipeline.destination/PublishLocal/files/start/start.yml
diff --git a/...ocal_localConn/PublishLocal_localConn.xml → ...File_localConn/PublishLocal_localConn.xml b/...ocal_localConn/PublishLocal_localConn.xml → ...File_localConn/PublishLocal_localConn.xml
@@ -1,27 +1,26 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <template encoding-version="1.3">
-    <description>Publish data to the local directory. 
-get the data from local pipeline (NOT remote pipeline)</description>
-    <groupId>828d601e-0171-1000-586c-8ce1c1d0d129</groupId>
+    <description>Template to publish save to a local directory as files. </description>
+    <groupId>75e8afa0-0172-1000-4806-fa75d35c18d8</groupId>
     <name>PublishLocal_localConn</name>
     <snippet>
         <processGroups>
-            <id>49b00e44-f13d-3526-0000-000000000000</id>
-            <parentGroupId>11e64e91-b867-3e4e-0000-000000000000</parentGroupId>
+            <id>c23b3755-718c-3377-0000-000000000000</id>
+            <parentGroupId>dac3b43b-d8ad-3768-0000-000000000000</parentGroupId>
             <position>
                 <x>0.0</x>
                 <y>0.0</y>
             </position>
             <comments></comments>
             <contents>
                 <connections>
-                    <id>1320577a-84dd-32be-0000-000000000000</id>
-                    <parentGroupId>49b00e44-f13d-3526-0000-000000000000</parentGroupId>
+                    <id>51c870fa-b814-344a-0000-000000000000</id>
+                    <parentGroupId>c23b3755-718c-3377-0000-000000000000</parentGroupId>
                     <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
                     <backPressureObjectThreshold>10000</backPressureObjectThreshold>
                     <destination>
-                        <groupId>49b00e44-f13d-3526-0000-000000000000</groupId>
-                        <id>ba3414cf-c6b2-3c56-0000-000000000000</id>
+                        <groupId>c23b3755-718c-3377-0000-000000000000</groupId>
+                        <id>dffb757f-2412-3e53-0000-000000000000</id>
                         <type>PROCESSOR</type>
                     </destination>
                     <flowFileExpiration>0 sec</flowFileExpiration>
@@ -32,15 +31,15 @@ get the data from local pipeline (NOT remote pipeline)</description>
                     <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
                     <name></name>
                     <source>
-                        <groupId>49b00e44-f13d-3526-0000-000000000000</groupId>
-                        <id>3f391761-49a9-3ed6-0000-000000000000</id>
+                        <groupId>c23b3755-718c-3377-0000-000000000000</groupId>
+                        <id>b9e58522-5ad4-3232-0000-000000000000</id>
                         <type>INPUT_PORT</type>
                     </source>
                     <zIndex>0</zIndex>
                 </connections>
                 <inputPorts>
-                    <id>3f391761-49a9-3ed6-0000-000000000000</id>
-                    <parentGroupId>49b00e44-f13d-3526-0000-000000000000</parentGroupId>
+                    <id>b9e58522-5ad4-3232-0000-000000000000</id>
+                    <parentGroupId>c23b3755-718c-3377-0000-000000000000</parentGroupId>
                     <position>
                         <x>-512.0</x>
                         <y>-152.0</y>
@@ -52,8 +51,8 @@ get the data from local pipeline (NOT remote pipeline)</description>
                     <validationErrors>'Port 'connLocal_IP'' is invalid because Port has no incoming connections</validationErrors>
                 </inputPorts>
                 <processors>
-                    <id>ba3414cf-c6b2-3c56-0000-000000000000</id>
-                    <parentGroupId>49b00e44-f13d-3526-0000-000000000000</parentGroupId>
+                    <id>dffb757f-2412-3e53-0000-000000000000</id>
+                    <parentGroupId>c23b3755-718c-3377-0000-000000000000</parentGroupId>
                     <position>
                         <x>-472.0</x>
                         <y>-16.0</y>
@@ -123,6 +122,7 @@ get the data from local pipeline (NOT remote pipeline)</description>
                         <properties>
                             <entry>
 <key>Directory</key>
+<value>/mnt/out</value>
                             </entry>
                             <entry>
 <key>Conflict Resolution Strategy</key>
@@ -156,11 +156,11 @@ get the data from local pipeline (NOT remote pipeline)</description>
                     <executionNodeRestricted>false</executionNodeRestricted>
                     <name>PutFile</name>
                     <relationships>
-                        <autoTerminate>false</autoTerminate>
+                        <autoTerminate>true</autoTerminate>
                         <name>failure</name>
                     </relationships>
                     <relationships>
-                        <autoTerminate>false</autoTerminate>
+                        <autoTerminate>true</autoTerminate>
                         <name>success</name>
                     </relationships>
                     <state>STOPPED</state>
@@ -172,5 +172,5 @@ get the data from local pipeline (NOT remote pipeline)</description>
             <variables/>
         </processGroups>
     </snippet>
-    <timestamp>05/20/2020 05:40:01 UTC</timestamp>
+    <timestamp>06/03/2020 06:14:18 UTC</timestamp>
 </template>
diff --git a/...ocalConn/AWSlambda_localConn.xml.mimetype → ...lConn/PublishLocal_localConn.xml.mimetype b/...ocalConn/AWSlambda_localConn.xml.mimetype → ...lConn/PublishLocal_localConn.xml.mimetype
diff --git a/...ishLocal_rmtConn/PublishLocal_rmtConn.xml → ...lateFile_rmtConn/PublishLocal_rmtConn.xml b/...ishLocal_rmtConn/PublishLocal_rmtConn.xml → ...lateFile_rmtConn/PublishLocal_rmtConn.xml
diff --git a/...ine.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml.mimetype b/...ine.destination/PublishLocal/files/templateFile_rmtConn/PublishLocal_rmtConn.xml.mimetype
@@ -0,0 +1 @@
+application/xml