From 60d95df74f9791e1c6a6c1a90a0c4c5989ed68df Mon Sep 17 00:00:00 2001 From: ganiruddha03 <103038659+ganiruddha03@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:16:20 +0800 Subject: [PATCH] final version of test pipeline 05/11/24 --- ...oad_pipeline_laptop_AB_hires_final_2.graph | 280 +++--------------- 1 file changed, 44 insertions(+), 236 deletions(-) diff --git a/dlg-graphs-final/casda_download_pipeline_laptop_AB_hires_final_2.graph b/dlg-graphs-final/casda_download_pipeline_laptop_AB_hires_final_2.graph index 6668713..1429cc3 100644 --- a/dlg-graphs-final/casda_download_pipeline_laptop_AB_hires_final_2.graph +++ b/dlg-graphs-final/casda_download_pipeline_laptop_AB_hires_final_2.graph @@ -19,7 +19,7 @@ "lastModifiedName": "", "lastModifiedEmail": "", "lastModifiedDatetime": 0, - "numLGNodes": 27 + "numLGNodes": 26 }, "nodeDataArray": [ { @@ -605,7 +605,7 @@ { "name": "num_of_copies", "value": 6, - "defaultValue": "4", + "defaultValue": "6", "description": "Specifies the number of replications that will be generated of the scatter construct's contents.", "readonly": false, "type": "Integer", @@ -1713,8 +1713,8 @@ "key": -16, "name": "CimagerStaticNew", "description": "A set of parameters, which can be set and modified in EAGLE and thus is part of the graph. Multiple serialisation formats are available.", - "x": 813.1197389399224, - "y": 1037.8723992970054, + "x": 802.8426044480382, + "y": 1089.2580717564263, "radius": 25, "collapsed": false, "subject": null, @@ -2843,7 +2843,7 @@ "name": "imcontsubStaticNew", "description": "A set of parameters, which can be set and modified in EAGLE and thus is part of the graph. Multiple serialisation formats are available.", "x": 985.0913931435751, - "y": 1045.5032107765235, + "y": 1126.00743096295, "radius": 25, "collapsed": false, "subject": null, @@ -3140,8 +3140,8 @@ "key": -18, "name": "linmosStaticNew", "description": "A set of parameters, which can be set and modified in EAGLE and thus is part of the graph. Multiple serialisation formats are available.", - "x": 1173.5389730274258, - "y": 1061.0798986912478, + "x": 1163.2618385355415, + "y": 1129.5941286371421, "radius": 25, "collapsed": false, "subject": null, @@ -4377,7 +4377,7 @@ "fields": [ { "name": "filepath", - "value": "$DLG_ROOT/testdata/casda_download/test_catalogue_processed_none.csv", + "value": "$DLG_ROOT/testdata/casda_download/test_catalogue_processed.csv", "defaultValue": "", "description": "File path for this file. In many cases this does not need to be specified. If it has a \\/ at the end it will be treated as a directory name and the filename will be generated. If it does not have a \\/, the last part will be treated as a filename. If filepath does not start with \\/ (relative path) then the session directory will be prepended to make the path absolute.\"", "readonly": false, @@ -5884,8 +5884,8 @@ "key": -26, "name": "mosaic", "description": "An application that wraps a simple python function. The inputs of the application are treated as the arguments of the function. Conversely, the output of the function is treated as the output of the application. If the application has more than one output, the result of calling the function is treated as an iterable, with each individual object being written to its corresponding output.", - "x": 1935.7057772545552, - "y": 785.6039035558017, + "x": 1873.9934290730444, + "y": 879.6172703967438, "radius": 25, "collapsed": false, "subject": null, @@ -5907,7 +5907,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "18b09e91-c941-4b16-96ad-3a57fa6b91b2", + "id": "69fbf2a7-e8c7-4dd6-ad00-200d886c3c00", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -5923,7 +5923,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "d6f3c9e4-3540-462f-8a8e-616f2bf5ea2f", + "id": "0f35cc44-ab01-465d-97d2-79c28f103552", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -5939,7 +5939,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "ca2bc0fa-1110-4433-b47c-f43e75a244e5", + "id": "6dce70cf-dc6a-4b01-88a0-6116ffa0b442", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -5955,7 +5955,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "db52e599-ef71-4134-8f95-552a27da39d5", + "id": "6c85f995-e837-4b64-aea2-597b6d7e2c1f", "parameterType": "ConstraintParameter", "usage": "NoPort" }, @@ -5971,7 +5971,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "25595b47-747f-4ced-8e23-23eae72f752c", + "id": "89980096-4e6e-4445-9e2b-3e08a575ae35", "parameterType": "ConstraintParameter", "usage": "NoPort" }, @@ -5987,7 +5987,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "232b637c-768e-4c9d-8b40-c67c4fd5d67d", + "id": "1b594d57-897f-4b05-be36-66812dd3014c", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6003,7 +6003,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "4e777956-eb7b-4547-8457-d7b93493aa5d", + "id": "e21355bc-1aad-4296-9b09-f0cd03aaabc4", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6019,7 +6019,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "5be6a363-9901-4ad8-ae6e-38c623fb3f33", + "id": "8ba948e6-d2b0-4efa-9b78-44c9b4f8f3da", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6042,7 +6042,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "a6325ecd-b3d5-4eac-87a7-7fd47d278ec6", + "id": "94cbab3a-1376-4dbe-8881-1019bfc1bceb", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6065,28 +6065,12 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "d8e20b3e-0de1-45ea-8f67-fa897824c1e6", + "id": "89dcc816-69fc-4907-bdd9-547603fb6493", "parameterType": "ComponentParameter", "usage": "NoPort" }, { - "name": "dummy1", - "value": "", - "defaultValue": "", - "description": "", - "readonly": false, - "type": "String", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": true, - "encoding": "pickle", - "id": "ea8fd512-d2b3-4922-9c7f-9b76e75627f0", - "parameterType": "ComponentParameter", - "usage": "InputOutput" - }, - { - "name": "dummy2", + "name": "dummy", "value": "", "defaultValue": "", "description": "", @@ -6097,184 +6081,8 @@ "positional": false, "keyAttribute": true, "encoding": "pickle", - "id": "eacf31bf-c1fe-432b-b21e-f7139c9ac0ec", - "parameterType": "ComponentParameter", - "usage": "OutputPort" - } - ], - "inputAppFields": [], - "outputAppFields": [], - "inputApplicationName": "", - "inputApplicationType": "None", - "inputApplicationKey": null, - "inputApplicationDescription": "", - "outputApplicationName": "", - "outputApplicationType": "None", - "outputApplicationKey": null, - "outputApplicationDescription": "" - }, - { - "category": "File", - "categoryType": "Data", - "isGroup": false, - "color": "#2c2c2c", - "drawOrderHint": 0, - "key": -28, - "name": "image_n.10arc.final_mosaic.fits", - "description": "A standard file on a filesystem mounted to the deployment machine", - "x": 2081.9826590656994, - "y": 693.0184320087992, - "radius": 25, - "collapsed": false, - "subject": null, - "expanded": true, - "repositoryUrl": "https://github.com/ICRAR/daliuge", - "commitHash": "96daba4", - "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", - "dataHash": "fb6b9cc58716743665932ec7d3eba94334ea8ab892708fdb4658751a1878ed22", - "fields": [ - { - "name": "filepath", - "value": "", - "defaultValue": "", - "description": "File path for this file. In many cases this does not need to be specified. If it has a \\/ at the end it will be treated as a directory name and the filename will be generated. If it does not have a \\/, the last part will be treated as a filename. If filepath does not start with \\/ (relative path) then the session directory will be prepended to make the path absolute.\"", - "readonly": false, - "type": "String", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "7607300f-5c31-498c-99c1-735f9fe5ea4c", - "parameterType": "ApplicationArgument", - "usage": "NoPort" - }, - { - "name": "check_filepath_exists", - "value": false, - "defaultValue": "False", - "description": "Perform a check to make sure the file path exists before proceeding with the application", - "readonly": false, - "type": "Boolean", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "7fa793d9-866b-45fa-a62c-225134c54402", - "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "dropclass", - "value": "dlg.data.drops.file.FileDROP", - "defaultValue": "dlg.data.drops.file.FileDROP", - "description": "Drop class", - "readonly": false, - "type": "String", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "79a2e7c7-1f96-4203-9e37-2f42616982df", - "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "base_name", - "value": "file", - "defaultValue": "file", - "description": "Base name of application class", - "readonly": false, - "type": "String", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "dc4b6f41-a034-4159-a998-8787b2311f13", - "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "streaming", - "value": false, - "defaultValue": "False", - "description": "Specifies whether this data component streams input and output data", - "readonly": false, - "type": "Boolean", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "1845c151-5499-4401-a705-e036037f65e3", - "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "persist", - "value": true, - "defaultValue": "True", - "description": "Specifies whether this data component contains data that should not be deleted after execution", - "readonly": false, - "type": "Boolean", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "b7e017c8-dfeb-4586-a7a7-96ed05e9aed7", - "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "data_volume", - "value": 5, - "defaultValue": "5", - "description": "Estimated size of the data contained in this node", - "readonly": false, - "type": "Float", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "e6888203-bebe-44db-8b04-ea75f17a2759", - "parameterType": "ConstraintParameter", - "usage": "NoPort" - }, - { - "name": "group_end", - "value": false, - "defaultValue": "False", - "description": "Is this node the end of a group?", - "readonly": false, - "type": "Boolean", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "1230b71d-9588-41fd-a7e4-e5772e249636", + "id": "8730d1a2-6433-4552-bc72-dc500946b9b3", "parameterType": "ComponentParameter", - "usage": "NoPort" - }, - { - "name": "dummy1", - "value": "", - "defaultValue": "", - "description": "Dummy port", - "readonly": false, - "type": "Object", - "precious": false, - "options": [], - "positional": false, - "keyAttribute": false, - "encoding": "pickle", - "id": "158ed104-207b-4f31-abbd-72c5ecd8a066", - "parameterType": "ApplicationArgument", "usage": "InputOutput" } ], @@ -6295,11 +6103,11 @@ "isGroup": false, "color": "#2c2c2c", "drawOrderHint": 0, - "key": -29, - "name": "weights_n.10arc.final_mosaic.fits", + "key": -27, + "name": "moasic-image.fits", "description": "A standard file on a filesystem mounted to the deployment machine", - "x": 2084.9651089451404, - "y": 888.9612176882473, + "x": 1999.5447777756358, + "y": 960.5053328689517, "radius": 25, "collapsed": false, "subject": null, @@ -6321,7 +6129,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "6c79fcc6-2886-4a98-8162-97e5f04d470f", + "id": "343882bc-f1b2-447b-b97f-e2ec1b2204ee", "parameterType": "ApplicationArgument", "usage": "NoPort" }, @@ -6337,7 +6145,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "4a2b79fa-d2a1-4a95-baeb-f822b9e107e7", + "id": "9790a70e-803a-409c-99f2-1b73d359e049", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6353,7 +6161,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "8f4a38c5-15e7-43f6-bece-6d929d724e9b", + "id": "259123a8-2251-48f7-8647-4088cb4161da", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6369,7 +6177,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "2ff5fedc-568a-439a-b82c-8f197d041da9", + "id": "a60eb87c-71b2-4db0-aa06-3eff9c98dc38", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6385,7 +6193,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "7f77ef3a-34e6-4f47-901d-847c211876de", + "id": "a6f99fdf-64e1-4e1d-9ffc-66c1f465c6c5", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6401,7 +6209,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "ce1a1a1d-3d20-4ec0-9e02-172e889b0680", + "id": "0e82241c-489f-47ff-b392-3f7f32c00f5f", "parameterType": "ComponentParameter", "usage": "NoPort" }, @@ -6417,7 +6225,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "7f9bbfe5-cbcf-407d-984e-235d7116cdaf", + "id": "8778253b-4eb6-4343-b48c-8ac7c2914b35", "parameterType": "ConstraintParameter", "usage": "NoPort" }, @@ -6433,12 +6241,12 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "cd2978dd-1210-4706-82fa-33d9e5bf17a4", + "id": "235fd702-b21d-4024-b921-c5c4970af75c", "parameterType": "ComponentParameter", "usage": "NoPort" }, { - "name": "dummy2", + "name": "dummy", "value": "", "defaultValue": "", "description": "Dummy port", @@ -6449,7 +6257,7 @@ "positional": false, "keyAttribute": false, "encoding": "pickle", - "id": "1b257d19-8841-4846-ae8a-720cc3d23ea3", + "id": "4d2f0da2-00b9-488f-a459-ec40aa68d757", "parameterType": "ApplicationArgument", "usage": "InputOutput" } @@ -6668,18 +6476,18 @@ "closesLoop": false }, { - "from": -26, - "fromPort": "ea8fd512-d2b3-4922-9c7f-9b76e75627f0", - "to": -28, - "toPort": "158ed104-207b-4f31-abbd-72c5ecd8a066", - "loop_aware": "0", + "from": -25, + "fromPort": "9c2a5438-d9e0-41e7-b0f7-18fe99b1f7cf", + "to": -26, + "toPort": "8730d1a2-6433-4552-bc72-dc500946b9b3", + "loop_aware": "1", "closesLoop": false }, { "from": -26, - "fromPort": "eacf31bf-c1fe-432b-b21e-f7139c9ac0ec", - "to": -29, - "toPort": "1b257d19-8841-4846-ae8a-720cc3d23ea3", + "fromPort": "8730d1a2-6433-4552-bc72-dc500946b9b3", + "to": -27, + "toPort": "4d2f0da2-00b9-488f-a459-ec40aa68d757", "loop_aware": "0", "closesLoop": false }