From 2886a8c46e2ec0b9aa426819465559b271e55379 Mon Sep 17 00:00:00 2001
From: Vincent Gao <vincent.gao@dpc.vic.gov.au>
Date: Thu, 18 Jul 2024 17:02:46 +1000
Subject: [PATCH] adds mutiple_value_processor plugin for datapipe

---
 .../MutipleValueProcessor.php                 |  74 ++++++++++++
 ...ide_data_pipelines_test.data_pipelines.yml |  44 +++++++
 .../tide_data_pipelines_test.info.yml         |  10 ++
 .../Transform/TideSearchTransformTest.php     | 107 ++++++++++++++++++
 .../test-pipeline-mutiple_value_processor.csv |   3 +
 5 files changed, 238 insertions(+)
 create mode 100644 modules/tide_data_pipeline/src/Plugin/DatasetTransform/MutipleValueProcessor.php
 create mode 100644 modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.data_pipelines.yml
 create mode 100644 modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.info.yml
 create mode 100644 modules/tide_data_pipeline/tests/src/Kernel/Transform/TideSearchTransformTest.php
 create mode 100644 modules/tide_data_pipeline/tests/src/fixtures/test-pipeline-mutiple_value_processor.csv

diff --git a/modules/tide_data_pipeline/src/Plugin/DatasetTransform/MutipleValueProcessor.php b/modules/tide_data_pipeline/src/Plugin/DatasetTransform/MutipleValueProcessor.php
new file mode 100644
index 0000000..f075f2c
--- /dev/null
+++ b/modules/tide_data_pipeline/src/Plugin/DatasetTransform/MutipleValueProcessor.php
@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\tide_data_pipeline\Plugin\DatasetTransform;
+
+use Drupal\data_pipelines\DatasetData;
+use Drupal\data_pipelines\Transform\TransformPluginBase;
+
+/**
+ * Splits a string into multiple values and optionally processes them.
+ *
+ * @DatasetTransform(
+ *   id="mutiple_value_processor",
+ *   fields=TRUE,
+ *   records=FALSE
+ * )
+ */
+class MutipleValueProcessor extends TransformPluginBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function defaultConfiguration() {
+    return parent::defaultConfiguration() + [
+      'separator' => '',
+      'callback' => NULL,
+      'parameters' => [],
+    ];
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function doTransformField(string $field_name, DatasetData $record): DatasetData {
+    $record = parent::doTransformRecord($record);
+    if ($record->offsetExists($field_name) && !empty($record[$field_name])) {
+      $separator = $this->configuration['separator'];
+      $callback = $this->configuration['callback'];
+      $parameters = $this->configuration['parameters'];
+      $parts = explode($separator, $record[$field_name]);
+      $cleaned_parts = array_values(array_filter(array_map('trim', $parts), function ($part) {
+        return $part !== '';
+      }));
+
+      // Process the parts if a callback is provided.
+      if (is_callable($callback)) {
+        $cleaned_parts = array_map(function ($value) use ($callback, $parameters) {
+          $typed_parameters = array_map([$this, 'convertParameter'], $parameters);
+          return call_user_func_array($callback, array_merge([$value], $typed_parameters));
+        }, $cleaned_parts);
+      }
+      $record[$field_name] = $cleaned_parts;
+    }
+    return $record;
+  }
+
+  /**
+   * Converts a parameter to its appropriate type.
+   */
+  private function convertParameter($parameter) {
+    if (is_numeric($parameter)) {
+      return $parameter + 0;
+    }
+    if ($parameter === 'true' || $parameter === 'false') {
+      return $parameter === 'true';
+    }
+    if (is_string($parameter) && defined($parameter)) {
+      return constant($parameter);
+    }
+    return $parameter;
+  }
+
+}
diff --git a/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.data_pipelines.yml b/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.data_pipelines.yml
new file mode 100644
index 0000000..1d65828
--- /dev/null
+++ b/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.data_pipelines.yml
@@ -0,0 +1,44 @@
+pipeline_with_mutiple_value_processor_transform:
+  label: 'Mutiple Value Processor transform'
+  transforms:
+    field:
+      Suburbs:
+        - plugin: mutiple_value_processor
+          separator: ';'
+          callback: str_pad
+          parameters:
+            - 10
+            - '0'
+            - STR_PAD_LEFT
+
+pipeline_with_strtoupper:
+  label: 'Mutiple Value Processor with strtoupper'
+  transforms:
+    field:
+      Suburbs:
+        - plugin: mutiple_value_processor
+          separator: ';'
+          callback: strtoupper
+
+pipeline_with_mb_convert_case:
+  label: 'Mutiple Value Processor with mb_convert_case'
+  transforms:
+    field:
+      Suburbs:
+        - plugin: mutiple_value_processor
+          separator: ';'
+          callback: mb_convert_case
+          parameters:
+            - 'MB_CASE_UPPER'
+
+pipeline_with_substr:
+  label: 'Mutiple Value Processor with substr'
+  transforms:
+    field:
+      Suburbs:
+        - plugin: mutiple_value_processor
+          separator: ';'
+          callback: substr
+          parameters:
+            - '0'
+            - 'true'
diff --git a/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.info.yml b/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.info.yml
new file mode 100644
index 0000000..f7e218b
--- /dev/null
+++ b/modules/tide_data_pipeline/tests/modules/tide_data_pipelines_test/tide_data_pipelines_test.info.yml
@@ -0,0 +1,10 @@
+name: Data Pipelines - Test Pipelines
+package: Testing
+description: 'Provides tests data set plugins'
+core_version_requirement: ^10.1
+php: 8.0
+type: module
+dependencies:
+  - data_pipelines:data_pipelines
+  - data_pipelines_test:data_pipelines_test
+  - tide_data_pipeline:tide_data_pipeline
diff --git a/modules/tide_data_pipeline/tests/src/Kernel/Transform/TideSearchTransformTest.php b/modules/tide_data_pipeline/tests/src/Kernel/Transform/TideSearchTransformTest.php
new file mode 100644
index 0000000..5406d86
--- /dev/null
+++ b/modules/tide_data_pipeline/tests/src/Kernel/Transform/TideSearchTransformTest.php
@@ -0,0 +1,107 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\Tests\tide_data_pipeline\Kernel\Transform;
+
+use Drupal\data_pipelines\Entity\Dataset;
+use Drupal\Tests\data_pipelines\Kernel\Transform\TransformTest;
+
+/**
+ * Defines a class for testing transform functionality.
+ *
+ * @group data_pipelines
+ *
+ * @covers \Drupal\tide_data_pipeline\Transform\MutipleValueProcessor
+ */
+class TideSearchTransformTest extends TransformTest {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'options',
+    'link',
+    'file',
+    'entity',
+    'data_pipelines',
+    'data_pipelines_test',
+    'user',
+    'system',
+    'tide_data_pipeline',
+    'tide_data_pipelines_test',
+  ];
+
+  /**
+   * Test mutiple_value_processor transform.
+   */
+  public function testMutipleValueProcessorTransform(): void {
+    $file = $this->getTestFile(dirname(__DIR__, 2) . '/fixtures/test-pipeline-mutiple_value_processor.csv');
+    $dataset = Dataset::create([
+      'source' => 'csv:file',
+      'name' => $this->randomMachineName(),
+      'machine_name' => mb_strtolower($this->randomMachineName()),
+      'pipeline' => 'pipeline_with_mutiple_value_processor_transform',
+      'csv_file' => $file,
+    ]);
+    $data = iterator_to_array($dataset->getDataIterator());
+    $this->assertCount(2, $data);
+    $this->assertEquals(['0Dandenong', 'Dandenong North'], $data[0]['Suburbs']);
+    $this->assertEquals(['00000Boneo', '000Outtrim'], $data[1]['Suburbs']);
+  }
+
+  /**
+   * Test multiple_value_processor transform with strtoupper callback.
+   */
+  public function testMutipleValueProcessorTransformWithStrtoupper(): void {
+    $file = $this->getTestFile(dirname(__DIR__, 2) . '/fixtures/test-pipeline-mutiple_value_processor.csv');
+    $dataset = Dataset::create([
+      'source' => 'csv:file',
+      'name' => $this->randomMachineName(),
+      'machine_name' => mb_strtolower($this->randomMachineName()),
+      'pipeline' => 'pipeline_with_strtoupper',
+      'csv_file' => $file,
+    ]);
+    $data = iterator_to_array($dataset->getDataIterator());
+    $this->assertCount(2, $data);
+    $this->assertEquals(['DANDENONG', 'DANDENONG NORTH'], $data[0]['Suburbs']);
+    $this->assertEquals(['BONEO', 'OUTTRIM'], $data[1]['Suburbs']);
+  }
+
+  /**
+   * Test multiple_value_processor transform with mb_convert_case.
+   */
+  public function testMutipleValueProcessorTransformWithConvertCase(): void {
+    $file = $this->getTestFile(dirname(__DIR__, 2) . '/fixtures/test-pipeline-mutiple_value_processor.csv');
+    $dataset = Dataset::create([
+      'source' => 'csv:file',
+      'name' => $this->randomMachineName(),
+      'machine_name' => mb_strtolower($this->randomMachineName()),
+      'pipeline' => 'pipeline_with_mb_convert_case',
+      'csv_file' => $file,
+    ]);
+    $data = iterator_to_array($dataset->getDataIterator());
+    $this->assertCount(2, $data);
+    $this->assertEquals(['DANDENONG', 'DANDENONG NORTH'], $data[0]['Suburbs']);
+    $this->assertEquals(['BONEO', 'OUTTRIM'], $data[1]['Suburbs']);
+  }
+
+  /**
+   * Test multiple_value_processor transform with substr.
+   */
+  public function testMutipleValueProcessorTransformWithSubstr(): void {
+    $file = $this->getTestFile(dirname(__DIR__, 2) . '/fixtures/test-pipeline-mutiple_value_processor.csv');
+    $dataset = Dataset::create([
+      'source' => 'csv:file',
+      'name' => $this->randomMachineName(),
+      'machine_name' => mb_strtolower($this->randomMachineName()),
+      'pipeline' => 'pipeline_with_substr',
+      'csv_file' => $file,
+    ]);
+    $data = iterator_to_array($dataset->getDataIterator());
+    $this->assertCount(2, $data);
+    $this->assertEquals(['D', 'D'], $data[0]['Suburbs']);
+    $this->assertEquals(['B', 'O'], $data[1]['Suburbs']);
+  }
+
+}
diff --git a/modules/tide_data_pipeline/tests/src/fixtures/test-pipeline-mutiple_value_processor.csv b/modules/tide_data_pipeline/tests/src/fixtures/test-pipeline-mutiple_value_processor.csv
new file mode 100644
index 0000000..18ed7e7
--- /dev/null
+++ b/modules/tide_data_pipeline/tests/src/fixtures/test-pipeline-mutiple_value_processor.csv
@@ -0,0 +1,3 @@
+Suburbs
+Dandenong; Dandenong North
+Boneo;Outtrim