add mig step file/load_csv

Core/Executor/FileExecutor.php

@@ -9,13 +9,16 @@
 class FileExecutor extends AbstractExecutor
 {
     use IgnorableStepExecutorTrait;
+    use NonScalarReferenceSetterTrait;
 
     protected $supportedStepTypes = array('file');
-    protected $supportedActions = array('load', 'save', 'copy', 'move', 'delete', 'append', 'prepend', 'exists');
+    protected $supportedActions = array('load', 'load_csv', 'save', 'copy', 'move', 'delete', 'append', 'prepend', 'exists');
 
     /** @var EmbeddedReferenceResolverBagInterface $referenceResolver */
     protected $referenceResolver;
 
+    protected $scalarReferences = array('count');
+
     /**
      * @param EmbeddedReferenceResolverBagInterface $referenceResolver
      */
@@ -45,7 +48,7 @@ public function execute(MigrationStep $step)
 
         $this->skipStepIfNeeded($step);
 
-        return $this->$action($step->dsl, $step->context);
+        return $action == 'load_csv' ? $this->$action($step) : $this->$action($step->dsl, $step->context);
     }
 
     /**
@@ -69,6 +72,61 @@ protected function load($dsl, $context)
         return file_get_contents($fileName);
     }
 
+    /**
+     * @param MigrationStep $step
+     * @return string[][]
+     * @throws InvalidStepDefinitionException
+     * @throws \Kaliop\eZMigrationBundle\API\Exception\InvalidMatchResultsNumberException
+     */
+    protected function load_csv($step)
+    {
+        if (!isset($step->dsl['expect'])) {
+            // for csv files, it makes sense that we expect them to have many rows
+            $step = new MigrationStep(
+                $step->type,
+                array_merge($step->dsl, array('expect' => self::$EXPECT_MANY)),
+                $step->context
+            );
+        }
+
+        $dsl = $step->dsl;
+
+        if (!isset($dsl['file'])) {
+            throw new InvalidStepDefinitionException("Can not load file: name missing");
+        }
+        $fileName = $this->referenceResolver->resolveReference($dsl['file']);
+        if (!file_exists($fileName)) {
+            throw new \Exception("Can not load '$fileName': file missing");
+        }
+
+
+
+        $separator = isset($dsl['separator']) ? $dsl['separator'] : ',';
+        $enclosure = isset($dsl['enclosure']) ? $dsl['enclosure'] : '"';
+        $escape = isset($dsl['escape']) ? $dsl['escape'] : '\\';
+
+        $singleResult = ($this->expectedResultsType($step) == self::$RESULT_TYPE_SINGLE);
+
+        $data = array();
+        if (($handle = fopen($fileName, "r")) !== FALSE) {
+            while (($row = fgetcsv($handle, 0, $separator, $enclosure, $escape)) !== FALSE) {
+                $data[] = $row;
+                if ($singleResult && count($data) > 1) {
+                    break;
+                }
+            }
+            fclose($handle);
+        } else {
+            throw new \Exception("Can not load '$fileName'");
+        }
+
+        $this->validateResultsCount($data, $step);
+        $this->setDataReferences($data, $dsl, $singleResult);
+
+        // NB: this is one of the very few places where we return a nested array...
+        return $data;
+    }
+
     /**
      * @param array $dsl
      * @param array $context
@@ -363,6 +421,47 @@ protected function setReferences($fileName, $dsl)
         return true;
     }
 
+    protected function setdataReferences($data, $dsl, $singleResult)
+    {
+        if (!array_key_exists('references', $dsl)) {
+            return false;
+        }
+
+        foreach ($dsl['references'] as $key => $reference) {
+            $reference = $this->parseReferenceDefinition($key, $reference);
+            switch ($reference['attribute']) {
+                case 'count':
+                    $value = count($data);
+                    break;
+                default:
+                    if (strpos($reference['attribute'], 'column.') !== 0) {
+                        throw new InvalidStepDefinitionException('File Executor does not support setting references for attribute ' . $reference['attribute']);
+                    }
+                    if (count($data)) {
+                        $colNum = substr($reference['attribute'], 7);
+                        if (!isset($data[0][$colNum])) {
+                            throw new \InvalidArgumentException('File Executor does not support setting references for attribute ' . $reference['attribute']);
+                        }
+                        $value = array_column($data, $colNum);
+                        if ($singleResult) {
+                            $value = reset($value);
+                        }
+                    } else {
+                        // we should validate the requested column name, but we can't...
+                        $value = array();
+                    }
+            }
+
+            $overwrite = false;
+            if (isset($reference['overwrite'])) {
+                $overwrite = $reference['overwrite'];
+            }
+            $this->referenceResolver->addReference($reference['identifier'], $value, $overwrite);
+        }
+
+        return true;
+    }
+
     /**
      * Replaces any references inside a string
      *
@@ -374,4 +473,13 @@ protected function resolveReferencesInText($text)
     {
         return $this->referenceResolver->ResolveEmbeddedReferences($text);
     }
+
+    /**
+     * @param array $referenceDefinition
+     * @return bool
+     */
+    protected function isScalarReference($referenceDefinition)
+    {
+        return in_array($referenceDefinition['attribute'], $this->scalarReferences);
+    }
 }

Resources/doc/DSL/Files.yml

@@ -18,6 +18,30 @@
         "reference:_ref_name": # name of a reference to be used for the test
             _operator_: value # allowed operators: eq, gt, gte, lt, lte, ne, count, length, regexp, satisfies
 
+-
+    # Load data from a csv file's contents into a set of references
+    type: file
+    mode: load_csv
+    file: xxx # string. Filename including path. References will be resolved
+    separator: , # optional
+    enclosure: '"' # optional
+    escape: '\' # optional
+    # The list in references tells the manager to store specific values for later use by other steps in the current migration.
+    # NB: these are NEW VARIABLES THAT YOU ARE CREATING. They are not used in the current migration step!
+    references: # Optional
+        # short syntax:
+        referenceId: attributeId # (possible values and meaning are explained for the 'long syntax')
+            # long syntax:
+            -
+                identifier: referenceId # A string used to identify the reference
+                attribute: attributeId # An attribute to get the value of for the reference.
+                # Supports: column.XX, where XX is a column index, count which is the number of rows read
+                overwrite: true|false # Optional, default false. If not set, and the reference already exists, an exception is thrown
+    expect: one|any|many # many by default
+    if: # Optional. If set, the migration step will be skipped unless the condition is matched
+        "reference:_ref_name": # name of a reference to be used for the test
+            _operator_: value # allowed operators: eq, gt, gte, lt, lte, ne, count, length, regexp, satisfies
+
 -
     # Check file existence and save the information into a reference
     type: file

WHATSNEW.md

@@ -7,6 +7,8 @@ Version 5.16.0
 * New: multiple migration steps `url_alias` and `url_wildcard` are now available to manage urls aliases. Please read
   their documentation in Resources/doc/DSL for details
 
+* New: migration step `file/load_csv`, allows to easily initialize references long list of values
+
 * New: everywhere a reference was previously resolved, ie. using `reference:myref` or `[reference:myref]` syntax
   it is now possible to use `eval:expression` or `[eval:expression]`.