Allow to set remote id on location creation; allow to save to file ge…

…nerated migrations; resolve more references
kaliop-uk · Nov 24, 2018 · b2c38b1 · b2c38b1
1 parent 5980e71
commit b2c38b1
Show file tree

Hide file tree

Showing 9 changed files with 145 additions and 18 deletions.
diff --git a/Core/Executor/LocationManager.php b/Core/Executor/LocationManager.php
@@ -70,19 +70,23 @@ protected function create($step)
                 $locationCreateStruct = $locationService->newLocationCreateStruct($parentLocationId);
 
                 if (isset($step->dsl['is_hidden'])) {
-                    $locationCreateStruct->hidden = $step->dsl['is_hidden'];
+                    $locationCreateStruct->hidden = $this->referenceResolver->resolveReference($step->dsl['is_hidden']);
                 }
 
                 if (isset($step->dsl['priority'])) {
-                    $locationCreateStruct->priority = $step->dsl['priority'];
+                    $locationCreateStruct->priority = $this->referenceResolver->resolveReference($step->dsl['priority']);
                 }
 
                 if (isset($step->dsl['sort_order'])) {
-                    $locationCreateStruct->sortOrder = $this->getSortOrder($step->dsl['sort_order']);
+                    $locationCreateStruct->sortOrder = $this->getSortOrder($this->referenceResolver->resolveReference($step->dsl['sort_order']));
                 }
 
                 if (isset($step->dsl['sort_field'])) {
-                    $locationCreateStruct->sortField = $this->getSortField($step->dsl['sort_field']);
+                    $locationCreateStruct->sortField = $this->getSortField($this->referenceResolver->resolveReference($step->dsl['sort_field']));
+                }
+
+                if (isset($step->dsl['remote_id'])) {
+                    $locationCreateStruct->remoteId = $this->referenceResolver->resolveReference($step->dsl['remote_id']);
                 }
 
                 $location = $locationService->createLocation($contentInfo, $locationCreateStruct);
@@ -147,19 +151,19 @@ protected function update($step)
                 $locationUpdateStruct = $locationService->newLocationUpdateStruct();
 
                 if (isset($step->dsl['priority'])) {
-                    $locationUpdateStruct->priority = $step->dsl['priority'];
+                    $locationUpdateStruct->priority = $this->referenceResolver->resolveReference($step->dsl['priority']);
                 }
 
                 if (isset($step->dsl['sort_field'])) {
-                    $locationUpdateStruct->sortField = $this->getSortField($step->dsl['sort_field'], $location->sortField);
+                    $locationUpdateStruct->sortField = $this->getSortField($this->referenceResolver->resolveReference($step->dsl['sort_field']), $location->sortField);
                 }
 
                 if (isset($step->dsl['sort_order'])) {
-                    $locationUpdateStruct->sortOrder = $this->getSortOrder($step->dsl['sort_order'], $location->sortOrder);
+                    $locationUpdateStruct->sortOrder = $this->getSortOrder($this->referenceResolver->resolveReference($step->dsl['sort_order']), $location->sortOrder);
                 }
 
                 if (isset($step->dsl['remote_id'])) {
-                    $locationUpdateStruct->remoteId = $step->dsl['remote_id'];
+                    $locationUpdateStruct->remoteId = $this->referenceResolver->resolveReference($step->dsl['remote_id']);
                 }
 
                 $location = $locationService->updateLocation($location, $locationUpdateStruct);

diff --git a/Core/Executor/MigrationDefinitionExecutor.php b/Core/Executor/MigrationDefinitionExecutor.php
@@ -4,9 +4,10 @@
 
 use Kaliop\eZMigrationBundle\API\Value\MigrationStep;
 use Kaliop\eZMigrationBundle\API\MatcherInterface;
-use Kaliop\eZMigrationBundle\API\ReferenceBagInterface;
+use Kaliop\eZMigrationBundle\API\ReferenceResolverBagInterface;
 use Kaliop\eZMigrationBundle\API\MigrationGeneratorInterface;
 use JmesPath\Env as JmesPath;
+use Symfony\Component\Yaml\Yaml;
 
 class MigrationDefinitionExecutor extends AbstractExecutor
 {
@@ -18,7 +19,7 @@ class MigrationDefinitionExecutor extends AbstractExecutor
     /** @var ReferenceBagInterface $referenceResolver */
     protected $referenceResolver;
 
-    public function __construct($migrationService, ReferenceBagInterface $referenceResolver)
+    public function __construct($migrationService, ReferenceResolverBagInterface $referenceResolver)
     {
         $this->migrationService = $migrationService;
         $this->referenceResolver = $referenceResolver;
@@ -58,11 +59,11 @@ protected function generate($dsl, $context)
         if (!isset($dsl['migration_type'])) {
             throw new \Exception("Invalid step definition: miss 'migration_type'");
         }
-        $migrationType = $dsl['migration_type'];
+        $migrationType = $this->referenceResolver->resolveReference($dsl['migration_type']);
         if (!isset($dsl['migration_mode'])) {
             throw new \Exception("Invalid step definition: miss 'migration_mode'");
         }
-        $migrationMode = $dsl['migration_mode'];
+        $migrationMode = $this->referenceResolver->resolveReference($dsl['migration_mode']);
         if (!isset($dsl['match']) || !is_array($dsl['match'])) {
             throw new \Exception("Invalid step definition: miss 'match' to determine what to generate migration definition for");
         }
@@ -76,15 +77,42 @@ protected function generate($dsl, $context)
 
         $context = array();
         if (isset($dsl['lang']) && $dsl['lang'] != '') {
-            $context['defaultLanguageCode'] = $dsl['lang'];
+            $context['defaultLanguageCode'] = $this->referenceResolver->resolveReference($dsl['lang']);
         }
 
-        $matchCondition = array($match['type'] => $match['value']);
+        $matchCondition = array($this->referenceResolver->resolveReference($match['type']) => $this->referenceResolver->resolveReference($match['value']));
         if (isset($match['except']) && $match['except']) {
             $matchCondition = array(MatcherInterface::MATCH_NOT => $matchCondition);
         }
+
         $result = $executor->generateMigration($matchCondition, $migrationMode, $context);
 
+        if (isset($dsl['file'])) {
+
+            $fileName = $this->referenceResolver->resolveReference($dsl['file']);
+
+            $ext = pathinfo(basename($fileName), PATHINFO_EXTENSION);
+
+            switch ($ext) {
+                case 'yml':
+                case 'yaml':
+                    $code = Yaml::dump($result, 5);
+                    break;
+                case 'json':
+                    $code = json_encode($result, JSON_PRETTY_PRINT);
+                    break;
+                default:
+                    throw new \Exception("Can not save generated migration to a file of type '$ext'");
+            }
+
+            $dir = dirname($fileName);
+            if (!is_dir($dir)) {
+                mkdir($dir, 0777, true);
+            }
+
+            file_put_contents($fileName, $code);
+        }
+
         $this->setReferences($result, $dsl);
 
         return $result;

diff --git a/Core/Matcher/UserMatcher.php b/Core/Matcher/UserMatcher.php
@@ -13,6 +13,7 @@ class UserMatcher extends RepositoryMatcher implements KeyMatcherInterface
     const MATCH_USER_ID = 'user_id';
     const MATCH_USER_LOGIN = 'login';
     const MATCH_USER_EMAIL = 'email';
+    const MATCH_USERGROUP_ID = 'usergroup_id';
 
     protected $allowedConditions = array(
         self::MATCH_AND, self::MATCH_OR,
@@ -53,6 +54,9 @@ public function matchUser(array $conditions)
                 case self::MATCH_USER_LOGIN:
                     return new UserCollection($this->findUsersByLogin($values));
 
+                case self::MATCH_USERGROUP_ID:
+                    return new UserCollection($this->findUsersByGroup($values));
+
                 case self::MATCH_USER_EMAIL:
                     return new UserCollection($this->findUsersByEmail($values));
 
@@ -138,4 +142,28 @@ protected function findUsersByEmail(array $emails)
 
         return $users;
     }
+
+    protected function findUsersByGroup(array $groupsIds)
+    {
+        $users = [];
+
+        foreach ($groupsIds as $groupId) {
+
+            $group = $this->repository->getUserService()->loadUserGroup($groupId);
+
+            $offset = 0;
+            $limit = 100;
+            do {
+                $matches = $this->repository->getUserService()->loadUsersOfUserGroup($group, $offset, $limit);
+                $offset += $limit;
+            } while (count($matches));
+
+            // return unique contents
+            foreach ($matches as $user) {
+                $users[$user->id] = $user;
+            }
+        }
+
+        return $users;
+    }
 }
diff --git a/Resources/doc/DSL/Files.yml b/Resources/doc/DSL/Files.yml
@@ -54,8 +54,8 @@
     type: file
     mode: append
     file: xxx # string. Filename including path. References will be resolved
-    body: "file contents" # string. References will be replaced as long as they are within square brackets
-                          # eg. to save to a file the value of reference 'abc', write: body: "[reference:abc]"
+    body: "appended file contents" # string. References will be replaced as long as they are within square brackets
+                                   # eg. to save to a file the value of reference 'abc', write: body: "[reference:abc]"
     template: path/to/file # optional. Alternative to using the "body" tag: load body from the template file.
                            # References will be resolved in the file path
                            # Path will be resolved as relative to $migrationFile/templates/ first and absolute 2nd
@@ -74,8 +74,8 @@
     type: file
     mode: prepend
     file: xxx # string. Filename including path. References will be resolved
-    body: "file contents" # string. References will be replaced as long as they are within square brackets
-                          # eg. to save to a file the value of reference 'abc', write: body: "[reference:abc]"
+    body: "prepended file contents" # string. References will be replaced as long as they are within square brackets
+                                    # eg. to save to a file the value of reference 'abc', write: body: "[reference:abc]"
     template: path/to/file # optional. Alternative to using the "body" tag: load body from the template file
                            # References will be resolved in the file path
                            # Path will be resolved as relative to $migrationFile/templates/ first and absolute 2nd

diff --git a/Resources/doc/DSL/Locations.yml b/Resources/doc/DSL/Locations.yml
@@ -28,6 +28,7 @@
                   # - contentobject_id
                   # - modified_subnode
     sort_order: ASC|DESC # Optional
+    remote_id: string # optional
     # The list in references tells the manager to store specific values for later use by other steps in the current migration.
     # NB: these are NEW VARIABLES THAT YOU ARE CREATING. They are not used in the current migration step!
     references: # Optional

diff --git a/Resources/doc/DSL/MigrationDefinitions.yml b/Resources/doc/DSL/MigrationDefinitions.yml
@@ -3,6 +3,8 @@
     mode: generate
     migration_type: string # content, content_type, etc...
     migration_mode: string # create, update, delete
+    file: string # path to the file to be generated. NB: if not set, no file is generated at all, and only references are set.
+                 # The filename must end with an extension of either .yml or .json
     match:
         type: xxx # depending on the migration_type, eg: content_id
         value: yyy

diff --git a/Resources/doc/DSL/UsersAndGroups.yml b/Resources/doc/DSL/UsersAndGroups.yml
@@ -29,6 +29,7 @@
         id: x # int|int[]
         email: xyz # string|string[]
         login: # string|string[]
+        usergroup_id: # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
@@ -39,6 +40,37 @@
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
+
+        # NB: if you need more complex matching conditions to select a set of users, you can achieve that using a slightly convoluted way:
+        #     1. use a content/load migration step to match the contents which represent your users
+        #     2. in that step, set an reference, of type array, where you store the content ids
+        #     3. in a second migration step, use a loop/over construct to iterate over the reference and
+        #        the use as substep a user/update where you match based on user_id
+        #        eg:
+        #    -
+        #        type: content
+        #        mode: load
+        #        match:
+        #            contenttype_identifier: editor_user
+        #        references_type: array
+        #        references:
+        #        -
+        #            identifier: editor_user_ids
+        #            attribute: content_id
+        #    -
+        #        type: loop
+        #        over: "reference:editor_user_ids"
+        #        steps:
+        #        -
+        #            type: user
+        #            mode: update
+        #            match:
+        #                user_id: "loop:value"
+        #            password: publish
+        #            attributes:
+        #                first_name: "faker: firstName"
+        #                last_name: "faker: lastName"
+
     email: xyz # Optional. NB: can only be set if the match definition latches a single user
     password: xyz # Optional
     enabled: true|false # Optional
@@ -66,6 +98,7 @@
         id: x # int|int[]
         email: xyz # string|string[]
         login: # string|string[]
+        usergroup_id: # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
@@ -76,6 +109,9 @@
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
+
+        # NB: if you need more complex matching conditions to select a set of users, see user/update above
+
     references: # Optional
          -
             identifier: referenceId # A string used to identify the reference
@@ -94,6 +130,7 @@
         id: x # int|int[]
         email: xyz # string|string[]
         login: # string|string[]
+        usergroup_id: # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
@@ -143,6 +180,7 @@
     match: # Define which groups to update. Only one match condition at a time supported
         id: x # int|int[]
         content_remote_id: yyy # string|string[]
+        parent_usergroup_id: yyy # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
@@ -153,6 +191,9 @@
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
+
+        # NB: if you need more complex matching conditions to select a set of usergroups, see user/update above
+
     name: xyz # Optional. Can only be used when the group to be updated is single
     description: xyz # Optional
     parent_group_id: x # Optional, the new parent user group ID or group's remote_id
@@ -180,6 +221,7 @@
     match: # Only one of the following can be specified, to define which groups to delete
         id: x # int|int[]
         content_remote_id: yyy # string|string[]
+        parent_usergroup_id: yyy # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
@@ -190,6 +232,9 @@
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'
+
+        # NB: if you need more complex matching conditions to select a set of usergroups, see user/update above
+
     # The list in references tells the manager to store specific values for later use by other steps in the current migration.
     # NB: these are NEW VARIABLES THAT YOU ARE CREATING. They are not used in the current migration step!
     references: # Optional
@@ -210,6 +255,7 @@
     match: # Only one of the following can be specified, to define which groups to delete
         id: x # int|int[]
         content_remote_id: yyy # string|string[]
+        parent_usergroup_id: yyy # string|string[]
         or: # match any of the conditions below. *NB:* less efficient that using the array notation for a single condition
             -
                 _condition_: value # where _condition_ can be any of ones specified above, including 'and' and 'or'

diff --git a/Resources/views/MigrationTemplate/dbMigration.sql.twig b/Resources/views/MigrationTemplate/dbMigration.sql.twig
@@ -1 +1,5 @@
 -- Auto-generated migration file. Please customise for your needs (and remove this line ;-)!
+
+-- NB: for mysql, you can add multiple statements in a single SQL file, separated by the semicolon character,
+--     however be careful about the total length, as you might go over the maximun statement length.
+--     In that case, either use multiple sql migrations, or a single yml migration where each step is a sql statement
diff --git a/WHATSNEW.md b/WHATSNEW.md
@@ -1,3 +1,17 @@
+Version 5.5
+===========
+
+* New: it is now possible to generate migration definition files via migration steps.
+    Also, more references are resolved in migrationdefinition/generate steps.
+
+* New: it is now possible to set the location remote_id when creating locations
+    Also, more references are resolved in location/create and location/update steps.
+
+* New: it is now possible to match users using their group id
+
+* New: it is now possible to match user groups using the parent group id
+
+
 Version 5.4.1
 =============