DB gateway (#81)

* Moved commands' db query functions to dedicated Gateway service * Fixed regression in 'PostgreSQL do not support alias in update queries (#78)' commit * Added tests for DB gateway service * Added Gateway tests on Travis * Added kernel 1.13 tests on Travis
ezsystems · Oct 26, 2018 · b1c9485 · b1c9485
1 parent 53fbc5e
commit b1c9485
Show file tree

Hide file tree

Showing 15 changed files with 1,382 additions and 262 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,17 @@
 language: php
 
+services:
+  - mysql
+  - postgresql
+
+# Mysql isn't installed on trusty (only client is), so we need to specifically install it
+addons:
+  apt:
+    packages:
+    - mysql-server-5.6
+    - mysql-client-core-5.6
+    - mysql-client-5.6
+
 matrix:
     include:
         - php: 5.6
@@ -14,12 +26,23 @@ matrix:
           env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="4.10.4" CORES_SETUP="single" SOLR_CORES="collection1" SOLR_CONFIG="vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/schema.xml vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/custom-fields-types.xml vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/language-fieldtypes.xml"
         - php: 7.1
           env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.6.0" CORES_SETUP="shared" SOLR_CONFIG="vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/schema.xml vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/custom-fields-types.xml vendor/ezsystems/ezplatform-solr-search-engine/lib/Resources/config/solr/language-fieldtypes.xml"
+        - php: 7.1
+          env: TEST_CONFIG="phpunit-integration-legacy-empty-db.xml" DB="postgresql" DATABASE="pgsql://postgres@localhost/testdb"
+        - php: 7.1
+          env: TEST_CONFIG="phpunit-integration-legacy-empty-db.xml" DB="mysql" DATABASE="mysql://root@localhost/testdb"
+        - php: 7.0
+          env: TEST_CONFIG="phpunit.xml" COMPOSER_REQUIRE="ezsystems/ezpublish-kernel:^6.13.6@dev"
 
 # test only master (+ Pull requests)
 branches:
     only:
         - master
 
+
+# setup requirements for running db tests
+before_install:
+  - if [ "$TEST_CONFIG" != "" ] ; then ./bin/.travis/prepare_unittest.sh ; fi
+
 before_script:
     # Disable memory_limit for composer
     - echo "memory_limit=-1" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
@@ -30,7 +53,7 @@ before_script:
 
 script:
     - php vendor/bin/phpunit --bootstrap tests/bootstrap.php -c $TEST_CONFIG
-    - if [ "$CHECK_CS" == "true" ]; then phpenv config-rm xdebug.ini && ./vendor/bin/php-cs-fixer fix -v --dry-run --diff --show-progress=estimating; fi
+    - if [ "$CHECK_CS" == "true" ]; then ./vendor/bin/php-cs-fixer fix -v --dry-run --diff --show-progress=estimating; fi
 
 notifications:
     email: false
diff --git a/bin/.travis/prepare_unittest.sh b/bin/.travis/prepare_unittest.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# File for setting up system for unit/integration testing
+
+# Disable xdebug to speed things up as we don't currently generate coverge on travis
+# And make sure we use UTF-8 encoding
+phpenv config-rm xdebug.ini
+echo "default_charset = UTF-8" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
+
+# Setup DB
+if [ "$DB" = "mysql" ] ; then
+    # https://github.com/travis-ci/travis-ci/issues/3049
+    # make sure we don't run out of entropy apparently (see link above)
+    sudo apt-get -y install haveged
+    sudo service haveged start
+    # make tmpfs and run MySQL on it for reasonable performance
+    sudo mkdir /mnt/ramdisk
+    sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk
+    sudo stop mysql
+    sudo mv /var/lib/mysql /mnt/ramdisk
+    sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql
+    sudo start mysql
+    # Install test db
+    mysql -e "CREATE DATABASE IF NOT EXISTS testdb DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;" -uroot
+fi
+if [ "$DB" = "postgresql" ] ; then psql -c "CREATE DATABASE testdb;" -U postgres ; psql -c "CREATE EXTENSION pgcrypto;" -U postgres testdb ; fi
+
+if [ "$COMPOSER_REQUIRE" != "" ] ; then composer require --no-update $COMPOSER_REQUIRE ; fi
+#travis_retry composer update --no-progress --no-interaction $COMPSER_ARGS
diff --git a/bundle/Command/ConvertXmlTextToRichTextCommand.php b/bundle/Command/ConvertXmlTextToRichTextCommand.php
@@ -15,7 +15,7 @@
 use Symfony\Component\Console\Style\SymfonyStyle;
 use eZ\Publish\Core\FieldType\XmlText\Value;
 use eZ\Publish\Core\FieldType\XmlText\Converter\RichText as RichTextConverter;
-use Doctrine\DBAL\Connection;
+use eZ\Publish\Core\FieldType\XmlText\Persistence\Legacy\ContentModelGateway as Gateway;
 use Symfony\Component\Debug\Exception\ContextErrorException;
 use Symfony\Component\Process\PhpExecutableFinder;
 use Symfony\Component\Process\ProcessBuilder;
@@ -27,9 +27,9 @@ class ConvertXmlTextToRichTextCommand extends ContainerAwareCommand
     const DEFAULT_REPOSITORY_USER = 'admin';
 
     /**
-     * @var \Doctrine\DBAL\Connection
+     * @var \eZ\Publish\Core\FieldType\XmlText\Persistence\Legacy\ContentModelGateway
      */
-    private $dbal;
+    private $gateway;
 
     /**
      * @var \Psr\Log\LoggerInterface
@@ -81,11 +81,11 @@ class ConvertXmlTextToRichTextCommand extends ContainerAwareCommand
      */
     protected $kernelCacheDir;
 
-    public function __construct(Connection $dbal, RichTextConverter $converter, $kernelCacheDir, LoggerInterface $logger)
+    public function __construct(Gateway $gateway, RichTextConverter $converter, $kernelCacheDir, LoggerInterface $logger)
     {
         parent::__construct();
 
-        $this->dbal = $dbal;
+        $this->gateway = $gateway;
         $this->converter = $converter;
         $this->kernelCacheDir = $kernelCacheDir;
         $this->logger = $logger;
@@ -258,7 +258,7 @@ protected function baseExecute(InputInterface $input, OutputInterface $output, &
         } else {
             $this->imageContentTypeIdentifiers = ['image'];
         }
-        $imageContentTypeIds = $this->getContentTypeIds($this->imageContentTypeIdentifiers);
+        $imageContentTypeIds = $this->gateway->getContentTypeIds($this->imageContentTypeIdentifiers);
         if (count($imageContentTypeIds) !== count($this->imageContentTypeIdentifiers)) {
             throw new RuntimeException('Unable to lookup all content type identifiers, not found: ' . implode(',', array_diff($this->imageContentTypeIdentifiers, array_keys($imageContentTypeIds))));
         }
@@ -378,7 +378,7 @@ protected function login()
 
     protected function fixEmbeddedImages($dryRun, $contentId, OutputInterface $output)
     {
-        $count = $this->getRowCountOfContentObjectAttributes('ezrichtext', $contentId);
+        $count = $this->gateway->getRowCountOfContentObjectAttributes('ezrichtext', $contentId);
 
         $output->writeln("Found $count field rows to convert.");
 
@@ -387,7 +387,7 @@ protected function fixEmbeddedImages($dryRun, $contentId, OutputInterface $outpu
         do {
             $limit = self::MAX_OBJECTS_PER_CHILD;
 
-            $statement = $this->getFieldRows('ezrichtext', $contentId, $offset, $limit);
+            $statement = $this->gateway->getFieldRows('ezrichtext', $contentId, $offset, $limit);
             while ($row = $statement->fetch(PDO::FETCH_ASSOC)) {
                 if (empty($row['data_text'])) {
                     $inputValue = Value::EMPTY_VALUE;
@@ -437,141 +437,21 @@ protected function fixEmbeddedImages($dryRun, $contentId, OutputInterface $outpu
         $output->writeln("Updated ezembed tags in $totalCount field(s)");
     }
 
-    protected function convertFieldDefinitions($dryRun, OutputInterface $output)
+    protected function convertFieldDefinitions($dryRun, $output)
     {
-        $query = $this->dbal->createQueryBuilder();
-        $query->select('count(a.id)')
-            ->from('ezcontentclass_attribute', 'a')
-            ->where(
-                $query->expr()->eq(
-                    'a.data_type_string',
-                    ':datatypestring'
-                )
-            )
-            ->setParameter(':datatypestring', 'ezxmltext');
-
-        $statement = $query->execute();
-        $count = (int) $statement->fetchColumn();
-
+        $count = $this->gateway->countContentTypeFieldsByFieldType('ezxmltext');
         $output->writeln("Found $count field definiton to convert.");
 
-        $updateQuery = $this->dbal->createQueryBuilder();
-        $updateQuery->update('ezcontentclass_attribute')
-            ->set('data_type_string', ':newdatatypestring')
-            // was tagPreset in ezxmltext, unused in RichText
-            ->set('data_text2', ':datatext2')
-            ->where(
-                $updateQuery->expr()->eq(
-                    'data_type_string',
-                    ':olddatatypestring'
-                )
-            )
-            ->setParameters([
-                ':newdatatypestring' => 'ezrichtext',
-                ':datatext2' => null,
-                ':olddatatypestring' => 'ezxmltext',
-            ]);
-
+        $updateQuery = $this->gateway->getContentTypeFieldTypeUpdateQuery('ezxmltext', 'ezrichtext');
         if (!$dryRun) {
             $updateQuery->execute();
         }
-
         $output->writeln("Converted $count ezxmltext field definitions to ezrichtext");
     }
 
-    protected function getRowCountOfContentObjectAttributes($datatypeString, $contentId)
-    {
-        $query = $this->dbal->createQueryBuilder();
-        $query->select('count(a.id)')
-            ->from('ezcontentobject_attribute', 'a')
-            ->where(
-                $query->expr()->eq(
-                    'a.data_type_string',
-                    ':datatypestring'
-                )
-            )
-            ->setParameter(':datatypestring', $datatypeString);
-
-        if ($contentId !== null) {
-            $query->andWhere(
-                $query->expr()->eq(
-                    'a.contentobject_id',
-                    ':contentid'
-                )
-            )
-                ->setParameter(':contentid', $contentId);
-        }
-
-        $statement = $query->execute();
-
-        return (int) $statement->fetchColumn();
-    }
-
-    /**
-     * Get the specified field rows.
-     * Note that if $contentId !== null, then $offset and $limit will be ignored.
-     *
-     * @param $datatypeString
-     * @param $contentId
-     * @param $offset
-     * @param $limit
-     * @return \Doctrine\DBAL\Driver\Statement|int
-     */
-    protected function getFieldRows($datatypeString, $contentId, $offset, $limit)
-    {
-        $query = $this->dbal->createQueryBuilder();
-        $query->select('a.*')
-            ->from('ezcontentobject_attribute', 'a')
-            ->where(
-                $query->expr()->eq(
-                    'a.data_type_string',
-                    ':datatypestring'
-                )
-            )
-            ->orderBy('a.id')
-            ->setParameter(':datatypestring', $datatypeString);
-
-        if ($contentId === null) {
-            $query->setFirstResult($offset)
-                ->setMaxResults($limit);
-        } else {
-            $query->andWhere(
-                $query->expr()->eq(
-                    'a.contentobject_id',
-                    ':contentid'
-                )
-            )
-                ->setParameter(':contentid', $contentId);
-        }
-
-        return $query->execute();
-    }
-
     protected function updateFieldRow($dryRun, $id, $version, $datatext)
     {
-        $updateQuery = $this->dbal->createQueryBuilder();
-        $updateQuery->update('ezcontentobject_attribute')
-            ->set('data_type_string', ':datatypestring')
-            ->set('data_text', ':datatext')
-            ->where(
-                $updateQuery->expr()->eq(
-                    'id',
-                    ':id'
-                )
-            )
-            ->andWhere(
-                $updateQuery->expr()->eq(
-                    'version',
-                    ':version'
-                )
-            )
-            ->setParameters([
-                ':datatypestring' => 'ezrichtext',
-                ':datatext' => $datatext,
-                ':id' => $id,
-                ':version' => $version,
-            ]);
-
+        $updateQuery = $this->gateway->getUpdateFieldRowQuery($id, $version, $datatext);
         if (!$dryRun) {
             $updateQuery->execute();
         }
@@ -709,7 +589,7 @@ protected function dumpOnErrors($errors, $dataText, $contentobjectId, $contentob
 
     protected function convertFields($dryRun, $contentId, $checkDuplicateIds, $checkIdValues, $offset, $limit)
     {
-        $statement = $this->getFieldRows('ezxmltext', $contentId, $offset, $limit);
+        $statement = $this->gateway->getFieldRows('ezxmltext', $contentId, $offset, $limit);
         while ($row = $statement->fetch(PDO::FETCH_ASSOC)) {
             if (empty($row['data_text'])) {
                 $inputValue = Value::EMPTY_VALUE;
@@ -745,7 +625,7 @@ protected function convertFields($dryRun, $contentId, $checkDuplicateIds, $check
 
     protected function processFields($dryRun, $checkDuplicateIds, $checkIdValues, OutputInterface $output)
     {
-        $count = $this->getRowCountOfContentObjectAttributes('ezxmltext', null);
+        $count = $this->gateway->getRowCountOfContentObjectAttributes('ezxmltext', null);
         $output->writeln("Found $count field rows to convert.");
 
         $offset = 0;