Merge pull request #47 from itk-dev/feature/1638_add_force_option_to_…

…import_command

Feature/1638 add force option to import command

CHANGELOG.md

@@ -50,6 +50,7 @@ See [keep a changelog] for information about writing changes to this log.
 - Update FeedDefaultsMapper to set default values for empty nested properties
 - Better error handling for import flow
 - Force UTC for all timestamps persisted in the database
+- Add `--force` option to `app:feed:import` command
 
 [keep a changelog]: https://keepachangelog.com/en/1.1.0/
 [unreleased]: https://github.com/itk-dev/event-database-imports/compare/main...develop

docker-compose.server.override.yml

@@ -5,6 +5,27 @@ services:
     depends_on:
       - rabbit
 
+  supervisor:
+    # @TODO: Create supervisor with ansible, when merged with Ture's PR
+    image: itkdev/supervisor-php8.3:alpine
+    restart: unless-stopped
+    stop_grace_period: 20s
+    environment:
+      - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_default
+      - APP_SUPERVISOR_WORKERS=4
+      - APP_SUPERVISOR_USER=deploy
+      - PHP_MAX_EXECUTION_TIME=30
+      - PHP_MEMORY_LIMIT=256M
+      - PHP_TIMEZONE=UTC
+    networks:
+      - app
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - .:/app
+    depends_on:
+      - rabbit
+
   nginx:
     environment:
       APP_PATH_PREFIX: ${APP_PATH_PREFIX}
@@ -28,23 +49,3 @@ services:
       - "RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE}"
     volumes:
       - "./.docker/data/rabbit:/var/lib/rabbitmq/mnesia"
-
-  supervisor:
-    # @TODO: Create supervisor with ansible, when merged with Ture's PR
-    image: itkdev/supervisor-php8.3:alpine
-    restart: unless-stopped
-    stop_grace_period: 20s
-    environment:
-      - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async
-      - APP_SUPERVISOR_WORKERS=4
-      - APP_SUPERVISOR_USER=deploy
-      - PHP_MAX_EXECUTION_TIME=30
-      - PHP_MEMORY_LIMIT=256M
-    networks:
-      - app
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - .:/app
-    depends_on:
-      - rabbit

src/Command/Feed/FeedImportCommand.php

@@ -45,7 +45,8 @@ public function __construct(
     protected function configure(): void
     {
         $this->addOption('feed-id', '', InputOption::VALUE_REQUIRED, 'Limit imports to the feed ID given', self::DEFAULT_OPTION)
-            ->addOption('limit', '', InputOption::VALUE_REQUIRED, 'Limit the number of items parsed pr. feed', self::DEFAULT_OPTION);
+            ->addOption('limit', '', InputOption::VALUE_REQUIRED, 'Limit the number of items parsed pr. feed', self::DEFAULT_OPTION)
+            ->addOption('force', '', InputOption::VALUE_NONE, 'Force update from feed ignoring hash');
     }
 
     /**
@@ -56,6 +57,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $io = new SymfonyStyle($input, $output);
         $feedId = (int) $input->getOption('feed-id');
         $limit = (int) $input->getOption('limit');
+        $force = $input->getOption('force');
 
         $feeds = $this->loadFeeds($io, $feedId);
         if (empty($feeds)) {
@@ -92,7 +94,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             foreach ($this->feedParser->parse($feed, $config->url, $config->rootPointer) as $item) {
                 $feedId = $feed->getId();
                 if (!is_null($feedId)) {
-                    $message = new FeedItemDataMessage($feedId, $config, $item);
+                    $message = new FeedItemDataMessage($feedId, $config, $item, $force);
                     try {
                         $this->messageBus->dispatch($message);
                     } catch (TransportException|\LogicException) {

src/Message/EventMessage.php

@@ -7,12 +7,18 @@
 final class EventMessage
 {
     public function __construct(
-        private readonly FeedItem $item
+        private readonly FeedItem $item,
+        private readonly bool $forceUpdate = false
     ) {
     }
 
     public function getItem(): FeedItem
     {
         return $this->item;
     }
+
+    public function isForceUpdate(): bool
+    {
+        return $this->forceUpdate;
+    }
 }

src/Message/FeedItemDataMessage.php

@@ -10,6 +10,7 @@ public function __construct(
         private readonly int $feedId,
         private readonly FeedConfiguration $configuration,
         private readonly array $data,
+        private readonly bool $forceUpdate = false
     ) {
     }
 
@@ -27,4 +28,9 @@ public function getData(): array
     {
         return $this->data;
     }
+
+    public function isForceUpdate(): bool
+    {
+        return $this->forceUpdate;
+    }
 }

src/Message/FeedNormalizationMessage.php

@@ -10,6 +10,7 @@ final class FeedNormalizationMessage
     public function __construct(
         private readonly FeedConfiguration $configuration,
         private readonly FeedItem $item,
+        private readonly bool $forceUpdate,
     ) {
     }
 
@@ -22,4 +23,9 @@ public function getItem(): FeedItem
     {
         return $this->item;
     }
+
+    public function isForceUpdate(): bool
+    {
+        return $this->forceUpdate;
+    }
 }

src/MessageHandler/EventHandler.php

@@ -23,7 +23,7 @@ public function __invoke(EventMessage $message): void
         $item = $message->getItem();
 
         try {
-            if ($this->eventFactory->isUpdatableOrNew($item)) {
+            if ($message->isForceUpdate() || $this->eventFactory->isUpdatableOrNew($item)) {
                 try {
                     $entity = $this->eventFactory->createOrUpdate($item);
 

src/MessageHandler/FeedItemDataHandler.php

@@ -28,6 +28,6 @@ public function __invoke(FeedItemDataMessage $message): void
             throw new UnrecoverableMessageHandlingException($e->getMessage(), $e->getCode(), $e);
         }
 
-        $this->messageBus->dispatch(new FeedNormalizationMessage($message->getConfiguration(), $feedItem));
+        $this->messageBus->dispatch(new FeedNormalizationMessage($message->getConfiguration(), $feedItem, $message->isForceUpdate()));
     }
 }

src/MessageHandler/FeedNormalizationHandler.php

@@ -49,6 +49,6 @@ public function __invoke(FeedNormalizationMessage $message): void
             }
         }
 
-        $this->messageBus->dispatch(new EventMessage($item));
+        $this->messageBus->dispatch(new EventMessage($item, $message->isForceUpdate()));
     }
 }

tests/OccurrencesFactoryTest.php → tests/Factory/OccurrencesFactoryTest.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Factory;
 
 use App\Entity\Event;
 use App\Entity\Occurrence;

tests/ContentNormalizationTest.php → tests/Service/ContentNormalizerTest.php

@@ -2,15 +2,15 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service;
 
 use App\Service\ContentNormalizer;
 use App\Tests\Utils\TestData;
 use PHPUnit\Framework\Attributes\CoversClass;
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(ContentNormalizer::class)]
-final class ContentNormalizationTest extends KernelTestCase
+final class ContentNormalizerTest extends KernelTestCase
 {
     /**
      * @throws \Exception
@@ -54,7 +54,7 @@ public function testTrimLength()
     private function getContentNormalizerService(): ContentNormalizer
     {
         self::bootKernel();
-        $container = ContentNormalizationTest::getContainer();
+        $container = ContentNormalizerTest::getContainer();
 
         return $container->get(ContentNormalizer::class);
     }

tests/FeedDefaultMapperTest.php → tests/Service/Feeds/FeedDefaultsMapperTest.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service\Feeds;
 
 use App\Model\Feed\FeedConfiguration;
 use App\Service\Feeds\FeedDefaultsMapper;
@@ -11,7 +11,7 @@
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(FeedDefaultsMapper::class)]
-final class FeedDefaultMapperTest extends KernelTestCase
+final class FeedDefaultsMapperTest extends KernelTestCase
 {
     /**
      * Test mapping of default values.
@@ -108,7 +108,7 @@ public function testApply(): void
     private function getFeedDefaultMapper(): FeedDefaultsMapper
     {
         self::bootKernel();
-        $container = FeedDefaultMapperTest::getContainer();
+        $container = FeedDefaultsMapperTest::getContainer();
 
         return $container->get(FeedDefaultsMapper::class);
     }

tests/FeedItemSourceTest.php → tests/Service/Feeds/Mapper/Source/FeedItemSourceTest.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service\Feeds\Mapper\Source;
 
 use App\Model\Feed\FeedConfiguration;
 use App\Service\Feeds\FeedDefaultsMapper;

tests/GeocoderServiceTest.php → tests/Service/GeocoderTest.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service;
 
 use App\Entity\Address;
 use App\Service\Geocoder;
@@ -11,7 +11,7 @@
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(Geocoder::class)]
-final class GeocoderServiceTest extends KernelTestCase
+final class GeocoderTest extends KernelTestCase
 {
     /**
      * @throws \Exception
@@ -43,7 +43,7 @@ public function testBuildQuery()
     private function getGeocoderService(): Geocoder
     {
         self::bootKernel();
-        $container = GeocoderServiceTest::getContainer();
+        $container = GeocoderTest::getContainer();
 
         return $container->get(Geocoder::class);
     }

tests/ImageServiceTest.php → tests/Service/ImageHandlerTest.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service;
 
 use App\Exception\ImageMineTypeException;
 use App\Service\ImageHandler;
@@ -11,7 +11,7 @@
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(ImageHandler::class)]
-final class ImageServiceTest extends KernelTestCase
+final class ImageHandlerTest extends KernelTestCase
 {
     /**
      * @throws \Exception
@@ -83,7 +83,7 @@ public function testMineTypesException()
     private function getImageService(): ImageHandler
     {
         self::bootKernel();
-        $container = ImageServiceTest::getContainer();
+        $container = ImageHandlerTest::getContainer();
 
         return $container->get(ImageHandler::class);
     }

tests/TagsNormalizationTest.php → tests/Service/TagsNormalizerTest.php

@@ -2,15 +2,15 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service;
 
 use App\Service\TagsNormalizer;
 use App\Tests\Utils\PhpUnitUtils;
 use PHPUnit\Framework\Attributes\CoversClass;
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(TagsNormalizer::class)]
-final class TagsNormalizationTest extends KernelTestCase
+final class TagsNormalizerTest extends KernelTestCase
 {
     /**
      * @throws \Exception
@@ -46,7 +46,7 @@ public function testTrimLength()
     private function getTagsNormalizerService(): TagsNormalizer
     {
         self::bootKernel();
-        $container = TagsNormalizationTest::getContainer();
+        $container = TagsNormalizerTest::getContainer();
 
         return $container->get(TagsNormalizer::class);
     }

tests/TimeServiceTest.php → tests/Service/TimeTest.php

@@ -2,14 +2,14 @@
 
 declare(strict_types=1);
 
-namespace App\Tests;
+namespace App\Tests\Service;
 
 use App\Service\Time;
 use PHPUnit\Framework\Attributes\CoversClass;
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
 
 #[CoversClass(Time::class)]
-final class TimeServiceTest extends KernelTestCase
+final class TimeTest extends KernelTestCase
 {
     /**
      * @throws \Exception
@@ -79,7 +79,7 @@ public function testTimeIntervalMultiDays()
     private function getTimeService(): Time
     {
         self::bootKernel();
-        $container = TimeServiceTest::getContainer();
+        $container = TimeTest::getContainer();
 
         return $container->get(Time::class);
     }