databox basket sync expose multipart

alchemy-fr · Nov 21, 2024 · a269bee · a269bee
1 parent 9e354e0
commit a269bee
Show file tree

Hide file tree

Showing 6 changed files with 214 additions and 25 deletions.
diff --git a/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php b/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php
@@ -2,16 +2,17 @@
 
 namespace App\Integration\Phrasea\Expose;
 
-use App\Asset\Attribute\AssetTitleResolver;
-use App\Asset\Attribute\AttributesResolver;
 use App\Asset\FileFetcher;
-use App\Attribute\AttributeInterface;
 use App\Entity\Core\Asset;
 use App\Entity\Core\Attribute;
-use App\Entity\Integration\IntegrationToken;
+use App\Storage\RenditionManager;
+use App\Attribute\AttributeInterface;
 use App\Integration\IntegrationConfig;
+use App\Asset\Attribute\AssetTitleResolver;
+use App\Asset\Attribute\AttributesResolver;
+use App\Entity\Integration\IntegrationToken;
+use Alchemy\StorageBundle\Upload\UploadManager;
 use App\Integration\Phrasea\PhraseaClientFactory;
-use App\Storage\RenditionManager;
 use Symfony\Contracts\HttpClient\HttpClientInterface;
 
 final readonly class ExposeClient
@@ -23,6 +24,7 @@ public function __construct(
         private AssetTitleResolver $assetTitleResolver,
         private AttributesResolver $attributesResolver,
         private RenditionManager $renditionManager,
+        private UploadManager $uploadManager
     ) {
     }
 
@@ -149,13 +151,86 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati
             ;
             $exposeAssetId = $pubAsset['id'];
 
-            $this->uploadClient->request('PUT', $pubAsset['uploadURL'], [
-                'headers' => [
-                    'Content-Type' => $source->getType(),
-                    'Content-Length' => filesize($fetchedFilePath),
-                ],
-                'body' => fopen($fetchedFilePath, 'r'),
-            ]);
+            $uploadsData = [
+                'filename' => $source->getOriginalName(),
+                'type' => $source->getType(),
+                'size' => (int)$source->getSize(),
+            ];
+
+            $resUploads = $this->create($config, $integrationToken)
+                ->request('POST', '/uploads', [
+                    'json' => $uploadsData,
+                ])
+                ->toArray()
+            ;
+
+            $mUploadId = $resUploads['id'];
+            $parts['Parts'] = [];
+            $isPartsComplete = false;
+
+            // Upload the file in parts.
+            try {
+                $file = fopen($fetchedFilePath, 'r');
+                $partNumber = 1;
+                while (!feof($file)) {
+                    $resUploadPart = $this->create($config, $integrationToken)
+                        ->request('POST', '/uploads/'. $mUploadId .'/part', [
+                            'json' => ['part' => $partNumber],
+                        ])
+                        ->toArray()
+                    ;
+
+                    try {
+                        $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [
+                                'body' => fread($file, 10 * 1024 * 1024), // 10Mo
+                            ])->getHeaders()
+                        ;
+
+                    } catch (\Exception $e) {
+                        sleep (1); // retry after 1 second
+                        try {
+                            $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [
+                                'body' => fread($file, 10 * 1024 * 1024), // 10Mo
+                            ])->getHeaders()
+                        ;
+                        } catch (\Exception $e) {
+                            throw $e;
+                        }
+                    }
+
+                    $parts['Parts'][$partNumber] = [
+                        'PartNumber'    => $partNumber,
+                        'ETag'          => current($headerPutPart['etag']),
+                    ];
+
+                    $partNumber++;
+                }
+
+                $isPartsComplete = true;
+                fclose($file);
+            } catch (\Exception $e) {
+                $this->create($config, $integrationToken)
+                    ->request('DELETE', '/uploads/'. $mUploadId);
+            }
+
+            if ($isPartsComplete) {
+                $result = $this->create($config, $integrationToken)
+                ->request('POST', '/uploads/'. $mUploadId.'/complete', [
+                    'json' => [
+                        'parts' => $parts['Parts']
+                    ],
+                ])
+                ->toArray()
+                ;
+
+                $this->create($config, $integrationToken)
+                    ->request('PUT', '/assets/'. $exposeAssetId, [
+                        'json' => [
+                            'path' => $result['path']
+                        ],
+                    ])
+                ;
+            }
 
             foreach ([
                 'preview',

diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php
@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Alchemy\StorageBundle\Controller;
+
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\HttpFoundation\Request;
+use Alchemy\StorageBundle\Upload\UploadManager;
+use Alchemy\StorageBundle\Entity\MultipartUpload;
+use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+
+class MultipartUploadCancelAction extends AbstractController
+{
+    public function __construct(private UploadManager $uploadManager, private EntityManagerInterface $em,)
+    {
+    }
+
+    public function __invoke(MultipartUpload $data, Request $request)
+    {
+        try {
+            $this->uploadManager->cancelMultipartUpload($data->getPath(), $data->getUploadId());
+        } catch (\Exception $e) {
+
+        }
+
+        $this->em->remove($data);
+    }
+}
diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php
@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Alchemy\StorageBundle\Controller;
+
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\HttpFoundation\Request;
+use Alchemy\StorageBundle\Upload\UploadManager;
+use Alchemy\StorageBundle\Entity\MultipartUpload;
+use Symfony\Component\HttpFoundation\JsonResponse;
+use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
+
+final class MultipartUploadCompleteAction extends AbstractController
+{
+    public function __construct(private UploadManager $uploadManager, private EntityManagerInterface $em,)
+    {
+    }
+
+    public function __invoke(MultipartUpload $data, Request $request)
+    {
+        $parts = $request->request->all('parts');
+
+        if (empty($parts)) {
+            throw new BadRequestHttpException('Missing parts');
+        }
+
+        $res = $this->uploadManager->markComplete($data->getUploadId(), $data->getPath(), (array) $parts);
+
+        $data->setComplete(true);
+        $this->em->persist($data);
+        $this->em->flush();
+
+        return new JsonResponse([
+            'path' => $res['Key'],
+        ]);
+    }
+}
diff --git a/lib/php/storage-bundle/Entity/MultipartUpload.php b/lib/php/storage-bundle/Entity/MultipartUpload.php
@@ -4,18 +4,20 @@
 
 namespace Alchemy\StorageBundle\Entity;
 
-use Alchemy\StorageBundle\Controller\MultipartUploadPartAction;
-use ApiPlatform\Metadata\ApiProperty;
-use ApiPlatform\Metadata\ApiResource;
-use ApiPlatform\Metadata\Delete;
+use Ramsey\Uuid\Uuid;
 use ApiPlatform\Metadata\Get;
-use ApiPlatform\Metadata\GetCollection;
 use ApiPlatform\Metadata\Post;
 use Doctrine\DBAL\Types\Types;
+use ApiPlatform\Metadata\Delete;
 use Doctrine\ORM\Mapping as ORM;
 use Ramsey\Uuid\Doctrine\UuidType;
-use Ramsey\Uuid\Uuid;
+use ApiPlatform\Metadata\ApiProperty;
+use ApiPlatform\Metadata\ApiResource;
+use ApiPlatform\Metadata\GetCollection;
 use Symfony\Component\Serializer\Annotation\Groups;
+use Alchemy\StorageBundle\Controller\MultipartUploadPartAction;
+use Alchemy\StorageBundle\Controller\MultipartUploadCancelAction;
+use Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction;
 
 #[ApiResource(
     shortName: 'Upload',
@@ -67,7 +69,42 @@
                         ]],
                 ],
             ]),
-        new Delete(openapiContext: ['summary' => 'Cancel an upload', 'description' => 'Cancel an upload.']),
+        new Post(
+            uriTemplate: '/uploads/{id}/complete',
+            controller: MultipartUploadCompleteAction::class,
+            openapiContext: [
+                'summary'       => 'Complete a multi part upload.',
+                'requestBody'   => [
+                    'content' => [
+                        'application/json' => [
+                            'schema' => [
+                                'type' => 'object',
+                                'properties' => [
+                                    'parts' => [
+                                        'type' => 'array',
+                                        'items' => [
+                                            'type' => 'object',
+                                            'properties' => [
+                                                'ETag'          => ['type' => 'string'],
+                                                'PartNumber'    => ['type' => 'integer'],
+                                            ],
+                                        ],
+                                    ]
+                                ]
+                            ]
+                        ]
+                    ]
+                ]
+            ]
+        ),    
+        new Delete(
+            controller: MultipartUploadCancelAction::class,
+            openapiContext: [
+                'summary' => 'Cancel an upload', 
+                'description' => 'Cancel an upload.'
+            ]
+        ),
+
         new GetCollection(security: 'is_granted(\'ROLE_ADMIN\')'),
     ],
     normalizationContext: ['groups' => ['upload:read']],
@@ -149,6 +186,7 @@ public function getUploadId(): string
 
     public function setUploadId(string $uploadId): void
     {
+        file_put_contents("log.txt", "nandalo setuploadid");
         $this->uploadId = $uploadId;
     }
 

diff --git a/lib/php/storage-bundle/Resources/config/services.yaml b/lib/php/storage-bundle/Resources/config/services.yaml
@@ -42,6 +42,14 @@ services:
         tags:
             - { name: controller.service_arguments }
 
+    Alchemy\StorageBundle\Controller\MultipartUploadCancelAction:
+        tags:
+            - { name: controller.service_arguments }        
+
+    Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction:
+        tags:
+            - { name: controller.service_arguments }        
+
     Alchemy\StorageBundle\Doctrine\MultipartUploadListener: ~
 
     Alchemy\StorageBundle\Storage\PathGenerator: ~

diff --git a/lib/php/storage-bundle/Upload/UploadManager.php b/lib/php/storage-bundle/Upload/UploadManager.php
@@ -4,14 +4,14 @@
 
 namespace Alchemy\StorageBundle\Upload;
 
-use Alchemy\StorageBundle\Entity\MultipartUpload;
-use Aws\Api\DateTimeResult;
 use Aws\S3\S3Client;
-use Doctrine\ORM\EntityManagerInterface;
+use Aws\Api\DateTimeResult;
 use Psr\Log\LoggerInterface;
+use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\HttpFoundation\Request;
-use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
+use Alchemy\StorageBundle\Entity\MultipartUpload;
 use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
 
 final readonly class UploadManager
 {
@@ -62,7 +62,7 @@ public function getSignedUrl(string $uploadId, string $path, int $partNumber): s
         return (string) $request->getUri();
     }
 
-    public function markComplete(string $uploadId, string $filename, array $parts): void
+    public function markComplete(string $uploadId, string $filename, array $parts)
     {
         $params = [
             'Bucket' => $this->uploadBucket,
@@ -73,7 +73,7 @@ public function markComplete(string $uploadId, string $filename, array $parts):
             'UploadId' => $uploadId,
         ];
 
-        $this->client->completeMultipartUpload($params);
+        return $this->client->completeMultipartUpload($params);
     }
 
     public function createPutObjectSignedURL(string $path, string $contentType): string