diff --git a/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php b/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php index 9196ce42f..352ae0d3a 100644 --- a/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php +++ b/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php @@ -130,27 +130,6 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati $fetchedFilePath = $this->fileFetcher->getFile($source); try { - $data = array_merge([ - 'publication_id' => $publicationId, - 'asset_id' => $asset->getId(), - 'title' => $resolvedTitle, - 'description' => $description, - 'translations' => $translations, - 'upload' => [ - 'type' => $source->getType(), - 'size' => $source->getSize(), - 'name' => $source->getOriginalName(), - ], - ], $extraData); - - $pubAsset = $this->create($config, $integrationToken) - ->request('POST', '/assets', [ - 'json' => $data, - ]) - ->toArray() - ; - $exposeAssetId = $pubAsset['id']; - $uploadsData = [ 'filename' => $source->getOriginalName(), 'type' => $source->getType(), @@ -165,13 +144,19 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati ; $mUploadId = $resUploads['id']; + $parts['Parts'] = []; - $isPartsComplete = false; - + // Upload the file in parts. try { $file = fopen($fetchedFilePath, 'r'); $partNumber = 1; + + // part size is up to 5Mo https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + $partSize = 10 * 1024 * 1024; // 10Mo + + $retryCount = 3; + while (!feof($file)) { $resUploadPart = $this->create($config, $integrationToken) ->request('POST', '/uploads/'. $mUploadId .'/part', [ @@ -179,24 +164,8 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati ]) ->toArray() ; - - try { - $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [ - 'body' => fread($file, 10 * 1024 * 1024), // 10Mo - ])->getHeaders() - ; - - } catch (\Exception $e) { - sleep (1); // retry after 1 second - try { - $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [ - 'body' => fread($file, 10 * 1024 * 1024), // 10Mo - ])->getHeaders() - ; - } catch (\Exception $e) { - throw $e; - } - } + + $headerPutPart = $this->putPart($resUploadPart['url'], $file, $partSize, $retryCount); $parts['Parts'][$partNumber] = [ 'PartNumber' => $partNumber, @@ -206,31 +175,34 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati $partNumber++; } - $isPartsComplete = true; fclose($file); - } catch (\Exception $e) { + } catch (\Throwable $e) { $this->create($config, $integrationToken) ->request('DELETE', '/uploads/'. $mUploadId); + + throw $e; } - if ($isPartsComplete) { - $result = $this->create($config, $integrationToken) - ->request('POST', '/uploads/'. $mUploadId.'/complete', [ - 'json' => [ - 'parts' => $parts['Parts'] - ], + $data = array_merge([ + 'publication_id' => $publicationId, + 'asset_id' => $asset->getId(), + 'title' => $resolvedTitle, + 'description' => $description, + 'translations' => $translations, + 'multipart' => [ + 'uploadId' => $mUploadId, + 'parts' => $parts['Parts'], + ], + ], $extraData); + + $pubAsset = $this->create($config, $integrationToken) + ->request('POST', '/assets', [ + 'json' => $data, ]) ->toArray() - ; + ; - $this->create($config, $integrationToken) - ->request('PUT', '/assets/'. $exposeAssetId, [ - 'json' => [ - 'path' => $result['path'] - ], - ]) - ; - } + $exposeAssetId = $pubAsset['id']; foreach ([ 'preview', @@ -282,4 +254,21 @@ public function deleteAsset(IntegrationConfig $config, IntegrationToken $integra ->request('DELETE', '/assets/'.$assetId) ; } + + private function putPart(string $url, mixed $handleFile, int $partSize, int $retryCount) + { + if ($retryCount > 0) { + $retryCount--; + try { + return $this->uploadClient->request('PUT', $url, [ + 'body' => fread($handleFile, 10 * 1024 * 1024), // 10Mo + ])->getHeaders(); + } catch (\Throwable $e) { + if ($retryCount == 0) { + throw $e; // retry unsuccess + } + $this->putPart($url, $handleFile, $partSize, $retryCount); + } + } + } } diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php index f57f26862..f8749bd12 100644 --- a/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php +++ b/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php @@ -20,8 +20,8 @@ public function __invoke(MultipartUpload $data, Request $request) { try { $this->uploadManager->cancelMultipartUpload($data->getPath(), $data->getUploadId()); - } catch (\Exception $e) { - + } catch (\Throwable $e) { + // S3 storage will clean up its uncomplete uploads automatically } $this->em->remove($data); diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php deleted file mode 100644 index aff6a8c38..000000000 --- a/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php +++ /dev/null @@ -1,39 +0,0 @@ -request->all('parts'); - - if (empty($parts)) { - throw new BadRequestHttpException('Missing parts'); - } - - $res = $this->uploadManager->markComplete($data->getUploadId(), $data->getPath(), (array) $parts); - - $data->setComplete(true); - $this->em->persist($data); - $this->em->flush(); - - return new JsonResponse([ - 'path' => $res['Key'], - ]); - } -} diff --git a/lib/php/storage-bundle/Entity/MultipartUpload.php b/lib/php/storage-bundle/Entity/MultipartUpload.php index 975cbb2c5..fae5288d4 100644 --- a/lib/php/storage-bundle/Entity/MultipartUpload.php +++ b/lib/php/storage-bundle/Entity/MultipartUpload.php @@ -69,34 +69,6 @@ ]], ], ]), - new Post( - uriTemplate: '/uploads/{id}/complete', - controller: MultipartUploadCompleteAction::class, - openapiContext: [ - 'summary' => 'Complete a multi part upload.', - 'requestBody' => [ - 'content' => [ - 'application/json' => [ - 'schema' => [ - 'type' => 'object', - 'properties' => [ - 'parts' => [ - 'type' => 'array', - 'items' => [ - 'type' => 'object', - 'properties' => [ - 'ETag' => ['type' => 'string'], - 'PartNumber' => ['type' => 'integer'], - ], - ], - ] - ] - ] - ] - ] - ] - ] - ), new Delete( controller: MultipartUploadCancelAction::class, openapiContext: [ diff --git a/lib/php/storage-bundle/Resources/config/services.yaml b/lib/php/storage-bundle/Resources/config/services.yaml index fe3869df2..d8e90a187 100644 --- a/lib/php/storage-bundle/Resources/config/services.yaml +++ b/lib/php/storage-bundle/Resources/config/services.yaml @@ -46,10 +46,6 @@ services: tags: - { name: controller.service_arguments } - Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction: - tags: - - { name: controller.service_arguments } - Alchemy\StorageBundle\Doctrine\MultipartUploadListener: ~ Alchemy\StorageBundle\Storage\PathGenerator: ~