Fix upload to R2

RikudouSage · Aug 27, 2024 · ef0b70e · ef0b70e
1 parent e28a0e4
commit ef0b70e
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 30 deletions.
diff --git a/composer.json b/composer.json
@@ -16,7 +16,8 @@
         "symfony/css-selector": "^6.0",
         "ext-simplexml": "*",
         "ext-pdo": "*",
-        "aws/aws-sdk-php": "^3.320"
+        "aws/aws-sdk-php": "^3.320",
+        "symfony/polyfill-php84": "^1.30"
     },
     "autoload": {
         "psr-4": {

diff --git a/composer.lock b/composer.lock
diff --git a/src/DTO/FileWriter/S3FileReference.php b/src/DTO/FileWriter/S3FileReference.php
@@ -6,7 +6,7 @@
 
 final class S3FileReference
 {
-    private const MINIMUM_PART_SIZE = 10 * 1024 * 1024;
+    private const PART_SIZE = 10 * 1024 * 1024;
 
     private int $partNumber = 0;
     private ?string $openedObjectId = null;
@@ -15,10 +15,13 @@ final class S3FileReference
     private array $parts = [];
     private string $buffer = '';
 
+    private readonly string $tempKey;
+
     public function __construct(
         public readonly string $bucket,
         public readonly string $key,
     ) {
+        $this->tempKey = $this->key . '.gog-downloader.tmp';
     }
 
     public function open(S3Client $client): void
@@ -27,7 +30,7 @@ public function open(S3Client $client): void
         if ($this->openedObjectId === null) {
             $this->openedObjectId = $client->createMultipartUpload([
                 'Bucket' => $this->bucket,
-                'Key' => $this->key,
+                'Key' => $this->tempKey,
             ])->get('UploadId');
         }
     }
@@ -36,7 +39,7 @@ public function writeChunk(S3Client $client, string $data, bool $forceWrite = fa
     {
         $this->client = $client;
 
-        if (strlen($this->buffer . $data) < self::MINIMUM_PART_SIZE && !$forceWrite) {
+        if (strlen($this->buffer . $data) < self::PART_SIZE && !$forceWrite) {
             $this->buffer .= $data;
             return;
         }
@@ -45,46 +48,51 @@ public function writeChunk(S3Client $client, string $data, bool $forceWrite = fa
         $this->open($client);
 
         $result = $client->uploadPart([
-            'Body' => $data,
+            'Body' => substr($data, 0, self::PART_SIZE),
             'UploadId' => $this->openedObjectId,
             'Bucket' => $this->bucket,
-            'Key' => $this->key,
+            'Key' => $this->tempKey,
             'PartNumber' => ++$this->partNumber,
         ]);
         $this->parts[] = [
             'PartNumber' => $this->partNumber,
             'ETag' => $result['ETag'],
         ];
-        $this->buffer = '';
+        $this->buffer = substr($data, self::PART_SIZE);
     }
 
     public function finalize(string $hash): void
     {
         if ($this->client !== null && $this->openedObjectId !== null  && count($this->parts)) {
-            if ($this->buffer) {
+            while ($this->buffer) {
                 $this->writeChunk($this->client, $this->buffer, true);
             }
 
             $this->client->completeMultipartUpload([
                 'UploadId' => $this->openedObjectId,
                 'Bucket' => $this->bucket,
-                'Key' => $this->key,
+                'Key' => $this->tempKey,
                 'MultipartUpload' => [
                     'Parts' => $this->parts,
                 ],
             ]);
 
-            $this->client->putObjectTagging([
-                'Bucket' => $this->bucket,
+            while (!$this->client->doesObjectExistV2($this->bucket, $this->tempKey)) {
+                sleep(1);
+            }
+
+            $this->client->copyObject([
                 'Key' => $this->key,
-                'Tagging' => [
-                    'TagSet' => [
-                        [
-                            'Key' => 'md5_hash',
-                            'Value' => $hash,
-                        ],
-                    ]
-                ]
+                'Bucket' => $this->bucket,
+                'CopySource' => "{$this->bucket}/{$this->tempKey}",
+                'MetadataDirective' => 'REPLACE',
+                'Metadata' => [
+                    'md5_hash' => $hash,
+                ],
+            ]);
+            $this->client->deleteObject([
+                'Bucket' => $this->bucket,
+                'Key' => $this->tempKey,
             ]);
         }
     }

diff --git a/src/Service/DownloadManager.php b/src/Service/DownloadManager.php
@@ -31,7 +31,7 @@ public function getFilename(DownloadDescription $download, int $httpTimeout = 3)
         );
         $url = $response->getHeaders(false)['location'][0];
 
-        return pathinfo($url, PATHINFO_BASENAME);
+        return urldecode(pathinfo($url, PATHINFO_BASENAME));
     }
 
     public function download(

diff --git a/src/Service/FileWriter/S3FileWriter.php b/src/Service/FileWriter/S3FileWriter.php
@@ -56,18 +56,14 @@ public function getSize(object $file): int
 
     public function getMd5Hash(object $file): string
     {
-        $tags = $this->client->getObjectTagging([
+        $object = $this->client->headObject([
             'Bucket' => $file->bucket,
             'Key' => $file->key,
-        ])->get('TagSet');
-
-        foreach ($tags as $tag) {
-            if ($tag['Key'] === 'md5_hash') {
-                return $tag['Value'];
-            }
-        }
+        ]);
 
-        return hash_final($this->getMd5HashContext($file));
+        return array_find($object->get('Metadata'), function (string $value, string $key): string {
+            return $key === 'md5_hash';
+        }) ?? hash_final($this->getMd5HashContext($file));
     }
 
     public function createDirectory(string $path): void