Merge pull request #34 from kurusugawa-computer/upload-force

コマンドライン引数に"--force" を指定して、入力データ、補助データを上書きして登録できるようにする

README.md

@@ -326,6 +326,8 @@ FLAGS
         カメラのhorizontal FOVの角度[degree] 指定が無い場合kittiのカメラ仕様を採用する
     --sensor_height=SENSOR_HEIGHT
         点群のセンサ(velodyne)の設置高。単位は点群の単位系（=kittiであれば[m]） 3dpc-editorは、この値を元に地面の高さを仮定する。 指定が無い場合はkittiのvelodyneの設置高を採用する
+    --force=FORCE
+        入力データと補助データを上書きしてアップロードするかどうか。
 
 NOTES
     You can also use flags syntax for POSITIONAL ARGUMENTS
@@ -389,6 +391,8 @@ FLAGS
         タスクを作る場合、１タスク辺り何個のinput_dataを登録するか。 省略した場合 シーン単位でタスクを作成
     --upload_kind=UPLOAD_KIND
         処理の種類　省略した場合 "annotation" // data => 入力データと補助データの登録のみを行う // task => 上記に加えて、タスクの生成を行う // annotation => 上記に加えて、アノテーションの登録を行う
+    --force=FORCE
+        入力データと補助データを上書きしてアップロードするかどうか。
 
 NOTES
     You can also use flags syntax for POSITIONAL ARGUMENTS

anno3d/annofab/uploader.py

@@ -1,9 +1,12 @@
 from dataclasses import dataclass
 from logging import getLogger
 from pathlib import Path
+from typing import Any, Optional
 
+import more_itertools
 import requests
 from annofabapi import AnnofabApi
+from annofabapi.utils import allow_404_error
 
 
 @dataclass(frozen=True)
@@ -19,9 +22,15 @@ class Uploader:
     _client: AnnofabApi
     _project: str
 
-    def __init__(self, client: AnnofabApi, project: str):
+    def __init__(self, client: AnnofabApi, project: str, force: bool = False):
         self._client = client
         self._project = project
+        self._force = force
+
+    @allow_404_error
+    def get_input_data(self, input_data_id: str) -> Optional[Any]:
+        result, _ = self._client.get_input_data(self._project, input_data_id)
+        return result
 
     def upload_tempdata(self, upload_file: Path) -> str:
         client = self._client
@@ -40,20 +49,32 @@ def upload_input_data(self, input_data_id: str, file: Path) -> str:
 
         data_id = input_data_id
         body = {"input_data_name": file.name, "input_data_path": path}
+        if self._force:
+            old_input_data = self.get_input_data(input_data_id)
+            if old_input_data is not None:
+                body["last_updated_datetime"] = old_input_data["updated_datetime"]
+
         input_data, _ = self._client.put_input_data(self._project, data_id, query_params=None, request_body=body)
 
         logger.debug("uploaded input data: %s", input_data)
         return data_id
 
     def upload_supplementary(self, input_data_id: str, supplementary_id: str, file: Path) -> str:
         path = self.upload_tempdata(file)
-
         body = {
             "supplementary_data_name": supplementary_id,
             "supplementary_data_path": path,
             "supplementary_data_type": "custom",
             "supplementary_data_number": 0,
         }
+        if self._force:
+            supplementary_list, _ = self._client.get_supplementary_data_list(self._project, input_data_id)
+            old_supplementary = more_itertools.first_true(
+                supplementary_list, pred=lambda e: e["supplementary_data_id"] == supplementary_id
+            )
+            if old_supplementary is not None:
+                body["last_updated_datetime"] = old_supplementary["updated_datetime"]
+
         supplementary, _ = self._client.put_supplementary_data(self._project, input_data_id, supplementary_id, body)
         logger.debug("uploaded supplementary data: %s", supplementary)
         return supplementary_id

anno3d/app.py

@@ -52,15 +52,17 @@ class Sandbox:
     """ sandboxの再現 """
 
     @staticmethod
-    def upload(annofab_id: str, annofab_pass: str, kitti_dir: str, skip: int = 0, size: int = 10) -> None:
+    def upload(
+        annofab_id: str, annofab_pass: str, kitti_dir: str, skip: int = 0, size: int = 10, force: bool = False
+    ) -> None:
         project = "66241367-9175-40e3-8f2f-391d6891590b"
 
         kitti_dir_path = Path(kitti_dir)
         loader = FilePathsLoader(kitti_dir_path, kitti_dir_path, kitti_dir_path)
         pathss = loader.load(FrameKind.testing)[skip : (skip + size)]
         client_loader = ClientLoader(annofab_id, annofab_pass)
         with client_loader.open_api() as api:
-            uploader = Uploader(api, project)
+            uploader = Uploader(api, project, force=force)
             for paths in pathss:
                 upload("", uploader, paths, [hidari, migi], None, None)
 
@@ -84,14 +86,15 @@ def upload_velodyne(
         velo_dir: str,
         data_id_prefix: str = "",
         sensor_height: Optional[float] = None,
+        force: bool = False,
     ) -> None:
         velo_files = [Path(velo_dir) / path for path in os.listdir(velo_dir)]
 
         client_loader = ClientLoader(annofab_id, annofab_pass)
         with client_loader.open_api() as api:
             with tempfile.TemporaryDirectory() as tempdir_str:
                 tempdir = Path(tempdir_str)
-                uploader = Uploader(api, project_id)
+                uploader = Uploader(api, project_id, force=force)
                 for velo_file in velo_files:
                     data_id = data_id_prefix + velo_file.name
                     uploader.upload_input_data(data_id, velo_file)
@@ -311,6 +314,7 @@ def upload_kitti_data(
         input_id_prefix: str = "",
         camera_horizontal_fov: Optional[int] = None,
         sensor_height: Optional[float] = None,
+        force: bool = False,
     ) -> None:
         """
         kitti 3d detection形式のファイル群を3dpc-editorに登録します。
@@ -325,6 +329,7 @@ def upload_kitti_data(
             camera_horizontal_fov: カメラのhorizontal FOVの角度[degree] 指定が無い場合はcalibデータから計算する。 calibデータも無い場合はkittiのカメラ仕様を採用する。
             sensor_height: 点群のセンサ(velodyne)の設置高。単位は点群の単位系（=kittiであれば[m]）
                            3dpc-editorは、この値を元に地面の高さを仮定する。 指定が無い場合はkittiのvelodyneの設置高を採用する
+            force: 入力データと補助データを上書きしてアップロードするかどうか。
         Returns:
 
         """
@@ -335,7 +340,7 @@ def upload_kitti_data(
         pathss = loader.load(None)[skip : (skip + size)]
         client_loader = ClientLoader(annofab_id, annofab_pass)
         with client_loader.open_api() as api:
-            uploader = Uploader(api, project)
+            uploader = Uploader(api, project, force=force)
             # fmt: off
             uploaded = [
                 (input_id, len(supps))
@@ -361,6 +366,7 @@ def upload_scene(
         sensor_height: Optional[float] = None,
         frame_per_task: Optional[int] = None,
         upload_kind: str = UploadKind.CREATE_ANNOTATION.value,
+        force: bool = False,
     ) -> None:
         """
         拡張kitti形式のファイル群をAnnoFabにアップロードします
@@ -379,11 +385,11 @@ def upload_scene(
                          data => 入力データと補助データの登録のみを行う //
                          task => 上記に加えて、タスクの生成を行う //
                          annotation => 上記に加えて、アノテーションの登録を行う
+            force: 入力データと補助データを上書きしてアップロードするかどうか。
 
         Returns:
 
         """
-
         client_loader = ClientLoader(annofab_id, annofab_pass)
         with client_loader.open_api() as api:
             uploader = SceneUploader(api)
@@ -395,7 +401,7 @@ def upload_scene(
                 task_id_prefix=task_id_prefix,
                 kind=_decode_enum(UploadKind, upload_kind),
             )
-            uploader.upload_from_path(Path(scene_path), uploader_input)
+            uploader.upload_from_path(Path(scene_path), uploader_input, force=force)
 
 
 class LocalCommand:

anno3d/kitti/scene_uploader.py

@@ -84,7 +84,7 @@ def _default_scene(path: Path) -> Scene:
             labels=[KittiLabelSeries(str(label_dir.absolute()), str(image_dir.absolute()), str(calib_dir.absolute()))],
         )
 
-    def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput) -> None:
+    def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput, force: bool = False) -> None:
         """
         Args:
             scene_path: 読み込み対象パス。　以下の何れかとなる
@@ -93,6 +93,7 @@ def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput)
                          * scene.metaが存在しないアップロード対象ディレクトリのパス
                              * "velodyne/image_2/calib/label_2" のディレクトリがあるという前提で、読み込みを行う
             uploader_input:
+            force:
 
         Returns:
 
@@ -102,7 +103,7 @@ def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput)
             file = scene_path / Defaults.scene_meta_file
 
         scene = Scene.decode_path(file) if file.is_file() else self._default_scene(scene_path)
-        return self.upload_scene(scene, uploader_input)
+        return self.upload_scene(scene, uploader_input, force=force)
 
     @staticmethod
     def _scene_to_paths(scene: Scene) -> List[FilePaths]:
@@ -224,10 +225,10 @@ def _create_annotations(
 
             task.put_cuboid_annotations(task_id, input_data_id, self._label_to_cuboids(id_to_label, transformed_labels))
 
-    def upload_scene(self, scene: Scene, uploader_input: SceneUploaderInput) -> None:
+    def upload_scene(self, scene: Scene, uploader_input: SceneUploaderInput, force: bool = False) -> None:
         logger.info("upload scene: %s", scene.to_json(indent=2, ensure_ascii=False))
 
-        uploader = Uploader(self._client, uploader_input.project_id)
+        uploader = Uploader(self._client, uploader_input.project_id, force=force)
         pathss = self._scene_to_paths(scene)
         specs = self._project.get_annotation_specs(uploader_input.project_id)
         annofab_labels = specs.labels