From f78c362c6134ce8bff227183e9550b05d6a69bd1 Mon Sep 17 00:00:00 2001 From: yuji38kwmt Date: Thu, 25 Feb 2021 01:02:38 +0900 Subject: [PATCH 1/2] =?UTF-8?q?"--force"=20=E3=82=92=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E3=81=99=E3=82=8C=E3=81=B0=E3=80=81=E5=85=A5=E5=8A=9B=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E3=80=81=E8=A3=9C=E5=8A=A9=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=82=92=E4=B8=8A=E6=9B=B8=E3=81=8D=E3=81=97=E3=81=A6?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- anno3d/annofab/uploader.py | 25 +++++++++++++++++++++++-- anno3d/app.py | 18 ++++++++++++------ anno3d/kitti/scene_uploader.py | 9 +++++---- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/anno3d/annofab/uploader.py b/anno3d/annofab/uploader.py index 5299a4f..522be6f 100644 --- a/anno3d/annofab/uploader.py +++ b/anno3d/annofab/uploader.py @@ -1,9 +1,12 @@ from dataclasses import dataclass from logging import getLogger from pathlib import Path +from typing import Any, Optional +import more_itertools import requests from annofabapi import AnnofabApi +from annofabapi.utils import allow_404_error @dataclass(frozen=True) @@ -19,9 +22,15 @@ class Uploader: _client: AnnofabApi _project: str - def __init__(self, client: AnnofabApi, project: str): + def __init__(self, client: AnnofabApi, project: str, force: bool = False): self._client = client self._project = project + self._force = force + + @allow_404_error + def get_input_data(self, input_data_id: str) -> Optional[Any]: + result, _ = self._client.get_input_data(self._project, input_data_id) + return result def upload_tempdata(self, upload_file: Path) -> str: client = self._client @@ -40,6 +49,11 @@ def upload_input_data(self, input_data_id: str, file: Path) -> str: data_id = input_data_id body = {"input_data_name": file.name, "input_data_path": path} + if self._force: + old_input_data = self.get_input_data(input_data_id) + if old_input_data is not None: + body["last_updated_datetime"] = old_input_data["updated_datetime"] + input_data, _ = self._client.put_input_data(self._project, data_id, query_params=None, request_body=body) logger.debug("uploaded input data: %s", input_data) @@ -47,13 +61,20 @@ def upload_input_data(self, input_data_id: str, file: Path) -> str: def upload_supplementary(self, input_data_id: str, supplementary_id: str, file: Path) -> str: path = self.upload_tempdata(file) - body = { "supplementary_data_name": supplementary_id, "supplementary_data_path": path, "supplementary_data_type": "custom", "supplementary_data_number": 0, } + if self._force: + supplementary_list, _ = self._client.get_supplementary_data_list(self._project, input_data_id) + old_supplementary = more_itertools.first_true( + supplementary_list, pred=lambda e: e["supplementary_data_id"] == supplementary_id + ) + if old_supplementary is not None: + body["last_updated_datetime"] = old_supplementary["updated_datetime"] + supplementary, _ = self._client.put_supplementary_data(self._project, input_data_id, supplementary_id, body) logger.debug("uploaded supplementary data: %s", supplementary) return supplementary_id diff --git a/anno3d/app.py b/anno3d/app.py index 465f0a0..d6638a3 100644 --- a/anno3d/app.py +++ b/anno3d/app.py @@ -52,7 +52,9 @@ class Sandbox: """ sandboxの再現 """ @staticmethod - def upload(annofab_id: str, annofab_pass: str, kitti_dir: str, skip: int = 0, size: int = 10) -> None: + def upload( + annofab_id: str, annofab_pass: str, kitti_dir: str, skip: int = 0, size: int = 10, force: bool = False + ) -> None: project = "66241367-9175-40e3-8f2f-391d6891590b" kitti_dir_path = Path(kitti_dir) @@ -60,7 +62,7 @@ def upload(annofab_id: str, annofab_pass: str, kitti_dir: str, skip: int = 0, si pathss = loader.load(FrameKind.testing)[skip : (skip + size)] client_loader = ClientLoader(annofab_id, annofab_pass) with client_loader.open_api() as api: - uploader = Uploader(api, project) + uploader = Uploader(api, project, force=force) for paths in pathss: upload("", uploader, paths, [hidari, migi], None, None) @@ -84,6 +86,7 @@ def upload_velodyne( velo_dir: str, data_id_prefix: str = "", sensor_height: Optional[float] = None, + force: bool = False, ) -> None: velo_files = [Path(velo_dir) / path for path in os.listdir(velo_dir)] @@ -91,7 +94,7 @@ def upload_velodyne( with client_loader.open_api() as api: with tempfile.TemporaryDirectory() as tempdir_str: tempdir = Path(tempdir_str) - uploader = Uploader(api, project_id) + uploader = Uploader(api, project_id, force=force) for velo_file in velo_files: data_id = data_id_prefix + velo_file.name uploader.upload_input_data(data_id, velo_file) @@ -311,6 +314,7 @@ def upload_kitti_data( input_id_prefix: str = "", camera_horizontal_fov: Optional[int] = None, sensor_height: Optional[float] = None, + force: bool = False, ) -> None: """ kitti 3d detection形式のファイル群を3dpc-editorに登録します。 @@ -325,6 +329,7 @@ def upload_kitti_data( camera_horizontal_fov: カメラのhorizontal FOVの角度[degree] 指定が無い場合はcalibデータから計算する。 calibデータも無い場合はkittiのカメラ仕様を採用する。 sensor_height: 点群のセンサ(velodyne)の設置高。単位は点群の単位系（=kittiであれば[m]） 3dpc-editorは、この値を元に地面の高さを仮定する。 指定が無い場合はkittiのvelodyneの設置高を採用する + force: 入力データと補助データを上書きしてアップロードするかどうか。 Returns: """ @@ -335,7 +340,7 @@ def upload_kitti_data( pathss = loader.load(None)[skip : (skip + size)] client_loader = ClientLoader(annofab_id, annofab_pass) with client_loader.open_api() as api: - uploader = Uploader(api, project) + uploader = Uploader(api, project, force=force) # fmt: off uploaded = [ (input_id, len(supps)) @@ -361,6 +366,7 @@ def upload_scene( sensor_height: Optional[float] = None, frame_per_task: Optional[int] = None, upload_kind: str = UploadKind.CREATE_ANNOTATION.value, + force: bool = False, ) -> None: """ 拡張kitti形式のファイル群をAnnoFabにアップロードします @@ -379,11 +385,11 @@ def upload_scene( data => 入力データと補助データの登録のみを行う // task => 上記に加えて、タスクの生成を行う // annotation => 上記に加えて、アノテーションの登録を行う + force: 入力データと補助データを上書きしてアップロードするかどうか。 Returns: """ - client_loader = ClientLoader(annofab_id, annofab_pass) with client_loader.open_api() as api: uploader = SceneUploader(api) @@ -395,7 +401,7 @@ def upload_scene( task_id_prefix=task_id_prefix, kind=_decode_enum(UploadKind, upload_kind), ) - uploader.upload_from_path(Path(scene_path), uploader_input) + uploader.upload_from_path(Path(scene_path), uploader_input, force=force) class LocalCommand: diff --git a/anno3d/kitti/scene_uploader.py b/anno3d/kitti/scene_uploader.py index a0189c5..add7756 100644 --- a/anno3d/kitti/scene_uploader.py +++ b/anno3d/kitti/scene_uploader.py @@ -84,7 +84,7 @@ def _default_scene(path: Path) -> Scene: labels=[KittiLabelSeries(str(label_dir.absolute()), str(image_dir.absolute()), str(calib_dir.absolute()))], ) - def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput) -> None: + def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput, force: bool = False) -> None: """ Args: scene_path: 読み込み対象パス。　以下の何れかとなる @@ -93,6 +93,7 @@ def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput) * scene.metaが存在しないアップロード対象ディレクトリのパス * "velodyne/image_2/calib/label_2" のディレクトリがあるという前提で、読み込みを行う uploader_input: + force: Returns: @@ -102,7 +103,7 @@ def upload_from_path(self, scene_path: Path, uploader_input: SceneUploaderInput) file = scene_path / Defaults.scene_meta_file scene = Scene.decode_path(file) if file.is_file() else self._default_scene(scene_path) - return self.upload_scene(scene, uploader_input) + return self.upload_scene(scene, uploader_input, force=force) @staticmethod def _scene_to_paths(scene: Scene) -> List[FilePaths]: @@ -224,10 +225,10 @@ def _create_annotations( task.put_cuboid_annotations(task_id, input_data_id, self._label_to_cuboids(id_to_label, transformed_labels)) - def upload_scene(self, scene: Scene, uploader_input: SceneUploaderInput) -> None: + def upload_scene(self, scene: Scene, uploader_input: SceneUploaderInput, force: bool = False) -> None: logger.info("upload scene: %s", scene.to_json(indent=2, ensure_ascii=False)) - uploader = Uploader(self._client, uploader_input.project_id) + uploader = Uploader(self._client, uploader_input.project_id, force=force) pathss = self._scene_to_paths(scene) specs = self._project.get_annotation_specs(uploader_input.project_id) annofab_labels = specs.labels From 16b138e0aad2014cb48771426de042e0afcb05cf Mon Sep 17 00:00:00 2001 From: yuji38kwmt Date: Thu, 25 Feb 2021 01:36:37 +0900 Subject: [PATCH 2/2] upload readume --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b6809cf..115e8a1 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,8 @@ FLAGS カメラのhorizontal FOVの角度[degree] 指定が無い場合kittiのカメラ仕様を採用する --sensor_height=SENSOR_HEIGHT 点群のセンサ(velodyne)の設置高。単位は点群の単位系（=kittiであれば[m]） 3dpc-editorは、この値を元に地面の高さを仮定する。 指定が無い場合はkittiのvelodyneの設置高を採用する + --force=FORCE + 入力データと補助データを上書きしてアップロードするかどうか。 NOTES You can also use flags syntax for POSITIONAL ARGUMENTS @@ -389,6 +391,8 @@ FLAGS タスクを作る場合、１タスク辺り何個のinput_dataを登録するか。 省略した場合 シーン単位でタスクを作成 --upload_kind=UPLOAD_KIND 処理の種類　省略した場合 "annotation" // data => 入力データと補助データの登録のみを行う // task => 上記に加えて、タスクの生成を行う // annotation => 上記に加えて、アノテーションの登録を行う + --force=FORCE + 入力データと補助データを上書きしてアップロードするかどうか。 NOTES You can also use flags syntax for POSITIONAL ARGUMENTS