Multi-GPU and CPU support

mmdet/apis/ote/apis/detection/config_utils.py

@@ -13,11 +13,13 @@
 # and limitations under the License.
 
 import copy
-import glob
 import os
 import tempfile
 from collections import defaultdict
+from typing import Any, Optional, List
+
 from mmcv import Config, ConfigDict
+from mmcv.runner import master_only
 from sc_sdk.entities.datasets import Dataset
 from sc_sdk.entities.label import Label
 from sc_sdk.logging import logger_factory
@@ -26,7 +28,8 @@
 
 from .configuration import OTEDetectionConfig
 
-logger = logger_factory.get_logger("OTEDetectionTask")
+
+logger = logger_factory.get_logger("OTEDetectionTask.config_utils")
 
 
 def patch_config(config: Config, work_dir: str, labels: List[Label], random_seed: Optional[int] = None):
@@ -89,10 +92,10 @@ def prepare_for_testing(config: Config, dataset: Dataset) -> Config:
 
 
 def prepare_for_training(config: Config, train_dataset: Dataset, val_dataset: Dataset,
-                         time_monitor: TimeMonitorCallback, learning_curves: defaultdict) -> Config:
+                         round_id: Any, time_monitor: TimeMonitorCallback, learning_curves: defaultdict) -> Config:
     config = copy.deepcopy(config)
 
-    prepare_work_dir(config)
+    prepare_work_dir(config, round_id)
 
     # config.data.test.ote_dataset = dataset.get_subset(Subset.TESTING)
     config.data.val.ote_dataset = val_dataset
@@ -124,7 +127,6 @@ def config_to_string(config: Config) -> str:
         config_copy.data.train.ote_dataset = None
     else:
         config_copy.data.train.dataset.ote_dataset = None
-    # config_copy.labels = [label.name for label in config.labels]
     return Config(config_copy).pretty_text
 
 
@@ -141,6 +143,7 @@ def config_from_string(config_string: str) -> Config:
         return Config.fromfile(temp_file.name)
 
 
+@master_only
 def save_config_to_file(config: Config):
     """ Dump the full config to a file. Filename is 'config.py', it is saved in the current work_dir. """
     filepath = os.path.join(config.work_dir, 'config.py')
@@ -149,16 +152,16 @@ def save_config_to_file(config: Config):
         f.write(config_string)
 
 
-def prepare_work_dir(config: Config) -> str:
+def prepare_work_dir(config: Config, round_id: Any = 0) -> str:
     base_work_dir = config.work_dir
-    checkpoint_dirs = glob.glob(os.path.join(base_work_dir, "checkpoints_round_*"))
-    train_round_checkpoint_dir = os.path.join(base_work_dir, f"checkpoints_round_{len(checkpoint_dirs)}")
-    os.makedirs(train_round_checkpoint_dir)
+
+    train_round_checkpoint_dir = os.path.join(base_work_dir, f"checkpoints_round_{round_id}")
+    os.makedirs(train_round_checkpoint_dir, exist_ok=True)
     logger.info(f"Checkpoints and logs for this training run are stored in {train_round_checkpoint_dir}")
     config.work_dir = train_round_checkpoint_dir
     if 'meta' not in config.runner:
         config.runner.meta = ConfigDict()
-    config.runner.meta.exp_name = f"train_round_{len(checkpoint_dirs)}"
+    config.runner.meta.exp_name = f"train_round_{round_id}"
     # Save training config for debugging. It is saved in the checkpoint dir for this training round
     save_config_to_file(config)
     return train_round_checkpoint_dir
@@ -184,8 +187,6 @@ def set_data_classes(config: Config, label_names: List[str]):
             config.model.roi_head.bbox_head.num_classes = num_classes
     elif 'bbox_head' in config.model:
         config.model.bbox_head.num_classes = num_classes
-    # FIXME. ?
-    # self.config.model.CLASSES = label_names
 
 
 def patch_datasets(config: Config):

mmdet/apis/ote/apis/detection/configuration.py

@@ -115,17 +115,5 @@ class __Postprocessing(ParameterGroup):
             affects_outcome_of=ModelLifecycle.INFERENCE
         )
 
-    @attrs
-    class __AlgoBackend(ParameterGroup):
-        header = string_attribute("Internal Algo Backend parameters")
-        description = header
-        visible_in_ui = boolean_attribute(False)
-
-        template = string_attribute("template.yaml")
-        model = string_attribute("model.py")
-        model_name = string_attribute("object detection model")
-        data_pipeline = string_attribute("ote_data_pipeline.py")
-
     learning_parameters = add_parameter_group(__LearningParameters)
-    algo_backend = add_parameter_group(__AlgoBackend)
     postprocessing = add_parameter_group(__Postprocessing)

mmdet/apis/ote/apis/detection/openvino_task.py

@@ -33,6 +33,8 @@
 from sc_sdk.usecases.tasks.interfaces.evaluate_interface import IEvaluationTask
 from sc_sdk.usecases.tasks.interfaces.inference_interface import IInferenceTask
 
+from mmcv.runner import master_only
+
 from .configuration import OTEDetectionConfig
 
 
@@ -165,19 +167,22 @@ def __init__(self, task_environment: TaskEnvironment):
         self.model = self.task_environment.model
         self.inferencer = self.load_inferencer()
 
+    @master_only
     def load_inferencer(self) -> OpenVINODetectionInferencer:
         labels = self.task_environment.label_schema.get_labels(include_empty=False)
         return OpenVINODetectionInferencer(self.hparams,
                                            labels,
                                            self.model.get_data("openvino.xml"),
                                            self.model.get_data("openvino.bin"))
 
+    @master_only
     def infer(self, dataset: Dataset, inference_parameters: Optional[InferenceParameters] = None) -> Dataset:
         from tqdm import tqdm
         for dataset_item in tqdm(dataset):
             dataset_item.annotation_scene = self.inferencer.predict(dataset_item.numpy)
         return dataset
 
+    @master_only
     def evaluate(self,
                  output_result_set: ResultSet,
                  evaluation_metric: Optional[str] = None):