From 2684d8aa140165df14ee8b0e5ebe0e006d98647e Mon Sep 17 00:00:00 2001 From: husichao Date: Mon, 18 Sep 2023 16:20:10 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9tk=E5=90=8D=E5=AD=97=E4=B8=BA?= =?UTF-8?q?mindpet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- {tk => mindpet}/__init__.py | 2 +- mindpet/delta/__init__.py | 12 +++++ {tk => mindpet}/delta/adapter.py | 44 ++++++++++--------- {tk => mindpet}/delta/delta_constants.py | 2 +- {tk => mindpet}/delta/lora.py | 21 ++++----- {tk => mindpet}/delta/low_rank_adapter.py | 38 ++++++++-------- {tk => mindpet}/delta/prefix_layer.py | 10 ++--- {tk => mindpet}/delta/r_drop.py | 8 ++-- {tk => mindpet}/graph/__init__.py | 4 +- {tk => mindpet}/graph/ckpt_util.py | 6 +-- {tk => mindpet}/graph/freeze_utils.py | 8 ++-- {tk => mindpet}/log/__init__.py | 2 +- {tk => mindpet}/log/log.py | 6 +-- {tk => mindpet}/log/log_utils.py | 4 +- {tk => mindpet}/security/__init__.py | 0 .../security/param_check/__init__.py | 0 .../security/param_check/base_check.py | 2 +- .../model_config_params_check_util.py | 6 +-- .../param_check/option_check_utils.py | 10 ++--- {tk => mindpet}/task/__init__.py | 0 .../task/evaluate_infer/__init__.py | 0 .../evaluate_infer/evaluate_infer_task.py | 14 +++--- .../task/evaluate_infer/result_file_check.py | 4 +- {tk => mindpet}/task/finetune/__init__.py | 0 .../task/finetune/finetune_options_check.py | 6 +-- .../task/finetune/finetune_task.py | 10 ++--- {tk => mindpet}/task/option_decorators.py | 2 +- mindpet/task/options/__init__.py | 12 +++++ .../task/options/boot_file_path_option.py | 6 +-- .../task/options/ckpt_path_option.py | 6 +-- .../task/options/data_path_option.py | 6 +-- .../task/options/model_config_path_option.py | 6 +-- .../task/options/output_path_option.py | 6 +-- .../task/options/path_check_param.py | 6 +-- .../options/pretrained_model_path_option.py | 6 +-- {tk => mindpet}/task/options/quiet_option.py | 0 .../task/options/timeout_option.py | 2 +- {tk => mindpet}/tk_main.py | 14 +++--- {tk => mindpet}/tk_sdk.py | 10 ++--- {tk => mindpet}/utils/__init__.py | 0 {tk => mindpet}/utils/constants.py | 0 {tk => mindpet}/utils/entrance_monitor.py | 0 {tk => mindpet}/utils/exceptions.py | 2 +- {tk => mindpet}/utils/io_utils.py | 4 +- {tk => mindpet}/utils/task_utils.py | 8 ++-- {tk => mindpet}/utils/version_control.py | 0 {tk => mindpet}/utils/version_utils.py | 0 set_up.py | 2 +- test/unit_test/delta/test_adapter.py | 10 +++-- test/unit_test/delta/test_lora.py | 9 ++-- test/unit_test/delta/test_low_rank_adapter.py | 9 ++-- test/unit_test/delta/test_prefix_layer.py | 12 ++--- test/unit_test/delta/test_rdrop.py | 9 ++-- test/unit_test/entry/test_tk_sdk.py | 4 +- test/unit_test/graph/test_freeze_utils.py | 4 +- test/unit_test/graph/test_save_ckpt.py | 4 +- test/unit_test/log/test_concurrent_log.py | 4 +- test/unit_test/log/test_logger.py | 8 ++-- test/unit_test/log/test_logger_validator.py | 4 +- .../security/param_check/test_base_check.py | 4 +- .../test_model_config_check_utils.py | 8 ++-- .../param_check/test_option_check_utils.py | 8 ++-- .../test_evaluate_infer_task.py | 6 +-- .../evaluate_infer/test_result_file_check.py | 8 ++-- .../finetune/test_finetune_options_check.py | 6 +-- .../task/finetune/test_finetune_task.py | 6 +-- test/unit_test/utils/test_entrance_monitor.py | 4 +- test/unit_test/utils/test_io_utils.py | 6 +-- test/unit_test/utils/test_task_utils.py | 4 +- tk/delta/__init__.py | 12 ----- tk/task/options/__init__.py | 12 ----- 72 files changed, 248 insertions(+), 232 deletions(-) rename {tk => mindpet}/__init__.py (82%) create mode 100644 mindpet/delta/__init__.py rename {tk => mindpet}/delta/adapter.py (94%) rename {tk => mindpet}/delta/delta_constants.py (90%) rename {tk => mindpet}/delta/lora.py (96%) rename {tk => mindpet}/delta/low_rank_adapter.py (97%) rename {tk => mindpet}/delta/prefix_layer.py (96%) rename {tk => mindpet}/delta/r_drop.py (96%) rename {tk => mindpet}/graph/__init__.py (59%) rename {tk => mindpet}/graph/ckpt_util.py (97%) rename {tk => mindpet}/graph/freeze_utils.py (97%) rename {tk => mindpet}/log/__init__.py (67%) rename {tk => mindpet}/log/log.py (98%) rename {tk => mindpet}/log/log_utils.py (98%) rename {tk => mindpet}/security/__init__.py (100%) rename {tk => mindpet}/security/param_check/__init__.py (100%) rename {tk => mindpet}/security/param_check/base_check.py (94%) rename {tk => mindpet}/security/param_check/model_config_params_check_util.py (93%) rename {tk => mindpet}/security/param_check/option_check_utils.py (98%) rename {tk => mindpet}/task/__init__.py (100%) rename {tk => mindpet}/task/evaluate_infer/__init__.py (100%) rename {tk => mindpet}/task/evaluate_infer/evaluate_infer_task.py (93%) rename {tk => mindpet}/task/evaluate_infer/result_file_check.py (91%) rename {tk => mindpet}/task/finetune/__init__.py (100%) rename {tk => mindpet}/task/finetune/finetune_options_check.py (94%) rename {tk => mindpet}/task/finetune/finetune_task.py (92%) rename {tk => mindpet}/task/option_decorators.py (93%) create mode 100644 mindpet/task/options/__init__.py rename {tk => mindpet}/task/options/boot_file_path_option.py (87%) rename {tk => mindpet}/task/options/ckpt_path_option.py (83%) rename {tk => mindpet}/task/options/data_path_option.py (83%) rename {tk => mindpet}/task/options/model_config_path_option.py (87%) rename {tk => mindpet}/task/options/output_path_option.py (83%) rename {tk => mindpet}/task/options/path_check_param.py (82%) rename {tk => mindpet}/task/options/pretrained_model_path_option.py (83%) rename {tk => mindpet}/task/options/quiet_option.py (100%) rename {tk => mindpet}/task/options/timeout_option.py (94%) rename {tk => mindpet}/tk_main.py (71%) rename {tk => mindpet}/tk_sdk.py (89%) rename {tk => mindpet}/utils/__init__.py (100%) rename {tk => mindpet}/utils/constants.py (100%) rename {tk => mindpet}/utils/entrance_monitor.py (100%) rename {tk => mindpet}/utils/exceptions.py (98%) rename {tk => mindpet}/utils/io_utils.py (92%) rename {tk => mindpet}/utils/task_utils.py (94%) rename {tk => mindpet}/utils/version_control.py (100%) rename {tk => mindpet}/utils/version_utils.py (100%) delete mode 100644 tk/delta/__init__.py delete mode 100644 tk/task/options/__init__.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 377e366..3dffbef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: pip install -r requirements/ci_requirements.txt - name: Analysing the code with pylint run: | - pylint tk --rcfile=.github/pylint.conf + pylint mindpet --rcfile=.github/pylint.conf - name: Test with unit test (UT) pytest run: | pytest test/unit_test diff --git a/tk/__init__.py b/mindpet/__init__.py similarity index 82% rename from tk/__init__.py rename to mindpet/__init__.py index 7731464..3eb0186 100644 --- a/tk/__init__.py +++ b/mindpet/__init__.py @@ -2,6 +2,6 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -import tk.tk_sdk as tk_sdk +import mindpet.tk_sdk as tk_sdk __all__ = ["tk_sdk"] diff --git a/mindpet/delta/__init__.py b/mindpet/delta/__init__.py new file mode 100644 index 0000000..605be9c --- /dev/null +++ b/mindpet/delta/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. +"""Mindpet delta init""" +from mindpet.delta.lora import LoRADense +from mindpet.delta.prefix_layer import PrefixLayer +from mindpet.delta.low_rank_adapter import LowRankAdapterDense, LowRankAdapterLayer +from mindpet.delta.adapter import AdapterDense, AdapterLayer +from mindpet.delta.r_drop import RDropLoss, rdrop_repeat + +__all__ = ['LoRADense', 'PrefixLayer', 'LowRankAdapterDense', 'LowRankAdapterLayer', + 'AdapterDense', 'AdapterLayer', 'RDropLoss', 'rdrop_repeat'] diff --git a/tk/delta/adapter.py b/mindpet/delta/adapter.py similarity index 94% rename from tk/delta/adapter.py rename to mindpet/delta/adapter.py index ca6c2c3..2c5e7e5 100644 --- a/tk/delta/adapter.py +++ b/mindpet/delta/adapter.py @@ -1,24 +1,23 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. - +"""Adapter Cell""" from collections import OrderedDict -import mindspore as ms -import mindspore.nn as nn +from mindspore import nn import mindspore.common.dtype as mstype from mindspore.nn.layer.activation import get_activation, _activation from mindspore.ops import operations as P from mindspore.ops import functional as F -from tk.delta.delta_constants import VALID_TENSOR_DATATYPE - try: from mindspore.nn.transformer.layers import _Linear, _args_type_validator_check, _valid_value_checks from mindspore._checkparam import Validator -except: - from mindformers.modules.layers import Linear, _args_type_validator_check, _valid_value_checks +except ImportError: import mindspore._checkparam as Validator + from mindformers.modules.layers import Linear, _args_type_validator_check, _valid_value_checks _Linear = Linear +from mindpet.delta.delta_constants import VALID_TENSOR_DATATYPE + class AdapterLayer(nn.Cell): """ @@ -48,7 +47,7 @@ def __init__( param_init_type: mstype = mstype.float32, compute_dtype: mstype = mstype.float16): - super(AdapterLayer, self).__init__() + super().__init__() self.bottleneck_size = bottleneck_size self.non_linearity_name = non_linearity @@ -71,6 +70,7 @@ def __init__( self.reshape = P.Reshape() def construct(self, input_tensor): + """Foward""" # get input_tensor info input_tensor_shape = self.shape(input_tensor) ori_dtype = F.dtype(input_tensor) @@ -97,6 +97,7 @@ def shard(self, strategy_matmul_up_sampler=None, strategy_bias_up_sampler=None, strategy_residual_add=None): + """Shard Method""" try: self.tk_delta_adapter_block.tk_delta_adapter_down_sampler.shard( strategy_matmul=strategy_matmul_down_sampler, strategy_bias=strategy_bias_down_sampler) @@ -114,7 +115,7 @@ def shard(self, raise ValueError("The 'LogSoftmax' function is not supported in semi auto parallel " "or auto parallel mode.") else: - getattr(self.tk_delta_adapter_block.tk_delta_adapter_non_linear, + getattr(self.tk_delta_adapter_block.tk_delta_adapter_non_linear, self.non_linearity_name).shard(strategy_non_linearity) self.tk_delta_adapter_block.tk_delta_adapter_up_sampler.shard(strategy_matmul=strategy_matmul_up_sampler, @@ -123,9 +124,10 @@ def shard(self, self.residual_add.shard(strategy_residual_add) except Exception as ex: + # pylint: disable=W0719 raise Exception(f"Exception occurred when set the shard for AdapterLayer, error message: \ {str(ex)}") from ex - + class AdapterDense(nn.Dense): """ @@ -184,15 +186,13 @@ def __init__(self, bottleneck_size: int = 64, non_linearity: str = "gelu", param_init_type: mstype = mstype.float32, - compute_dtype: mstype = mstype.float16, - **kwargs): - - super(AdapterDense, self).__init__(in_channels=in_channels, - out_channels=out_channels, - weight_init=weight_init, - bias_init=bias_init, - has_bias=has_bias, - activation=activation) + compute_dtype: mstype = mstype.float16): + super().__init__(in_channels=in_channels, + out_channels=out_channels, + weight_init=weight_init, + bias_init=bias_init, + has_bias=has_bias, + activation=activation) self.tk_delta_adapter = AdapterLayer(hidden_size=out_channels, bottleneck_size=bottleneck_size, @@ -206,7 +206,7 @@ def __init__(self, self.act_name = activation def construct(self, input_tensor): - + """Foward""" # get input_x info x_shape = self.shape_op(input_tensor) ori_dtype = F.dtype(input_tensor) @@ -234,7 +234,7 @@ def construct(self, input_tensor): output = self.cast(input_tensor, ori_dtype) return output - def shard(self, + def shard(self, strategy_matmul_org=None, strategy_bias_org=None, strategy_activation_org=None, @@ -244,6 +244,7 @@ def shard(self, strategy_matmul_up_sampler=None, strategy_bias_up_sampler=None, strategy_residual_add=None): + """Shard Method""" try: # set origin dense strategy self.matmul.shard(strategy_matmul_org) @@ -274,5 +275,6 @@ def shard(self, strategy_residual_add=strategy_residual_add) except Exception as ex: + # pylint: disable=W0719 raise Exception(f"Exception occurred when set the shard for AdapterDense, error message: \ {str(ex)}") from ex diff --git a/tk/delta/delta_constants.py b/mindpet/delta/delta_constants.py similarity index 90% rename from tk/delta/delta_constants.py rename to mindpet/delta/delta_constants.py index 5b07b4c..e05a709 100644 --- a/tk/delta/delta_constants.py +++ b/mindpet/delta/delta_constants.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. - +"""Constants for delta""" from mindspore import dtype as mstype # 算法精度与类型控制 diff --git a/tk/delta/lora.py b/mindpet/delta/lora.py similarity index 96% rename from tk/delta/lora.py rename to mindpet/delta/lora.py index 957ddcd..de7bb5a 100644 --- a/tk/delta/lora.py +++ b/mindpet/delta/lora.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. - +"""LoRA Cell""" import math -import mindspore as ms -import mindspore.nn as nn +from mindspore import nn from mindspore import ops from mindspore import Parameter from mindspore.common.tensor import Tensor from mindspore import dtype as mstype from mindspore.ops import operations as P from mindspore.ops import functional as F -from mindspore.common.initializer import initializer, HeUniform -from tk.delta.delta_constants import VALID_TENSOR_DATATYPE -from tk.utils.version_control import get_dropout - try: from mindspore._checkparam import Validator, Rel INC_LEFT = Rel.INC_LEFT -except: +except ImportError: import mindspore._checkparam as Validator INC_LEFT = Validator.INC_LEFT +from mindspore.common.initializer import initializer, HeUniform +from mindpet.delta.delta_constants import VALID_TENSOR_DATATYPE +from mindpet.utils.version_control import get_dropout + class LoRADense(nn.Dense): """Define a dense layer with LoRA structure. @@ -49,7 +48,7 @@ def __init__( compute_dtype=mstype.float16, **kwargs ): - super(LoRADense, self).__init__(in_channels, out_channels, **kwargs) + super().__init__(in_channels, out_channels, **kwargs) # Check params self._check_num(lora_rank, lora_alpha, lora_dropout) @@ -76,6 +75,7 @@ def __init__( self.lora_b_matmul = P.MatMul(transpose_b=True) def construct(self, input_tensor): + """Foward""" # Data type operation ori_dtype = F.dtype(input_tensor) input_tensor = self.cast(input_tensor, self.dtype) @@ -164,6 +164,7 @@ def shard(self, strategy_org_dense_matmul=None, else: getattr(self.activation, self.act_name).shard(strategy_activation) except Exception as ex: + # pylint: disable=W0719 raise Exception(f"Exception occurred when set the shard for LoRADense, error message: {str(ex)}") from ex def _check_num(self, lora_rank, lora_alpha, lora_dropout): @@ -179,6 +180,7 @@ def _check_num(self, lora_rank, lora_alpha, lora_dropout): rel=INC_LEFT, arg_name='lora_dropout', prim_name=self.cls_name) def _check_init(self, lora_a_init, lora_b_init, lora_rank): + """check init arguments""" if isinstance(lora_a_init, Tensor): if lora_a_init.ndim != 2 or lora_a_init.shape[0] != lora_rank or \ lora_a_init.shape[1] != self.in_channels: @@ -201,4 +203,3 @@ def _check_type_of_data(self, param_init_type, compute_dtype): if compute_dtype not in VALID_TENSOR_DATATYPE: raise TypeError(f"For {self.cls_name}, the 'compute_dtype' must be mindspore.dtype.float16 or " f"mindspore.dtype.float32, but got {compute_dtype}.") - diff --git a/tk/delta/low_rank_adapter.py b/mindpet/delta/low_rank_adapter.py similarity index 97% rename from tk/delta/low_rank_adapter.py rename to mindpet/delta/low_rank_adapter.py index cf6a00f..a42056b 100644 --- a/tk/delta/low_rank_adapter.py +++ b/mindpet/delta/low_rank_adapter.py @@ -1,27 +1,22 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. - -import numbers - -import mindspore as ms -import mindspore.nn as nn +"""Low Rank Adapter Cell""" +from mindspore import nn import mindspore.common.dtype as mstype from mindspore.common.initializer import initializer, Initializer from mindspore.common.parameter import Parameter from mindspore import Tensor - from mindspore.nn.layer.activation import get_activation, _activation from mindspore.ops import operations as P from mindspore.ops import functional as F -from tk.delta.delta_constants import VALID_TENSOR_DATATYPE - try: - from mindspore.nn.transformer.layers import _args_type_validator_check, _valid_type_checks, _valid_value_checks + from mindspore.nn.transformer.layers import _args_type_validator_check, _valid_value_checks from mindspore._checkparam import Validator -except: - from mindformers.modules.layers import _args_type_validator_check, _valid_type_checks, _valid_value_checks +except ImportError: import mindspore._checkparam as Validator + from mindformers.modules.layers import _args_type_validator_check, _valid_value_checks +from mindpet.delta.delta_constants import VALID_TENSOR_DATATYPE class LowRankLinear(nn.Cell): @@ -63,7 +58,7 @@ def __init__(self, has_bias: bool = True, param_init_type: mstype = mstype.float32, compute_dtype: mstype = mstype.float16): - super(LowRankLinear, self).__init__() + super().__init__() self.in_channels = in_channels self.out_channels = out_channels @@ -95,6 +90,7 @@ def __init__(self, self.shape_op = P.Shape() def construct(self, input_tensor): + """Forward""" # get input_x info x_shape = self.shape_op(input_tensor) ori_dtype = F.dtype(input_tensor) @@ -178,7 +174,7 @@ def __init__( compute_dtype: mstype = mstype.float16): self._check_low_rank_w_init(low_rank_w_init) self._check_reduction_factor(hidden_size, reduction_factor) - super(LowRankAdapterLayer, self).__init__() + super().__init__() self.bottleneck_size = hidden_size // reduction_factor self.non_linearity = non_linearity @@ -200,6 +196,7 @@ def __init__( self.residual_add = P.Add() def construct(self, input_tensor): + """Forward""" # get input_x info x_shape = P.Shape()(input_tensor) ori_dtype = F.dtype(input_tensor) @@ -284,6 +281,7 @@ def shard(self, strategy_matmul_down_sampler_weight=None, self.residual_add.shard(strategy_residual_add) except Exception as ex: + # pylint: disable=W0719 raise Exception( f"Exception occurred when set the shard for LowRankAdapterLayer, error message: {str(ex)}") from ex @@ -368,12 +366,12 @@ def __init__(self, non_linearity: str = "gelu", param_init_type: mstype = mstype.float32, compute_dtype: mstype = mstype.float16): - super(LowRankAdapterDense, self).__init__(in_channels=in_channels, - out_channels=out_channels, - weight_init=weight_init, - bias_init=bias_init, - has_bias=has_bias, - activation=activation) + super().__init__(in_channels=in_channels, + out_channels=out_channels, + weight_init=weight_init, + bias_init=bias_init, + has_bias=has_bias, + activation=activation) self.tk_delta_low_rank_adapter = LowRankAdapterLayer(hidden_size=out_channels, reduction_factor=reduction_factor, low_rank_size=low_rank_size, @@ -386,6 +384,7 @@ def __init__(self, self.act_name = activation def construct(self, input_tensor): + """Forward""" # get input_x info x_shape = self.shape_op(input_tensor) ori_dtype = F.dtype(input_tensor) @@ -482,6 +481,7 @@ def shard(self, strategy_matmul_org=None, strategy_bias_up_sampler, strategy_residual_add) except Exception as ex: + # pylint: disable=W0719 raise Exception( f"Exception occurred when set the shard for LowRankAdapterDense, error message: {str(ex)}") from ex diff --git a/tk/delta/prefix_layer.py b/mindpet/delta/prefix_layer.py similarity index 96% rename from tk/delta/prefix_layer.py rename to mindpet/delta/prefix_layer.py index 6eb7ec2..aa279bf 100644 --- a/tk/delta/prefix_layer.py +++ b/mindpet/delta/prefix_layer.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2010-2022. All rights reserved. - +"""Prefix Layer Cell""" import mindspore as ms -import mindspore.nn as nn - -from tk.utils.version_control import get_dropout - +from mindspore import nn try: from mindspore._checkparam import Validator, Rel INC_LEFT = Rel.INC_LEFT -except: +except ImportError: import mindspore._checkparam as Validator INC_LEFT = Validator.INC_LEFT +from mindpet.utils.version_control import get_dropout def check_multiple(param_dividend, value_dividend, param_divisor, value_divisor): diff --git a/tk/delta/r_drop.py b/mindpet/delta/r_drop.py similarity index 96% rename from tk/delta/r_drop.py rename to mindpet/delta/r_drop.py index d4d4195..af3b50a 100644 --- a/tk/delta/r_drop.py +++ b/mindpet/delta/r_drop.py @@ -3,18 +3,18 @@ # Copyright (c) Microsoft Corporation, 2022-2023, All rights reserved. # Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. # Copyright © Huawei Technologies Co., Ltd. 2010-2023. All rights reserved. - +"""R drop Cell""" import numpy as np from mindspore import nn from mindspore.common.tensor import Tensor from mindspore.common import dtype as mstype -import mindspore.ops as ops +from mindspore import ops class RDropLoss(nn.Cell): """The loss between input and target with r_drop.""" def __init__(self): - super(RDropLoss, self).__init__() + super().__init__() self.cross_entropy_loss = nn.CrossEntropyLoss() self.log_softmax = nn.LogSoftmax(axis=-1) self.softmax = nn.Softmax(axis=-1) @@ -55,7 +55,7 @@ def construct(self, logits, label_ids, alpha=4): def rdrop_repeat(*items): - res = list() + res = [] for item in items: res.append(item.repeat(2, axis=0)) return tuple(res) diff --git a/tk/graph/__init__.py b/mindpet/graph/__init__.py similarity index 59% rename from tk/graph/__init__.py rename to mindpet/graph/__init__.py index 6e40fc8..3ca06e0 100644 --- a/tk/graph/__init__.py +++ b/mindpet/graph/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.graph.freeze_utils import freeze_modules, freeze_delta, freeze_from_config -from tk.graph.ckpt_util import TrainableParamsCheckPoint +from mindpet.graph.freeze_utils import freeze_modules, freeze_delta, freeze_from_config +from mindpet.graph.ckpt_util import TrainableParamsCheckPoint __all__ = ['freeze_modules', 'freeze_delta', 'freeze_from_config', 'TrainableParamsCheckPoint'] diff --git a/tk/graph/ckpt_util.py b/mindpet/graph/ckpt_util.py similarity index 97% rename from tk/graph/ckpt_util.py rename to mindpet/graph/ckpt_util.py index 7085cf6..334f5dd 100644 --- a/tk/graph/ckpt_util.py +++ b/mindpet/graph/ckpt_util.py @@ -15,9 +15,9 @@ from mindspore.train.callback import ModelCheckpoint from mindspore.train.serialization import save_checkpoint, _get_merged_param_data -from tk.utils.exceptions import AbsolutePathError, LinkPathError -from tk.utils.io_utils import is_link_path -from tk.log.log import logger +from mindpet.utils.exceptions import AbsolutePathError, LinkPathError +from mindpet.utils.io_utils import is_link_path +from mindpet.log.log import logger class TrainableParamsCheckPoint(ModelCheckpoint): diff --git a/tk/graph/freeze_utils.py b/mindpet/graph/freeze_utils.py similarity index 97% rename from tk/graph/freeze_utils.py rename to mindpet/graph/freeze_utils.py index 45c488d..667d80b 100644 --- a/tk/graph/freeze_utils.py +++ b/mindpet/graph/freeze_utils.py @@ -10,10 +10,10 @@ from fnmatch import fnmatch import mindspore.nn as nn -from tk.log.log import logger -from tk.utils.constants import DELTA_LIST -from tk.utils.exceptions import ReadYamlFileError, ModelConfigFreezeInfoError, LinkPathError, UnexpectedError -from tk.utils.io_utils import read_yaml_file +from mindpet.log.log import logger +from mindpet.utils.constants import DELTA_LIST +from mindpet.utils.exceptions import ReadYamlFileError, ModelConfigFreezeInfoError, LinkPathError, UnexpectedError +from mindpet.utils.io_utils import read_yaml_file def freeze_modules(model: nn.Cell, diff --git a/tk/log/__init__.py b/mindpet/log/__init__.py similarity index 67% rename from tk/log/__init__.py rename to mindpet/log/__init__.py index cdc0c98..386cb80 100644 --- a/tk/log/__init__.py +++ b/mindpet/log/__init__.py @@ -2,6 +2,6 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.log.log import get_logger, logger, operation_logger_without_std, operation_logger, logger_without_std +from mindpet.log.log import get_logger, logger, operation_logger_without_std, operation_logger, logger_without_std __all__ = ['get_logger', 'logger', 'logger_without_std', 'operation_logger', 'operation_logger_without_std'] diff --git a/tk/log/log.py b/mindpet/log/log.py similarity index 98% rename from tk/log/log.py rename to mindpet/log/log.py index e68a761..15b3130 100644 --- a/tk/log/log.py +++ b/mindpet/log/log.py @@ -10,15 +10,15 @@ from typing import Dict, List, Tuple, Union import traceback -from tk.utils.constants import DEFAULT_MAX_LOG_FILE_NUM, DEFAULT_MAX_LOG_FILE_SIZE, ABNORMAL_EXIT_CODE -from tk.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError +from mindpet.utils.constants import DEFAULT_MAX_LOG_FILE_NUM, DEFAULT_MAX_LOG_FILE_SIZE, ABNORMAL_EXIT_CODE +from mindpet.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError try: from concurrent_log_handler import ConcurrentRotatingFileHandler as RFHandler except ImportError: from logging.handlers import RotatingFileHandler as RFHandler -from tk.log.log_utils import check_list, const, convert_nodes_devices_input, create_dirs, \ +from mindpet.log.log_utils import check_list, const, convert_nodes_devices_input, create_dirs, \ generate_rank_list, get_num_nodes_devices, get_rank_info, log_args_black_list_characters_replace, check_link_path logger_list = {} diff --git a/tk/log/log_utils.py b/mindpet/log/log_utils.py similarity index 98% rename from tk/log/log_utils.py rename to mindpet/log/log_utils.py index f472319..f217c6b 100644 --- a/tk/log/log_utils.py +++ b/mindpet/log/log_utils.py @@ -7,8 +7,8 @@ import platform from typing import Dict, List, Tuple, Union -from tk.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError -from tk.utils.constants import EMPTY_STRING +from mindpet.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError +from mindpet.utils.constants import EMPTY_STRING LOG_CONTENT_BLACK_LIST = ('\r', '\n', '\t', '\f', '\v', '\b', '\u000A', '\u000D', '\u000C', '\u000B', '\u0008', '\u007F', '\u0009') diff --git a/tk/security/__init__.py b/mindpet/security/__init__.py similarity index 100% rename from tk/security/__init__.py rename to mindpet/security/__init__.py diff --git a/tk/security/param_check/__init__.py b/mindpet/security/param_check/__init__.py similarity index 100% rename from tk/security/param_check/__init__.py rename to mindpet/security/param_check/__init__.py diff --git a/tk/security/param_check/base_check.py b/mindpet/security/param_check/base_check.py similarity index 94% rename from tk/security/param_check/base_check.py rename to mindpet/security/param_check/base_check.py index 53d2067..1979b2f 100644 --- a/tk/security/param_check/base_check.py +++ b/mindpet/security/param_check/base_check.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.security.param_check.option_check_utils import PathRightEscalationCheck, FileSizeCheck +from mindpet.security.param_check.option_check_utils import PathRightEscalationCheck, FileSizeCheck class BaseCheckParam: diff --git a/tk/security/param_check/model_config_params_check_util.py b/mindpet/security/param_check/model_config_params_check_util.py similarity index 93% rename from tk/security/param_check/model_config_params_check_util.py rename to mindpet/security/param_check/model_config_params_check_util.py index 76faef3..8ac3f38 100644 --- a/tk/security/param_check/model_config_params_check_util.py +++ b/mindpet/security/param_check/model_config_params_check_util.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.log.log import logger -from tk.utils.exceptions import ModelConfigParamsInfoError -from tk.utils.constants import MODEL_CONFIG_PARAMS_CHARACTER_BLACKLIST, MODEL_CONFIG_LEN_LIMIT, \ +from mindpet.log.log import logger +from mindpet.utils.exceptions import ModelConfigParamsInfoError +from mindpet.utils.constants import MODEL_CONFIG_PARAMS_CHARACTER_BLACKLIST, MODEL_CONFIG_LEN_LIMIT, \ TK_DEFINED_PARAM_NAME, INVALID_CUSTOM_PARAM_KEY_PREFIX, INVALID_CUSTOM_PARAM_VAL_PREFIX diff --git a/tk/security/param_check/option_check_utils.py b/mindpet/security/param_check/option_check_utils.py similarity index 98% rename from tk/security/param_check/option_check_utils.py rename to mindpet/security/param_check/option_check_utils.py index b6fcd7c..2d4d8f3 100644 --- a/tk/security/param_check/option_check_utils.py +++ b/mindpet/security/param_check/option_check_utils.py @@ -3,12 +3,12 @@ # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. import os -from tk.log.log import logger -from tk.utils.io_utils import is_link_path -from tk.utils.entrance_monitor import entrance_monitor -from tk.utils.constants import EMPTY_STRING, BLACKLIST_CHARACTERS, BASE_WHITELIST_CHARACTERS, MAX_FILE_MB_SIZE, \ +from mindpet.log.log import logger +from mindpet.utils.io_utils import is_link_path +from mindpet.utils.entrance_monitor import entrance_monitor +from mindpet.utils.constants import EMPTY_STRING, BLACKLIST_CHARACTERS, BASE_WHITELIST_CHARACTERS, MAX_FILE_MB_SIZE, \ MB_SIZE, ENTRANCE_TYPE, MIN_MODE, MAX_MODE, GB_SIZE -from tk.utils.exceptions import FileOversizeError, LinkPathError, LowDiskFreeSizeRiskError, PathContentError, \ +from mindpet.utils.exceptions import FileOversizeError, LinkPathError, LowDiskFreeSizeRiskError, PathContentError, \ AbsolutePathError, PathLengthError, FileNameLengthError, PathGranularityError, PathOwnerError, PathModeError, \ PathRightEscalationError, ManualCancelError diff --git a/tk/task/__init__.py b/mindpet/task/__init__.py similarity index 100% rename from tk/task/__init__.py rename to mindpet/task/__init__.py diff --git a/tk/task/evaluate_infer/__init__.py b/mindpet/task/evaluate_infer/__init__.py similarity index 100% rename from tk/task/evaluate_infer/__init__.py rename to mindpet/task/evaluate_infer/__init__.py diff --git a/tk/task/evaluate_infer/evaluate_infer_task.py b/mindpet/task/evaluate_infer/evaluate_infer_task.py similarity index 93% rename from tk/task/evaluate_infer/evaluate_infer_task.py rename to mindpet/task/evaluate_infer/evaluate_infer_task.py index 008b016..cee7f8a 100644 --- a/tk/task/evaluate_infer/evaluate_infer_task.py +++ b/mindpet/task/evaluate_infer/evaluate_infer_task.py @@ -5,16 +5,16 @@ import os import json import subprocess -from tk.utils.io_utils import read_yaml_file, read_json_file -from tk.security.param_check.option_check_utils import PathContentCheckParam -from tk.task.evaluate_infer.result_file_check import ResultFileCheckParam, ResultFileCheck -from tk.log.log import logger, operation_logger_without_std, record_operation_and_service_info_log, \ +from mindpet.utils.io_utils import read_yaml_file, read_json_file +from mindpet.security.param_check.option_check_utils import PathContentCheckParam +from mindpet.task.evaluate_infer.result_file_check import ResultFileCheckParam, ResultFileCheck +from mindpet.log.log import logger, operation_logger_without_std, record_operation_and_service_info_log, \ record_operation_and_service_error_log, record_operation_and_service_warning_log -from tk.utils.constants import MODEL_CONFIG_ROOT_KEYS, EVAL_RESULT_FILE_NAME, INFER_RESULT_FILE_NAME, SPACE_CHARACTER, \ +from mindpet.utils.constants import MODEL_CONFIG_ROOT_KEYS, EVAL_RESULT_FILE_NAME, INFER_RESULT_FILE_NAME, SPACE_CHARACTER, \ PATH_MODE_LIMIT, EVAL_INFER_TASK_NAMES, EVALUATE_TASK_NAME, EMPTY_STRING -from tk.utils.task_utils import create_output_path_subdir_with_uuid, model_config_params_check_item, \ +from mindpet.utils.task_utils import create_output_path_subdir_with_uuid, model_config_params_check_item, \ monitor_process_rsp_code, model_config_keys_check_item, path_in_out_loop_check -from tk.utils.exceptions import LinkPathError, ReadYamlFileError, UnexpectedError, MonitorProcessRspError, \ +from mindpet.utils.exceptions import LinkPathError, ReadYamlFileError, UnexpectedError, MonitorProcessRspError, \ CreateProcessError, TaskError, FileOversizeError, PathRightEscalationError diff --git a/tk/task/evaluate_infer/result_file_check.py b/mindpet/task/evaluate_infer/result_file_check.py similarity index 91% rename from tk/task/evaluate_infer/result_file_check.py rename to mindpet/task/evaluate_infer/result_file_check.py index f2e1a2e..9a5e43d 100644 --- a/tk/task/evaluate_infer/result_file_check.py +++ b/mindpet/task/evaluate_infer/result_file_check.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.security.param_check.base_check import BaseCheckParam, BaseCheck -from tk.security.param_check.option_check_utils import PathExistCheck, LinkPathCheck, get_real_path +from mindpet.security.param_check.base_check import BaseCheckParam, BaseCheck +from mindpet.security.param_check.option_check_utils import PathExistCheck, LinkPathCheck, get_real_path class ResultFileCheckParam(BaseCheckParam): diff --git a/tk/task/finetune/__init__.py b/mindpet/task/finetune/__init__.py similarity index 100% rename from tk/task/finetune/__init__.py rename to mindpet/task/finetune/__init__.py diff --git a/tk/task/finetune/finetune_options_check.py b/mindpet/task/finetune/finetune_options_check.py similarity index 94% rename from tk/task/finetune/finetune_options_check.py rename to mindpet/task/finetune/finetune_options_check.py index 999a7de..3d00cba 100644 --- a/tk/task/finetune/finetune_options_check.py +++ b/mindpet/task/finetune/finetune_options_check.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.utils.constants import GB_SIZE -from tk.security.param_check.base_check import BaseCheckParam, BaseCheck -from tk.security.param_check.option_check_utils import PathContentBlacklistCharactersCheck, AbsolutePathCheck, \ +from mindpet.utils.constants import GB_SIZE +from mindpet.security.param_check.base_check import BaseCheckParam, BaseCheck +from mindpet.security.param_check.option_check_utils import PathContentBlacklistCharactersCheck, AbsolutePathCheck, \ PathExistCheck, LinkPathCheck, PathContentLengthCheck, PathContentCharacterCheck, PathGranularityCheck, \ DiskFreeSpaceCheck, get_real_path diff --git a/tk/task/finetune/finetune_task.py b/mindpet/task/finetune/finetune_task.py similarity index 92% rename from tk/task/finetune/finetune_task.py rename to mindpet/task/finetune/finetune_task.py index 6cb5699..9d91802 100644 --- a/tk/task/finetune/finetune_task.py +++ b/mindpet/task/finetune/finetune_task.py @@ -4,13 +4,13 @@ import os import subprocess -from tk.utils.io_utils import read_yaml_file -from tk.utils.constants import MODEL_CONFIG_ROOT_KEYS -from tk.utils.exceptions import LinkPathError, TaskError, ReadYamlFileError, UnexpectedError, CreateProcessError, \ +from mindpet.utils.io_utils import read_yaml_file +from mindpet.utils.constants import MODEL_CONFIG_ROOT_KEYS +from mindpet.utils.exceptions import LinkPathError, TaskError, ReadYamlFileError, UnexpectedError, CreateProcessError, \ MonitorProcessRspError -from tk.log.log import record_operation_and_service_error_log, record_operation_and_service_info_log, \ +from mindpet.log.log import record_operation_and_service_error_log, record_operation_and_service_info_log, \ operation_logger_without_std -from tk.utils.task_utils import create_output_path_subdir_with_uuid, model_config_params_check_item, \ +from mindpet.utils.task_utils import create_output_path_subdir_with_uuid, model_config_params_check_item, \ extend_model_config_freeze_command, monitor_process_rsp_code, model_config_keys_check_item, path_in_out_loop_check diff --git a/tk/task/option_decorators.py b/mindpet/task/option_decorators.py similarity index 93% rename from tk/task/option_decorators.py rename to mindpet/task/option_decorators.py index 8c76d2b..1193a1c 100644 --- a/tk/task/option_decorators.py +++ b/mindpet/task/option_decorators.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.task.options import DataPathOption, PretrainedModelPathOption, OutputPathOption, BootFilePathOption, \ +from mindpet.task.options import DataPathOption, PretrainedModelPathOption, OutputPathOption, BootFilePathOption, \ ModelConfigPathOption, QuietOption, CkptPathOption, TimeoutOption diff --git a/mindpet/task/options/__init__.py b/mindpet/task/options/__init__.py new file mode 100644 index 0000000..6d2cfcb --- /dev/null +++ b/mindpet/task/options/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + +from mindpet.task.options.data_path_option import DataPathOption +from mindpet.task.options.pretrained_model_path_option import PretrainedModelPathOption +from mindpet.task.options.output_path_option import OutputPathOption +from mindpet.task.options.boot_file_path_option import BootFilePathOption +from mindpet.task.options.model_config_path_option import ModelConfigPathOption +from mindpet.task.options.quiet_option import QuietOption +from mindpet.task.options.ckpt_path_option import CkptPathOption +from mindpet.task.options.timeout_option import TimeoutOption diff --git a/tk/task/options/boot_file_path_option.py b/mindpet/task/options/boot_file_path_option.py similarity index 87% rename from tk/task/options/boot_file_path_option.py rename to mindpet/task/options/boot_file_path_option.py index d8ba92b..b2a5709 100644 --- a/tk/task/options/boot_file_path_option.py +++ b/mindpet/task/options/boot_file_path_option.py @@ -5,9 +5,9 @@ import os import click from click.exceptions import MissingParameter, BadParameter -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_file_length_check_param, default_file_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_file_length_check_param, default_file_content_check_param class BootFilePathOption(click.core.Option): diff --git a/tk/task/options/ckpt_path_option.py b/mindpet/task/options/ckpt_path_option.py similarity index 83% rename from tk/task/options/ckpt_path_option.py rename to mindpet/task/options/ckpt_path_option.py index 0ad9aba..43da9e5 100644 --- a/tk/task/options/ckpt_path_option.py +++ b/mindpet/task/options/ckpt_path_option.py @@ -4,9 +4,9 @@ import click from click.exceptions import MissingParameter -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param class CkptPathOption(click.core.Option): diff --git a/tk/task/options/data_path_option.py b/mindpet/task/options/data_path_option.py similarity index 83% rename from tk/task/options/data_path_option.py rename to mindpet/task/options/data_path_option.py index 3aa8a61..fefb6fe 100644 --- a/tk/task/options/data_path_option.py +++ b/mindpet/task/options/data_path_option.py @@ -4,9 +4,9 @@ import click from click.exceptions import MissingParameter -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param class DataPathOption(click.core.Option): diff --git a/tk/task/options/model_config_path_option.py b/mindpet/task/options/model_config_path_option.py similarity index 87% rename from tk/task/options/model_config_path_option.py rename to mindpet/task/options/model_config_path_option.py index c0eb00b..f664fc7 100644 --- a/tk/task/options/model_config_path_option.py +++ b/mindpet/task/options/model_config_path_option.py @@ -4,9 +4,9 @@ import click from click.exceptions import BadParameter -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_file_length_check_param, default_file_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_file_length_check_param, default_file_content_check_param class ModelConfigPathOption(click.core.Option): diff --git a/tk/task/options/output_path_option.py b/mindpet/task/options/output_path_option.py similarity index 83% rename from tk/task/options/output_path_option.py rename to mindpet/task/options/output_path_option.py index f6f8fd1..10555d7 100644 --- a/tk/task/options/output_path_option.py +++ b/mindpet/task/options/output_path_option.py @@ -4,9 +4,9 @@ import click from click.exceptions import MissingParameter -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param class OutputPathOption(click.core.Option): diff --git a/tk/task/options/path_check_param.py b/mindpet/task/options/path_check_param.py similarity index 82% rename from tk/task/options/path_check_param.py rename to mindpet/task/options/path_check_param.py index c85f998..50e2df0 100644 --- a/tk/task/options/path_check_param.py +++ b/mindpet/task/options/path_check_param.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.utils.constants import SPACE_CHARACTER -from tk.security.param_check.option_check_utils import PathLengthCheckParam, PathContentCheckParam -from tk.utils.constants import DEFAULT_PATH_LEN_MIN_LIMIT, DEFAULT_PATH_LEN_MAX_LIMIT, DEFAULT_FILE_LEN_MIN_LIMIT, \ +from mindpet.utils.constants import SPACE_CHARACTER +from mindpet.security.param_check.option_check_utils import PathLengthCheckParam, PathContentCheckParam +from mindpet.utils.constants import DEFAULT_PATH_LEN_MIN_LIMIT, DEFAULT_PATH_LEN_MAX_LIMIT, DEFAULT_FILE_LEN_MIN_LIMIT, \ DEFAULT_FILE_LEN_MAX_LIMIT default_folder_length_check_param = PathLengthCheckParam(path_min_limit=DEFAULT_PATH_LEN_MIN_LIMIT, diff --git a/tk/task/options/pretrained_model_path_option.py b/mindpet/task/options/pretrained_model_path_option.py similarity index 83% rename from tk/task/options/pretrained_model_path_option.py rename to mindpet/task/options/pretrained_model_path_option.py index 10e9129..1004e12 100644 --- a/tk/task/options/pretrained_model_path_option.py +++ b/mindpet/task/options/pretrained_model_path_option.py @@ -3,9 +3,9 @@ # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. import click -from tk.utils.constants import PATH_MODE_LIMIT -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck -from tk.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param +from mindpet.utils.constants import PATH_MODE_LIMIT +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheckParam, FinetuneOptionsCheck +from mindpet.task.options.path_check_param import default_folder_length_check_param, default_folder_content_check_param class PretrainedModelPathOption(click.core.Option): diff --git a/tk/task/options/quiet_option.py b/mindpet/task/options/quiet_option.py similarity index 100% rename from tk/task/options/quiet_option.py rename to mindpet/task/options/quiet_option.py diff --git a/tk/task/options/timeout_option.py b/mindpet/task/options/timeout_option.py similarity index 94% rename from tk/task/options/timeout_option.py rename to mindpet/task/options/timeout_option.py index 42d2070..7a27d02 100644 --- a/tk/task/options/timeout_option.py +++ b/mindpet/task/options/timeout_option.py @@ -5,7 +5,7 @@ import re import click from click.exceptions import BadParameter -from tk.utils.constants import TIMEOUT_REGEX, HOURS_PER_DAY, SPACE_CHARACTER, EMPTY_STRING, SECONDS_PER_HOUR +from mindpet.utils.constants import TIMEOUT_REGEX, HOURS_PER_DAY, SPACE_CHARACTER, EMPTY_STRING, SECONDS_PER_HOUR class TimeoutOption(click.core.Option): diff --git a/tk/tk_main.py b/mindpet/tk_main.py similarity index 71% rename from tk/tk_main.py rename to mindpet/tk_main.py index 3f6254d..a474442 100644 --- a/tk/tk_main.py +++ b/mindpet/tk_main.py @@ -4,13 +4,13 @@ import sys import click -from tk.utils.task_utils import handle_exception_log -from tk.utils.entrance_monitor import entrance_monitor -from tk.task.finetune.finetune_task import FinetuneTask -from tk.task.evaluate_infer.evaluate_infer_task import EvaluateInferTask -from tk.log.log import set_logger_property, record_operation_and_service_info_log -from tk.task.option_decorators import finetune_options, evaluate_options, infer_options -from tk.utils.constants import ENTRANCE_TYPE, EVALUATE_TASK_NAME, INFER_TASK_NAME, ABNORMAL_EXIT_CODE +from mindpet.utils.task_utils import handle_exception_log +from mindpet.utils.entrance_monitor import entrance_monitor +from mindpet.task.finetune.finetune_task import FinetuneTask +from mindpet.task.evaluate_infer.evaluate_infer_task import EvaluateInferTask +from mindpet.log.log import set_logger_property, record_operation_and_service_info_log +from mindpet.task.option_decorators import finetune_options, evaluate_options, infer_options +from mindpet.utils.constants import ENTRANCE_TYPE, EVALUATE_TASK_NAME, INFER_TASK_NAME, ABNORMAL_EXIT_CODE def cli_wrapper(): diff --git a/tk/tk_sdk.py b/mindpet/tk_sdk.py similarity index 89% rename from tk/tk_sdk.py rename to mindpet/tk_sdk.py index dbc5487..9fd56ef 100644 --- a/tk/tk_sdk.py +++ b/mindpet/tk_sdk.py @@ -2,11 +2,11 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.tk_main import cli -from tk.log.log import logger, set_logger_property -from tk.utils.task_utils import handle_exception_log -from tk.utils.entrance_monitor import entrance_monitor -from tk.utils.constants import ENTRANCE_TYPE, EMPTY_STRING, ARG_NAMES, TK_SDK_INTERFACE_NAMES +from mindpet.tk_main import cli +from mindpet.log.log import logger, set_logger_property +from mindpet.utils.task_utils import handle_exception_log +from mindpet.utils.entrance_monitor import entrance_monitor +from mindpet.utils.constants import ENTRANCE_TYPE, EMPTY_STRING, ARG_NAMES, TK_SDK_INTERFACE_NAMES entrance_monitor.set_value(ENTRANCE_TYPE, 'SDK') set_logger_property('SDK') diff --git a/tk/utils/__init__.py b/mindpet/utils/__init__.py similarity index 100% rename from tk/utils/__init__.py rename to mindpet/utils/__init__.py diff --git a/tk/utils/constants.py b/mindpet/utils/constants.py similarity index 100% rename from tk/utils/constants.py rename to mindpet/utils/constants.py diff --git a/tk/utils/entrance_monitor.py b/mindpet/utils/entrance_monitor.py similarity index 100% rename from tk/utils/entrance_monitor.py rename to mindpet/utils/entrance_monitor.py diff --git a/tk/utils/exceptions.py b/mindpet/utils/exceptions.py similarity index 98% rename from tk/utils/exceptions.py rename to mindpet/utils/exceptions.py index ce93d29..e06bb80 100644 --- a/tk/utils/exceptions.py +++ b/mindpet/utils/exceptions.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. -from tk.utils.constants import EMPTY_STRING +from mindpet.utils.constants import EMPTY_STRING # mxTuningKit包基础异常 diff --git a/tk/utils/io_utils.py b/mindpet/utils/io_utils.py similarity index 92% rename from tk/utils/io_utils.py rename to mindpet/utils/io_utils.py index dd9c7ed..4b38422 100644 --- a/tk/utils/io_utils.py +++ b/mindpet/utils/io_utils.py @@ -5,8 +5,8 @@ import os import json import yaml -from tk.utils.exceptions import LinkPathError -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES, EMPTY_STRING +from mindpet.utils.exceptions import LinkPathError +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES, EMPTY_STRING def is_link_path(path): diff --git a/tk/utils/task_utils.py b/mindpet/utils/task_utils.py similarity index 94% rename from tk/utils/task_utils.py rename to mindpet/utils/task_utils.py index 3a51831..fa9df50 100644 --- a/tk/utils/task_utils.py +++ b/mindpet/utils/task_utils.py @@ -9,10 +9,10 @@ import multiprocessing from pathlib import Path import click -from tk.security.param_check.model_config_params_check_util import ModelConfigParamsChecker -from tk.utils.exceptions import MakeDirError, PathLoopError, ModelConfigKeysInfoError, ModelConfigParamsInfoError -from tk.log.log import logger, logger_without_std, record_operation_and_service_info_log -from tk.utils.constants import OUTPUT_PATH_RANDOM_DIR_MODE, TIME_MONITOR_SLEEP_TIME, SECONDS_PER_HOUR, HOURS_PER_DAY +from mindpet.security.param_check.model_config_params_check_util import ModelConfigParamsChecker +from mindpet.utils.exceptions import MakeDirError, PathLoopError, ModelConfigKeysInfoError, ModelConfigParamsInfoError +from mindpet.log.log import logger, logger_without_std, record_operation_and_service_info_log +from mindpet.utils.constants import OUTPUT_PATH_RANDOM_DIR_MODE, TIME_MONITOR_SLEEP_TIME, SECONDS_PER_HOUR, HOURS_PER_DAY def monitor_process_rsp_code(task_name, task_process, timeout=None): diff --git a/tk/utils/version_control.py b/mindpet/utils/version_control.py similarity index 100% rename from tk/utils/version_control.py rename to mindpet/utils/version_control.py diff --git a/tk/utils/version_utils.py b/mindpet/utils/version_utils.py similarity index 100% rename from tk/utils/version_utils.py rename to mindpet/utils/version_utils.py diff --git a/set_up.py b/set_up.py index 83bf541..6538322 100644 --- a/set_up.py +++ b/set_up.py @@ -11,7 +11,7 @@ def get_version(): - version = '1.0.1' + version = '1.0.2' return version diff --git a/test/unit_test/delta/test_adapter.py b/test/unit_test/delta/test_adapter.py index bcb7915..b339d69 100644 --- a/test/unit_test/delta/test_adapter.py +++ b/test/unit_test/delta/test_adapter.py @@ -5,6 +5,7 @@ import os import logging import unittest +import argparse import mindspore import numpy as np @@ -12,11 +13,10 @@ from mindspore import dtype as mstype from mindspore.common.tensor import Tensor -from tk.delta.adapter import AdapterDense -from tk.delta.adapter import AdapterLayer +from mindpet.delta.adapter import AdapterDense +from mindpet.delta.adapter import AdapterLayer logging.getLogger().setLevel(logging.INFO) -mindspore.set_context(device_id=1) class TestAdapter(unittest.TestCase): # _check_in_channels @@ -393,5 +393,9 @@ def test_shard_with_invalid_type_strategy(self): logging.info('Finish test_shard_with_invalid_type_strategy') if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--device_id', type=int, default=0) + args = parser.parse_args() + mindspore.set_context(device_id=args.device_id) pytest.main(["-s", os.path.abspath(__file__)]) \ No newline at end of file diff --git a/test/unit_test/delta/test_lora.py b/test/unit_test/delta/test_lora.py index 1cd2e61..a2610c8 100644 --- a/test/unit_test/delta/test_lora.py +++ b/test/unit_test/delta/test_lora.py @@ -5,7 +5,7 @@ import os import logging import unittest - +import argparse import mindspore import numpy import pytest @@ -13,10 +13,9 @@ from mindspore.common.tensor import Tensor from mindspore.common.initializer import One -from tk.delta.lora import LoRADense +from mindpet.delta.lora import LoRADense logging.getLogger().setLevel(logging.INFO) -mindspore.set_context(device_id=1) class TestLoRADense(unittest.TestCase): # _check_num @@ -259,4 +258,8 @@ def test_lora_shard_with_invalid_type_strategy(self): logging.info('Finish test_lora_shard_with_invalid_type_strategy') if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--device_id', type=int, default=0) + args = parser.parse_args() + mindspore.set_context(device_id=args.device_id) pytest.main(["-s", os.path.abspath(__file__)]) diff --git a/test/unit_test/delta/test_low_rank_adapter.py b/test/unit_test/delta/test_low_rank_adapter.py index a89729f..f11bd45 100644 --- a/test/unit_test/delta/test_low_rank_adapter.py +++ b/test/unit_test/delta/test_low_rank_adapter.py @@ -6,7 +6,7 @@ import logging import unittest import operator - +import argparse import mindspore import numpy @@ -15,10 +15,9 @@ from mindspore.common.tensor import Tensor from mindspore.common.initializer import One -from tk.delta.low_rank_adapter import LowRankAdapterDense, LowRankAdapterLayer +from mindpet.delta.low_rank_adapter import LowRankAdapterDense, LowRankAdapterLayer logging.getLogger().setLevel(logging.INFO) -mindspore.set_context(device_id=1) class TestLowRankAdapterDense(unittest.TestCase): # _check_origin_params @@ -1031,4 +1030,8 @@ def test_layer_shard_with_invalid_type_strategy(self): if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--device_id', type=int, default=0) + args = parser.parse_args() + mindspore.set_context(device_id=args.device_id) pytest.main(["-s", os.path.abspath(__file__)]) diff --git a/test/unit_test/delta/test_prefix_layer.py b/test/unit_test/delta/test_prefix_layer.py index 7d2d0d3..279e841 100644 --- a/test/unit_test/delta/test_prefix_layer.py +++ b/test/unit_test/delta/test_prefix_layer.py @@ -7,13 +7,11 @@ import shutil import unittest import pytest - +import argparse import mindspore -from tk.delta.prefix_layer import PrefixLayer -from tk.utils.constants import DEFAULT_MODES, DEFAULT_FLAGS - -mindspore.set_context(device_id=1) +from mindpet.delta.prefix_layer import PrefixLayer +from mindpet.utils.constants import DEFAULT_MODES, DEFAULT_FLAGS logging.getLogger().setLevel(logging.INFO) LOCAL_PATH = os.path.join('/', 'tmp', 'ut').replace('\\', '/') @@ -183,4 +181,8 @@ def test_dropout_rate_is_one(self): if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--device_id', type=int, default=0) + args = parser.parse_args() + mindspore.set_context(device_id=args.device_id) pytest.main(["-s", os.path.abspath(__file__)]) diff --git a/test/unit_test/delta/test_rdrop.py b/test/unit_test/delta/test_rdrop.py index 1966bcd..d59990c 100644 --- a/test/unit_test/delta/test_rdrop.py +++ b/test/unit_test/delta/test_rdrop.py @@ -5,16 +5,15 @@ import logging import os import unittest - +import argparse import mindspore import numpy import pytest from mindspore import Tensor -from tk.delta.r_drop import rdrop_repeat, RDropLoss +from mindpet.delta.r_drop import rdrop_repeat, RDropLoss logging.getLogger().setLevel(logging.INFO) -mindspore.set_context(device_id=1) class TestRDrop(unittest.TestCase): def test_repeat_success(self): @@ -53,4 +52,8 @@ def test_init_rdrop(self): if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--device_id', type=int, default=0) + args = parser.parse_args() + mindspore.set_context(device_id=args.device_id) pytest.main(["-s", os.path.abspath(__file__)]) diff --git a/test/unit_test/entry/test_tk_sdk.py b/test/unit_test/entry/test_tk_sdk.py index 05e7b7d..6aefbd4 100644 --- a/test/unit_test/entry/test_tk_sdk.py +++ b/test/unit_test/entry/test_tk_sdk.py @@ -7,8 +7,8 @@ import shutil import unittest import pytest -import tk.tk_sdk as sdk -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES +import mindpet.tk_sdk as sdk +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES class TestTkSdk(unittest.TestCase): diff --git a/test/unit_test/graph/test_freeze_utils.py b/test/unit_test/graph/test_freeze_utils.py index 591328c..4871ec1 100644 --- a/test/unit_test/graph/test_freeze_utils.py +++ b/test/unit_test/graph/test_freeze_utils.py @@ -12,8 +12,8 @@ import pytest from mindspore import nn -from tk.graph.freeze_utils import freeze_modules, freeze_delta, freeze_from_config -from tk.utils.exceptions import ModelConfigFreezeInfoError +from mindpet.graph.freeze_utils import freeze_modules, freeze_delta, freeze_from_config +from mindpet.utils.exceptions import ModelConfigFreezeInfoError logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/graph/test_save_ckpt.py b/test/unit_test/graph/test_save_ckpt.py index 4dda4a5..d1b43ae 100644 --- a/test/unit_test/graph/test_save_ckpt.py +++ b/test/unit_test/graph/test_save_ckpt.py @@ -24,8 +24,8 @@ from mindspore.dataset.vision import Inter from mindspore import dtype as mstype, LossMonitor, Model, CheckpointConfig -from tk.graph.ckpt_util import TrainableParamsCheckPoint -from tk.utils.exceptions import AbsolutePathError, LinkPathError +from mindpet.graph.ckpt_util import TrainableParamsCheckPoint +from mindpet.utils.exceptions import AbsolutePathError, LinkPathError logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/log/test_concurrent_log.py b/test/unit_test/log/test_concurrent_log.py index 9ed85d5..04f0ca8 100644 --- a/test/unit_test/log/test_concurrent_log.py +++ b/test/unit_test/log/test_concurrent_log.py @@ -11,8 +11,8 @@ import multiprocessing import pytest -from tk.log.log import get_logger -from tk.log.log_utils import get_env_info +from mindpet.log.log import get_logger +from mindpet.log.log_utils import get_env_info CONFIG_TEST_BASE_DIR = os.path.expanduser('~/.cache/Huawei/mxTuningKit/log') diff --git a/test/unit_test/log/test_logger.py b/test/unit_test/log/test_logger.py index b1bfde0..63b4bab 100644 --- a/test/unit_test/log/test_logger.py +++ b/test/unit_test/log/test_logger.py @@ -9,11 +9,11 @@ from unittest import TestCase, mock import pytest -from tk.log.log import get_logger, get_file_path_list, set_logger_property, CustomizedRotatingFileHandler, \ +from mindpet.log.log import get_logger, get_file_path_list, set_logger_property, CustomizedRotatingFileHandler, \ LOG_RECORD_MAX_LEN -from tk.log.log_utils import log_args_black_list_characters_replace, wrap_local_working_directory, const -from tk.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError -from tk.utils.constants import DEFAULT_MAX_LOG_FILE_NUM, EMPTY_STRING +from mindpet.log.log_utils import log_args_black_list_characters_replace, wrap_local_working_directory, const +from mindpet.utils.exceptions import MakeDirError, UnsupportedPlatformError, PathOwnerError, PathModeError +from mindpet.utils.constants import DEFAULT_MAX_LOG_FILE_NUM, EMPTY_STRING MODE640 = 0o640 diff --git a/test/unit_test/log/test_logger_validator.py b/test/unit_test/log/test_logger_validator.py index 4a1bf12..bb90932 100644 --- a/test/unit_test/log/test_logger_validator.py +++ b/test/unit_test/log/test_logger_validator.py @@ -8,8 +8,8 @@ import logging from unittest import TestCase import pytest -import tk.log.log as ac -from tk.log.log import validate_file_input_format +import mindpet.log.log as ac +from mindpet.log.log import validate_file_input_format LOG_STD_INPUT_PARAM = { 'to_std': True, diff --git a/test/unit_test/security/param_check/test_base_check.py b/test/unit_test/security/param_check/test_base_check.py index 12332e1..b3e1344 100644 --- a/test/unit_test/security/param_check/test_base_check.py +++ b/test/unit_test/security/param_check/test_base_check.py @@ -6,8 +6,8 @@ import logging import unittest import pytest -from tk.security.param_check.base_check import BaseCheckParam, BaseCheck -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES +from mindpet.security.param_check.base_check import BaseCheckParam, BaseCheck +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/security/param_check/test_model_config_check_utils.py b/test/unit_test/security/param_check/test_model_config_check_utils.py index ea16f46..70de6b3 100644 --- a/test/unit_test/security/param_check/test_model_config_check_utils.py +++ b/test/unit_test/security/param_check/test_model_config_check_utils.py @@ -7,10 +7,10 @@ import os.path import unittest import pytest -from tk.task.finetune.finetune_task import FinetuneTask -from tk.security.param_check.model_config_params_check_util import ModelConfigParamsChecker -from tk.utils.exceptions import ModelConfigParamsInfoError -from tk.utils.constants import MODEL_CONFIG_LEN_LIMIT, MODEL_CONFIG_PARAMS_CHARACTER_BLACKLIST +from mindpet.task.finetune.finetune_task import FinetuneTask +from mindpet.security.param_check.model_config_params_check_util import ModelConfigParamsChecker +from mindpet.utils.exceptions import ModelConfigParamsInfoError +from mindpet.utils.constants import MODEL_CONFIG_LEN_LIMIT, MODEL_CONFIG_PARAMS_CHARACTER_BLACKLIST logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/security/param_check/test_option_check_utils.py b/test/unit_test/security/param_check/test_option_check_utils.py index 7d9523f..e122964 100644 --- a/test/unit_test/security/param_check/test_option_check_utils.py +++ b/test/unit_test/security/param_check/test_option_check_utils.py @@ -7,11 +7,11 @@ import unittest import logging import pytest -from tk.utils.entrance_monitor import entrance_monitor -from tk.utils.constants import ENTRANCE_TYPE, DEFAULT_MODES, DEFAULT_FLAGS, BLACKLIST_CHARACTERS, GB_SIZE -from tk.utils.exceptions import FileOversizeError, LinkPathError, AbsolutePathError, PathContentError, \ +from mindpet.utils.entrance_monitor import entrance_monitor +from mindpet.utils.constants import ENTRANCE_TYPE, DEFAULT_MODES, DEFAULT_FLAGS, BLACKLIST_CHARACTERS, GB_SIZE +from mindpet.utils.exceptions import FileOversizeError, LinkPathError, AbsolutePathError, PathContentError, \ FileNameLengthError, PathLengthError, PathGranularityError, PathRightEscalationError, LowDiskFreeSizeRiskError -from tk.security.param_check.option_check_utils import OptionBase, PathContentBlacklistCharactersCheck, \ +from mindpet.security.param_check.option_check_utils import OptionBase, PathContentBlacklistCharactersCheck, \ AbsolutePathCheck, PathExistCheck, LinkPathCheck, PathContentLengthCheck, PathContentCharacterCheck, \ PathGranularityCheck, PathRightEscalationCheck, FileSizeCheck, InteractionByEntrance, DiskFreeSpaceCheck, \ get_real_path diff --git a/test/unit_test/task/evaluate_infer/test_evaluate_infer_task.py b/test/unit_test/task/evaluate_infer/test_evaluate_infer_task.py index e499bdd..d5f2823 100644 --- a/test/unit_test/task/evaluate_infer/test_evaluate_infer_task.py +++ b/test/unit_test/task/evaluate_infer/test_evaluate_infer_task.py @@ -10,9 +10,9 @@ import unittest.mock as mock import pytest import yaml -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES -from tk.task.evaluate_infer.evaluate_infer_task import EvaluateInferTask -from tk.utils.exceptions import ReadYamlFileError, CreateProcessError, TaskError, MonitorProcessRspError, \ +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES +from mindpet.task.evaluate_infer.evaluate_infer_task import EvaluateInferTask +from mindpet.utils.exceptions import ReadYamlFileError, CreateProcessError, TaskError, MonitorProcessRspError, \ PathRightEscalationError, UnexpectedError logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/task/evaluate_infer/test_result_file_check.py b/test/unit_test/task/evaluate_infer/test_result_file_check.py index 8715da1..fa23041 100644 --- a/test/unit_test/task/evaluate_infer/test_result_file_check.py +++ b/test/unit_test/task/evaluate_infer/test_result_file_check.py @@ -9,10 +9,10 @@ import json import unittest import pytest -from tk.utils.constants import SPACE_CHARACTER, PATH_MODE_LIMIT -from tk.security.param_check.option_check_utils import PathContentCheckParam -from tk.task.evaluate_infer.result_file_check import ResultFileCheckParam, ResultFileCheck -from tk.utils.exceptions import LinkPathError, FileOversizeError, PathRightEscalationError +from mindpet.utils.constants import SPACE_CHARACTER, PATH_MODE_LIMIT +from mindpet.security.param_check.option_check_utils import PathContentCheckParam +from mindpet.task.evaluate_infer.result_file_check import ResultFileCheckParam, ResultFileCheck +from mindpet.utils.exceptions import LinkPathError, FileOversizeError, PathRightEscalationError class TestEvaluateResultCheck(unittest.TestCase): diff --git a/test/unit_test/task/finetune/test_finetune_options_check.py b/test/unit_test/task/finetune/test_finetune_options_check.py index 6280a7b..9b84a32 100644 --- a/test/unit_test/task/finetune/test_finetune_options_check.py +++ b/test/unit_test/task/finetune/test_finetune_options_check.py @@ -6,9 +6,9 @@ import os import unittest import pytest -from tk.task.finetune.finetune_options_check import FinetuneOptionsCheck, FinetuneOptionsCheckParam -from tk.security.param_check.option_check_utils import PathLengthCheckParam, PathContentCheckParam -from tk.utils.constants import DEFAULT_PATH_LEN_MIN_LIMIT, DEFAULT_PATH_LEN_MAX_LIMIT, PATH_MODE_LIMIT, \ +from mindpet.task.finetune.finetune_options_check import FinetuneOptionsCheck, FinetuneOptionsCheckParam +from mindpet.security.param_check.option_check_utils import PathLengthCheckParam, PathContentCheckParam +from mindpet.utils.constants import DEFAULT_PATH_LEN_MIN_LIMIT, DEFAULT_PATH_LEN_MAX_LIMIT, PATH_MODE_LIMIT, \ DEFAULT_FLAGS, DEFAULT_MODES, DEFAULT_FILE_LEN_MIN_LIMIT, DEFAULT_FILE_LEN_MAX_LIMIT diff --git a/test/unit_test/task/finetune/test_finetune_task.py b/test/unit_test/task/finetune/test_finetune_task.py index 3c37291..9c53ff7 100644 --- a/test/unit_test/task/finetune/test_finetune_task.py +++ b/test/unit_test/task/finetune/test_finetune_task.py @@ -9,9 +9,9 @@ from unittest import mock import pytest import yaml -from tk.task.finetune.finetune_task import FinetuneTask -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES -from tk.utils.exceptions import PathLoopError, ReadYamlFileError, ModelConfigKeysInfoError, \ +from mindpet.task.finetune.finetune_task import FinetuneTask +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES +from mindpet.utils.exceptions import PathLoopError, ReadYamlFileError, ModelConfigKeysInfoError, \ ModelConfigParamsInfoError, TaskError, CreateProcessError, MonitorProcessRspError, UnexpectedError logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/utils/test_entrance_monitor.py b/test/unit_test/utils/test_entrance_monitor.py index 84774c9..c4781ec 100644 --- a/test/unit_test/utils/test_entrance_monitor.py +++ b/test/unit_test/utils/test_entrance_monitor.py @@ -6,8 +6,8 @@ import unittest import logging import pytest -from tk.utils.entrance_monitor import EntranceMonitor -from tk.utils.constants import ENTRANCE_TYPE +from mindpet.utils.entrance_monitor import EntranceMonitor +from mindpet.utils.constants import ENTRANCE_TYPE logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/utils/test_io_utils.py b/test/unit_test/utils/test_io_utils.py index 53f95d8..97fcfe5 100644 --- a/test/unit_test/utils/test_io_utils.py +++ b/test/unit_test/utils/test_io_utils.py @@ -10,9 +10,9 @@ import logging import yaml import pytest -from tk.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES, EMPTY_STRING -from tk.utils.io_utils import read_json_file, read_yaml_file -from tk.utils.exceptions import LinkPathError +from mindpet.utils.constants import DEFAULT_FLAGS, DEFAULT_MODES, EMPTY_STRING +from mindpet.utils.io_utils import read_json_file, read_yaml_file +from mindpet.utils.exceptions import LinkPathError logging.getLogger().setLevel(logging.INFO) diff --git a/test/unit_test/utils/test_task_utils.py b/test/unit_test/utils/test_task_utils.py index a02bb74..9fddfaa 100644 --- a/test/unit_test/utils/test_task_utils.py +++ b/test/unit_test/utils/test_task_utils.py @@ -10,8 +10,8 @@ import unittest.mock as mock import click import pytest -from tk.utils.exceptions import MakeDirError -from tk.utils.task_utils import timeout_monitor, create_output_path_subdir_with_uuid, model_config_keys_check_item, \ +from mindpet.utils.exceptions import MakeDirError +from mindpet.utils.task_utils import timeout_monitor, create_output_path_subdir_with_uuid, model_config_keys_check_item, \ model_config_params_check_item, extend_model_config_freeze_command, handle_exception_log logging.getLogger().setLevel(logging.INFO) diff --git a/tk/delta/__init__.py b/tk/delta/__init__.py deleted file mode 100644 index 6fcb928..0000000 --- a/tk/delta/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. - -from tk.delta.lora import LoRADense -from tk.delta.prefix_layer import PrefixLayer -from tk.delta.low_rank_adapter import LowRankAdapterDense, LowRankAdapterLayer -from tk.delta.adapter import AdapterDense, AdapterLayer -from tk.delta.r_drop import RDropLoss, rdrop_repeat - -__all__ = ['LoRADense', 'PrefixLayer', 'LowRankAdapterDense', 'LowRankAdapterLayer', - 'AdapterDense', 'AdapterLayer', 'RDropLoss', 'rdrop_repeat'] diff --git a/tk/task/options/__init__.py b/tk/task/options/__init__.py deleted file mode 100644 index ea10dd4..0000000 --- a/tk/task/options/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright © Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. - -from tk.task.options.data_path_option import DataPathOption -from tk.task.options.pretrained_model_path_option import PretrainedModelPathOption -from tk.task.options.output_path_option import OutputPathOption -from tk.task.options.boot_file_path_option import BootFilePathOption -from tk.task.options.model_config_path_option import ModelConfigPathOption -from tk.task.options.quiet_option import QuietOption -from tk.task.options.ckpt_path_option import CkptPathOption -from tk.task.options.timeout_option import TimeoutOption