From 0da3190ff03a34a9f48a7efa1abce2936390e9c8 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Sat, 23 Mar 2024 10:14:31 +0800 Subject: [PATCH 01/12] sort reused steps whose startedAt are identical by key Signed-off-by: zjgemi --- dpgen2/entrypoint/submit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dpgen2/entrypoint/submit.py b/dpgen2/entrypoint/submit.py index 8190d2bd..e3f7b6db 100644 --- a/dpgen2/entrypoint/submit.py +++ b/dpgen2/entrypoint/submit.py @@ -734,7 +734,10 @@ def print_list_steps( def successful_step_keys(wf): all_step_keys = [] - for step in wf.query_step(): + steps = wf.query_step() + # For reused steps whose startedAt are identical, sort them by key + steps.sort(key=lambda x: "%s-%s" % (x.startedAt, x.key)) + for step in steps: if step.key is not None and step.phase == "Succeeded": all_step_keys.append(step.key) return all_step_keys From d9f9255c7c7c99773803a84d539b146ef6322725 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Sun, 24 Mar 2024 09:58:34 +0800 Subject: [PATCH 02/12] sort reused steps whose startedAt are identical by key in resubmit Signed-off-by: zjgemi --- dpgen2/entrypoint/submit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dpgen2/entrypoint/submit.py b/dpgen2/entrypoint/submit.py index e3f7b6db..f69d5319 100644 --- a/dpgen2/entrypoint/submit.py +++ b/dpgen2/entrypoint/submit.py @@ -871,6 +871,8 @@ def resubmit_concurrent_learning( reused_folded_keys[k] = [k] reused_keys = sum(reused_folded_keys.values(), []) reuse_step = old_wf.query_step(key=reused_keys) + # For reused steps whose startedAt are identical, sort them by key + reuse_step.sort(key=lambda x: "%s-%s" % (x.startedAt, x.key)) wf = submit_concurrent_learning( wf_config, From 8ab36059880ac1493f26043292c25396d8be9d55 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Wed, 27 Mar 2024 11:57:00 +0800 Subject: [PATCH 03/12] fix dpgen and multitask-dpgen for deepmd v3 Signed-off-by: zjgemi --- dpgen2/constants.py | 2 +- dpgen2/entrypoint/submit.py | 2 +- dpgen2/op/prep_dp_train.py | 8 ++++---- dpgen2/op/run_dp_train.py | 4 ++-- dpgen2/op/run_lmp.py | 7 ++++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/dpgen2/constants.py b/dpgen2/constants.py index 12dbe29f..8107e41f 100644 --- a/dpgen2/constants.py +++ b/dpgen2/constants.py @@ -3,7 +3,7 @@ train_script_name = "input.json" train_log_name = "train.log" model_name_pattern = "model.%03d" -model_name_match_pattern = r"model\.[0-9]{3,}" +model_name_match_pattern = r"model\.[0-9]{3,}(|\.pb|\.pth)" lmp_index_pattern = "%06d" lmp_task_pattern = "task." + lmp_index_pattern lmp_conf_name = "conf.lmp" diff --git a/dpgen2/entrypoint/submit.py b/dpgen2/entrypoint/submit.py index 61c8b883..67d73b94 100644 --- a/dpgen2/entrypoint/submit.py +++ b/dpgen2/entrypoint/submit.py @@ -552,7 +552,7 @@ def workflow_concurrent_learning( train_config["multitask"] = True train_config["head"] = head train_config["multi_init_data_idx"] = multi_init_data_idx - lmp_config["head"] = head + explore_config["head"] = head else: init_data_prefix = config["inputs"]["init_data_prefix"] init_data = config["inputs"]["init_data_sys"] diff --git a/dpgen2/op/prep_dp_train.py b/dpgen2/op/prep_dp_train.py index 3fdaf9f4..508f0e6a 100644 --- a/dpgen2/op/prep_dp_train.py +++ b/dpgen2/op/prep_dp_train.py @@ -110,7 +110,7 @@ def execute( def _set_desc_seed(self, desc): if desc["type"] == "hybrid": for desc in desc["list"]: - desc["seed"] = random.randrange(sys.maxsize) % (2**32) + self._set_desc_seed(desc) elif desc["type"] not in ["dpa1", "dpa2"]: desc["seed"] = random.randrange(sys.maxsize) % (2**32) @@ -119,10 +119,10 @@ def _script_rand_seed( input_dict, ): jtmp = input_dict.copy() - if "shared_dict" in jtmp["model"] and "model_dict" in jtmp["model"]: - if "dpa1_dpau_descriptor_1" in jtmp["model"]["shared_dict"]: - self._set_desc_seed(jtmp["model"]["shared_dict"]["dpa1_dpau_descriptor_1"]) + if "model_dict" in jtmp["model"]: for d in jtmp["model"]["model_dict"].values(): + if isinstance(d["descriptor"], str): + self._set_desc_seed(jtmp["model"]["shared_dict"][d["descriptor"]]) d["fitting_net"]["seed"] = random.randrange(sys.maxsize) % (2**32) else: self._set_desc_seed(jtmp["model"]["descriptor"]) diff --git a/dpgen2/op/run_dp_train.py b/dpgen2/op/run_dp_train.py index 78cdc35e..a7c0f826 100644 --- a/dpgen2/op/run_dp_train.py +++ b/dpgen2/op/run_dp_train.py @@ -309,7 +309,6 @@ def write_data_to_input_script( v["training_data"]["systems"] += [str(init_data[ii]) for ii in multi_init_data_idx[k]] if k == head: v["training_data"]["systems"] += [str(ii) for ii in iter_data] - v.pop("validation_data", None) return odict data_list = [str(ii) for ii in init_data] + [str(ii) for ii in iter_data] if major_version == "1": @@ -420,7 +419,7 @@ def decide_init_model( @staticmethod def training_args(): - doc_impl = "The implementation of DP. It can be 'tensorflow' or 'pytorch'. 'tensorflow' for default." + doc_impl = "The implementation/backend of DP. It can be 'tensorflow' or 'pytorch'. 'tensorflow' for default." doc_init_model_policy = "The policy of init-model training. It can be\n\n\ - 'no': No init-model training. Traing from scratch.\n\n\ - 'yes': Do init-model training.\n\n\ @@ -449,6 +448,7 @@ def training_args(): optional=True, default="tensorflow", doc=doc_impl, + alias=["backend"], ), Argument( "init_model_policy", diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index fdbf461d..e6fba1e6 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -143,12 +143,12 @@ def execute( model_names = [] for idx, mm in enumerate(model_files): ext = os.path.splitext(mm)[-1] - mname = model_name_pattern % (idx) + ext - model_names.append(mname) if ext == ".pb": + mname = model_name_pattern % (idx) + ".pb" Path(mname).symlink_to(mm) elif ext == ".pt": # freeze model + mname = model_name_pattern % (idx) + ".pth" freeze_args = "-o %s" % mname if config.get("head") is not None: freeze_args += " --head %s" % config["head"] @@ -167,6 +167,7 @@ def execute( "\n", ))) raise TransientError("freeze failed") + model_names.append(mname) if shuffle_models: random.shuffle(model_names) @@ -275,7 +276,7 @@ def set_models(lmp_input_name: str, model_names: List[str]): f"in line {lmp_input_lines[idx]}" ) new_line_split[match_first:match_last] = model_names - lmp_input_lines[idx] = " ".join(new_line_split) + lmp_input_lines[idx] = " ".join(new_line_split) + "\n" with open(lmp_input_name, "w", encoding="utf8") as f: f.write("".join(lmp_input_lines)) From 16ce26ebc06ddfc51565014c643cf173be8b7955 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 09:52:18 +0800 Subject: [PATCH 04/12] fix multitask init model Signed-off-by: zjgemi --- dpgen2/op/run_dp_train.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dpgen2/op/run_dp_train.py b/dpgen2/op/run_dp_train.py index a7c0f826..1d39fe3b 100644 --- a/dpgen2/op/run_dp_train.py +++ b/dpgen2/op/run_dp_train.py @@ -348,9 +348,16 @@ def write_other_to_input_script( odict["training"]["disp_file"] = "lcurve.out" if do_init_model: odict["learning_rate"]["start_lr"] = config["init_model_start_lr"] - odict["loss"]["start_pref_e"] = config["init_model_start_pref_e"] - odict["loss"]["start_pref_f"] = config["init_model_start_pref_f"] - odict["loss"]["start_pref_v"] = config["init_model_start_pref_v"] + if "loss_dict" in odict: + for v in odict["loss_dict"].values(): + if isinstance(v, dict): + v["start_pref_e"] = config["init_model_start_pref_e"] + v["start_pref_f"] = config["init_model_start_pref_f"] + v["start_pref_v"] = config["init_model_start_pref_v"] + else: + odict["loss"]["start_pref_e"] = config["init_model_start_pref_e"] + odict["loss"]["start_pref_f"] = config["init_model_start_pref_f"] + odict["loss"]["start_pref_v"] = config["init_model_start_pref_v"] if major_version == "1": odict["training"]["stop_batch"] = config["init_model_numb_steps"] elif major_version == "2": From 3221a6b79cf99b34b1848fd64f6888a5dfa110b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 01:53:18 +0000 Subject: [PATCH 05/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dpgen2/op/prep_dp_train.py | 4 ++- dpgen2/op/run_dp_train.py | 61 ++++++++++++++++++++++++++++---------- dpgen2/op/run_lmp.py | 26 +++++++++------- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/dpgen2/op/prep_dp_train.py b/dpgen2/op/prep_dp_train.py index 508f0e6a..b7b287a1 100644 --- a/dpgen2/op/prep_dp_train.py +++ b/dpgen2/op/prep_dp_train.py @@ -126,6 +126,8 @@ def _script_rand_seed( d["fitting_net"]["seed"] = random.randrange(sys.maxsize) % (2**32) else: self._set_desc_seed(jtmp["model"]["descriptor"]) - jtmp["model"]["fitting_net"]["seed"] = random.randrange(sys.maxsize) % (2**32) + jtmp["model"]["fitting_net"]["seed"] = random.randrange(sys.maxsize) % ( + 2**32 + ) jtmp["training"]["seed"] = random.randrange(sys.maxsize) % (2**32) return jtmp diff --git a/dpgen2/op/run_dp_train.py b/dpgen2/op/run_dp_train.py index 1d39fe3b..b6c47996 100644 --- a/dpgen2/op/run_dp_train.py +++ b/dpgen2/op/run_dp_train.py @@ -167,7 +167,13 @@ def execute( # update the input dict train_dict = RunDPTrain.write_data_to_input_script( - train_dict, config, init_data, iter_data_exp, auto_prob_str, major_version, valid_data + train_dict, + config, + init_data, + iter_data_exp, + auto_prob_str, + major_version, + valid_data, ) train_dict = RunDPTrain.write_other_to_input_script( train_dict, config, do_init_model, major_version @@ -198,11 +204,25 @@ def clean_before_quit(): # train model if impl == "tensorflow" and os.path.isfile("checkpoint"): - command = dp_command + ["train", "--restart", "model.ckpt", train_script_name] + command = dp_command + [ + "train", + "--restart", + "model.ckpt", + train_script_name, + ] elif impl == "pytorch" and len(glob.glob("model.ckpt-[0-9]*.pt")) > 0: - checkpoint = "model.ckpt-%s.pt" % max([int(f[11:-3]) for f in glob.glob("model.ckpt-[0-9]*.pt")]) - command = dp_command + ["train", "--restart", checkpoint, train_script_name] - elif (do_init_model or finetune_mode == "train-init") and not config["init_model_with_finetune"]: + checkpoint = "model.ckpt-%s.pt" % max( + [int(f[11:-3]) for f in glob.glob("model.ckpt-[0-9]*.pt")] + ) + command = dp_command + [ + "train", + "--restart", + checkpoint, + train_script_name, + ] + elif (do_init_model or finetune_mode == "train-init") and not config[ + "init_model_with_finetune" + ]: if impl == "tensorflow": command = dp_command + [ "train", @@ -217,13 +237,20 @@ def clean_before_quit(): str(init_model), train_script_name, ] - elif finetune_mode == "finetune" or ((do_init_model or finetune_mode == "train-init") and config["init_model_with_finetune"]): - command = dp_command + [ - "train", - train_script_name, - "--finetune", - str(init_model), - ] + finetune_args.split() + elif finetune_mode == "finetune" or ( + (do_init_model or finetune_mode == "train-init") + and config["init_model_with_finetune"] + ): + command = ( + dp_command + + [ + "train", + train_script_name, + "--finetune", + str(init_model), + ] + + finetune_args.split() + ) else: command = dp_command + ["train", train_script_name] ret, out, err = run_command(command) @@ -306,7 +333,9 @@ def write_data_to_input_script( for k, v in odict["training"]["data_dict"].items(): v["training_data"]["systems"] = [] if k in multi_init_data_idx: - v["training_data"]["systems"] += [str(init_data[ii]) for ii in multi_init_data_idx[k]] + v["training_data"]["systems"] += [ + str(init_data[ii]) for ii in multi_init_data_idx[k] + ] if k == head: v["training_data"]["systems"] += [str(ii) for ii in iter_data] return odict @@ -446,7 +475,9 @@ def training_args(): doc_finetune_args = "Extra arguments for finetuning" doc_multitask = "Do multitask training" doc_head = "Head to use in the multitask training" - doc_multi_init_data_idx = "A dict mapping from task name to list of indices in the init data" + doc_multi_init_data_idx = ( + "A dict mapping from task name to list of indices in the init data" + ) doc_init_model_with_finetune = "Use finetune for init model" return [ Argument( @@ -541,7 +572,7 @@ def training_args(): optional=True, default=None, doc=doc_multi_init_data_idx, - ) + ), ] @staticmethod diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index e6fba1e6..1ea37f51 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -155,17 +155,21 @@ def execute( freeze_cmd = "dp --pt freeze -c %s %s" % (mm, freeze_args) ret, out, err = run_command(freeze_cmd, shell=True) if ret != 0: - logging.error("".join(( - "freeze failed\n", - "command was", - freeze_cmd, - "out msg", - out, - "\n", - "err msg", - err, - "\n", - ))) + logging.error( + "".join( + ( + "freeze failed\n", + "command was", + freeze_cmd, + "out msg", + out, + "\n", + "err msg", + err, + "\n", + ) + ) + ) raise TransientError("freeze failed") model_names.append(mname) From 501890ad1cdddc64923d882d6c7b262535f620d8 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 10:52:32 +0800 Subject: [PATCH 06/12] fix UT Signed-off-by: zjgemi --- dpgen2/constants.py | 5 +++-- dpgen2/op/run_lmp.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dpgen2/constants.py b/dpgen2/constants.py index 8107e41f..fe1bffbf 100644 --- a/dpgen2/constants.py +++ b/dpgen2/constants.py @@ -2,8 +2,9 @@ train_task_pattern = "task." + train_index_pattern train_script_name = "input.json" train_log_name = "train.log" -model_name_pattern = "model.%03d" -model_name_match_pattern = r"model\.[0-9]{3,}(|\.pb|\.pth)" +model_name_pattern = "model.%03d.pb" +pytorch_model_name_pattern = "model.%03d.pth" +model_name_match_pattern = r"model\.[0-9]{3,}(\.pb|\.pth)" lmp_index_pattern = "%06d" lmp_task_pattern = "task." + lmp_index_pattern lmp_conf_name = "conf.lmp" diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index 1ea37f51..5523fbd9 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -38,6 +38,7 @@ model_name_match_pattern, model_name_pattern, plm_output_name, + pytorch_model_name_pattern, ) from dpgen2.utils import ( BinaryFileInput, @@ -144,11 +145,11 @@ def execute( for idx, mm in enumerate(model_files): ext = os.path.splitext(mm)[-1] if ext == ".pb": - mname = model_name_pattern % (idx) + ".pb" + mname = model_name_pattern % (idx) Path(mname).symlink_to(mm) elif ext == ".pt": # freeze model - mname = model_name_pattern % (idx) + ".pth" + mname = pytorch_model_name_pattern % (idx) freeze_args = "-o %s" % mname if config.get("head") is not None: freeze_args += " --head %s" % config["head"] From 23bf3b9f568c8fe1a8fe3c5ef5ebe404c03f6d64 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 11:21:00 +0800 Subject: [PATCH 07/12] fix UT and pyright Signed-off-by: zjgemi --- dpgen2/op/run_dp_train.py | 21 +++++++++++---------- dpgen2/op/run_lmp.py | 2 ++ tests/op/test_run_dp_train.py | 6 +++++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dpgen2/op/run_dp_train.py b/dpgen2/op/run_dp_train.py index b6c47996..f3e25e12 100644 --- a/dpgen2/op/run_dp_train.py +++ b/dpgen2/op/run_dp_train.py @@ -124,10 +124,11 @@ def execute( finetune_mode = ip["optional_parameter"]["finetune_mode"] config = ip["config"] if ip["config"] is not None else {} impl = ip["config"].get("impl", "tensorflow") - if impl == "tensorflow": - dp_command = ["dp"] - elif impl == "pytorch": + assert impl in ["tensorflow", "pytorch"] + if impl == "pytorch": dp_command = ["dp", "--pt"] + else: + dp_command = ["dp"] finetune_args = config.get("finetune_args", "") config = RunDPTrain.normalize_config(config) task_name = ip["task_name"] @@ -223,17 +224,17 @@ def clean_before_quit(): elif (do_init_model or finetune_mode == "train-init") and not config[ "init_model_with_finetune" ]: - if impl == "tensorflow": + if impl == "pytorch": command = dp_command + [ "train", - "--init-frz-model", + "--init-model", str(init_model), train_script_name, ] - elif impl == "pytorch": + else: command = dp_command + [ "train", - "--init-model", + "--init-frz-model", str(init_model), train_script_name, ] @@ -279,7 +280,9 @@ def clean_before_quit(): shutil.copy2("input_v2_compat.json", train_script_name) # freeze model - if impl == "tensorflow": + if impl == "pytorch": + model_file = "model.ckpt.pt" + else: ret, out, err = run_command(["dp", "freeze", "-o", "frozen_model.pb"]) if ret != 0: clean_before_quit() @@ -298,8 +301,6 @@ def clean_before_quit(): ) raise FatalError("dp freeze failed") model_file = "frozen_model.pb" - elif impl == "pytorch": - model_file = "model.ckpt.pt" fplog.write("#=================== freeze std out ===================\n") fplog.write(out) fplog.write("#=================== freeze std err ===================\n") diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index 5523fbd9..563ed422 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -172,6 +172,8 @@ def execute( ) ) raise TransientError("freeze failed") + else: + raise RuntimeError("Model file with extension '%s' is not supported" % ext) model_names.append(mname) if shuffle_models: diff --git a/tests/op/test_run_dp_train.py b/tests/op/test_run_dp_train.py index 330a0c13..04222a54 100644 --- a/tests/op/test_run_dp_train.py +++ b/tests/op/test_run_dp_train.py @@ -314,6 +314,7 @@ def test_decide_init_model_config_larger_than_yes(self): def test_update_input_dict_v1_init_model(self): odict = RunDPTrain.write_data_to_input_script( self.idict_v1, + self.config, self.init_data, self.iter_data_exp, auto_prob_str="prob_sys_size; 0:4:0.9; 4:7:0.1", @@ -329,6 +330,7 @@ def test_update_input_dict_v1_init_model(self): def test_update_input_dict_v1(self): odict = RunDPTrain.write_data_to_input_script( self.idict_v1, + self.config, self.init_data, self.iter_data_exp, auto_prob_str="prob_sys_size", @@ -345,6 +347,7 @@ def test_update_input_dict_v2_init_model(self): idict = self.idict_v2 odict = RunDPTrain.write_data_to_input_script( idict, + self.config, self.init_data, self.iter_data_exp, auto_prob_str="prob_sys_size; 0:4:0.9; 4:7:0.1", @@ -361,6 +364,7 @@ def test_update_input_dict_v2(self): idict = self.idict_v2 odict = RunDPTrain.write_data_to_input_script( idict, + self.config, self.init_data, self.iter_data_exp, auto_prob_str="prob_sys_size", @@ -820,7 +824,7 @@ def tearDown(self): def test_update_input_dict_v2_empty_list(self): idict = self.idict_v2 odict = RunDPTrain.write_data_to_input_script( - idict, self.init_data, [], auto_prob_str="prob_sys_size", major_version="2" + idict, self.config, self.init_data, [], auto_prob_str="prob_sys_size", major_version="2" ) config = self.config.copy() config["init_model_policy"] = "no" From cf59e849b561eebd40b6ff5016d8d34890ed8962 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 03:21:13 +0000 Subject: [PATCH 08/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dpgen2/op/run_lmp.py | 4 +++- tests/op/test_run_dp_train.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index 563ed422..ba7b40d8 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -173,7 +173,9 @@ def execute( ) raise TransientError("freeze failed") else: - raise RuntimeError("Model file with extension '%s' is not supported" % ext) + raise RuntimeError( + "Model file with extension '%s' is not supported" % ext + ) model_names.append(mname) if shuffle_models: diff --git a/tests/op/test_run_dp_train.py b/tests/op/test_run_dp_train.py index 04222a54..ae3ba56a 100644 --- a/tests/op/test_run_dp_train.py +++ b/tests/op/test_run_dp_train.py @@ -824,7 +824,12 @@ def tearDown(self): def test_update_input_dict_v2_empty_list(self): idict = self.idict_v2 odict = RunDPTrain.write_data_to_input_script( - idict, self.config, self.init_data, [], auto_prob_str="prob_sys_size", major_version="2" + idict, + self.config, + self.init_data, + [], + auto_prob_str="prob_sys_size", + major_version="2", ) config = self.config.copy() config["init_model_policy"] = "no" From 08a712bfd0919b06241186e0aa185ee667759314 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 12:17:39 +0800 Subject: [PATCH 09/12] fix UT Signed-off-by: zjgemi --- tests/op/test_run_dp_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/op/test_run_dp_train.py b/tests/op/test_run_dp_train.py index ae3ba56a..d4366df3 100644 --- a/tests/op/test_run_dp_train.py +++ b/tests/op/test_run_dp_train.py @@ -866,7 +866,7 @@ def test_exec_v2_empty_list(self): ) ) self.assertEqual(out["script"], work_dir / train_script_name) - self.assertEqual(out["model"], work_dir / "frozen_model.pb") + self.assertEqual(out["model"], self.init_model) self.assertEqual(out["lcurve"], work_dir / "lcurve.out") self.assertEqual(out["log"], work_dir / "train.log") From ad5f24aefc6860b897a58f43300f284daa222692 Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 14:51:41 +0800 Subject: [PATCH 10/12] fix UT Signed-off-by: zjgemi --- dpgen2/op/run_lmp.py | 11 ++++++++--- tests/op/test_run_lmp.py | 25 ++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index ba7b40d8..9ba71e84 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -259,7 +259,9 @@ def set_models(lmp_input_name: str, model_names: List[str]): with open(lmp_input_name, encoding="utf8") as f: lmp_input_lines = f.readlines() - idx = find_only_one_key(lmp_input_lines, ["pair_style", "deepmd"]) + idx = find_only_one_key(lmp_input_lines, ["pair_style", "deepmd"], raise_not_found=False) + if idx is None: + return new_line_split = lmp_input_lines[idx].split() match_first = -1 match_last = -1 @@ -291,7 +293,7 @@ def set_models(lmp_input_name: str, model_names: List[str]): f.write("".join(lmp_input_lines)) -def find_only_one_key(lmp_lines, key): +def find_only_one_key(lmp_lines, key, raise_not_found=True): found = [] for idx in range(len(lmp_lines)): words = lmp_lines[idx].split() @@ -301,5 +303,8 @@ def find_only_one_key(lmp_lines, key): if len(found) > 1: raise RuntimeError("found %d keywords %s" % (len(found), key)) if len(found) == 0: - raise RuntimeError("failed to find keyword %s" % (key)) + if raise_not_found: + raise RuntimeError("failed to find keyword %s" % (key)) + else: + return None return found[0] diff --git a/tests/op/test_run_lmp.py b/tests/op/test_run_lmp.py index 185b735f..f40ebca2 100644 --- a/tests/op/test_run_lmp.py +++ b/tests/op/test_run_lmp.py @@ -34,7 +34,7 @@ ) from dpgen2.op.run_lmp import ( RunLmp, - randomly_shuffle_models, + set_models, ) from dpgen2.utils import ( BinaryFileInput, @@ -206,7 +206,7 @@ def test_success(self, mocked_run): self.assertEqual((work_dir / lmp_conf_name).read_text(), "foo") lmp_config = TestRunLmpDist.lmp_config.replace( - "model.000.pb", "model.000.pb model.001.pb" + "pair_style deepmd model.000.pb", "pair_style deepmd model.000.pb model.001.pb" ) self.assertEqual((work_dir / lmp_input_name).read_text(), lmp_config) @@ -231,35 +231,34 @@ def swap_element(arg): arg[0] = bk[1] -class TestRandomShuffleModels(unittest.TestCase): +class TestSetModels(unittest.TestCase): def setUp(self): self.input_name = Path("lmp.input") + self.model_names = ["model.000.pth", "model.001.pb"] def tearDown(self): os.remove(self.input_name) - @patch("dpgen2.op.run_lmp.random.shuffle") - def test(self, mock_shuffle): - mock_shuffle.side_effect = swap_element - lmp_config = "pair_style deepmd model.000.pb model.001.pb out_freq 10 out_file model_devi.out" - expected_output = "pair_style deepmd model.001.pb model.000.pb out_freq 10 out_file model_devi.out" + def test(self): + lmp_config = "pair_style deepmd model.000.pb model.001.pb out_freq 10 out_file model_devi.out\n" + expected_output = "pair_style deepmd model.000.pth model.001.pb out_freq 10 out_file model_devi.out\n" input_name = self.input_name input_name.write_text(lmp_config) - randomly_shuffle_models(input_name) + set_models(input_name, self.model_names) self.assertEqual(input_name.read_text(), expected_output) def test_failed(self): - lmp_config = "pair_style deepmd model.000.pb model.001.pb out_freq 10 out_file model_devi.out model.002.pb" + lmp_config = "pair_style deepmd model.000.pb model.001.pb out_freq 10 out_file model_devi.out model.002.pb\n" input_name = self.input_name input_name = Path("lmp.input") input_name.write_text(lmp_config) with self.assertRaises(RuntimeError) as re: - randomly_shuffle_models(input_name) + set_models(input_name, self.model_names) def test_failed_no_matching(self): - lmp_config = "pair_style deepmd out_freq 10 out_file model_devi.out" + lmp_config = "pair_style deepmd out_freq 10 out_file model_devi.out\n" input_name = self.input_name input_name = Path("lmp.input") input_name.write_text(lmp_config) with self.assertRaises(RuntimeError) as re: - randomly_shuffle_models(input_name) + set_models(input_name, self.model_names) From bc6eba3e5bcfcd9485b775da0d6e24be6dde7a64 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:51:54 +0000 Subject: [PATCH 11/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dpgen2/op/run_lmp.py | 4 +++- tests/op/test_run_lmp.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dpgen2/op/run_lmp.py b/dpgen2/op/run_lmp.py index 9ba71e84..997407d8 100644 --- a/dpgen2/op/run_lmp.py +++ b/dpgen2/op/run_lmp.py @@ -259,7 +259,9 @@ def set_models(lmp_input_name: str, model_names: List[str]): with open(lmp_input_name, encoding="utf8") as f: lmp_input_lines = f.readlines() - idx = find_only_one_key(lmp_input_lines, ["pair_style", "deepmd"], raise_not_found=False) + idx = find_only_one_key( + lmp_input_lines, ["pair_style", "deepmd"], raise_not_found=False + ) if idx is None: return new_line_split = lmp_input_lines[idx].split() diff --git a/tests/op/test_run_lmp.py b/tests/op/test_run_lmp.py index f40ebca2..de885ce7 100644 --- a/tests/op/test_run_lmp.py +++ b/tests/op/test_run_lmp.py @@ -206,7 +206,8 @@ def test_success(self, mocked_run): self.assertEqual((work_dir / lmp_conf_name).read_text(), "foo") lmp_config = TestRunLmpDist.lmp_config.replace( - "pair_style deepmd model.000.pb", "pair_style deepmd model.000.pb model.001.pb" + "pair_style deepmd model.000.pb", + "pair_style deepmd model.000.pb model.001.pb", ) self.assertEqual((work_dir / lmp_input_name).read_text(), lmp_config) From 0cbbbccd7612170df4059003b64c9b7382e98a8b Mon Sep 17 00:00:00 2001 From: zjgemi Date: Thu, 28 Mar 2024 15:14:49 +0800 Subject: [PATCH 12/12] fix UT Signed-off-by: zjgemi --- tests/test_prep_run_dp_labeling.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_prep_run_dp_labeling.py b/tests/test_prep_run_dp_labeling.py index 59c34efb..67256f86 100644 --- a/tests/test_prep_run_dp_labeling.py +++ b/tests/test_prep_run_dp_labeling.py @@ -23,7 +23,6 @@ PrepDeepmd, RunDeepmd, deepmd_input_path, - deepmd_teacher_model, deepmd_temp_path, ) from dpgen2.utils import ( @@ -97,8 +96,6 @@ def tearDown(self): shutil.rmtree(deepmd_temp_path, ignore_errors=True) - shutil.rmtree(deepmd_teacher_model, ignore_errors=True) - def test_prep_input(self): run_deepmd = RunDeepmd() out_name = self.task_path / "test_out" @@ -151,8 +148,6 @@ def test_get_dp_model(self): _dp, _type_map = run_deepmd._get_dp_model(self.teacher_model) self.assertTrue(_dp is dp) self.assertEqual(_type_map, ["H", "C"]) - deepmd.infer.DeepPot.assert_called_once_with(Path(deepmd_teacher_model)) - self.assertFalse(Path(deepmd_teacher_model).is_file()) def test_dp_infer(self): self.system.to("deepmd/npy", deepmd_input_path)