From 89ce24b3964a915554f9ff529f1084a91c253653 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Tue, 30 Apr 2024 18:36:27 +0900 Subject: [PATCH 01/24] introduce evaluate method to solvers that merges prepare-run-get_results methods --- extra/leed/src/leed/leed.py | 9 +++++++++ .../src/sim_trhepd_rheed/sim_trhepd_rheed.py | 9 +++++++++ extra/sxrd/src/sxrd/sxrd.py | 9 +++++++++ src/py2dmat/_runner.py | 13 +++++++------ src/py2dmat/solver/function.py | 11 ++++++++++- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/extra/leed/src/leed/leed.py b/extra/leed/src/leed/leed.py index 041355b4..5c596e5d 100644 --- a/extra/leed/src/leed/leed.py +++ b/extra/leed/src/leed/leed.py @@ -106,6 +106,15 @@ def check_keywords(key, segment, registered_list): def name(self) -> str: return self._name + def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(message) + cwd = os.getcwd() + os.chdir(self.work_dir) + self.run(nprocs, nthreads) + os.chdir(cwd) + result = self.get_results() + return result + def prepare(self, message: py2dmat.Message) -> None: self.work_dir = self.proc_dir for dir in [self.path_to_base_dir]: diff --git a/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py b/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py index 098830c8..a22c23df 100644 --- a/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py +++ b/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py @@ -122,6 +122,15 @@ def command(self) -> List[str]: """Command to invoke solver""" return [str(self.path_to_solver)] + def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(message) + cwd = os.getcwd() + os.chdir(self.work_dir) + self.run(nprocs, nthreads) + os.chdir(cwd) + result = self.get_results() + return result + def prepare(self, message: py2dmat.Message) -> None: fitted_x_list, subdir = self.input.prepare(message) self.work_dir = self.proc_dir / Path(subdir) diff --git a/extra/sxrd/src/sxrd/sxrd.py b/extra/sxrd/src/sxrd/sxrd.py index 8ccbad47..0c0bad91 100644 --- a/extra/sxrd/src/sxrd/sxrd.py +++ b/extra/sxrd/src/sxrd/sxrd.py @@ -122,6 +122,15 @@ def check_keywords(key, segment, registered_list): def name(self) -> str: return self._name + def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(message) + cwd = os.getcwd() + os.chdir(self.work_dir) + self.run(nprocs, nthreads) + os.chdir(cwd) + result = self.get_results() + return result + def prepare(self, message: py2dmat.Message) -> None: self.work_dir = self.proc_dir self.input.prepare(message) diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index 50d54a10..9b559d79 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -243,12 +243,13 @@ def submit( if self.limitation.judge(message.x): x = self.mapping(message.x) message_indeed = py2dmat.Message(x, message.step, message.set) - self.solver.prepare(message_indeed) - cwd = os.getcwd() - os.chdir(self.solver.work_dir) - self.solver.run(nprocs, nthreads) - os.chdir(cwd) - result = self.solver.get_results() + # self.solver.prepare(message_indeed) + # cwd = os.getcwd() + # os.chdir(self.solver.work_dir) + # self.solver.run(nprocs, nthreads) + # os.chdir(cwd) + # result = self.solver.get_results() + result = self.solver.evaluate(message_indeed) else: result = np.inf self.logger.count(message, result) diff --git a/src/py2dmat/solver/function.py b/src/py2dmat/solver/function.py index bd71175c..d1f5ede0 100644 --- a/src/py2dmat/solver/function.py +++ b/src/py2dmat/solver/function.py @@ -14,8 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. +import os import numpy as np - import py2dmat # type hints @@ -65,6 +65,15 @@ def __init__(self, info: py2dmat.Info) -> None: def name(self) -> str: return self._name + def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(message) + cwd = os.getcwd() + os.chdir(self.work_dir) + self.run(nprocs, nthreads) + os.chdir(cwd) + result = self.get_results() + return result + def prepare(self, message: py2dmat.Message) -> None: self.x = message.x From e752610bac0aa0ab09936126f49bc28579b4ffa3 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Fri, 26 Apr 2024 19:14:43 +0900 Subject: [PATCH 02/24] added from_file method to Info class --- src/py2dmat/_info.py | 15 +++++++++++++++ src/py2dmat/_main.py | 13 +++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/py2dmat/_info.py b/src/py2dmat/_info.py index dc6d85ab..f2b4d85d 100644 --- a/src/py2dmat/_info.py +++ b/src/py2dmat/_info.py @@ -18,7 +18,10 @@ from typing import MutableMapping, Optional from pathlib import Path +from fnmatch import fnmatch +from .util import toml +from . import mpi from . import exception @@ -61,3 +64,15 @@ def _cleanup(self) -> None: self.algorithm = {} self.solver = {} self.runner = {} + + @classmethod + def from_file(cls, file_name, fmt="", **kwargs): + if fmt == "toml" or fnmatch(file_name.lower(), "*.toml"): + inp = {} + if mpi.rank() == 0: + inp = toml.load(file_name) + if mpi.size() > 1: + inp = mpi.comm().bcast(inp, root=0) + return cls(inp) + else: + raise TypeError("unsupported file format: {}".format(file_name)) diff --git a/src/py2dmat/_main.py b/src/py2dmat/_main.py index 969b9ad8..4b20ea1c 100644 --- a/src/py2dmat/_main.py +++ b/src/py2dmat/_main.py @@ -36,12 +36,13 @@ def main(): args = parser.parse_args() file_name = args.inputfile - inp = {} - if py2dmat.mpi.rank() == 0: - inp = py2dmat.util.toml.load(file_name) - if py2dmat.mpi.size() > 1: - inp = py2dmat.mpi.comm().bcast(inp, root=0) - info = py2dmat.Info(inp) + # inp = {} + # if py2dmat.mpi.rank() == 0: + # inp = py2dmat.util.toml.load(file_name) + # if py2dmat.mpi.size() > 1: + # inp = py2dmat.mpi.comm().bcast(inp, root=0) + # info = py2dmat.Info(inp) + info = py2dmat.Info.from_file(file_name) algname = info.algorithm["name"] if algname == "mapper": From c2532b17ce90212bab3e3d8752ed399517186403 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Fri, 26 Apr 2024 21:52:16 +0900 Subject: [PATCH 03/24] changed exception type --- src/py2dmat/_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py2dmat/_info.py b/src/py2dmat/_info.py index f2b4d85d..37b0caab 100644 --- a/src/py2dmat/_info.py +++ b/src/py2dmat/_info.py @@ -75,4 +75,4 @@ def from_file(cls, file_name, fmt="", **kwargs): inp = mpi.comm().bcast(inp, root=0) return cls(inp) else: - raise TypeError("unsupported file format: {}".format(file_name)) + raise ValueError("unsupported file format: {}".format(file_name)) From 1eec16565f951b2ba8986013f5698b867131a0a8 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 10:55:25 +0900 Subject: [PATCH 04/24] extract logger class to a separate file --- src/py2dmat/_runner.py | 93 +++------------------------------- src/py2dmat/util/logger.py | 101 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 87 deletions(-) create mode 100644 src/py2dmat/util/logger.py diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index 9b559d79..c53b08d4 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -16,9 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. -import os -import subprocess -import time from abc import ABCMeta, abstractmethod import numpy as np @@ -27,6 +24,7 @@ import py2dmat.util.read_matrix import py2dmat.util.mapping import py2dmat.util.limitation +from py2dmat.util.logger import Logger from py2dmat.exception import InputError # type hints @@ -56,94 +54,15 @@ def submit(self, solver): pass -class Logger: - logfile: Path - buffer_index: int - buffer_size: int - buffer: List[str] - num_calls: int - time_start: float - time_previous: float - to_write_result: bool - to_write_x: bool - - def __init__(self, info: Optional[py2dmat.Info] = None) -> None: - if info is None: - self.buffer_size = 0 - return - info_log = info.runner.get("log", {}) - self.buffer_size = info_log.get("interval", 0) - if self.buffer_size <= 0: - return - self.filename = info_log.get("filename", "runner.log") - self.time_start = time.perf_counter() - self.time_previous = self.time_start - self.num_calls = 0 - self.buffer_index = 0 - self.buffer = [""] * self.buffer_size - self.to_write_result = info_log.get("write_result", False) - self.to_write_x = info_log.get("write_input", False) - - def disabled(self) -> bool: - return self.buffer_size <= 0 - - def prepare(self, proc_dir: Path) -> None: - if self.disabled(): - return - self.logfile = proc_dir / self.filename - if self.logfile.exists(): - self.logfile.unlink() - with open(self.logfile, "w") as f: - f.write("# $1: num_calls\n") - f.write("# $2: elapsed_time_from_last_call\n") - f.write("# $3: elapsed_time_from_start\n") - if self.to_write_result: - f.write("# $4: result\n") - i = 4 - else: - i = 5 - if self.to_write_x: - f.write(f"# ${i}-: input\n") - f.write("\n") - - def count(self, message: py2dmat.Message, result: float) -> None: - if self.disabled(): - return - self.num_calls += 1 - t = time.perf_counter() - fields = [self.num_calls, t - self.time_previous, t - self.time_start] - if self.to_write_result: - fields.append(result) - if self.to_write_x: - for x in message.x: - fields.append(x) - fields.append("\n") - self.buffer[self.buffer_index] = " ".join(map(str, fields)) - self.time_previous = t - self.buffer_index += 1 - if self.buffer_index == self.buffer_size: - self.write() - - def write(self) -> None: - if self.disabled(): - return - with open(self.logfile, "a") as f: - for i in range(self.buffer_index): - f.write(self.buffer[i]) - self.buffer_index = 0 - - class Runner(object): #solver: "py2dmat.solver.SolverBase" logger: Logger - def __init__( - self, - solver, #: py2dmat.solver.SolverBase, - info: Optional[py2dmat.Info] = None, - mapping=None, - limitation=None, - ): + def __init__(self, + solver, + info: Optional[py2dmat.Info] = None, + mapping = None, + limitation = None) -> None: """ Parameters diff --git a/src/py2dmat/util/logger.py b/src/py2dmat/util/logger.py new file mode 100644 index 00000000..a21bdeaa --- /dev/null +++ b/src/py2dmat/util/logger.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure +# Copyright (C) 2020- The University of Tokyo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +import time +import py2dmat + +# type hints +from pathlib import Path +from typing import List, Optional + +class Logger: + logfile: Path + buffer_index: int + buffer_size: int + buffer: List[str] + num_calls: int + time_start: float + time_previous: float + to_write_result: bool + to_write_x: bool + + def __init__(self, info: Optional[py2dmat.Info] = None) -> None: + if info is None: + self.buffer_size = 0 + return + info_log = info.runner.get("log", {}) + self.buffer_size = info_log.get("interval", 0) + if self.buffer_size <= 0: + return + self.filename = info_log.get("filename", "runner.log") + self.time_start = time.perf_counter() + self.time_previous = self.time_start + self.num_calls = 0 + self.buffer_index = 0 + self.buffer = [""] * self.buffer_size + self.to_write_result = info_log.get("write_result", False) + self.to_write_x = info_log.get("write_input", False) + + def disabled(self) -> bool: + return self.buffer_size <= 0 + + def prepare(self, proc_dir: Path) -> None: + if self.disabled(): + return + self.logfile = proc_dir / self.filename + if self.logfile.exists(): + self.logfile.unlink() + with open(self.logfile, "w") as f: + f.write("# $1: num_calls\n") + f.write("# $2: elapsed_time_from_last_call\n") + f.write("# $3: elapsed_time_from_start\n") + if self.to_write_result: + f.write("# $4: result\n") + i = 4 + else: + i = 5 + if self.to_write_x: + f.write(f"# ${i}-: input\n") + f.write("\n") + + def count(self, message: py2dmat.Message, result: float) -> None: + if self.disabled(): + return + self.num_calls += 1 + t = time.perf_counter() + fields = [self.num_calls, t - self.time_previous, t - self.time_start] + if self.to_write_result: + fields.append(result) + if self.to_write_x: + for x in message.x: + fields.append(x) + fields.append("\n") + self.buffer[self.buffer_index] = " ".join(map(str, fields)) + self.time_previous = t + self.buffer_index += 1 + if self.buffer_index == self.buffer_size: + self.write() + + def write(self) -> None: + if self.disabled(): + return + with open(self.logfile, "a") as f: + for i in range(self.buffer_index): + f.write(self.buffer[i]) + self.buffer_index = 0 + From 3c9f8cb1a7c7d6350fff94d99311177bddf4bc59 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sun, 28 Apr 2024 00:17:19 +0900 Subject: [PATCH 05/24] fixed comment line in logger output --- src/py2dmat/util/logger.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/py2dmat/util/logger.py b/src/py2dmat/util/logger.py index a21bdeaa..b3dd4497 100644 --- a/src/py2dmat/util/logger.py +++ b/src/py2dmat/util/logger.py @@ -23,6 +23,14 @@ from pathlib import Path from typing import List, Optional +# Parameters +# ---------- +# [runner.log] +# interval +# filename +# write_input +# write_result + class Logger: logfile: Path buffer_index: int @@ -66,9 +74,9 @@ def prepare(self, proc_dir: Path) -> None: f.write("# $3: elapsed_time_from_start\n") if self.to_write_result: f.write("# $4: result\n") - i = 4 - else: i = 5 + else: + i = 4 if self.to_write_x: f.write(f"# ${i}-: input\n") f.write("\n") From 8ef181f27f215e6ddb824cd94b6c7547dfd46513 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 19:57:34 +0900 Subject: [PATCH 06/24] modified mapping in runner class --- src/py2dmat/_runner.py | 34 ++++++++--------------------- src/py2dmat/util/mapping.py | 43 ++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index 9b559d79..7eadebd5 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -154,34 +154,18 @@ def __init__( self.solver_name = solver.name self.logger = Logger(info) - if mapping is None: - info_mapping = info.runner.get("mapping", {}) - A: Optional[np.ndarray] = py2dmat.util.read_matrix.read_matrix( - info_mapping.get("A", []) - ) - b: Optional[np.ndarray] = py2dmat.util.read_matrix.read_matrix( - info_mapping.get("b", []) - ) - if A is not None: - if A.size == 0: - A = None - elif A.ndim != 2: - raise InputError("A should be a matrix") - if b is not None: - if b.size == 0: - b = None - elif b.ndim == 2: - if b.shape[1] == 1: - b = b.reshape(-1) - else: - raise InputError("b should be a vector") - elif b.ndim > 2: - raise InputError("b should be a vector") - self.mapping = py2dmat.util.mapping.Affine(A=A, b=b) - else: + if mapping is not None: self.mapping = mapping + elif "mapping" in info.runner: + info_mapping = info.runner["mapping"] + # N.B.: only Affine mapping is supported at present + self.mapping = py2dmat.util.mapping.Affine.from_dict(info_mapping) + else: + # trivial mapping + self.mapping = lambda xs: xs self.ndim = info.base["dimension"] + if limitation is None: info_limitation = info.runner.get("limitation",{}) co_a: np.ndarray = py2dmat.util.read_matrix.read_matrix( diff --git a/src/py2dmat/util/mapping.py b/src/py2dmat/util/mapping.py index a2f6d9c5..a311fbc0 100644 --- a/src/py2dmat/util/mapping.py +++ b/src/py2dmat/util/mapping.py @@ -19,17 +19,31 @@ import copy import numpy as np +from .read_matrix import read_matrix, read_vector + # type hints from typing import Optional - class Affine: A: Optional[np.ndarray] b: Optional[np.ndarray] def __init__(self, A: Optional[np.ndarray] = None, b: Optional[np.ndarray] = None): - self.A = A - self.b = b + # copy arguments + self.A = np.array(A) if A is not None else None + self.b = np.array(b) if b is not None else None + + # check + if self.A is not None: + if not self.A.ndim == 2: + raise ValueError("A is not a matrix") + if self.b is not None: + if not self.b.ndim == 1: + raise ValueError("b is not a vector") + if self.A is not None and self.b is not None: + if not self.A.shape[0] == self.b.shape[0]: + raise ValueError("shape of A and b mismatch") + def __call__(self, x: np.ndarray) -> np.ndarray: if self.A is None: @@ -40,3 +54,26 @@ def __call__(self, x: np.ndarray) -> np.ndarray: return ret else: return ret + self.b + + @classmethod + def from_dict(cls, d): + A: Optional[np.ndarray] = read_matrix(d.get("A", [])) + b: Optional[np.ndarray] = read_vector(d.get("b", [])) + + if A is not None: + if A.size == 0: + A = None + elif A.ndim != 2: + raise ValueError("A should be a matrix") + if b is not None: + if b.size == 0: + b = None + elif b.ndim == 2: + if b.shape[1] == 1: + b = b.reshape(-1) + else: + raise ValueError("b should be a vector") + elif b.ndim > 2: + raise ValueError("b should be a vector") + + return cls(A, b) From 040f92fdda7cc8a8a9b01981946559c0e3254053 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 22:46:02 +0900 Subject: [PATCH 07/24] modified mapping to conform to input convention --- src/py2dmat/util/mapping.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/py2dmat/util/mapping.py b/src/py2dmat/util/mapping.py index a311fbc0..e28bfe59 100644 --- a/src/py2dmat/util/mapping.py +++ b/src/py2dmat/util/mapping.py @@ -58,22 +58,25 @@ def __call__(self, x: np.ndarray) -> np.ndarray: @classmethod def from_dict(cls, d): A: Optional[np.ndarray] = read_matrix(d.get("A", [])) - b: Optional[np.ndarray] = read_vector(d.get("b", [])) + b: Optional[np.ndarray] = read_matrix(d.get("b", [])) - if A is not None: - if A.size == 0: - A = None - elif A.ndim != 2: + if A is None: + pass + elif A.size == 0: + A = None + else: + if not A.ndim == 2: raise ValueError("A should be a matrix") - if b is not None: - if b.size == 0: - b = None - elif b.ndim == 2: - if b.shape[1] == 1: - b = b.reshape(-1) - else: - raise ValueError("b should be a vector") - elif b.ndim > 2: - raise ValueError("b should be a vector") + + if b is None: + pass + elif b.size == 0: + b = None + else: + if not (b.ndim == 2 and b.shape[1] == 1): + raise ValueError("b should be a column vector") + if not (A is not None and b.shape[0] == A.shape[0]): + raise ValueError("shape of A and b does not match") + b = b.reshape(-1) return cls(A, b) From 196da3143641a5713df423b98fd0d791ff95e4c2 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Fri, 31 May 2024 14:18:51 +0900 Subject: [PATCH 08/24] class tree introduced to mapping class --- src/py2dmat/_runner.py | 2 +- src/py2dmat/util/mapping.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index 7eadebd5..1f8d4784 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -162,7 +162,7 @@ def __init__( self.mapping = py2dmat.util.mapping.Affine.from_dict(info_mapping) else: # trivial mapping - self.mapping = lambda xs: xs + self.mapping = py2dmat.util.mapping.TrivialMapping() self.ndim = info.base["dimension"] diff --git a/src/py2dmat/util/mapping.py b/src/py2dmat/util/mapping.py index e28bfe59..059f404b 100644 --- a/src/py2dmat/util/mapping.py +++ b/src/py2dmat/util/mapping.py @@ -24,7 +24,23 @@ # type hints from typing import Optional -class Affine: +class MappingBase: + def __init__(self): + pass + + def __call__(self, x: np.ndarray) -> np.ndarray: + raise NotImplemented + + +class TrivialMapping(MappingBase): + def __init__(self): + super().__init__() + + def __call__(self, x: np.ndarray) -> np.ndarray: + return x + + +class Affine(MappingBase): A: Optional[np.ndarray] b: Optional[np.ndarray] From 3d76e1ef24f2ab0047ff22c5c5e76bcdbc973bde Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 20:12:33 +0900 Subject: [PATCH 09/24] modified constructor of limitation classes --- src/py2dmat/util/limitation.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index bc352fe6..4b21afeb 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -5,23 +5,22 @@ class LimitationBase(metaclass=ABCMeta): @abstractmethod - def __init__(self, a: np.ndarray, b: np.ndarray, is_limitary: bool): - + def __init__(self, is_limitary: bool): self.islimitary = is_limitary - if self.islimitary: - self.a = a - self.minusb = -b - self.n_formura = a.shape[0] - self.ndim = a.shape[1] @abstractmethod def judge(self, x: np.ndarray) -> bool: - pass + raise NotImplementedError class Inequality(LimitationBase): def __init__(self, a: np.ndarray, b: np.ndarray, is_limitary: bool): - super().__init__(a, b, is_limitary) + super().__init__(is_limitary) + if self.islimitary: + self.a = a + self.minusb = -b + self.n_formura = a.shape[0] + self.ndim = a.shape[1] def judge(self, x: np.ndarray) -> bool: if self.islimitary: From e95e4467d55313185ee299120b947a906c4e0140 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 21:07:07 +0900 Subject: [PATCH 10/24] move limitation construction from runner class --- src/py2dmat/_runner.py | 58 ++++------------------------------ src/py2dmat/util/limitation.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index df7c1500..4bc172f6 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -85,57 +85,13 @@ def __init__(self, self.ndim = info.base["dimension"] - if limitation is None: - info_limitation = info.runner.get("limitation",{}) - co_a: np.ndarray = py2dmat.util.read_matrix.read_matrix( - info_limitation.get("co_a", []) - ) - co_b: np.ndarray = py2dmat.util.read_matrix.read_matrix( - info_limitation.get("co_b", []) - ) - if co_a.size == 0: - is_set_co_a = False - else: - is_set_co_a = True - if co_a.ndim != 2: - raise InputError("co_a should be a matrix") - if co_a.shape[1] != self.ndim: - msg ='The number of columns in co_a should be equal to' - msg+='the value of "dimension" in the [base] section' - raise InputError(msg) - n_row_co_a = co_a.shape[0] - if co_b.size == 0: - if not is_set_co_a : - is_set_co_b = False - else: # is_set_co_a is True - msg = "ERROR: co_a is defined but co_b is not." - raise InputError(msg) - elif co_b.ndim == 2: - if is_set_co_a: - if co_b.shape[0] == 1 or co_b.shape[1] == 1: - is_set_co_b = True - co_b = co_b.reshape(-1) - else: - raise InputError("co_b should be a vector") - if co_b.size != n_row_co_a: - msg ='The number of row in co_a should be equal to' - msg+='the number of size in co_b' - raise InputError(msg) - else: # not is_set_co_a: - msg = "ERROR: co_b is defined but co_a is not." - raise InputError(msg) - elif co_b.ndim > 2: - raise InputError("co_b should be a vector") - - if is_set_co_a and is_set_co_b: - is_limitation = True - elif (not is_set_co_a) and (not is_set_co_b): - is_limitation = False - else: - msg = "ERROR: Both co_a and co_b must be defined." - raise InputError(msg) - - self.limitation = py2dmat.util.limitation.Inequality(co_a, co_b, is_limitation) + if limitation is not None: + self.limitation = limitation + elif "limitation" in info.runner: + info_limitation = info.runner["limitation"] + self.limitation = py2dmat.util.limitation.Inequality.from_dict(info_limitation, self.ndim) + else: + self.limitation = py2dmat.util.limitation.Unlimited() def prepare(self, proc_dir: Path): self.logger.prepare(proc_dir) diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index 4b21afeb..8acfc262 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -2,6 +2,8 @@ import numpy as np +from .read_matrix import read_matrix, read_vector + class LimitationBase(metaclass=ABCMeta): @abstractmethod @@ -12,6 +14,11 @@ def __init__(self, is_limitary: bool): def judge(self, x: np.ndarray) -> bool: raise NotImplementedError +class Unlimited(LimitationBase): + def __init__(self): + super().__init__(False) + def judge(self, x: np.ndarray) -> bool: + return True class Inequality(LimitationBase): def __init__(self, a: np.ndarray, b: np.ndarray, is_limitary: bool): @@ -29,3 +36,52 @@ def judge(self, x: np.ndarray) -> bool: else: judge_result = True return judge_result + + @classmethod + def from_dict(cls, d, dimension): + co_a: np.ndarray = read_matrix(d.get("co_a", [])) + co_b: np.ndarray = read_matrix(d.get("co_b", [])) + + if co_a.size == 0: + is_set_co_a = False + else: + is_set_co_a = True + if co_a.ndim != 2: + raise ValueError("co_a should be a matrix") + if co_a.shape[1] != dimension: + msg ='The number of columns in co_a should be equal to' + msg+='the value of "dimension" in the [base] section' + raise ValueError(msg) + n_row_co_a = co_a.shape[0] + if co_b.size == 0: + if not is_set_co_a : + is_set_co_b = False + else: # is_set_co_a is True + msg = "ERROR: co_a is defined but co_b is not." + raise ValueError(msg) + elif co_b.ndim == 2: + if is_set_co_a: + if co_b.shape[0] == 1 or co_b.shape[1] == 1: + is_set_co_b = True + co_b = co_b.reshape(-1) + else: + raise ValueError("co_b should be a vector") + if co_b.size != n_row_co_a: + msg ='The number of row in co_a should be equal to' + msg+='the number of size in co_b' + raise ValueError(msg) + else: # not is_set_co_a: + msg = "ERROR: co_b is defined but co_a is not." + raise ValueError(msg) + elif co_b.ndim > 2: + raise ValueError("co_b should be a vector") + + if is_set_co_a and is_set_co_b: + is_limitation = True + elif (not is_set_co_a) and (not is_set_co_b): + is_limitation = False + else: + msg = "ERROR: Both co_a and co_b must be defined." + raise ValueError(msg) + + return cls(co_a, co_b, is_limitation) From 5a8ca70ef29d52d2be395aca04499e493dc4f8d3 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 27 Apr 2024 21:25:27 +0900 Subject: [PATCH 11/24] modified limitation construction --- src/py2dmat/util/limitation.py | 45 ++++++++++++---------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index 8acfc262..ca9f1e2f 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -42,39 +42,24 @@ def from_dict(cls, d, dimension): co_a: np.ndarray = read_matrix(d.get("co_a", [])) co_b: np.ndarray = read_matrix(d.get("co_b", [])) + # is_set_co_a = (co_a.size > 0 and co_a.ndim == 2 and co_a.shape[1] == dimension) + # is_set_co_b = (co_b.size > 0 and co_b.ndim == 2 and co_b.shape == (co_a.shape[0], 1)) + if co_a.size == 0: is_set_co_a = False else: - is_set_co_a = True - if co_a.ndim != 2: - raise ValueError("co_a should be a matrix") - if co_a.shape[1] != dimension: - msg ='The number of columns in co_a should be equal to' - msg+='the value of "dimension" in the [base] section' - raise ValueError(msg) - n_row_co_a = co_a.shape[0] + if co_a.ndim == 2 and co_a.shape[1] == dimension: + is_set_co_a = True + else: + raise ValueError("co_a should be a matrix of size equal to number of constraints times dimension") + if co_b.size == 0: - if not is_set_co_a : - is_set_co_b = False - else: # is_set_co_a is True - msg = "ERROR: co_a is defined but co_b is not." - raise ValueError(msg) - elif co_b.ndim == 2: - if is_set_co_a: - if co_b.shape[0] == 1 or co_b.shape[1] == 1: - is_set_co_b = True - co_b = co_b.reshape(-1) - else: - raise ValueError("co_b should be a vector") - if co_b.size != n_row_co_a: - msg ='The number of row in co_a should be equal to' - msg+='the number of size in co_b' - raise ValueError(msg) - else: # not is_set_co_a: - msg = "ERROR: co_b is defined but co_a is not." - raise ValueError(msg) - elif co_b.ndim > 2: - raise ValueError("co_b should be a vector") + is_set_co_b = False + else: + if co_b.ndim == 2 and co_b.shape == (co_a.shape[0], 1): + is_set_co_b = True + else: + raise ValueError("co_b should be a column vector of size equal to number of constraints") if is_set_co_a and is_set_co_b: is_limitation = True @@ -84,4 +69,4 @@ def from_dict(cls, d, dimension): msg = "ERROR: Both co_a and co_b must be defined." raise ValueError(msg) - return cls(co_a, co_b, is_limitation) + return cls(co_a, co_b.reshape(-1), is_limitation) From 0cfd4886a56cd8a902f63c7ccc236e7e93cf086a Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 1 Jun 2024 22:05:42 +0900 Subject: [PATCH 12/24] changed names of variables in limitation --- src/py2dmat/util/limitation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index ca9f1e2f..cc76f532 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -8,7 +8,7 @@ class LimitationBase(metaclass=ABCMeta): @abstractmethod def __init__(self, is_limitary: bool): - self.islimitary = is_limitary + self.is_limitary = is_limitary @abstractmethod def judge(self, x: np.ndarray) -> bool: @@ -23,14 +23,14 @@ def judge(self, x: np.ndarray) -> bool: class Inequality(LimitationBase): def __init__(self, a: np.ndarray, b: np.ndarray, is_limitary: bool): super().__init__(is_limitary) - if self.islimitary: + if self.is_limitary: self.a = a self.minusb = -b self.n_formura = a.shape[0] self.ndim = a.shape[1] def judge(self, x: np.ndarray) -> bool: - if self.islimitary: + if self.is_limitary: Ax = np.einsum("ij,j->i", self.a, x) judge_result = all(Ax > self.minusb) else: @@ -62,11 +62,11 @@ def from_dict(cls, d, dimension): raise ValueError("co_b should be a column vector of size equal to number of constraints") if is_set_co_a and is_set_co_b: - is_limitation = True + is_limitary = True elif (not is_set_co_a) and (not is_set_co_b): - is_limitation = False + is_limitary = False else: msg = "ERROR: Both co_a and co_b must be defined." raise ValueError(msg) - return cls(co_a, co_b.reshape(-1), is_limitation) + return cls(co_a, co_b.reshape(-1), is_limitary) From f3dcc4b4315c6be6fc8ec1ad4e2a11af3eff1c7f Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 1 Jun 2024 23:09:00 +0900 Subject: [PATCH 13/24] modify limitation and runner to remove dimension parameter dependence --- src/py2dmat/_runner.py | 4 +--- src/py2dmat/util/limitation.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index 4bc172f6..f97222f2 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -83,13 +83,11 @@ def __init__(self, # trivial mapping self.mapping = py2dmat.util.mapping.TrivialMapping() - self.ndim = info.base["dimension"] - if limitation is not None: self.limitation = limitation elif "limitation" in info.runner: info_limitation = info.runner["limitation"] - self.limitation = py2dmat.util.limitation.Inequality.from_dict(info_limitation, self.ndim) + self.limitation = py2dmat.util.limitation.Inequality.from_dict(info_limitation) else: self.limitation = py2dmat.util.limitation.Unlimited() diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index cc76f532..9629d1c9 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -38,7 +38,7 @@ def judge(self, x: np.ndarray) -> bool: return judge_result @classmethod - def from_dict(cls, d, dimension): + def from_dict(cls, d): co_a: np.ndarray = read_matrix(d.get("co_a", [])) co_b: np.ndarray = read_matrix(d.get("co_b", [])) @@ -48,7 +48,7 @@ def from_dict(cls, d, dimension): if co_a.size == 0: is_set_co_a = False else: - if co_a.ndim == 2 and co_a.shape[1] == dimension: + if co_a.ndim == 2: is_set_co_a = True else: raise ValueError("co_a should be a matrix of size equal to number of constraints times dimension") From 63218c7bab3c2898ada8940d7bce002f9a2fdbd6 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Mon, 29 Apr 2024 19:28:01 +0900 Subject: [PATCH 14/24] replace message class by parameter values and optional arguments --- src/py2dmat/__init__.py | 1 - src/py2dmat/_message.py | 29 --------- src/py2dmat/_runner.py | 17 ++---- src/py2dmat/algorithm/bayes.py | 5 +- src/py2dmat/algorithm/mapper_mpi.py | 7 +-- src/py2dmat/algorithm/min_search.py | 4 +- src/py2dmat/algorithm/montecarlo.py | 4 +- src/py2dmat/solver/function.py | 8 +-- src/py2dmat/util/logger.py | 93 ++++++++++++++++------------- 9 files changed, 71 insertions(+), 97 deletions(-) delete mode 100644 src/py2dmat/_message.py diff --git a/src/py2dmat/__init__.py b/src/py2dmat/__init__.py index f4b6ca69..cc9e8437 100644 --- a/src/py2dmat/__init__.py +++ b/src/py2dmat/__init__.py @@ -17,7 +17,6 @@ # Pay attention to the dependencies and the order of imports! # For example, Runner depends on solver. -from ._message import Message from ._info import Info from . import solver from ._runner import Runner diff --git a/src/py2dmat/_message.py b/src/py2dmat/_message.py deleted file mode 100644 index 3a73c233..00000000 --- a/src/py2dmat/_message.py +++ /dev/null @@ -1,29 +0,0 @@ -# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure -# Copyright (C) 2020- The University of Tokyo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import numpy as np -from typing import Iterable, Union - - -class Message: - x: np.ndarray - step: int - set: int - - def __init__(self, x: Union[np.ndarray, Iterable], step: int, set: int) -> None: - self.x = x - self.step = step - self.set = set diff --git a/src/py2dmat/_runner.py b/src/py2dmat/_runner.py index f97222f2..497aad98 100644 --- a/src/py2dmat/_runner.py +++ b/src/py2dmat/_runner.py @@ -95,21 +95,14 @@ def prepare(self, proc_dir: Path): self.logger.prepare(proc_dir) def submit( - self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1 + self, x: np.ndarray, args = (), nprocs: int = 1, nthreads: int = 1 ) -> float: - if self.limitation.judge(message.x): - x = self.mapping(message.x) - message_indeed = py2dmat.Message(x, message.step, message.set) - # self.solver.prepare(message_indeed) - # cwd = os.getcwd() - # os.chdir(self.solver.work_dir) - # self.solver.run(nprocs, nthreads) - # os.chdir(cwd) - # result = self.solver.get_results() - result = self.solver.evaluate(message_indeed) + if self.limitation.judge(x): + xp = self.mapping(x) + result = self.solver.evaluate(xp, args) else: result = np.inf - self.logger.count(message, result) + self.logger.count(x, args, result) return result def post(self) -> None: diff --git a/src/py2dmat/algorithm/bayes.py b/src/py2dmat/algorithm/bayes.py index e1e6ddb1..3f23f44a 100644 --- a/src/py2dmat/algorithm/bayes.py +++ b/src/py2dmat/algorithm/bayes.py @@ -86,8 +86,9 @@ def _run(self) -> None: class simulator: def __call__(self, action: np.ndarray) -> float: a = int(action[0]) - message = py2dmat.Message(mesh_list[a, 1:], a, 0) - fx = runner.submit(message) + args = (a, 0) + x = mesh_list[a, 1:] + fx = runner.submit(x, args) fx_list.append(fx) param_list.append(mesh_list[a]) return -fx diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index b2f6b136..d036d1ce 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -48,7 +48,6 @@ def _run(self) -> None: self.timer["run"]["file_CM"] = time_end - time_sta self.timer["run"]["submit"] = 0.0 - message = py2dmat.Message([], 0, 0) iterations = len(self.mesh_list) for iteration_count, mesh in enumerate(self.mesh_list): print("Iteration : {}/{}".format(iteration_count + 1, iterations)) @@ -61,11 +60,11 @@ def _run(self) -> None: self.timer["run"]["file_CM"] += time_end - time_sta # update information - message.step = int(mesh[0]) - message.x = mesh[1:] + args = (int(mesh[0]),) + x = mesh[1:] time_sta = time.perf_counter() - fx = run.submit(message) + fx = run.submit(x, args) time_end = time.perf_counter() self.timer["run"]["submit"] += time_end - time_sta diff --git a/src/py2dmat/algorithm/min_search.py b/src/py2dmat/algorithm/min_search.py index 9e069b70..b9b22a07 100644 --- a/src/py2dmat/algorithm/min_search.py +++ b/src/py2dmat/algorithm/min_search.py @@ -106,8 +106,8 @@ def _f_calc(x_list: np.ndarray, extra_data: bool = False) -> float: if not out_of_range: step[0] += 1 set = 1 if extra_data else 0 - message = py2dmat.Message(x_list, step[0], set) - y = run.submit(message) + args = (step[0], set) + y = run.submit(x_list, args) if not extra_data: callback_list.append([step[0], *x_list, y]) return y diff --git a/src/py2dmat/algorithm/montecarlo.py b/src/py2dmat/algorithm/montecarlo.py index fd3cb3a2..85542aa9 100644 --- a/src/py2dmat/algorithm/montecarlo.py +++ b/src/py2dmat/algorithm/montecarlo.py @@ -247,10 +247,10 @@ def _evaluate(self, in_range: np.ndarray = None) -> np.ndarray: for iwalker in range(self.nwalkers): x = self.x[iwalker, :] if in_range is None or in_range[iwalker]: - message = py2dmat.Message(x, self.istep, iwalker) + args = (self.istep, iwalker) time_sta = time.perf_counter() - self.fx[iwalker] = self.runner.submit(message) + self.fx[iwalker] = self.runner.submit(x, args) time_end = time.perf_counter() self.timer["run"]["submit"] += time_end - time_sta else: diff --git a/src/py2dmat/solver/function.py b/src/py2dmat/solver/function.py index d1f5ede0..680b1ba6 100644 --- a/src/py2dmat/solver/function.py +++ b/src/py2dmat/solver/function.py @@ -65,8 +65,8 @@ def __init__(self, info: py2dmat.Info) -> None: def name(self) -> str: return self._name - def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: - self.prepare(message) + def evaluate(self, x: np.ndarray, args = (), nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(x, args) cwd = os.getcwd() os.chdir(self.work_dir) self.run(nprocs, nthreads) @@ -74,8 +74,8 @@ def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) result = self.get_results() return result - def prepare(self, message: py2dmat.Message) -> None: - self.x = message.x + def prepare(self, x: np.ndarray, args = ()) -> None: + self.x = x def run(self, nprocs: int = 1, nthreads: int = 1) -> None: if self._func is None: diff --git a/src/py2dmat/util/logger.py b/src/py2dmat/util/logger.py index b3dd4497..d7baf11c 100644 --- a/src/py2dmat/util/logger.py +++ b/src/py2dmat/util/logger.py @@ -18,10 +18,11 @@ import time import py2dmat +import numpy as np # type hints from pathlib import Path -from typing import List, Optional +from typing import List, Dict, Any, Optional # Parameters # ---------- @@ -33,77 +34,87 @@ class Logger: logfile: Path - buffer_index: int buffer_size: int buffer: List[str] num_calls: int time_start: float time_previous: float to_write_result: bool - to_write_x: bool + to_write_input: bool + + def __init__(self, info: Optional[py2dmat.Info] = None, + *, + buffer_size: int = 0, + filename: str = "runner.log", + write_input: bool = False, + write_result: bool = False, + params: Optional[Dict[str,Any]] = None, + **rest) -> None: + + if info is not None: + info_log = info.runner.get("log", {}) + else: + info_log = params + + self.buffer_size = info_log.get("interval", buffer_size) + self.filename = info_log.get("filename", filename) + self.to_write_input = info_log.get("write_input", write_input) + self.to_write_result = info_log.get("write_result", write_result) - def __init__(self, info: Optional[py2dmat.Info] = None) -> None: - if info is None: - self.buffer_size = 0 - return - info_log = info.runner.get("log", {}) - self.buffer_size = info_log.get("interval", 0) - if self.buffer_size <= 0: - return - self.filename = info_log.get("filename", "runner.log") self.time_start = time.perf_counter() self.time_previous = self.time_start self.num_calls = 0 - self.buffer_index = 0 - self.buffer = [""] * self.buffer_size - self.to_write_result = info_log.get("write_result", False) - self.to_write_x = info_log.get("write_input", False) + self.buffer = [] - def disabled(self) -> bool: - return self.buffer_size <= 0 + def is_active(self) -> bool: + return self.buffer_size > 0 def prepare(self, proc_dir: Path) -> None: - if self.disabled(): + if not self.is_active(): return + self.logfile = proc_dir / self.filename if self.logfile.exists(): self.logfile.unlink() + with open(self.logfile, "w") as f: f.write("# $1: num_calls\n") - f.write("# $2: elapsed_time_from_last_call\n") - f.write("# $3: elapsed_time_from_start\n") + f.write("# $2: elapsed time from last call\n") + f.write("# $3: elapsed time from start\n") if self.to_write_result: f.write("# $4: result\n") - i = 5 - else: - i = 4 - if self.to_write_x: - f.write(f"# ${i}-: input\n") + if self.to_write_input: + f.write("# ${}-: input\n".format(5 if self.to_write_result else 4)) f.write("\n") - def count(self, message: py2dmat.Message, result: float) -> None: - if self.disabled(): + def count(self, x: np.ndarray, args, result: float) -> None: + if not self.is_active(): return + self.num_calls += 1 + t = time.perf_counter() - fields = [self.num_calls, t - self.time_previous, t - self.time_start] + + fields = [] + fields.append(str(self.num_calls).ljust(6)) + fields.append("{:.6f}".format(t - self.time_previous)) + fields.append("{:.6f}".format(t - self.time_start)) if self.to_write_result: fields.append(result) - if self.to_write_x: - for x in message.x: - fields.append(x) - fields.append("\n") - self.buffer[self.buffer_index] = " ".join(map(str, fields)) + if self.to_write_input: + for val in x: + fields.append(val) + self.buffer.append(" ".join(map(str, fields)) + "\n") + self.time_previous = t - self.buffer_index += 1 - if self.buffer_index == self.buffer_size: + + if len(self.buffer) >= self.buffer_size: self.write() def write(self) -> None: - if self.disabled(): + if not self.is_active(): return with open(self.logfile, "a") as f: - for i in range(self.buffer_index): - f.write(self.buffer[i]) - self.buffer_index = 0 - + for w in self.buffer: + f.write(w) + self.buffer.clear() From 389247f9abb532e3bbb5c510fc12254544b360d4 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Mon, 29 Apr 2024 20:37:14 +0900 Subject: [PATCH 15/24] fix mapper for args --- src/py2dmat/algorithm/mapper_mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index d036d1ce..e9581f80 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -60,7 +60,7 @@ def _run(self) -> None: self.timer["run"]["file_CM"] += time_end - time_sta # update information - args = (int(mesh[0]),) + args = (int(mesh[0]), 0) x = mesh[1:] time_sta = time.perf_counter() From 5dcaea58f66da8a64168b90464a73548ae2b2b98 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Mon, 29 Apr 2024 20:38:18 +0900 Subject: [PATCH 16/24] modified solver modules to replace message to parameters and extra arguments --- extra/leed/src/leed/leed.py | 17 +++++++++-------- .../src/sim_trhepd_rheed/sim_trhepd_rheed.py | 19 +++++++++++-------- extra/sxrd/src/sxrd/sxrd.py | 16 ++++++++-------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/extra/leed/src/leed/leed.py b/extra/leed/src/leed/leed.py index 5c596e5d..5d098038 100644 --- a/extra/leed/src/leed/leed.py +++ b/extra/leed/src/leed/leed.py @@ -106,8 +106,8 @@ def check_keywords(key, segment, registered_list): def name(self) -> str: return self._name - def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: - self.prepare(message) + def evaluate(self, x: np.ndarray, args = (), nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(x, args) cwd = os.getcwd() os.chdir(self.work_dir) self.run(nprocs, nthreads) @@ -115,11 +115,11 @@ def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) result = self.get_results() return result - def prepare(self, message: py2dmat.Message) -> None: + def prepare(self, x: np.ndarray, args) -> None: self.work_dir = self.proc_dir for dir in [self.path_to_base_dir]: copy_tree(os.path.join(self.root_dir, dir), os.path.join(self.work_dir)) - self.input.prepare(message) + self.input.prepare(x, args) def run(self, nprocs: int = 1, nthreads: int = 1) -> None: self._run_by_subprocess([str(self.path_to_solver)]) @@ -158,10 +158,11 @@ def __init__(self, info): self.root_dir = info.base["root_dir"] self.output_dir = info.base["output_dir"] - def prepare(self, message: py2dmat.Message): - x_list = message.x - step = message.step - extra = message.set > 0 + def prepare(self, x: np.ndarray, args): + x_list = x + #step, iset = args + #extra = iset > 0 + # Delete output files delete_files = ["search.s", "gleed.o"] for file in delete_files: diff --git a/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py b/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py index a22c23df..dd09f798 100644 --- a/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py +++ b/extra/sim-trhepd-rheed/src/sim_trhepd_rheed/sim_trhepd_rheed.py @@ -122,8 +122,8 @@ def command(self) -> List[str]: """Command to invoke solver""" return [str(self.path_to_solver)] - def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: - self.prepare(message) + def evaluate(self, x: np.ndarray, args = (), nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(x, args) cwd = os.getcwd() os.chdir(self.work_dir) self.run(nprocs, nthreads) @@ -131,8 +131,9 @@ def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) result = self.get_results() return result - def prepare(self, message: py2dmat.Message) -> None: - fitted_x_list, subdir = self.input.prepare(message) + def prepare(self, x: np.ndarray, args) -> None: + # fitted_x_list, subdir = self.input.prepare(message) + fitted_x_list, subdir = self.input.prepare(x, args) self.work_dir = self.proc_dir / Path(subdir) self.output.prepare(fitted_x_list) @@ -315,13 +316,15 @@ def load_bulk_output_file(self, filename): bulk_f = np.array(bulk_file) return bulk_f - def prepare(self, message: py2dmat.Message): + def prepare(self, x: np.ndarray, args): if self.isLogmode: time_sta = time.perf_counter() - x_list = message.x - step = message.step - iset = message.set + # x_list = message.x + # step = message.step + # iset = message.set + x_list = x + step, iset = args dimension = self.dimension string_list = self.string_list diff --git a/extra/sxrd/src/sxrd/sxrd.py b/extra/sxrd/src/sxrd/sxrd.py index 0c0bad91..3c402e8a 100644 --- a/extra/sxrd/src/sxrd/sxrd.py +++ b/extra/sxrd/src/sxrd/sxrd.py @@ -122,8 +122,8 @@ def check_keywords(key, segment, registered_list): def name(self) -> str: return self._name - def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) -> float: - self.prepare(message) + def evaluate(self, x: np.ndarray, args = (), nprocs: int = 1, nthreads: int = 1) -> float: + self.prepare(x, args) cwd = os.getcwd() os.chdir(self.work_dir) self.run(nprocs, nthreads) @@ -131,9 +131,9 @@ def evaluate(self, message: py2dmat.Message, nprocs: int = 1, nthreads: int = 1) result = self.get_results() return result - def prepare(self, message: py2dmat.Message) -> None: + def prepare(self, x: np.ndarray, args) -> None: self.work_dir = self.proc_dir - self.input.prepare(message) + self.input.prepare(x, args) import shutil for file in ["lsfit.in", self.path_to_f_in, self.path_to_bulk]: @@ -189,10 +189,10 @@ def __init__(self, info): info_s["config"], info_s["reference"], info_s["param"]["domain"] ) - def prepare(self, message: py2dmat.Message): - x_list = message.x - step = message.step - extra = message.set > 0 + def prepare(self, x: np.ndarray, args): + x_list = x + #step, iset = args + #extra = iset > 0 # Generate fit file # Add variables by numpy array.(Variables are updated in optimization process). From ceedb147f2a5385e10824749178b75b6dda88fbf Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Fri, 31 May 2024 15:18:26 +0900 Subject: [PATCH 17/24] move call to runner.prepare --- src/py2dmat/algorithm/_algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py2dmat/algorithm/_algorithm.py b/src/py2dmat/algorithm/_algorithm.py index 054b1c83..fe24efd9 100644 --- a/src/py2dmat/algorithm/_algorithm.py +++ b/src/py2dmat/algorithm/_algorithm.py @@ -296,7 +296,6 @@ def _meshgrid( def set_runner(self, runner: py2dmat.Runner) -> None: self.runner = runner - self.runner.prepare(self.proc_dir) def prepare(self) -> None: if self.runner is None: @@ -315,6 +314,7 @@ def run(self) -> None: raise RuntimeError(msg) original_dir = os.getcwd() os.chdir(self.proc_dir) + self.runner.prepare(self.proc_dir) self._run() self.runner.post() os.chdir(original_dir) From 8db6c476385c47ec807a84493a56074548283746 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 1 Jun 2024 23:07:27 +0900 Subject: [PATCH 18/24] modify algorithm base to find dimension parameter --- src/py2dmat/algorithm/_algorithm.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/py2dmat/algorithm/_algorithm.py b/src/py2dmat/algorithm/_algorithm.py index fe24efd9..b409eb64 100644 --- a/src/py2dmat/algorithm/_algorithm.py +++ b/src/py2dmat/algorithm/_algorithm.py @@ -72,33 +72,22 @@ def __init__( self.timer["init"]["total"] = 0.0 self.status = AlgorithmStatus.INIT - if "dimension" not in info.base: - raise exception.InputError( - "ERROR: base.dimension is not defined in the input" - ) - try: - self.dimension = int(str(info.base["dimension"])) - except ValueError: - raise exception.InputError( - "ERROR: base.dimension should be positive integer" - ) - if self.dimension < 1: - raise exception.InputError( - "ERROR: base.dimension should be positive integer" - ) + self.dimension = info.algorithm.get("dimension") or info.base.get("dimension") + if not self.dimension: + raise ValueError("ERROR: dimension is not defined") if "label_list" in info.algorithm: label = info.algorithm["label_list"] if len(label) != self.dimension: - raise exception.InputError( - f"ERROR: len(label_list) != dimension ({len(label)} != {self.dimension})" - ) + raise ValueError(f"ERROR: length of label_list and dimension do not match ({len(label)} != {self.dimension})") self.label_list = label else: self.label_list = [f"x{d+1}" for d in range(self.dimension)] + # initialize random number generator self.__init_rng(info) + # directories self.root_dir = info.base["root_dir"] self.output_dir = info.base["output_dir"] self.proc_dir = self.output_dir / str(self.mpirank) @@ -109,6 +98,8 @@ def __init__( time.sleep(0.1) if self.mpisize > 1: self.mpicomm.Barrier() + + # runner if runner is not None: self.set_runner(runner) From ed9876ec4070cd99d73e1761c6c97c40d9becf39 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sun, 9 Jun 2024 14:54:19 +0900 Subject: [PATCH 19/24] minor modifications to inequality limitation class --- src/py2dmat/util/limitation.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/py2dmat/util/limitation.py b/src/py2dmat/util/limitation.py index 9629d1c9..8a26c067 100644 --- a/src/py2dmat/util/limitation.py +++ b/src/py2dmat/util/limitation.py @@ -24,15 +24,16 @@ class Inequality(LimitationBase): def __init__(self, a: np.ndarray, b: np.ndarray, is_limitary: bool): super().__init__(is_limitary) if self.is_limitary: - self.a = a - self.minusb = -b - self.n_formura = a.shape[0] + self.a = np.array(a) + self.b = np.array(b) + self.minusb = -np.array(b) + self.n_formula = a.shape[0] self.ndim = a.shape[1] def judge(self, x: np.ndarray) -> bool: if self.is_limitary: - Ax = np.einsum("ij,j->i", self.a, x) - judge_result = all(Ax > self.minusb) + Ax_b = np.dot(self.a, x) + self.b + judge_result = np.all(Ax_b > 0) else: judge_result = True return judge_result From 3a539f37e528b0b6cf9c9dbddbd857cd474eba90 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sun, 9 Jun 2024 14:51:46 +0900 Subject: [PATCH 20/24] domain classes introduced --- src/py2dmat/domain/__init__.py | 18 ++++ src/py2dmat/domain/_domain.py | 35 ++++++++ src/py2dmat/domain/meshgrid.py | 153 +++++++++++++++++++++++++++++++++ src/py2dmat/domain/region.py | 115 +++++++++++++++++++++++++ 4 files changed, 321 insertions(+) create mode 100644 src/py2dmat/domain/__init__.py create mode 100644 src/py2dmat/domain/_domain.py create mode 100644 src/py2dmat/domain/meshgrid.py create mode 100644 src/py2dmat/domain/region.py diff --git a/src/py2dmat/domain/__init__.py b/src/py2dmat/domain/__init__.py new file mode 100644 index 00000000..a582d85f --- /dev/null +++ b/src/py2dmat/domain/__init__.py @@ -0,0 +1,18 @@ +# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure +# Copyright (C) 2020- The University of Tokyo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +from .meshgrid import MeshGrid +from .region import Region diff --git a/src/py2dmat/domain/_domain.py b/src/py2dmat/domain/_domain.py new file mode 100644 index 00000000..8d7319dc --- /dev/null +++ b/src/py2dmat/domain/_domain.py @@ -0,0 +1,35 @@ +# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure +# Copyright (C) 2020- The University of Tokyo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +from typing import List, Dict, Union, Any + +from pathlib import Path +import numpy as np + +import py2dmat + +class DomainBase: + def __init__(self, info: py2dmat.Info = None): + if info: + self.root_dir = info.base["root_dir"] + self.output_dir = info.base["output_dir"] + else: + self.root_dir = Path(".") + self.output_dir = Path(".") + + self.mpisize = py2dmat.mpi.size() + self.mpirank = py2dmat.mpi.rank() + diff --git a/src/py2dmat/domain/meshgrid.py b/src/py2dmat/domain/meshgrid.py new file mode 100644 index 00000000..3b3f78e9 --- /dev/null +++ b/src/py2dmat/domain/meshgrid.py @@ -0,0 +1,153 @@ +# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure +# Copyright (C) 2020- The University of Tokyo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +from typing import List, Dict, Union, Any + +from pathlib import Path +import numpy as np + +import py2dmat +from ._domain import DomainBase + +class MeshGrid(DomainBase): + grid: List[Union[int, float]] = [] + grid_local: List[Union[int, float]] = [] + candicates: int + + def __init__(self, info: py2dmat.Info = None, + *, + param: Dict[str, Any] = None): + super().__init__(info) + + if info: + if "param" in info.algorithm: + self._setup(info.algorithm["param"]) + else: + raise ValueError("ERROR: algorithm.param not defined") + elif param: + self._setup(param) + else: + pass + + + def do_split(self): + if self.mpisize > 1: + index = [idx for idx, *v in self.grid] + index_local = np.array_split(index, self.mpisize)[self.mpirank] + self.grid_local = [[idx, *v] for idx, *v in self.grid if idx in index_local] + else: + self.grid_local = self.grid + + + def _setup(self, info_param): + if "mesh_path" in info_param: + self._setup_from_file(info_param) + else: + self._setup_grid(info_param) + + self.ncandicates = len(self.grid) + + + def _setup_from_file(self, info_param): + if "mesh_path" not in info_param: + raise ValueError("ERROR: mesh_path not defined") + mesh_path = self.root_dir / Path(info_param["mesh_path"]).expanduser() + + if not mesh_path.exists(): + raise FileNotFoundError("mesh_path not found: {}".format(mesh_path)) + + comments = info_param.get("comments", "#") + delimiter = info_param.get("delimiter", None) + skiprows = info_param.get("skiprows", 0) + + if self.mpirank == 0: + data = np.loadtxt(mesh_path, comments=comments, delimiter=delimiter, skiprows=skiprows) + if data.ndim == 1: + data = data.reshape(1, -1) + + # old format: index x1 x2 ... -> omit index + data = data[:,1:] + else: + data = None + + if self.mpisize > 1: + data = py2dmat.mpi.comm().bcast(data, root=0) + + self.grid = [[idx, *v] for idx, v in enumerate(data)] + + + def _setup_grid(self, info_param): + if "min_list" not in info_param: + raise ValueError("ERROR: algorithm.param.min_list is not defined in the input") + min_list = np.array(info_param["min_list"], dtype=float) + + if "max_list" not in info_param: + raise ValueError("ERROR: algorithm.param.max_list is not defined in the input") + max_list = np.array(info_param["max_list"], dtype=float) + + if "num_list" not in info_param: + raise ValueError("ERROR: algorithm.param.num_list is not defined in the input") + num_list = np.array(info_param["num_list"], dtype=int) + + if len(min_list) != len(max_list) or len(min_list) != len(num_list): + raise ValueError("ERROR: lengths of min_list, max_list, num_list do not match") + + xs = [ + np.linspace(mn, mx, num=nm) + for mn, mx, nm in zip(min_list, max_list, num_list) + ] + + self.grid = [ + [idx, *v] for idx, v in enumerate( + np.array( + np.meshgrid(*xs, indexing='xy') + ).reshape(len(xs), -1).transpose() + ) + ] + + + def store_file(self, store_path, *, header=""): + if self.mpirank == 0: + np.savetxt(store_path, [[*v] for idx, *v in self.grid], header=header) + + + @classmethod + def from_file(cls, mesh_path): + return cls(param={"mesh_path": mesh_path}) + + + @classmethod + def from_dict(cls, param): + return cls(param=param) + + + +if __name__ == "__main__": + ms = MeshGrid.from_dict({ + 'min_list': [0,0,0], + 'max_list': [1,1,1], + 'num_list': [5,5,5], + }) + ms.store_file("meshfile.dat", header="sample mesh data") + + ms2 = MeshGrid.from_file("meshfile.dat") + ms2.do_split() + + if py2dmat.mpi.rank() == 0: + print(ms2.grid) + print(py2dmat.mpi.rank(), ms2.grid_local) + + ms2.store_file("meshfile2.dat", header="store again") diff --git a/src/py2dmat/domain/region.py b/src/py2dmat/domain/region.py new file mode 100644 index 00000000..23d2d332 --- /dev/null +++ b/src/py2dmat/domain/region.py @@ -0,0 +1,115 @@ +# 2DMAT -- Data-analysis software of quantum beam diffraction experiments for 2D material structure +# Copyright (C) 2020- The University of Tokyo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +from typing import List, Dict, Union, Any + +from pathlib import Path +import numpy as np + +import py2dmat +from ._domain import DomainBase + +class Region(DomainBase): + min_list: np.array + max_list: np.array + unit_list: np.array + initial_list: np.array + + def __init__(self, info: py2dmat.Info = None, + *, + param: Dict[str, Any] = None, + num_walkers: int = 1): + super().__init__(info) + + if info: + if "param" in info.algorithm: + self._setup(info.algorithm["param"], num_walkers) + else: + raise ValueError("ERROR: algorithm.param not defined") + elif param: + self._setup(param, num_walkers) + else: + pass + + + def _setup(self, info_param, num_walkers: int = 1): + self.num_walkers = num_walkers + + if "min_list" not in info_param: + raise ValueError("ERROR: algorithm.param.min_list is not defined in the input") + min_list = np.array(info_param["min_list"]) + + if "max_list" not in info_param: + raise ValueError("ERROR: algorithm.param.max_list is not defined in the input") + max_list = np.array(info_param["max_list"]) + + if len(min_list) != len(max_list): + raise ValueError("ERROR: lengths of min_list and max_list do not match") + + self.dimension = len(min_list) + + unit_list = np.array(info_param.get("unit_list", [1.0] * self.dimension)) + + self.min_list = min_list + self.max_list = max_list + self.unit_list = unit_list + + initial_list = np.array(info_param.get("initial_list", [])) + if initial_list.ndim == 1: + initial_list = initial_list.reshape(1, -1) + + if initial_list.size > 0: + if initial_list.shape != (num_walkers, self.dimension): + raise ValueError("ERROR: shape of initial_list do not match number of walkers times dimension") + + self.initial_list = initial_list + + def initialize(self, + rng=np.random, + limitation=py2dmat.util.limitation.Unlimited()): + if self.initial_list.size > 0: + pass + else: + self._init_random(rng=rng, limitation=limitation) + + def _init_random(self, + rng=np.random, + limitation=py2dmat.util.limitation.Unlimited(), + max_count=100): + initial_list = np.zeros((self.num_walkers, self.dimension), dtype=float) + is_ok = np.full(self.num_walkers, False) + count = 0 + while (not np.all(is_ok)): + count += 1 + initial_list[~is_ok] = self.min_list + (self.max_list - self.min_list) * rng.rand(np.count_nonzero(~is_ok), self.dimension) + is_ok = np.array([limitation.judge(v) for v in initial_list]) + if count >= max_count: + raise RuntimeError("ERROR: init_random: trial count exceeds {}".format(max_count)) + self.initial_list = initial_list + + +if __name__ == "__main__": + reg = Region(param={ + "min_list": [0.0, 0.0, 0.0], + "max_list": [1.0, 1.0, 1.0], + }, num_walkers=4) + + #lim = py2dmat.util.limitation.Unlimited() + lim = py2dmat.util.limitation.Inequality(a=np.array([[1,0,0],[-1,-1,-1]]),b=np.array([0,1]),is_limitary=True) + + reg.init_random(np.random, lim) + + print(reg.min_list, reg.max_list, reg.unit_list, reg.initial_list) From 60cbd385729fee86f45f5dbf39f602225b619eb0 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sun, 9 Jun 2024 14:55:03 +0900 Subject: [PATCH 21/24] algorithms classes modified to use domain classes --- src/py2dmat/algorithm/_algorithm.py | 173 ---------------------------- src/py2dmat/algorithm/bayes.py | 13 ++- src/py2dmat/algorithm/mapper_mpi.py | 27 +++-- src/py2dmat/algorithm/min_search.py | 23 ++-- src/py2dmat/algorithm/montecarlo.py | 90 ++++++++++----- 5 files changed, 104 insertions(+), 222 deletions(-) diff --git a/src/py2dmat/algorithm/_algorithm.py b/src/py2dmat/algorithm/_algorithm.py index b409eb64..e8059cec 100644 --- a/src/py2dmat/algorithm/_algorithm.py +++ b/src/py2dmat/algorithm/_algorithm.py @@ -112,179 +112,6 @@ def __init_rng(self, info: py2dmat.Info) -> None: else: self.rng = np.random.RandomState(seed + self.mpirank * seed_delta) - def _read_param( - self, info: py2dmat.Info, num_walkers: int = 1 - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - """Generate continuous data from info - - Returns - ======= - initial_list: np.ndarray - num_walkers \\times dimension array - min_list - max_list - unit_list - """ - if "param" not in info.algorithm: - raise exception.InputError( - "ERROR: [algorithm.param] is not defined in the input" - ) - info_param = info.algorithm["param"] - - if "min_list" not in info_param: - raise exception.InputError( - "ERROR: algorithm.param.min_list is not defined in the input" - ) - min_list = np.array(info_param["min_list"]) - if len(min_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(min_list) != dimension ({len(min_list)} != {self.dimension})" - ) - - if "max_list" not in info_param: - raise exception.InputError( - "ERROR: algorithm.param.max_list is not defined in the input" - ) - max_list = np.array(info_param["max_list"]) - if len(max_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(max_list) != dimension ({len(max_list)} != {self.dimension})" - ) - - unit_list = np.array(info_param.get("unit_list", [1.0] * self.dimension)) - if len(unit_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(unit_list) != dimension ({len(unit_list)} != {self.dimension})" - ) - - initial_list = np.array(info_param.get("initial_list", [])) - if initial_list.ndim == 1: - initial_list = initial_list.reshape(1, -1) - if initial_list.size == 0: - initial_list = min_list + (max_list - min_list) * self.rng.rand( - num_walkers, self.dimension - ) - # Repeat until an "initial_list" is generated - # that satisfies the constraint expression. - # If "co_a" and "co_b" are not set in [runner.limitation], - # all(isOK_judge) = true and do not repeat. - loop_count = 0 - isOK_judge = np.full(num_walkers, False) - while True: - for index in np.where(~isOK_judge)[0]: - isOK_judge[index] = self.runner.limitation.judge( - initial_list[index,:] - ) - if np.all(isOK_judge): - break - else: - initial_list[~isOK_judge] = ( - min_list + (max_list - min_list) * self.rng.rand( - np.count_nonzero(~isOK_judge), self.dimension - ) ) - loop_count += 1 - if initial_list.shape[0] != num_walkers: - raise exception.InputError( - f"ERROR: initial_list.shape[0] != num_walkers ({initial_list.shape[0]} != {num_walkers})" - ) - if initial_list.shape[1] != self.dimension: - raise exception.InputError( - f"ERROR: initial_list.shape[1] != dimension ({initial_list.shape[1]} != {self.dimension})" ) - judge_result = [] - for walker_index in range(num_walkers): - judge = self.runner.limitation.judge( - initial_list[walker_index,:]) - judge_result.append(judge) - if not all(judge_result): - raise exception.InputError( - "ERROR: initial_list does not satisfy the constraint formula." - ) - return initial_list, min_list, max_list, unit_list - - def _meshgrid( - self, info: py2dmat.Info, split: bool = False - ) -> Tuple[np.ndarray, np.ndarray]: - """Generate discrete data from info - - Arguments - ========== - info: - split: - if True, splits data into mpisize parts and returns mpirank-th one - (default: False) - - Returns - ======= - grid: - Ncandidate x dimension - id_list: - """ - - if "param" not in info.algorithm: - raise exception.InputError( - "ERROR: [algorithm.param] is not defined in the input" - ) - info_param = info.algorithm["param"] - - if "mesh_path" in info_param: - mesh_path = ( - self.root_dir / pathlib.Path(info_param["mesh_path"]).expanduser() - ) - comments = info_param.get("comments", "#") - delimiter = info_param.get("delimiter", None) - skiprows = info_param.get("skiprows", 0) - - data = np.loadtxt( - mesh_path, comments=comments, delimiter=delimiter, skiprows=skiprows, - ) - if data.ndim == 1: - data = data.reshape(1, -1) - grid = data - else: - if "min_list" not in info_param: - raise exception.InputError( - "ERROR: algorithm.param.min_list is not defined in the input" - ) - min_list = np.array(info_param["min_list"], dtype=float) - if len(min_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(min_list) != dimension ({len(min_list)} != {self.dimension})" - ) - - if "max_list" not in info_param: - raise exception.InputError( - "ERROR: algorithm.param.max_list is not defined in the input" - ) - max_list = np.array(info_param["max_list"], dtype=float) - if len(max_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(max_list) != dimension ({len(max_list)} != {self.dimension})" - ) - - if "num_list" not in info_param: - raise exception.InputError( - "ERROR: algorithm.param.num_list is not defined in the input" - ) - num_list = np.array(info_param["num_list"], dtype=int) - if len(num_list) != self.dimension: - raise exception.InputError( - f"ERROR: len(num_list) != dimension ({len(num_list)} != {self.dimension})" - ) - - xs = [ - np.linspace(mn, mx, num=nm) - for mn, mx, nm in zip(min_list, max_list, num_list) - ] - data = np.array([g.flatten() for g in np.meshgrid(*xs)]).transpose() - grid = np.array([np.hstack([i, d]) for i, d in enumerate(data)]) - ncandidates = grid.shape[0] - ns_total = np.arange(ncandidates) - if split: - id_list = np.array_split(ns_total, self.mpisize)[self.mpirank] - return grid[id_list, :], id_list - else: - return grid, ns_total - def set_runner(self, runner: py2dmat.Runner) -> None: self.runner = runner diff --git a/src/py2dmat/algorithm/bayes.py b/src/py2dmat/algorithm/bayes.py index 3f23f44a..8e72a968 100644 --- a/src/py2dmat/algorithm/bayes.py +++ b/src/py2dmat/algorithm/bayes.py @@ -21,6 +21,7 @@ import numpy as np import py2dmat +import py2dmat.domain class Algorithm(py2dmat.algorithm.AlgorithmBase): @@ -43,7 +44,9 @@ class Algorithm(py2dmat.algorithm.AlgorithmBase): fx_list: List[float] param_list: List[np.ndarray] - def __init__(self, info: py2dmat.Info, runner: py2dmat.Runner = None) -> None: + def __init__(self, info: py2dmat.Info, + runner: py2dmat.Runner = None, + domain = None) -> None: super().__init__(info=info, runner=runner) info_param = info.algorithm.get("param", {}) @@ -67,7 +70,13 @@ def __init__(self, info: py2dmat.Info, runner: py2dmat.Runner = None) -> None: print(f"interval = {self.interval}") print(f"num_rand_basis = {self.num_rand_basis}") - self.mesh_list, actions = self._meshgrid(info, split=False) + #self.mesh_list, actions = self._meshgrid(info, split=False) + if domain and isinstance(domain, py2dmat.domain.MeshGrid): + self.domain = domain + else: + self.domain = py2dmat.domain.MeshGrid(info) + self.mesh_list = np.array(self.domain.grid) + X_normalized = physbo.misc.centering(self.mesh_list[:, 1:]) comm = self.mpicomm if self.mpisize > 1 else None self.policy = physbo.search.discrete.policy(test_X=X_normalized, comm=comm) diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index e9581f80..c26fd306 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. +from typing import List, Union + from pathlib import Path from io import open import numpy as np @@ -21,14 +23,25 @@ import time import py2dmat - +import py2dmat.domain class Algorithm(py2dmat.algorithm.AlgorithmBase): - mesh_list: np.ndarray + #mesh_list: np.ndarray + mesh_list: List[Union[int, float]] - def __init__(self, info: py2dmat.Info, runner: py2dmat.Runner = None) -> None: + def __init__(self, info: py2dmat.Info, + runner: py2dmat.Runner = None, + domain = None) -> None: super().__init__(info=info, runner=runner) - self.mesh_list, actions = self._meshgrid(info, split=True) + + if domain and isinstance(domain, py2dmat.domain.MeshGrid): + self.domain = domain + else: + self.domain = py2dmat.domain.MeshGrid(info) + + self.domain.do_split() + self.mesh_list = self.domain.grid_local + def _run(self) -> None: # Make ColorMap @@ -51,7 +64,7 @@ def _run(self) -> None: iterations = len(self.mesh_list) for iteration_count, mesh in enumerate(self.mesh_list): print("Iteration : {}/{}".format(iteration_count + 1, iterations)) - print("mesh before:", mesh) + # print("mesh before:", mesh) time_sta = time.perf_counter() for value in mesh[1:]: @@ -61,7 +74,7 @@ def _run(self) -> None: # update information args = (int(mesh[0]), 0) - x = mesh[1:] + x = np.array(mesh[1:]) time_sta = time.perf_counter() fx = run.submit(x, args) @@ -74,7 +87,7 @@ def _run(self) -> None: time_end = time.perf_counter() self.timer["run"]["file_CM"] += time_end - time_sta - print("mesh after:", mesh) + # print("mesh after:", mesh) if iterations > 0: fx_order = np.argsort(fx_list) diff --git a/src/py2dmat/algorithm/min_search.py b/src/py2dmat/algorithm/min_search.py index b9b22a07..4090b94f 100644 --- a/src/py2dmat/algorithm/min_search.py +++ b/src/py2dmat/algorithm/min_search.py @@ -21,6 +21,7 @@ from scipy.optimize import minimize import py2dmat +import py2dmat.domain class Algorithm(py2dmat.algorithm.AlgorithmBase): @@ -46,16 +47,22 @@ class Algorithm(py2dmat.algorithm.AlgorithmBase): fx_for_simplex_list: List[float] callback_list: List[List[int]] - def __init__(self, info: py2dmat.Info, runner: py2dmat.Runner = None) -> None: + def __init__(self, info: py2dmat.Info, + runner: py2dmat.Runner = None, + domain = None) -> None: super().__init__(info=info, runner=runner) - ( - self.initial_list, - self.min_list, - self.max_list, - self.unit_list, - ) = self._read_param(info) - self.initial_list = self.initial_list.flatten() + if domain and isinstance(domain, py2dmat.domain.Region): + self.domain = domain + else: + self.domain = py2dmat.domain.Region(info, num_walkers=self.mpisize) + + self.min_list = self.domain.min_list + self.max_list = self.domain.max_list + self.unit_list = self.domain.unit_list + + self.domain.initialize(rng=self.rng, limitation=runner.limitation) + self.initial_list = self.domain.initial_list[self.mpirank] info_minimize = info.algorithm.get("minimize", {}) self.initial_scale_list = info_minimize.get( diff --git a/src/py2dmat/algorithm/montecarlo.py b/src/py2dmat/algorithm/montecarlo.py index 85542aa9..769d9f12 100644 --- a/src/py2dmat/algorithm/montecarlo.py +++ b/src/py2dmat/algorithm/montecarlo.py @@ -18,13 +18,14 @@ from typing import TextIO, Union, List, Tuple import copy import time -import pathlib +from pathlib import Path import numpy as np import py2dmat from py2dmat.util.neighborlist import load_neighbor_list import py2dmat.util.graph +import py2dmat.domain class AlgorithmBase(py2dmat.algorithm.AlgorithmBase): @@ -91,46 +92,48 @@ class AlgorithmBase(py2dmat.algorithm.AlgorithmBase): ntrial: int naccepted: int - def __init__( - self, info: py2dmat.Info, runner: py2dmat.Runner = None, nwalkers: int = 1 + def __init__(self, info: py2dmat.Info, + runner: py2dmat.Runner = None, + domain = None, + nwalkers: int = 1 ) -> None: time_sta = time.perf_counter() super().__init__(info=info, runner=runner) self.nwalkers = nwalkers - info_param = info.algorithm["param"] - if "mesh_path" in info_param: - self.iscontinuous = False - self.node_coordinates = self._meshgrid(info)[0][:, 1:] + + if domain: + if isinstance(domain, py2dmat.domain.MeshGrid): + self.iscontinuous = False + elif isinstance(domain, py2dmat.domain.Region): + self.iscontinuous = True + else: + raise ValueError("ERROR: unsupoorted domain type {}".format(type(domain))) + self.domain = domain + else: + info_param = info.algorithm["param"] + if "mesh_path" in info_param: + self.iscontinuous = False + self.domain = py2dmat.domain.MeshGrid(info) + + else: + self.iscontinuous = True + self.domain = py2dmat.domain.Region(info, num_walkers=nwalkers) + + if self.iscontinuous: + self.domain.initialize(rng=self.rng, limitation=self.runner.limitation) + self.x = self.domain.initial_list + self.xmin = self.domain.min_list + self.xmax = self.domain.max_list + self.xunit = self.domain.unit_list + + else: + self.node_coordinates = np.array(self.domain.grid)[:, 1:] self.nnodes = self.node_coordinates.shape[0] self.inode = self.rng.randint(self.nnodes, size=self.nwalkers) self.x = self.node_coordinates[self.inode, :] - if "neighborlist_path" not in info_param: - msg = ( - "ERROR: Parameter algorithm.param.neighborlist_path does not exist." - ) - raise RuntimeError(msg) - nn_path = ( - self.root_dir - / pathlib.Path(info_param["neighborlist_path"]).expanduser() - ) - self.neighbor_list = load_neighbor_list(nn_path, nnodes=self.nnodes) - if not py2dmat.util.graph.is_connected(self.neighbor_list): - msg = "ERROR: The transition graph made from neighbor list is not connected." - msg += "\nHINT: Increase neighborhood radius." - raise RuntimeError(msg) - if not py2dmat.util.graph.is_bidirectional(self.neighbor_list): - msg = "ERROR: The transition graph made from neighbor list is not bidirectional." - raise RuntimeError(msg) - self.ncandidates = np.array( - [len(ns) - 1 for ns in self.neighbor_list], dtype=np.int64 - ) + self._setup_neighbour(info_param) - else: - self.iscontinuous = True - self.x, self.xmin, self.xmax, self.xunit = self._read_param( - info, num_walkers=nwalkers - ) self.fx = np.zeros(self.nwalkers) self.best_fx = 0.0 self.best_istep = 0 @@ -142,6 +145,29 @@ def __init__( self.naccepted = 0 self.ntrial = 0 + def _setup_neighbour(self, info_param): + if "neighborlist_path" in info_param: + nn_path = self.root_dir / Path(info_param["neighborlist_path"]).expanduser() + self.neighbor_list = load_neighbor_list(nn_path, nnodes=self.nnodes) + + # checks + if not py2dmat.util.graph.is_connected(self.neighbor_list): + raise RuntimeError( + "ERROR: The transition graph made from neighbor list is not connected." + "\nHINT: Increase neighborhood radius." + ) + if not py2dmat.util.graph.is_bidirectional(self.neighbor_list): + raise RuntimeError( + "ERROR: The transition graph made from neighbor list is not bidirectional." + ) + + self.ncandidates = np.array([len(ns) - 1 for ns in self.neighbor_list], dtype=np.int64) + else: + raise ValueError( + "ERROR: Parameter algorithm.param.neighborlist_path does not exist." + ) + # otherwise find neighbourlist + def read_Ts(self, info: dict, numT: int = None) -> np.ndarray: """ From b5f1d8d172e5525d7b54b4c92df046bb61b9362c Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sun, 9 Jun 2024 15:24:23 +0900 Subject: [PATCH 22/24] fix minsearch --- src/py2dmat/algorithm/min_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py2dmat/algorithm/min_search.py b/src/py2dmat/algorithm/min_search.py index 4090b94f..649e7b04 100644 --- a/src/py2dmat/algorithm/min_search.py +++ b/src/py2dmat/algorithm/min_search.py @@ -55,14 +55,14 @@ def __init__(self, info: py2dmat.Info, if domain and isinstance(domain, py2dmat.domain.Region): self.domain = domain else: - self.domain = py2dmat.domain.Region(info, num_walkers=self.mpisize) + self.domain = py2dmat.domain.Region(info) self.min_list = self.domain.min_list self.max_list = self.domain.max_list self.unit_list = self.domain.unit_list self.domain.initialize(rng=self.rng, limitation=runner.limitation) - self.initial_list = self.domain.initial_list[self.mpirank] + self.initial_list = self.domain.initial_list[0] info_minimize = info.algorithm.get("minimize", {}) self.initial_scale_list = info_minimize.get( From 616456f6a08f351115babab04550f87613370165 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Tue, 11 Jun 2024 17:15:35 +0900 Subject: [PATCH 23/24] modify region in generating random initial list --- src/py2dmat/domain/region.py | 44 ++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/py2dmat/domain/region.py b/src/py2dmat/domain/region.py index 23d2d332..b10c1f6d 100644 --- a/src/py2dmat/domain/region.py +++ b/src/py2dmat/domain/region.py @@ -30,24 +30,21 @@ class Region(DomainBase): def __init__(self, info: py2dmat.Info = None, *, - param: Dict[str, Any] = None, - num_walkers: int = 1): + param: Dict[str, Any] = None): super().__init__(info) if info: if "param" in info.algorithm: - self._setup(info.algorithm["param"], num_walkers) + self._setup(info.algorithm["param"]) else: raise ValueError("ERROR: algorithm.param not defined") elif param: - self._setup(param, num_walkers) + self._setup(param) else: pass - def _setup(self, info_param, num_walkers: int = 1): - self.num_walkers = num_walkers - + def _setup(self, info_param): if "min_list" not in info_param: raise ValueError("ERROR: algorithm.param.min_list is not defined in the input") min_list = np.array(info_param["min_list"]) @@ -72,15 +69,22 @@ def _setup(self, info_param, num_walkers: int = 1): initial_list = initial_list.reshape(1, -1) if initial_list.size > 0: - if initial_list.shape != (num_walkers, self.dimension): - raise ValueError("ERROR: shape of initial_list do not match number of walkers times dimension") + if initial_list.shape[1] != self.dimension: + raise ValueError("ERROR: dimension of initial_list is incorrect") + self.num_walkers = initial_list.shape[0] + else: + self.num_walkers = 0 self.initial_list = initial_list def initialize(self, rng=np.random, - limitation=py2dmat.util.limitation.Unlimited()): - if self.initial_list.size > 0: + limitation=py2dmat.util.limitation.Unlimited(), + num_walkers: int = 1): + if num_walkers > self.num_walkers: + self.num_walkers = num_walkers + + if self.initial_list.size > 0 and self.initial_list.shape[0] >= num_walkers: pass else: self._init_random(rng=rng, limitation=limitation) @@ -91,6 +95,12 @@ def _init_random(self, max_count=100): initial_list = np.zeros((self.num_walkers, self.dimension), dtype=float) is_ok = np.full(self.num_walkers, False) + + if self.initial_list.size > 0: + nitem = min(self.num_walkers, self.initial_list.shape[0]) + initial_list[0:nitem] = self.initial_list[0:nitem] + is_ok[0:nitem] = True + count = 0 while (not np.all(is_ok)): count += 1 @@ -105,11 +115,17 @@ def _init_random(self, reg = Region(param={ "min_list": [0.0, 0.0, 0.0], "max_list": [1.0, 1.0, 1.0], - }, num_walkers=4) + "initial_list": [[0.1, 0.2, 0.3], + [0.2, 0.3, 0.1], + [0.3, 0.1, 0.2], + [0.2, 0.1, 0.3], + [0.1, 0.3, 0.2], + ], + }) #lim = py2dmat.util.limitation.Unlimited() lim = py2dmat.util.limitation.Inequality(a=np.array([[1,0,0],[-1,-1,-1]]),b=np.array([0,1]),is_limitary=True) - reg.init_random(np.random, lim) + reg.initialize(np.random, lim, 8) - print(reg.min_list, reg.max_list, reg.unit_list, reg.initial_list) + print(reg.min_list, reg.max_list, reg.unit_list, reg.initial_list, reg.num_walkers) From 3e10d8fb78d60f9205247cef2dbc8411408c9710 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Tue, 11 Jun 2024 17:16:02 +0900 Subject: [PATCH 24/24] modify according to changes to region class --- src/py2dmat/algorithm/montecarlo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py2dmat/algorithm/montecarlo.py b/src/py2dmat/algorithm/montecarlo.py index 769d9f12..f382f7c4 100644 --- a/src/py2dmat/algorithm/montecarlo.py +++ b/src/py2dmat/algorithm/montecarlo.py @@ -117,10 +117,10 @@ def __init__(self, info: py2dmat.Info, else: self.iscontinuous = True - self.domain = py2dmat.domain.Region(info, num_walkers=nwalkers) + self.domain = py2dmat.domain.Region(info) if self.iscontinuous: - self.domain.initialize(rng=self.rng, limitation=self.runner.limitation) + self.domain.initialize(rng=self.rng, limitation=self.runner.limitation, num_walkers=nwalkers) self.x = self.domain.initial_list self.xmin = self.domain.min_list self.xmax = self.domain.max_list