From 6e7a101916855e93f263734225af06153bb2b030 Mon Sep 17 00:00:00 2001 From: Carlos Garcia Jurado Suarez Date: Thu, 11 Jan 2024 10:32:10 -0800 Subject: [PATCH] Do deterministic inference --- azureml/eval.yml | 4 +- data/autora/code1.txt | 69 ++++++++++++++++++++++++++++ data/autora/data.jsonl | 1 + data/autora/text1.txt | 7 +++ data/{ => sweetpea}/data.jsonl | 0 src/autora/doc/pipelines/main.py | 12 +++++ src/autora/doc/runtime/predict_hf.py | 9 ++-- tests/test_main.py | 2 +- 8 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 data/autora/code1.txt create mode 100644 data/autora/data.jsonl create mode 100644 data/autora/text1.txt rename data/{ => sweetpea}/data.jsonl (100%) diff --git a/azureml/eval.yml b/azureml/eval.yml index 1f31ea4..f252020 100644 --- a/azureml/eval.yml +++ b/azureml/eval.yml @@ -18,9 +18,9 @@ inputs: # type: uri_folder # path: azureml://datastores/workspaceblobstore/paths/base_models model_path: meta-llama/Llama-2-7b-chat-hf - temperature: 0.7 + temperature: 0.01 top_p: 0.95 - top_k: 40 + top_k: 1 sys_id: SYS_1 instruc_id: INSTR_SWEETP_1 # using a curated environment doesn't work because we need additional packages diff --git a/data/autora/code1.txt b/data/autora/code1.txt new file mode 100644 index 0000000..735bb20 --- /dev/null +++ b/data/autora/code1.txt @@ -0,0 +1,69 @@ +import numpy as np +import pandas as pd +import sympy as sp +from autora.experiment_runner.synthetic.abstract.equation import equation_experiment +from autora.experimentalist.random import random_pool +from autora.state import StandardState, estimator_on_state, on_state +from autora.theorist.bms import BMSRegressor +from autora.variable import ValueType, Variable, VariableCollection + +#################################################################################### +## Define initial data +#################################################################################### + +#### Define variable data #### +iv = Variable(name="x", value_range=(0, 2 * np.pi), allowed_values=np.linspace(0, 2 * np.pi, 30)) +dv = Variable(name="y", type=ValueType.REAL) +variables = VariableCollection(independent_variables=[iv], dependent_variables=[dv]) + +#### Define seed condition data #### +conditions = random_pool(variables, num_samples=10, random_state=0) + +#################################################################################### +## Define experimentalist +#################################################################################### + +experimentalist = on_state(random_pool, output=["conditions"]) + +#################################################################################### +## Define experiment runner +#################################################################################### + +sin_experiment = equation_experiment( + sp.simplify("sin(x)"), variables.independent_variables, variables.dependent_variables[0] +) +sin_runner = sin_experiment.experiment_runner + +experiment_runner = on_state(sin_runner, output=["experiment_data"]) + +#################################################################################### +## Define theorist +#################################################################################### + +theorist = estimator_on_state(BMSRegressor(epochs=100)) + +#################################################################################### +## Define state +#################################################################################### + +s = StandardState( + variables=variables, conditions=conditions, experiment_data=pd.DataFrame(columns=["x", "y"]) +) + +#################################################################################### +## Cycle through the state +#################################################################################### + +print("Pre-Defined State:") +print(f"Number of datapoints collected: {len(s['experiment_data'])}") +print(f"Derived models: {s['models']}") +print("\n") + +for i in range(5): + s = experimentalist(s, num_samples=10, random_state=42) + s = experiment_runner(s, added_noise=1.0, random_state=42) + s = theorist(s) + print(f"\nCycle {i+1} Results:") + print(f"Number of datapoints collected: {len(s['experiment_data'])}") + print(f"Derived models: {s['models']}") + print("\n") diff --git a/data/autora/data.jsonl b/data/autora/data.jsonl new file mode 100644 index 0000000..b2a5702 --- /dev/null +++ b/data/autora/data.jsonl @@ -0,0 +1 @@ +{"instruction": "import numpy as np\nimport pandas as pd\nimport sympy as sp\nfrom autora.experiment_runner.synthetic.abstract.equation import equation_experiment\nfrom autora.experimentalist.random import random_pool\nfrom autora.state import StandardState, estimator_on_state, on_state\nfrom autora.theorist.bms import BMSRegressor\nfrom autora.variable import ValueType, Variable, VariableCollection\n\n####################################################################################\n## Define initial data\n####################################################################################\n\n#### Define variable data ####\niv = Variable(name=\"x\", value_range=(0, 2 * np.pi), allowed_values=np.linspace(0, 2 * np.pi, 30))\ndv = Variable(name=\"y\", type=ValueType.REAL)\nvariables = VariableCollection(independent_variables=[iv], dependent_variables=[dv])\n\n#### Define seed condition data ####\nconditions = random_pool(variables, num_samples=10, random_state=0)\n\n####################################################################################\n## Define experimentalist\n####################################################################################\n\nexperimentalist = on_state(random_pool, output=[\"conditions\"])\n\n####################################################################################\n## Define experiment runner\n####################################################################################\n\nsin_experiment = equation_experiment(\n sp.simplify(\"sin(x)\"), variables.independent_variables, variables.dependent_variables[0]\n)\nsin_runner = sin_experiment.experiment_runner\n\nexperiment_runner = on_state(sin_runner, output=[\"experiment_data\"])\n\n####################################################################################\n## Define theorist\n####################################################################################\n\ntheorist = estimator_on_state(BMSRegressor(epochs=100))\n\n####################################################################################\n## Define state\n####################################################################################\n\ns = StandardState(\n variables=variables, conditions=conditions, experiment_data=pd.DataFrame(columns=[\"x\", \"y\"])\n)\n\n####################################################################################\n## Cycle through the state\n####################################################################################\n\nprint(\"Pre-Defined State:\")\nprint(f\"Number of datapoints collected: {len(s['experiment_data'])}\")\nprint(f\"Derived models: {s['models']}\")\nprint(\"\\n\")\n\nfor i in range(5):\n s = experimentalist(s, num_samples=10, random_state=42)\n s = experiment_runner(s, added_noise=1.0, random_state=42)\n s = theorist(s)\n print(f\"\\nCycle {i+1} Results:\")\n print(f\"Number of datapoints collected: {len(s['experiment_data'])}\")\n print(f\"Derived models: {s['models']}\")\n print(\"\\n\")\n", "output": "The following example demonstrates how to use AutoRA to automate the process of model discovery, experimental design, and data collection. \n\nThe discovery problem is defined by a single independent variable $x \\in [0, 2 \\pi]$ and dependent variable $y$.\nThe experiment amounts to a simple sine wave, $y = \\sin(x)$, which is the model we are trying to discover.\n\nThe discovery cycle iterates between the experimentalist, experiment runner, and theorist. Here, we us a \"random\" experimentalist, which samples novel experimental conditions for $x$ every cycle. \nThe experiment runner then collects data for the corresponding $y$ values. Finally, the theorist uses a [Bayesian Machine Scientist](https://autoresearch.github.io/autora/user-guide/theorists/bms/) (BMS; Guimerà et al., in Science Advances) to identify a scientific model that explains the data. "} diff --git a/data/autora/text1.txt b/data/autora/text1.txt new file mode 100644 index 0000000..2da4e3b --- /dev/null +++ b/data/autora/text1.txt @@ -0,0 +1,7 @@ +The following example demonstrates how to use AutoRA to automate the process of model discovery, experimental design, and data collection. + +The discovery problem is defined by a single independent variable $x \in [0, 2 \pi]$ and dependent variable $y$. +The experiment amounts to a simple sine wave, $y = \sin(x)$, which is the model we are trying to discover. + +The discovery cycle iterates between the experimentalist, experiment runner, and theorist. Here, we us a "random" experimentalist, which samples novel experimental conditions for $x$ every cycle. +The experiment runner then collects data for the corresponding $y$ values. Finally, the theorist uses a [Bayesian Machine Scientist](https://autoresearch.github.io/autora/user-guide/theorists/bms/) (BMS; Guimerà et al., in Science Advances) to identify a scientific model that explains the data. diff --git a/data/data.jsonl b/data/sweetpea/data.jsonl similarity index 100% rename from data/data.jsonl rename to data/sweetpea/data.jsonl diff --git a/src/autora/doc/pipelines/main.py b/src/autora/doc/pipelines/main.py index 5afc6bf..d8149d7 100644 --- a/src/autora/doc/pipelines/main.py +++ b/src/autora/doc/pipelines/main.py @@ -108,6 +108,18 @@ def import_model(model_name: str) -> None: pass +@app.command() +def import_data(code_file: str, text_file: str, output_file: str = "data.jsonl") -> None: + from pathlib import Path + + import jsonlines + + # alpaca jsonl format: + d = {"instruction": Path(code_file).read_text(), "output": Path(text_file).read_text()} + with jsonlines.open(output_file, "a") as file: + file.write(d) + + if __name__ == "__main__": logger.info(f"Torch version: {torch.__version__} , Cuda available: {torch.cuda.is_available()}") diff --git a/src/autora/doc/runtime/predict_hf.py b/src/autora/doc/runtime/predict_hf.py index 307c99e..8a29519 100644 --- a/src/autora/doc/runtime/predict_hf.py +++ b/src/autora/doc/runtime/predict_hf.py @@ -32,9 +32,9 @@ def predict( sys: str, instr: str, inputs: List[str], - temperature: float = 0.6, + temperature: float = 0.0, top_p: float = 0.95, - top_k: float = 40, + top_k: float = 1, max_length: float = 2048, num_ret_seq: float = 1, ) -> List[List[str]]: @@ -45,10 +45,7 @@ def predict( prompts = [TEMP_LLAMA2.format(sys=sys, instr=instr, input=input) for input in inputs] sequences = self.pipeline( prompts, - do_sample=True, - temperature=temperature, - top_p=top_p, - top_k=int(top_k), + do_sample=False, num_return_sequences=int(num_ret_seq), eos_token_id=self.tokenizer.eos_token_id, max_length=int(max_length), diff --git a/tests/test_main.py b/tests/test_main.py index 097e8c7..2f89f5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,7 +8,7 @@ def test_predict() -> None: - data = Path(__file__).parent.joinpath("../data/data.jsonl").resolve() + data = Path(__file__).parent.joinpath("../data/sweetpea/data.jsonl").resolve() outputs = eval(str(data), TEST_HF_MODEL, SystemPrompts.SYS_1, InstructionPrompts.INSTR_SWEETP_1, []) assert len(outputs) == 3, "Expected 3 outputs" for output in outputs: