Do deterministic inference

azureml/eval.yml

@@ -18,9 +18,9 @@ inputs:
   #   type: uri_folder
   #   path: azureml://datastores/workspaceblobstore/paths/base_models
   model_path: meta-llama/Llama-2-7b-chat-hf
-  temperature: 0.7
+  temperature: 0.01
   top_p: 0.95
-  top_k: 40
+  top_k: 1
   sys_id: SYS_1
   instruc_id: INSTR_SWEETP_1
 # using a curated environment doesn't work because we need additional packages

data/autora/code1.txt

@@ -0,0 +1,69 @@
+import numpy as np
+import pandas as pd
+import sympy as sp
+from autora.experiment_runner.synthetic.abstract.equation import equation_experiment
+from autora.experimentalist.random import random_pool
+from autora.state import StandardState, estimator_on_state, on_state
+from autora.theorist.bms import BMSRegressor
+from autora.variable import ValueType, Variable, VariableCollection
+
+####################################################################################
+## Define initial data
+####################################################################################
+
+#### Define variable data ####
+iv = Variable(name="x", value_range=(0, 2 * np.pi), allowed_values=np.linspace(0, 2 * np.pi, 30))
+dv = Variable(name="y", type=ValueType.REAL)
+variables = VariableCollection(independent_variables=[iv], dependent_variables=[dv])
+
+#### Define seed condition data ####
+conditions = random_pool(variables, num_samples=10, random_state=0)
+
+####################################################################################
+## Define experimentalist
+####################################################################################
+
+experimentalist = on_state(random_pool, output=["conditions"])
+
+####################################################################################
+## Define experiment runner
+####################################################################################
+
+sin_experiment = equation_experiment(
+    sp.simplify("sin(x)"), variables.independent_variables, variables.dependent_variables[0]
+)
+sin_runner = sin_experiment.experiment_runner
+
+experiment_runner = on_state(sin_runner, output=["experiment_data"])
+
+####################################################################################
+## Define theorist
+####################################################################################
+
+theorist = estimator_on_state(BMSRegressor(epochs=100))
+
+####################################################################################
+## Define state
+####################################################################################
+
+s = StandardState(
+    variables=variables, conditions=conditions, experiment_data=pd.DataFrame(columns=["x", "y"])
+)
+
+####################################################################################
+## Cycle through the state
+####################################################################################
+
+print("Pre-Defined State:")
+print(f"Number of datapoints collected: {len(s['experiment_data'])}")
+print(f"Derived models: {s['models']}")
+print("\n")
+
+for i in range(5):
+    s = experimentalist(s, num_samples=10, random_state=42)
+    s = experiment_runner(s, added_noise=1.0, random_state=42)
+    s = theorist(s)
+    print(f"\nCycle {i+1} Results:")
+    print(f"Number of datapoints collected: {len(s['experiment_data'])}")
+    print(f"Derived models: {s['models']}")
+    print("\n")

data/autora/data.jsonl

@@ -0,0 +1 @@
+{"instruction": "import numpy as np\nimport pandas as pd\nimport sympy as sp\nfrom autora.experiment_runner.synthetic.abstract.equation import equation_experiment\nfrom autora.experimentalist.random import random_pool\nfrom autora.state import StandardState, estimator_on_state, on_state\nfrom autora.theorist.bms import BMSRegressor\nfrom autora.variable import ValueType, Variable, VariableCollection\n\n####################################################################################\n## Define initial data\n####################################################################################\n\n#### Define variable data ####\niv = Variable(name=\"x\", value_range=(0, 2 * np.pi), allowed_values=np.linspace(0, 2 * np.pi, 30))\ndv = Variable(name=\"y\", type=ValueType.REAL)\nvariables = VariableCollection(independent_variables=[iv], dependent_variables=[dv])\n\n#### Define seed condition data ####\nconditions = random_pool(variables, num_samples=10, random_state=0)\n\n####################################################################################\n## Define experimentalist\n####################################################################################\n\nexperimentalist = on_state(random_pool, output=[\"conditions\"])\n\n####################################################################################\n## Define experiment runner\n####################################################################################\n\nsin_experiment = equation_experiment(\n    sp.simplify(\"sin(x)\"), variables.independent_variables, variables.dependent_variables[0]\n)\nsin_runner = sin_experiment.experiment_runner\n\nexperiment_runner = on_state(sin_runner, output=[\"experiment_data\"])\n\n####################################################################################\n## Define theorist\n####################################################################################\n\ntheorist = estimator_on_state(BMSRegressor(epochs=100))\n\n####################################################################################\n## Define state\n####################################################################################\n\ns = StandardState(\n    variables=variables, conditions=conditions, experiment_data=pd.DataFrame(columns=[\"x\", \"y\"])\n)\n\n####################################################################################\n## Cycle through the state\n####################################################################################\n\nprint(\"Pre-Defined State:\")\nprint(f\"Number of datapoints collected: {len(s['experiment_data'])}\")\nprint(f\"Derived models: {s['models']}\")\nprint(\"\\n\")\n\nfor i in range(5):\n    s = experimentalist(s, num_samples=10, random_state=42)\n    s = experiment_runner(s, added_noise=1.0, random_state=42)\n    s = theorist(s)\n    print(f\"\\nCycle {i+1} Results:\")\n    print(f\"Number of datapoints collected: {len(s['experiment_data'])}\")\n    print(f\"Derived models: {s['models']}\")\n    print(\"\\n\")\n", "output": "The following example demonstrates how to use AutoRA to automate the process of model discovery, experimental design, and data collection. \n\nThe discovery problem is defined by a single independent variable $x \\in [0, 2 \\pi]$ and dependent variable $y$.\nThe experiment amounts to a simple sine wave, $y = \\sin(x)$, which is the model we are trying to discover.\n\nThe discovery cycle iterates between the experimentalist, experiment runner, and theorist. Here, we us a \"random\" experimentalist, which samples novel experimental conditions for $x$ every cycle. \nThe experiment runner then collects data for the corresponding $y$ values. Finally, the theorist uses a [Bayesian Machine Scientist](https://autoresearch.github.io/autora/user-guide/theorists/bms/) (BMS; Guimerà et al., in Science Advances) to identify a scientific model that explains the data. "}

data/autora/text1.txt

@@ -0,0 +1,7 @@
+The following example demonstrates how to use AutoRA to automate the process of model discovery, experimental design, and data collection.
+
+The discovery problem is defined by a single independent variable $x \in [0, 2 \pi]$ and dependent variable $y$.
+The experiment amounts to a simple sine wave, $y = \sin(x)$, which is the model we are trying to discover.
+
+The discovery cycle iterates between the experimentalist, experiment runner, and theorist. Here, we us a "random" experimentalist, which samples novel experimental conditions for $x$ every cycle.
+The experiment runner then collects data for the corresponding $y$ values. Finally, the theorist uses a [Bayesian Machine Scientist](https://autoresearch.github.io/autora/user-guide/theorists/bms/) (BMS; Guimerà et al., in Science Advances) to identify a scientific model that explains the data.

src/autora/doc/pipelines/main.py

@@ -108,6 +108,18 @@ def import_model(model_name: str) -> None:
     pass
 
 
+@app.command()
+def import_data(code_file: str, text_file: str, output_file: str = "data.jsonl") -> None:
+    from pathlib import Path
+
+    import jsonlines
+
+    # alpaca jsonl format:
+    d = {"instruction": Path(code_file).read_text(), "output": Path(text_file).read_text()}
+    with jsonlines.open(output_file, "a") as file:
+        file.write(d)
+
+
 if __name__ == "__main__":
     logger.info(f"Torch version: {torch.__version__} , Cuda available: {torch.cuda.is_available()}")
 

src/autora/doc/runtime/predict_hf.py

@@ -32,9 +32,9 @@ def predict(
         sys: str,
         instr: str,
         inputs: List[str],
-        temperature: float = 0.6,
+        temperature: float = 0.0,
         top_p: float = 0.95,
-        top_k: float = 40,
+        top_k: float = 1,
         max_length: float = 2048,
         num_ret_seq: float = 1,
     ) -> List[List[str]]:
@@ -45,10 +45,7 @@ def predict(
         prompts = [TEMP_LLAMA2.format(sys=sys, instr=instr, input=input) for input in inputs]
         sequences = self.pipeline(
             prompts,
-            do_sample=True,
-            temperature=temperature,
-            top_p=top_p,
-            top_k=int(top_k),
+            do_sample=False,
             num_return_sequences=int(num_ret_seq),
             eos_token_id=self.tokenizer.eos_token_id,
             max_length=int(max_length),

tests/test_main.py

@@ -8,7 +8,7 @@
 
 
 def test_predict() -> None:
-    data = Path(__file__).parent.joinpath("../data/data.jsonl").resolve()
+    data = Path(__file__).parent.joinpath("../data/sweetpea/data.jsonl").resolve()
     outputs = eval(str(data), TEST_HF_MODEL, SystemPrompts.SYS_1, InstructionPrompts.INSTR_SWEETP_1, [])
     assert len(outputs) == 3, "Expected 3 outputs"
     for output in outputs: