From 107bf5e8d86d37b46b76e40817a4868193fe8195 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Mon, 4 Nov 2024 16:37:13 +0000 Subject: [PATCH 1/2] Allow analysis scripts to read data.json --- .../Scripting/Analysis/AnalysisPyWindow.tsx | 16 +++++++++----- .../Scripting/Analysis/AnalysisRWindow.tsx | 6 +++-- .../Scripting/Analysis/useAnalysisState.ts | 12 ++++++++++ .../Scripting/DataGeneration/DataPyWindow.tsx | 7 +++--- .../app/Scripting/pyodide/pyodideWorker.ts | 16 +++++++++++++- .../Scripting/pyodide/pyodideWorkerTypes.ts | 1 + .../app/Scripting/pyodide/usePyodideWorker.ts | 22 +++++++++---------- gui/src/app/Scripting/webR/useWebR.ts | 14 +++++++++++- gui/src/app/StanSampler/StanSampler.ts | 2 +- gui/src/app/StanSampler/useStanSampler.ts | 3 +++ 10 files changed, 74 insertions(+), 25 deletions(-) diff --git a/gui/src/app/Scripting/Analysis/AnalysisPyWindow.tsx b/gui/src/app/Scripting/Analysis/AnalysisPyWindow.tsx index 2adf7f0c..7ce845a5 100644 --- a/gui/src/app/Scripting/Analysis/AnalysisPyWindow.tsx +++ b/gui/src/app/Scripting/Analysis/AnalysisPyWindow.tsx @@ -34,6 +34,7 @@ const AnalysisPyWindow: FunctionComponent = ({ onStatus, runnable, notRunnableReason, + files, } = useAnalysisState(latestRun); const callbacks = useMemo( @@ -51,13 +52,18 @@ const AnalysisPyWindow: FunctionComponent = ({ const handleRun = useCallback( (code: string) => { clearOutputDivs(consoleRef, imagesRef); - run(code, spData, { - loadsDraws: true, - showsPlots: true, - producesData: false, + run({ + code, + spData, + spRunSettings: { + loadsDraws: true, + showsPlots: true, + producesData: false, + }, + files, }); }, - [consoleRef, imagesRef, run, spData], + [consoleRef, imagesRef, run, spData, files], ); const contentOnEmpty = useTemplatedFillerText( diff --git a/gui/src/app/Scripting/Analysis/AnalysisRWindow.tsx b/gui/src/app/Scripting/Analysis/AnalysisRWindow.tsx index d21b4c33..220ed8c6 100644 --- a/gui/src/app/Scripting/Analysis/AnalysisRWindow.tsx +++ b/gui/src/app/Scripting/Analysis/AnalysisRWindow.tsx @@ -26,16 +26,18 @@ const AnalysisRWindow: FunctionComponent = ({ onStatus, runnable, notRunnableReason, + files, } = useAnalysisState(latestRun); const { run } = useWebR({ consoleRef, imagesRef, onStatus }); + const handleRun = useCallback( async (userCode: string) => { clearOutputDivs(consoleRef, imagesRef); const code = loadDrawsCode + userCode; - await run({ code, spData }); + await run({ code, spData, files }); }, - [consoleRef, imagesRef, run, spData], + [consoleRef, imagesRef, run, spData, files], ); const contentOnEmpty = useTemplatedFillerText( diff --git a/gui/src/app/Scripting/Analysis/useAnalysisState.ts b/gui/src/app/Scripting/Analysis/useAnalysisState.ts index 7bdec3f7..74495674 100644 --- a/gui/src/app/Scripting/Analysis/useAnalysisState.ts +++ b/gui/src/app/Scripting/Analysis/useAnalysisState.ts @@ -1,3 +1,4 @@ +import { FileNames } from "@SpCore/FileMapping"; import { InterpreterStatus, isInterpreterBusy, @@ -44,6 +45,16 @@ const useAnalysisState = (latestRun: StanRun) => { const isDataDefined = useMemo(() => spData !== undefined, [spData]); + const files = useMemo(() => { + if (latestRun.data === undefined) { + return undefined; + } else { + return { + [FileNames.DATAFILE]: latestRun.data, + }; + } + }, [latestRun.data]); + useEffect(() => { if (!isDataDefined) { setRunnable(false); @@ -65,6 +76,7 @@ const useAnalysisState = (latestRun: StanRun) => { onStatus: setStatus, runnable, notRunnableReason, + files, }; }; diff --git a/gui/src/app/Scripting/DataGeneration/DataPyWindow.tsx b/gui/src/app/Scripting/DataGeneration/DataPyWindow.tsx index 4f2f8bf0..988c88ad 100644 --- a/gui/src/app/Scripting/DataGeneration/DataPyWindow.tsx +++ b/gui/src/app/Scripting/DataGeneration/DataPyWindow.tsx @@ -39,15 +39,14 @@ const DataPyWindow: FunctionComponent = () => { const handleRun = useCallback( (code: string) => { clearOutputDivs(consoleRef); - run( + run({ code, - {}, - { + spRunSettings: { loadsDraws: false, showsPlots: false, producesData: true, }, - ); + }); }, [consoleRef, run], ); diff --git a/gui/src/app/Scripting/pyodide/pyodideWorker.ts b/gui/src/app/Scripting/pyodide/pyodideWorker.ts index 7600c827..4c0737f1 100644 --- a/gui/src/app/Scripting/pyodide/pyodideWorker.ts +++ b/gui/src/app/Scripting/pyodide/pyodideWorker.ts @@ -67,13 +67,14 @@ self.onmessage = async (e) => { return; } const message = e.data; - await run(message.code, message.spData, message.spRunSettings); + await run(message.code, message.spData, message.spRunSettings, message.files); }; const run = async ( code: string, spData: Record | undefined, spPySettings: PyodideRunSettings, + files: Record | undefined, ) => { setStatus("loading"); try { @@ -113,12 +114,25 @@ const run = async ( await f; } + if (files) { + const encoder = new TextEncoder(); + for (const [filename, content] of Object.entries(files)) { + await pyodide.FS.writeFile(filename, encoder.encode(content + "\n")); + } + } + setStatus("running"); pyodide.runPython(script, { globals }); succeeded = true; } catch (e: any) { console.error(e); sendStderr(e.toString()); + } finally { + if (files) { + for (const filename of Object.keys(files)) { + await pyodide.FS.unlink(filename); + } + } } if (spPySettings.producesData) { diff --git a/gui/src/app/Scripting/pyodide/pyodideWorkerTypes.ts b/gui/src/app/Scripting/pyodide/pyodideWorkerTypes.ts index 5e869d57..9830344e 100644 --- a/gui/src/app/Scripting/pyodide/pyodideWorkerTypes.ts +++ b/gui/src/app/Scripting/pyodide/pyodideWorkerTypes.ts @@ -15,6 +15,7 @@ export type MessageToPyodideWorker = { code: string; spData: Record | undefined; spRunSettings: PyodideRunSettings; + files: Record | undefined; }; export const isMessageToPyodideWorker = ( diff --git a/gui/src/app/Scripting/pyodide/usePyodideWorker.ts b/gui/src/app/Scripting/pyodide/usePyodideWorker.ts index 8b1151b1..114ee3b1 100644 --- a/gui/src/app/Scripting/pyodide/usePyodideWorker.ts +++ b/gui/src/app/Scripting/pyodide/usePyodideWorker.ts @@ -18,6 +18,13 @@ type PyodideWorkerCallbacks = { onImage?: (image: string) => void; }; +type RunPyProps = { + code: string; + spData?: Record; + spRunSettings: PyodideRunSettings; + files?: Record; +}; + class PyodideWorkerInterface { #worker: Worker | undefined; @@ -71,16 +78,13 @@ class PyodideWorkerInterface { return { worker, cleanup }; } - run( - code: string, - spData: Record | undefined, - spRunSettings: PyodideRunSettings, - ) { + run({ code, spData, spRunSettings, files }: RunPyProps) { const msg: MessageToPyodideWorker = { type: "run", code, spData, spRunSettings, + files, }; if (this.#worker) { this.#worker.postMessage(msg); @@ -114,13 +118,9 @@ const usePyodideWorker = (callbacks: { }, [callbacks]); const run = useCallback( - ( - code: string, - spData: Record | undefined, - spRunSettings: PyodideRunSettings, - ) => { + (p: RunPyProps) => { if (worker) { - worker.run(code, spData, spRunSettings); + worker.run(p); } else { throw new Error("pyodide worker is not defined"); } diff --git a/gui/src/app/Scripting/webR/useWebR.ts b/gui/src/app/Scripting/webR/useWebR.ts index 84c3c15e..47d62657 100644 --- a/gui/src/app/Scripting/webR/useWebR.ts +++ b/gui/src/app/Scripting/webR/useWebR.ts @@ -23,6 +23,7 @@ type useWebRProps = { type RunRProps = { code: string; spData?: Record; + files?: Record; }; const useWebR = ({ imagesRef, consoleRef, onStatus, onData }: useWebRProps) => { @@ -63,7 +64,7 @@ const useWebR = ({ imagesRef, consoleRef, onStatus, onData }: useWebRProps) => { }, [consoleRef, imagesRef, onStatus, webR]); const run = useCallback( - async ({ code, spData }: RunRProps) => { + async ({ code, spData, files }: RunRProps) => { try { const webR = await loadWebRInstance(); const shelter = await new webR.Shelter(); @@ -86,6 +87,12 @@ const useWebR = ({ imagesRef, consoleRef, onStatus, onData }: useWebRProps) => { const options = { ...captureOutputOptions, env }; + if (files) { + const encoder = new TextEncoder(); + for (const [name, content] of Object.entries(files)) { + await webR.FS.writeFile(name, encoder.encode(content + "\n")); + } + } // setup await webR.evalRVoid(webRPreamble, options); await webR.evalRVoid(code, options); @@ -98,6 +105,11 @@ const useWebR = ({ imagesRef, consoleRef, onStatus, onData }: useWebRProps) => { } } finally { shelter.purge(); + if (files) { + for (const [name] of Object.entries(files)) { + await webR.FS.unlink(name); + } + } } onStatus("completed"); } catch (e: any) { diff --git a/gui/src/app/StanSampler/StanSampler.ts b/gui/src/app/StanSampler/StanSampler.ts index b6f1586b..69004520 100644 --- a/gui/src/app/StanSampler/StanSampler.ts +++ b/gui/src/app/StanSampler/StanSampler.ts @@ -100,7 +100,7 @@ class StanSampler { }; if (!this.#stanWorker) throw new Error("model worker is undefined"); - this.update({ type: "startSampling", samplingOpts }); + this.update({ type: "startSampling", samplingOpts, data }); this.#samplingStartTimeSec = Date.now() / 1000; this.postMessage({ purpose: Requests.Sample, sampleConfig }); diff --git a/gui/src/app/StanSampler/useStanSampler.ts b/gui/src/app/StanSampler/useStanSampler.ts index a4bf4e19..a79d56a4 100644 --- a/gui/src/app/StanSampler/useStanSampler.ts +++ b/gui/src/app/StanSampler/useStanSampler.ts @@ -11,6 +11,7 @@ export type StanRun = { errorMessage: string; progress?: Progress; samplingOpts?: SamplingOpts; + data?: string; draws?: number[][]; paramNames?: string[]; computeTimeSec?: number; @@ -35,6 +36,7 @@ export type StanRunAction = | { type: "startSampling"; samplingOpts: SamplingOpts; + data: string; } | { type: "samplerReturn"; @@ -65,6 +67,7 @@ export const StanRunReducer = ( status: "sampling", errorMessage: "", samplingOpts: action.samplingOpts, + data: action.data, }; case "samplerReturn": return { From 4733e23621d18d8dcd036599a8663d08ba6faa1f Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Mon, 4 Nov 2024 16:48:45 +0000 Subject: [PATCH 2/2] Update example scripts --- gui/src/app/Scripting/Analysis/analysis_template.R | 4 ++++ gui/src/app/Scripting/Analysis/analysis_template.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/gui/src/app/Scripting/Analysis/analysis_template.R b/gui/src/app/Scripting/Analysis/analysis_template.R index ce85ff9e..80c59eac 100644 --- a/gui/src/app/Scripting/Analysis/analysis_template.R +++ b/gui/src/app/Scripting/Analysis/analysis_template.R @@ -5,3 +5,7 @@ install.packages("bayesplot") library(bayesplot) mcmc_hist(draws, pars = c("lp__")) + +# can also access data.json +data <- jsonlite::read_json("./data.json") +data diff --git a/gui/src/app/Scripting/Analysis/analysis_template.py b/gui/src/app/Scripting/Analysis/analysis_template.py index ec932233..fe4318dc 100644 --- a/gui/src/app/Scripting/Analysis/analysis_template.py +++ b/gui/src/app/Scripting/Analysis/analysis_template.py @@ -12,3 +12,10 @@ plt.hist(samples.ravel(), bins=30) plt.title("lp__") plt.show() + +# you can also read data.json +import json + +with open("data.json") as f: + data = json.load(f) +print(data)