Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Zip import/export (based from zip-import-export branch) #80

Merged
merged 14 commits into from
Jun 26, 2024
2 changes: 2 additions & 0 deletions gui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
"plotly.js-cartesian-dist": "^2.33.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-markdown": "^8",
WardBrian marked this conversation as resolved.
Show resolved Hide resolved
"react-plotly.js": "^2.6.0",
"react-router-dom": "^6.17.0",
"react-visibility-sensor": "^5.1.1",
Expand Down
105 changes: 105 additions & 0 deletions gui/src/app/SPAnalysis/FileMapping.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { SPAnalysisDataModel, SPAnalysisPersistentDataModel, stringifyField } from "./SPAnalysisDataModel"

// This code exists to provide rigorous definitions for the mappings between
// the in-memory representation of a Stan Playground project (i.e. the
// SPAnalysisDataModel) and the on-disk representation of its parts, as (for example)
// when downloading or uploading a zip.
//
// Effectively, we need to map among three things:
// 1. the fields of the in-memory data model
// 2. the names of the on-disk/in-zip files
// 3. the actual contents of those on-disk files
// We need the link between 1-2 to serialize the data model fields to files, and
// between 2-3 for deserialization from files.

// Mechanically, we'll also want an exhaustive list of the filenames we will use
// (that's the FileNames enum).

export enum FileNames {
META = 'meta.json',
SAMPLING = 'sampling_opts.json',
STANFILE = 'main.stan',
DATAFILE = 'data.json',
}

// FileMapType enforces an exhaustive mapping from data-model fields to the
// known file names that store those fields. (This is the 1-2 leg of the
// triangle).
type FileMapType = {
[name in keyof SPAnalysisPersistentDataModel]: FileNames
}

// This dictionary stores the actual (global) fields-to-file-names map.
// Because it's of type FileMapType, it enforces that every key in the
// data model (except the "ephemera" key, which is not to be preserved)
// maps to some file name
export const SPAnalysisFileMap: FileMapType = {
meta: FileNames.META,
samplingOpts: FileNames.SAMPLING,
stanFileContent: FileNames.STANFILE,
dataFileContent: FileNames.DATAFILE,
}

// The FileRegistry is the 2-3 leg of the triangle: it maps the known file names
// to their actual contents when read from disk.
// Since we don't *actually* want to mandate that all the known files
// are present, it'll almost always be used in a Partial<>.
// But this way, during deserialization, we can associate the (string) data with
// the file it came from, and the file with the field of the data model, so we
// know how to (re)populate the data model.
export type FileRegistry = {
[name in FileNames]: string
}

// This is a serialization function that maps a data model to a FileRegistry,
// i.e. a dictionary mapping the intended file names to their intended contents.
export const mapModelToFileManifest = (data: SPAnalysisDataModel): Partial<FileRegistry> => {
const fileManifest: Partial<FileRegistry> = {};
const fields = Object.keys(SPAnalysisFileMap) as (keyof SPAnalysisDataModel)[]
fields.forEach((k) => {
if (k === "ephemera") return;
const key = SPAnalysisFileMap[k]
fileManifest[key] = stringifyField(data, k)
})
return fileManifest
}

// This is used during deserialization as an intermediate representation.
// It maps the (named) fields of the data model to the string representation of their
// contents as was written into the file representation.
// During actual deserialization, special case files can be deserialized as needed,
// and the actual file list can just be mapped directly.
export type FieldsContentsMap = {
[name in keyof SPAnalysisPersistentDataModel]: string
}

// This is the inverse of the SPAnalysisFileMap dictionary; with the bonus that it actually
// populates the fields.
export const mapFileContentsToModel = (files: Partial<FileRegistry>): Partial<FieldsContentsMap> => {
const fields = Object.keys(files)
const theMap: Partial<FieldsContentsMap> = {}
fields.forEach(f => {
switch (f) {
case FileNames.META: {
theMap.meta = files[f]
break;
}
case FileNames.DATAFILE: {
theMap.dataFileContent = files[f]
break;
}
case FileNames.STANFILE: {
theMap.stanFileContent = files[f]
break;
}
case FileNames.SAMPLING: {
theMap.samplingOpts = files[f]
break;
}
default:
// Don't do anything for unrecognized filenames
break;
}
})
return theMap
}
7 changes: 4 additions & 3 deletions gui/src/app/SPAnalysis/SPAnalysisContextProvider.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { createContext, FunctionComponent, PropsWithChildren, useEffect, useReducer } from "react"
import { deserializeAnalysis, initialDataModel, serializeAnalysis, SPAnalysisDataModel } from "./SPAnalysisDataModel"
import { initialDataModel, SPAnalysisDataModel } from "./SPAnalysisDataModel"
import { SPAnalysisReducer, SPAnalysisReducerAction, SPAnalysisReducerType } from "./SPAnalysisReducer"
import { deserializeAnalysisFromLocalStorage, serializeAnalysisToLocalStorage } from "./SPAnalysisSerialization"

type SPAnalysisContextType = {
data: SPAnalysisDataModel
Expand All @@ -27,7 +28,7 @@ const SPAnalysisContextProvider: FunctionComponent<PropsWithChildren<SPAnalysisC
useEffect(() => {
// as user reloads the page or closes the tab, save state to local storage
const handleBeforeUnload = () => {
const state = serializeAnalysis(data)
const state = serializeAnalysisToLocalStorage(data)
localStorage.setItem('stan-playground-saved-state', state)
};
window.addEventListener('beforeunload', handleBeforeUnload);
Expand All @@ -41,7 +42,7 @@ const SPAnalysisContextProvider: FunctionComponent<PropsWithChildren<SPAnalysisC
// load the saved state on first load
const savedState = localStorage.getItem('stan-playground-saved-state')
if (!savedState) return
const parsedData = deserializeAnalysis(savedState)
const parsedData = deserializeAnalysisFromLocalStorage(savedState)
update({ type: 'loadLocalStorage', state: parsedData })
}, [])
////////////////////////////////////////////////////////////////////////////////////////
Expand Down
33 changes: 22 additions & 11 deletions gui/src/app/SPAnalysis/SPAnalysisDataModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ export type SPAnalysisDataModel = SPAnalysisBase &
ephemera: SPAnalysisEphemeralData
}

export type SPAnalysisPersistentDataModel = Omit<SPAnalysisDataModel, "ephemera">

export const initialDataModel: SPAnalysisDataModel = {
meta: { title: "Undefined" },
ephemera: {
Expand All @@ -42,17 +44,26 @@ export const initialDataModel: SPAnalysisDataModel = {
samplingOpts: defaultSamplingOpts
}

export const serializeAnalysis = (data: SPAnalysisDataModel): string => {
const intermediary = {
...data, ephemera: undefined }
return JSON.stringify(intermediary)
export const persistStateToEphemera = (data: SPAnalysisDataModel): SPAnalysisDataModel => {
WardBrian marked this conversation as resolved.
Show resolved Hide resolved
const newEphemera = { ...data.ephemera }
getStringKnownFileKeys().forEach(k => newEphemera[k] = data[k])
return {
...data,
ephemera: newEphemera
}
}

export const deserializeAnalysis = (serialized: string): SPAnalysisDataModel => {
const intermediary = JSON.parse(serialized)
// Not sure if this is strictly necessary
intermediary.ephemera = {}
const stringFileKeys = Object.values(SPAnalysisKnownFiles).filter((v) => isNaN(Number(v)));
stringFileKeys.forEach((k) => intermediary.ephemera[k] = intermediary[k]);
return intermediary as SPAnalysisDataModel
export const getStringKnownFileKeys = () => Object.values(SPAnalysisKnownFiles).filter((v) => isNaN(Number(v)));
WardBrian marked this conversation as resolved.
Show resolved Hide resolved

export const modelHasUnsavedChanges = (data: SPAnalysisDataModel): boolean => {
const stringFileKeys = getStringKnownFileKeys()
return stringFileKeys.every((k) => data[k] !== data.ephemera[k])
WardBrian marked this conversation as resolved.
Show resolved Hide resolved
}

export const stringifyField = (data: SPAnalysisDataModel, field: keyof SPAnalysisDataModel): string => {
if (field === 'ephemera') return ''
const value = data[field]
if (typeof value === 'string') return value
return JSON.stringify(value)
}

45 changes: 44 additions & 1 deletion gui/src/app/SPAnalysis/SPAnalysisReducer.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import { Reducer } from "react"
import { Stanie } from "../exampleStanies/exampleStanies"
import { defaultSamplingOpts, SamplingOpts } from '../StanSampler/StanSampler'
import { initialDataModel, SPAnalysisDataModel, SPAnalysisKnownFiles } from "./SPAnalysisDataModel"
import { FieldsContentsMap } from "./FileMapping"
import { initialDataModel, persistStateToEphemera, SPAnalysisDataModel, SPAnalysisKnownFiles } from "./SPAnalysisDataModel"


export type SPAnalysisReducerType = Reducer<SPAnalysisDataModel, SPAnalysisReducerAction>

export type SPAnalysisReducerAction = {
type: 'loadStanie',
stanie: Stanie
} | {
type: 'loadFiles',
files: Partial<FieldsContentsMap>,
clearExisting: boolean
} | {
type: 'retitle',
title: string
Expand Down Expand Up @@ -45,6 +50,9 @@ export const SPAnalysisReducer: SPAnalysisReducerType = (s: SPAnalysisDataModel,
}
}
}
case "loadFiles": {
return loadFromProjectFiles(s, a.files, a.clearExisting)
}
case "retitle": {
return {
...s,
Expand Down Expand Up @@ -73,3 +81,38 @@ export const SPAnalysisReducer: SPAnalysisReducerType = (s: SPAnalysisDataModel,
}
}

const loadMetaFromString = (data: SPAnalysisDataModel, json: string, clearExisting: boolean = false): SPAnalysisDataModel => {
const newMeta = JSON.parse(json)
// TODO: properly check type of deserialized meta
const newMetaMember = clearExisting ? { ...newMeta } : { ...data.meta, ...newMeta }
return { ...data, meta: newMetaMember }
}

const loadSamplingOptsFromString = (data: SPAnalysisDataModel, json: string, clearExisting: boolean = false): SPAnalysisDataModel => {
const newSampling = JSON.parse(json)
// TODO: properly check type/fields of deserialized sampling opts
const newSamplingOptsMember = clearExisting ? { ...newSampling } : { ...data.samplingOpts, ...newSampling }
return { ...data, samplingOpts: newSamplingOptsMember }
}

const loadFileFromString = (data: SPAnalysisDataModel, field: SPAnalysisKnownFiles, contents: string, replaceProject: boolean = false): SPAnalysisDataModel => {
const newData = replaceProject ? { ...initialDataModel } : { ...data }
newData[field] = contents
return newData
}

const loadFromProjectFiles = (data: SPAnalysisDataModel, files: Partial<FieldsContentsMap>, clearExisting: boolean = false): SPAnalysisDataModel => {
let newData = clearExisting ? initialDataModel : data
if (Object.keys(files).includes('meta')) {
newData = loadMetaFromString(newData, files.meta ?? '')
delete files['meta']
}
if (Object.keys(files).includes('samplingOpts')) {
newData = loadSamplingOptsFromString(newData, files.samplingOpts ?? '')
delete files['samplingOpts']
}
const fileKeys = Object.keys(files) as SPAnalysisKnownFiles[]
newData = fileKeys.reduce((currData, currField) => loadFileFromString(currData, currField, files[currField] ?? ''), newData)
newData = persistStateToEphemera(newData)
return newData
}
72 changes: 72 additions & 0 deletions gui/src/app/SPAnalysis/SPAnalysisSerialization.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import JSZip from "jszip"
import { replaceSpacesWithUnderscores } from "../util/replaceSpaces"
import { FileNames, FileRegistry, mapFileContentsToModel, mapModelToFileManifest, SPAnalysisFileMap } from "./FileMapping"
import { getStringKnownFileKeys, SPAnalysisDataModel } from "./SPAnalysisDataModel"

export const serializeAnalysisToLocalStorage = (data: SPAnalysisDataModel): string => {
const intermediary = {
...data, ephemera: undefined }
return JSON.stringify(intermediary)
}

export const deserializeAnalysisFromLocalStorage = (serialized: string): SPAnalysisDataModel => {
const intermediary = JSON.parse(serialized)
// Not sure if this is strictly necessary
intermediary.ephemera = {}
const stringFileKeys = getStringKnownFileKeys()
stringFileKeys.forEach((k) => intermediary.ephemera[k] = intermediary[k]);
return intermediary as SPAnalysisDataModel
}

export const serializeAsZip = async (data: SPAnalysisDataModel): Promise<[Blob, string]> => {
const fileManifest = mapModelToFileManifest(data)
const folderName = replaceSpacesWithUnderscores(data.meta.title)
const zip = new JSZip()
const folder = zip.folder(folderName)
if (!folder) {
throw new Error('Error creating folder in zip file')
}
Object.entries(fileManifest).forEach(([name, content]) => {
folder.file(name, content)
})
const zipBlob = await zip.generateAsync({type: 'blob'})

return [zipBlob, folderName]
}

export const parseFile = (fileBuffer: ArrayBuffer) => {
const content = new TextDecoder().decode(fileBuffer)
return content
}

export const deserializeZipToFiles = async (zipBuffer: ArrayBuffer) => {
const zip = await JSZip.loadAsync(zipBuffer)
const dirNames: string[] = []
zip.forEach((relpath, file) => file.dir && dirNames.push(relpath))
const folderName = dirNames[0] ?? ''
if (! dirNames.every(n => n === folderName)) {
throw new Error('Multiple directories in zip file')
}
zip.forEach((_, file) => {
if (!file.name.startsWith(folderName)) {
throw new Error('Files are not all in a single folder')
}
})
const folderLength = folderName.length
const files: {[name: string]: string} = {}
// we want to use a traditional for loop here, since async doesn't do nicely with higher-order callbacks
for (const name in zip.files) {
const file = zip.files[name]
if (file.dir) continue
const basename = name.substring(folderLength)
if (Object.values(SPAnalysisFileMap).includes(basename as FileNames)) {
const content = await file.async('arraybuffer')
const decoded = new TextDecoder().decode(content)
files[basename] = decoded
} else {
throw new Error(`Unrecognized file in zip: ${file.name} (basename ${basename})`)
}

}
return mapFileContentsToModel(files as Partial<FileRegistry>)
}
13 changes: 5 additions & 8 deletions gui/src/app/SamplerOutputView/SamplerOutputView.tsx
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import { SmallIconButton } from "@fi-sci/misc"
import { Download } from "@mui/icons-material"
import JSZip from 'jszip'
import { FunctionComponent, useCallback, useMemo, useState } from "react"
import StanSampler from "../StanSampler/StanSampler"
import { useSamplerOutput } from "../StanSampler/useStanSampler"
import TabWidget from "../TabWidget/TabWidget"
import TracePlotsView from "./TracePlotsView"
import SummaryView from "./SummaryView"
import { triggerDownload } from "../util/triggerDownload"
import HistsView from "./HistsView"
import JSZip from 'jszip'
import SummaryView from "./SummaryView"
import TracePlotsView from "./TracePlotsView"

type SamplerOutputViewProps = {
width: number
Expand Down Expand Up @@ -250,11 +251,7 @@ const createZipBlobForMultipleCsvs = async (csvTexts: string[], uniqueChainIds:

const downloadTextFile = (text: string, filename: string) => {
const blob = new Blob([text], {type: 'text/plain'});
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = filename;
a.click();
triggerDownload(blob, filename, () => {});
}

export default SamplerOutputView
Loading