131 make a simulated detector that can write hdf files (#144)

* add the sim module

* day progress

* more exploration

* day's work

* incremental changes

* an attempt to put features together

* incremental changes

* try SimSample

* make pattern generator do sensible things

* make lint happy

* attempt at tests

* WIP setting up pytest with fake paths

* write more test outlines

* work on tests

* WIP fix async tests

* WIP make the tests work better

* raw refactor

* refactoring proof of concept

* delete refactoring attempts

* tests improvement

* add basic streaming plan

* rename to sim driver

* change into SimDriver

* fix formatting

* work on the tests

* correct 19 out of 32 errors

* down to 10 errors

* down to 4 errors, all due to observe_value decision and the NDfileHDF

* swmr error

* fix lint

* setup for asyncio

* still error with async

* closing writer and file handle issue

* move back to pattern generator

* got to descriptor validation error

* async setup for all IO failed

* fix writer descriptor test. also component not found async error

* fix the test

* pass all tests

* ran black

* make lint happy

* missed one

* make mypy happy

* fix black

* add declare to the failing plan

* update pyproject for wrong pyside6

* fix the flyer test

* delete mistaken test file

* rollback some unintended changes

* make lint happy

* delete sim sample to lower the test lines

* delete empty test

* mistake double test

* day work (not done yet)

* add snake case@

* lock pyside6 to 6.6.2 to fix the svg indent issue in upstream

* pyproject syntax error fix

.devcontainer/devcontainer.json

@@ -1,47 +1,44 @@
 // For format details, see https://containers.dev/implementors/json_reference/
 {
- "name": "Python 3 Developer Container",
- "build": {
- "dockerfile": "Dockerfile",
- "target": "build",
- // Only upgrade pip, we will install the project below
- "args": {
- "PIP_OPTIONS": "--upgrade pip"
- },
- },
- "remoteEnv": {
- "DISPLAY": "${localEnv:DISPLAY}"
- },
- // Set *default* container specific settings.json values on container create.
- "settings": {
- "python.defaultInterpreterPath": "/venv/bin/python"
- },
- "customizations": {
- "vscode": {
- // Add the IDs of extensions you want installed when the container is created.
- "extensions": [
- "ms-python.python",
- "tamasfe.even-better-toml",
- "redhat.vscode-yaml",
- "ryanluker.vscode-coverage-gutters"
- ]
- }
- },
- // Make sure the files we are mapping into the container exist on the host
- "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'",
- "runArgs": [
- "--net=host",
- "--security-opt=label=type:container_runtime_t"
- ],
- "mounts": [
- "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind",
- "source=${localEnv:HOME}/.inputrc,target=/root/.inputrc,type=bind",
- // map in home directory - not strictly necessary but useful
- "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached"
- ],
- // make the workspace folder the same inside and outside of the container
- "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
- "workspaceFolder": "${localWorkspaceFolder}",
- // After the container is created, install the python project in editable form
- "postCreateCommand": "pip install -e .[dev] --config-settings editable_mode=compat"
+ "name": "Python 3 Developer Container",
+ "build": {
+ "dockerfile": "Dockerfile",
+ "target": "build",
+ // Only upgrade pip, we will install the project below
+ "args": {
+ "PIP_OPTIONS": "--upgrade pip"
+ }
+ },
+ "remoteEnv": {
+ "DISPLAY": "${localEnv:DISPLAY}"
+ },
+ "customizations": {
+ "vscode": {
+ // Set *default* container specific settings.json values on container create.
+ // "settings": {
+ // "python.defaultInterpreterPath": "/venv/bin/python"
+ // },
+ // Add the IDs of extensions you want installed when the container is created.
+ "extensions": [
+ "ms-python.python",
+ "tamasfe.even-better-toml",
+ "redhat.vscode-yaml",
+ "ryanluker.vscode-coverage-gutters"
+ ]
+ }
+ },
+ // Make sure the files we are mapping into the container exist on the host
+ "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'",
+ "runArgs": ["--net=host", "--security-opt=label=type:container_runtime_t"],
+ "mounts": [
+ "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind",
+ "source=${localEnv:HOME}/.inputrc,target=/root/.inputrc,type=bind",
+ // map in home directory - not strictly necessary but useful
+ "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached"
+ ],
+ // make the workspace folder the same inside and outside of the container
+ "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+ "workspaceFolder": "${localWorkspaceFolder}",
+ // After the container is created, install the python project in editable form
+ "postCreateCommand": "pip install -e .[dev] --config-settings editable_mode=compat"
 }

pyproject.toml

@@ -52,8 +52,7 @@ dev = [
  "pre-commit",
  "pydata-sphinx-theme>=0.12",
  "pyepics>=3.4.2",
- "pyside6",
- "pyside6-stubs",
+ "pyside6==6.6.2",
  "pytest",
  "pytest-asyncio",
  "pytest-cov",

src/ophyd_async/epics/areadetector/writers/hdf_writer.py

@@ -109,12 +109,17 @@ async def collect_stream_docs(
  await self.hdf.flush_now.set(True)
  if indices_written:
  if not self._file:
+ path = Path(await self.hdf.full_file_name.get_value())
  self._file = _HDFFile(
  self._directory_provider(),
  # See https://github.com/bluesky/ophyd-async/issues/122
- Path(await self.hdf.full_file_name.get_value()),
+ path,
  self._datasets,
  )
+ # stream resource says "here is a dataset",
+ # stream datum says "here are N frames in that stream resource",
+ # you get one stream resource and many stream datums per scan
+
  for doc in self._file.stream_resources():
  yield "stream_resource", doc
  for doc in self._file.stream_data(indices_written):

src/ophyd_async/sim/__init__.py

@@ -0,0 +1,11 @@
+from .pattern_generator import PatternGenerator
+from .sim_pattern_detector_control import SimPatternDetectorControl
+from .sim_pattern_detector_writer import SimPatternDetectorWriter
+from .sim_pattern_generator import SimPatternDetector
+
+__all__ = [
+ "PatternGenerator",
+ "SimPatternDetectorControl",
+ "SimPatternDetectorWriter",
+ "SimPatternDetector",
+]