From 78632cbca72b297e813d5fa2b59032415ec84582 Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Tue, 18 Jul 2023 17:34:37 +0100 Subject: [PATCH 1/9] Add requirements for uploading hdf5 files via API --- pyproject.toml | 4 +++- requirements-dev.txt | 2 ++ requirements.txt | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4d4dca..c7ac9cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ requires-python = ">=3.10" dependencies = [ "pandas[excel]", "fastapi", - "uvicorn" + "uvicorn", + "python-multipart", + "h5py" ] [project.optional-dependencies] diff --git a/requirements-dev.txt b/requirements-dev.txt index 13d419c..9e59c81 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -116,6 +116,8 @@ pytest-mypy==0.10.3 # via datahub (pyproject.toml) python-dateutil==2.8.2 # via pandas +python-multipart==0.0.6 + # via datahub (pyproject.toml) pytz==2023.3 # via pandas pyxlsb==1.0.10 diff --git a/requirements.txt b/requirements.txt index 2faa371..de5429a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,10 +18,14 @@ fastapi==0.99.1 # via datahub (pyproject.toml) h11==0.14.0 # via uvicorn +h5py==3.9.0 + # via datahub (pyproject.toml) idna==3.4 # via anyio numpy==1.25.0 - # via pandas + # via + # h5py + # pandas odfpy==1.4.1 # via pandas openpyxl==3.1.2 @@ -32,6 +36,8 @@ pydantic==1.10.10 # via fastapi python-dateutil==2.8.2 # via pandas +python-multipart==0.0.6 + # via datahub (pyproject.toml) pytz==2023.3 # via pandas pyxlsb==1.0.10 From 6ef98a0b4aebc63e3384ae67924ebb00655f6dc0 Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Tue, 18 Jul 2023 18:32:29 +0100 Subject: [PATCH 2/9] Create file upload api for dsr data --- datahub/dsr.py | 53 ++++++++++++++++++++++++++----------------------- datahub/main.py | 30 +++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/datahub/dsr.py b/datahub/dsr.py index 863f786..392f1a4 100644 --- a/datahub/dsr.py +++ b/datahub/dsr.py @@ -1,29 +1,30 @@ """This module defines the data structures for the MEDUSA Demand Simulator model.""" import numpy as np +from numpy.typing import NDArray from pydantic import BaseModel, Field class DSRModel(BaseModel): """Define required key values for Demand Side Response data.""" - amount: list = Field(alias="Amount", size=(13,)) - cost: list = Field(alias="Cost", size=(1440, 13)) - kwh_cost: list = Field(alias="kWh Cost", size=(2,)) - activities: list = Field(alias="Activities", size=(1440, 7)) + amount: list = Field(alias="Amount", shape=(13,)) + cost: list = Field(alias="Cost", shape=(1440, 13)) + kwh_cost: list = Field(alias="kWh Cost", shape=(2,)) + activities: list = Field(alias="Activities", shape=(1440, 7)) activities_outside_home: list = Field( - alias="Activities Outside Home", size=(1440, 7) + alias="Activities Outside Home", shape=(1440, 7) ) - activity_types: list = Field(alias="Activity Types", size=(7,)) - ev_id_matrix: list = Field(alias="EV ID Matrix", default=None, size=(1440, 4329)) - ev_dt: list = Field(alias="EV DT", size=(1440, 2)) - ev_locations: list = Field(alias="EV Locations", default=None, size=(1440, 4329)) - ev_battery: list = Field(alias="EV Battery", default=None, size=(1440, 4329)) - ev_state: list = Field(alias="EV State", size=(1440, 4329)) - ev_mask: list = Field(alias="EV Mask", default=None, size=(1440, 4329)) - baseline_ev: list = Field(alias="Baseline EV", size=(1440,)) - baseline_non_ev: list = Field(alias="Baseline Non-EV", size=(1440,)) - actual_ev: list = Field(alias="Actual EV", size=(1440,)) - actual_non_ev: list = Field(alias="Actual Non-EV", size=(1440,)) + activity_types: list = Field(alias="Activity Types", shape=(7,)) + ev_id_matrix: list = Field(alias="EV ID Matrix", default=None, shape=(1440, 4329)) + ev_dt: list = Field(alias="EV DT", shape=(1440, 2)) + ev_locations: list = Field(alias="EV Locations", default=None, shape=(1440, 4329)) + ev_battery: list = Field(alias="EV Battery", default=None, shape=(1440, 4329)) + ev_state: list = Field(alias="EV State", shape=(1440, 4329)) + ev_mask: list = Field(alias="EV Mask", default=None, shape=(1440, 4329)) + baseline_ev: list = Field(alias="Baseline EV", shape=(1440,)) + baseline_non_ev: list = Field(alias="Baseline Non-EV", shape=(1440,)) + actual_ev: list = Field(alias="Actual EV", shape=(1440,)) + actual_non_ev: list = Field(alias="Actual Non-EV", shape=(1440,)) name: str = Field(alias="Name", default="") warn: str = Field(alias="Warn", default="") @@ -33,8 +34,8 @@ class Config: allow_population_by_field_name = True -def validate_dsr_arrays(data: dict[str, str | list]) -> list[str]: - """Validate the sizes of the arrays in the DSR data. +def validate_dsr_arrays(data: dict[str, str | NDArray]) -> list[str]: + """Validate the shapes of the arrays in the DSR data. Args: data: The dictionary representation of the DSR Data. The keys are field aliases. @@ -45,12 +46,14 @@ def validate_dsr_arrays(data: dict[str, str | list]) -> list[str]: """ aliases = [] for alias, field in DSRModel.schema()["properties"].items(): - if field["type"] == "array": - try: - array = np.array(data[alias]) - except ValueError: - aliases.append(alias) - continue - if array.shape != field["size"] or array.dtype != np.float64: + try: + element = data[alias] + except ValueError: + aliases.append(alias) + continue + if isinstance(element, np.ndarray): + if element.shape != field["shape"] or not np.issubdtype( + element.dtype, np.number + ): aliases.append(alias) return aliases diff --git a/datahub/main.py b/datahub/main.py index 67a028c..fa44046 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -1,7 +1,8 @@ """Script for running Datahub API.""" from typing import Any, Hashable -from fastapi import FastAPI, HTTPException +import h5py +from fastapi import FastAPI, HTTPException, UploadFile from pydantic import BaseModel from . import data as dt @@ -87,6 +88,33 @@ def get_opal_data( # type: ignore[misc] return {"data": data} +@app.post("/upload-dsr-file") +def upload_dsr(file: UploadFile) -> dict[str, str | None]: + """POST method for appending data to the DSR list. + + This takes a HDF5 file as input. + + \f + + Args: + file (UploadFile): A HDF5 file with the DSR data. + + Raises: + HTTPException: If the data is invalid + + Returns: + dict[str, str]: dictionary with the filename + """ # noqa: D301 + with h5py.File(file.file, "r") as h5file: + data = {key: value[...] for key, value in h5file.items()} + if alias := validate_dsr_arrays(data): + raise HTTPException( + status_code=400, detail=f"Invalid size for: {', '.join(alias)}." + ) + dt.dsr_data.append(data) + return {"filename": file.filename} + + @app.post("/dsr") def update_dsr_data(data: DSRModel) -> dict[str, str]: """POST method function for appending data to the DSR list. From 4167f4b9003fc6bc7afda299a165dc2da4dcef21 Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Tue, 18 Jul 2023 19:02:04 +0100 Subject: [PATCH 3/9] Remove old /dsr post api --- datahub/main.py | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index fa44046..16a10af 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -7,7 +7,7 @@ from . import data as dt from . import log -from .dsr import DSRModel, validate_dsr_arrays +from .dsr import validate_dsr_arrays from .opal import OpalModel from .wesim import get_wesim @@ -88,7 +88,7 @@ def get_opal_data( # type: ignore[misc] return {"data": data} -@app.post("/upload-dsr-file") +@app.post("/dsr") def upload_dsr(file: UploadFile) -> dict[str, str | None]: """POST method for appending data to the DSR list. @@ -105,42 +105,19 @@ def upload_dsr(file: UploadFile) -> dict[str, str | None]: Returns: dict[str, str]: dictionary with the filename """ # noqa: D301 + log.info("Recieved Opal data.") with h5py.File(file.file, "r") as h5file: data = {key: value[...] for key, value in h5file.items()} if alias := validate_dsr_arrays(data): raise HTTPException( status_code=400, detail=f"Invalid size for: {', '.join(alias)}." ) - dt.dsr_data.append(data) - return {"filename": file.filename} - - -@app.post("/dsr") -def update_dsr_data(data: DSRModel) -> dict[str, str]: - """POST method function for appending data to the DSR list. - - Args: - data: The DSR Data - - Returns: - A dictionary with a success message - - Raises: - A HTTPException if the data is invalid - """ - log.info("Recieved Opal data.") - data_dict = data.dict(by_alias=True) - if alias := validate_dsr_arrays(data_dict): - message = f"Invalid size for: {', '.join(alias)}." - log.error(message) - raise HTTPException(status_code=400, detail=message) - log.info("Appending new data...") log.debug(f"Current DSR data length: {len(dt.dsr_data)}") - dt.dsr_data.append(data_dict) + dt.dsr_data.append(data) log.debug(f"Updated DSR data length: {len(dt.dsr_data)}") - return {"message": "Data submitted successfully."} + return {"filename": file.filename} @app.get("/dsr") From e991e973781b347cd5ed443b2d0b780ad6ac2f05 Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Tue, 18 Jul 2023 19:03:04 +0100 Subject: [PATCH 4/9] Update conftest to use dsr as a temporary file --- tests/conftest.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9c32426..000f242 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ import random +import h5py import numpy as np import pytest from fastapi.testclient import TestClient @@ -38,12 +39,33 @@ def opal_data_array(): @pytest.fixture -def dsr_data(): - """Pytest Fixture for random Opal data input.""" - data = {} - for field in list(DSRModel.__fields__.values()): - if field.annotation == str: - data[field.alias] = "Name or Warning" - else: - data[field.alias] = np.random.rand(*field.field_info.extra["size"]).tolist() +def dsr_data(dsr_data_path): + """Pytest Fixture for DSR data as a dictionary.""" + with h5py.File(dsr_data_path, "r") as h5file: + data = {key: value[...] for key, value in h5file.items()} return data + + +@pytest.fixture +def dsr_data_path(tmp_path): + """The path to a temporary HDF5 file with first-time-only generated DSR data.""" + # Define the file path within the temporary directory + file_path = tmp_path / "data.h5" + + # Check if the file already exists + if file_path.is_file(): + # If the file exists, return its path + return file_path + + # Otherwise, create and write data to the file + with h5py.File(file_path, "w") as h5file: + for field in list(DSRModel.__fields__.values()): + if field.annotation == str: + h5file[field.alias] = "Name or Warning" + else: + h5file[field.alias] = np.random.rand( + *field.field_info.extra["shape"] + ).astype("float16") + + # Return the path to the file + return file_path From aeac309112096c9e2922ed6915d110a0da58859c Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Tue, 18 Jul 2023 19:03:30 +0100 Subject: [PATCH 5/9] Use new file upload for dsr in tests --- datahub/dsr.py | 10 ++++---- tests/test_dsr.py | 62 +++++++++++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/datahub/dsr.py b/datahub/dsr.py index 392f1a4..dd3d16b 100644 --- a/datahub/dsr.py +++ b/datahub/dsr.py @@ -34,7 +34,7 @@ class Config: allow_population_by_field_name = True -def validate_dsr_arrays(data: dict[str, str | NDArray]) -> list[str]: +def validate_dsr_arrays(data: dict[str, NDArray]) -> list[str]: """Validate the shapes of the arrays in the DSR data. Args: @@ -47,13 +47,13 @@ def validate_dsr_arrays(data: dict[str, str | NDArray]) -> list[str]: aliases = [] for alias, field in DSRModel.schema()["properties"].items(): try: - element = data[alias] + array = data[alias] except ValueError: aliases.append(alias) continue - if isinstance(element, np.ndarray): - if element.shape != field["shape"] or not np.issubdtype( - element.dtype, np.number + if field["type"] == "array": + if array.shape != field["shape"] or not np.issubdtype( + array.dtype, np.number ): aliases.append(alias) return aliases diff --git a/tests/test_dsr.py b/tests/test_dsr.py index eac86d3..23d3349 100644 --- a/tests/test_dsr.py +++ b/tests/test_dsr.py @@ -1,5 +1,5 @@ -import json - +import h5py +import numpy as np import pytest from fastapi.testclient import TestClient @@ -7,7 +7,6 @@ from datahub.main import app client = TestClient(app) -client.headers["Content-Type"] = "application/json" @pytest.fixture(autouse=True) @@ -18,45 +17,50 @@ def reset_dsr_data(): def test_validate_dsr_arrays(dsr_data): """Tests the validate_dsr_arrays function.""" - from datahub.dsr import DSRModel, validate_dsr_arrays + from datahub.dsr import validate_dsr_arrays - dsr = DSRModel(**dsr_data) + assert validate_dsr_arrays(dsr_data) == [] - assert validate_dsr_arrays(dsr.dict(by_alias=True)) == [] + dsr_data["Amount"] = np.append(dsr_data["Amount"], 1.0) + dsr_data["Cost"] = np.delete(dsr_data["Cost"], 1) - dsr.amount.append(1.0) - dsr.cost.pop() + assert validate_dsr_arrays(dsr_data) == ["Amount", "Cost"] - assert validate_dsr_arrays(dsr.dict(by_alias=True)) == ["Amount", "Cost"] +def test_post_dsr_api(dsr_data_path): + """Tests POSTing DSR data as a hdf5 file to the API.""" + with open(dsr_data_path, "rb") as dsr_data: + response = client.post("/dsr", files={"file": dsr_data}) -def test_post_dsr_api(dsr_data): - """Tests POSTing DSR data to API.""" - # Checks that a POST request can be successfully made - response = client.post("/dsr", data=json.dumps(dsr_data)) assert response.status_code == 200 - assert response.json() == {"message": "Data submitted successfully."} + assert response.json() == {"filename": dsr_data_path.name} # Checks that the DSR global variable has been updated assert len(dt.dsr_data) == 1 -def test_post_dsr_api_invalid(dsr_data): +def test_post_dsr_api_invalid(dsr_data_path): """Tests POSTing DSR data to API.""" - # Checks invalid array lengths raises an error - dsr_data["Amount"].append(1.0) - dsr_data["Cost"][0].pop() - - response = client.post("/dsr", data=json.dumps(dsr_data)) - assert response.status_code == 400 - assert response.json() == {"detail": "Invalid size for: Amount, Cost."} - - # Checks missing fields raises an error - dsr_data.pop("Amount") - - response = client.post("/dsr", data=json.dumps(dsr_data)) - assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "field required" + with h5py.File(dsr_data_path, "r+") as dsr_data: + # Checks invalid array lengths raises an error + amount = dsr_data.pop("Amount")[...] + dsr_data["Amount"] = np.append(amount, 1.0) + cost = dsr_data.pop("Cost")[...] + dsr_data["Cost"] = cost[1:] + + with open(dsr_data_path, "rb") as dsr_data: + response = client.post("/dsr", files={"file": dsr_data}) + assert response.status_code == 400 + assert response.json() == {"detail": "Invalid size for: Amount, Cost."} + + with h5py.File(dsr_data_path, "r+") as dsr_data: + # Checks missing fields raises an error + dsr_data.pop("Amount") + + with open(dsr_data_path, "rb") as dsr_data: + response = client.post("/dsr", files={"file": dsr_data}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "field required" def test_get_dsr_api(): From 3a2fcc0acc04548a2ac071f9b1cd3ae7ad5ddc9c Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Wed, 19 Jul 2023 12:48:16 +0100 Subject: [PATCH 6/9] Validate for missing fields in dsr file upload --- datahub/dsr.py | 27 +++++++++++++++++++++------ datahub/main.py | 12 ++++++------ tests/test_dsr.py | 37 ++++++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/datahub/dsr.py b/datahub/dsr.py index dd3d16b..e8a615a 100644 --- a/datahub/dsr.py +++ b/datahub/dsr.py @@ -1,5 +1,6 @@ """This module defines the data structures for the MEDUSA Demand Simulator model.""" import numpy as np +from fastapi import HTTPException from numpy.typing import NDArray from pydantic import BaseModel, Field @@ -34,26 +35,40 @@ class Config: allow_population_by_field_name = True -def validate_dsr_arrays(data: dict[str, NDArray]) -> list[str]: +def validate_dsr_data(data: dict[str, NDArray]) -> None: """Validate the shapes of the arrays in the DSR data. Args: data: The dictionary representation of the DSR Data. The keys are field aliases. It is generated with the data.dict(by_alias=True) where data is a DSRModel. - Returns: - An empty list if there are no issues. A list of the failing fields if there are. + Raises: + A HTTPException is there are mising failing fields if there are. """ + missing_fields = [ + field for field in DSRModel.schema()["required"] if field not in data.keys() + ] + if missing_fields: + raise HTTPException( + status_code=422, + detail=f"Missing required fields: {', '.join(missing_fields)}.", + ) + aliases = [] for alias, field in DSRModel.schema()["properties"].items(): try: array = data[alias] - except ValueError: - aliases.append(alias) + except KeyError: + if field: + aliases.append(alias) continue if field["type"] == "array": if array.shape != field["shape"] or not np.issubdtype( array.dtype, np.number ): aliases.append(alias) - return aliases + if aliases: + raise HTTPException( + status_code=422, + detail=f"Invalid size for: {', '.join(aliases)}.", + ) diff --git a/datahub/main.py b/datahub/main.py index 16a10af..7688d93 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -7,7 +7,7 @@ from . import data as dt from . import log -from .dsr import validate_dsr_arrays +from .dsr import validate_dsr_data from .opal import OpalModel from .wesim import get_wesim @@ -92,7 +92,8 @@ def get_opal_data( # type: ignore[misc] def upload_dsr(file: UploadFile) -> dict[str, str | None]: """POST method for appending data to the DSR list. - This takes a HDF5 file as input. + This takes a HDF5 file as input. Data specification can be found at + https://github.com/ImperialCollegeLondon/gridlington-datahub/wiki/Agent-model-data#output \f @@ -108,10 +109,9 @@ def upload_dsr(file: UploadFile) -> dict[str, str | None]: log.info("Recieved Opal data.") with h5py.File(file.file, "r") as h5file: data = {key: value[...] for key, value in h5file.items()} - if alias := validate_dsr_arrays(data): - raise HTTPException( - status_code=400, detail=f"Invalid size for: {', '.join(alias)}." - ) + + validate_dsr_data(data) + log.info("Appending new data...") log.debug(f"Current DSR data length: {len(dt.dsr_data)}") dt.dsr_data.append(data) diff --git a/tests/test_dsr.py b/tests/test_dsr.py index 23d3349..dee0696 100644 --- a/tests/test_dsr.py +++ b/tests/test_dsr.py @@ -15,16 +15,28 @@ def reset_dsr_data(): dt.dsr_data = [] -def test_validate_dsr_arrays(dsr_data): +def test_validate_dsr_data(dsr_data): """Tests the validate_dsr_arrays function.""" - from datahub.dsr import validate_dsr_arrays + from fastapi import HTTPException - assert validate_dsr_arrays(dsr_data) == [] + from datahub.dsr import validate_dsr_data + # Confirm no errors are raised + validate_dsr_data(dsr_data) + + # Check invalid array lengths raises an error dsr_data["Amount"] = np.append(dsr_data["Amount"], 1.0) - dsr_data["Cost"] = np.delete(dsr_data["Cost"], 1) + dsr_data["Cost"] = dsr_data["Cost"][1:] + + with pytest.raises(HTTPException) as err: + validate_dsr_data(dsr_data) + assert err.value.detail == "Invalid size for: Amount, Cost." + + dsr_data.pop("Amount") - assert validate_dsr_arrays(dsr_data) == ["Amount", "Cost"] + with pytest.raises(HTTPException) as err: + validate_dsr_data(dsr_data) + assert err.value.detail == "Missing required fields: Amount." def test_post_dsr_api(dsr_data_path): @@ -40,9 +52,9 @@ def test_post_dsr_api(dsr_data_path): def test_post_dsr_api_invalid(dsr_data_path): - """Tests POSTing DSR data to API.""" + """Tests POSTing invalid DSR data to API.""" + # Check invalid array lengths raises an error with h5py.File(dsr_data_path, "r+") as dsr_data: - # Checks invalid array lengths raises an error amount = dsr_data.pop("Amount")[...] dsr_data["Amount"] = np.append(amount, 1.0) cost = dsr_data.pop("Cost")[...] @@ -50,17 +62,20 @@ def test_post_dsr_api_invalid(dsr_data_path): with open(dsr_data_path, "rb") as dsr_data: response = client.post("/dsr", files={"file": dsr_data}) - assert response.status_code == 400 - assert response.json() == {"detail": "Invalid size for: Amount, Cost."} + assert response.status_code == 422 + assert response.json()["detail"] == "Invalid size for: Amount, Cost." + # Check missing fields raises an error with h5py.File(dsr_data_path, "r+") as dsr_data: - # Checks missing fields raises an error dsr_data.pop("Amount") with open(dsr_data_path, "rb") as dsr_data: response = client.post("/dsr", files={"file": dsr_data}) assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "field required" + assert response.json()["detail"] == "Missing required fields: Amount." + + # Checks that the DSR global variable has not been updated + assert len(dt.dsr_data) == 0 def test_get_dsr_api(): From 9e505cadd14f0fd0e473eeee1c60d86a5139dc2b Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Wed, 19 Jul 2023 13:20:53 +0100 Subject: [PATCH 7/9] Add detail to DSR docstring for API docs --- datahub/main.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index 7688d93..cedf21d 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -63,7 +63,6 @@ def get_opal_data( # type: ignore[misc] Args: start: Starting index for exported Dataframe - end: Last index that will be included in exported Dataframe Returns: @@ -92,8 +91,33 @@ def get_opal_data( # type: ignore[misc] def upload_dsr(file: UploadFile) -> dict[str, str | None]: """POST method for appending data to the DSR list. - This takes a HDF5 file as input. Data specification can be found at - https://github.com/ImperialCollegeLondon/gridlington-datahub/wiki/Agent-model-data#output + This takes a HDF5 file as input. This file has a flat structure, with each dataset + available at the top level. + + The required fields (datasets) are: + - Amount (13 x 1) + - Cost (1440 x 13) + - kWh Cost (2 x 1) + - Activities (1440 x 7) + - Activities Outside Home (1440 x 7) + - Activity Types (7 x 1) + - EV DT (1440 x 2) + - EV State (1440 x 4329) + - Baseline EV (1440 x 1) + - Baseline Non-EV (1440 x 1) + - Actual EV (1440 x 1) + - Actual Non-EV (1440 x 1) + + The optional fields are: + - EV ID Matrix (1440 x 4329) + - EV Locations (1440 x 4329) + - EV Battery (1440 x 4329) + - EV Mask (1440 x 4329) + - Name (str) + - Warn (str) + + Further details for the DSR data specification can be found in + [the GitHub wiki.](https://github.com/ImperialCollegeLondon/gridlington-datahub/wiki/Agent-model-data#output) \f @@ -128,7 +152,6 @@ def get_dsr_data( # type: ignore[misc] Args: start: Starting index for exported list - end: Last index that will be included in exported list Returns: From 059e7f4d7f64c19bb69750165794cb066f3fc2ef Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Wed, 19 Jul 2023 13:22:37 +0100 Subject: [PATCH 8/9] Bring requirement-dev up to date --- requirements-dev.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9e59c81..c98a2ad 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,6 +47,8 @@ h11==0.14.0 # via # httpcore # uvicorn +h5py==3.9.0 + # via datahub (pyproject.toml) httpcore==0.17.2 # via httpx httpx==0.24.1 @@ -71,6 +73,7 @@ nodeenv==1.8.0 # via pre-commit numpy==1.25.0 # via + # h5py # pandas # pandas-stubs odfpy==1.4.1 From 11bbd02f63786eb63d686370e5ea40a002bfd85b Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Wed, 19 Jul 2023 13:33:41 +0100 Subject: [PATCH 9/9] Ignore mypy for stub-less h5py module --- datahub/main.py | 2 +- tests/conftest.py | 2 +- tests/test_dsr.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index cedf21d..a272f6a 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -1,7 +1,7 @@ """Script for running Datahub API.""" from typing import Any, Hashable -import h5py +import h5py # type: ignore from fastapi import FastAPI, HTTPException, UploadFile from pydantic import BaseModel diff --git a/tests/conftest.py b/tests/conftest.py index 000f242..4d76bf3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import random -import h5py +import h5py # type: ignore import numpy as np import pytest from fastapi.testclient import TestClient diff --git a/tests/test_dsr.py b/tests/test_dsr.py index dee0696..4cacf1c 100644 --- a/tests/test_dsr.py +++ b/tests/test_dsr.py @@ -1,4 +1,4 @@ -import h5py +import h5py # type: ignore import numpy as np import pytest from fastapi.testclient import TestClient