hello world

.gitignore

@@ -0,0 +1,2 @@
+.pytest_cache/
+__pycache__/

README.md

@@ -0,0 +1,64 @@
+# Tofu
+
+<a><img src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/03/Japanese_SilkyTofu_%28Kinugoshi_Tofu%29.JPG/1920px-Japanese_SilkyTofu_%28Kinugoshi_Tofu%29.JPG" 
+title="tofu" alt="tofu" width="20%" height="20%"></a>
+
+Tofu is a Python library for generating synthetic UK Biobank data.
+
+The [UK Biobank](https://www.ukbiobank.ac.uk/) is a large open-access prospective research cohort study 
+of 500,000 middle aged participants recruited in England, Scotland and Wales. The study has collected and continues to collect extensive phenotypic and genotypic detail about its participants, including data from questionnaires, physical measures, sample assays, accelerometry, multimodal imaging, genome-wide genotyping and longitudinal follow-up for a wide range of health-related outcomes. 
+
+Tofu will generate synthetic data which conform to the structure of the baseline data UK Biobank sends researchers by generating random values:
+* For categorical variables (single or multiple choices), a random value will be picked from the UK Biobank data dictionary for that field.
+* For continous variables, a random value will be generated based on the distribution of values reported for that field on the UK Biobank showcase.
+* For date and date/time fields, a random date will be generated.
+* For all other fields, such as polymorphic fields, no data will be generated.
+
+Some general observations:
+* The ```lookups``` directory contains lookups downloaded from the UK Biobank showcase - they might need to be updated when new fields become available.
+* Data conform to the _structure_ and _schema_ of the baseline file but are otherwise nonsensical: no checks have been implemented across fields.
+* All eid's (patient identifiers) generated from this tool are prefaced with 'fake' in order to avoid confusion with legitimate datasets.
+* Dates randomly generated are between 1910 and 1990 again to avoid confusion with real data.
+
+You can find more information on the UK Biobank here:
+
+* The protocol paper in [PLOS Medicine](https://journals.plos.org/plosmedicine/article?id=10.1371/journal.pmed.1001779)
+* Presentations from the [UK Biobank Scientific Conference](https://www.youtube.com/watch?v=_OG9aXf-Pd0&list=PLretMgaKD12883K_GZPWzQUwDBVz5dCfM) contain a lot of information on various aspects of the dataset.
+* The [UK Biobank Showcase](http://biobank.ctsu.ox.ac.uk/crystal/) contains information for each field.
+* The [data dictionaries](https://biobank.ctsu.ox.ac.uk/crystal/exinfo.cgi?src=DataDictionary) contain machine-readable metadata for each field.
+
+## Usage
+
+Generate synthetic data for 100 patients across all baseline fields (not advised unless you really have the entire dataset):
+
+```bash
+python tofu.py -n 100
+Wrote synthetic-20200128181342.csv shape (100,21946).
+```
+
+Generate synthetic data for 100 patients for fields _3_ and _20002_.
+
+```bash
+python tofu.py -n 100 --field 3 20002
+Wrote synthetic-20200128171143.csv shape (10,103).
+```
+
+Generate synthetic data for 100 patients for fields _3_ and _20002_ 
+and make 10% of values missing.
+
+```bash
+python tofu.py -n 100 --field 3 20002 -j 10
+Wrote synthetic-20200128191124.csv shape (100,103).
+```
+
+## Contributing
+Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
+
+Please make sure to update tests as appropriate.
+
+## License
+[MIT](https://choosealicense.com/licenses/mit/)
+
+
+# 
+

helpers.py

@@ -0,0 +1,240 @@
+"""
+Collection of helper functions for working with UK Biobank
+data dictionaries and generating fake values for synthetic
+data.
+"""
+
+import numpy as np
+import pandas as pd
+import datetime
+
+MIN_DATE = pd.to_datetime('1910-01-01')
+MAX_DATE = pd.to_datetime('1990-12-31')
+
+FILE_FIELDS = "lookups/df_lkp_fields.tsv.gz"
+FILE_STATS = "lookups/df_showcase_desc_stats.csv.gz"
+FILE_ENCODINGS = "lookups/df_lkp_encodings.csv.gz"
+
+DF_ENCODINGS = pd.read_csv(FILE_ENCODINGS, encoding='latin-1')
+DF_STATS = pd.read_csv(FILE_STATS)
+DF_FIELDS = pd.read_csv(FILE_FIELDS, sep="\t", encoding='latin-1')
+
+
+def get_field_ids() -> list:
+    """
+    Returns a list of all field id's.
+
+    Output
+    ------
+        all valid field id numbers (list)
+
+    """
+
+    return DF_FIELDS.field_id.values
+
+
+def get_field_metadata(field_id) -> dict:
+    """
+    Returns a dictonary of field metadata
+    stored in the fields data dictionary.
+
+    Arguments
+    ---------
+        field_id : field id number (int)
+
+    Output
+    ------
+        metadata for field (dict)
+
+    """
+
+    df = DF_FIELDS[DF_FIELDS['field_id'] == field_id]
+    if len(df) == 0:
+        return None
+    else:
+        return df.to_dict(orient='records')[0]
+
+
+def gen_fake_ids(n) -> list:
+    """
+    Return a list of fake identifiers.
+
+    Arguments
+    ---------
+        n : number of identifiers to return (int)
+
+    Output
+    ------
+        fake identifiers (list)
+
+    """
+
+    return ["fake%d" % x for x in np.arange(1, n+1, 1)]
+
+
+def get_encoding_id_values(field_encoding_id) -> list:
+    """
+    Returns a list of all valid values for a field
+    based on the UK Biobank encodings dictionary.
+    Will only return values where "selectable" is true
+    as defined in the lookup file supplied by the UK
+    Biobank.
+
+    Arguments
+    ---------
+        field_encoding_id : encoding id number (int)
+
+    Output
+    ------
+        valid values (list)
+
+    """
+
+    # Get all values from the lookup_dictionary
+    # but limit to selectable values.
+    m = (DF_ENCODINGS['encoding_id'] == field_encoding_id)
+    m = m & (DF_ENCODINGS['selectable'] != 0)
+
+    df_value_lookup = DF_ENCODINGS[m]
+
+    # Generate N fake values
+    possible_field_values = df_value_lookup['value'].values
+    return possible_field_values
+
+
+def gen_dummy_data_for_field(field_id, n) -> list:
+
+    """
+    Generate and return a list of fake values for a
+    UK Biobank field. The function supports date,
+    categorical and continuous data type fields (and
+    will return an empty array in other cases, such as
+    for example for fields of polymoprhic value).
+
+    Arguments
+    ----------
+
+        field_id = UK Biobank field identifider (int)
+        n = number of fake fields to be generated (int)
+
+    Output
+    -------
+
+        List of fake values (list)
+
+    """
+
+    field_metadata = get_field_metadata(field_id)
+    field_encoding_id = field_metadata['encoding_id']
+    field_data_type = field_metadata['value_type']
+
+    # Date (51) and time (61)
+    if field_data_type in (51, 61):
+        d = (MAX_DATE - MIN_DATE).days + 1
+        dummy_values = MIN_DATE + pd.to_timedelta(
+                                    pd.np.random.randint(d, size=n),
+                                    unit='d')
+
+    # Categorical single choice (21) or multiple (22)
+    elif field_data_type in (21, 22):
+
+        # Get all values from the lookup_dictionary
+        possible_field_values = get_encoding_id_values(field_encoding_id)
+        dummy_values = np.random.choice(possible_field_values, n).tolist()
+
+    # Float (31) or Integer (11)
+    elif field_data_type in (31, 11):
+
+        # Lookup mean and sd
+        df_entry = DF_STATS[DF_STATS['field_id'] == field_id]
+        mean = 0
+        sd = 1
+
+        if df_entry.empty is False:
+            mean = df_entry['mean'].values[0]
+            sd = df_entry['sd'].values[0]
+
+        # Round all floats to four digits for
+        # continious measurements and to no digits
+        # for integer measurements.
+
+        dec = 4
+        if field_data_type == 11:
+            dec = None
+
+        dummy_values = [
+            round(x, dec) for x in np.random.normal(loc=mean, scale=sd, size=n)
+        ]
+
+    # Everything else
+    else:
+        # Generate an empty array
+        dummy_values = np.empty(n)
+
+    return dummy_values
+
+
+def gen_field_name(field_id, i, a) -> str:
+    """
+    Generate field column name composed of the field id
+    the instance number and the array index.
+
+    Arguments
+    ----------
+
+        field_id = UK Biobank field identifier (int)
+        i = instance id (int)
+        a = array index (int)
+
+    Output
+    -------
+
+        Field name (string)
+
+    """
+
+    return "%d-%d.%d" % (field_id, i, a)
+
+
+def get_now() -> str:
+    """
+    Return current timestamp in YYYYMMDDHHMMSS format.
+    """
+
+    return datetime.datetime.today().strftime('%Y%m%d%H%M%S')
+
+
+def gen_output_filename() -> str:
+    """
+    Return timestamped output filename.
+    """
+
+    return "synthetic-%s.csv" % get_now()
+
+
+def insert_missingness(a, perc) -> list:
+    """
+    Replaces a percentage of items from
+    the list of values with np.nan.
+
+    Arguments
+    ----------
+        a = values of data (list)
+        perc = % of items to be removed (float)
+
+    Output
+    -------
+
+        Values with missingness introduced (list)
+
+    """
+
+    total_values = len(a)
+    howmany = int(total_values * perc / 100)
+    random_ind_to_empty = np.random.randint(0,
+                                            total_values,
+                                            howmany)
+    for i in random_ind_to_empty:
+        a[i] = np.nan
+
+    return a

requirements.txt

@@ -0,0 +1,4 @@
+pandas==0.25.3
+numpy==1.16.4
+scipy==1.3.0
+tqdm==4.32.1

test_all.py

@@ -0,0 +1,70 @@
+
+import helpers
+import numpy as np
+import pytest
+
+
+def test_fake_ids():
+
+    n = 10
+    fake_ids = helpers.gen_fake_ids(n)
+    assert len(fake_ids) == 10
+
+    for x in fake_ids:
+        assert(x.startswith('fake'))
+
+
+def test_field_metadata():
+
+    expected = {'field_id': 3,
+                'title': 'Verbal interview duration',
+                'availability': 0,
+                'stability': 0,
+                'private': 0,
+                'value_type': 11,
+                'base_type': 0,
+                'item_type': 0,
+                'strata': 2,
+                'instanced': 1,
+                'arrayed': 0,
+                'sexed': 0,
+                'units': 'seconds',
+                'main_category': 152,
+                'encoding_id': 0,
+                'instance_id': 2,
+                'instance_min': 0,
+                'instance_max': 3,
+                'array_min': 0,
+                'array_max': 0,
+                'notes': 'Time taken for interview',
+                'debut': '2012-01-05',
+                'version': '2019-09-05',
+                'num_participants': 501673,
+                'item_count': 561869,
+                'showcase_order': 0.0}
+
+    assert helpers.get_field_metadata(3) == expected
+    assert helpers.get_field_metadata(999999) is None
+
+
+def test_get_field_ids():
+    assert len(helpers.get_field_ids()) > 0
+
+
+def test_dummy_data():
+    diseases = helpers.gen_dummy_data_for_field(20002, 100)
+    allowed_values = helpers.get_encoding_id_values(6)
+    assert len(diseases) == 100
+
+    for x in diseases:
+        assert str(x) in allowed_values
+
+    ages = helpers.gen_dummy_data_for_field(21022, 100)
+    assert len(ages) == 100
+    assert pytest.approx(np.median(ages), 2) == 55
+
+
+def test_missing():
+    a = np.random.randint(0, 100, 100).tolist()
+    new_a = helpers.insert_missingness(a, 10)
+    assert pytest.approx(new_a.count(np.nan), 2) == 10