From 24555a64a562eb58035bbe29221cd48975d2ca2f Mon Sep 17 00:00:00 2001 From: Torben <59419684+entorb@users.noreply.github.com> Date: Wed, 13 Sep 2023 08:23:11 +0200 Subject: [PATCH] API v2 migration start --- 1fetch.py => 1fetch_v1.py | 0 1fetch_v2.py | 66 ++++++ 2analyze.py => 2analyze_v1.py | 0 2analyze_v2.py | 376 ++++++++++++++++++++++++++++++++++ 4 files changed, 442 insertions(+) rename 1fetch.py => 1fetch_v1.py (100%) create mode 100755 1fetch_v2.py rename 2analyze.py => 2analyze_v1.py (100%) create mode 100755 2analyze_v2.py diff --git a/1fetch.py b/1fetch_v1.py similarity index 100% rename from 1fetch.py rename to 1fetch_v1.py diff --git a/1fetch_v2.py b/1fetch_v2.py new file mode 100755 index 0000000..673f27d --- /dev/null +++ b/1fetch_v2.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# by Dr. Torben Menke https://entorb.net +# https://github.com/entorb/analyze-oura +""" +Fetch Oura day-summary data from Oura Cloud API. + +requires a personal access token from https://cloud.ouraring.com/personal-access-tokens +provide your personal access token in file token.txt +set the start date in config.json +fetched data is stored in data/ +""" +# standard modules +import json +import os + +import requests + +# external modules + +os.makedirs("data", exist_ok=True) + +with open(file="config.json", encoding="utf-8") as fh: + config = json.load(fh) + +with open(file="token.txt") as fh: + token = fh.read() + token = token.strip() # trim spaces + + +def fetch_data_summaries() -> None: + """ + Fetch data from Oura API. + """ + for data_summary_set in ("sleep",): # , "activity", "readiness" + print(f"fetching {data_summary_set} data") + # url = "https://api.ouraring.com/v1/sleep" + # -> last week + url = f"https://api.ouraring.com/v2/usercollection/{data_summary_set}?start_date={config['date_start']}" # noqa: E501 + # start=YYYY-MM-DD + # end=YYYY-MM-DD + headers = { + # "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0 ", # noqa: E501 + "Authorization": f"Bearer {token}", + } + cont = requests.get(url, headers=headers, timeout=3).content + cont = cont.decode("utf-8") + + with open( + file=f"data/data_raw_{data_summary_set}.json", + mode="w", + encoding="utf-8", + newline="\n", + ) as fh: + fh.write(cont) + with open( + f"data/data_formatted_{data_summary_set}.json", + mode="w", + encoding="utf-8", + newline="\n", + ) as fh: + d = json.loads(cont) + json.dump(d, fh, ensure_ascii=False, sort_keys=False, indent=True) + + +if __name__ == "__main__": + fetch_data_summaries() diff --git a/2analyze.py b/2analyze_v1.py similarity index 100% rename from 2analyze.py rename to 2analyze_v1.py diff --git a/2analyze_v2.py b/2analyze_v2.py new file mode 100755 index 0000000..fe5b106 --- /dev/null +++ b/2analyze_v2.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +# by Dr. Torben Menke https://entorb.net +# https://github.com/entorb/analyze-oura +""" +Analyze data of Oura daily summaries fetched from Oura Cloud API. + +fetched data is read from data/ +a sleep_report.txt is generated +some charts of correlating data are generated in plot/ +""" +import json +import os + +import matplotlib.pyplot as plt +import pandas as pd + +# from turtle import color + +# import numpy as np +# import matplotlib.ticker as mtick + +os.makedirs("plot", exist_ok=True) + +# empty file +fh_report = open( # noqa: SIM115 + file="sleep_report.txt", + mode="w", + encoding="utf-8", + newline="\n", +) + +# fields: see https://cloud.ouraring.com/docs/sleep + + +# hypnogram_5min: +# '1' = deep (N3) sleep +# '2' = light (N1 or N2) sleep - +# '3' = REM sleep - +# '4' = awake + + +def prep_data_sleep() -> pd.DataFrame: + """ + Prepare sleep data. + """ + with open(file="data/data_raw_sleep.json", encoding="utf-8") as fh: + d_json = json.load(fh) + d_json = d_json["data"] # drop first level + # print(d) + + # df = pd.read_json("data_raw.json") + df = pd.DataFrame.from_dict(d_json) + + # remove 5min-interval time series + df.drop(columns=["heart_rate", "hrv", "movement_30_sec"], inplace=True) + + print(df["bedtime_start"]) + df["bedtime_start"] = pd.to_datetime( + df["bedtime_start"], format="%Y-%m-%dT%H:%M:%S%z" + ) + print(df["bedtime_start"]) + # df["bedtime_start"] = pd.to_datetime(df["bedtime_start"]) + df["bedtime_start"] = df["bedtime_start"].dt.tz_convert(None) + print(df["bedtime_start"]) + exit() + # convert to date format + # print(df.info()) + # TODO: why is manual conversion to ISO 8601 format needed? + # "bedtime_start": "2021-12-30T23:38:05+01:00", + print(df["bedtime_start"]) + for col in ("day", "bedtime_end", "bedtime_start"): + df[col] = pd.to_datetime(df[col]) # , format="ISO8601" + for col in ("bedtime_end", "bedtime_start"): + # Remove the timezone information by converting to None + df[col] = df[col].dt.tz_convert(None) + # print(df.info()) + print(df["bedtime_start"]) + exit() + + df["dayofweek"] = df["day"].dt.dayofweek + + # set date as index + df = df.set_index(["day"]) + + df.to_csv( + path_or_buf="data/data_sleep_orig.tsv", + sep="\t", + lineterminator="\n", + ) + + # Adding/calcing some data fields + + df["REM sleep %"] = df["rem_sleep_duration"] / df["total_sleep_duration"] * 100 + df["deep sleep %"] = df["deep_sleep_duration"] / df["total_sleep_duration"] * 100 + df["light sleep %"] = df["light_sleep_duration"] / df["total_sleep_duration"] * 100 + + # calc start of sleep as seconds since midnight + df["start of sleep"] = ( + df["bedtime_start"].dt.hour * 3600 + + df["bedtime_start"].dt.minute * 60 + + df["bedtime_start"].dt.second + ) + print(df[["bedtime_start", "start of sleep"]]) + exit() + + df["start of sleep"] = df["start of sleep"] / 3600 + # -2 -> 22:00 + # +2 -> 02:00 + + df["duration of sleep"] = df["total_sleep_duration"] / 3600 + + df["efficiency %"] = df["efficiency"] * 100 + + df["time to fall asleep"] = df["onset_latency"] / 60 + + # df["time to fall asleep"].where(df["time to fall asleep"] + # > 100, 100, inplace=True) + + df["time awake"] = df["awake"] / 60 + + df.drop( + columns=[ + "bedtime_start_delta", + "total", + "efficiency", + "onset_latency", + "awake", + ], + inplace=True, + ) + + # rename some columns + df.rename( + columns={ + "rmssd": "HRV average", + "hr_average": "HR average", + "hr_lowest": "HR min", + }, + inplace=True, + ) + + df.to_csv( + path_or_buf="data/data_sleep_modified.tsv", + sep="\t", + lineterminator="\n", + ) + return df + + +def corrlation_tester(df, was, interesting_properties): + """ + Tester for Correlations. + """ + s = f"=== Effect of {was} ===" + print(s) + fh_report.write(s + "\n") + + d_results = {} + max_corr = 0.2 + + for column in interesting_properties: + if column == was: + continue + corr = round(df[was].corr(df[column]), 3) + # print(f"{column}: {corr}") + d_results[column] = corr + + l_corr_pos = [] + l_corr_neg = [] + l_corr_none = [] + # sort by absolute value + for column, value in sorted( + d_results.items(), + key=lambda item: abs(item[1]), + reverse=True, + ): + s = f"{d_results[column]:+1.3f} : {column}" + print(s) + fh_report.write(s + "\n") + if value >= max_corr: + l_corr_pos.append(column) + elif value <= -max_corr: + l_corr_neg.append(column) + else: + l_corr_none.append({column}) + fh_report.write("\n") + + return d_results, l_corr_pos, l_corr_neg + + +def plotit(df, was, d_results, l_corr_pos, l_corr_neg) -> None: + """ + Plot the data. + """ + colors = ("#1f77b4", "green") + # colors = ("#1f77b4", "#ff7f0e") + # from + # colors = axes[0].lines[0].get_color(), axes[0].right_ax.lines[0].get_color() + + for pos_neg in ("positive", "negative"): + # pos correlation + + list_of_variables = () + if pos_neg == "positive": + list_of_variables = l_corr_pos + elif pos_neg == "negative": + list_of_variables = l_corr_neg + + numplots = len(list_of_variables) + fig, axes = plt.subplots( + nrows=numplots, + ncols=1, + sharex=True, + dpi=100, + figsize=( + 8, + numplots * 3, + ), + ) + + fig.suptitle( + f"effect of '{was}' is {pos_neg} on ...", + color=colors[1], + ) + i = 0 + for column in list_of_variables: + axes[i].set_title( + f"{column}: {d_results[column]}", + color=colors[0], + ) + df[column].plot( + ax=axes[i], + linewidth=2.0, + color=colors[0], + ) + df[was].plot( + ax=axes[i], + linewidth=2.0, + secondary_y=True, + color=colors[1], + ) + + # layout + if pos_neg == "negative": + axes[i].invert_yaxis() + + # tic color + axes[i].tick_params(axis="y", colors=colors[0]) + axes[i].right_ax.tick_params(axis="y", colors=colors[1]) + # grid + axes[i].grid(zorder=0) + + # y labels + axes[i].set_ylabel(column, color=colors[0]) + axes[i].right_ax.set_ylabel(was, color=colors[1]) + + i += 1 + + # top tics + # axes[0].tick_params( + # axis="x", bottom=False, top=True, labelbottom=False, labeltop=True + # ) + axes[i - 1].set_xlabel("") + fig.set_tight_layout(True) + fig.savefig(fname=f"plot/sleep-{was}-{pos_neg}.png", format="png") + plt.close() + + # # neg correlation + # numplots = len(l_corr_neg) + # fig, axes = plt.subplots( + # nrows=numplots, ncols=1, sharex=True, dpi=100, figsize=(8, numplots * 3) + # ) + + # fig.suptitle(f"Effect of '{was}' is {pos_neg} on ...") + # i = 0 + # for column in l_corr_neg: + # axes[i].set_title(f"{column}: {d_results[column]}") + # df[column].plot( + # ax=axes[i], + # linewidth=2.0, + # ) + # df[was].plot( + # ax=axes[i], + # linewidth=2.0, + # secondary_y=True, + # ) + # # axes[i].right_ax.invert_yaxis() + # if pos_neg == "negative": + # axes[i].invert_yaxis() + # # tic color + # axes[i].tick_params(axis="y", colors=axes[i].lines[0].get_color()) + # axes[i].right_ax.tick_params( + # axis="y", colors=axes[i].right_ax.lines[0].get_color() + # ) + # axes[i].grid(zorder=0) + # i += 1 + # # a = axes[i - 1] + + # # top tics + # # axes[0].tick_params( + # # axis="x", bottom=False, top=True, labelbottom=False, labeltop=True + # # ) + # axes[i - 1].set_xlabel("") + # fig.set_tight_layout(True) + # fig.savefig(fname=f"plot/sleep-{was}-{pos_neg}.png", format="png") + # plt.close() + + +df = prep_data_sleep() + +interesting_properties = ( + "duration of sleep", + "HR average", + "HR min", + "HRV average", + "time to fall asleep", + "time awake", + "efficiency %", + "REM sleep %", + "deep sleep %", + "light sleep %", + "breath_average", + "restless", + "temperature_delta", +) + + +# 1. analize influece of start of sleep +was = "start of sleep" + + +d_results, l_corr_pos, l_corr_neg = corrlation_tester( + df=df, + was=was, + interesting_properties=interesting_properties, +) +plotit(df, was, d_results, l_corr_pos, l_corr_neg) + + +# my results: +# time start sleep up (=later) -> +# total sleep time down +# hr up +# mrssd down +# REM sleep % down + +# print("\n") + +# 2. analize influece of sleep duration + +was = "duration of sleep" + + +d_results, l_corr_pos, l_corr_neg = corrlation_tester( + df=df, + was=was, + interesting_properties=interesting_properties, +) +plotit(df, was, d_results, l_corr_pos, l_corr_neg) + + +fh_report.close() + + +# scatter plots +fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(8, 6)) +axes = df.plot.scatter( + x="start of sleep", + y="HR min", + c="dayofweek", + colormap="viridis", +) +axes.grid(zorder=0) +plt.savefig(fname="plot/scatter1.png", format="png") + +exit()