From c9c2ac173f46639a023a856b8ca785a6b0546b40 Mon Sep 17 00:00:00 2001 From: akimrx Date: Sun, 2 Jul 2023 10:17:07 +0300 Subject: [PATCH 1/3] fix(metrics): fixed bug with workdays metrics, replaced lib --- requirements.txt | 3 +-- tracker_exporter/__version__.py | 4 ++-- tracker_exporter/defaults.py | 1 + tracker_exporter/exporter.py | 4 ++-- tracker_exporter/main.py | 5 ++++- tracker_exporter/utils/helpers.py | 21 ++++++++++++++++----- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0deb057..c6e3f79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,10 @@ yandex_tracker_client==2.3 -clickhouse_driver==0.2.* datadog==0.44.* apscheduler==3.9.* requests==2.27.* pandas==1.3.* numpy==1.21.* -businesstime==0.3.* +businesstimedelta==1.0.* holidays==0.14.* sentry-sdk==1.6.* python-dotenv==0.20.* \ No newline at end of file diff --git a/tracker_exporter/__version__.py b/tracker_exporter/__version__.py index 7a2218d..05f18ec 100644 --- a/tracker_exporter/__version__.py +++ b/tracker_exporter/__version__.py @@ -1,9 +1,9 @@ # pylint: disable=C0103,W0622 -version = "0.1.17" +version = "0.1.18" url = "https://github.com/akimrx/yandex-tracker-exporter" download_url = "https://pypi.org/project/tracker-exporter/" appname = "yandex_tracker_exporter" description = "Yandex.Tracker issue metrics exporter" author = "Akim Faskhutdinov" -author_email = "a.faskhutdinov@yclients.tech" +author_email = "akimstrong@yandex.ru" license = "MIT" diff --git a/tracker_exporter/defaults.py b/tracker_exporter/defaults.py index 14b9586..5e3ddf8 100644 --- a/tracker_exporter/defaults.py +++ b/tracker_exporter/defaults.py @@ -9,6 +9,7 @@ BUSINESS_HOURS_START = dt.time(9) BUSINESS_HOURS_END = dt.time(22) WEEKENDS = (5, 6,) # Monday is 0, Sunday is 6 +WORKDAYS = [0, 1, 2, 3, 4] # From Monday to Friday # Monitoring settings MONITORING_ENABLED = os.environ.get("EXPORTER_MONITORING_ENABLED", "false").lower() in ("true", "yes") diff --git a/tracker_exporter/exporter.py b/tracker_exporter/exporter.py index 15f9372..b58e060 100644 --- a/tracker_exporter/exporter.py +++ b/tracker_exporter/exporter.py @@ -32,8 +32,8 @@ class Exporter: # TODO: configure class instance # TODO: parse migration from sprint to sprint by changelog (field changed), # by default exported only last sprint (tracker logic) - def __init__(self): - self.clickhouse = ClickhouseClient( + def __init__(self, clickhouse_client: ClickhouseClient = None): + self.clickhouse = clickhouse_client or ClickhouseClient( host=CLICKHOUSE_HOST, port=CLICKHOUSE_HTTP_PORT, user=CLICKHOUSE_USER, diff --git a/tracker_exporter/main.py b/tracker_exporter/main.py index c4e866e..275d990 100644 --- a/tracker_exporter/main.py +++ b/tracker_exporter/main.py @@ -24,9 +24,12 @@ help="Path to .env file" ) args = parser.parse_args() -load_dotenv(args.env_file) warnings.filterwarnings("ignore") +if args.env_file: + load_dotenv(args.env_file) + + # pylint: disable=C0413 from .errors import ExportError from .services.monitoring import DogStatsdClient, sentry_events_filter diff --git a/tracker_exporter/utils/helpers.py b/tracker_exporter/utils/helpers.py index 7b26237..01e5d23 100644 --- a/tracker_exporter/utils/helpers.py +++ b/tracker_exporter/utils/helpers.py @@ -8,12 +8,12 @@ import holidays import pandas as pd +import businesstimedelta -from businesstime import BusinessTime from tracker_exporter.models.enums import TimeDeltaOut from tracker_exporter.defaults import ( NOT_NULLABLE_FIELDS, - WEEKENDS, + WORKDAYS, BUSINESS_HOURS_START, BUSINESS_HOURS_END, TRACKER_DEFAULT_DATETIME_FORMAT @@ -40,26 +40,37 @@ def get_timedelta(end_time: datetime, def calculate_time_spent(start_date: datetime, end_date: datetime, busdays_only: bool = False, - weekends: Tuple[int] = WEEKENDS, + workdays: list = WORKDAYS, business_hours: Tuple = (BUSINESS_HOURS_START, BUSINESS_HOURS_END,)) -> int: """ Calculate timedelta between dates with business days support. Weekdays: Monday is 0, Sunday is 6, so weekends (5, 6) mean (Sat, Sun). + Returns: seconds """ if not isinstance(start_date, datetime): start_date = pd.to_datetime(start_date) if not isinstance(end_date, datetime): end_date = pd.to_datetime(end_date) + holiday_rules = businesstimedelta.HolidayRule(holidays.RU()) + workday_rules = businesstimedelta.WorkDayRule( + start_time=business_hours[0], + end_time=business_hours[1], + working_days=workdays) + + if busdays_only: - bt = BusinessTime(business_hours=business_hours, weekends=weekends, holidays=holidays.RU()) # pylint: disable=C0103 - result = bt.businesstimedelta(start_date, end_date).total_seconds() + logger.debug(f"Calculating workhours. Business hours: {business_hours}. {start_date}, {end_date}") + bt = businesstimedelta.Rules([workday_rules, holiday_rules]) + result = bt.difference(start_date, end_date).timedelta.total_seconds() else: + logger.debug(f"Calculating regular hours") result = (end_date - start_date).total_seconds() return abs(int(result)) + def fix_null_dates(data: dict) -> dict: to_remove = [] From 65fb901501843445a1c4a3e709c9c5e656a1e077 Mon Sep 17 00:00:00 2001 From: akimrx Date: Sun, 2 Jul 2023 10:18:03 +0300 Subject: [PATCH 2/3] chore: removed useless const --- tracker_exporter/defaults.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tracker_exporter/defaults.py b/tracker_exporter/defaults.py index 5e3ddf8..3186af0 100644 --- a/tracker_exporter/defaults.py +++ b/tracker_exporter/defaults.py @@ -8,7 +8,6 @@ # Business days settings BUSINESS_HOURS_START = dt.time(9) BUSINESS_HOURS_END = dt.time(22) -WEEKENDS = (5, 6,) # Monday is 0, Sunday is 6 WORKDAYS = [0, 1, 2, 3, 4] # From Monday to Friday # Monitoring settings From c643471b6e2b894ff68c3fb6f9ca022b2214abb7 Mon Sep 17 00:00:00 2001 From: akimrx Date: Sun, 2 Jul 2023 10:25:06 +0300 Subject: [PATCH 3/3] chore: added new env variables for workhours --- README.md | 2 ++ tracker_exporter/defaults.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d521144..8d149e8 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,8 @@ run_migration | `EXPORTER_CLICKHOUSE_DATABASE` | ❌ | `agile` | Database for exporter CH tables | | `EXPORTER_CLICKHOUSE_ISSUES_TABLE` | ❌ | `issues` | Table when store issues metadata | | `EXPORTER_CLICKHOUSE_ISSUE_METRICS_TABLE` | ❌ | `issue_metrics` | Table when store issue metrics | +| `EXPORTER_WORKHOURS_START` | ❌ | 9 | The beginning of working hours for calculating business hour metrics | +| `EXPORTER_WORKHOURS_END` | ❌ | 22 | The end of working hours for calculating business hour metrics | # Monitoring diff --git a/tracker_exporter/defaults.py b/tracker_exporter/defaults.py index 3186af0..a021276 100644 --- a/tracker_exporter/defaults.py +++ b/tracker_exporter/defaults.py @@ -6,9 +6,13 @@ UPLOAD_TO_STORAGE = os.environ.get("EXPORTER_ENABLE_UPLOAD", "false").lower() in ("true", "yes") # Business days settings -BUSINESS_HOURS_START = dt.time(9) -BUSINESS_HOURS_END = dt.time(22) WORKDAYS = [0, 1, 2, 3, 4] # From Monday to Friday +try: + BUSINESS_HOURS_START = dt.time(int(os.environ.get("EXPORTER_WORKHOURS_START", "9"))) + BUSINESS_HOURS_END = dt.time(int(os.environ.get("EXPORTER_WORKHOURS_END", "22"))) +except (ValueError, TypeError): + BUSINESS_HOURS_START = dt.time(9) + BUSINESS_HOURS_END = dt.time(22) # Monitoring settings MONITORING_ENABLED = os.environ.get("EXPORTER_MONITORING_ENABLED", "false").lower() in ("true", "yes")