-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CrateDB] Add support for the data export subsystem
- Loading branch information
Showing
9 changed files
with
187 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,25 +1,29 @@ | ||
# -*- coding: utf-8 -*- | ||
# (c) 2023 Andreas Motl <[email protected]> | ||
import calendar | ||
import functools | ||
import json | ||
from collections import OrderedDict | ||
from decimal import Decimal | ||
from copy import deepcopy | ||
from datetime import datetime, date | ||
from datetime import datetime, date, timezone | ||
|
||
import crate.client.http | ||
import pytz | ||
import requests | ||
from crate import client | ||
from crate.client.converter import DefaultTypeConverter | ||
from crate.client.exceptions import ProgrammingError | ||
from funcy import project | ||
from munch import Munch | ||
from twisted.logger import Logger | ||
|
||
from kotori.daq.storage.util import format_chunk | ||
|
||
log = Logger() | ||
|
||
|
||
class CrateDBAdapter(object): | ||
class CrateDBAdapter: | ||
""" | ||
Kotori database backend adapter for CrateDB. | ||
|
@@ -86,6 +90,79 @@ def create_table(self, tablename): | |
cursor.execute(sql_ddl) | ||
cursor.close() | ||
|
||
def query(self, expression: str, tdata: Munch = None): | ||
""" | ||
Query CrateDB and respond with results in suitable shape. | ||
Make sure to synchronize data by using `REFRESH TABLE ...` before running | ||
the actual `SELECT` statement. This is applicable in test case scenarios. | ||
Response format:: | ||
[ | ||
{ | ||
"time": ..., | ||
"tags": {"city": "berlin", "location": "balcony"}, | ||
"fields": {"temperature": 42.42, "humidity": 84.84}, | ||
}, | ||
... | ||
] | ||
TODO: Unify with `kotori.test.util:CrateDBWrapper.query`. | ||
""" | ||
|
||
log.info(f"Database query: {expression}") | ||
|
||
tdata = tdata or {} | ||
|
||
# Before reading data from CrateDB, synchronize it. | ||
# Currently, it is mostly needed to satisfy synchronization constraints when running the test suite. | ||
# However, users also may expect to see data "immediately". On the other hand, in order to satisfy | ||
# different needs, this should be made configurable per realm, channel and/or request. | ||
# TODO: Maybe just _optionally_ synchronize with the database when reading data. | ||
if tdata: | ||
refresh_sql = f"REFRESH TABLE {self.get_tablename(tdata)}" | ||
self.execute(refresh_sql) | ||
|
||
def dict_from_row(columns, row): | ||
""" | ||
https://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query | ||
https://stackoverflow.com/questions/4147707/python-mysqldb-sqlite-result-as-dictionary | ||
""" | ||
return dict(zip(columns, row)) | ||
|
||
def record_from_dict(item): | ||
record = OrderedDict() | ||
record.update({"time": item["time"]}) | ||
record.update(item["tags"]) | ||
record.update(item["fields"]) | ||
return record | ||
|
||
# Query database, with convenience data type converters. Assume timestamps to be in UTC. | ||
cursor = self.db_client.cursor(converter=DefaultTypeConverter(), time_zone=timezone.utc) | ||
cursor.execute(expression) | ||
data_raw = cursor.fetchall() | ||
|
||
# Provide fully-qualified records to downstream components, including column names. | ||
column_names = [column_info[0] for column_info in cursor.description] | ||
data_tags_fields = map(functools.partial(dict_from_row, column_names), data_raw) | ||
|
||
# Bring results into classic "records" shape. | ||
data_records = map(record_from_dict, data_tags_fields) | ||
|
||
cursor.close() | ||
return data_records | ||
|
||
def execute(self, expression: str): | ||
""" | ||
Execute a database query, using a cursor, and return its results. | ||
""" | ||
cursor = self.db_client.cursor() | ||
cursor.execute(expression) | ||
result = cursor._result | ||
cursor.close() | ||
return result | ||
|
||
def write(self, meta, data): | ||
""" | ||
Format ingress data chunk and store it into database table. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# -*- coding: utf-8 -*- | ||
# (c) 2023 Andreas Motl <[email protected]> | ||
from twisted.logger import Logger | ||
|
||
from kotori.io.protocol.util import compute_daterange | ||
|
||
log = Logger() | ||
|
||
|
||
class QueryTransformer: | ||
|
||
@classmethod | ||
def transform(cls, data): | ||
""" | ||
Compute CrateDB query expression from data in transformation dictionary. | ||
Also compute date range from query parameters "from" and "to". | ||
""" | ||
|
||
log.info(f"Querying database: {data}") | ||
|
||
# The PyInfluxQL query generator is versatile enough to be used for all SQL databases. | ||
from pyinfluxql import Query | ||
|
||
# TODO: Use ".date_range" API method | ||
time_begin, time_end = compute_daterange(data.get('from'), data.get('to')) | ||
|
||
# TODO: Add querying by tags. | ||
tags = {} | ||
# tags = CrateDBAdapter.get_tags(data) | ||
|
||
table = f"{data.database}.{data.measurement}" | ||
expression = Query('*').from_(table).where(time__gte=time_begin, time__lte=time_end, **tags) | ||
|
||
result = { | ||
'expression': str(expression), | ||
'time_begin': time_begin, | ||
'time_end': time_end, | ||
} | ||
|
||
return result |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters