test

data_watchtower/api/handlers/watchtower.py

@@ -51,11 +51,12 @@ def get(self):
             data_loader = item['data_loader']
             if isinstance(data_loader, dict):
                 item['data_loader'] = data_loader.pop('__class__')
-                item['data_loader_cls'] = item['data_loader'].split(':')[-1]
-                item['data_loader_params'] = data_loader
+                item['cls_name'] = item['data_loader'].split(':')[-1]
+                # item['data_loader_params'] = data_loader
+                item['cls_params'] = data_loader
                 data_loader_maps = get_registered_data_loader_maps()
                 # todo 如果data_loader被删除， 则会报错
-                data_loader_cls = data_loader_maps[item['data_loader_cls']]
+                data_loader_cls = data_loader_maps[item['cls_name']]
                 item['data_loader_schema'] = data_loader_cls.to_schema()
             if isinstance(item.get('params'), dict):
                 for k, v in item['params'].items():
@@ -72,6 +73,16 @@ def post(self):
 
 
 class ValidatorRelationHandler(BaseHandler):
+    def get(self):
+        name = self.get_argument('name')
+        validators = self.database.get_validators_of_watchtower(name)
+        result = []
+        for item in validators:
+            item['cls_name'] = item['__class__'].split(':')[-1]
+            item['cls_params'] = item.pop('params')
+            result.append(item)
+        self.json(dict(records=result))
+
     def post(self):
         """
         处理POST请求，用于向数据库中的watchtower添加新的validator。
@@ -120,3 +131,10 @@ def post(self):
 
         # 返回成功的JSON响应
         return self.json()
+
+    def delete(self):
+        body = self.json_loads(self.request.body)
+        name = body['name']  # watchtower的名称
+        validator_id = body['validator_id']
+        self.database.remove_validator_from_watchtower(name, validator_id)
+        return self.json()

data_watchtower/api/url.py

@@ -5,6 +5,7 @@
 URLS = [
     (r"/", validator.ValidatorListHandler),
     (r"/data_watchtower/v1/watchtower", watchtower.WatchtowerHandler),
+    (r"/data_watchtower/v1/watchtower/validator", watchtower.ValidatorRelationHandler),
     (r"/data_watchtower/v1/watchtowers", watchtower.WatchtowerListHandler),
     (r"/data_watchtower/v1/data_loaders", data_loader.DataLoaderListHandler),
     (r"/data_watchtower/v1/validators", validator.ValidatorListHandler),

data_watchtower/core/validators.py

@@ -266,7 +266,7 @@ class ExpectColumnDistinctValuesToContainSet(Validator):
     class Params:
         column = field(type=str, metadata={'help': 'The column name'})
         value_set = field(
-            type=set, metadata={'help': 'A set of objects used for comparison.'},
+            type=list, metadata={'help': 'A set of objects used for comparison.'},
             converter=lambda x: set(x),
         )
 
@@ -299,7 +299,7 @@ class ExpectColumnDistinctValuesToEqualSet(Validator):
     class Params:
         column = field(type=str, metadata={'help': 'The column name'})
         value_set = field(
-            type=set, metadata={'help': 'A set of objects used for comparison.'},
+            type=list, metadata={'help': 'A set of objects used for comparison.'},
             converter=lambda x: set(x),
         )
 
@@ -334,7 +334,7 @@ class ExpectColumnDistinctValuesToBeInSet(Validator):
     class Params:
         column = field(type=str, metadata={'help': 'The column name'})
         value_set = field(
-            type=set, metadata={'help': 'A set of objects used for comparison.'},
+            type=list, metadata={'help': 'A set of objects used for comparison.'},
             converter=lambda x: set(x),
         )
 

data_watchtower/model/models.py

@@ -9,7 +9,11 @@
 from ..utils import json_loads
 
 database_proxy = DatabaseProxy()
+import logging
 
+logger = logging.getLogger('peewee')
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.DEBUG)
 
 class BaseModel(Model):
 

data_watchtower/model/services.py

@@ -58,16 +58,30 @@ def get_watchtowers(self):
         :return:
         """
         result = []
+
+        join_query = ValidatorRelationModel.select(ValidatorRelationModel.wt_name,
+                                                   fn.COUNT(ValidatorRelationModel.validator).alias(
+                                                       'validator_count')).group_by(
+            ValidatorRelationModel.wt_name).alias('join_query')
+        predicate = (WatchtowerModel.name == join_query.c.wt_name)
         query = WatchtowerModel.select(
             WatchtowerModel.name,
             WatchtowerModel.success,
             WatchtowerModel.run_time,
             WatchtowerModel.data_loader,
             WatchtowerModel.params,
+            join_query.c.validator_count,
+        ).join(
+            join_query, on=predicate
         )
         for item in query:
             row = item.to_dict(fields_from_query=query)
             row['params'] = json_loads(row['params'])
+            try:
+                # 不确定这么取validator_count的值是否合理，先这样
+                row['validator_count'] = item.validatorrelationmodel.validator_count
+            except AttributeError:
+                row['validator_count'] = 0
             result.append(row)
         return result
 
@@ -237,8 +251,17 @@ def get_validators_of_watchtower(self, name):
         result = []
         for validator_item in validators:
             validator = dict(
+                id=validator_item.id,
                 params=json_loads(validator_item.params),
                 __class__=validator_item.validator,
             )
             result.append(validator)
         return result
+
+    def remove_validator_from_watchtower(self, wt_name, validator_id):
+        inst = ValidatorRelationModel(
+            wt_name=wt_name,
+            id=validator_id,
+        )
+
+        return inst.delete_instance()

pyproject.toml

@@ -32,6 +32,7 @@ pyarrow = [
     { version = "^16.0.0", python = ">=3.8" }
 ]
 connectorx = [
+    { version = "0.3.0", python = "<=3.7" },
     { version = "0.3.1", python = "<=3.8" },
     { version = "^0.3.2", python = ">3.8" }
 ]