diff --git a/bodspipelines/infrastructure/caching.py b/bodspipelines/infrastructure/caching.py new file mode 100644 index 0000000..bac2460 --- /dev/null +++ b/bodspipelines/infrastructure/caching.py @@ -0,0 +1,288 @@ +#from functools import wraps +import inspect + +def get_id(storage, item_type, item): + """Get item id given item and item_type""" + return storage.storage.indexes[item_type]['id'](item) + +class Caching(): + """Caching for updates""" + def __init__(self): + """Setup cache""" + self.initialised = False + self.cache = {"latest": {}, "references": {}, "exceptions": {}} + self.batch = {"latest": {}, "references": {}, "exceptions": {}} + + async def load(self, storage): + """Load data into cache""" + for item_type in self.cache: + print(f"Loading cache for {item_type}") + async for item in storage.stream_items(item_type): + #print(item_type, item) + item_id = get_id(storage, item_type, item) + self.cache[item_type][item_id] = item + self.initialised = True + + def save(self, item_type, item, item_id): + """Save item in cache""" + self.cache[item_type][item_id] = item + + def batch_item(self, item_type, item, item_id, overwrite=False): + """Batch to be written later""" + if overwrite: + self.batch[item_type][item_id] = ('index', item) + else: + self.batch[item_type][item_id] = ('create', item) + + def check_batch_item(self, item_type, item_id): + """Check batch for item""" + return self.batch[item_type].get(item_id) + + def unbatch_item(self, item_type, item_id): + """Remove item from batch""" + del self.batch[item_type][item_id] + + async def write_batch(self, storage, item_type): + """Write batch to storage""" + items = [self.batch[item_type][item_id] for item_id in self.batch[item_type]] + await storage.add_batch(item_type, items) + self.batch[item_type] = {} + + async def check_batch(self, storage): + """Check if any batches need writing""" + for item_type in self.batch: + if len(self.batch[item_type]) > 485: + await self.write_batch(storage, item_type) + + async def flush_batch(self, storage): + """Check if any batches need writing""" + for item_type in self.batch: + if len(self.batch[item_type]) > 0: + await self.write_batch(storage, item_type) + + def read(self, item_type, item_id): + """Read item from cache""" + return self.cache[item_type].get(item_id) + + def delete(self, item_type, item_id): + """Delete item from cache""" + del self.cache[item_type][item_id] + +#cache = Caching() + +#def cached(func): +# """Apply caching to function""" +# @wraps(func) +# def wrapper(*args, **kwargs): +# storage = func.__self__ +# if not cache.initialised: +# cache.load(storage) +# if func.__name__ == "add_item": +# item = args[0] +# item_type = args[1] +# item_id = get_id(storage, item_type, item) +# cache.save(item_type, item, item_id) +# return func(*args, **kwargs) +# elif func.__name__ == "get_item": +# item_id = args[0] +# item_type = args[1] +# item = cache.read(item_type, item_id) +# if item: return item +# return func(*args, **kwargs) +# return wrapper + +#async def load_cache(storage): +# await cache.load(storage) + +#async def flush_cache(storage): +# await cache.flush_batch(storage) + +#async def cached(func, *args, **kwargs): +# """Apply caching to function call""" +# #if inspect.isclass(func): +# # storage = func +# #else: +# storage = func.__self__ +# #if "initialise" in kwargs and kwargs["initialise"]: +# # cache.load(storage) +# if "batch" in kwargs: +# batch = kwargs["batch"] +# del kwargs["batch"] +# else: +# batch = False +# if "overwrite" in kwargs: +# overwrite = kwargs["overwrite"] +# del kwargs["overwrite"] +# else: +# overwrite = False +# #if batch == "finished": +# # cache.flush_batch(storage) +# # return +# #if not cache.initialised: +# # cache.load(storage) +# out = None +# if func.__name__ == "add_item": +# item = args[0] +# item_type = args[1] +# item_id = get_id(storage, item_type, item) +# cache.save(item_type, item, item_id) +# if batch: +# cache.batch_item(item_type, item, item_id, overwrite=overwrite) +# else: +# out = await func(*args, **kwargs) +# elif func.__name__ == "get_item": +# item_id = args[0] +# item_type = args[1] +# item = cache.read(item_type, item_id) +# if item: +# out = item +# else: +# out = await func(*args, **kwargs) +# elif func.__name__ == "delete_item": +# item_id = args[0] +# item_type = args[1] +# cache.delete(item_type, item_id) +# if cache.check_batch_item(item_type, item_id): +# cache.unbatch_item(item_type, item_id) +# else: +# out = await func(*args, **kwargs) +# if batch: await cache.check_batch(storage) +# return out + +class Caching(): + """Caching for updates""" + def __init__(self, storage, batching=False): + """Setup cache""" + self.initialised = False + self.cache = {"latest": {}, "references": {}, "exceptions": {}, "updates": {}} + self.batch = {"latest": {}, "references": {}, "exceptions": {}, "updates": {}} if batching else None + self.batch_size = batching if batching else None + self.memory_only = ["updates"] + self.storage = storage + + async def load(self): + """Load data into cache""" + for item_type in self.cache: + if not item_type in self.memory_only: + print(f"Loading cache for {item_type}") + async for item in self.storage.stream_items(item_type): + #print(item_type, item) + item_id = get_id(self.storage, item_type, item) + self.cache[item_type][item_id] = item + self.initialised = True + + def _save(self, item_type, item, item_id, overwrite=False): + """Save item in cache""" + if item_id in self.cache[item_type]: + if overwrite: + self.cache[item_type][item_id] = item + if item_id in self.batch[item_type]: + # ??? + self._batch_item(item_type, item, item_id, overwrite=True) + else: + self._batch_item(item_type, item, item_id, overwrite=True) + else: + raise Exception("Cannot overwrite item {item_id} of type {item_type}") + else: + self.cache[item_type][item_id] = item + self._batch_item(item_type, item, item_id, overwrite=overwrite) + + def _batch_item(self, item_type, item, item_id, overwrite=False): + """Batch to be written later""" + if overwrite: + self.batch[item_type][item_id] = ('index', item) + else: + self.batch[item_type][item_id] = ('index', item) + + def _check_batch_item(self, item_type, item_id): + """Check batch for item""" + return self.batch[item_type].get(item_id) + + def _unbatch_item(self, item_type, item_id): + """Remove item from batch""" + del self.batch[item_type][item_id] + + async def _generate_items(self, items): + for item in items: + yield item[1] + + async def _write_batch(self, item_type): + """Write batch to storage""" + for action in ('index', 'update', 'delete'): + items = [self.batch[item_type][item_id] for item_id in self.batch[item_type] + if self.batch[item_type][item_id][0] == action] + #print(f"{action}: {items}") + if items: + print(f"Flushing {action}: {len(items)} items") + await self.storage.dump_stream(item_type, action, self._generate_items(items)) + self.batch[item_type] = {} + + async def _check_batch(self): + """Check if any batches need writing""" + for item_type in self.batch: + if not item_type in self.memory_only: + if len(self.batch[item_type]) > self.batch_size: + await self._write_batch(item_type) + + async def flush(self): + """Check if any item need writing""" + print("Flushing cache") + for item_type in self.batch: + if not item_type in self.memory_only: + #print(f"{item_type}: {len(self.batch[item_type])} items in batch") + if len(self.batch[item_type]) > 0: + await self._write_batch(item_type) + + def _read(self, item_type, item_id): + """Read item from cache""" + return self.cache[item_type].get(item_id) + + def _delete(self, item_type, item_id, if_exists=False): + """Delete item from cache""" + if if_exists and not item_id in self.cache[item_type]: + return + item = self.cache[item_type][item_id] + del self.cache[item_type][item_id] + if not item_type in self.memory_only: + if item_id in self.batch[item_type] and self.batch[item_type][item_id][0] == 'index': + del self.batch[item_type][item_id] + else: + self.batch[item_type][item_id] = ('delete', item) + + #async def flush_cache(storage): + # await self._flush_batch() + + async def add(self, item, item_type, overwrite=False): + """Apply caching to function call""" + item_id = get_id(self.storage, item_type, item) + self._save(item_type, item, item_id, overwrite=overwrite) + if not self.batch is None: + self._batch_item(item_type, item, item_id, overwrite=overwrite) + else: + out = await self.storage.add_item(*args, **kwargs) + + async def get(self, item_id, item_type): + """Get cached item""" + #print(item_id, item_type) + item = self._read(item_type, item_id) + #if item: + # out = item + #else: + # out = await self.storage.get_item(item, item_type) + return item + + async def delete(self, item_id, item_type, if_exists=False): + """Delete acched item""" + self._delete(item_type, item_id, if_exists=if_exists) + #if self._check_batch_item(item_type, item_id): + # self._unbatch_item(item_type, item_id) + #else: + # out = await self.storage.delete_item(item_id, item_type) + + async def stream(self, item_type): + """Get cached items""" + for item_id in self.cache[item_type]: + yield self.cache[item_type].get(item_id) + + def count(self, item_type): + return len(self.cache[item_type]) diff --git a/bodspipelines/infrastructure/clients/elasticsearch_client.py b/bodspipelines/infrastructure/clients/elasticsearch_client.py index 020b157..89c72f7 100644 --- a/bodspipelines/infrastructure/clients/elasticsearch_client.py +++ b/bodspipelines/infrastructure/clients/elasticsearch_client.py @@ -1,18 +1,27 @@ import os import json -from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk, streaming_bulk +import asyncio +import elastic_transport +from elasticsearch import AsyncElasticsearch +from elasticsearch.helpers import async_streaming_bulk, async_scan, async_bulk -def create_client(): +async def create_client(): """Create Elasticsearch client""" protocol = os.getenv('ELASTICSEARCH_PROTOCOL') host = os.getenv('ELASTICSEARCH_HOST') port = os.getenv('ELASTICSEARCH_PORT') password = os.getenv('ELASTICSEARCH_PASSWORD') if password: - return Elasticsearch(f"{protocol}://{host}:{port}", basic_auth=('elastic', password), timeout=30, max_retries=10, retry_on_timeout=True) + return AsyncElasticsearch(f"{protocol}://{host}:{port}", + basic_auth=('elastic', password), + timeout=30, + max_retries=10, + retry_on_timeout=True) else: - return Elasticsearch(f"{protocol}://{host}:{port}", timeout=30, max_retries=10, retry_on_timeout=True) #, basic_auth=('elastic', password)) + return AsyncElasticsearch(f"{protocol}://{host}:{port}", + timeout=30, + max_retries=10, + retry_on_timeout=True) def index_definition(record, out): """Create index definition from record""" @@ -26,16 +35,20 @@ def index_definition(record, out): class ElasticsearchClient: """ElasticsearchClient class""" - def __init__(self): + def __init__(self, indexes): """Initial setup""" - self.client = create_client() + self.client = None + self.indexes = indexes self.index_name = None + async def create_client(self): + self.client = await create_client() + def set_index(self, index_name): """Set index name""" self.index_name = index_name - def create_index(self, index_name, properties): + async def create_index(self, index_name, properties): """Create index""" self.set_index(index_name) # index settings @@ -43,31 +56,68 @@ def create_index(self, index_name, properties): "number_of_replicas": 0} mappings = {"dynamic": "strict", "properties": properties} - if not self.client.indices.exists(index=self.index_name): + if not await self.client.indices.exists(index=self.index_name): # Ignore 400 means to ignore "Index Already Exist" error. - self.client.options(ignore_status=400).indices.create(index=self.index_name, settings=settings, mappings=mappings) + await self.client.options(ignore_status=400).indices.create(index=self.index_name, + settings=settings, + mappings=mappings) print('Elasticserach created Index') + async def setup_indexes(self): + """Setup indexes""" + done = False + if not self.client: await self.setup() + while not done: + try: + await self.create_indexes() + done = True + except elastic_transport.ConnectionError: + print("Waiting for Elasticsearch to start ...") + await asyncio.sleep(5) + await self.close() + def delete_index(self): """Delete index""" self.client.options(ignore_status=[400, 404]).indices.delete(index=self.index_name) - def stats(self, index_name): - """Get index statistics""" - return self.client.indices.stats(index=index_name) + async def create_indexes(self): + """Moved from storage""" + for index_name in self.indexes: + await self.create_index(index_name, self.indexes[index_name]['properties']) - def store_data(self, data): + async def statistics(self, index_name): + """Get index statistics""" + #count = 0 + stats = {} + #for index_name in self.indexes: + await self.client.indices.refresh(index=index_name) + result = await self.client.cat.count(index=index_name, params={"format": "json"}) + #result = await self.client.indices.stats(index=index_name) + #print(result) + #stats[index_name] = result['_all']['primaries']['docs']['count'] + #count += result['_all']['primaries']['docs']['count'] + #stats['total'] = result['_all']['primaries']['indexing']['index_total'] + stats['total'] = int(result[0]['count']) + return stats + + async def store_data(self, data, id=None): """Store data in index""" if isinstance(data, list): for d in data: - self.client.index(index=self.index_name, document=d) + await self.client.index(index=self.index_name, document=d) else: - self.client.index(index=self.index_name, document=data) + #print(f"Storing in {self.index_name}: {data}") + await self.client.index(index=self.index_name, document=data, id=id) + + async def update_data(self, data, id): + """Update data in index""" + #print(f"Updating {self.index_name} ({id}): {data}") + await self.client.update(index=self.index_name, id=id, doc=data) def bulk_store_data(self, actions, index_name): """Store bulk data in index""" for ok, item in streaming_bulk(client=self.client, index=index_name, actions=actions): - print(ok, item) + #print(ok, item) if not ok: yield False else: @@ -75,38 +125,89 @@ def bulk_store_data(self, actions, index_name): def batch_store_data(self, actions, index_name): """Store bulk data in index""" - #ok, errors = bulk(client=self.client, index=index_name, actions=actions) errors = self.client.bulk(index=index_name, operations=actions) print("Bulk:", errors) return errors - def batch_store_data(self, actions, batch, index_name): + async def batch_store_data(self, actions, batch, index_name): """Store bulk data in index""" + #await self.create_client() record_count = 0 new_records = 0 - for ok, result in streaming_bulk(client=self.client, actions=actions, raise_on_error=False): #index=index_name, + #for b in batch: + # print(b['_id'], b['_index']) + async for ok, result in async_streaming_bulk(client=self.client, actions=actions, raise_on_error=False): record_count += 1 #print(ok, result) - #print(batch[0]) if ok: new_records += 1 - match = [i for i in batch if i['_id'] == result['create']['_id']] - yield match[0]['_source'] + #match = [i for i in batch if i['_id'] == result['create']['_id']] + match = [i for i in batch if i['_id'] == result[i['_op_type']]['_id']] + if match[0]['_op_type'] == 'delete': + yield True + else: + yield match[0]['_source'] else: print(ok, result) - # - #if not ok: - # yield False - #else: - # yield item if callable(index_name): - print(f"Storing in {index_name(batch[0]['_source'])}: {record_count} records; {new_records} new records") + index_name = index_name(batch[0]['_source']) + #print(f"Storing in {index_name(batch[0]['_source'])}: {record_count} records; {new_records} new records") + if batch[0]['_op_type'] == 'delete': + print(f"Deleting in {index_name}: {record_count} records; {new_records} records deleted") else: print(f"Storing in {index_name}: {record_count} records; {new_records} new records") - def search(self, search): + async def search(self, search): """Search index""" - return self.client.search(index=self.index_name, query=search) + #print(f"ES search ({self.index_name}): {search}") + return await self.client.search(index=self.index_name, query=search) + + async def get(self, id): + """Get by id""" + #match = await self.search({"query": {"match": {"_id": id}}}) + match = await self.search({"match": {"_id": id}}) + result = match['hits']['hits'] + if result: + return result[0]['_source'] + else: + return None + + async def delete(self, id): + """Delete by id""" + return await self.client.delete(index=self.index_name, id=id) + + async def delete_all(self, index): + """Delete all documents in index""" + await self.client.delete_by_query(index=index, query={"query":{"match_all":{}}}) + + async def scan_index(self, index): + """Scan index""" + async for doc in async_scan(client=self.client, + query={"query": {"match_all": {}}}, + index=index): + yield doc + + async def _generate_actions(self, index_name, action_type, items): + #if action_type == 'update': + # metadata = {'_op_type': action_type, + # '_type': 'document', + # "_index": index_name} + #else: + metadata = {'_op_type': action_type, + "_index": index_name} + async for item in items: + if action_type == 'delete': + action = metadata | {'_id': self.indexes[index_name]["id"](item)} + elif action_type == 'update': + action = metadata | {'_id': self.indexes[index_name]["id"](item)} | item + else: + action = metadata | {'_id': self.indexes[index_name]["id"](item)} | item + #print(action) + yield action + + async def dump_stream(self, index_name, action_type, items): + await async_bulk(client=self.client, + actions=self._generate_actions(index_name, action_type, items)) def list_indexes(self): """List indexes""" @@ -117,5 +218,13 @@ def get_mapping(self, index_name): return self.client.indices.get_mapping(index=index_name) def check_new(self, data): + """Dummy method""" pass + async def setup(self): + """Setup Elasticsearch client""" + self.client = await create_client() + + async def close(self): + """Close Elasticsearch client""" + await self.client.transport.close() diff --git a/bodspipelines/infrastructure/clients/kinesis_client.py b/bodspipelines/infrastructure/clients/kinesis_client.py index c94125d..5fb8642 100644 --- a/bodspipelines/infrastructure/clients/kinesis_client.py +++ b/bodspipelines/infrastructure/clients/kinesis_client.py @@ -2,19 +2,28 @@ import time import json import gzip -import boto3 +#import boto3 -def create_client(service): +from pathlib import Path +from aiobotocore.session import get_session + +async def create_client(service): """Create AWS client for specified service""" - return boto3.client(service, region_name=os.getenv('BODS_AWS_REGION'), aws_access_key_id=os.environ.get('BODS_AWS_ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('BODS_AWS_SECRET_ACCESS_KEY')) + #return boto3.client(service, region_name=os.getenv('BODS_AWS_REGION'), aws_access_key_id=os.environ.get('BODS_AWS_ACCESS_KEY_ID'), + # aws_secret_access_key=os.environ.get('BODS_AWS_SECRET_ACCESS_KEY')) + session = get_session() + return await session.create_client(service, region_name=os.getenv('BODS_AWS_REGION'), + aws_secret_access_key=os.environ["BODS_AWS_SECRET_ACCESS_KEY"], + aws_access_key_id=os.environ["BODS_AWS_ACCESS_KEY_ID"]).__aenter__() + -def get_stream_arn(client, stream_name): - return client.describe_stream(StreamName=stream_name)["StreamDescription"]["StreamARN"] +async def get_stream_arn(client, stream_name): + data = await client.describe_stream(StreamName=stream_name) + return data["StreamDescription"]["StreamARN"] -def shard_id(client, stream_arn): +async def shard_id(client, stream_arn): """Generate shard id""" - response = client.describe_stream(StreamARN=stream_arn) + response = await client.describe_stream(StreamARN=stream_arn) return response['StreamDescription']['Shards'][0]['ShardId'] def unpack_records(record_response): @@ -25,23 +34,50 @@ def unpack_records(record_response): records.append(json.loads(record['Data'])) return records +def save_last_seqno(stream_name, last_seqno): + status_dir = os.getenv('KINESIS_STATUS_DIRECTORY') + path = Path(f"{status_dir}/{stream_name}") + if last_seqno: + with open(path, 'w') as file: + file.write(last_seqno) + +def load_last_seqno(stream_name): + status_dir = os.getenv('KINESIS_STATUS_DIRECTORY') + path = Path(f"{status_dir}/{stream_name}") + if path.is_file(): + with open(path, 'r') as file: + return file.read() + else: + return None + class KinesisStream: """Kinesis Stream class""" def __init__(self, stream_name=None, shard_count=1): """Initial setup""" - self.client = create_client('kinesis') - self.stream_arn = get_stream_arn(self.client, stream_name) - self.shard_id = shard_id(self.client, self.stream_arn) + self.stream_name = stream_name + #self.stream_arn = get_stream_arn(self.client, stream_name) + #self.shard_id = shard_id(self.client, self.stream_arn) self.shard_count = shard_count self.records = [] self.waiting_bytes = 0 + self.last_seqno = load_last_seqno(self.stream_name) - def send_records(self): + async def setup(self): + """Setup Kinesis client""" + self.client = await create_client('kinesis') + self.stream_arn = await get_stream_arn(self.client, self.stream_name) + self.shard_id = await shard_id(self.client, self.stream_arn) + + def save_last_seqno(self, response): + if response and 'Records' in response and len(response['Records']) > 0: + self.last_seqno = response['Records'][-1]['SequenceNumber'] + + async def send_records(self): """Send accumulated records""" print(f"Sending {len(self.records)} records to {self.stream_arn}") failed = len(self.records) while failed == len(self.records): - response = self.client.put_records(Records=self.records, StreamARN=self.stream_arn) #, StreamARN='string') + response = await self.client.put_records(Records=self.records, StreamARN=self.stream_arn) #, StreamARN='string') failed = response['FailedRecordCount'] if failed == len(self.records): time.sleep(1) @@ -59,7 +95,7 @@ def send_records(self): self.waiting_bytes = 0 break - def add_record(self, record): + async def add_record(self, record): """Add record to stream""" json_data = json.dumps(record) + "\n" #encoded_data = bytes(json_data, 'utf-8') @@ -69,28 +105,36 @@ def add_record(self, record): self.waiting_bytes += num_bytes #print(f"Added {num_bytes} byte record ...") #print(f"Batched records {len(self.records)}") - if self.waiting_bytes > 500000 or len(self.records) > 485: self.send_records() + if self.waiting_bytes > 500000 or len(self.records) > 485: await self.send_records() - def finish_write(self): + async def finish_write(self): """Write any remaining records""" - if len(self.records) > 0: self.send_records() + if len(self.records) > 0: await self.send_records() - def read_stream(self): + async def read_stream(self): """Read records from stream""" - shard_iterator = self.client.get_shard_iterator(StreamARN=self.stream_arn, + if self.last_seqno: + shard_iterator = await self.client.get_shard_iterator(StreamARN=self.stream_arn, + ShardId=self.shard_id, + ShardIteratorType='AFTER_SEQUENCE_NUMBER', + StartingSequenceNumber=self.last_seqno) + else: + shard_iterator = await self.client.get_shard_iterator(StreamARN=self.stream_arn, ShardId=self.shard_id, ShardIteratorType='TRIM_HORIZON') - #ShardIteratorType='LATEST') shard_iterator = shard_iterator['ShardIterator'] empty = 0 while True: - record_response = self.client.get_records(ShardIterator=shard_iterator, Limit=100) + record_response = await self.client.get_records(ShardIterator=shard_iterator, Limit=500) + self.save_last_seqno(record_response) #print(record_response) if len(record_response['Records']) == 0 and record_response['MillisBehindLatest'] == 0: empty += 1 else: if len(record_response['Records']) > 0: empty = 0 - yield unpack_records(record_response) + print(f"Read {len(record_response['Records'])} records from {self.stream_arn}") + for item in unpack_records(record_response): + yield item if empty > 250: print(f"No records found in {self.stream_arn} after {empty} retries") break @@ -99,6 +143,12 @@ def read_stream(self): else: break + async def close(self): + """Close Kinesis client""" + if self.client: + await self.client.__aexit__(None, None, None) + save_last_seqno(self.stream_name, self.last_seqno) + # def read_stream(self): # """Read records from stream""" # stream = self.client.describe_stream(StreamName=self.stream_name) diff --git a/bodspipelines/infrastructure/clients/redis_client.py b/bodspipelines/infrastructure/clients/redis_client.py new file mode 100644 index 0000000..1bf372f --- /dev/null +++ b/bodspipelines/infrastructure/clients/redis_client.py @@ -0,0 +1,87 @@ +import os +import json + +from redis.asyncio import Redis, RedisError + +def create_client(): + """Create redis client""" + host = os.getenv('REDIS_HOST') + port = os.getenv('REDIS_PORT') + return Redis(host=host, port=port) + +def get_key(index, id): + return f"{index}-{id}" + +class RedisClient: + """RedisClient class""" + def __init__(self, indexes): + """Initial setup""" + self.client = create_client() + self.indexes = indexes + self.index_name = None + + def set_index(self, index_name): + """Set index name""" + self.index_name = index_name + + async def batch_store_data(self, actions, batch, index_name, output_new=True): + """Store bulk data in index""" + record_count = 0 + new_records = 0 + async with self.client.pipeline() as pipe: + async for item in actions: + key = get_key(index_name, item['_id']) + await pipe.setnx(key, json.dumps(item['_source'])) + results = await pipe.execute() + for i, result in enumerate(results): + if result is True and output_new: + new_records += 1 + yield batch[i]['_source'] + #else: + # print(result, batch[i]['_id']) + + async def get(self, id): + """Search index""" + key = get_key(self.index_name, id) + try: + value = await self.client.get(key) + except RedisError: + return None + return json.loads(value) + + async def store_data(self, data): + """Store data in index""" + if isinstance(data, list): + for d in data: + key = get_key(self.index_name, d['_id']) + await self.client.set(key, d['_source']) + else: + key = get_key(self.index_name, data['_id']) + await self.client.set(key, data['_source']) + + async def count_keys(self, pattern: str) -> int: + """Counts the number of keys matching a pattern.""" + keys_count = self.client.register_script( + """ + return #redis.call('KEYS', ARGV[1]) + """ + ) + return await keys_count(args=[pattern]) + + async def statistics(self) -> dict: + """Calculate storage statistics""" + stats = {} + for index in self.indexes: + keys = await self.count_keys(f"{index}*") + stats[index] = keys + total_keys = await self.client.dbsize() + stats['total'] = total_keys + return stats + + async def setup(self): + """Dummy setup method""" + pass + + async def setup_indexes(self): + """Dummy setup indexes""" + pass diff --git a/bodspipelines/infrastructure/indexes.py b/bodspipelines/infrastructure/indexes.py index e1e00e1..1efc617 100644 --- a/bodspipelines/infrastructure/indexes.py +++ b/bodspipelines/infrastructure/indexes.py @@ -4,6 +4,7 @@ 'statementDate': {'type': 'text'}, 'entityType': {'type': 'text'}, 'name': {'type': 'text'}, + 'isComponent': {"type": "boolean"}, 'incorporatedInJurisdiction': {'type': 'object', 'properties': {'name': {'type': 'text'}, 'code': {'type': 'text'}}}, @@ -29,7 +30,17 @@ 'url': {'type': 'text'}}}}}, 'source': {'type': 'object', 'properties': {'type': {'type': 'text'}, - 'description': {'type': 'text'}}}} + 'description': {'type': 'text'}}}, + 'annotations': {'type': 'object', + 'properties': {'motivation': {'type': 'text'}, + 'description': {'type': 'text'}, + 'statementPointerTarget': {'type': 'text'}, + 'creationDate': {'type': 'text'}, + 'createdBy': {'type': 'object', + 'properties': {'name': {'type': 'text'}, + 'uri': {'type': 'text'}}}}}, + 'replacesStatements': {'type': 'text'} + } # BODS Entity Statement Elasticsearch Properties @@ -37,6 +48,7 @@ 'statementType': {'type': 'text'}, 'statementDate': {'type': 'text'}, 'personType': {'type': 'text'}, + 'isComponent': {"type": "boolean"}, 'unspecifiedPersonDetails': {'type': 'object', 'properties': {'reason': {'type': 'text'}, 'description': {'type': 'text'}}}, @@ -49,12 +61,23 @@ 'url': {'type': 'text'}}}}}, 'source': {'type': 'object', 'properties': {'type': {'type': 'text'}, - 'description': {'type': 'text'}}}} + 'description': {'type': 'text'}}}, + 'annotations': {'type': 'object', + 'properties': {'motivation': {'type': 'text'}, + 'description': {'type': 'text'}, + 'statementPointerTarget': {'type': 'text'}, + 'creationDate': {'type': 'text'}, + 'createdBy': {'type': 'object', + 'properties': {'name': {'type': 'text'}, + 'uri': {'type': 'text'}}}}}, + 'replacesStatements': {'type': 'text'} + } # BODS Ownership Or Control Statement ownership_statement_properties = {'statementID': {'type': 'text'}, 'statementType': {'type': 'text'}, 'statementDate': {'type': 'text'}, + 'isComponent': {"type": "boolean"}, 'subject': {'type': 'object', 'properties': {'describedByEntityStatement': {'type': 'text'}}}, 'interestedParty': {'type': 'object', @@ -85,8 +108,42 @@ 'creationDate': {'type': 'text'}, 'createdBy': {'type': 'object', 'properties': {'name': {'type': 'text'}, - 'uri': {'type': 'text'}}}}}} + 'uri': {'type': 'text'}}}}}, + 'replacesStatements': {'type': 'text'} + } + + +# Additional indexes for managing updates +latest_properties = {'latest_id': {'type': 'text'}, + 'statement_id': {'type': 'text'}, + 'reason': {'type': 'text'}} +references_properties = {'statement_id': {'type': 'text'}, + 'references_id': {'type': 'object', + 'properties': {'statement_id': {'type': 'text'}, + 'latest_id': {'type': 'text'}}} + } +#updates_properties = {'referencing_id': {'type': 'text'}, +# 'old_statement_id': {'type': 'text'}, +# 'new_statement_id': {'type': 'text'}} +updates_properties = {'referencing_id': {'type': 'text'}, + 'latest_id': {'type': 'text'}, + 'updates': {'type': 'object', + 'properties': {'old_statement_id': {'type': 'text'}, + 'new_statement_id': {'type': 'text'}}} + #{'type': 'text'} + } +exceptions_properties = {'latest_id': {'type': 'text'}, + 'statement_id': {'type': 'text'}, + 'other_id': {'type': 'text'}, + 'reason': {'type': 'text'}, + 'reference': {'type': 'text'}, + 'entity_type': {'type': 'text'}} + +# Properties for logging pipeline runs +pipeline_run_properties = {'stage_name': {'type': 'text'}, + 'start_timestamp': {"type": "text"}, + 'end_timestamp': {"type": "text"}} def match_entity(item): return {"match": {"statementID": item["statementID"]}} @@ -97,6 +154,21 @@ def match_person(item): def match_ownership(item): return {"match": {"statementID": item["statementID"]}} +def match_latest(item): + return {"match": {"latest_id": item["latest_id"]}} + +def match_references(item): + return {"match": {"statement_id": item["statement_id"]}} + +def match_updates(item): + return {"match": {"old_statement_id": item["old_statement_id"]}} + +def match_exceptions(item): + return {"match": {"latest_id": item["latest_id"]}} + +def match_run(item): + return {"match": {"end_timestamp": item["end_timestamp"]}} + def id_entity(item): return item["statementID"] @@ -105,3 +177,28 @@ def id_person(item): def id_ownership(item): return item["statementID"] + +def id_latest(item): + return item["latest_id"] + +def id_references(item): + return item["statement_id"] + +def id_updates(item): + return item["referencing_id"] + +def id_exceptions(item): + return item["latest_id"] + +def id_run(item): + return item["end_timestamp"] + +# Elasticsearch indexes for BODS data +bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity}, + "person": {"properties": person_statement_properties, "match": match_person, "id": id_person}, + "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership}, + "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest}, + "references": {"properties": references_properties, "match": match_references, "id": id_references}, + "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates}, + "exceptions": {"properties": exceptions_properties, "match": match_exceptions, "id": id_exceptions}, + "runs": {"properties": pipeline_run_properties, "match": match_run, "id": id_run}} diff --git a/bodspipelines/infrastructure/inputs.py b/bodspipelines/infrastructure/inputs.py index 62f4895..d403312 100644 --- a/bodspipelines/infrastructure/inputs.py +++ b/bodspipelines/infrastructure/inputs.py @@ -6,8 +6,16 @@ def __init__(self, stream_name=None): self.stream_name = stream_name self.stream = KinesisStream(self.stream_name) - def process(self): - for records in self.stream.read_stream(): - for record in records: - if isinstance(record, dict): - yield record + async def process(self): + async for record in self.stream.read_stream(): + #for record in records: + # if isinstance(record, dict): + yield record + + async def setup(self): + if hasattr(self.stream, 'setup'): + await self.stream.setup() + + async def close(self): + if hasattr(self.stream, 'close'): + await self.stream.close() diff --git a/bodspipelines/infrastructure/memory_debugging.py b/bodspipelines/infrastructure/memory_debugging.py new file mode 100644 index 0000000..37fa533 --- /dev/null +++ b/bodspipelines/infrastructure/memory_debugging.py @@ -0,0 +1,18 @@ +import tracemalloc +import os, psutil +from loguru import logger + +# Configure log file +logger.add("memory.log") + +# Start tracing +tracemalloc.start() + +def log_memory(): + """Log current memory usage and largest objects""" + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('lineno') + + logger.info(f"Process size: {psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2} MB") + for stat in top_stats[:10]: + logger.info(stat) diff --git a/bodspipelines/infrastructure/outputs.py b/bodspipelines/infrastructure/outputs.py index a3f4154..fa9e1f8 100644 --- a/bodspipelines/infrastructure/outputs.py +++ b/bodspipelines/infrastructure/outputs.py @@ -47,12 +47,25 @@ def process(self, item, item_type): self.new_count += 1 #print(f"Processed: {self.processed_count}, New: {self.new_count}") - def process_stream(self, stream, item_type): + async def process_stream(self, stream, item_type): if self.identify: item_type = self.identify - for item in self.storage.process_batch(stream, item_type): + async for item in self.storage.process_batch(stream, item_type): if item: - self.output.process(item, item_type) - self.output.finish() + await self.output.process(item, item_type) + await self.output.finish() + + async def setup(self): + if hasattr(self.storage, 'setup'): + await self.storage.setup() + if hasattr(self.output, 'setup'): + await self.output.setup() + + async def close(self): + if hasattr(self.storage, 'close'): + await self.storage.close() + if hasattr(self.output, 'close'): + await self.output.close() + class KinesisOutput: """Output to Kinesis Stream""" @@ -61,8 +74,16 @@ def __init__(self, stream_name=None): self.stream_name = stream_name self.stream = KinesisStream(self.stream_name) - def process(self, item, item_type): - self.stream.add_record(item) + async def process(self, item, item_type): + await self.stream.add_record(item) + + async def finish(self): + await self.stream.finish_write() + + async def setup(self): + if hasattr(self.stream, 'setup'): + await self.stream.setup() - def finish(self): - self.stream.finish_write() + async def close(self): + if hasattr(self.stream, 'close'): + await self.stream.close() diff --git a/bodspipelines/infrastructure/pipeline.py b/bodspipelines/infrastructure/pipeline.py index 28d3b4f..60811c9 100644 --- a/bodspipelines/infrastructure/pipeline.py +++ b/bodspipelines/infrastructure/pipeline.py @@ -1,10 +1,14 @@ +import time +import asyncio from typing import List, Union from pathlib import Path -from bodspipelines.infrastructure.processing.bulk_data import BulkData -from bodspipelines.infrastructure.processing.xml_data import XMLData +#from bodspipelines.infrastructure.processing.bulk_data import BulkData +#from bodspipelines.infrastructure.processing.xml_data import XMLData -from bodspipelines.infrastructure.storage import ElasticStorage +#from bodspipelines.infrastructure.storage import ElasticStorage + +#from .memory_debugging import log_memory class Source: """Data source definition class""" @@ -14,15 +18,27 @@ def __init__(self, name=None, origin=None, datatype=None): self.origin = origin self.datatype = datatype - def process(self, stage_dir): + async def process(self, stage_dir, updates=False): """Iterate over source items""" if hasattr(self.origin, "prepare"): - data = self.origin.prepare(stage_dir, self.name) - for item in self.datatype.process(data): - yield item + for data in self.origin.prepare(stage_dir, self.name, updates=updates): + async for header, item in self.datatype.process(data): + yield header, item else: - for item in self.origin.process(): - yield self.datatype.process(item) + async for item in self.origin.process(): + header, item = self.datatype.process(item) + yield header, item + + async def setup(self): + """Run origin setup""" + if hasattr(self.origin, 'setup'): + await self.origin.setup() + + async def close(self): + """Close origin""" + if hasattr(self.origin, 'close'): + await self.origin.close() + class Stage: """Pipeline stage definition class""" @@ -32,7 +48,7 @@ def __init__(self, name=None, sources=None, processors=None, outputs=None): self.name = name self.sources = sources self.processors = processors - self.outputs = outputs + self.outputs = outputs def directory(self, parent_dir) -> Path: """Return subdirectory path after ensuring exists""" @@ -40,41 +56,74 @@ def directory(self, parent_dir) -> Path: path.mkdir(exist_ok=True) return path - def source_processing(self, source, stage_dir): + async def source_processing(self, source, stage_dir, updates=False): """Iterate over items from source, with processing""" - for item in source.process(stage_dir): + #count = 0 + async for header, item in source.process(stage_dir, updates=updates): + #print(header, item) if self.processors: + items = [item] for processor in self.processors: - for out in processor.process(item, source.name): - #print(out) - yield out + new_items = [] + for current_item in items: + #print("Processor:", processor) + async for out in processor.process(current_item, source.name, header, updates=updates): + #print(out) + #yield out + new_items.append(out) + items = new_items + for current_item in items: + yield current_item else: yield item + #count += 1 + #if count % 100000 == 0: + # log_memory() + yield {"flush": True} + await asyncio.sleep(5) + for processor in self.processors: + print("Processor:", hasattr(processor, "finish_updates"), updates) + if hasattr(processor, "finish_updates") and updates: + async for out in processor.finish_updates(updates=updates): + yield out - #def process_source(self, source, stage_dir): - # """Iterate over items from source, with processing and output""" - # for item in source.process(stage_dir): - # for processor in self.processors: - # item = processor.process(item, source.name) - - def process_source(self, source, stage_dir): + async def process_source(self, source, stage_dir, updates=False): """Iterate over items from source, and output""" + print("Process source:", len(self.outputs) > 1, not self.outputs[0].streaming) if len(self.outputs) > 1 or not self.outputs[0].streaming: - for item in self.source_processing(source, stage_dir): + print("Interating:") + async for item in self.source_processing(source, stage_dir, updates=updates): for output in self.outputs: output.process(item, source.name) else: - self.outputs[0].process_stream(self.source_processing(source, stage_dir), source.name) + print("Streaming:") + await self.outputs[0].process_stream(self.source_processing(source, stage_dir, updates=updates), + source.name) - def process(self, pipeline_dir): + async def process(self, pipeline_dir, updates=False): """Process all sources for stage""" print(f"Running {self.name} pipeline stage") stage_dir = self.directory(pipeline_dir) for source in self.sources: print(f"Processing {source.name} source") - self.process_source(source, stage_dir) + await self.process_source(source, stage_dir, updates=updates) print(f"Finished {self.name} pipeline stage") + async def setup(self): + """Setup stage components""" + for components in (self.sources, self.processors, self.outputs): + for component in components: + if hasattr(component, 'setup'): + await component.setup() + + async def close(self): + """Shutdown stage components""" + for components in (self.sources, self.processors, self.outputs): + for component in components: + if hasattr(component, 'close'): + await component.close() + + class Pipeline: """Pipeline definition class""" def __init__(self, name=None, stages=None): @@ -95,8 +144,16 @@ def get_stage(self, name): return stage return None - def process(self, stage_name): + async def process_stage(self, stage_name, updates=False): """Process specified pipeline stage""" stage = self.get_stage(stage_name) pipeline_dir = self.directory() - stage.process(pipeline_dir) + await stage.setup() + await stage.process(pipeline_dir, updates=updates) + await stage.close() + + def process(self, stage_name, updates=False): + """Process specified pipeline stage""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.process_stage(stage_name, updates=updates)) diff --git a/bodspipelines/infrastructure/processing/bulk_data.py b/bodspipelines/infrastructure/processing/bulk_data.py index 2309ff9..7f03be2 100644 --- a/bodspipelines/infrastructure/processing/bulk_data.py +++ b/bodspipelines/infrastructure/processing/bulk_data.py @@ -8,10 +8,10 @@ class BulkData: """Bulk data definition class""" - def __init__(self, display=None, url=None, size=None, directory=None): + def __init__(self, display=None, data=None, size=None, directory=None): """Initial setup""" self.display = display - self.url = url + self.data = data self.size = size self.directory = directory @@ -26,39 +26,73 @@ def manifest_file(self, path) -> Path: def create_manifest(self, path, name): """Create manifest file""" manifest_file = self.manifest_file(path) - if callable(self.url): - url = self.url(name) - else: - url = self.url + manifest = [] with open(manifest_file, "w") as outfile: - json.dump({"url": url, "timestamp": time.time()}, outfile) + for data in self.data.sources(): + if callable(data): + url = data(name) + else: + url = data + manifest.append({"url": url, "timestamp": time.time()}) + json.dump(manifest, outfile) - def check_manifest(self, path, name): - """Check manifest file exists and up-to-date""" + def read_manifest(self, path): + """Read manifest file if exists""" manifest_file = self.manifest_file(path) if manifest_file.exists(): with open(manifest_file, 'r') as openfile: try: - manifest = json.load(openfile) + return json.load(openfile) except json.decoder.JSONDecodeError: return False - if callable(self.url): - url = self.url(name) + else: + return None + + def source_data(self, name, last_update=None, delta_type=False): + """Yield urls for source""" + for data in self.data.sources(last_update=last_update, delta_type=delta_type): + if callable(data): + url = data(name) else: - url = self.url - if manifest["url"] == url and abs(manifest["timestamp"] - time.time()) < 24*60*60: - return True + url = data + yield url + + def check_manifest(self, path, name, updates=False): + """Check manifest file exists and up-to-date""" + #manifest_file = self.manifest_file(path) + manifest = self.read_manifest(path) + if updates and manifest: + d, t, *_ = manifest['url'].split('/')[-1].split('-') + last_update = f"{d[:4]}-{d[4:6]}-{d[6:8]}" + else: + last_update = False + if last_update: + if updates in ("month", "week", "day"): + # Special case for testing (usually True/False) + delta_type = updates else: - return False + delta_type = None + yield from self.source_data(name, last_update=last_update, delta_type=delta_type) else: - return False + if manifest: #manifest_file.exists(): + #with open(manifest_file, 'r') as openfile: + # try: + # manifest = json.load(openfile) + # except json.decoder.JSONDecodeError: + # return False + for data in self.data.sources(): + if callable(data): + url = data(name) + else: + url = data + match = [m for m in manifest if m["url"] == url] + if not match or abs(m["timestamp"] - time.time()) > 24*60*60: + yield url + else: + yield from self.source_data(name, last_update=last_update) - def download_large(self, directory, name): + def download_large(self, directory, name, url): """Download file to specified directory""" - if callable(self.url): - url = self.url(name) - else: - url = self.url with requests.get(url, stream=True) as r: r.raise_for_status() if 'content-disposition' in r.headers: @@ -78,28 +112,83 @@ def download_large(self, directory, name): def unzip_data(self, filename, directory): """Unzip specified file to directory""" with zipfile.ZipFile(filename, 'r') as zip_ref: - zip_ref.extractall(directory) + for fn in zip_ref.namelist(): + zip_ref.extract(fn, path=directory) + yield fn - def delete_old_data(self, directory): + def delete_old_data_all(self, directory): + """Delete all data files""" for file in directory.glob('*'): print(f"Deleting {file.name} ...") file.unlink() + def delete_old_data(self, directory, url): + """Delete filename for specified url""" + fn = url.rsplit('/', 1)[-1] + for file in directory.glob('*'): + print(file.name, fn) + if file.name == fn: + print(f"Deleting {file.name} ...") + file.unlink() + + def delete_unused_data(self, directory, files): + """Delete files not in list""" + for file in directory.glob('*'): + if not file.name in files: + print(f"Deleting {file.name} ...") + file.unlink() + + def delete_zip_data(self, directory, url): + """Delete filename for specified url""" + fn = url.rsplit('/', 1)[-1] + for file in directory.glob('*'): + if file.name == fn: + print(f"Deleting {file.name} ...") + file.unlink() + + def download_data(self, directory, name): + """Download data files""" + for data in self.data.sources(): + if callable(data): + url = data(name) + else: + url = data + zip = self.download_large(directory, name, url) + for fn in self.unzip_data(zip, directory): + yield fn + def download_extract_data(self, path, name): """Download and extract data""" - directory = self.data_dir(path) - directory.mkdir(exist_ok=True) + #directory = self.data_dir(path) + #directory.mkdir(exist_ok=True) self.delete_old_data(directory) - zip = self.download_large(directory, name) - self.unzip_data(zip, directory) + #zip = self.download_large(directory, name) + #self.unzip_data(zip, directory) + for fn in self.download_data(directory, name): + yield fn + + def download_extract_data(self, directory, name, url): + """Download and unzip data files""" + self.delete_old_data(directory, url) + zip = self.download_large(directory, name, url) + for fn in self.unzip_data(zip, directory): + self.delete_zip_data(directory, url) + yield fn - def prepare(self, path, name) -> Path: + def prepare(self, path, name, updates=False) -> Path: """Prepare data for use""" - if not self.check_manifest(path, name): - self.download_extract_data(path, name) - self.create_manifest(path, name) + directory = self.data_dir(path) + directory.mkdir(exist_ok=True) + files = [] + if list(directory.glob("*.xml")): # and not list(directory.glob("*golden-copy.xml")): + for f in directory.glob("*.xml"): + fn = f.name + files.append(fn) + yield directory / fn else: - print(f"{self.display} data up-to-date ...") - for file in self.data_dir(path).glob('*.xml'): - return file - + for url in self.check_manifest(path, name, updates=updates): + for fn in self.download_extract_data(directory, name, url): + files.append(fn) + yield directory / fn + print("Files:", files) + self.create_manifest(path, name) diff --git a/bodspipelines/infrastructure/processing/json_data.py b/bodspipelines/infrastructure/processing/json_data.py index cff85aa..bd587d3 100644 --- a/bodspipelines/infrastructure/processing/json_data.py +++ b/bodspipelines/infrastructure/processing/json_data.py @@ -4,4 +4,4 @@ class JSONData: def process(self, item): """Return item""" #print(item) - return item + return None, item diff --git a/bodspipelines/infrastructure/processing/xml_data.py b/bodspipelines/infrastructure/processing/xml_data.py index 23db3a6..37ef168 100644 --- a/bodspipelines/infrastructure/processing/xml_data.py +++ b/bodspipelines/infrastructure/processing/xml_data.py @@ -1,6 +1,21 @@ -from lxml import etree +import aiofiles +#from lxml import etree +import xml.etree.ElementTree as etree + + +async def stream_file(filename): + async with aiofiles.open(filename, mode='r') as f: + while True: + data = await f.read(65536) + if data: + yield data + else: + yield "" + break + def is_plural(tag, child_tag): + """Is tag name plural""" if tag == child_tag + "s": return True elif tag == child_tag + "es": @@ -9,69 +24,118 @@ def is_plural(tag, child_tag): return True return False + +def is_plural(tag, child_tag): + """Is tag name plural""" + if tag.endswith(child_tag + "s"): + return True + elif tag.endswith(child_tag + "es"): + return True + elif tag.endswith(child_tag[:-1] + 'ies'): + return True + return False + + +def get_tag(element, pos): + """Return tag name without namespace""" + for ns in pos: + if ns in element.tag: + return element.tag[pos[ns]:] + + +def handle_event(event, element, tag_name, skip, stack, pos, filter): + out = None + tag = get_tag(element, pos) + if event == 'start': + if skip: + pass + elif tag in filter: + skip = True + elif element.tag == tag_name or stack: + stack.append([element.tag, {}]) + elif event == 'end': + if skip: + if tag in filter: + skip = False + elif element.tag == tag_name: + element.clear() + # Also eliminate now-empty references from the root node to elem + #for ancestor in element.xpath('ancestor-or-self::*'): + # while ancestor.getprevious() is not None: + # del ancestor.getparent()[0] + elem = stack.pop() + out = elem[1] + elif stack: + elem = stack.pop() + if elem[1]: + val = elem[1] + else: + val = element.text + if stack[-1][1]: + if isinstance(stack[-1][1], list): + if 'type' in element.attrib: + if isinstance(val, dict): + val['type'] = element.attrib['type'] + stack[-1][1].append(val) + else: + stack[-1][1].append({'type': element.attrib['type'], tag: val}) + else: + stack[-1][1].append(val) + else: + stack[-1][1][tag] = val + else: + if is_plural(stack[-1][0], tag): + if 'type' in element.attrib: + if isinstance(val, dict): + val['type'] = element.attrib['type'] + stack[-1][1] = [val] + else: + stack[-1][1] = [{'type': element.attrib['type'], tag: val}] + else: + if isinstance(stack[-1][1], list): + stack[-1][1].append(val) + else: + stack[-1][1] = [val] + else: + stack[-1][1][tag] = val + return out, skip + + +async def data_stream(filename, tag_name, namespaces, filter=[]): + """Stream items from XML file""" + skip = False + stack = [] + pos = {namespaces[ns]: len(namespaces[ns])+2 for ns in namespaces} + parser = etree.XMLPullParser(('start', 'end',)) + async for chunk in stream_file(filename): + parser.feed(chunk) + for event, element in parser.read_events(): + out, skip = handle_event(event, element, tag_name, skip, stack, pos, filter) + if out: yield out + + class XMLData: - """XML data definition class""" + """XML data parser configuration""" - def __init__(self, item_tag=None, namespace=None, filter=[]): + def __init__(self, item_tag=None, header_tag=None, namespace=None, filter=[]): """Initial setup""" self.item_tag = item_tag + self.header_tag = header_tag self.namespace = namespace self.filter = filter - def data_stream(self, filename): - """Stream parsed XML elements from file""" - print(f"Parsing {filename}") - ns = self.namespace[next(iter(self.namespace))] - tag_name = f"{{{ns}}}{self.item_tag}" - for event, element in etree.iterparse(filename, events=('end',), tag=tag_name): - yield element - - def is_array(self, tag, child): - """Check if is array """ - child_tag = etree.QName(child[0]).localname - if is_plural(tag, child_tag): - #print("Array!!!!") - return True + async def extract_header(self, filename): + """Extract header""" + if self.header_tag: + tag_name = f"{{{self.namespace[next(iter(self.namespace))]}}}{self.header_tag}" + async for item in data_stream(filename, tag_name, self.namespace, filter=self.filter): + return item else: - return False - - def add_element(self, out, tag, value): - if not tag in self.filter and value: - out[tag] = value - - def process_item(self, item, out): - """Process XML item to dict""" - for child in item: - tag = etree.QName(child).localname - #print(tag, len(child)) - if len(child) > 0: - if self.is_array(tag, child): - parent_data = [] - else: - parent_data = {} - child_data = self.process_item(child, parent_data) - if isinstance(out, list): - out.append(child_data) - else: - self.add_element(out, tag, child_data) - #out[tag] = child_data - else: - try: - child_value = child.xpath("./text()", namespaces=self.namespace)[0] - except IndexError: - child_value = False - if isinstance(out, list): - out.append(child_value) - else: - self.add_element(out, tag, child_value) - #out[tag] = child_value - return out + return None - def process(self, filename): + async def process(self, filename): """Iterate over processed items from file""" - for element in self.data_stream(filename): - item = self.process_item(element, {}) - element.clear() - #print(item) - yield item - #break # Remove + header = await self.extract_header(filename) + tag_name = f"{{{self.namespace[next(iter(self.namespace))]}}}{self.item_tag}" + async for item in data_stream(filename, tag_name, self.namespace, filter=self.filter): + yield header, item diff --git a/bodspipelines/infrastructure/storage.py b/bodspipelines/infrastructure/storage.py index 6538d52..46a7786 100644 --- a/bodspipelines/infrastructure/storage.py +++ b/bodspipelines/infrastructure/storage.py @@ -1,116 +1,145 @@ from typing import List, Union, Optional from dataclasses import dataclass -from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +class Storage: + """Storage definition class""" -class ElasticStorage: - """Elasticsearch storage definition class""" - def __init__(self, indexes): - self.indexes = indexes - self.storage = ElasticsearchClient() - self.current_index = None + def __init__(self, storage): + """Initialise storage""" + self.storage = storage - def setup_indexes(self): - for index_name in self.indexes: - self.storage.create_index(index_name, self.indexes[index_name]['properties']) - - def create_action(self, index_name, item): - #print(index_name, item) - if callable(index_name): index_name = index_name(item) - #return {"create": { "_index" : index_name, "_id" : self.indexes[index_name]["id"](item)}} - return {"_id": self.indexes[index_name]["id"](item), '_index': index_name, '_op_type': 'create', "_source": item} - - def action_stream(self, stream, index_name): - for item in stream: - yield self.create_action(index_name, item) - - def batch_stream(self, stream, index_name): - for item in stream: - yield self.create_action(index_name, item), item - - def create_batch(self, batch): - def func(): - for i in batch: - yield i - return func(), batch - - def batch_stream(self, stream, index_name): - batch = [] - for item in stream: - batch.append(self.create_action(index_name, item)) - if len(batch) > 485: - yield self.create_batch(batch) - batch = [] - if len(batch) > 0: - yield self.create_batch(batch) + async def setup(self): + """Setup storage""" + await self.storage.setup() def list_indexes(self): + """List indexes""" return self.storage.list_indexes() def list_index_details(self, index_name): + """List details for specified index""" return self.storage.get_mapping(index_name) - def stats(self): - for index_name in self.indexes: - print(f"Index {index_name}:", self.storage.stats(index_name)) + async def statistics(self): + """Print storage statistics""" + print("Storage:") + print("") + statistics = await self.storage.statistics() + for index_name in statistics: + if index_name != "total": + print(f"{index_name} items:", statistics[index_name]) + print("") + print("Total items:", statistics["total"]) def set_index(self, index_name): + """Set current index""" self.current_index = index_name self.storage.set_index(index_name) def delete_index(self, index_name): + """Delete index""" self.current_index = index_name self.storage.set_index(index_name) self.storage.delete_index() def delete_all(self, index_name): + """Delete index""" self.current_index = index_name self.storage.set_index(index_name) self.storage.delete_index() self.storage.create_index(index_name, self.indexes[index_name]['properties']) - def add_item(self, item, item_type): - #print(item_type, self.indexes[item_type]) - query = self.indexes[item_type]['match'](item) - #print("Query:", query) - match = self.storage.search(query) - #print(match) - if not match['hits']['hits']: - out = self.storage.store_data(item) - #print(out) - return item + def create_action(self, index_name, item, action_type='create'): + """Build create action for item""" + if callable(index_name): index_name = index_name(item) + if action_type == 'delete': + return {"_id": self.storage.indexes[index_name]["id"](item), + '_index': index_name, + '_op_type': action_type} + else: + return {"_id": self.storage.indexes[index_name]["id"](item), + '_index': index_name, + '_op_type': action_type, + "_source": item} + + async def get_item(self, id, item_type): + """Get item from index""" + self.storage.set_index(item_type) + return await self.storage.get(id) + + async def add_item(self, item, item_type, overwrite=False): + """Add item to index""" + self.storage.set_index(item_type) + id = self.storage.indexes[item_type]['id'](item) + result = await self.storage.get(id) + #print(result) + if overwrite or not result: + if overwrite and result: + #print(f"Updating: {item}") + out = await self.storage.update_data(item, id) + return item + else: + #print(f"Creating: {item}") + #action = self.create_action(item_type, item) + out = await self.storage.store_data(item, id=id) + return item else: return False - def process(self, item, item_type): - if item_type != self.current_index: - self.set_index(item_type) - return self.add_item(item, item_type) - - def process_stream(self, stream, item_type): - for item in self.storage.bulk_store_data(self.action_stream(stream, item_type), item_type): - yield item + async def delete_item(self, id, item_type): + """Delete item with id in index""" + self.storage.set_index(item_type) + await self.storage.delete(id) + + async def stream_items(self, index): + """Stream items in index""" + async for item in self.storage.scan_index(index): + yield item['_source'] + + async def process(self, item, item_type): + """Add item to index""" + if item_type != self.storage.index_name: + self.storage.set_index(item_type) + return await self.add_item(item, item_type) + + async def create_batch(self, batch): + """Create iterator that yields batch""" + async def func(): + for i in batch: + yield i + return func(), batch - def process_batch(self, stream, item_type): + async def batch_stream(self, stream, index_name): + """Create stream of batched actions""" batch = [] - for action, item in self.batch_stream(stream, item_type): - batch.append(action) - batch.append(item) - if len(batch) > 499: - for item in self.storage.batch_store_data(batch, item_type): - yield item - batch = [] - - def process_batch(self, stream, item_type): - for actions, items in self.batch_stream(stream, item_type): - for item in self.storage.batch_store_data(actions, items, item_type): + async for item in stream: + if "flush" in item and item["flush"] is True: + if len(batch) > 0: + yield await self.create_batch(batch) + else: + batch.append(self.create_action(index_name, item)) + if len(batch) > 485: + yield await self.create_batch(batch) + batch = [] + if len(batch) > 0: + yield await self.create_batch(batch) + + async def process_batch(self, stream, item_type): + """Store items from stream in batches""" + async for actions, items in self.batch_stream(stream, item_type): + async for item in self.storage.batch_store_data(actions, items, item_type): yield item - #top_mem() - def query(self, index_name, query): - self.storage.set_index(index_name) - return self.storage.search(query) + async def setup_indexes(self): + """Setup indexes""" + await self.storage.setup_indexes() - def get_all(self, index_name): - self.storage.set_index(index_name) - return self.storage.search({'match_all': {}}) + async def add_batch(self, item_type, items): + #print(f"Write batch of {len(items)} item for {item_type}") + actions = [self.create_action(item_type, current_item[1], action_type=current_item[0]) + for current_item in items] + async for current_item in self.storage.batch_store_data(actions, actions, item_type): + pass + + async def dump_stream(self, index_name, action_type, items): + await self.storage.dump_stream(index_name, action_type, items) diff --git a/bodspipelines/infrastructure/updates.py b/bodspipelines/infrastructure/updates.py new file mode 100644 index 0000000..2781e50 --- /dev/null +++ b/bodspipelines/infrastructure/updates.py @@ -0,0 +1,452 @@ +from bodspipelines.pipelines.gleif.indexes import id_rr as rr_id +from bodspipelines.infrastructure.utils import (current_date_iso, generate_statement_id, + random_string, format_date) +from bodspipelines.infrastructure.caching import Caching + +def convert_rel_type(rel_type): + """Convert Relationship Type To Exception Type""" + conv = {"IS_DIRECTLY_CONSOLIDATED_BY": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "IS_ULTIMATELY_CONSOLIDATED_BY": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT"} + return conv[rel_type] + + +def convert_except_type(except_type): + """Convert Exception Type To Relationship Type""" + conv = {"DIRECT_ACCOUNTING_CONSOLIDATION_PARENT": "IS_DIRECTLY_CONSOLIDATED_BY", + "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT": "IS_ULTIMATELY_CONSOLIDATED_BY"} + return conv[except_type] + + +def build_latest(lei, bods_id, reason=False): + """Build latest object""" + return {'latest_id': lei, # Source id (e.g. LEI) + 'statement_id': bods_id, # Latest statement id + 'reason': reason} # Unused + + +def build_references(statement_id, referencing_ids): + """Build references object""" + return {'statement_id': statement_id, # Statement id + 'references_id': referencing_ids} # Referencing statement ids + + +def build_update(referencing_id, latest_id, updates): + """Build updates object""" + return {'referencing_id': referencing_id, + 'latest_id': latest_id, + 'updates': [{'old_statement_id': old, 'new_statement_id': updates[old]} for old in updates]} + + +def build_exception(latest, occ_id, other_id, reason, reference, entity_type): + """Build exception object""" + return {'latest_id': latest, 'statement_id': occ_id, 'other_id': other_id, 'reason': reason, + 'reference': reference, 'entity_type': entity_type} + + +async def latest_save(cache, lei, bods_id, reason=False, updates=False): + """Save latest statement id for LEI/RR/Repex""" + await cache.add(build_latest(lei, bods_id, reason=reason), + "latest", overwrite=True) + +async def latest_lookup(cache, lei, updates=False): + """Lookup latest statement id for LEI/RR/Repex""" + data = await cache.get(lei, "latest") + if data: + return data['statement_id'], data['reason'] + else: + return None, None + + +async def latest_delete(cache, old_statement_id, updates=False): + """Delete latest statement id for LEI/RR/Repex""" + await cache.delete(old_statement_id, "updates") + + +async def exception_save(cache, latest, ooc_id, other_id, reason, reference, entity_type, updates=False): + """Save latest exception""" + await cache.add(build_exception(latest, ooc_id, other_id, reason, reference, entity_type), + "exceptions", + overwrite=True) + +async def exception_lookup(cache, latest, updates=False): + """Lookup latest exception""" + data = await cache.get(latest, "exceptions") + if data: + return data['statement_id'], data['other_id'], data['reason'], data['reference'], data['entity_type'] + else: + return None, None, None, None, None + +async def exception_delete(cache, old_statement_id, updates=False): + """Delete exception id""" + await cache.delete(old_statement_id, "exceptions") + +def build_referencing(referencing_ids): + """Build referencing object""" + out = [] + for statement_id in referencing_ids: + out.append({'statement_id': statement_id, 'latest_id': referencing_ids[statement_id]}) + return out + +async def references_save(cache, statement_id, referencing_ids, updates=False, overwrite=False): + """Save list of statement ids referencing statement""" + await cache.add(build_references(statement_id, build_referencing(referencing_ids)), + "references", + overwrite=overwrite) + +def translate_references(references): + """Translate references""" + out = {} + if isinstance(references, list): + for ref in references: + out[ref['statement_id']] = ref['latest_id'] + else: + out[references['statement_id']] = references['latest_id'] + return out + +async def lookup_references(cache, statement_id, updates=False): + """Lookup list of statement ids referencing statement""" + data = await cache.get(statement_id, "references") + if data: + return translate_references(data['references_id']) + else: + return {} + +async def references_update(cache, referenced_id, statement_id, latest_id, updates=False): + """Update list of statement ids referencing statement""" + referencing_ids = await lookup_references(cache, referenced_id, updates=updates) + referencing_ids[statement_id] = latest_id + await references_save(cache, referenced_id, referencing_ids, updates=updates, overwrite=True) + +async def references_remove(cache, referenced_id, statement_id, updates=False): + """Update list of statement ids referencing statement""" + referencing_ids = await lookup_references(cache, referenced_id, updates=updates) + if statement_id in referencing_ids: del referencing_ids[statement_id] + await references_save(cache, referenced_id, referencing_ids, updates=updates, overwrite=True) + +async def updates_save(cache, referencing_id, latest_id, updates): + """Save statement to updates""" + await cache.add(build_update(referencing_id, latest_id, updates), "updates", overwrite=True) + +async def updates_delete(cache, old_statement_id, if_exists=False): + """Delete statement to updates""" + await cache.delete(old_statement_id, "updates", if_exists=if_exists) + +async def lookup_updates(cache, old_statement_id): + """Lookup statement to updates""" + data = await cache.get(old_statement_id, "updates") + if data: + return {update['old_statement_id']: update['new_statement_id'] for update in data['updates']} + else: + return {} + +async def updates_update(cache, referencing_id, latest_id, old_statement_id, new_statement_id): + """Save statement to update""" + updates = await lookup_updates(cache, referencing_id) + updates[old_statement_id] = new_statement_id + await updates_save(cache, referencing_id, latest_id, updates) + +async def process_updates(cache): + """Stream updates from index""" + async for update in cache.stream("updates"): + updates = {data['old_statement_id']: data['new_statement_id'] for data in update['updates']} + yield update['referencing_id'], update['latest_id'], updates + +async def retrieve_statement(storage, statement_type, statement_id): + """Retrive statement using statement_id""" + data = await storage.get_item(statement_id, statement_type) + return data + +def fix_statement_reference(statement, updates, latest_id): + """Update ownershipOrControlStatement with new_id""" + for old_id in updates: + new_id = updates[old_id] + if "subject" in statement and "describedByEntityStatement" in statement["subject"]: + if statement["subject"]["describedByEntityStatement"] == old_id: + statement["subject"]["describedByEntityStatement"] = new_id + if "interestedParty" in statement and "describedByEntityStatement" in statement["interestedParty"]: + if statement["interestedParty"]["describedByEntityStatement"] == old_id: + statement["interestedParty"]["describedByEntityStatement"] = new_id + old_statement_id = statement['statementID'] + subject = statement["subject"]["describedByEntityStatement"] + interested = statement["interestedParty"]["describedByEntityStatement"] + statement_id = generate_statement_id(f"{latest_id}_{subject}_{interested}", 'ownershipOrControlStatement') + statement['statementID'] = statement_id + return old_statement_id + +def source_id(data, id_name): + """Extract identifier for specified type""" + for id in data['identifiers']: + if id['scheme'] == id_name: + return id['id'] + return None + +def referenced_ids(statement, item=None): + """Collect statement ids referenced by statement""" + out = [] + if "subject" in statement and "describedByEntityStatement" in statement["subject"]: + out.append(statement["subject"]["describedByEntityStatement"]) + if ("interestedParty" in statement and "describedByEntityStatement" in statement["interestedParty"] and + (not item or item["Registration"]["RegistrationStatus"] == "PUBLISHED")): + out.append(statement["interestedParty"]["describedByEntityStatement"]) + return out + +async def calculate_mapping(cache, item, updates=False): + """Calculate mapping lei and latest statementID""" + out = {} + if "Relationship" in item: + for lei in (item["Relationship"]["StartNode"]["NodeID"], item["Relationship"]["EndNode"]["NodeID"]): + latest_id, _ = await latest_lookup(cache, lei, updates=updates) + if latest_id: out[lei] = latest_id + else: + latest_id, _ = await latest_lookup(cache, item["LEI"], updates=updates) + if latest_id: out[item["LEI"]] = latest_id + return out + +async def item_setup(cache, item, updates=False): + """Setup exception and relationship mapping""" + if "ExceptionCategory" in item: + except_lei = item["LEI"] + except_type = item["ExceptionCategory"] + except_reason = item["ExceptionReason"] + except_reference = item["ExceptionReference"] if "ExceptionReference" in item else None + old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type = \ + await exception_lookup(cache, f"{except_lei}_{except_type}", updates=updates) + else: + old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, except_type, \ + except_reason, except_reference = None, None, None, None, None, None, None, None, None + if "Relationship" in item or "ExceptionCategory" in item: + mapping = await calculate_mapping(cache, item, updates=updates) + else: + mapping = {} + return mapping, old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, \ + except_type, except_reason, except_reference + +async def process_entity_lei(statement_id, statement, item, lei, updates, mapping, data_type, cache): + """Process entity statement from LEI""" + if updates: + latest_id, _ = await latest_lookup(cache, lei, updates=updates) # Latest statement + if latest_id: + if item["Registration"]["RegistrationStatus"] == 'RETIRED': + update_date = item['Registration']['LastUpdateDate'] + statement = data_type.void_entity_retired(latest_id, + update_date, + item["LEI"], + item["Registration"]["RegistrationStatus"]) + statement_id = statement['statementID'] if statement else None + else: + data_type.add_replaces(statement, latest_id) # Add replaces statement + referencing_ids = await lookup_references(cache, latest_id, updates=updates) + else: + referencing_ids = [] + for ref_id in referencing_ids: + await updates_update(cache, + ref_id, + referencing_ids[ref_id], + latest_id, + statement_id) # Save statements to update + if statement_id: + await latest_save(cache, lei, statement_id, updates=updates) # Save new latest + return statement_id, statement + +async def process_entity_repex(statement_id, statement, item, except_lei, except_type, except_reason, + old_reason, old_other_id, old_entity_type, data_type, mapping, + cache, updates=False): + """Process entity statement from reporting exception""" + void_statement = None + if "Extension" in item and "Deletion" in item["Extension"]: + latest_id, _ = await latest_lookup(cache, f"{except_lei}_{except_type}_{except_reason}_entity", updates=updates) + if latest_id: + statement = data_type.void_entity_deletion(latest_id, + item["Extension"]["Deletion"]["DeletedAt"], + item["LEI"], + item["ExceptionReason"]) + statement_id = statement['statementID'] if statement else None + else: + statement, statement_id = None, None + elif (old_reason and except_reason != old_reason): + update_date = current_date_iso() + void_statement = data_type.void_entity_changed(old_other_id, + update_date, + old_entity_type, + except_lei, + old_reason) + if statement_id: + await latest_save(cache, f"{except_lei}_{except_type}_{except_reason}_entity", statement_id, updates=updates) + return statement_id, statement, void_statement + +async def process_ooc_rr(statement_id, statement, item, start, end, rel_type, entity_voided, + updates, data_type, cache): + """Process ownership or control statement for relationship""" + void_statement = None + if updates: + latest_id, _ = await latest_lookup(cache, f"{start}_{end}_{rel_type}", updates=updates) + if latest_id: + # Check if deleted + if "Extension" in item and "Deletion" in item["Extension"]: + statement = data_type.void_ooc_relationship_deletion(latest_id, + item["Extension"]["Deletion"]["DeletedAt"], + start, + end) + statement_id = statement['statementID'] if statement else None + elif item["Registration"]["RegistrationStatus"] == 'RETIRED': + update_date = item['Registration']['LastUpdateDate'] + statement = data_type.void_ooc_relationship_retired(latest_id, + update_date, + start, + end) + statement_id = statement['statementID'] if statement else None + else: + data_type.add_replaces(statement, latest_id) # Add replaces statement + await updates_delete(cache, latest_id, if_exists=True) + # Check if replacing exception + if rel_type in ("IS_DIRECTLY_CONSOLIDATED_BY", "IS_ULTIMATELY_CONSOLIDATED_BY"): + except_type = convert_rel_type(rel_type) + except_ooc_id, except_other_id, except_reason, except_ref, except_entity_type \ + = await exception_lookup(cache, f"{start}_{except_type}", updates=updates) + if except_ooc_id and not entity_voided: + update_date = current_date_iso() + void_statement = data_type.void_entity_replaced(except_other_id, + update_date, + except_entity_type, + start, + except_reason) + await exception_delete(cache, f"{start}_{except_type}", updates=updates) + await latest_save(cache, f"{start}_{except_type}_{except_reason}_entity", None, updates=updates) + # Update references + if statement_id: + ref_statement_ids = referenced_ids(statement, item=item) # Statements Referenced By OOC + for ref_id in ref_statement_ids: + if ref_id: + if latest_id: + await references_remove(cache, ref_id, latest_id, updates=updates) + await references_update(cache, ref_id, statement_id, f"{start}_{end}_{rel_type}", updates=updates) + # Save statementID in latest + if statement_id: + await latest_save(cache, f"{start}_{end}_{rel_type}", statement_id, updates=updates) + return statement_id, statement, void_statement + +async def process_ooc_repex(statement_id, statement, item, except_lei, except_type, except_reason, + except_reference, old_reason, old_reference, old_ooc_id, old_other_id, + data_type, cache, updates=False): + """Process ownership or control statement for reporting exception""" + if "Extension" in item and "Deletion" in item["Extension"]: + latest_id, _ = await latest_lookup(cache, f"{except_lei}_{except_type}_{except_reason}_ownership", updates=updates) + statement = data_type.void_ooc_exception_deletion(latest_id, + item["Extension"]["Deletion"]["DeletedAt"], + item["LEI"], + item["ExceptionReason"]) + statement_id = statement['statementID'] if statement else None + await updates_delete(cache, latest_id, if_exists=True) + elif (old_reason and except_reason != old_reason): + data_type.add_replaces(statement, old_ooc_id) # Add replaces statement + await updates_delete(cache, old_ooc_id, if_exists=True) + elif (old_reference and except_reference != old_reference): + data_type.add_replaces(statement, old_ooc_id) # Add replaces statement + if statement['statementID'] == old_other_id: + statement['statementID'] = generate_statement_id(statement['statementID'], "ownership") + statement_id = statement['statementID'] + await updates_delete(cache, old_ooc_id, if_exists=True) + if statement_id: + await latest_save(cache, f"{except_lei}_{except_type}_{except_reason}_ownership", statement_id, updates=updates) + return statement_id, statement + +class ProcessUpdates: + """Data processor definition class""" + def __init__(self, id_name=None, transform=None, updates=None, storage=None): + """Initial setup""" + self.transform = transform + self.updates = updates + self.id_name = id_name + self.storage = storage + self.cache = Caching(self.storage, batching=-1) + + async def setup(self): + """Load data into cache""" + await self.storage.setup() + await self.cache.load() + + async def process(self, item, item_type, header, updates=False): + """Process updates if applicable""" + print(f"Processing - updates: {updates}") + entity_voided = False + entity_type = None + mapping, old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, \ + except_type, except_reason, except_reference = await item_setup(self.cache, item, updates=updates) + async for statement in self.transform.process(item, item_type, header, mapping=mapping): + statement_id = statement['statementID'] + if statement['statementType'] in ('entityStatement', 'personStatement'): + entity_type = statement['statementType'] + if not "ExceptionCategory" in item: + lei = source_id(statement, self.id_name) # LEI for statement + statement_id, statement = await process_entity_lei(statement_id, + statement, + item, + lei, + updates, + mapping, + self.updates, + self.cache) + else: + statement_id, statement, void_statement = await process_entity_repex(statement_id, + statement, item, except_lei, except_type, except_reason, + old_reason, old_other_id, old_entity_type, self.updates, + mapping, self.cache, updates=updates) + if void_statement: + yield void_statement + entity_voided = True + other_id = statement_id + elif statement['statementType'] == 'ownershipOrControlStatement': + # Get identifying features + if "ExceptionCategory" in item: + start = except_lei + end = "None" + rel_type = convert_except_type(except_type) + else: + start = item["Relationship"]["StartNode"]["NodeID"] + end = item["Relationship"]["EndNode"]["NodeID"] + rel_type = item["Relationship"]["RelationshipType"] + # Save references + if not "ExceptionCategory" in item: + statement_id, statement, void_statement = await process_ooc_rr(statement_id, statement, item, + start, end, rel_type, entity_voided, + updates, self.updates, self.cache) + if void_statement: yield void_statement + else: + statement_id, statement = await process_ooc_repex(statement_id, statement, item, + except_lei, except_type, except_reason, + except_reference, old_reason, old_reference, + old_ooc_id, old_other_id, self.updates, + self.cache, updates=updates) + ooc_id = statement_id + if statement: yield statement + if "ExceptionCategory" in item: + except_lei = item["LEI"] + except_type = item["ExceptionCategory"] + except_reason = item["ExceptionReason"] + except_reference = item["ExceptionReference"] if "ExceptionReference" in item else None + await exception_save(self.cache, f"{except_lei}_{except_type}", ooc_id, + other_id, except_reason, except_reference, entity_type, updates=updates) + + async def finish_updates(self, updates=False): + """Process updates to referencing statements""" + print("In finish_updates") + data_type = self.updates + if updates: + done_updates = [] + async for ref_id, latest_id, todo_updates in process_updates(self.cache): + statement = await retrieve_statement(self.storage, "ownership", ref_id) + old_statement_id = fix_statement_reference(statement, todo_updates, latest_id) + statement_id = statement["statementID"] + data_type.add_replaces(statement, old_statement_id) + await latest_save(self.cache, latest_id, statement_id, updates=updates) + # Update references + ref_statement_ids = referenced_ids(statement) # Statements Referenced By OOC + for new_ref_id in ref_statement_ids: + if new_ref_id: + await references_remove(self.cache, new_ref_id, old_statement_id, updates=updates) + await references_update(self.cache, new_ref_id, statement_id, latest_id, updates=updates) + done_updates.append(old_statement_id) + yield statement + for statement_id in done_updates: + await updates_delete(self.cache, statement_id) + await self.cache.flush() diff --git a/bodspipelines/infrastructure/utils.py b/bodspipelines/infrastructure/utils.py index 0216fa6..e5ede00 100644 --- a/bodspipelines/infrastructure/utils.py +++ b/bodspipelines/infrastructure/utils.py @@ -1,8 +1,40 @@ +import datetime +import dateutil +import pytz +import string +import random +import hashlib +import uuid import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from functools import partial +def random_string(length): + """Generate random string of specified length""" + characters = string.ascii_letters + string.digits + return ''.join(random.choice(characters) for _ in range(length)) + +def format_date(d): + """Format date in ISO 8601""" + return dateutil.parser.isoparse(d).strftime("%Y-%m-%d") + + +def current_date_iso(): + """Generate current date in ISO 8601""" + return datetime.datetime.now(pytz.timezone('Europe/London')).strftime("%Y-%m-%d") + + +def generate_statement_id(name, role, version=None): + """Generate statement ID deterministically""" + if version: + seed = '-'.join([name, role, version]) + else: + seed = '-'.join([name, role]) + m = hashlib.md5() + m.update(seed.encode('utf-8')) + return str(uuid.UUID(m.hexdigest())) + def requests_retry_session( retries=3, @@ -10,6 +42,7 @@ def requests_retry_session( status_forcelist=(500, 502, 504), session=None, ): + """Requests session with automatic retries""" session = session or requests.Session() retry = Retry( total=retries, @@ -24,10 +57,40 @@ def requests_retry_session( return session +def download(url): + """Download url""" + with requests_retry_session().get(url) as r: + r.raise_for_status() + return r + + def download_delayed(url, func): + """Return partial download function""" def download(url, func, param): with requests_retry_session().get(url) as r: r.raise_for_status() out = func(r, param) return out return partial(download, url, func) + +def identify_bods(item): + """Identify type of BODS data""" + if item['statementType'] == 'entityStatement': + return 'entity' + elif item['statementType'] == 'personStatement': + return 'person' + elif item['statementType'] == 'ownershipOrControlStatement': + return 'ownership' + +async def load_last_run(storage, name=None): + """Load data about last pipeline run""" + runs = [] + async for run in storage.stream_items("runs"): + runs.append(run) + if name: + runs = [run for run in runs if run['stage_name'] == name] + return sorted(runs, key=lambda x: float(x["end_timestamp"]))[-1] + +async def save_run(storage, data): + """Save data about last pipeline run""" + await storage.add_item(data, "runs") diff --git a/bodspipelines/pipelines/gleif/annotations.py b/bodspipelines/pipelines/gleif/annotations.py new file mode 100644 index 0000000..8612f2e --- /dev/null +++ b/bodspipelines/pipelines/gleif/annotations.py @@ -0,0 +1,66 @@ +from bodspipelines.infrastructure.utils import current_date_iso + +def add_annotation(annotations, description, pointer): + """Add commenting annotation to statement""" + annotation = {'motivation': 'commenting', + 'description': description, + 'statementPointerTarget': pointer, + 'creationDate': current_date_iso(), + 'createdBy': {'name': 'Open Ownership', + 'uri': "https://www.openownership.org"}} + annotations.append(annotation) + +def add_repex_ooc_annotation(annotations): + """Annotation for all reporting exception ownership or control statements""" + add_annotation(annotations, + "The nature of this interest is unknown", + "/interests/0/type") + +def add_lei_annotation(annotations, lei, registration_status): + """Annotation of status for all entity statements (not generated as a result + of a reporting exception)""" + add_annotation(annotations, + f"GLEIF data for this entity - LEI: {lei}; Registration Status: {registration_status}", + "/") + +def add_rr_annotation_status(annotations, subject, interested): + """Annotation for all ownership or control statements""" + add_annotation(annotations, + f"Describes GLEIF relationship: {subject} is subject, {interested} is interested party", + "/") + +def add_annotation_retired(annotations): + """Annotation for retired statements""" + add_annotation(annotations, + "GLEIF RegistrationStatus set to RETIRED on this statementDate.", + "/") + +def add_rr_annotation_deleted(annotations): + """Annotation for deleted ownership or control statements""" + add_annotation(annotations, + "GLEIF relationship deleted on this statementDate.", + "/") + +def add_repex_annotation_reason(annotations, reason, lei): + """Annotation for all statements created as a reasult of a reporting exception""" + add_annotation(annotations, + f"This statement was created due to a {reason} GLEIF Reporting Exception for {lei}", + "/") + +def add_repex_annotation_changed(annotations, reason, lei): + """Annotation for all statements when reporting exception changed""" + add_annotation(annotations, + f"Statement retired due to change in a {reason} GLEIF Reporting Exception for {lei}", + "/") + +def add_repex_annotation_replaced(annotations, reason, lei): + """Annotation for all statements when reporting exception replaced with relationship""" + add_annotation(annotations, + f"Statement series retired due to replacement of a {reason} GLEIF Reporting Exception for {lei}", + "/") + +def add_repex_annotation_deleted(annotations, reason, lei): + """Annotation for ownership or control statement when reporting exception deleted""" + add_annotation(annotations, + f"Statement series retired due to deletion of a {reason} GLEIF Reporting Exception for {lei}", + "/") diff --git a/bodspipelines/pipelines/gleif/config.py b/bodspipelines/pipelines/gleif/config.py index c1ddf86..4d7fda6 100644 --- a/bodspipelines/pipelines/gleif/config.py +++ b/bodspipelines/pipelines/gleif/config.py @@ -1,129 +1,132 @@ +import os import time import elastic_transport +import asyncio +from datetime import datetime from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline from bodspipelines.infrastructure.inputs import KinesisInput -from bodspipelines.infrastructure.storage import ElasticStorage +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +from bodspipelines.infrastructure.clients.redis_client import RedisClient from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput from bodspipelines.infrastructure.processing.bulk_data import BulkData from bodspipelines.infrastructure.processing.xml_data import XMLData from bodspipelines.infrastructure.processing.json_data import JSONData +from bodspipelines.infrastructure.updates import ProcessUpdates -from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties, ownership_statement_properties, - match_entity, match_person, match_ownership, - id_entity, id_person, id_ownership) +from bodspipelines.pipelines.gleif.indexes import gleif_index_properties +from bodspipelines.infrastructure.indexes import bods_index_properties -from bodspipelines.pipelines.gleif.transforms import Gleif2Bods +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, AddContentDate, RemoveEmptyExtension from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, match_lei, match_rr, match_repex, id_lei, id_rr, id_repex) -from bodspipelines.pipelines.gleif.utils import gleif_download_link +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData, identify_gleif +from bodspipelines.pipelines.gleif.updates import GleifUpdates +from bodspipelines.infrastructure.utils import identify_bods, load_last_run, save_run # Defintion of LEI-CDF v3.1 XML date source lei_source = Source(name="lei", - origin=BulkData(display="LEI-CDF v3.1", - #url='https://leidata.gleif.org/api/v1/concatenated-files/lei2/get/30447/zip', - url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), - size=41491, - directory="lei-cdf"), - datatype=XMLData(item_tag="LEIRecord", - namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016"}, - filter=['NextVersion', 'Extension'])) + origin=BulkData(display="LEI-CDF v3.1", + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest", + data_date="2024-01-01"), + size=41491, + directory="lei-cdf"), + datatype=XMLData(item_tag="LEIRecord", + namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016", + "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"}, + filter=['NextVersion', 'Extension'])) # Defintion of RR-CDF v2.1 XML date source rr_source = Source(name="rr", origin=BulkData(display="RR-CDF v2.1", - #url='https://leidata.gleif.org/api/v1/concatenated-files/rr/get/30450/zip', - url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), - size=2823, - directory="rr-cdf"), + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/rr/latest", + data_date="2024-01-01"), + size=2823, + directory="rr-cdf"), datatype=XMLData(item_tag="RelationshipRecord", - namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"}, - filter=['Extension'])) + namespace={"rr": "http://www.gleif.org/data/schema/rr/2016", + "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"}, + filter=['NextVersion', ])) # Defintion of Reporting Exceptions v2.1 XML date source repex_source = Source(name="repex", origin=BulkData(display="Reporting Exceptions v2.1", - #url='https://leidata.gleif.org/api/v1/concatenated-files/repex/get/30453/zip', - url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), - size=3954, - directory="rep-ex"), + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/repex/latest", + data_date="2024-01-01"), + size=3954, + directory="rep-ex"), datatype=XMLData(item_tag="Exception", - namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"}, - filter=['NextVersion', 'Extension'])) + header_tag="Header", + namespace={"repex": "http://www.gleif.org/data/schema/repex/2016", + "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"}, + filter=['NextVersion', ])) -# Console Output -#output_console = Output(name="console", target=OutputConsole(name="gleif-ingest")) - -# Elasticsearch indexes for GLEIF data -index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, - "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, - "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} +# Easticsearch storage for GLEIF data +gleif_storage = ElasticsearchClient(indexes=gleif_index_properties) # GLEIF data: Store in Easticsearch and output new to Kinesis stream -output_new = NewOutput(storage=ElasticStorage(indexes=index_properties), - output=KinesisOutput(stream_name="gleif-prod")) +output_new = NewOutput(storage=Storage(storage=gleif_storage), + output=KinesisOutput(stream_name=os.environ.get('GLEIF_KINESIS_STREAM'))) # Definition of GLEIF data pipeline ingest stage ingest_stage = Stage(name="ingest", sources=[lei_source, rr_source, repex_source], - processors=[], - outputs=[output_new] -) + processors=[AddContentDate(identify=identify_gleif), + RemoveEmptyExtension(identify=identify_gleif)], + outputs=[output_new]) # Kinesis stream of GLEIF data from ingest stage gleif_source = Source(name="gleif", - origin=KinesisInput(stream_name="gleif-prod"), + origin=KinesisInput(stream_name=os.environ.get('GLEIF_KINESIS_STREAM')), datatype=JSONData()) -# Elasticsearch indexes for BODS data -bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity}, - "person": {"properties": person_statement_properties, "match": match_person, "id": id_person}, - "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership}} - -# Identify type of GLEIF data -def identify_gleif(item): - if 'Entity' in item: - return 'lei' - elif 'Relationship' in item: - return 'rr' - elif 'ExceptionCategory' in item: - return 'repex' - -# Identify type of BODS data -def identify_bods(item): - if item['statementType'] == 'entityStatement': - return 'entity' - elif item['statementType'] == 'personStatement': - return 'person' - elif item['statementType'] == 'ownershipOrControlStatement': - return 'ownership' +# Easticsearch storage for BODS data +bods_storage = ElasticsearchClient(indexes=bods_index_properties) # BODS data: Store in Easticsearch and output new to Kinesis stream -bods_output_new = NewOutput(storage=ElasticStorage(indexes=bods_index_properties), - output=KinesisOutput(stream_name="bods-gleif-prod"), +bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name=os.environ.get('BODS_KINESIS_STREAM')), identify=identify_bods) # Definition of GLEIF data pipeline transform stage transform_stage = Stage(name="transform", sources=[gleif_source], - processors=[Gleif2Bods(identify=identify_gleif)], - outputs=[bods_output_new] -) + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage), + updates=GleifUpdates())], + outputs=[bods_output_new]) # Definition of GLEIF data pipeline pipeline = Pipeline(name="gleif", stages=[ingest_stage, transform_stage]) -# Setup Elasticsearch indexes +# Setup storage indexes +async def setup_indexes(): + await gleif_storage.setup_indexes() + await bods_storage.setup_indexes() + +# Load run +async def load_previous(name): + bods_storage_run = ElasticsearchClient(indexes=bods_index_properties) + await bods_storage_run.setup() + storage_run = Storage(storage=bods_storage_run) + return await load_last_run(storage_run, name=name) + +# Save data on current pipeline run +async def save_current_run(name, start_timestamp): + bods_storage_run = ElasticsearchClient(indexes=bods_index_properties) + await bods_storage_run.setup() + storage_run = Storage(storage=bods_storage_run) + run_data = {'stage_name': name, + 'start_timestamp': str(start_timestamp), + 'end_timestamp': datetime.now().timestamp()} + await save_run(storage_run, run_data) + +# Setup pipeline storage def setup(): - done = False - while not done: - try: - ElasticStorage(indexes=index_properties).setup_indexes() - ElasticStorage(indexes=bods_index_properties).setup_indexes() - done = True - except elastic_transport.ConnectionError: - print("Waiting for Elasticsearch to start ...") - time.sleep(5) - -#stats = ElasticStorage(indexes=index_properties).stats + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(setup_indexes()) + diff --git a/bodspipelines/pipelines/gleif/indexes.py b/bodspipelines/pipelines/gleif/indexes.py index 76a2759..6972463 100644 --- a/bodspipelines/pipelines/gleif/indexes.py +++ b/bodspipelines/pipelines/gleif/indexes.py @@ -1,11 +1,16 @@ +import hashlib + # GLEIF LEI Elasticsearch Properties lei_properties = {'LEI': {'type': 'text'}, 'Entity': {'type': 'object', 'properties': {'LegalName': {'type': 'text'}, - 'OtherEntityNames': {'type': 'text'}, -# 'TransliteratedOtherEntityNames': {'type': 'object', -# 'properties': {'TransliteratedOtherEntityName': {'type': 'text'}}}, - 'TransliteratedOtherEntityNames': {'type': 'text'}, + 'OtherEntityNames': #{'type': 'text'}, + {'type': 'object', + 'properties': {'OtherEntityName': {'type': 'text'}, + 'type': {'type': 'text'}}}, + 'TransliteratedOtherEntityNames': {'type': 'object', + 'properties': {'TransliteratedOtherEntityName': {'type': 'text'}, + 'type': {'type': 'text'}}}, 'LegalAddress': {'type': 'object', 'properties': {'FirstAddressLine': {'type': 'text'}, 'AdditionalAddressLine': {'type': 'text'}, @@ -35,7 +40,8 @@ 'City': {'type': 'text'}, 'Region': {'type': 'text'}, 'Country': {'type': 'text'}, - 'PostalCode': {'type': 'text'}}}, + 'PostalCode': {'type': 'text'}, + 'type': {'type': 'text'}}}, 'TransliteratedOtherAddresses': {'type': 'object', 'properties': {'FirstAddressLine': {'type': 'text'}, 'AdditionalAddressLine': {'type': 'text'}, @@ -45,7 +51,8 @@ 'City': {'type': 'text'}, 'Region': {'type': 'text'}, 'Country': {'type': 'text'}, - 'PostalCode': {'type': 'text'}}}, + 'PostalCode': {'type': 'text'}, + 'type': {'type': 'text'}}}, 'RegistrationAuthority': {'type': 'object', 'properties': {'RegistrationAuthorityID': {'type': 'text'}, 'RegistrationAuthorityEntityID': {'type': 'text'}, @@ -117,12 +124,21 @@ 'properties': {'ValidationAuthorityID': {'type': 'text'}, 'OtherValidationAuthorityID': {'type': 'text'}, 'ValidationAuthorityEntityID': {'type': 'text'}}} - }}} + }}, + 'Extension': {'type': 'object', + 'properties': {'Deletion': {'type': 'object', + 'properties': {'DeletedAt': {'type': 'text'}}}}} + } repex_properties = {'LEI': {'type': 'text'}, 'ExceptionCategory': {'type': 'text'}, 'ExceptionReason': {'type': 'text'}, - 'ExceptionReference': {'type': 'text'}} + 'ExceptionReference': {'type': 'text'}, + 'ContentDate': {'type': 'text'}, + 'Extension': {'type': 'object', + 'properties': {'Deletion': {'type': 'object', + 'properties': {'DeletedAt': {'type': 'text'}}}}} + } def match_lei(item): return {"match": {"LEI": item["LEI"]}} @@ -131,24 +147,28 @@ def match_rr(item): return {'bool': {'must': [{"match": {'Relationship.StartNode.NodeID': item['Relationship']['StartNode']['NodeID']}}, {"match": {'Relationship.EndNode.NodeID': item['Relationship']['EndNode']['NodeID']}}, {"match": {'Relationship.RelationshipType': item['Relationship']['RelationshipType']}}]}} -#{"bool": {"must": [{"term": {'Relationship.StartNode.NodeID': item['Relationship']['StartNode']['NodeID']}}, -# {"term": {'Relationship.EndNode.NodeID': item['Relationship']['EndNode']['NodeID']}}, -# {"term": {'Relationship.RelationshipType': item['Relationship']['RelationshipType']}}]}} def match_repex(item): return {'bool': {'must': [{"match": {'LEI': item["LEI"]}}, {"match": {'ExceptionCategory': item["ExceptionCategory"]}}, {"match": {'ExceptionReason': item["ExceptionReason"]}}]}} -#{"bool": {"must": [{"term": {'ExceptionCategory': item["ExceptionCategory"]}}, -# {"term": {'ExceptionReason': item["ExceptionReason"]}}, -# {"term": {'LEI': item["LEI"]}}]}} def id_lei(item): - return item["LEI"] + return f"{item['LEI']}_{item['Registration']['LastUpdateDate']}" def id_rr(item): - return f"{item['Relationship']['StartNode']['NodeID']}_{item['Relationship']['EndNode']['NodeID']}_{item['Relationship']['RelationshipType']}" + return f"{item['Relationship']['StartNode']['NodeID']}_{item['Relationship']['EndNode']['NodeID']}_{item['Relationship']['RelationshipType']}_{item['Registration']['LastUpdateDate']}" def id_repex(item): - return f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}" + if "ExceptionReference" in item: + ref_hash = hashlib.sha256(bytes(item['ExceptionReference'], 'utf8')).hexdigest() + item_id = f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}_{ref_hash}_{item['ContentDate']}" + else: + item_id = f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}_None_{item['ContentDate']}" + #print(item_id, len(item_id), item) + return item_id +# Elasticsearch indexes for GLEIF data +gleif_index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, + "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, + "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} diff --git a/bodspipelines/pipelines/gleif/transforms.py b/bodspipelines/pipelines/gleif/transforms.py index 6b5f33f..50d5eec 100644 --- a/bodspipelines/pipelines/gleif/transforms.py +++ b/bodspipelines/pipelines/gleif/transforms.py @@ -7,49 +7,75 @@ from typing import List, Union import pycountry +#from bodspipelines.infrastructure.caching import cached +from bodspipelines.infrastructure.utils import format_date, current_date_iso, generate_statement_id +from .annotations import (add_lei_annotation, add_rr_annotation_status, + add_repex_annotation_reason, add_repex_ooc_annotation) +from .indexes import id_lei as entity_id +from .indexes import id_rr as rr_id +from .indexes import id_repex as repex_id + +#def entity_id(data): +# """Create ID for entity""" +# return f"{data['LEI']}_{data['Registration']['LastUpdateDate']}" + +#def rr_id(data): +# """Create ID for relationship""" +# return f"{data['Relationship']['StartNode']['NodeID']}_{data['Relationship']['EndNode']['NodeID']}_{data['Relationship']['RelationshipType']}_{data['Registration']['LastUpdateDate']}" + +#def repex_id(data): +# """Create ID for reporting exception""" +# if 'ExceptionReference' in data: +# return f"{data['LEI']}_{data['ExceptionCategory']}_{data['ExceptionReason']}_{data['ExceptionReference']}" +# else: +# return f"{data['LEI']}_{data['ExceptionCategory']}_{data['ExceptionReason']}_None" + +#async def subject_id(lei): +# async def get_item(x): +# return x +# get_item.__self__ = None +# data = await cached(get_item, lei, "latest", batch=False) +# return data['statement_id'] + def format_address(address_type, address): """Format address structure""" address_string = ", ".join([address['FirstAddressLine'], address['City']]) out = {'type': address_type,'address': address_string, 'country': address['Country']} if 'PostalCode' in address: out['postCode'] = address['PostalCode'] return out -def generate_statement_id(name, role, version=None): - """Generate statement ID deterministically""" - if version: - seed = '-'.join([name, role, version]) - else: - seed = '-'.join([name, role]) - m = hashlib.md5() - m.update(seed.encode('utf-8')) - return str(uuid.UUID(m.hexdigest())) - -def format_date(d): - """Format date in ISO 8601""" - return dateutil.parser.isoparse(d).strftime("%Y-%m-%d") #.isoformat(timespec='seconds') - -def current_date_iso(): - """Generate current date in ISO 8601""" - return datetime.datetime.now(pytz.timezone('Europe/London')).strftime("%Y-%m-%d") #.isoformat(timespec='seconds') def publication_details(): """Generate publication details""" return {'publicationDate': current_date_iso(), # TODO: fix publication date 'bodsVersion': "0.2", - 'license': "https://register.openownership.org/terms-and-conditions", - 'publisher': {"name": "OpenOwnership Register", - "url": "https://register.openownership.org"}} + 'license': "https://creativecommons.org/publicdomain/zero/1.0/", + 'publisher': {"name": "Open Ownership", + "url": "https://www.openownership.org"}} + +def jurisdiction_name(data): + try: + if "-" in data['Entity']['LegalJurisdiction']: + subdivision = pycountry.subdivisions.get(code=data['Entity']['LegalJurisdiction']) + name = f"{subdivision.name}, {subdivision.country.name}" + else: + name = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name + except AttributeError: + name = data['Entity']['LegalJurisdiction'] + return name def transform_lei(data): """Transform LEI-CDF v3.1 data to BODS statement""" - statementID = generate_statement_id(data['LEI'], 'entityStatement') + #print("Transforming LEI:") + statementID = generate_statement_id(entity_id(data), 'entityStatement') statementType = 'entityStatement' statementDate = format_date(data['Registration']['LastUpdateDate']) entityType = 'registeredEntity' name = data['Entity']['LegalName'] - try: - country = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name - except AttributeError: - country = data['Entity']['LegalJurisdiction'] + country = jurisdiction_name(data) + #try: + # country = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name + #except AttributeError: + # country = data['Entity']['LegalJurisdiction'] jurisdiction = {'name': country, 'code': data['Entity']['LegalJurisdiction']} identifiers = [{'id': data['LEI'], 'scheme':'XI-LEI', 'schemeName':'Global Legal Entity Identifier Index'}] if 'RegistrationAuthority' in data['Entity']: @@ -59,9 +85,6 @@ def transform_lei(data): if 'RegistrationAuthorityID' in data['Entity']['RegistrationAuthority']: authority['schemeName'] = data['Entity']['RegistrationAuthority']['RegistrationAuthorityID'] if authority: identifiers.append(authority) - #identifiers.append({'id': data['Entity']['RegistrationAuthority']['RegistrationAuthorityEntityID'], - # 'schemeName': data['Entity']['RegistrationAuthority']['RegistrationAuthorityID']}) - #foundingDate = data['Entity']['EntityCreationDate'] registeredAddress = format_address('registered', data['Entity']['LegalAddress']) businessAddress = format_address('business', data['Entity']['HeadquartersAddress']) if 'ValidationSources' in data['Registration']: @@ -69,6 +92,8 @@ def transform_lei(data): else: sourceType = ['officialRegister'] sourceDescription = 'GLEIF' + annotations = [] + add_lei_annotation(annotations, data['LEI'], data["Registration"]["RegistrationStatus"]) out = {'statementID': statementID, 'statementType': statementType, 'statementDate': statementDate, @@ -76,8 +101,8 @@ def transform_lei(data): 'name': name, 'incorporatedInJurisdiction': jurisdiction, 'identifiers': identifiers, - #'foundingDate': foundingDate, 'addresses': [registeredAddress,businessAddress], + 'annotations': annotations, 'publicationDetails': publication_details(), 'source': {'type':sourceType,'description':sourceDescription}} if 'EntityCreationDate' in data['Entity']: out['foundingDate'] = data['Entity']['EntityCreationDate'] @@ -92,20 +117,27 @@ def interest_level(relationship_type, default): else: return default # Other options in data -def transform_rr(data): +def calc_statement_id(lei, mapping): + """Calculate statementID for lei using mapping if available""" + #print("calc_statement_id:", lei, mapping) + if lei in mapping: + return mapping[lei] + else: + return generate_statement_id(lei, 'entityStatement') + +def transform_rr(data, mapping): """Transform RR-CDF v2.1 data to BODS statement""" - statementID = generate_statement_id(data['Relationship']['StartNode']['NodeID'] + - data['Relationship']['EndNode']['NodeID'] + - data['Relationship']['RelationshipType'], 'ownershipOrControlStatement') + statementID = generate_statement_id(rr_id(data), 'ownershipOrControlStatement') statementType = 'ownershipOrControlStatement' statementDate = format_date(data['Registration']['LastUpdateDate']) - subjectDescribedByEntityStatement = generate_statement_id(data['Relationship']['StartNode']['NodeID'], 'entityStatement') - interestedPartyDescribedByEntityStatement = generate_statement_id(data['Relationship']['EndNode']['NodeID'], 'entityStatement') - #interestType = 'otherInfluenceOrControl' + subjectDescribedByEntityStatement = calc_statement_id(data['Relationship']['StartNode']['NodeID'], mapping) + interestedPartyDescribedByEntityStatement = calc_statement_id(data['Relationship']['EndNode']['NodeID'], mapping) interestType = 'other-influence-or-control' - interestLevel = interest_level(data['Relationship']['RelationshipType'], 'unknown') + #interestLevel = interest_level(data['Relationship']['RelationshipType'], 'unknown') + interestLevel = "unknown" #periods = data['Relationship']['RelationshipPeriods'] interestStartDate = False + interestDetails = f"LEI RelationshipType: {data['Relationship']['RelationshipType']}" start_date = False if 'RelationshipPeriods' in data['Relationship']: periods = data['Relationship']['RelationshipPeriods'] @@ -122,6 +154,10 @@ def transform_rr(data): beneficialOwnershipOrControl = False sourceType = ['officialRegister'] if not data['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' else ['officialRegister', 'verified'] sourceDescription = 'GLEIF' + annotations = [] + add_rr_annotation_status(annotations, + data['Relationship']['StartNode']['NodeID'], + data['Relationship']['EndNode']['NodeID']) out = {'statementID': statementID, 'statementType':statementType, 'statementDate':statementDate, @@ -130,7 +166,9 @@ def transform_rr(data): 'interests':[{'type': interestType, 'interestLevel': interestLevel, 'beneficialOwnershipOrControl': beneficialOwnershipOrControl, - 'startDate': interestStartDate}], + 'startDate': interestStartDate, + 'details': interestDetails}], + 'annotations': annotations, 'publicationDetails': publication_details(), 'source':{'type': sourceType, 'description': sourceDescription}} return out @@ -143,14 +181,22 @@ def transform_repex_entity(data, description, person=False): else: statementType = 'entityStatement' entityType = "unknownEntity" - statementID = generate_statement_id(data['LEI'] + data['ExceptionCategory'] + data['ExceptionReason'], statementType) + statementID = generate_statement_id(repex_id(data), statementType) + statementDate = format_date(data['ContentDate']) #isComponent = 'false' Required? unspecified_reason = 'interested-party-exempt-from-disclosure' - unspecified_description = description + if "ExceptionReference" in data: + unspecified_description = f"{description} ExemptionReference provided: {data['ExceptionReference']}" + else: + unspecified_description = description sourceType = ['officialRegister'] sourceDescription = 'GLEIF' + annotations = [] + add_repex_annotation_reason(annotations, data["ExceptionReason"], data["LEI"]) out = {'statementID': statementID, 'statementType': statementType, + 'statementDate': statementDate, + 'annotations': annotations, 'publicationDetails': publication_details(), 'source':{'type':sourceType,'description':sourceDescription}} if person: @@ -161,29 +207,28 @@ def transform_repex_entity(data, description, person=False): out['unspecifiedEntityDetails'] = {'reason': unspecified_reason, 'description': unspecified_description} return out, statementID -def transform_repex_ooc(data, interested=None, person=False): +def transform_repex_ooc(data, mapping, interested=None, person=False): """Transform Reporting Exception to Ownership or Control statement""" - statementID = generate_statement_id(data['LEI'] + data['ExceptionCategory'] + data['ExceptionReason'], 'ownershipOrControlStatement') + statementID = generate_statement_id(repex_id(data), 'ownershipOrControlStatement') statementType = 'ownershipOrControlStatement' - subjectDescribedByEntityStatement = generate_statement_id(data['LEI'], 'entityStatement') + statementDate = format_date(data['ContentDate']) + #subjectDescribedByEntityStatement = generate_statement_id(data['LEI'], 'entityStatement') + subjectDescribedByEntityStatement = calc_statement_id(data['LEI'], mapping) if interested: interestedParty = interested else: interestedParty = data['ExceptionReason'] - #interestType = 'unknownInterest' interestType = 'other-influence-or-control' - annotation = {'motivation': 'commenting', - 'description': "The nature of this interest is unknown", - 'statementPointerTarget': "/interests/0/type", - 'creationDate': current_date_iso(), - 'createdBy': {'name': 'Open Ownership', - 'uri': "https://www.openownership.org"}} + annotations = [] + add_repex_ooc_annotation(annotations) + add_repex_annotation_reason(annotations, data["ExceptionReason"], data["LEI"]) if data['ExceptionCategory'] == "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT": interestLevel = 'indirect' elif data['ExceptionCategory'] == "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT": interestLevel = 'direct' else: interestLevel = 'unknown' + interestDetails = f"LEI ExceptionCategory: {data['ExceptionCategory']}" sourceType = ['officialRegister'] sourceDescription = 'GLEIF' if interested: @@ -195,69 +240,80 @@ def transform_repex_ooc(data, interested=None, person=False): interestedParty = {'unspecified': {'reason': interestedParty}} out = {'statementID': statementID, 'statementType':statementType, + 'statementDate': statementDate, 'subject': {'describedByEntityStatement': subjectDescribedByEntityStatement}, 'interestedParty': interestedParty, 'interests':[{'type': interestType, 'interestLevel': interestLevel, 'beneficialOwnershipOrControl': False, 'details': "A controlling interest."}], + 'annotations': annotations, 'publicationDetails': publication_details(), - 'source':{'type': sourceType, 'description': sourceDescription}, - 'annotations': [annotation]} + 'source':{'type': sourceType, 'description': sourceDescription}} return out, statementID -def transform_repex_no_lei(data): +def transform_repex_no_lei(data, mapping): """Transform NO_LEI Reporting Exception""" for func in (transform_repex_entity, transform_repex_ooc): if func == transform_repex_ooc: - statement, statement_id = func(data, interested=statement_id) + statement, statement_id = func(data, mapping, interested=statement_id) else: statement, statement_id = func(data, "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf.") yield statement -def transform_repex_natural_persons(data): +def transform_repex_natural_persons(data, mapping): """Transform NATURAL_PERSONS Reporting Exception""" for func in (transform_repex_entity, transform_repex_ooc): if func == transform_repex_ooc: - statement, statement_id = func(data, interested=statement_id, person=True) + statement, statement_id = func(data, mapping, interested=statement_id, person=True) else: statement, statement_id = func(data, "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity.", person=True) yield statement -def transform_repex_non_consolidating(data): +def transform_repex_non_consolidating(data, mapping): """Transform NON_CONSOLIDATING Reporting Exception""" for func in (transform_repex_entity, transform_repex_ooc): if func == transform_repex_ooc: - statement, statement_id = func(data, interested=statement_id) + statement, statement_id = func(data, mapping, interested=statement_id) else: statement, statement_id = func(data, "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control.") yield statement -def transform_repex_non_public(data): +def transform_repex_non_public(data, mapping): """Transform NON_PUBLIC Reporting Exception""" for func in (transform_repex_entity, transform_repex_ooc): if func == transform_repex_ooc: - statement, statement_id = func(data, interested=statement_id) + statement, statement_id = func(data, mapping, interested=statement_id) else: statement, statement_id = func(data, "From LEI ExemptionReason `NON_PUBLIC` or related deprecated values. The legal entity’s relationship information with an entity it controls is non-public. There are therefore obstacles to releasing this information.") yield statement -def transform_repex(data): +def transform_repex_no_known(data, mapping): + """Transform NO_KNOWN_PERSON Reporting Exception""" + for func in (transform_repex_entity, transform_repex_ooc): + if func == transform_repex_ooc: + statement, statement_id = func(data, mapping, interested=statement_id, person=True) + else: + statement, statement_id = func(data, "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity.", person=True) + yield statement + +def transform_repex(data, mapping): """Transform Reporting Exceptions to BODS statements""" - #print(data['ExceptionReason']) if data['ExceptionReason'] == "NO_LEI": - #print("Got here!") - for statement in transform_repex_no_lei(data): + for statement in transform_repex_no_lei(data, mapping): yield statement elif data['ExceptionReason'] == "NATURAL_PERSONS": - for statement in transform_repex_natural_persons(data): + for statement in transform_repex_natural_persons(data, mapping): yield statement elif data['ExceptionReason'] == "NON_CONSOLIDATING": - for statement in transform_repex_non_consolidating(data): + for statement in transform_repex_non_consolidating(data, mapping): yield statement elif data['ExceptionReason'] in ('NON_PUBLIC', 'BINDING_LEGAL_COMMITMENTS', 'LEGAL_OBSTACLES', - 'DISCLOSURE_DETRIMENTAL', 'DETRIMENT_NOT_EXCLUDED', 'DETRIMENT_NOT_EXCLUDED'): - for statement in transform_repex_non_public(data): + 'DISCLOSURE_DETRIMENTAL', 'DETRIMENT_NOT_EXCLUDED', 'CONSENT_NOT_OBTAINED'): + for statement in transform_repex_non_public(data, mapping): + yield statement + elif data['ExceptionReason'] == "NO_KNOWN_PERSON": + for statement in transform_repex_no_known(data, mapping): yield statement class Gleif2Bods: @@ -266,12 +322,41 @@ def __init__(self, identify=None): """Initial setup""" self.identify = identify - def process(self, item, item_type): + async def process(self, item, item_type, header, mapping={}, updates=False): + """Process item""" if self.identify: item_type = self.identify(item) + #print("Gleif2Bods:", item_type) if item_type == 'lei': yield transform_lei(item) elif item_type == 'rr': - yield transform_rr(item) + yield transform_rr(item, mapping) elif item_type == 'repex': - yield from transform_repex(item) + for statement in transform_repex(item, mapping): + yield statement +class AddContentDate: + """Data processor to add ContentDate""" + def __init__(self, identify=None): + """Initial setup""" + self.identify = identify + + async def process(self, item, item_type, header, mapping={}, updates=False): + """Process item""" + if self.identify: item_type = self.identify(item) + if item_type == 'repex': + item["ContentDate"] = header["ContentDate"] + yield item + +class RemoveEmptyExtension: + """Data processor to remove empty Extension""" + def __init__(self, identify=None): + """Initial setup""" + self.identify = identify + + async def process(self, item, item_type, header, mapping={}, updates=False): + """Process item""" + if self.identify: item_type = self.identify(item) + if item_type == 'repex': + if "Extension" in item and not isinstance(item["Extension"], dict): + del item["Extension"] + yield item diff --git a/bodspipelines/pipelines/gleif/updates.py b/bodspipelines/pipelines/gleif/updates.py new file mode 100644 index 0000000..eb12d17 --- /dev/null +++ b/bodspipelines/pipelines/gleif/updates.py @@ -0,0 +1,149 @@ +from bodspipelines.infrastructure.utils import (current_date_iso, generate_statement_id, + random_string, format_date) +from .annotations import (add_lei_annotation, add_repex_annotation_changed, + add_repex_annotation_replaced, add_repex_ooc_annotation, add_rr_annotation_deleted, + add_annotation_retired, add_rr_annotation_status, add_repex_annotation_deleted) + +def void_entity_statement(reason, statement_id, status, update_date, entity_type, lei, unknown=False): + """Create BODS statemnt to void entity/person""" + id = generate_statement_id(f"{statement_id}", "voided") + #print(f"Voiding entity ({statement_id}):", id) + statement = {"statementID": id, + "statementDate": format_date(update_date), + "publicationDetails": { + "publicationDate": current_date_iso(), + "bodsVersion": "0.2", + "publisher": { + "name": "GLEIF" + } + }, + "statementType": entity_type, + #"entityType": "registeredEntity", + "isComponent": False, + "replacesStatements": [statement_id] + } + if entity_type == "entityStatement": + if unknown: + statement["entityType"] = "unknownEntity" + else: + statement["entityType"] = "registeredEntity" + else: + statement["personType"] = "unknownPerson" + annotations = [] + if reason == "RETIRED": + add_lei_annotation(annotations, lei, status) + elif reason == "DELETION": + add_repex_annotation_deleted(annotations, status, lei) + elif reason == "CHANGED": + add_repex_annotation_changed(annotations, status, lei) + elif reason == "REPLACED": + add_repex_annotation_replaced(annotations, status, lei) + statement["annotations"] = annotations + return statement + +def void_ooc_statement(reason, statement_id, status, update_date, lei, lei2): + """Create BODS statement to void ownership or control statement""" + id = generate_statement_id(f"{statement_id}", "voided_ownershipOrControlStatement") + #print(f"Voiding OOC ({statement_id}):", id) + statement = {"statementID": id, + "statementDate": format_date(update_date), + "publicationDetails": { + "publicationDate": current_date_iso(), + "bodsVersion": "0.2", + "publisher": { + "name": "GLEIF" + } + }, + "statementType": "ownershipOrControlStatement", + "interestedParty":{ + "describedByEntityStatement": "" + }, + "isComponent": False, + "subject": { + "describedByEntityStatement": "" + }, + "replacesStatements": [statement_id] + } + annotations = [] + if reason == "RR_DELETION": + add_rr_annotation_deleted(annotations) + add_rr_annotation_status(annotations, lei, lei2) + elif reason == "RETIRED": + add_annotation_retired(annotations) + add_rr_annotation_status(annotations, lei, lei2) + elif reason == "REPEX_DELETION": + add_repex_annotation_deleted(annotations, status, lei) + statement["annotations"] = annotations + return statement + +class GleifUpdates(): + """GLEIF specific updates""" + + def __init__(self): + self.already_voided = [] + self.already_replaced = [] + + def void_entity_retired(self, latest_id, update_date, lei, status): + """Void entity statement for retired LEI""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_entity_statement("RETIRED", latest_id, status, update_date, "entityStatement", lei) + + def void_entity_deletion(self, latest_id, update_date, lei, status): + """Void entity statement for deleted reporting exception""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_entity_statement("DELETION", latest_id, status, update_date, "entityStatement", lei, unknown=True) + + def void_entity_changed(self, latest_id, update_date, entity_type, lei, status): + """Void entity statement for changed reporting exception""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_entity_statement("CHANGED", latest_id, status, update_date, entity_type, lei, unknown=True) + + def void_ooc_relationship_deletion(self, latest_id, update_date, start, end): + """Void ownership or control statement for deleted relationship record""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_ooc_statement("RR_DELETION", latest_id, None, update_date, start, end) + + def void_ooc_relationship_retired(self, latest_id, update_date, start, end): + """Void ownership or control statement for retired relationship record """ + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_ooc_statement("RETIRED", latest_id, None, update_date, start, end) + + def void_entity_replaced(self, latest_id, update_date, entity_type, lei, status): + """Void entity statement for replaced (by relationship) reporting exception""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_entity_statement("REPLACED", latest_id, status, update_date, entity_type, lei, unknown=True) + + def void_ooc_exception_deletion(self, latest_id, update_date, lei, status): + """Void ownership or control statement for deleted reporting exception""" + if latest_id in self.already_voided or latest_id in self.already_replaced: + return None + else: + self.already_voided.append(latest_id) + return void_ooc_statement("REPEX_DELETION", latest_id, status, update_date, lei, None) + + def add_replaces(self, statement, old_statement_id): + """Add replacesStatements to statement object""" + if old_statement_id in self.already_replaced: + return None + else: + self.already_replaced.append(old_statement_id) + statement["replacesStatements"] = [old_statement_id] + return statement diff --git a/bodspipelines/pipelines/gleif/utils.py b/bodspipelines/pipelines/gleif/utils.py index 70bf062..8f34cb8 100644 --- a/bodspipelines/pipelines/gleif/utils.py +++ b/bodspipelines/pipelines/gleif/utils.py @@ -1,15 +1,240 @@ -from bodspipelines.infrastructure.utils import download_delayed +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta -def get_source(r, name): +from bodspipelines.infrastructure.utils import download_delayed, download + +def source_metadata(r): + """Get metadata from request""" data = r.json() - if name == 'lei': - url = data['data']['lei2']['full_file']['xml']['url'] - elif name == 'rr': - url = data['data']['rr']['full_file']['xml']['url'] - elif name == 'repex': - url = data['data']['repex']['full_file']['xml']['url'] + return data['data'] + + +def get_source(r, name): + """Extract source url from metadata""" + data = source_metadata(r) + url = data['full_file']['xml']['url'] print(f"Using: {url}") return url +def step_date(date, time, period_name, count): + """Step date forward based on delta file type""" + if period_name == 'IntraDay': + d = datetime.strptime(f"{date} {time}", "%Y%m%d %H%M") + delta = relativedelta(hours=-8*count) + return (d + delta).strftime("%Y%m%d"), (d + delta).strftime("%H%M") + else: + d = datetime.strptime(date, "%Y%m%d") + if period_name == "LastMonth": + delta = relativedelta(months=-1) + elif period_name == "LastWeek": + delta = relativedelta(days=-7) + elif period_name == "LastDay": + delta = relativedelta(days=-1) + return (d + delta).strftime("%Y%m%d"), time + + +def delta_days(date1, date2): + """Days between two dates""" + delta = datetime.strptime(date2, "%Y%m%d") - datetime.strptime(date1, "%Y%m%d") + return delta.days + + +def step_period(data, base, period_name, count): + """Step through count periods""" + for _ in range(count): + url = data['delta_files'][period_name]['xml']['url'] + date, time, *_ = url.split('/')[-1].split('-') + date, time = step_date(date, time, period_name, 1) + request = download(f"{base}/{date}-{time}") + data = source_metadata(request) + yield url, data + + +def get_sources_int(data, base, last_update): + """Get urls for sources (internal)""" + current_date = data['publish_date'] + print(current_date, last_update) + delta = relativedelta(datetime.strptime(current_date, "%Y-%m-%d %H:%M:%S"), + datetime.strptime(last_update, "%Y-%m-%d %H:%M:%S")) + print(current_date, delta) + periods = {'months': 'LastMonth', 'days': 'LastDay', 'hours': 'IntraDay'} + done = 0 + if getattr(delta, 'months') > 0: + for url, data in step_period(data, base, 'LastMonth', 1): + yield url + done = -1 + for period in periods: + if done < 0: + if period == 'days': + if getattr(delta, period) > 0: + for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)*3): + yield url + elif period == 'hours': + if getattr(delta, period) > 0: + for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)//8): + yield url + else: + if getattr(delta, period) - done > 0: + date1, time, *_ = url.split('/')[-1].split('-') + date2, time = step_date(date1, time, 'LastMonth', getattr(delta, period) - done) + month_days = delta_days(date1, date2) + for url, data in step_period(data, base, 'IntraDay', month_days*3): + yield url + else: + if period == 'days': + if getattr(delta, period) > 6: + for url, data in step_period(data, base, 'LastWeek', getattr(delta, period)//7): + yield url + extra_days = getattr(delta, period)%7 + else: + extra_days = getattr(delta, period) + if extra_days > 0: + for url, data in step_period(data, base, 'LastDay', extra_days): + yield url + elif period == 'hours': + if getattr(delta, period) > 0: + for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)//8): + yield url + else: + if getattr(delta, period) > 0: + for url, data in step_period(data, base, periods[period], getattr(delta, period)): + yield url + +def get_sources(url, last_update): + """Get urls for sources""" + base = url.rsplit('/', 1)[0] + request = download(url) + data = source_metadata(request) + yield from get_sources_int(data, base, last_update) + +def get_data_type(url): + """Get type of data""" + if "/lei" in url: + data_type = "lei2" + elif "/rr" in url: + data_type = "rr" + elif "/repex" in url: + data_type = "repex" + return data_type + +def get_url_for_target_date(url, data, target_date, target_time, delta_type=None): + """Get url for specified date""" + for item in data: + item_date = datetime.strptime(item["publish_date"].split()[0], "%Y-%m-%d") + item_time = item["publish_date"].split()[1] + if item_date == target_date and item_time == target_time: + data_type = get_data_type(url) + if delta_type: + if delta_type == "month": + return item[data_type]["delta_files"]["LastMonth"]["xml"]["url"] + elif delta_type == "week": + return item[data_type]["delta_files"]["LastWeek"]["xml"]["url"] + elif delta_type == "day": + return item[data_type]["delta_files"]["LastDay"]["xml"]["url"] + else: + return item[data_type]["full_file"]["xml"]["url"] + +def find_urls_for_target_date(target_date): + """Find urls for specified date""" + page = 1 + while True: + page_url = f"https://goldencopy.gleif.org/api/v2/golden-copies/publishes?page={page}&per_page=10" + response = download(page_url) + data = source_metadata(response) + end_date = datetime.strptime(data[0]["publish_date"].split()[0], "%Y-%m-%d") + start_date = datetime.strptime(data[9]["publish_date"].split()[0], "%Y-%m-%d") + if target_date < start_date: + delta = start_date - target_date + print(delta.days) + page = page + max(1, int(delta.days/3)) + elif target_date > end_date: + delta = target_date - end_date + print(delta.days) + page = page - max(1, int(delta.days/3)) + else: + return data, page + +def get_source_by_date(url, data_date, delta_type=None): + """Get source data url for specified date""" + target_date = datetime.strptime(data_date.split()[0], "%Y-%m-%d") + if len(data_date.split()) > 1 and ":" in data_date.split()[1]: + target_time = data_date.split()[1] + else: + target_time = "00:00:00" + if delta_type == "month": + target_date = target_date + timedelta(days=31) + elif delta_type == "week": + target_date = target_date + timedelta(days=7) + elif delta_type == "day": + target_date = target_date + timedelta(days=1) + data, page = find_urls_for_target_date(target_date) + target_url = get_url_for_target_date(url, data, target_date, target_time, delta_type=delta_type) + return target_url + +def stream_delta_file(item, target_date, target_time, data_type): + """Yield delta file""" + item_date = datetime.strptime(item["publish_date"].split()[0], "%Y-%m-%d") + item_time = item["publish_date"].split()[1] + if item_date > target_date or (item_date == target_date and + int(item_time.split(":")[0]) > int(target_time.split(":")[0])): + yield item[data_type]["delta_files"]["IntraDay"]["xml"]["url"] + +def stream_delta_files(url, data, target_date, target_time, page): + """Yield delta files""" + data_type = get_data_type(url) + for item in data: + yield from stream_delta_file(item, target_date, target_time, data_type) + page = page - 1 + while page > 0: + page_url = f"https://goldencopy.gleif.org/api/v2/golden-copies/publishes?page={page}&per_page=10" + response = download(page_url) + data = source_metadata(response) + for item in data: + yield from stream_delta_file(item, target_date, target_time, data_type) + page = page - 1 + +def get_source_from_date(url, data_date, delta_type=None): + """Get source data url for specified date""" + target_date = datetime.strptime(data_date.split()[0], "%Y-%m-%d") + if len(data_date.split()) > 1 and ":" in data_date.split()[1]: + target_time = data_date.split()[1] + else: + target_time = "00:00:00" + data, page = data, page = find_urls_for_target_date(target_date) + yield from stream_delta_files(url, data, target_date, target_time, page) + def gleif_download_link(url): + """Returns callable to download data""" return download_delayed(url, get_source) + +class GLEIFData: + def __init__(self, url=None, data_date=None): + """Initialise with url""" + self.url = url + self.data_date = data_date + + def sources(self, last_update=False, delta_type=None): + """Yield data sources""" + if last_update: + print("Updating ...") + if delta_type: + if delta_type == "stream": + yield from get_source_from_date(self.url, self.data_date, delta_type=delta_type) + else: + yield get_source_by_date(self.url, self.data_date, delta_type=delta_type) + else: + yield from get_sources(self.url, last_update) + else: + if self.data_date: + yield get_source_by_date(self.url, self.data_date) + else: + yield gleif_download_link(self.url) + +def identify_gleif(item): + """Identify type of GLEIF data""" + if 'Entity' in item: + return 'lei' + elif 'Relationship' in item: + return 'rr' + elif 'ExceptionCategory' in item: + return 'repex' diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..0170318 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --docker-compose=tests/docker-compose.yml diff --git a/requirements.txt b/requirements.txt index 7c2b518..9fd8bcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,20 @@ -boto3==1.26.66 -elasticsearch==7.17.9 -elastic_transport==7.16.0 +aiobotocore==2.13.1 +elasticsearch==8.12.1 +elastic_transport==8.12.0 +aiohttp==3.9.3 lxml==4.9.2 parsel==1.7.0 progress==1.6 -pytest==7.2.1 +pytest==7.2.2 +pytest-docker-compose-v2==0.1.1 +pytest-asyncio==0.21.1 python-dateutil==2.8.2 requests==2.28.2 tqdm==4.64.1 pytz==2022.7.1 pycountry==22.3.5 +redis==4.6.0 +aiofiles==24.1.0 +# Debugging +loguru==0.7.2 +psutil==5.9.8 diff --git a/setup.py b/setup.py index 2586564..34957c6 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from setuptools.command.install import install install_requires = [ + "async-kinesis", "boto3", "elasticsearch", "elastic_transport", @@ -14,7 +15,12 @@ "requests", "tqdm", "pytz", - "pycountry" + "pycountry", + "aiohttp", + "aiofiles", + "redis", + "psutil", + "loguru" ] setup( diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 0000000..b05ff28 --- /dev/null +++ b/tests/config.py @@ -0,0 +1,11 @@ +import os + +def set_environment_variables(): + """Set environment variables""" + os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' + os.environ['ELASTICSEARCH_HOST'] = 'localhost' + os.environ['ELASTICSEARCH_PORT'] = '9876' + os.environ['ELASTICSEARCH_PASSWORD'] = '********' + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..5d23eb3 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.9' + +services: + bods_ingester_gleif_es: + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.1 + environment: + - 'discovery.type=single-node' + - 'cluster.name=gleif-elasticsearch' + - 'bootstrap.memory_lock=true' + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + - 'xpack.security.enabled=false' + ports: + - 9200:9200 diff --git a/tests/fixtures/lei-data-out.json b/tests/fixtures/lei-data-out.json new file mode 100644 index 0000000..637fff5 --- /dev/null +++ b/tests/fixtures/lei-data-out.json @@ -0,0 +1,262 @@ +[ + { + "statementID": "04c98917-93f2-bd6d-1c9b-40411dc9b8b3", + "statementType": "entityStatement", + "statementDate": "2023-06-18", + "entityType": "registeredEntity", + "name": "Fidelity Advisor Leveraged Company Stock Fund", + "incorporatedInJurisdiction": { + "name": "US-MA", + "code": "US-MA" + }, + "identifiers": [ + { + "id": "001GPB6A9XPE8XJICC14", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "S000005113", + "schemeName": "RA000665" + } + ], + "addresses": [ + { + "type": "registered", + "address": "245 SUMMER STREET, BOSTON", + "country": "US", + "postCode": "02210" + }, + { + "type": "business", + "address": "C/O Fidelity Management & Research Company LLC, Boston", + "country": "US", + "postCode": "02210" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-11-29T00:00:00.000Z" + }, + { + "statementID": "41c4522b-f5e2-4f19-5f6b-ec5e54258401", + "statementType": "entityStatement", + "statementDate": "2022-08-22", + "entityType": "registeredEntity", + "name": "Vanguard Russell 1000 Growth Index Trust", + "incorporatedInJurisdiction": { + "name": "US-PA", + "code": "US-PA" + }, + "identifiers": [ + { + "id": "00EHHQ2ZHDCFXJCPCL46", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "schemeName": "RA999999" + } + ], + "addresses": [ + { + "type": "registered", + "address": "C/O VANGUARD FIDUCIARY TRUST COMPANY, MALVERN", + "country": "US", + "postCode": "19355" + }, + { + "type": "business", + "address": "C/O Vanguard Fiduciary Trust Company, Valley Forge", + "country": "US", + "postCode": "19482" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-10-05T00:00:00.000Z" + }, + { + "statementID": "ede4fef6-2aa2-8caa-b69a-330187c1750f", + "statementType": "entityStatement", + "statementDate": "2023-06-03", + "entityType": "registeredEntity", + "name": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", + "incorporatedInJurisdiction": { + "name": "US-DE", + "code": "US-DE" + }, + "identifiers": [ + { + "id": "00QDBXDXLLF3W3JJJO36", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "2835527", + "schemeName": "RA000602" + } + ], + "addresses": [ + { + "type": "registered", + "address": "CORPORATION TRUST CENTER 1209 ORANGE ST, WILMINGTON", + "country": "US", + "postCode": "19801" + }, + { + "type": "business", + "address": "655 Broad Street, Newark", + "country": "US", + "postCode": "07102" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1997-12-18T05:00:00.000Z" + }, + { + "statementID": "96a93ace-30e2-016f-5ac1-6e88f320a1b6", + "statementType": "entityStatement", + "statementDate": "2020-08-17", + "entityType": "registeredEntity", + "name": "GE Pacific-3 Holdings, Inc.", + "incorporatedInJurisdiction": { + "name": "US-DE", + "code": "US-DE" + }, + "identifiers": [ + { + "id": "00W0SLGGVF0QQ5Q36N03", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "schemeName": "RA999999" + } + ], + "addresses": [ + { + "type": "registered", + "address": "C/O The Corporation Trust Company, Wilmington", + "country": "US", + "postCode": "19801" + }, + { + "type": "business", + "address": "C/O The Corporation Trust Company, Wilmington", + "country": "US", + "postCode": "19801" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "11739b48-9df9-2d05-34aa-79c04d5cca06", + "statementType": "entityStatement", + "statementDate": "2023-03-02", + "entityType": "registeredEntity", + "name": "DENTSU INTERNATIONAL LIMITED", + "incorporatedInJurisdiction": { + "name": "United Kingdom", + "code": "GB" + }, + "identifiers": [ + { + "id": "213800FERQ5LE3H7WJ58", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "01403668", + "schemeName": "RA000585" + } + ], + "addresses": [ + { + "type": "registered", + "address": "10 TRITON STREET, LONDON", + "country": "GB", + "postCode": "NW1 3BF" + }, + { + "type": "business", + "address": "10 TRITON STREET, LONDON", + "country": "GB", + "postCode": "NW1 3BF" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1978-12-05T00:00:00Z" + } +] diff --git a/tests/fixtures/lei-data.json b/tests/fixtures/lei-data.json index 6779727..6b45e5c 100644 --- a/tests/fixtures/lei-data.json +++ b/tests/fixtures/lei-data.json @@ -1 +1 @@ -[{"LEI": "001GPB6A9XPE8XJICC14", "Entity": {"LegalName": "Fidelity Advisor Leveraged Company Stock Fund", "OtherEntityNames": ["FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"], "LegalAddress": {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC", "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-29T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-29T16:33:00.000Z", "LastUpdateDate": "2023-05-18T15:41:20.212Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-18T15:48:53.604Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"}}}, {"LEI": "004L5FPTUREIWK9T2N63", "Entity": {"LegalName": "Hutchin Hill Capital, LP", "LegalAddress": {"FirstAddressLine": "C/O Corporation Service Company", "AdditionalAddressLine": "Suite 400", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19808"}, "HeadquartersAddress": {"FirstAddressLine": "22nd Floor", "AdditionalAddressLine": "888 7th Avenue", "City": "New York", "Region": "US-NY", "Country": "US", "PostalCode": "10106"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "4386463"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "T91T"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2018-05-08T13:46:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "4386463"}}}, {"LEI": "00EHHQ2ZHDCFXJCPCL46", "Entity": {"LegalName": "Vanguard Russell 1000 Growth Index Trust", "LegalAddress": {"FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY", "AdditionalAddressLine": "100 VANGUARD BLVD", "City": "MALVERN", "Region": "US-PA", "Country": "US", "PostalCode": "19355"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Vanguard Fiduciary Trust Company", "AdditionalAddressLine": "PO Box 2600", "City": "Valley Forge", "Region": "US-PA", "Country": "US", "PostalCode": "19482"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-PA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-10-05T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-10-05T20:30:00.000Z", "LastUpdateDate": "2022-07-22T09:32:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-07-22T07:35:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00GBW0Z2GYIER7DHDS71", "Entity": {"LegalName": "ARISTEIA CAPITAL, L.L.C.", "LegalAddress": {"FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY", "AdditionalAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "3rd Floor", "AdditionalAddressLine": "One Greenwich Plaza", "City": "Greenwich", "Region": "US-CT", "Country": "US", "PostalCode": "06830"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2758229"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "HZEH"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-03T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:55:00.000Z", "LastUpdateDate": "2022-10-24T21:31:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-09-23T19:58:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "2758229"}}}, {"LEI": "00KLB2PFTM3060S2N216", "Entity": {"LegalName": "Oakmark International Fund", "OtherEntityNames": ["HARRIS ASSOCIATES INVESTMENT TRUST - Oakmark International Fund"], "LegalAddress": {"FirstAddressLine": "155 FEDERAL ST., SUITE 700", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02110"}, "HeadquartersAddress": {"FirstAddressLine": "111 South Wacker Drive", "City": "Chicago", "Region": "US-IL", "Country": "US", "PostalCode": "60606-4319"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000002762"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-04T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-05T18:17:00.000Z", "LastUpdateDate": "2023-05-18T17:24:00.540Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-11T18:37:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000002762"}}}, {"LEI": "00QDBXDXLLF3W3JJJO36", "Entity": {"LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", "OtherEntityNames": ["Jennison 20/20 Focus Fund", "Prudential 20/20 Focus Fund", "Prudential 20/20 Fund", "Prudential Jennison 20/20 Focus Fund"], "LegalAddress": {"FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "655 Broad Street", "City": "Newark", "Region": "US-NJ", "Country": "US", "PostalCode": "07102"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2835527"}, "LegalJurisdiction": "US-DE", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "4FSX"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-12-18T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:58:00.000Z", "LastUpdateDate": "2023-05-03T07:03:05.620Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-04-21T15:23:48.020Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "0001052118"}}}, {"LEI": "00TR8NKAEL48RGTZEW89", "Entity": {"LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", "LegalAddress": {"FirstAddressLine": "89 Nexus Way", "City": "Camana Bay", "Country": "KY", "PostalCode": "KY1-9009"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", "AdditionalAddressLine": "Kaya Flamboyan 9", "City": "Willemstad", "Country": "CW"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000087", "RegistrationAuthorityEntityID": "574206"}, "LegalJurisdiction": "KY", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"}, "EntityStatus": "INACTIVE", "LegalEntityEvents": [{"LegalEntityEventType": "DISSOLUTION", "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000Z", "LegalEntityEventRecordedDate": "2022-03-24T19:41:39.000Z", "ValidationDocuments": "SUPPORTING_DOCUMENTS"}]}, "Registration": {"InitialRegistrationDate": "2012-08-24T18:53:00.000Z", "LastUpdateDate": "2019-04-22T21:31:00.000Z", "RegistrationStatus": "RETIRED", "NextRenewalDate": "2019-02-14T00:32:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000087", "ValidationAuthorityEntityID": "574206"}}}, {"LEI": "00TV1D5YIV5IDUGWBW29", "Entity": {"LegalName": "Ceridian Corporation Retirement Plan Trust", "LegalAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "TRUST"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-13T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-14T19:43:00.000Z", "LastUpdateDate": "2023-05-10T04:42:18.790Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-20T00:00:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00W0SLGGVF0QQ5Q36N03", "Entity": {"LegalName": "GE Pacific-3 Holdings, Inc.", "LegalAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XTIQ"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2016-01-12T15:45:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00X5RQKJQQJFFX0WPA53", "Entity": {"LegalName": "The Greater Morristown Young Men's Christian Association, Inc.", "OtherEntityNames": ["The Greater Morristown YMCA Inc"], "LegalAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "HeadquartersAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-NJ", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XSNP"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-11-22T10:55:00.000Z", "LastUpdateDate": "2020-07-24T19:29:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2013-11-22T10:38:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}] +[{"LEI": "001GPB6A9XPE8XJICC14", "Entity": {"LegalName": "Fidelity Advisor Leveraged Company Stock Fund", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"}], "LegalAddress": {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC", "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-29T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-29T16:33:00.000Z", "LastUpdateDate": "2023-05-18T15:41:20.212Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-18T15:48:53.604Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"}}}, {"LEI": "004L5FPTUREIWK9T2N63", "Entity": {"LegalName": "Hutchin Hill Capital, LP", "LegalAddress": {"FirstAddressLine": "C/O Corporation Service Company", "AdditionalAddressLine": "Suite 400", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19808"}, "HeadquartersAddress": {"FirstAddressLine": "22nd Floor", "AdditionalAddressLine": "888 7th Avenue", "City": "New York", "Region": "US-NY", "Country": "US", "PostalCode": "10106"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "4386463"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "T91T"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2018-05-08T13:46:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "4386463"}}}, {"LEI": "00EHHQ2ZHDCFXJCPCL46", "Entity": {"LegalName": "Vanguard Russell 1000 Growth Index Trust", "LegalAddress": {"FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY", "AdditionalAddressLine": "100 VANGUARD BLVD", "City": "MALVERN", "Region": "US-PA", "Country": "US", "PostalCode": "19355"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Vanguard Fiduciary Trust Company", "AdditionalAddressLine": "PO Box 2600", "City": "Valley Forge", "Region": "US-PA", "Country": "US", "PostalCode": "19482"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-PA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-10-05T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-10-05T20:30:00.000Z", "LastUpdateDate": "2022-07-22T09:32:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-07-22T07:35:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00GBW0Z2GYIER7DHDS71", "Entity": {"LegalName": "ARISTEIA CAPITAL, L.L.C.", "LegalAddress": {"FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY", "AdditionalAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "3rd Floor", "AdditionalAddressLine": "One Greenwich Plaza", "City": "Greenwich", "Region": "US-CT", "Country": "US", "PostalCode": "06830"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2758229"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "HZEH"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-03T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:55:00.000Z", "LastUpdateDate": "2022-10-24T21:31:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-09-23T19:58:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "2758229"}}}, {"LEI": "00KLB2PFTM3060S2N216", "Entity": {"LegalName": "Oakmark International Fund", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "HARRIS ASSOCIATES INVESTMENT TRUST - Oakmark International Fund"}], "LegalAddress": {"FirstAddressLine": "155 FEDERAL ST., SUITE 700", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02110"}, "HeadquartersAddress": {"FirstAddressLine": "111 South Wacker Drive", "City": "Chicago", "Region": "US-IL", "Country": "US", "PostalCode": "60606-4319"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000002762"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-04T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-05T18:17:00.000Z", "LastUpdateDate": "2023-05-18T17:24:00.540Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-11T18:37:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000002762"}}}, {"LEI": "00QDBXDXLLF3W3JJJO36", "Entity": {"LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Jennison 20/20 Focus Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential 20/20 Focus Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential 20/20 Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential Jennison 20/20 Focus Fund"}], "LegalAddress": {"FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "655 Broad Street", "City": "Newark", "Region": "US-NJ", "Country": "US", "PostalCode": "07102"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2835527"}, "LegalJurisdiction": "US-DE", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "4FSX"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-12-18T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:58:00.000Z", "LastUpdateDate": "2023-05-03T07:03:05.620Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-04-21T15:23:48.020Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "0001052118"}}}, {"LEI": "00TR8NKAEL48RGTZEW89", "Entity": {"LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", "LegalAddress": {"FirstAddressLine": "89 Nexus Way", "City": "Camana Bay", "Country": "KY", "PostalCode": "KY1-9009"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", "AdditionalAddressLine": "Kaya Flamboyan 9", "City": "Willemstad", "Country": "CW"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000087", "RegistrationAuthorityEntityID": "574206"}, "LegalJurisdiction": "KY", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"}, "EntityStatus": "INACTIVE", "LegalEntityEvents": [{"LegalEntityEventType": "DISSOLUTION", "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000Z", "LegalEntityEventRecordedDate": "2022-03-24T19:41:39.000Z", "ValidationDocuments": "SUPPORTING_DOCUMENTS"}]}, "Registration": {"InitialRegistrationDate": "2012-08-24T18:53:00.000Z", "LastUpdateDate": "2019-04-22T21:31:00.000Z", "RegistrationStatus": "RETIRED", "NextRenewalDate": "2019-02-14T00:32:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000087", "ValidationAuthorityEntityID": "574206"}}}, {"LEI": "00TV1D5YIV5IDUGWBW29", "Entity": {"LegalName": "Ceridian Corporation Retirement Plan Trust", "LegalAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "TRUST"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-13T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-14T19:43:00.000Z", "LastUpdateDate": "2023-05-10T04:42:18.790Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-20T00:00:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00W0SLGGVF0QQ5Q36N03", "Entity": {"LegalName": "GE Pacific-3 Holdings, Inc.", "LegalAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XTIQ"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2016-01-12T15:45:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00X5RQKJQQJFFX0WPA53", "Entity": {"LegalName": "The Greater Morristown Young Men's Christian Association, Inc.", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "The Greater Morristown YMCA Inc"}], "LegalAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "HeadquartersAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-NJ", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XSNP"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-11-22T10:55:00.000Z", "LastUpdateDate": "2020-07-24T19:29:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2013-11-22T10:38:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "1595D0QCK7Y15293JK84", "Entity": {"LegalName": "GALAPAGOS CONSERVATION TRUST", "LegalAddress": {"FirstAddressLine": "7-14 Great Dover Street", "City": "London", "Country": "GB", "PostalCode": "SE1 4YR"}, "HeadquartersAddress": {"FirstAddressLine": "7-14 Great Dover Street", "City": "London", "Country": "GB", "PostalCode": "SE1 4YR"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000585", "RegistrationAuthorityEntityID": "03004112"}, "LegalJurisdiction": "GB", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "G12F"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1994-12-21T00:00:00+01:00"}, "Registration": {"InitialRegistrationDate": "2023-02-13T22:13:11+01:00", "LastUpdateDate": "2023-03-10T13:08:56+01:00", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-02-13T22:13:11+01:00", "ManagingLOU": "98450045AN5EB5FDC780", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000585", "ValidationAuthorityEntityID": "03004112"}, "OtherValidationAuthorities": [{"ValidationAuthorityID": "RA000589", "ValidationAuthorityEntityID": "1043470"}]}}, {"LEI": "213800FERQ5LE3H7WJ58", "Entity": {"LegalName": "DENTSU INTERNATIONAL LIMITED", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "DENTSU AEGIS NETWORK LTD."}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "AEGIS GROUP PLC"}], "LegalAddress": {"FirstAddressLine": "10 TRITON STREET", "AdditionalAddressLine": "REGENT'S PLACE", "City": "LONDON", "Region": "GB-LND", "Country": "GB", "PostalCode": "NW1 3BF"}, "HeadquartersAddress": {"FirstAddressLine": "10 TRITON STREET", "AdditionalAddressLine": "REGENT'S PLACE", "City": "LONDON", "Region": "GB-LND", "Country": "GB", "PostalCode": "NW1 3BF"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000585", "RegistrationAuthorityEntityID": "01403668"}, "LegalJurisdiction": "GB", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "H0PO"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1978-12-05T00:00:00Z"}, "Registration": {"InitialRegistrationDate": "2014-02-10T00:00:00Z", "LastUpdateDate": "2023-02-02T09:07:52.390Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-02-17T00:00:00Z", "ManagingLOU": "213800WAVVOPS85N2205", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000585", "ValidationAuthorityEntityID": "01403668"}}}, {"LEI": "213800BJPX8V9HVY1Y11", "Entity": {"LegalName": "Swedeit Italian Aktiebolag", "LegalAddress": {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "V\u00e4stra Fr\u00f6lunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "V\u00e4stra Fr\u00f6lunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76"}, "TransliteratedOtherAddresses": [{"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "Vastra Frolunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76", "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS"}, {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "Vastra Frolunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76", "type": "AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS"}], "RegistrationAuthority": {"RegistrationAuthorityID": "RA000544", "RegistrationAuthorityEntityID": "556543-1193"}, "LegalJurisdiction": "SE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XJHM"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-05T02:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2014-04-09T00:00:00Z", "LastUpdateDate": "2023-04-25T13:18:00Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-12T06:59:39Z", "ManagingLOU": "549300O897ZC5H7CY412", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000544", "ValidationAuthorityEntityID": "556543-1193"}}}] diff --git a/tests/fixtures/lei-data.xml b/tests/fixtures/lei-data.xml index 0f62498..bc84d35 100644 --- a/tests/fixtures/lei-data.xml +++ b/tests/fixtures/lei-data.xml @@ -5,7 +5,6 @@ 2394586 - @@ -762,6 +761,178 @@ USA + + + 1595D0QCK7Y15293JK84 + + GALAPAGOS CONSERVATION TRUST + + 7-14 Great Dover Street + London + GB + SE1 4YR + + + 7-14 Great Dover Street + London + GB + SE1 4YR + + + RA000585 + 03004112 + + GB + GENERAL + + G12F + + ACTIVE + 1994-12-21T00:00:00+01:00 + + + 2023-02-13T22:13:11+01:00 + 2023-03-10T13:08:56+01:00 + ISSUED + 2024-02-13T22:13:11+01:00 + 98450045AN5EB5FDC780 + FULLY_CORROBORATED + + RA000585 + 03004112 + + + + RA000589 + 1043470 + + + + + + + 213800FERQ5LE3H7WJ58 + + DENTSU INTERNATIONAL LIMITED + + DENTSU AEGIS NETWORK LTD. + AEGIS GROUP PLC + + + 10 TRITON STREET + REGENT'S PLACE + LONDON + GB-LND + GB + NW1 3BF + + + 10 TRITON STREET + REGENT'S PLACE + LONDON + GB-LND + GB + NW1 3BF + + + RA000585 + 01403668 + + GB + GENERAL + + H0PO + + ACTIVE + 1978-12-05T00:00:00Z + + + 2014-02-10T00:00:00Z + 2023-02-02T09:07:52.390Z + ISSUED + 2024-02-17T00:00:00Z + 213800WAVVOPS85N2205 + FULLY_CORROBORATED + + RA000585 + 01403668 + + + + + + 213800BJPX8V9HVY1Y11 + + Swedeit Italian Aktiebolag + + C/O Anita Lindberg + C/O Anita Lindberg + Fortgatan 11 + Västra Frölunda + SE-O + SE + 426 76 + + + C/O Anita Lindberg + C/O Anita Lindberg + Fortgatan 11 + Västra Frölunda + SE-O + SE + 426 76 + + + + C/O Anita Lindberg + C/O Anita Lindberg + Fortgatan 11 + Vastra Frolunda + SE-O + SE + 426 76 + + + C/O Anita Lindberg + C/O Anita Lindberg + Fortgatan 11 + Vastra Frolunda + SE-O + SE + 426 76 + + + + RA000544 + 556543-1193 + + SE + GENERAL + + XJHM + + ACTIVE + 1997-06-05T02:00:00+02:00 + + + 2014-04-09T00:00:00Z + 2023-04-25T13:18:00Z + ISSUED + 2024-05-12T06:59:39Z + 549300O897ZC5H7CY412 + FULLY_CORROBORATED + + RA000544 + 556543-1193 + + + diff --git a/tests/fixtures/lei-updates-data-out.json b/tests/fixtures/lei-updates-data-out.json new file mode 100644 index 0000000..637fff5 --- /dev/null +++ b/tests/fixtures/lei-updates-data-out.json @@ -0,0 +1,262 @@ +[ + { + "statementID": "04c98917-93f2-bd6d-1c9b-40411dc9b8b3", + "statementType": "entityStatement", + "statementDate": "2023-06-18", + "entityType": "registeredEntity", + "name": "Fidelity Advisor Leveraged Company Stock Fund", + "incorporatedInJurisdiction": { + "name": "US-MA", + "code": "US-MA" + }, + "identifiers": [ + { + "id": "001GPB6A9XPE8XJICC14", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "S000005113", + "schemeName": "RA000665" + } + ], + "addresses": [ + { + "type": "registered", + "address": "245 SUMMER STREET, BOSTON", + "country": "US", + "postCode": "02210" + }, + { + "type": "business", + "address": "C/O Fidelity Management & Research Company LLC, Boston", + "country": "US", + "postCode": "02210" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-11-29T00:00:00.000Z" + }, + { + "statementID": "41c4522b-f5e2-4f19-5f6b-ec5e54258401", + "statementType": "entityStatement", + "statementDate": "2022-08-22", + "entityType": "registeredEntity", + "name": "Vanguard Russell 1000 Growth Index Trust", + "incorporatedInJurisdiction": { + "name": "US-PA", + "code": "US-PA" + }, + "identifiers": [ + { + "id": "00EHHQ2ZHDCFXJCPCL46", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "schemeName": "RA999999" + } + ], + "addresses": [ + { + "type": "registered", + "address": "C/O VANGUARD FIDUCIARY TRUST COMPANY, MALVERN", + "country": "US", + "postCode": "19355" + }, + { + "type": "business", + "address": "C/O Vanguard Fiduciary Trust Company, Valley Forge", + "country": "US", + "postCode": "19482" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-10-05T00:00:00.000Z" + }, + { + "statementID": "ede4fef6-2aa2-8caa-b69a-330187c1750f", + "statementType": "entityStatement", + "statementDate": "2023-06-03", + "entityType": "registeredEntity", + "name": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", + "incorporatedInJurisdiction": { + "name": "US-DE", + "code": "US-DE" + }, + "identifiers": [ + { + "id": "00QDBXDXLLF3W3JJJO36", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "2835527", + "schemeName": "RA000602" + } + ], + "addresses": [ + { + "type": "registered", + "address": "CORPORATION TRUST CENTER 1209 ORANGE ST, WILMINGTON", + "country": "US", + "postCode": "19801" + }, + { + "type": "business", + "address": "655 Broad Street, Newark", + "country": "US", + "postCode": "07102" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1997-12-18T05:00:00.000Z" + }, + { + "statementID": "96a93ace-30e2-016f-5ac1-6e88f320a1b6", + "statementType": "entityStatement", + "statementDate": "2020-08-17", + "entityType": "registeredEntity", + "name": "GE Pacific-3 Holdings, Inc.", + "incorporatedInJurisdiction": { + "name": "US-DE", + "code": "US-DE" + }, + "identifiers": [ + { + "id": "00W0SLGGVF0QQ5Q36N03", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "schemeName": "RA999999" + } + ], + "addresses": [ + { + "type": "registered", + "address": "C/O The Corporation Trust Company, Wilmington", + "country": "US", + "postCode": "19801" + }, + { + "type": "business", + "address": "C/O The Corporation Trust Company, Wilmington", + "country": "US", + "postCode": "19801" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "11739b48-9df9-2d05-34aa-79c04d5cca06", + "statementType": "entityStatement", + "statementDate": "2023-03-02", + "entityType": "registeredEntity", + "name": "DENTSU INTERNATIONAL LIMITED", + "incorporatedInJurisdiction": { + "name": "United Kingdom", + "code": "GB" + }, + "identifiers": [ + { + "id": "213800FERQ5LE3H7WJ58", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "01403668", + "schemeName": "RA000585" + } + ], + "addresses": [ + { + "type": "registered", + "address": "10 TRITON STREET, LONDON", + "country": "GB", + "postCode": "NW1 3BF" + }, + { + "type": "business", + "address": "10 TRITON STREET, LONDON", + "country": "GB", + "postCode": "NW1 3BF" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-06", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1978-12-05T00:00:00Z" + } +] diff --git a/tests/fixtures/lei-updates-data.json b/tests/fixtures/lei-updates-data.json new file mode 100644 index 0000000..0f43884 --- /dev/null +++ b/tests/fixtures/lei-updates-data.json @@ -0,0 +1,254 @@ +[ + { + "LEI": "001GPB6A9XPE8XJICC14", + "Entity": { + "LegalName": "Fidelity Advisor Leveraged Company Stock Fund", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund" + } + ], + "LegalAddress": { + "FirstAddressLine": "245 SUMMER STREET", + "City": "BOSTON", + "Region": "US-MA", + "Country": "US", + "PostalCode": "02210" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O Fidelity Management & Research Company LLC", + "City": "Boston", + "Region": "US-MA", + "Country": "US", + "PostalCode": "02210" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000665", + "RegistrationAuthorityEntityID": "S000005113" + }, + "LegalJurisdiction": "US-MA", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "8888", + "OtherLegalForm": "FUND" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2012-11-29T00:00:00.000Z" + }, + "Registration": { + "InitialRegistrationDate": "2012-11-29T16:33:00.000Z", + "LastUpdateDate": "2023-06-18T15:41:20.212Z", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-05-18T15:48:53.604Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000665", + "ValidationAuthorityEntityID": "S000005113" + } + } + }, + { + "LEI": "00EHHQ2ZHDCFXJCPCL46", + "Entity": { + "LegalName": "Vanguard Russell 1000 Growth Index Trust", + "LegalAddress": { + "FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY", + "AdditionalAddressLine": "100 VANGUARD BLVD", + "City": "MALVERN", + "Region": "US-PA", + "Country": "US", + "PostalCode": "19355" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O Vanguard Fiduciary Trust Company", + "AdditionalAddressLine": "PO Box 2600", + "City": "Valley Forge", + "Region": "US-PA", + "Country": "US", + "PostalCode": "19482" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA999999" + }, + "LegalJurisdiction": "US-PA", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "8888", + "OtherLegalForm": "FUND" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2012-10-05T00:00:00.000Z" + }, + "Registration": { + "InitialRegistrationDate": "2012-10-05T20:30:00.000Z", + "LastUpdateDate": "2022-08-22T09:32:00.000Z", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2023-07-22T07:35:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "ENTITY_SUPPLIED_ONLY", + "ValidationAuthority": { + "ValidationAuthorityID": "RA999999" + } + } + }, + { + "LEI": "00QDBXDXLLF3W3JJJO36", + "Entity": { + "LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "Jennison 20/20 Focus Fund" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "Prudential 20/20 Focus Fund" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "Prudential 20/20 Fund" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "Prudential Jennison 20/20 Focus Fund" + } + ], + "LegalAddress": { + "FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", + "City": "WILMINGTON", + "Region": "US-DE", + "Country": "US", + "PostalCode": "19801" + }, + "HeadquartersAddress": { + "FirstAddressLine": "655 Broad Street", + "City": "Newark", + "Region": "US-NJ", + "Country": "US", + "PostalCode": "07102" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000602", + "RegistrationAuthorityEntityID": "2835527" + }, + "LegalJurisdiction": "US-DE", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "4FSX" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "1997-12-18T05:00:00.000Z" + }, + "Registration": { + "InitialRegistrationDate": "2012-06-06T15:58:00.000Z", + "LastUpdateDate": "2023-06-03T07:03:05.620Z", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-04-21T15:23:48.020Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000665", + "ValidationAuthorityEntityID": "0001052118" + } + } + }, + { + "LEI": "00W0SLGGVF0QQ5Q36N03", + "Entity": { + "LegalName": "GE Pacific-3 Holdings, Inc.", + "LegalAddress": { + "FirstAddressLine": "C/O The Corporation Trust Company", + "AdditionalAddressLine": "1209 Orange Street", + "City": "Wilmington", + "Region": "US-DE", + "Country": "US", + "PostalCode": "19801" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O The Corporation Trust Company", + "AdditionalAddressLine": "1209 Orange Street", + "City": "Wilmington", + "Region": "US-DE", + "Country": "US", + "PostalCode": "19801" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA999999" + }, + "LegalJurisdiction": "US-DE", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "XTIQ" + }, + "EntityStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-06-06T15:56:00.000Z", + "LastUpdateDate": "2020-08-17T12:40:00.000Z", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2016-01-12T15:45:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "ENTITY_SUPPLIED_ONLY", + "ValidationAuthority": { + "ValidationAuthorityID": "RA999999" + } + } + }, + { + "LEI": "213800FERQ5LE3H7WJ58", + "Entity": { + "LegalName": "DENTSU INTERNATIONAL LIMITED", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "DENTSU AEGIS NETWORK LTD." + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "AEGIS GROUP PLC" + } + ], + "LegalAddress": { + "FirstAddressLine": "10 TRITON STREET", + "AdditionalAddressLine": "REGENT'S PLACE", + "City": "LONDON", + "Region": "GB-LND", + "Country": "GB", + "PostalCode": "NW1 3BF" + }, + "HeadquartersAddress": { + "FirstAddressLine": "10 TRITON STREET", + "AdditionalAddressLine": "REGENT'S PLACE", + "City": "LONDON", + "Region": "GB-LND", + "Country": "GB", + "PostalCode": "NW1 3BF" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000585", + "RegistrationAuthorityEntityID": "01403668" + }, + "LegalJurisdiction": "GB", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "H0PO" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "1978-12-05T00:00:00Z" + }, + "Registration": { + "InitialRegistrationDate": "2014-02-10T00:00:00Z", + "LastUpdateDate": "2023-03-02T09:07:52.390Z", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-02-17T00:00:00Z", + "ManagingLOU": "213800WAVVOPS85N2205", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000585", + "ValidationAuthorityEntityID": "01403668" + } + } + } +] diff --git a/tests/fixtures/lei-updates-data2-new-out.json b/tests/fixtures/lei-updates-data2-new-out.json new file mode 100644 index 0000000..24ef4d2 --- /dev/null +++ b/tests/fixtures/lei-updates-data2-new-out.json @@ -0,0 +1,636 @@ +[ + { + "statementID": "179355f9-6d62-3ccc-1779-f09db959a042", + "statementType": "entityStatement", + "statementDate": "2023-09-12", + "entityType": "registeredEntity", + "name": "Delaware Ivy Value Fund", + "incorporatedInJurisdiction": { + "name": "United States", + "code": "US" + }, + "identifiers": [ + { + "id": "ZZZWS8Y6XVKM0Q9RJQ79", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "S000024832", + "schemeName": "RA000665" + } + ], + "addresses": [ + { + "type": "registered", + "address": "C/O THE CORPORATION TRUST COMPANY, WILMINGTON", + "country": "US", + "postCode": "19801" + }, + { + "type": "business", + "address": "C/O Delaware Management Company, Inc., WILMINGTON", + "country": "US", + "postCode": "19808" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-06-06T00:00:00.000Z" + }, + { + "statementID": "68b6a4dc-cc3d-3c8d-d44c-47b29ab0935c", + "statementType": "entityStatement", + "statementDate": "2023-09-03", + "entityType": "registeredEntity", + "name": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", + "incorporatedInJurisdiction": { + "name": "Cayman Islands", + "code": "KY" + }, + "identifiers": [ + { + "id": "00TR8NKAEL48RGTZEW89", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "574206", + "schemeName": "RA000087" + } + ], + "addresses": [ + { + "type": "registered", + "address": "89 Nexus Way, Camana Bay", + "country": "KY", + "postCode": "KY1-9009" + }, + { + "type": "business", + "address": "C/O Citco Fund Services (Curacao) B.V., Willemstad", + "country": "CW" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "296865b4-00a0-2f0e-47ac-c20037da1f30", + "statementType": "entityStatement", + "statementDate": "2023-06-25", + "entityType": "registeredEntity", + "name": "VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BHF50000076475", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "35903058", + "schemeName": "RA000526" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2004-10-09T08:00:00+02:00" + }, + { + "statementID": "d400beea-3cdc-a894-ce0d-9ce8fa67c30b", + "statementType": "entityStatement", + "statementDate": "2023-09-17", + "entityType": "registeredEntity", + "name": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002337", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB04", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-23T08:00:00+01:00" + }, + { + "statementID": "19fb3b00-c801-7ff9-fae7-a1955c772e71", + "statementType": "entityStatement", + "statementDate": "2023-09-17", + "entityType": "registeredEntity", + "name": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002143", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB01", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-04-02T08:00:00+02:00" + }, + { + "statementID": "1ff23b39-62f3-4744-da92-b11c2c3f8ff3", + "statementType": "entityStatement", + "statementDate": "2023-09-17", + "entityType": "registeredEntity", + "name": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002240", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB02", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-22T08:00:00+01:00" + }, + { + "statementID": "a8866f73-572f-5ec5-6b42-2fe78b72f128", + "statementType": "entityStatement", + "statementDate": "2023-09-17", + "entityType": "registeredEntity", + "name": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002434", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB03", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-22T08:00:00+01:00" + }, + { + "statementID": "426b5ddd-267f-0020-ae05-5dbc1e0f3341", + "statementType": "entityStatement", + "statementDate": "2023-10-12", + "entityType": "registeredEntity", + "name": "Všeobecná úverová banka, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "549300JB1P61FUTPEZ75", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "31320155", + "schemeName": "RA000526" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "829 90" + }, + { + "type": "business", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "829 90" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1992-04-01T08:00:00+02:00" + }, + { + "statementID": "42b2a5f0-c7ff-a8c8-fd77-9b0292530830", + "statementType": "entityStatement", + "statementDate": "2023-02-25", + "entityType": "registeredEntity", + "name": "Corporate Business Solutions Partners S.à r.l.", + "incorporatedInJurisdiction": { + "name": "Luxembourg", + "code": "LU" + }, + "identifiers": [ + { + "id": "9845002B9874D50A4531", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "B191847", + "schemeName": "RA000432" + } + ], + "addresses": [ + { + "type": "registered", + "address": "23 Am Scheerleck, Wecker", + "country": "LU", + "postCode": "6868" + }, + { + "type": "business", + "address": "23 Am Scheerleck, Wecker", + "country": "LU", + "postCode": "6868" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2014-11-17T00:00:00+00:00" + }, + { + "statementID": "657a9791-1cf0-e35c-a65f-0785e4685749", + "statementType": "entityStatement", + "statementDate": "2023-02-11", + "entityType": "registeredEntity", + "name": "MONOOVA GLOBAL PAYMENTS PTY LTD", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "2138002YAB7Z9KO21397", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "106 249 852", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "UNIT L7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2003-09-09T00:00:00+00:00" + }, + { + "statementID": "8304570e-5bbf-d8df-1674-51f9d2a1f354", + "statementType": "entityStatement", + "statementDate": "2022-03-23", + "entityType": "registeredEntity", + "name": "MONEYTECH GROUP LIMITED", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "984500501A1B1045PB30", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "611 393 554", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2016-03-18T00:00:00+00:00" + }, + { + "statementID": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81", + "statementType": "entityStatement", + "statementDate": "2023-04-30", + "entityType": "registeredEntity", + "name": "MONOOVA FX PTY LTD", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "98450051BS9C610A8T78", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "151 337 852", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "LEVEL 7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "LEVEL 7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2011-06-07T00:00:00+00:00" + } +] diff --git a/tests/fixtures/lei-updates-data2-new.json b/tests/fixtures/lei-updates-data2-new.json new file mode 100644 index 0000000..b60903e --- /dev/null +++ b/tests/fixtures/lei-updates-data2-new.json @@ -0,0 +1,629 @@ +[ + { + "LEI": "ZZZWS8Y6XVKM0Q9RJQ79", + "Entity": { + "LegalName": "Delaware Ivy Value Fund", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "IVY FUNDS - Delaware Ivy Value Fund" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "IVY FUNDS - Ivy Value Fund" + } + ], + "LegalAddress": { + "FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY", + "AdditionalAddressLine": "1209 ORANGE ST", + "City": "WILMINGTON", + "Region": "US-DE", + "Country": "US", + "PostalCode": "19801" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O Delaware Management Company, Inc.", + "AdditionalAddressLine": "251 LITTLE FALLS DRIVE", + "City": "WILMINGTON", + "Region": "US-DE", + "Country": "US", + "PostalCode": "19808" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000665", + "RegistrationAuthorityEntityID": "S000024832" + }, + "LegalJurisdiction": "US", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "8888", + "OtherLegalForm": "FUND" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2012-06-06T00:00:00.000Z" + }, + "Registration": { + "InitialRegistrationDate": "2012-06-06T15:56:00.000Z", + "LastUpdateDate": "2023-09-12T20:51:51.031Z", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-12T20:51:51.032Z", + "ManagingLOU": "5493001KJTIIGC8Y1R12", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000665", + "ValidationAuthorityEntityID": "S000024832" + } + } + }, + { + "LEI": "00TR8NKAEL48RGTZEW89", + "Entity": { + "LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", + "LegalAddress": { + "FirstAddressLine": "89 Nexus Way", + "City": "Camana Bay", + "Country": "KY", + "PostalCode": "KY1-9009" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", + "AdditionalAddressLine": "Kaya Flamboyan 9", + "City": "Willemstad", + "Country": "CW" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000087", + "RegistrationAuthorityEntityID": "574206" + }, + "LegalJurisdiction": "KY", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "9999", + "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY" + }, + "EntityStatus": "INACTIVE", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "DISSOLUTION", + "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000+01:00", + "LegalEntityEventRecordedDate": "2022-03-24T20:41:39.000+01:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2012-08-24T20:53:00.000+02:00", + "LastUpdateDate": "2023-09-03T08:48:03.080+02:00", + "RegistrationStatus": "RETIRED", + "NextRenewalDate": "2019-02-14T01:32:00.000+01:00", + "ManagingLOU": "39120001KULK7200U106", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000087", + "ValidationAuthorityEntityID": "574206" + } + } + }, + { + "LEI": "097900BHF50000076475", + "Entity": { + "LegalName": "VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000526", + "RegistrationAuthorityEntityID": "35903058" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "2EEG" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2004-10-09T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2017-05-30T00:00:00+02:00", + "LastUpdateDate": "2023-06-25T08:30:56.059+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-05-30T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000526", + "ValidationAuthorityEntityID": "35903058" + } + } + }, + { + "LEI": "097900BEJX0000002337", + "Entity": { + "LegalName": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s." + } + ], + "TransliteratedOtherEntityNames": [ + { + "type": "PREFERRED_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s." + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB04" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-23T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:04.159+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB04" + } + } + }, + { + "LEI": "097900BEJX0000002143", + "Entity": { + "LegalName": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB01" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2012-04-02T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:02.482+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB01" + } + } + }, + { + "LEI": "097900BEJX0000002240", + "Entity": { + "LegalName": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB02" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-22T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:03.502+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB02" + } + } + }, + { + "LEI": "097900BEJX0000002434", + "Entity": { + "LegalName": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB03" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-22T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:04.659+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB03" + } + } + }, + { + "LEI": "549300JB1P61FUTPEZ75", + "Entity": { + "LegalName": "Všeobecná úverová banka, a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "Vseobecna uverova banka, as" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "829 90" + }, + "HeadquartersAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "829 90" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000526", + "RegistrationAuthorityEntityID": "31320155" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "2EEG" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "1992-04-01T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2013-10-02T00:00:00+02:00", + "LastUpdateDate": "2023-10-12T08:30:46.285+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-10-03T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000526", + "ValidationAuthorityEntityID": "31320155" + } + } + }, + { + "LEI": "9845002B9874D50A4531", + "Entity": { + "LegalName": "Corporate Business Solutions Partners S.à r.l.", + "LegalAddress": { + "FirstAddressLine": "23 Am Scheerleck", + "City": "Wecker", + "Country": "LU", + "PostalCode": "6868" + }, + "HeadquartersAddress": { + "FirstAddressLine": "23 Am Scheerleck", + "City": "Wecker", + "Country": "LU", + "PostalCode": "6868" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000432", + "RegistrationAuthorityEntityID": "B191847" + }, + "LegalJurisdiction": "LU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "DVXS" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2014-11-17T00:00:00+00:00" + }, + "Registration": { + "InitialRegistrationDate": "2020-03-17T08:22:48+00:00", + "LastUpdateDate": "2023-02-25T09:13:32+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-03-16T08:36:11+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000432", + "ValidationAuthorityEntityID": "B191847" + } + } + }, + { + "LEI": "2138002YAB7Z9KO21397", + "Entity": { + "LegalName": "MONOOVA GLOBAL PAYMENTS PTY LTD", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONOOVA GLOBAL PAYMENTS LIMITED" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONEYTECH LIMITED" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "SQUADRON FINANCIAL SERVICES LIMITED" + } + ], + "LegalAddress": { + "FirstAddressLine": "UNIT L7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "106 249 852" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "TXVC" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2003-09-09T00:00:00+00:00", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "CHANGE_LEGAL_FORM", + "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + }, + { + "LegalEntityEventType": "CHANGE_LEGAL_NAME", + "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-01-14T00:00:00+00:00", + "LastUpdateDate": "2023-02-11T11:21:01+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "106 249 852" + } + } + }, + { + "LEI": "984500501A1B1045PB30", + "Entity": { + "LegalName": "MONEYTECH GROUP LIMITED", + "LegalAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "611 393 554" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "R4KK" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2016-03-18T00:00:00+00:00" + }, + "Registration": { + "InitialRegistrationDate": "2021-07-27T07:07:14+00:00", + "LastUpdateDate": "2022-03-23T15:28:31+00:00", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-07-27T07:07:14+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "611 393 554" + } + } + }, + { + "LEI": "98450051BS9C610A8T78", + "Entity": { + "LegalName": "MONOOVA FX PTY LTD", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONEYTECH FX PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "360 MARKETS PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "360FX PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "CAPITAL AND FINANCE FOREX PTY LTD" + } + ], + "LegalAddress": { + "FirstAddressLine": "LEVEL 7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "LEVEL 7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "151 337 852" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "TXVC" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2011-06-07T00:00:00+00:00", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "CHANGE_LEGAL_ADDRESS", + "LegalEntityEventEffectiveDate": "2022-09-13T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-03-30T14:18:04+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T12:21:06+00:00", + "LastUpdateDate": "2023-04-30T14:18:04+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "151 337 852" + } + } + } +] diff --git a/tests/fixtures/lei-updates-data2-new.xml b/tests/fixtures/lei-updates-data2-new.xml new file mode 100644 index 0000000..65bf57c --- /dev/null +++ b/tests/fixtures/lei-updates-data2-new.xml @@ -0,0 +1,649 @@ + + + 2023-11-06T08:20:40Z + GLEIF_FULL_PUBLISHED + 2495059 + + + + ZZZWS8Y6XVKM0Q9RJQ79 + + Delaware Ivy Value Fund + + IVY FUNDS - Delaware Ivy Value Fund + IVY FUNDS - Ivy Value Fund + + + C/O THE CORPORATION TRUST COMPANY + 1209 ORANGE ST + WILMINGTON + US-DE + US + 19801 + + + C/O Delaware Management Company, Inc. + CORPORATION SERVICE COMPANY + 251 LITTLE FALLS DRIVE + WILMINGTON + US-DE + US + 19808 + + + RA000665 + S000024832 + + US + FUND + + 8888 + FUND + + ACTIVE + 2012-06-06T00:00:00.000Z + + + 2012-06-06T15:56:00.000Z + 2023-09-12T20:51:51.031Z + ISSUED + 2024-09-12T20:51:51.032Z + 5493001KJTIIGC8Y1R12 + FULLY_CORROBORATED + + RA000665 + S000024832 + + + + + + 00TR8NKAEL48RGTZEW89 + + ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD. + + 89 Nexus Way + Camana Bay + KY + KY1-9009 + + + C/O Citco Fund Services (Curacao) B.V. + Kaya Flamboyan 9 + Willemstad + CW + + + RA000087 + 574206 + + KY + GENERAL + + 9999 + CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY + + INACTIVE + + + DISSOLUTION + 2018-11-09T00:00:00.000+01:00 + 2022-03-24T20:41:39.000+01:00 + SUPPORTING_DOCUMENTS + + + + + 2012-08-24T20:53:00.000+02:00 + 2023-09-03T08:48:03.080+02:00 + RETIRED + 2019-02-14T01:32:00.000+01:00 + 39120001KULK7200U106 + FULLY_CORROBORATED + + RA000087 + 574206 + + + + + + 097900BHF50000076475 + + VÚB Generali dôchodková správcovská spoločnosť, a.s. + + VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000526 + 35903058 + + SK + GENERAL + + 2EEG + + ACTIVE + 2004-10-09T08:00:00+02:00 + + + 2017-05-30T00:00:00+02:00 + 2023-06-25T08:30:56.059+02:00 + ISSUED + 2024-05-30T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000526 + 35903058 + + + + + Mlynské nivy 1, null, null, null, SK, null, 820 04, Bratislava + 0.94 + pointAddress + 48.14687 + 17.12492 + 2016-05-17T10:50:50 + TopLeft.Latitude: 48.1479942, TopLeft.Longitude: 17.1232352, BottomRight.Latitude: 48.1457458, BottomRight.Longitude: 17.1266048 + houseNumber + Mlynské nivy 170/1, 821 09 Bratislava, Slovenská Republika + NT_kzljYRG5OuCE2R6Jrp7U1D_xcDMvED + Mlynské nivy + 170/1 + 821 09 + Bratislava + Bratislava I + Bratislavský kraj + SVK + + + + + 097900BEJX0000002337 + + SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s. + + MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + + SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s. + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB04 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-23T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:04.159+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB04 + + + + + + 097900BEJX0000002143 + + INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s. + + INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB01 + + SK + FUND + + 7EG2 + + ACTIVE + 2012-04-02T08:00:00+02:00 + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:02.482+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB01 + + + + + + 097900BEJX0000002240 + + KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB02 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-22T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:03.502+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB02 + + + + + + 097900BEJX0000002434 + + PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB03 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-22T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:04.659+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB03 + + + + + + 549300JB1P61FUTPEZ75 + + Všeobecná úverová banka, a.s. + + Vseobecna uverova banka, as + + + Mlynské nivy 1 + Bratislava + SK + 829 90 + + + Mlynské nivy 1 + Bratislava + SK + 829 90 + + + RA000526 + 31320155 + + SK + GENERAL + + 2EEG + + ACTIVE + 1992-04-01T08:00:00+02:00 + + + 2013-10-02T00:00:00+02:00 + 2023-10-12T08:30:46.285+02:00 + ISSUED + 2024-10-03T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000526 + 31320155 + + + + + + 9845002B9874D50A4531 + + Corporate Business Solutions Partners S.à r.l. + + 23 Am Scheerleck + Wecker + LU + 6868 + + + 23 Am Scheerleck + Wecker + LU + 6868 + + + RA000432 + B191847 + + LU + GENERAL + + DVXS + + ACTIVE + 2014-11-17T00:00:00+00:00 + + + 2020-03-17T08:22:48+00:00 + 2023-02-25T09:13:32+00:00 + ISSUED + 2024-03-16T08:36:11+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000432 + B191847 + + + + + + 2138002YAB7Z9KO21397 + + MONOOVA GLOBAL PAYMENTS PTY LTD + + MONOOVA GLOBAL PAYMENTS LIMITED + MONEYTECH LIMITED + SQUADRON FINANCIAL SERVICES LIMITED + + + UNIT L7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 106 249 852 + + AU + GENERAL + + TXVC + + ACTIVE + 2003-09-09T00:00:00+00:00 + + + CHANGE_LEGAL_FORM + 2022-09-16T00:00:00+00:00 + 2023-01-11T09:59:14+00:00 + SUPPORTING_DOCUMENTS + + + CHANGE_LEGAL_NAME + 2022-09-16T00:00:00+00:00 + 2023-01-11T09:59:14+00:00 + SUPPORTING_DOCUMENTS + + + + + 2019-01-14T00:00:00+00:00 + 2023-02-11T11:21:01+00:00 + ISSUED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 106 249 852 + + + + + + 984500501A1B1045PB30 + + MONEYTECH GROUP LIMITED + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 611 393 554 + + AU + GENERAL + + R4KK + + ACTIVE + 2016-03-18T00:00:00+00:00 + + + 2021-07-27T07:07:14+00:00 + 2022-03-23T15:28:31+00:00 + LAPSED + 2022-07-27T07:07:14+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 611 393 554 + + + + + + 98450051BS9C610A8T78 + + MONOOVA FX PTY LTD + + MONEYTECH FX PTY LTD + 360 MARKETS PTY LTD + 360FX PTY LTD + CAPITAL AND FINANCE FOREX PTY LTD + + + LEVEL 7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + LEVEL 7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 151 337 852 + + AU + GENERAL + + TXVC + + ACTIVE + 2011-06-07T00:00:00+00:00 + + + CHANGE_LEGAL_ADDRESS + 2022-09-13T00:00:00+00:00 + 2023-03-30T14:18:04+00:00 + SUPPORTING_DOCUMENTS + + + + + 2019-05-15T12:21:06+00:00 + 2023-04-30T14:18:04+00:00 + ISSUED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 151 337 852 + + + + + + diff --git a/tests/fixtures/lei-updates-data2-out.json b/tests/fixtures/lei-updates-data2-out.json new file mode 100644 index 0000000..d6b602d --- /dev/null +++ b/tests/fixtures/lei-updates-data2-out.json @@ -0,0 +1,636 @@ +[ + { + "statementID": "790b76d5-0db6-9d26-bb84-1d57963c4638", + "statementType": "entityStatement", + "statementDate": "2023-08-03", + "entityType": "registeredEntity", + "name": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", + "incorporatedInJurisdiction": { + "name": "Cayman Islands", + "code": "KY" + }, + "identifiers": [ + { + "id": "00TR8NKAEL48RGTZEW89", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "574206", + "schemeName": "RA000087" + } + ], + "addresses": [ + { + "type": "registered", + "address": "89 Nexus Way, Camana Bay", + "country": "KY", + "postCode": "KY1-9009" + }, + { + "type": "business", + "address": "C/O Citco Fund Services (Curacao) B.V., Willemstad", + "country": "CW" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "1865157c-c635-bb1b-cebf-9edcf88c7938", + "statementType": "entityStatement", + "statementDate": "2023-05-25", + "entityType": "registeredEntity", + "name": "VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BHF50000076475", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "35903058", + "schemeName": "RA000526" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2004-10-09T08:00:00+02:00" + }, + { + "statementID": "8a8e8ee2-5bb4-7582-9b79-d531b6b59c12", + "statementType": "entityStatement", + "statementDate": "2023-08-17", + "entityType": "registeredEntity", + "name": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002337", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB04", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-23T08:00:00+01:00" + }, + { + "statementID": "d60fa665-032c-6992-895f-66cdc7509909", + "statementType": "entityStatement", + "statementDate": "2023-08-17", + "entityType": "registeredEntity", + "name": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002143", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB01", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2012-04-02T08:00:00+02:00" + }, + { + "statementID": "64b41c35-f336-0010-ae81-785d33e65ffd", + "statementType": "entityStatement", + "statementDate": "2023-08-17", + "entityType": "registeredEntity", + "name": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002240", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB02", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-22T08:00:00+01:00" + }, + { + "statementID": "4f32f254-ec84-a32a-79d1-a036dfa789e6", + "statementType": "entityStatement", + "statementDate": "2023-08-17", + "entityType": "registeredEntity", + "name": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BEJX0000002434", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "SK35903058VUB03", + "schemeName": "RA000706" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "820 04" + }, + { + "type": "business", + "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava", + "country": "SK", + "postCode": "820 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2005-03-22T08:00:00+01:00" + }, + { + "statementID": "eca994a4-c50a-62b4-1308-9077410ee7b2", + "statementType": "entityStatement", + "statementDate": "2023-09-12", + "entityType": "registeredEntity", + "name": "Všeobecná úverová banka, a.s.", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "549300JB1P61FUTPEZ75", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "31320155", + "schemeName": "RA000526" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "829 90" + }, + { + "type": "business", + "address": "Mlynské nivy 1, Bratislava", + "country": "SK", + "postCode": "829 90" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "1992-04-01T08:00:00+02:00" + }, + { + "statementID": "71da391b-9385-4a1c-c5fc-e040a762b172", + "statementType": "entityStatement", + "statementDate": "2023-09-27", + "entityType": "registeredEntity", + "name": "Aquila Real Asset Finance III a. s. v likvidácii", + "incorporatedInJurisdiction": { + "name": "Slovakia", + "code": "SK" + }, + "identifiers": [ + { + "id": "097900BJGO0000201513", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "52143058", + "schemeName": "RA000526" + } + ], + "addresses": [ + { + "type": "registered", + "address": "Dúbravská cesta 14, Bratislava - mestská časť Karlova Ves", + "country": "SK", + "postCode": "841 04" + }, + { + "type": "business", + "address": "Dúbravská cesta 14, Bratislava - mestská časť Karlova Ves", + "country": "SK", + "postCode": "841 04" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2019-02-02T08:00:00+01:00" + }, + { + "statementID": "fd9f3a92-ed00-95f2-b005-45772e4f40b7", + "statementType": "entityStatement", + "statementDate": "2023-01-25", + "entityType": "registeredEntity", + "name": "Corporate Business Solutions Partners S.à r.l.", + "incorporatedInJurisdiction": { + "name": "Luxembourg", + "code": "LU" + }, + "identifiers": [ + { + "id": "9845002B9874D50A4531", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "B191847", + "schemeName": "RA000432" + } + ], + "addresses": [ + { + "type": "registered", + "address": "23 Am Scheerleck, Wecker", + "country": "LU", + "postCode": "6868" + }, + { + "type": "business", + "address": "23 Am Scheerleck, Wecker", + "country": "LU", + "postCode": "6868" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2014-11-17T00:00:00+00:00" + }, + { + "statementID": "63a68db5-2ba9-2d26-0acc-2303ca07035e", + "statementType": "entityStatement", + "statementDate": "2023-01-11", + "entityType": "registeredEntity", + "name": "MONOOVA GLOBAL PAYMENTS PTY LTD", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "2138002YAB7Z9KO21397", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "106 249 852", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "UNIT L7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2003-09-09T00:00:00+00:00" + }, + { + "statementID": "6c029f06-1c31-3864-3508-8c393bd6f61e", + "statementType": "entityStatement", + "statementDate": "2022-02-23", + "entityType": "registeredEntity", + "name": "MONEYTECH GROUP LIMITED", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "984500501A1B1045PB30", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "611 393 554", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "'L7', NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2016-03-18T00:00:00+00:00" + }, + { + "statementID": "78205d04-b700-2c35-ea8c-af8cb534bc11", + "statementType": "entityStatement", + "statementDate": "2023-03-30", + "entityType": "registeredEntity", + "name": "MONOOVA FX PTY LTD", + "incorporatedInJurisdiction": { + "name": "Australia", + "code": "AU" + }, + "identifiers": [ + { + "id": "98450051BS9C610A8T78", + "scheme": "XI-LEI", + "schemeName": "Global Legal Entity Identifier Index" + }, + { + "id": "151 337 852", + "schemeName": "RA000014" + } + ], + "addresses": [ + { + "type": "registered", + "address": "LEVEL 7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + }, + { + "type": "business", + "address": "LEVEL 7, NORTH SYDNEY", + "country": "AU", + "postCode": "2060" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-18", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + }, + "foundingDate": "2011-06-07T00:00:00+00:00" + } +] diff --git a/tests/fixtures/lei-updates-data2.json b/tests/fixtures/lei-updates-data2.json new file mode 100644 index 0000000..fcb36bb --- /dev/null +++ b/tests/fixtures/lei-updates-data2.json @@ -0,0 +1,623 @@ +[ + { + "LEI": "00TR8NKAEL48RGTZEW89", + "Entity": { + "LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", + "LegalAddress": { + "FirstAddressLine": "89 Nexus Way", + "City": "Camana Bay", + "Country": "KY", + "PostalCode": "KY1-9009" + }, + "HeadquartersAddress": { + "FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", + "AdditionalAddressLine": "Kaya Flamboyan 9", + "City": "Willemstad", + "Country": "CW" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000087", + "RegistrationAuthorityEntityID": "574206" + }, + "LegalJurisdiction": "KY", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "9999", + "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY" + }, + "EntityStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-08-24T20:53:00.000+02:00", + "LastUpdateDate": "2023-08-03T08:48:03.080+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2019-02-14T01:32:00.000+01:00", + "ManagingLOU": "39120001KULK7200U106", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000087", + "ValidationAuthorityEntityID": "574206" + } + } + }, + { + "LEI": "097900BHF50000076475", + "Entity": { + "LegalName": "VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000526", + "RegistrationAuthorityEntityID": "35903058" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "2EEG" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2004-10-09T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2017-05-30T00:00:00+02:00", + "LastUpdateDate": "2023-05-25T08:30:56.059+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-05-30T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000526", + "ValidationAuthorityEntityID": "35903058" + } + } + }, + { + "LEI": "097900BEJX0000002337", + "Entity": { + "LegalName": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s." + } + ], + "TransliteratedOtherEntityNames": [ + { + "type": "PREFERRED_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s." + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB04" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-23T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:04.159+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB04" + } + } + }, + { + "LEI": "097900BEJX0000002143", + "Entity": { + "LegalName": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB01" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2012-04-02T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:02.482+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB01" + } + } + }, + { + "LEI": "097900BEJX0000002240", + "Entity": { + "LegalName": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB02" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-22T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:03.502+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB02" + } + } + }, + { + "LEI": "097900BEJX0000002434", + "Entity": { + "LegalName": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.", + "AdditionalAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "820 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000706", + "RegistrationAuthorityEntityID": "SK35903058VUB03" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "FUND", + "LegalForm": { + "EntityLegalFormCode": "7EG2" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2005-03-22T08:00:00+01:00" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:04.659+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000706", + "ValidationAuthorityEntityID": "SK35903058VUB03" + } + } + }, + { + "LEI": "549300JB1P61FUTPEZ75", + "Entity": { + "LegalName": "Všeobecná úverová banka, a.s.", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "Vseobecna uverova banka, as" + } + ], + "LegalAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "829 90" + }, + "HeadquartersAddress": { + "FirstAddressLine": "Mlynské nivy 1", + "City": "Bratislava", + "Country": "SK", + "PostalCode": "829 90" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000526", + "RegistrationAuthorityEntityID": "31320155" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "2EEG" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "1992-04-01T08:00:00+02:00" + }, + "Registration": { + "InitialRegistrationDate": "2013-10-02T00:00:00+02:00", + "LastUpdateDate": "2023-09-12T08:30:46.285+02:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-10-03T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000526", + "ValidationAuthorityEntityID": "31320155" + } + } + }, + { + "LEI": "097900BJGO0000201513", + "Entity": { + "LegalName": "Aquila Real Asset Finance III a. s. v likvidácii", + "TransliteratedOtherEntityNames": [ + { + "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + "TransliteratedOtherEntityName": "AQUILA REAL ASSET FINANCE III A S V LIKVIDACII" + } + ], + "LegalAddress": { + "FirstAddressLine": "Dúbravská cesta 14", + "City": "Bratislava - mestská časť Karlova Ves", + "Country": "SK", + "PostalCode": "841 04" + }, + "HeadquartersAddress": { + "FirstAddressLine": "Dúbravská cesta 14", + "City": "Bratislava - mestská časť Karlova Ves", + "Country": "SK", + "PostalCode": "841 04" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000526", + "RegistrationAuthorityEntityID": "52143058" + }, + "LegalJurisdiction": "SK", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "2EEG" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2019-02-02T08:00:00+01:00", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "LIQUIDATION", + "LegalEntityEventEffectiveDate": "2023-04-13T08:00:00+02:00", + "LegalEntityEventRecordedDate": "2023-09-27T12:34:16.118+02:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "AffectedFields": [ + "Aquila Real Asset Finance III a. s. v likvidácii" + ] + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-06-14T00:00:00+02:00", + "LastUpdateDate": "2023-09-27T12:34:16.118+02:00", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2023-06-18T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000526", + "ValidationAuthorityEntityID": "52143058" + } + } + }, + { + "LEI": "9845002B9874D50A4531", + "Entity": { + "LegalName": "Corporate Business Solutions Partners S.à r.l.", + "LegalAddress": { + "FirstAddressLine": "23 Am Scheerleck", + "City": "Wecker", + "Country": "LU", + "PostalCode": "6868" + }, + "HeadquartersAddress": { + "FirstAddressLine": "23 Am Scheerleck", + "City": "Wecker", + "Country": "LU", + "PostalCode": "6868" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000432", + "RegistrationAuthorityEntityID": "B191847" + }, + "LegalJurisdiction": "LU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "DVXS" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2014-11-17T00:00:00+00:00" + }, + "Registration": { + "InitialRegistrationDate": "2020-03-17T08:22:48+00:00", + "LastUpdateDate": "2023-01-25T09:13:32+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-03-16T08:36:11+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000432", + "ValidationAuthorityEntityID": "B191847" + } + } + }, + { + "LEI": "2138002YAB7Z9KO21397", + "Entity": { + "LegalName": "MONOOVA GLOBAL PAYMENTS PTY LTD", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONOOVA GLOBAL PAYMENTS LIMITED" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONEYTECH LIMITED" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "SQUADRON FINANCIAL SERVICES LIMITED" + } + ], + "LegalAddress": { + "FirstAddressLine": "UNIT L7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "106 249 852" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "TXVC" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2003-09-09T00:00:00+00:00", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "CHANGE_LEGAL_FORM", + "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + }, + { + "LegalEntityEventType": "CHANGE_LEGAL_NAME", + "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-01-14T00:00:00+00:00", + "LastUpdateDate": "2023-01-11T11:21:01+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "106 249 852" + } + } + }, + { + "LEI": "984500501A1B1045PB30", + "Entity": { + "LegalName": "MONEYTECH GROUP LIMITED", + "LegalAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "'L7'", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "611 393 554" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "R4KK" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2016-03-18T00:00:00+00:00" + }, + "Registration": { + "InitialRegistrationDate": "2021-07-27T07:07:14+00:00", + "LastUpdateDate": "2022-02-23T15:28:31+00:00", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-07-27T07:07:14+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "611 393 554" + } + } + }, + { + "LEI": "98450051BS9C610A8T78", + "Entity": { + "LegalName": "MONOOVA FX PTY LTD", + "OtherEntityNames": [ + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "MONEYTECH FX PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "360 MARKETS PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "360FX PTY LTD" + }, + { + "type": "PREVIOUS_LEGAL_NAME", + "OtherEntityName": "CAPITAL AND FINANCE FOREX PTY LTD" + } + ], + "LegalAddress": { + "FirstAddressLine": "LEVEL 7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "HeadquartersAddress": { + "FirstAddressLine": "LEVEL 7", + "AdditionalAddressLine": "80 PACIFIC HIGHWAY", + "City": "NORTH SYDNEY", + "Region": "AU-NSW", + "Country": "AU", + "PostalCode": "2060" + }, + "RegistrationAuthority": { + "RegistrationAuthorityID": "RA000014", + "RegistrationAuthorityEntityID": "151 337 852" + }, + "LegalJurisdiction": "AU", + "EntityCategory": "GENERAL", + "LegalForm": { + "EntityLegalFormCode": "TXVC" + }, + "EntityStatus": "ACTIVE", + "EntityCreationDate": "2011-06-07T00:00:00+00:00", + "LegalEntityEvents": [ + { + "LegalEntityEventType": "CHANGE_LEGAL_ADDRESS", + "LegalEntityEventEffectiveDate": "2022-09-13T00:00:00+00:00", + "LegalEntityEventRecordedDate": "2023-03-30T14:18:04+00:00", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T12:21:06+00:00", + "LastUpdateDate": "2023-03-30T14:18:04+00:00", + "RegistrationStatus": "ISSUED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationAuthority": { + "ValidationAuthorityID": "RA000014", + "ValidationAuthorityEntityID": "151 337 852" + } + } + } +] diff --git a/tests/fixtures/lei-updates-data2.xml b/tests/fixtures/lei-updates-data2.xml new file mode 100644 index 0000000..4924cb8 --- /dev/null +++ b/tests/fixtures/lei-updates-data2.xml @@ -0,0 +1,645 @@ + + + 2023-11-06T08:20:40Z + GLEIF_FULL_PUBLISHED + 2495059 + + + + 00TR8NKAEL48RGTZEW89 + + ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD. + + 89 Nexus Way + Camana Bay + KY + KY1-9009 + + + C/O Citco Fund Services (Curacao) B.V. + Kaya Flamboyan 9 + Willemstad + CW + + + RA000087 + 574206 + + KY + GENERAL + + 9999 + CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY + + ACTIVE + + + 2012-08-24T20:53:00.000+02:00 + 2023-08-03T08:48:03.080+02:00 + ISSUED + 2019-02-14T01:32:00.000+01:00 + 39120001KULK7200U106 + FULLY_CORROBORATED + + RA000087 + 574206 + + + + + + 097900BHF50000076475 + + VÚB Generali dôchodková správcovská spoločnosť, a.s. + + VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000526 + 35903058 + + SK + GENERAL + + 2EEG + + ACTIVE + 2004-10-09T08:00:00+02:00 + + + 2017-05-30T00:00:00+02:00 + 2023-05-25T08:30:56.059+02:00 + ISSUED + 2024-05-30T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000526 + 35903058 + + + + + Mlynské nivy 1, null, null, null, SK, null, 820 04, Bratislava + 0.94 + pointAddress + 48.14687 + 17.12492 + 2016-05-17T10:50:50 + TopLeft.Latitude: 48.1479942, TopLeft.Longitude: 17.1232352, BottomRight.Latitude: 48.1457458, BottomRight.Longitude: 17.1266048 + houseNumber + Mlynské nivy 170/1, 821 09 Bratislava, Slovenská Republika + NT_kzljYRG5OuCE2R6Jrp7U1D_xcDMvED + Mlynské nivy + 170/1 + 821 09 + Bratislava + Bratislava I + Bratislavský kraj + SVK + + + + + 097900BEJX0000002337 + + SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s. + + MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + + SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s. + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB04 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-23T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:04.159+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB04 + + + + + + 097900BEJX0000002143 + + INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s. + + INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB01 + + SK + FUND + + 7EG2 + + ACTIVE + 2012-04-02T08:00:00+02:00 + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:02.482+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB01 + + + + + + 097900BEJX0000002240 + + KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB02 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-22T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:03.502+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB02 + + + + + + 097900BEJX0000002434 + + PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s. + + PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS + + + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + c/o VÚB Generali dôchodková správcovská spoločnosť, a.s. + Mlynské nivy 1 + Bratislava + SK + 820 04 + + + RA000706 + SK35903058VUB03 + + SK + FUND + + 7EG2 + + ACTIVE + 2005-03-22T08:00:00+01:00 + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:04.659+02:00 + ISSUED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000706 + SK35903058VUB03 + + + + + + 549300JB1P61FUTPEZ75 + + Všeobecná úverová banka, a.s. + + Vseobecna uverova banka, as + + + Mlynské nivy 1 + Bratislava + SK + 829 90 + + + Mlynské nivy 1 + Bratislava + SK + 829 90 + + + RA000526 + 31320155 + + SK + GENERAL + + 2EEG + + ACTIVE + 1992-04-01T08:00:00+02:00 + + + 2013-10-02T00:00:00+02:00 + 2023-09-12T08:30:46.285+02:00 + ISSUED + 2024-10-03T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000526 + 31320155 + + + + + + 097900BJGO0000201513 + + Aquila Real Asset Finance III a. s. v likvidácii + + AQUILA REAL ASSET FINANCE III A S V LIKVIDACII + + + Dúbravská cesta 14 + Bratislava - mestská časť Karlova Ves + SK + 841 04 + + + Dúbravská cesta 14 + Bratislava - mestská časť Karlova Ves + SK + 841 04 + + + RA000526 + 52143058 + + SK + GENERAL + + 2EEG + + ACTIVE + 2019-02-02T08:00:00+01:00 + + + LIQUIDATION + 2023-04-13T08:00:00+02:00 + 2023-09-27T12:34:16.118+02:00 + SUPPORTING_DOCUMENTS + + Aquila Real Asset Finance III a. s. v likvidácii + + + + + + 2019-06-14T00:00:00+02:00 + 2023-09-27T12:34:16.118+02:00 + LAPSED + 2023-06-18T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + + RA000526 + 52143058 + + + + + + 9845002B9874D50A4531 + + Corporate Business Solutions Partners S.à r.l. + + 23 Am Scheerleck + Wecker + LU + 6868 + + + 23 Am Scheerleck + Wecker + LU + 6868 + + + RA000432 + B191847 + + LU + GENERAL + + DVXS + + ACTIVE + 2014-11-17T00:00:00+00:00 + + + 2020-03-17T08:22:48+00:00 + 2023-01-25T09:13:32+00:00 + ISSUED + 2024-03-16T08:36:11+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000432 + B191847 + + + + + + 2138002YAB7Z9KO21397 + + MONOOVA GLOBAL PAYMENTS PTY LTD + + MONOOVA GLOBAL PAYMENTS LIMITED + MONEYTECH LIMITED + SQUADRON FINANCIAL SERVICES LIMITED + + + UNIT L7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 106 249 852 + + AU + GENERAL + + TXVC + + ACTIVE + 2003-09-09T00:00:00+00:00 + + + CHANGE_LEGAL_FORM + 2022-09-16T00:00:00+00:00 + 2023-01-11T09:59:14+00:00 + SUPPORTING_DOCUMENTS + + + CHANGE_LEGAL_NAME + 2022-09-16T00:00:00+00:00 + 2023-01-11T09:59:14+00:00 + SUPPORTING_DOCUMENTS + + + + + 2019-01-14T00:00:00+00:00 + 2023-01-11T11:21:01+00:00 + ISSUED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 106 249 852 + + + + + + 984500501A1B1045PB30 + + MONEYTECH GROUP LIMITED + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + 'L7' + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 611 393 554 + + AU + GENERAL + + R4KK + + ACTIVE + 2016-03-18T00:00:00+00:00 + + + 2021-07-27T07:07:14+00:00 + 2022-02-23T15:28:31+00:00 + LAPSED + 2022-07-27T07:07:14+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 611 393 554 + + + + + + 98450051BS9C610A8T78 + + MONOOVA FX PTY LTD + + MONEYTECH FX PTY LTD + 360 MARKETS PTY LTD + 360FX PTY LTD + CAPITAL AND FINANCE FOREX PTY LTD + + + LEVEL 7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + LEVEL 7 + 80 PACIFIC HIGHWAY + NORTH SYDNEY + AU-NSW + AU + 2060 + + + RA000014 + 151 337 852 + + AU + GENERAL + + TXVC + + ACTIVE + 2011-06-07T00:00:00+00:00 + + + CHANGE_LEGAL_ADDRESS + 2022-09-13T00:00:00+00:00 + 2023-03-30T14:18:04+00:00 + SUPPORTING_DOCUMENTS + + + + + 2019-05-15T12:21:06+00:00 + 2023-03-30T14:18:04+00:00 + ISSUED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + + RA000014 + 151 337 852 + + + + + + diff --git a/tests/fixtures/lei-updates-replaces-data.json b/tests/fixtures/lei-updates-replaces-data.json new file mode 100644 index 0000000..a243b90 --- /dev/null +++ b/tests/fixtures/lei-updates-replaces-data.json @@ -0,0 +1,4 @@ +[ +{"LEI": "549300JV7BZB002LHI61", "Entity": {"LegalName": "Helsingin Hammaslaboratorio Oy", "LegalAddress": {"FirstAddressLine": "c/o Eskola", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "Kulosaarentie 4-8 C 20", "City": "Helsinki", "Region": "FI-18", "Country": "FI", "PostalCode": "00570"}, "HeadquartersAddress": {"FirstAddressLine": "c/o Eskola", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "Kulosaarentie 4-8 C 20", "City": "Helsinki", "Region": "FI-18", "Country": "FI", "PostalCode": "00570"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000188", "RegistrationAuthorityEntityID": "0535671-9"}, "LegalJurisdiction": "FI", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "DKUW"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1983-09-15T02:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2017-10-17T03:14:00Z", "LastUpdateDate": "2023-12-29T09:43:20Z", "RegistrationStatus": "PENDING_TRANSFER", "NextRenewalDate": "2024-01-23T15:15:02Z", "ManagingLOU": "549300O897ZC5H7CY412", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000188", "ValidationAuthorityEntityID": "0535671-9"}}}, +{"LEI": "549300JV7BZB002LHI61", "Entity": {"LegalName": "Helsingin Hammaslaboratorio Oy", "LegalAddress": {"FirstAddressLine": "Kulosaarentie 4-8 C 20", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "00570 HELSINKI", "City": "HELSINKI", "Country": "FI", "PostalCode": "FI-00570"}, "HeadquartersAddress": {"FirstAddressLine": "Kulosaarentie 4-8 C 20", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "00570 HELSINKI", "City": "HELSINKI", "Country": "FI", "PostalCode": "FI-00570"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000188", "RegistrationAuthorityEntityID": "0535671-9"}, "LegalJurisdiction": "FI", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "DKUW"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1983-12-19T00:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2017-10-17T03:14:00+03:00", "LastUpdateDate": "2024-01-03T21:10:02+02:00", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2025-01-03T12:00:00+02:00", "ManagingLOU": "743700OO8O2N3TQKJC81", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000188", "ValidationAuthorityEntityID": "0535671-9"}}} +] diff --git a/tests/fixtures/relationship-update-data.json b/tests/fixtures/relationship-update-data.json new file mode 100644 index 0000000..4c9502a --- /dev/null +++ b/tests/fixtures/relationship-update-data.json @@ -0,0 +1,8 @@ +[ +{"LEI": "0292003540H0S4VA7A50", "Entity": {"LegalName": "FBNQUEST ASSET MANAGEMENT LIMITED", "LegalAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "HeadquartersAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI, LAGOS", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000469", "RegistrationAuthorityEntityID": "978831"}, "LegalJurisdiction": "NG", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "LIMITED"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2006-01-01T12:30:00Z"}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000469", "ValidationAuthorityEntityID": "978831"}}}, +{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}}, +{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}}, +{"LEI": "0292003540H0S4VA7A50", "Entity": {"LegalName": "FBNQUEST ASSET MANAGEMENT LIMITED", "LegalAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "HeadquartersAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI, LAGOS", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000469", "RegistrationAuthorityEntityID": "978831"}, "LegalJurisdiction": "NG", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "LIMITED"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2006-01-01T12:30:00Z"}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000469", "ValidationAuthorityEntityID": "978831"}}}, +{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}}, +{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}} +] diff --git a/tests/fixtures/repex-data.json b/tests/fixtures/repex-data.json new file mode 100644 index 0000000..91afde7 --- /dev/null +++ b/tests/fixtures/repex-data.json @@ -0,0 +1,62 @@ +[ + { + "LEI": "001GPB6A9XPE8XJICC14", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_KNOWN_PERSON", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "004L5FPTUREIWK9T2N63", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00EHHQ2ZHDCFXJCPCL46", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00GBW0Z2GYIER7DHDS71", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00KLB2PFTM3060S2N216", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_KNOWN_PERSON", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00QDBXDXLLF3W3JJJO36", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_KNOWN_PERSON", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00TV1D5YIV5IDUGWBW29", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "00X8DSV26QKJPKUT5B34", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "010G7UHBHEI87EKP0Q97", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-06-09T09:03:29Z" + }, + { + "LEI": "01370W6ZIY66KQ4J3570", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_PUBLIC", + "ContentDate": "2023-06-09T09:03:29Z" + } +] diff --git a/tests/fixtures/repex-updates-data2-new-out.json b/tests/fixtures/repex-updates-data2-new-out.json new file mode 100644 index 0000000..0b6e2be --- /dev/null +++ b/tests/fixtures/repex-updates-data2-new-out.json @@ -0,0 +1,653 @@ +[ + { + "statementID": "efc817ce-d25e-cae0-80f2-1405326c0cf6", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 159516LKTEGWITQUIE78", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "e8b77ad5-7ab6-629f-09a5-dcd040ba87ae", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "078879e7-147c-30de-163a-b13d7d5002f8" + }, + "interestedParty": { + "describedByEntityStatement": "efc817ce-d25e-cae0-80f2-1405326c0cf6" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 159516LKTEGWITQUIE78", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "f9b8a63c-8071-5d72-032f-8f790313a8e2", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 0292001398F7K0YJI588", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "a535fc20-3f6b-1502-e066-650a7eb73006", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "24d30d0b-3f58-7118-ec6a-0772ddb7dfa3" + }, + "interestedParty": { + "describedByEntityStatement": "f9b8a63c-8071-5d72-032f-8f790313a8e2" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 0292001398F7K0YJI588", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "cbb59de1-4612-07d8-59c9-78eb080dc028", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity." + } + }, + { + "statementID": "65706612-e115-032d-daca-059196e046c8", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "a12cd374-be9e-e6f3-eb97-2e9b143a049d" + }, + "interestedParty": { + "describedByPersonStatement": "cbb59de1-4612-07d8-59c9-78eb080dc028" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "960d8dad-3fb8-270c-7396-7f1800d3181d", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity." + } + }, + { + "statementID": "e521936e-a800-f55b-1cf4-c0e85d16d810", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "30332285-0d9a-7256-449c-d917de1685c2" + }, + "interestedParty": { + "describedByPersonStatement": "960d8dad-3fb8-270c-7396-7f1800d3181d" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "58a27735-dda3-7175-8521-d0ea1b4d1fba", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 0292001018B2N4DBI146", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf." + } + }, + { + "statementID": "13325e39-b4a4-8ce2-bd34-b12e1e22ce7d", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "187cceb8-30de-1aed-0723-d006c6f47260" + }, + "interestedParty": { + "describedByEntityStatement": "58a27735-dda3-7175-8521-d0ea1b4d1fba" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 0292001018B2N4DBI146", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "db0e963d-8d2d-3afb-e836-28b76dab3c6e", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "282cd39f-4c8d-3ca4-dd39-85919c649f6a" + }, + "interestedParty": { + "describedByEntityStatement": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "f4869861-cb38-f5a8-66a3-82ddefb10f83", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity. ExemptionReference provided: Datenschutz!" + } + }, + { + "statementID": "15c1486d-9cdb-eae6-b1fb-3fa0456d070b", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "6d2a451e-8449-37c7-1cb2-44a1633fd8d8" + }, + "interestedParty": { + "describedByPersonStatement": "f4869861-cb38-f5a8-66a3-82ddefb10f83" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/repex-updates-data2-new.json b/tests/fixtures/repex-updates-data2-new.json new file mode 100644 index 0000000..4e39b50 --- /dev/null +++ b/tests/fixtures/repex-updates-data2-new.json @@ -0,0 +1,50 @@ +[ + { + "LEI": "159516LKTEGWITQUIE78", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "0292001398F7K0YJI588", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "004L5FPTUREIWK9T2N63", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_KNOWN_PERSON", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "020BQJXAXCZNLKIN7326", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NATURAL_PERSONS", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "0292001018B2N4DBI146", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_LEI", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "2549007VY3IWUVGW7A82", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "Extension": { + "Deletion": { + "DeletedAt": "2023-11-24T02:07:59Z" + } + }, + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "159515PKYKQYJLT0KF16", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NATURAL_PERSONS", + "ExceptionReference": "Datenschutz!", + "ContentDate": "2023-11-06T09:13:21Z" + } +] diff --git a/tests/fixtures/repex-updates-data2-new.xml b/tests/fixtures/repex-updates-data2-new.xml new file mode 100644 index 0000000..ac15677 --- /dev/null +++ b/tests/fixtures/repex-updates-data2-new.xml @@ -0,0 +1,58 @@ + + + 2023-11-06T09:13:21Z + GLEIF_FULL_PUBLISHED + 4458620 + + + + + 159516LKTEGWITQUIE78 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + + 0292001398F7K0YJI588 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + + 004L5FPTUREIWK9T2N63 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NO_KNOWN_PERSON + + + + 020BQJXAXCZNLKIN7326 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NATURAL_PERSONS + + + + 0292001018B2N4DBI146 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NO_LEI + + + + 2549007VY3IWUVGW7A82 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + 2023-11-24T02:07:59Z + + + + + + 159515PKYKQYJLT0KF16 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NATURAL_PERSONS + Datenschutz! + + + + diff --git a/tests/fixtures/repex-updates-data2-out.json b/tests/fixtures/repex-updates-data2-out.json new file mode 100644 index 0000000..38296cc --- /dev/null +++ b/tests/fixtures/repex-updates-data2-out.json @@ -0,0 +1,653 @@ +[ + { + "statementID": "854a1c24-5b4e-1587-9dea-222bc503163b", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 0292001398F7K0YJI588", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity." + } + }, + { + "statementID": "c33246ac-e412-9e13-6f17-410c94ca303e", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "24d30d0b-3f58-7118-ec6a-0772ddb7dfa3" + }, + "interestedParty": { + "describedByPersonStatement": "854a1c24-5b4e-1587-9dea-222bc503163b" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 0292001398F7K0YJI588", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "b964a2b7-9a2b-9d5e-4f1d-7802fb8bba82", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "0fd27404-0f10-f9e7-1cb1-46c7d6e1cb4e", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "a12cd374-be9e-e6f3-eb97-2e9b143a049d" + }, + "interestedParty": { + "describedByEntityStatement": "b964a2b7-9a2b-9d5e-4f1d-7802fb8bba82" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "faaa96d7-b2bb-73e6-5d77-a0b623f9c2c5", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf." + } + }, + { + "statementID": "8638ee52-50f6-b3d9-bb81-a7e5f5b9c070", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "30332285-0d9a-7256-449c-d917de1685c2" + }, + "interestedParty": { + "describedByEntityStatement": "faaa96d7-b2bb-73e6-5d77-a0b623f9c2c5" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "c635057b-a7a5-1bf7-e323-0b8aef4e3347", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 0292001018B2N4DBI146", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity." + } + }, + { + "statementID": "25effa14-462b-7559-361c-1e62bb33fe5d", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "187cceb8-30de-1aed-0723-d006c6f47260" + }, + "interestedParty": { + "describedByPersonStatement": "c635057b-a7a5-1bf7-e323-0b8aef4e3347" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 0292001018B2N4DBI146", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "894e8dc1-432c-d922-6517-e84923ee7b50", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for ZZZWS8Y6XVKM0Q9RJQ79", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "ae1f191a-8a87-e58c-7a60-4fa7d727cbbb", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "6844aee7-92e6-9b58-111f-c0685b52d81e" + }, + "interestedParty": { + "describedByEntityStatement": "894e8dc1-432c-d922-6517-e84923ee7b50" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "indirect", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for ZZZWS8Y6XVKM0Q9RJQ79", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708", + "statementType": "entityStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "entityType": "unknownEntity", + "unspecifiedEntityDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } + }, + { + "statementID": "db0e963d-8d2d-3afb-e836-28b76dab3c6e", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "282cd39f-4c8d-3ca4-dd39-85919c649f6a" + }, + "interestedParty": { + "describedByEntityStatement": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "9f8997ec-809e-2ed1-b56b-93269498eecf", + "statementType": "personStatement", + "statementDate": "2023-11-06", + "annotations": [ + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + }, + "personType": "unknownPerson", + "unspecifiedPersonDetails": { + "reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity. ExemptionReference provided: Datenschutz" + } + }, + { + "statementID": "3fadbdb5-dd32-800e-c2d1-5dc1cd28754f", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-11-06", + "subject": { + "describedByEntityStatement": "6d2a451e-8449-37c7-1cb2-44a1633fd8d8" + }, + "interestedParty": { + "describedByPersonStatement": "9f8997ec-809e-2ed1-b56b-93269498eecf" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "details": "A controlling interest." + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "The nature of this interest is unknown", + "statementPointerTarget": "/interests/0/type", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + }, + { + "motivation": "commenting", + "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16", + "statementPointerTarget": "/", + "creationDate": "2024-02-15", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-15", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/repex-updates-data2.json b/tests/fixtures/repex-updates-data2.json new file mode 100644 index 0000000..06d2df2 --- /dev/null +++ b/tests/fixtures/repex-updates-data2.json @@ -0,0 +1,45 @@ +[ + { + "LEI": "0292001398F7K0YJI588", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_KNOWN_PERSON", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "004L5FPTUREIWK9T2N63", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "020BQJXAXCZNLKIN7326", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NO_LEI", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "0292001018B2N4DBI146", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NATURAL_PERSONS", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "ZZZWS8Y6XVKM0Q9RJQ79", + "ExceptionCategory": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "2549007VY3IWUVGW7A82", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NON_CONSOLIDATING", + "ContentDate": "2023-11-06T09:13:21Z" + }, + { + "LEI": "159515PKYKQYJLT0KF16", + "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", + "ExceptionReason": "NATURAL_PERSONS", + "ExceptionReference": "Datenschutz", + "ContentDate": "2023-11-06T09:13:21Z" + } +] diff --git a/tests/fixtures/repex-updates-data2.xml b/tests/fixtures/repex-updates-data2.xml new file mode 100644 index 0000000..10732cd --- /dev/null +++ b/tests/fixtures/repex-updates-data2.xml @@ -0,0 +1,54 @@ + + + 2023-11-06T09:13:21Z + GLEIF_FULL_PUBLISHED + 4458620 + + + + + 0292001398F7K0YJI588 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NO_KNOWN_PERSON + + + + 004L5FPTUREIWK9T2N63 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + + 020BQJXAXCZNLKIN7326 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NO_LEI + + + + 0292001018B2N4DBI146 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NATURAL_PERSONS + + + + + ZZZWS8Y6XVKM0Q9RJQ79 + ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + + 2549007VY3IWUVGW7A82 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NON_CONSOLIDATING + + + + 159515PKYKQYJLT0KF16 + DIRECT_ACCOUNTING_CONSOLIDATION_PARENT + NATURAL_PERSONS + Datenschutz + + + + diff --git a/tests/fixtures/reporting-exceptions-data.json b/tests/fixtures/reporting-exceptions-data.json new file mode 100644 index 0000000..fa32627 --- /dev/null +++ b/tests/fixtures/reporting-exceptions-data.json @@ -0,0 +1,5 @@ +[ +{"LEI": "959800W87BKGBUDPF915", "Entity": {"LegalName": "COMUNIDAD HEREDITARIA FORAL DE MARIA BEGO\u00d1A BENITO PE\u00d1A", "LegalAddress": {"FirstAddressLine": "JUAN BAUTISTA URIARTE 9 4-B", "AddressNumber": "9", "AddressNumberWithinBuilding": "4-B", "City": "Galdakao", "Country": "ES", "PostalCode": "48960"}, "HeadquartersAddress": {"FirstAddressLine": "JUAN BAUTISTA URIARTE 9 4-B", "AddressNumber": "9", "AddressNumberWithinBuilding": "4-B", "City": "Galdakao", "Country": "ES", "PostalCode": "48960"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000535", "RegistrationAuthorityEntityID": "V95881967"}, "LegalJurisdiction": "ES", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "COMUNIDAD HEREDITARIA FORAL"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2018-01-26T13:04:07.145+01:00", "LastUpdateDate": "2022-01-25T23:30:01.829+01:00", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2022-01-25T16:24:15.251+01:00", "ManagingLOU": "959800R2X69K6Y6MX775", "ValidationSources": "PARTIALLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000535", "ValidationAuthorityEntityID": "V95881967"}}}, +{"LEI": "959800W87BKGBUDPF915", "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", "ExceptionReason": "NON_CONSOLIDATING", "ContentDate": "2024-01-01T01:18:56Z"}, +{"LEI": "959800W87BKGBUDPF915", "ExceptionCategory": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT", "ExceptionReason": "NON_CONSOLIDATING", "ContentDate": "2024-01-01T01:18:56Z"} +] diff --git a/tests/fixtures/rr-data-out.json b/tests/fixtures/rr-data-out.json new file mode 100644 index 0000000..9bcf3f7 --- /dev/null +++ b/tests/fixtures/rr-data-out.json @@ -0,0 +1,351 @@ +[ + { + "statementID": "d459ada3-ee9d-5ee9-f40a-ac5aadecdd64", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-18", + "subject": { + "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419" + }, + "interestedParty": { + "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2012-11-29T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "ce4b78b2-f797-cf87-c853-49f972f516d0", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-18", + "subject": { + "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419" + }, + "interestedParty": { + "describedByEntityStatement": "9687e19c-1d19-5cdf-f4c6-954aa05ea842" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2012-11-29T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "bc2c8a8f-a32d-92f0-0d6c-43b4d2980287", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-18", + "subject": { + "describedByEntityStatement": "17b29ec4-1a80-0acf-7207-50148bef2e43" + }, + "interestedParty": { + "describedByEntityStatement": "ac13a555-2a3e-4bf8-e71d-f968e65f30ef" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-18T08:47:10.475Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "060c7f33-7ea5-769e-164a-d2683eebf983", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-18", + "subject": { + "describedByEntityStatement": "17b29ec4-1a80-0acf-7207-50148bef2e43" + }, + "interestedParty": { + "describedByEntityStatement": "81e07925-b7f6-9d2f-0b6e-7b6889561510" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-18T08:47:10.570Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "95002ea4-69a1-da3f-4d34-4fad484a6fb7", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-02", + "subject": { + "describedByEntityStatement": "8ca194d4-ee72-56c4-e39d-0447dbcf0d17" + }, + "interestedParty": { + "describedByEntityStatement": "7f9cc65e-8195-cb0a-08d3-6b99cce86690" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-02T06:05:38.072Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "ecc27092-3fda-2b48-4500-6cfcb61964f8", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-01", + "subject": { + "describedByEntityStatement": "099fd796-e9c1-d2a4-4efe-8843d421d827" + }, + "interestedParty": { + "describedByEntityStatement": "95bec03f-6cf6-ba02-cda7-75ef614f9798" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-01T05:07:23.076Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "8729e7c2-9037-f289-057f-cb9a6e42f8e9", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-01", + "subject": { + "describedByEntityStatement": "099fd796-e9c1-d2a4-4efe-8843d421d827" + }, + "interestedParty": { + "describedByEntityStatement": "2335891d-0bd6-e0ba-e0b2-27c58e383a38" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-01T05:07:23.152Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "4138bbc8-dc3c-e8ee-dfe8-f9876cb83500", + "statementType": "ownershipOrControlStatement", + "statementDate": "2022-06-04", + "subject": { + "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8" + }, + "interestedParty": { + "describedByEntityStatement": "f910bb52-6772-f634-698e-392402bd3dca" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2018-02-06T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "3cd06b78-3c1c-0cab-881b-cbbc36af1741", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-01-24", + "subject": { + "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8" + }, + "interestedParty": { + "describedByEntityStatement": "20815b26-efdc-a516-a905-7abdfa63d128" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "indirect", + "beneficialOwnershipOrControl": false, + "startDate": "2018-02-06T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "208d41ba-977c-6760-801e-77fdc9d47307", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-16", + "subject": { + "describedByEntityStatement": "06999bda-3717-8acb-719e-d6dd6d145c2a" + }, + "interestedParty": { + "describedByEntityStatement": "5bd23434-6df3-b2cf-4f28-83f16e6fea7a" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2023-05-09T06:14:06.846Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-13", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/rr-data.json b/tests/fixtures/rr-data.json new file mode 100644 index 0000000..e30e317 --- /dev/null +++ b/tests/fixtures/rr-data.json @@ -0,0 +1,327 @@ +[ + { + "Relationship": { + "StartNode": { + "NodeID": "001GPB6A9XPE8XJICC14", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "5493001Z012YSB2A0K51", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2012-11-29T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2022-11-14T09:59:48.000Z", + "LastUpdateDate": "2023-05-18T15:41:20.212Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-18T15:48:53.604Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "001GPB6A9XPE8XJICC14", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "C7J4FOV6ELAVE39B7M82", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_SUBFUND_OF", + "RelationshipPeriods": [ + { + "StartDate": "2012-11-29T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2022-11-14T09:59:48.000Z", + "LastUpdateDate": "2023-05-18T15:41:20.212Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-18T15:48:53.604Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/cgi-bin/browse-edgar?scd=series&CIK=0000722574&action=getcompany" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "00KLB2PFTM3060S2N216", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "YVC19L6NOWRA51XIKW06", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-18T08:47:10.475Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-11-05T18:17:00.000Z", + "LastUpdateDate": "2023-05-18T08:47:10.462Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-11T18:37:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/872323/000110465923007476/tm231669d1_485bpos.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "00KLB2PFTM3060S2N216", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "54930037R1JTV3ZYFY28", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_SUBFUND_OF", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-18T08:47:10.570Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-11-05T18:17:00.000Z", + "LastUpdateDate": "2023-05-18T08:47:10.559Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-11T18:37:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/cgi-bin/series?company=HARRIS+ASSOCIATES+INVESTMENT+TRUST" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "00QDBXDXLLF3W3JJJO36", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "549300FF2KD1XH54I851", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-02T06:05:38.072Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2021-03-23T21:33:00.000Z", + "LastUpdateDate": "2023-05-02T06:08:13.031Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-04-21T15:23:48.020Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/0001052118/000168386323000377/f24169d0.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "00X8DSV26QKJPKUT5B34", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "7HTL8AEQSEDX602FBU63", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-01T05:07:23.076Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-06-06T15:51:00.000Z", + "LastUpdateDate": "2023-05-01T05:07:23.062Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2023-12-22T00:00:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/Archives/edgar/data/912029/000174177323000891/c497k.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "00X8DSV26QKJPKUT5B34", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "549300S84WGKQCATYO76", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_SUBFUND_OF", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-01T05:07:23.152Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2017-08-25T01:37:00.000Z", + "LastUpdateDate": "2023-05-01T05:07:23.140Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2023-12-22T00:00:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/cgi-bin/browse-edgar?CIK=0000912029&action=getcompany&scd=series" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "010CMKZ3VON21WF2ZD45", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "ZZG38T0MDR3QY1ETUA76", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-01-01T00:00:00.000Z", + "EndDate": "2018-12-31T00:00:00.000Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-02-06T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2012-10-01T11:09:00.000Z", + "LastUpdateDate": "2022-06-04T01:00:00.000Z", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-06-03T13:04:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "ENTITY_SUPPLIED_ONLY", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "010CMKZ3VON21WF2ZD45", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "3C7474T6CDKPR9K6YT90", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2020-01-01T00:00:00.000Z", + "EndDate": "2020-12-31T00:00:00.000Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-02-06T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2020-01-01T00:00:00.000Z", + "EndDate": "2020-12-31T00:00:00.000Z", + "PeriodType": "DOCUMENT_FILING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "US_GAAP" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2012-10-01T11:09:00.000Z", + "LastUpdateDate": "2023-01-24T15:32:00.000Z", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-06-03T13:04:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "REGULATORY_FILING", + "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/40545/000004054521000011/ge-20201231.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "010G7UHBHEI87EKP0Q97", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "549300GC5MDF1KXYMP06", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2023-05-09T06:14:06.846Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2012-06-06T15:57:00.000Z", + "LastUpdateDate": "2023-05-16T14:21:08.886Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-06-27T00:00:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/Archives/edgar/data/1458460/000140508622000264/xslFormDX01/primary_doc.xml" + } + } +] diff --git a/tests/fixtures/rr-updates-data-out.json b/tests/fixtures/rr-updates-data-out.json new file mode 100644 index 0000000..4fdc133 --- /dev/null +++ b/tests/fixtures/rr-updates-data-out.json @@ -0,0 +1,106 @@ +[ + { + "statementID": "00420996-34d4-e085-1366-d9bae2178613", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-07-18", + "subject": { + "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419" + }, + "interestedParty": { + "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2012-11-29T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-20", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "502d54ca-ca55-843b-9db2-e516643563d6", + "statementType": "ownershipOrControlStatement", + "statementDate": "2022-08-04", + "subject": { + "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8" + }, + "interestedParty": { + "describedByEntityStatement": "f910bb52-6772-f634-698e-392402bd3dca" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2018-02-06T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-20", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister" + ], + "description": "GLEIF" + } + }, + { + "statementID": "8fec0d1c-b31b-7632-7c47-901e10bcd367", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-03-24", + "subject": { + "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8" + }, + "interestedParty": { + "describedByEntityStatement": "20815b26-efdc-a516-a905-7abdfa63d128" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "indirect", + "beneficialOwnershipOrControl": false, + "startDate": "2018-02-06T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-11-20", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/rr-updates-data.json b/tests/fixtures/rr-updates-data.json new file mode 100644 index 0000000..78023d7 --- /dev/null +++ b/tests/fixtures/rr-updates-data.json @@ -0,0 +1,117 @@ +[ + { + "Relationship": { + "StartNode": { + "NodeID": "001GPB6A9XPE8XJICC14", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "5493001Z012YSB2A0K51", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2012-11-29T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2022-11-14T09:59:48.000Z", + "LastUpdateDate": "2023-07-18T15:41:20.212Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-18T15:48:53.604Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "010CMKZ3VON21WF2ZD45", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "ZZG38T0MDR3QY1ETUA76", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-01-01T00:00:00.000Z", + "EndDate": "2018-12-31T00:00:00.000Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-02-06T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2012-10-01T11:09:00.000Z", + "LastUpdateDate": "2022-08-04T01:00:00.000Z", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-06-03T13:04:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "ENTITY_SUPPLIED_ONLY", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "010CMKZ3VON21WF2ZD45", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "3C7474T6CDKPR9K6YT90", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2020-01-01T00:00:00.000Z", + "EndDate": "2020-12-31T00:00:00.000Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-02-06T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2020-01-01T00:00:00.000Z", + "EndDate": "2020-12-31T00:00:00.000Z", + "PeriodType": "DOCUMENT_FILING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "US_GAAP" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2012-10-01T11:09:00.000Z", + "LastUpdateDate": "2023-03-24T15:32:00.000Z", + "RegistrationStatus": "LAPSED", + "NextRenewalDate": "2022-06-03T13:04:00.000Z", + "ManagingLOU": "EVK05KS7XY1DEII3R011", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "REGULATORY_FILING", + "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/40545/000004054521000011/ge-20201231.htm" + } + } +] diff --git a/tests/fixtures/rr-updates-data2-new-out.json b/tests/fixtures/rr-updates-data2-new-out.json new file mode 100644 index 0000000..7f43921 --- /dev/null +++ b/tests/fixtures/rr-updates-data2-new-out.json @@ -0,0 +1,352 @@ +[ + { + "statementID": "9b2197d7-2d52-5b77-1356-c9d62b5ce670", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-08-23", + "subject": { + "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419" + }, + "interestedParty": { + "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2012-11-29T00:00:00.000Z" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "724871b2-e240-68c9-030c-c85370efa7c4", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-09-17", + "subject": { + "describedByEntityStatement": "d400beea-3cdc-a894-ce0d-9ce8fa67c30b" + }, + "interestedParty": { + "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-23T08:00:00+01:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "46af8fa4-1b3d-00db-c440-7b53bee06464", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-09-17", + "subject": { + "describedByEntityStatement": "19fb3b00-c801-7ff9-fae7-a1955c772e71" + }, + "interestedParty": { + "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2012-04-02T08:00:00+02:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "90ccd99f-086c-1bfe-211e-330266d50cb1", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-09-17", + "subject": { + "describedByEntityStatement": "1ff23b39-62f3-4744-da92-b11c2c3f8ff3" + }, + "interestedParty": { + "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-22T08:00:00+01:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "5729d9a2-f80d-b945-930f-0ed3c210f222", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-09-17", + "subject": { + "describedByEntityStatement": "a8866f73-572f-5ec5-6b42-2fe78b72f128" + }, + "interestedParty": { + "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-22T08:00:00+01:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "41e51ae9-3a31-e7ed-10a2-7bb13375ccf6", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-06-25", + "subject": { + "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30" + }, + "interestedParty": { + "describedByEntityStatement": "426b5ddd-267f-0020-ae05-5dbc1e0f3341" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2022-10-24T08:00:00+02:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "c1aa1715-1cd3-b105-842a-1da48c0b5e30", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-02-11", + "subject": { + "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749" + }, + "interestedParty": { + "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2019-06-30T00:00:00+00:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "82d4a9f1-5ee6-d620-1a85-e45323563dde", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-02-11", + "subject": { + "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749" + }, + "interestedParty": { + "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "indirect", + "beneficialOwnershipOrControl": false, + "startDate": "2019-06-30T00:00:00+00:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "e6afa469-a581-82cd-e9b8-63763fae365d", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-04-30", + "subject": { + "describedByEntityStatement": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81" + }, + "interestedParty": { + "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "direct", + "beneficialOwnershipOrControl": false, + "startDate": "2018-06-30T00:00:00+00:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "5c126e1c-83f9-fece-2f82-70af13007c69", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-04-30", + "subject": { + "describedByEntityStatement": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81" + }, + "interestedParty": { + "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "indirect", + "beneficialOwnershipOrControl": false, + "startDate": "2018-06-30T00:00:00+00:00" + } + ], + "publicationDetails": { + "publicationDate": "2023-12-11", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/rr-updates-data2-new.json b/tests/fixtures/rr-updates-data2-new.json new file mode 100644 index 0000000..1d7fa9f --- /dev/null +++ b/tests/fixtures/rr-updates-data2-new.json @@ -0,0 +1,374 @@ +[ + { + "Relationship": { + "StartNode": { + "NodeID": "001GPB6A9XPE8XJICC14", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "5493001Z012YSB2A0K51", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2012-11-29T00:00:00.000Z", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2022-11-14T09:59:48.000Z", + "LastUpdateDate": "2023-08-23T18:46:01.213Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-18T15:48:53.604Z", + "ManagingLOU": "5493001KJTIIGC8Y1R12", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS", + "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002337", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-23T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:04.159+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002143", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2012-04-02T08:00:00+02:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:02.482+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002240", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-22T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:03.502+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002434", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-22T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-09-17T08:31:04.659+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "549300JB1P61FUTPEZ75", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2022-10-24T08:00:00+02:00", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2022-01-01T08:00:00+01:00", + "EndDate": "2022-12-31T08:00:00+01:00", + "PeriodType": "ACCOUNTING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "IFRS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-05-22T12:27:52.655+02:00", + "LastUpdateDate": "2023-06-25T08:30:56.059+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-30T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-06-30T00:00:00+00:00", + "EndDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-01-10T17:29:10+00:00", + "LastUpdateDate": "2023-02-11T11:21:01+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-06-30T00:00:00+00:00", + "EndDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-01-10T17:29:10+00:00", + "LastUpdateDate": "2023-02-11T11:21:01+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "98450051BS9C610A8T78", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2019-06-30T00:00:00+00:00", + "EndDate": "2020-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T00:00:00+00:00", + "LastUpdateDate": "2023-04-30T14:18:04+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "98450051BS9C610A8T78", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2019-06-30T00:00:00+00:00", + "EndDate": "2020-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T00:00:00+00:00", + "LastUpdateDate": "2023-04-30T14:18:04+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + }, + "Extension": { + "Deletion": { + "DeletedAt": "2023-11-01T01:39:19Z" + } + } + } +] diff --git a/tests/fixtures/rr-updates-data2-new.xml b/tests/fixtures/rr-updates-data2-new.xml new file mode 100644 index 0000000..19ea704 --- /dev/null +++ b/tests/fixtures/rr-updates-data2-new.xml @@ -0,0 +1,411 @@ + + + 2023-11-06T09:29:01Z + GLEIF_FULL_PUBLISHED + 407916 + + + + + + 001GPB6A9XPE8XJICC14 + LEI + + + 5493001Z012YSB2A0K51 + LEI + + IS_FUND-MANAGED_BY + + + 2012-11-29T00:00:00.000Z + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2022-11-14T09:59:48.000Z + 2023-08-23T18:46:01.213Z + PUBLISHED + 2024-05-18T15:48:53.604Z + 5493001KJTIIGC8Y1R12 + FULLY_CORROBORATED + SUPPORTING_DOCUMENTS + https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm + + + + + + 097900BEJX0000002337 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-23T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:04.159+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002143 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2012-04-02T08:00:00+02:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:02.482+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002240 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-22T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:03.502+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002434 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-22T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-09-17T08:31:04.659+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BHF50000076475 + LEI + + + 549300JB1P61FUTPEZ75 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2022-10-24T08:00:00+02:00 + RELATIONSHIP_PERIOD + + + 2022-01-01T08:00:00+01:00 + 2022-12-31T08:00:00+01:00 + ACCOUNTING_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + IFRS + + + + + 2023-05-22T12:27:52.655+02:00 + 2023-06-25T08:30:56.059+02:00 + PUBLISHED + 2024-05-30T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + SUPPORTING_DOCUMENTS + + + + + + 2138002YAB7Z9KO21397 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2018-06-30T00:00:00+00:00 + 2019-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2019-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2023-01-10T17:29:10+00:00 + 2023-02-11T11:21:01+00:00 + PUBLISHED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 2138002YAB7Z9KO21397 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_ULTIMATELY_CONSOLIDATED_BY + + + 2018-06-30T00:00:00+00:00 + 2019-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2019-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2023-01-10T17:29:10+00:00 + 2023-02-11T11:21:01+00:00 + PUBLISHED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 98450051BS9C610A8T78 + LEI + + + 2138002YAB7Z9KO21397 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2019-06-30T00:00:00+00:00 + 2020-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2018-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + 2019-05-15T00:00:00+00:00 + 2023-04-30T14:18:04+00:00 + PUBLISHED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 98450051BS9C610A8T78 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_ULTIMATELY_CONSOLIDATED_BY + + + 2019-06-30T00:00:00+00:00 + 2020-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2018-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2019-05-15T00:00:00+00:00 + 2023-04-30T14:18:04+00:00 + PUBLISHED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + 2023-11-01T01:39:19Z + + + + + diff --git a/tests/fixtures/rr-updates-data2-out.json b/tests/fixtures/rr-updates-data2-out.json new file mode 100644 index 0000000..c7a275f --- /dev/null +++ b/tests/fixtures/rr-updates-data2-out.json @@ -0,0 +1,530 @@ +[ + { + "statementID": "d4e3f1bd-05dc-6a36-6341-ed769a2672a4", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-08-17", + "subject": { + "describedByEntityStatement": "8a8e8ee2-5bb4-7582-9b79-d531b6b59c12" + }, + "interestedParty": { + "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-23T08:00:00+01:00", + "details": "LEI RelationshipType: IS_FUND-MANAGED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BEJX0000002337 is subject, 097900BHF50000076475 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "5ebf3d86-dcba-7749-b419-73c3e91cb497", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-08-17", + "subject": { + "describedByEntityStatement": "d60fa665-032c-6992-895f-66cdc7509909" + }, + "interestedParty": { + "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2012-04-02T08:00:00+02:00", + "details": "LEI RelationshipType: IS_FUND-MANAGED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BEJX0000002143 is subject, 097900BHF50000076475 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "7ed9c1c0-5f2d-acb9-a351-0dce16a4ca67", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-08-17", + "subject": { + "describedByEntityStatement": "64b41c35-f336-0010-ae81-785d33e65ffd" + }, + "interestedParty": { + "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-22T08:00:00+01:00", + "details": "LEI RelationshipType: IS_FUND-MANAGED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BEJX0000002240 is subject, 097900BHF50000076475 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "b08a654d-6a50-16c2-f93f-8dcb54621f55", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-08-17", + "subject": { + "describedByEntityStatement": "4f32f254-ec84-a32a-79d1-a036dfa789e6" + }, + "interestedParty": { + "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2005-03-22T08:00:00+01:00", + "details": "LEI RelationshipType: IS_FUND-MANAGED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BEJX0000002434 is subject, 097900BHF50000076475 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "458cea07-b906-bcf2-1257-ab270a8010b0", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-05-25", + "subject": { + "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938" + }, + "interestedParty": { + "describedByEntityStatement": "eca994a4-c50a-62b4-1308-9077410ee7b2" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2022-10-24T08:00:00+02:00", + "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BHF50000076475 is subject, 549300JB1P61FUTPEZ75 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "e313cd90-2781-30bd-cfd3-39243fbf401d", + "statementType": "ownershipOrControlStatement", + "statementDate": "2020-07-16", + "subject": { + "describedByEntityStatement": "71da391b-9385-4a1c-c5fc-e040a762b172" + }, + "interestedParty": { + "describedByEntityStatement": "fd9f3a92-ed00-95f2-b005-45772e4f40b7" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2014-11-06T00:00:00Z", + "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BJGO0000201513 is subject, 9845002B9874D50A4531 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "f3af408c-b46e-491c-0ccc-8928fcca3f98", + "statementType": "ownershipOrControlStatement", + "statementDate": "2020-07-16", + "subject": { + "describedByEntityStatement": "71da391b-9385-4a1c-c5fc-e040a762b172" + }, + "interestedParty": { + "describedByEntityStatement": "fd9f3a92-ed00-95f2-b005-45772e4f40b7" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2014-11-06T00:00:00Z", + "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 097900BJGO0000201513 is subject, 9845002B9874D50A4531 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "23f5dca9-1954-91e8-787e-8e80460d5aeb", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-01-11", + "subject": { + "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e" + }, + "interestedParty": { + "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2019-06-30T00:00:00+00:00", + "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 2138002YAB7Z9KO21397 is subject, 984500501A1B1045PB30 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "10b394b0-8403-9183-1b28-26ea4ccf4567", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-01-11", + "subject": { + "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e" + }, + "interestedParty": { + "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2019-06-30T00:00:00+00:00", + "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 2138002YAB7Z9KO21397 is subject, 984500501A1B1045PB30 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "65665354-d0c9-457d-f8b7-11eff538a891", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-03-30", + "subject": { + "describedByEntityStatement": "78205d04-b700-2c35-ea8c-af8cb534bc11" + }, + "interestedParty": { + "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2018-06-30T00:00:00+00:00", + "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 98450051BS9C610A8T78 is subject, 2138002YAB7Z9KO21397 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + }, + { + "statementID": "77e6d2fb-f1e9-af58-3d47-45f261609ba0", + "statementType": "ownershipOrControlStatement", + "statementDate": "2023-03-30", + "subject": { + "describedByEntityStatement": "78205d04-b700-2c35-ea8c-af8cb534bc11" + }, + "interestedParty": { + "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e" + }, + "interests": [ + { + "type": "other-influence-or-control", + "interestLevel": "unknown", + "beneficialOwnershipOrControl": false, + "startDate": "2018-06-30T00:00:00+00:00", + "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY" + } + ], + "annotations": [ + { + "motivation": "commenting", + "description": "Describes GLEIF relationship: 98450051BS9C610A8T78 is subject, 984500501A1B1045PB30 is interested party", + "statementPointerTarget": "/", + "creationDate": "2024-02-16", + "createdBy": { + "name": "Open Ownership", + "uri": "https://www.openownership.org" + } + } + ], + "publicationDetails": { + "publicationDate": "2024-02-16", + "bodsVersion": "0.2", + "license": "https://register.openownership.org/terms-and-conditions", + "publisher": { + "name": "OpenOwnership Register", + "url": "https://register.openownership.org" + } + }, + "source": { + "type": [ + "officialRegister", + "verified" + ], + "description": "GLEIF" + } + } +] diff --git a/tests/fixtures/rr-updates-data2.json b/tests/fixtures/rr-updates-data2.json new file mode 100644 index 0000000..c37d4b5 --- /dev/null +++ b/tests/fixtures/rr-updates-data2.json @@ -0,0 +1,439 @@ +[ + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002337", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-23T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:04.159+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002143", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2012-04-02T08:00:00+02:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:02.482+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002240", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-22T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:03.502+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BEJX0000002434", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_FUND-MANAGED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2005-03-22T08:00:00+01:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE" + }, + "Registration": { + "InitialRegistrationDate": "2014-09-23T00:00:00+02:00", + "LastUpdateDate": "2023-08-17T08:31:04.659+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-09-26T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BHF50000076475", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "549300JB1P61FUTPEZ75", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2022-10-24T08:00:00+02:00", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2022-01-01T08:00:00+01:00", + "EndDate": "2022-12-31T08:00:00+01:00", + "PeriodType": "ACCOUNTING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "IFRS" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-05-22T12:27:52.655+02:00", + "LastUpdateDate": "2023-05-25T08:30:56.059+02:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-30T00:00:00+02:00", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "SUPPORTING_DOCUMENTS" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BJGO0000201513", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "9845002B9874D50A4531", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2020-07-16T00:00:00Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2014-11-06T00:00:00Z", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2020-07-16T00:00:00Z", + "PeriodType": "DOCUMENT_FILING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "IFRS" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "1.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2020-07-16T00:00:00Z", + "LastUpdateDate": "2020-07-16T00:00:00Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2021-06-18T00:00:00Z", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "097900BJGO0000201513", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "9845002B9874D50A4531", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2020-07-16T00:00:00Z", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2014-11-06T00:00:00Z", + "PeriodType": "RELATIONSHIP_PERIOD" + }, + { + "StartDate": "2020-07-16T00:00:00Z", + "PeriodType": "DOCUMENT_FILING_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "IFRS" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "1.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2020-07-16T00:00:00Z", + "LastUpdateDate": "2020-07-16T00:00:00Z", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2021-06-18T00:00:00Z", + "ManagingLOU": "097900BEFH0000000217", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-06-30T00:00:00+00:00", + "EndDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-01-10T17:29:10+00:00", + "LastUpdateDate": "2023-01-11T11:21:01+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2018-06-30T00:00:00+00:00", + "EndDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2019-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2023-01-10T17:29:10+00:00", + "LastUpdateDate": "2023-01-11T11:21:01+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-01-13T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "98450051BS9C610A8T78", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "2138002YAB7Z9KO21397", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2019-06-30T00:00:00+00:00", + "EndDate": "2020-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T00:00:00+00:00", + "LastUpdateDate": "2023-03-30T14:18:04+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + }, + { + "Relationship": { + "StartNode": { + "NodeID": "98450051BS9C610A8T78", + "NodeIDType": "LEI" + }, + "EndNode": { + "NodeID": "984500501A1B1045PB30", + "NodeIDType": "LEI" + }, + "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", + "RelationshipPeriods": [ + { + "StartDate": "2019-06-30T00:00:00+00:00", + "EndDate": "2020-06-30T00:00:00+00:00", + "PeriodType": "ACCOUNTING_PERIOD" + }, + { + "StartDate": "2018-06-30T00:00:00+00:00", + "PeriodType": "RELATIONSHIP_PERIOD" + } + ], + "RelationshipStatus": "ACTIVE", + "RelationshipQualifiers": [ + { + "QualifierDimension": "ACCOUNTING_STANDARD", + "QualifierCategory": "OTHER_ACCOUNTING_STANDARD" + } + ], + "RelationshipQuantifiers": [ + { + "MeasurementMethod": "ACCOUNTING_CONSOLIDATION", + "QuantifierAmount": "100.00", + "QuantifierUnits": "PERCENTAGE" + } + ] + }, + "Registration": { + "InitialRegistrationDate": "2019-05-15T00:00:00+00:00", + "LastUpdateDate": "2023-03-30T14:18:04+00:00", + "RegistrationStatus": "PUBLISHED", + "NextRenewalDate": "2024-05-15T00:00:00+00:00", + "ManagingLOU": "529900T8BM49AURSDO55", + "ValidationSources": "FULLY_CORROBORATED", + "ValidationDocuments": "ACCOUNTS_FILING" + } + } +] diff --git a/tests/fixtures/rr-updates-data2.xml b/tests/fixtures/rr-updates-data2.xml new file mode 100644 index 0000000..c72b60d --- /dev/null +++ b/tests/fixtures/rr-updates-data2.xml @@ -0,0 +1,479 @@ + + + 2023-11-06T09:29:01Z + GLEIF_FULL_PUBLISHED + 407916 + + + + + + 097900BEJX0000002337 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-23T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:04.159+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002143 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2012-04-02T08:00:00+02:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:02.482+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002240 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-22T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:03.502+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BEJX0000002434 + LEI + + + 097900BHF50000076475 + LEI + + IS_FUND-MANAGED_BY + + + 2005-03-22T08:00:00+01:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + 2014-09-23T00:00:00+02:00 + 2023-08-17T08:31:04.659+02:00 + PUBLISHED + 2024-09-26T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + OTHER_OFFICIAL_DOCUMENTS + + + + + + 097900BHF50000076475 + LEI + + + 549300JB1P61FUTPEZ75 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2022-10-24T08:00:00+02:00 + RELATIONSHIP_PERIOD + + + 2022-01-01T08:00:00+01:00 + 2022-12-31T08:00:00+01:00 + ACCOUNTING_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + IFRS + + + + + 2023-05-22T12:27:52.655+02:00 + 2023-05-25T08:30:56.059+02:00 + PUBLISHED + 2024-05-30T00:00:00+02:00 + 097900BEFH0000000217 + FULLY_CORROBORATED + SUPPORTING_DOCUMENTS + + + + + + 097900BJGO0000201513 + LEI + + + 9845002B9874D50A4531 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2020-07-16T00:00:00Z + ACCOUNTING_PERIOD + + + 2014-11-06T00:00:00Z + RELATIONSHIP_PERIOD + + + 2020-07-16T00:00:00Z + DOCUMENT_FILING_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + IFRS + + + + + ACCOUNTING_CONSOLIDATION + 1.00 + PERCENTAGE + + + + + 2020-07-16T00:00:00Z + 2020-07-16T00:00:00Z + PUBLISHED + 2021-06-18T00:00:00Z + 097900BEFH0000000217 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 097900BJGO0000201513 + LEI + + + 9845002B9874D50A4531 + LEI + + IS_ULTIMATELY_CONSOLIDATED_BY + + + 2020-07-16T00:00:00Z + ACCOUNTING_PERIOD + + + 2014-11-06T00:00:00Z + RELATIONSHIP_PERIOD + + + 2020-07-16T00:00:00Z + DOCUMENT_FILING_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + IFRS + + + + + ACCOUNTING_CONSOLIDATION + 1.00 + PERCENTAGE + + + + + 2020-07-16T00:00:00Z + 2020-07-16T00:00:00Z + PUBLISHED + 2021-06-18T00:00:00Z + 097900BEFH0000000217 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 2138002YAB7Z9KO21397 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2018-06-30T00:00:00+00:00 + 2019-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2019-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2023-01-10T17:29:10+00:00 + 2023-01-11T11:21:01+00:00 + PUBLISHED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 2138002YAB7Z9KO21397 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_ULTIMATELY_CONSOLIDATED_BY + + + 2018-06-30T00:00:00+00:00 + 2019-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2019-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2023-01-10T17:29:10+00:00 + 2023-01-11T11:21:01+00:00 + PUBLISHED + 2024-01-13T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 98450051BS9C610A8T78 + LEI + + + 2138002YAB7Z9KO21397 + LEI + + IS_DIRECTLY_CONSOLIDATED_BY + + + 2019-06-30T00:00:00+00:00 + 2020-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2018-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + 2019-05-15T00:00:00+00:00 + 2023-03-30T14:18:04+00:00 + PUBLISHED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + + + 98450051BS9C610A8T78 + LEI + + + 984500501A1B1045PB30 + LEI + + IS_ULTIMATELY_CONSOLIDATED_BY + + + 2019-06-30T00:00:00+00:00 + 2020-06-30T00:00:00+00:00 + ACCOUNTING_PERIOD + + + 2018-06-30T00:00:00+00:00 + RELATIONSHIP_PERIOD + + + ACTIVE + + + ACCOUNTING_STANDARD + OTHER_ACCOUNTING_STANDARD + + + + + ACCOUNTING_CONSOLIDATION + 100.00 + PERCENTAGE + + + + + 2019-05-15T00:00:00+00:00 + 2023-03-30T14:18:04+00:00 + PUBLISHED + 2024-05-15T00:00:00+00:00 + 529900T8BM49AURSDO55 + FULLY_CORROBORATED + ACCOUNTS_FILING + + + + diff --git a/tests/test_elasticsearch.py b/tests/test_elasticsearch.py index 8924c24..0a54cd9 100644 --- a/tests/test_elasticsearch.py +++ b/tests/test_elasticsearch.py @@ -1,10 +1,13 @@ import os import sys import time +import json from unittest.mock import patch, Mock +import asyncio import pytest -from bodspipelines.infrastructure.storage import ElasticStorage +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, match_lei, match_rr, match_repex, id_lei, id_rr, id_repex) @@ -14,6 +17,7 @@ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} def set_environment_variables(): + """Setup environment variables""" os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' os.environ['ELASTICSEARCH_HOST'] = 'localhost' os.environ['ELASTICSEARCH_PORT'] = '9876' @@ -41,18 +45,98 @@ def lei_item(): 'ValidationSources': 'FULLY_CORROBORATED', 'ValidationAuthority': {'ValidationAuthorityID': 'RA000670', 'ValidationAuthorityEntityID': '43846696'}}} -def test_lei_storage_new(lei_item): + +@pytest.fixture +def lei_list(): + """List of entity LEIs""" + return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216', + '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53', + '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11'] + +@pytest.fixture +def last_update_list(): + """List of last update datetimes""" + return ["2023-05-18T15:41:20.212Z", "2020-07-17T12:40:00.000Z", "2022-07-22T09:32:00.000Z", "2022-10-24T21:31:00.000Z", + "2023-05-18T17:24:00.540Z", "2023-05-03T07:03:05.620Z", "2019-04-22T21:31:00.000Z", "2023-05-10T04:42:18.790Z", + "2020-07-17T12:40:00.000Z", "2020-07-24T19:29:00.000Z", "2023-03-10T13:08:56+01:00", "2023-02-02T09:07:52.390Z", + "2023-04-25T13:18:00Z"] + + +@pytest.fixture +def json_data(): + """LEI JSON data""" + with open("tests/fixtures/lei-data.json", "r") as read_file: + return json.load(read_file) + + +@pytest.mark.asyncio +async def test_lei_storage_new(lei_item): """Test storing a new LEI-CDF v3.1 record in elasticsearch""" - with patch('bodspipelines.infrastructure.clients.elasticsearch_client.Elasticsearch') as mock_es: - mock_es.return_value.search.return_value = {"hits": {"total": {"value": 1, "relation": "eq"}, "hits": []}} + with patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es: + search_future = asyncio.Future() + search_future.set_result({"hits": {"total": {"value": 1, "relation": "eq"}, "hits": []}}) + mock_es.return_value.search.return_value = search_future + index_future = asyncio.Future() + index_future.set_result(None) + mock_es.return_value.index.return_value = index_future set_environment_variables() - storage = ElasticStorage(indexes=index_properties) - assert storage.process(lei_item, 'lei') == lei_item + storage = Storage(storage=ElasticsearchClient(indexes=index_properties)) + await storage.setup() + assert await storage.process(lei_item, 'lei') == lei_item + -def test_lei_storage_existing(lei_item): +@pytest.mark.asyncio +async def test_lei_storage_existing(lei_item): """Test trying to store LEI-CDF v3.1 record which is already in elasticsearch""" - with patch('bodspipelines.infrastructure.clients.elasticsearch_client.Elasticsearch') as mock_es: - mock_es.return_value.search.return_value = {"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [lei_item]}} + with patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es: + search_future = asyncio.Future() + search_future.set_result({"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [lei_item]}}) + mock_es.return_value.search.return_value = search_future + index_future = asyncio.Future() + index_future.set_result(None) + mock_es.return_value.index.return_value = index_future + set_environment_variables() + storage = Storage(storage=ElasticsearchClient(indexes=index_properties)) + await storage.setup() + assert await storage.process(lei_item, 'lei') == False + + +@pytest.mark.asyncio +async def test_lei_bulk_storage_new(lei_list, last_update_list, json_data): + """Test ingest pipeline stage on LEI-CDF v3.1 records""" + with patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb: + async def result(): + for lei, last in zip(lei_list, last_update_list): + yield (True, {'create': {'_id': f"{lei}_{last}"}}) + mock_sb.return_value = result() + set_environment_variables() + storage = Storage(storage=ElasticsearchClient(indexes=index_properties)) + await storage.setup() + async def json_stream(): + for d in json_data: + yield d + count = 0 + async for result in storage.process_batch(json_stream(), 'lei'): + assert result == json_data[count] + count += 1 + assert count == 13 + + +@pytest.mark.asyncio +async def test_lei_bulk_storage_existing(lei_list, last_update_list, json_data): + """Test ingest pipeline stage on LEI-CDF v3.1 records""" + with patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb: + async def result(): + for lei, last in zip(lei_list, last_update_list): + yield (False, {'create': {'_id': f"{lei}_{last}"}}) + mock_sb.return_value = result() set_environment_variables() - storage = ElasticStorage(indexes=index_properties) - assert storage.process(lei_item, 'lei') == False + storage = Storage(storage=ElasticsearchClient(indexes=index_properties)) + await storage.setup() + async def json_stream(): + for d in json_data: + yield d + count = 0 + async for result in storage.process_batch(json_stream(), 'lei'): + count += 1 + assert count == 0 diff --git a/tests/test_gleif_data.py b/tests/test_gleif_data.py new file mode 100644 index 0000000..1fb8f03 --- /dev/null +++ b/tests/test_gleif_data.py @@ -0,0 +1,225 @@ +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta +from pathlib import Path +import random +import zipfile +import io +import pytest + +from bodspipelines.infrastructure.processing.bulk_data import BulkData + +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData + +from unittest.mock import patch, Mock, MagicMock + +class TestGLEIFData: + """Test download of single file""" + source = 'lei' + + @pytest.fixture(scope="class") + def temp_dir(self, tmp_path_factory): + """Fixture to create temporary directory""" + return tmp_path_factory.getbasetemp() + + @pytest.fixture(scope="class") + def stage_dir(self, temp_dir): + """Fixture to create subdirectory""" + output_dir = Path(temp_dir) / "data" / self.source + output_dir.mkdir(parents=True) + return output_dir + + def test_gleif_data(self, stage_dir): + """Test downloading GLEIF data""" + with (patch('bodspipelines.infrastructure.utils.download_delayed') as mock_download, + patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests): + def download_delayed(url): + def download(param): + fn = param.rsplit('/', 1)[-1] + print(fn) + return fn + return partial(download, url) + mock_download.side_effect = download_delayed + def zip_file(fn): + file = io.BytesIO() + zf = zipfile.ZipFile(file, mode='w') + zf.writestr(fn, '') + zf.close() + return bytes(file.getbuffer()) + def requests_get(url, stream=True): + fn = url.rsplit('/', 1)[-1] + xfn = fn.split('.zip')[0] + mock_return = Mock() + mock_return.headers = {'content-disposition': f"filename={fn}", + 'content-length': 10000} + mock_return.iter_content.return_value = iter([zip_file(xfn)]) + mock_context = Mock() + mock_context.__enter__ = Mock(return_value=mock_return) + mock_context.__exit__ = Mock(return_value=False) + return mock_context + mock_requests.side_effect = requests_get + + origin=BulkData(display="LEI-CDF v3.1", + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"), + size=41491, + directory="lei-cdf") + count = 0 + for fn in origin.prepare(stage_dir, self.source): + assert fn.endswith("-gleif-goldencopy-lei2-golden-copy.xml") + print([x for x in (stage_dir/'lei-cdf').iterdir()]) + assert (stage_dir/'lei-cdf'/fn).exists() + count += 1 + assert count == 1 + +class TestGLEIFDataUpdateMonths: + """Test download of updates after over a month""" + source = 'lei' + + @pytest.fixture(scope="class") + def temp_dir(self, tmp_path_factory): + """Fixture to create temporary directory""" + return tmp_path_factory.getbasetemp() + + @pytest.fixture(scope="class") + def stage_dir(self, temp_dir): + """Fixture to create subdirectory""" + output_dir = Path(temp_dir) / "data2" / self.source + output_dir.mkdir(parents=True) + return output_dir + + def test_gleif_data(self, stage_dir): + """Test downloading GLEIF data""" + with (patch('bodspipelines.infrastructure.utils.download') as mock_download, + patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests): + def download(url): + url_base = "https://goldencopy.gleif.org/storage/golden-copy-files/2023/06/13/795336/" + d = datetime.now().strftime("%Y%m%d") + t = f'{[x for x in (0, 800, 1600) if x < int(datetime.now().strftime("%H%M"))][-1]:04d}' + return {'data': {'publish_date': f"{d} {t}", + 'delta_files': { + 'LastMonth': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-month.xml.zip"}}, + 'LastWeek': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-week.xml.zip"}}, + 'LastDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-day.xml.zip"}}, + 'IntraDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-intra-day.xml.zip"}}, + }}} + mock_download.side_effect = download + def zip_file(fn): + file = io.BytesIO() + zf = zipfile.ZipFile(file, mode='w') + zf.writestr(fn, '') + zf.close() + return bytes(file.getbuffer()) + def requests_get(url, stream=True): + fn = url.rsplit('/', 1)[-1] + xfn = fn.split('.zip')[0] + mock_return = Mock() + mock_return.headers = {'content-disposition': f"filename={fn}", + 'content-length': 10000} + mock_return.iter_content.return_value = iter([zip_file(xfn)]) + mock_context = Mock() + mock_context.__enter__ = Mock(return_value=mock_return) + mock_context.__exit__ = Mock(return_value=False) + return mock_context + mock_requests.side_effect = requests_get + last_update_date = (datetime.now() + relativedelta(months=-1) - timedelta(days=5)).strftime("%Y-%m-%d") + last_update_time = f"{[x for x in (0, 8, 16) if int(datetime.now().strftime('%H00')) > x][-1]:02d}:00:00" + + origin=BulkData(display="LEI-CDF v3.1", + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"), + size=41491, + directory="lei-cdf") + print(f'{last_update_date} {last_update_time}') + count = 0 + for fn in origin.prepare(stage_dir, self.source, last_update=f'{last_update_date} {last_update_time}'): + print(fn) + if count == 0: + assert fn.endswith("-gleif-goldencopy-lei2-last-month.xml") + first = fn + else: + assert fn.endswith("-gleif-goldencopy-lei2-intra-day.xml") + print([x for x in (stage_dir/'lei-cdf').iterdir()]) + assert (stage_dir/'lei-cdf'/fn).exists() + count += 1 + first_date = datetime.strptime(first.split("-gleif")[0], "%Y%m%d-%H%M") + last_date = datetime.strptime(f'{last_update_date} {last_update_time}', "%Y-%m-%d %H:%M:%S") + delta = relativedelta(first_date, last_date) + assert count == 1 + 3*delta.days + delta.hours/8 + assert len([x for x in (stage_dir/'lei-cdf').iterdir() if x.suffix == ".xml"]) == 1 + 3*delta.days + delta.hours/8 + + +class TestGLEIFDataUpdateWeeks: + """Test download of updates after less than a month""" + source = 'lei' + + @pytest.fixture(scope="class") + def temp_dir(self, tmp_path_factory): + """Fixture to create temporary directory""" + return tmp_path_factory.getbasetemp() + + @pytest.fixture(scope="class") + def stage_dir(self, temp_dir): + """Fixture to create subdirectory""" + output_dir = Path(temp_dir) / "data3" / self.source + output_dir.mkdir(parents=True) + return output_dir + + def test_gleif_data(self, stage_dir): + """Test downloading GLEIF data""" + with (patch('bodspipelines.infrastructure.utils.download') as mock_download, + patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests): + def download(url): + url_base = "https://goldencopy.gleif.org/storage/golden-copy-files/2023/06/13/795336/" + d = datetime.now().strftime("%Y%m%d") + t = f'{[x for x in (0, 800, 1600) if x < int(datetime.now().strftime("%H%M"))][-1]:04d}' + return {'data': {'publish_date': f"{d} {t}", + 'delta_files': { + 'LastMonth': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-month.xml.zip"}}, + 'LastWeek': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-week.xml.zip"}}, + 'LastDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-day.xml.zip"}}, + 'IntraDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-intra-day.xml.zip"}}, + }}} + mock_download.side_effect = download + def zip_file(fn): + file = io.BytesIO() + zf = zipfile.ZipFile(file, mode='w') + zf.writestr(fn, '') + zf.close() + return bytes(file.getbuffer()) + def requests_get(url, stream=True): + fn = url.rsplit('/', 1)[-1] + xfn = fn.split('.zip')[0] + mock_return = Mock() + mock_return.headers = {'content-disposition': f"filename={fn}", + 'content-length': 10000} + mock_return.iter_content.return_value = iter([zip_file(xfn)]) + mock_context = Mock() + mock_context.__enter__ = Mock(return_value=mock_return) + mock_context.__exit__ = Mock(return_value=False) + return mock_context + mock_requests.side_effect = requests_get + last_update_date = (datetime.now() - timedelta(days=19)).strftime("%Y-%m-%d") + last_update_time = f"{[x for x in (0, 8, 16) if int(datetime.now().strftime('%H00')) > x][-1]:02d}:00:00" + + origin=BulkData(display="LEI-CDF v3.1", + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"), + size=41491, + directory="lei-cdf") + print(f'{last_update_date} {last_update_time}') + count = 0 + for fn in origin.prepare(stage_dir, self.source, last_update=f'{last_update_date} {last_update_time}'): + print(fn) + if count < 2: + assert fn.endswith("-gleif-goldencopy-lei2-last-week.xml") + if count == 0: first = fn + #elif count < 7: + # assert fn.endswith("-gleif-goldencopy-lei2-last-day.xml") + else: + assert fn.endswith("-gleif-goldencopy-lei2-intra-day.xml") or fn.endswith("-gleif-goldencopy-lei2-last-day.xml") + print([x for x in (stage_dir/'lei-cdf').iterdir()]) + assert (stage_dir/'lei-cdf'/fn).exists() + count += 1 + first_date = datetime.strptime(first.split("-gleif")[0], "%Y%m%d-%H%M") + last_date = datetime.strptime(f'{last_update_date} {last_update_time}', "%Y-%m-%d %H:%M:%S") + delta = relativedelta(first_date, last_date) + print(delta) + assert count == delta.days//7 + delta.days%7 + delta.hours/8 + assert len([x for x in (stage_dir/'lei-cdf').iterdir() if x.suffix == ".xml"]) == delta.days//7 + delta.days%7 + delta.hours/8 diff --git a/tests/test_gleif_source.py b/tests/test_gleif_source.py new file mode 100644 index 0000000..6339d85 --- /dev/null +++ b/tests/test_gleif_source.py @@ -0,0 +1,137 @@ +import os +import sys +import time +import datetime +from pathlib import Path +import json +from unittest.mock import patch, Mock +import asyncio +import pytest + +from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline +from bodspipelines.infrastructure.inputs import KinesisInput +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput +from bodspipelines.infrastructure.processing.bulk_data import BulkData +from bodspipelines.infrastructure.processing.xml_data import XMLData +from bodspipelines.infrastructure.processing.json_data import JSONData +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, AddContentDate + +from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, + match_lei, match_rr, match_repex, + id_lei, id_rr, id_repex) + +def set_environment_variables(): + """Set environment variables""" + os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' + os.environ['ELASTICSEARCH_HOST'] = 'localhost' + os.environ['ELASTICSEARCH_PORT'] = '9876' + os.environ['ELASTICSEARCH_PASSWORD'] = '********' + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" + +index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, + "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, + "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} + +# Identify type of GLEIF data +def identify_gleif(item): + if 'Entity' in item: + return 'lei' + elif 'Relationship' in item: + return 'rr' + elif 'ExceptionCategory' in item: + return 'repex' + +@pytest.fixture +def repex_xml_data_file(): + """GLEIF Repex XML data""" + return Path("tests/fixtures/repex-data.xml") + +@pytest.fixture +def repex_list(): + """GLEIF Repex id list""" + return ["001GPB6A9XPE8XJICC14_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None", + "004L5FPTUREIWK9T2N63_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "00EHHQ2ZHDCFXJCPCL46_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "00GBW0Z2GYIER7DHDS71_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "00KLB2PFTM3060S2N216_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None", + "00QDBXDXLLF3W3JJJO36_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None", + "00TV1D5YIV5IDUGWBW29_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "00X8DSV26QKJPKUT5B34_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "010G7UHBHEI87EKP0Q97_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None", + "01370W6ZIY66KQ4J3570_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_PUBLIC_None"] + +def test_repex_ingest_stage(repex_list, repex_xml_data_file): + """Test ingest pipeline stage on Reporting Exceptions v2.1 XML records""" + with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd, + patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + mock_bd.return_value = repex_xml_data_file + mock_pdr.return_value = None + mock_sdr.return_value = None + async def result(): + for repex_id in repex_list: + yield (True, {'create': {'_id': f"{repex_id}"}}) + mock_sb.return_value = result() + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + consumer_close_future = asyncio.Future() + consumer_close_future.set_result(None) + mock_kni.return_value.close.return_value = consumer_close_future + + set_environment_variables() + + # Defintion of Reporting Exceptions v2.1 XML date source + repex_source = Source(name="repex", + origin=BulkData(display="Reporting Exceptions v2.1", + #url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/repex/latest"), + size=3954, + directory="rep-ex"), + datatype=XMLData(item_tag="Exception", + header_tag="Header", + namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"}, + filter=['NextVersion', 'Extension'])) + + # GLEIF data: Store in Easticsearch and output new to Kinesis stream + output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)), + output=KinesisOutput(stream_name="gleif-test")) + + # Definition of GLEIF data pipeline ingest stage + ingest_stage = Stage(name="ingest-test", + sources=[repex_source], + processors=[AddContentDate(identify=identify_gleif)], + outputs=[output_new] + ) + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif-test", stages=[ingest_stage]) + + # Run pipelne + pipeline.process("ingest-test") + + # Check item count + assert mock_kno.return_value.put.call_count == 10 + + item = mock_kno.return_value.put.call_args.args[0] + + print(item) + + assert item['LEI'] == '01370W6ZIY66KQ4J3570' + assert item['ExceptionCategory'] == 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT' + assert item['ExceptionReason'] == 'NON_PUBLIC' + assert item['ContentDate'] == '2023-06-09T09:03:29Z' diff --git a/tests/test_gleif_transforms.py b/tests/test_gleif_transforms.py index 4c22940..f606f06 100644 --- a/tests/test_gleif_transforms.py +++ b/tests/test_gleif_transforms.py @@ -2,9 +2,10 @@ import datetime -from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, generate_statement_id +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, generate_statement_id, entity_id def validate_datetime(d): + """Test is valid datetime""" try: datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') return True @@ -12,11 +13,13 @@ def validate_datetime(d): return False def validate_date_now(d): + """Test is today's date""" return d == datetime.date.today().strftime('%Y-%m-%d') def test_generate_statement_id(): """Test the deterministic generation of statement IDs""" - assert generate_statement_id('2138008UKA1QH5L5XM10','entity') == '9edfc53c-34c7-4e3f-b427-bbbf65806dd3' + data = {"LEI": '2138008UKA1QH5L5XM10', 'Registration': {'LastUpdateDate': '2022-10-25T08:00:54.268+02:00'}} + assert generate_statement_id(entity_id(data),'entity') == '9d35c198-74f2-92bb-9205-a7bf73aff594' @pytest.fixture def lei_data(): @@ -81,43 +84,49 @@ def repex_data_no_known_person(): return {'LEI': '029200013A5N6ZD0F605', 'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT', 'ExceptionReason': 'NO_KNOWN_PERSON', - 'NextVersion': '', - 'Extension': ''} + #'NextVersion': '', + #'Extension': '', + 'ContentDate': "2023-06-09T09:03:29Z"} @pytest.fixture def repex_data_non_consolidating(): """Example NON_CONSOLIDATING Reporting Exception""" return {'LEI': '2138008UKA1QH5L5XM10', 'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT', - 'ExceptionReason': 'NON_CONSOLIDATING'} + 'ExceptionReason': 'NON_CONSOLIDATING', + 'ContentDate': "2023-06-09T09:03:29Z"} @pytest.fixture def repex_data_natural_persons(): """Example NATURAL_PERSONS Reporting Exception""" return {'LEI': '213800RJPV1SI7G2HW19', 'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT', - 'ExceptionReason': 'NATURAL_PERSONS'} + 'ExceptionReason': 'NATURAL_PERSONS', + 'ContentDate': "2023-06-09T09:03:29Z"} @pytest.fixture def repex_data_non_public(): """Example NON_PUBLIC Reporting Exception""" return {'LEI': '213800ZSKA23GF6L3F24', 'ExceptionCategory': 'ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT', - 'ExceptionReason': 'NON_PUBLIC'} + 'ExceptionReason': 'NON_PUBLIC', + 'ContentDate': "2023-06-09T09:03:29Z"} @pytest.fixture def repex_data_no_lei(): """Example NO_LEI Reporting Exception""" return {'LEI': '213800QTG16IUUZ6WD32', 'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT', - 'ExceptionReason': 'NO_LEI'} + 'ExceptionReason': 'NO_LEI', + 'ContentDate': "2023-06-09T09:03:29Z"} -def test_lei_transform(lei_data): +@pytest.mark.asyncio +async def test_lei_transform(lei_data): """Test transformation of LEI-CDF v3.1 data to BODS entity statement""" transform = Gleif2Bods() - for bods_data in transform.process(lei_data, 'lei'): + async for bods_data in transform.process(lei_data, 'lei', {}): print(bods_data) - assert bods_data['statementID'] == '659b3c3a-3ac8-b5bd-6480-e56dff141fa9' + assert bods_data['statementID'] == 'ca229e51-cb6b-11b8-4ed7-48935aed30ad' assert bods_data['statementType'] == 'entityStatement' assert bods_data['statementDate'] == '2022-10-25' assert bods_data['entityType'] == 'registeredEntity' @@ -139,17 +148,19 @@ def test_lei_transform(lei_data): if not 'bods_data' in locals(): raise Exception("No statements produced") -def test_rr_transform(rr_data): +@pytest.mark.asyncio +async def test_rr_transform(rr_data): """Test transformation of RR-CDF v2.1 data to BODS ownership or control statement""" transform = Gleif2Bods() - for bods_data in transform.process(rr_data, 'rr'): - assert bods_data['statementID'] == '7dc21488-bc9e-407c-293b-05578875b12f' + async for bods_data in transform.process(rr_data, 'rr', {}): + assert bods_data['statementID'] == '3f31f4cb-16dd-a251-144f-b871b330d5eb' assert bods_data['statementType'] == 'ownershipOrControlStatement' assert bods_data['statementDate'] == '2018-03-23' assert bods_data['subject'] == {'describedByEntityStatement': 'c892c662-2904-0c7d-5978-07108d102c33'} assert bods_data['interestedParty'] == {'describedByEntityStatement': '7d72f25e-910f-1f28-714d-7a761cbbc5de'} assert bods_data['interests'] == [{'type': 'other-influence-or-control', - 'interestLevel': 'indirect', + 'interestLevel': 'unknown', + 'details': 'LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY', 'beneficialOwnershipOrControl': False, 'startDate': '2018-03-23T00:00:00Z'}] assert validate_date_now(bods_data['publicationDetails']['publicationDate']) @@ -161,20 +172,21 @@ def test_rr_transform(rr_data): if not 'bods_data' in locals(): raise Exception("No statements produced") -def test_repex_transform_non_consolidating(repex_data_non_consolidating): +@pytest.mark.asyncio +async def test_repex_transform_non_consolidating(repex_data_non_consolidating): """Test transformation of NON_CONSOLIDATING Reporting Exception data to BODS statements""" transform = Gleif2Bods() - for bods_data in transform.process(repex_data_non_consolidating, 'repex'): + async for bods_data in transform.process(repex_data_non_consolidating, 'repex', {}): print(bods_data) if bods_data['statementType'] == 'entityStatement': - assert bods_data['statementID'] == '27b6f7b1-4e16-618c-9a19-3a7ad016ee91' + assert bods_data['statementID'] == '35694dcd-31ba-23dd-65d6-5f97839ef3aa' assert bods_data['unspecifiedEntityDetails'] == {'reason': 'interested-party-exempt-from-disclosure', 'description': 'From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control.'} assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'} assert bods_data['entityType'] == 'unknownEntity' statementID = bods_data['statementID'] elif bods_data['statementType'] == 'ownershipOrControlStatement': - assert bods_data['statementID'] == '47ab1442-5eb7-4d09-5745-e6330b89e6ed' + assert bods_data['statementID'] == '15ff987f-9e7b-2fb3-893c-f89acaa4ef6b' assert bods_data['subject'] == {'describedByEntityStatement': '5cc33d9b-ee78-a7e1-de3a-8082f06c4798'} assert bods_data['interestedParty'] == {'describedByEntityStatement': statementID} assert bods_data['interests'] == [{'type': 'other-influence-or-control', @@ -191,20 +203,21 @@ def test_repex_transform_non_consolidating(repex_data_non_consolidating): if not 'bods_data' in locals(): raise Exception("No statements produced") -def test_repex_transform_natural_persons(repex_data_natural_persons): +@pytest.mark.asyncio +async def test_repex_transform_natural_persons(repex_data_natural_persons): """Test transformation of NATURAL_PERSONS Reporting Exception data to BODS statements""" transform = Gleif2Bods() - for bods_data in transform.process(repex_data_natural_persons, 'repex'): + async for bods_data in transform.process(repex_data_natural_persons, 'repex', {}): print(bods_data) if bods_data['statementType'] == 'personStatement': - assert bods_data['statementID'] == '12186e42-9384-8a9e-f7b3-279a399770fe' + assert bods_data['statementID'] == '4df7870a-3215-d80e-6bd4-a9c5e8c5b1f9' assert bods_data['unspecifiedPersonDetails'] == {'reason': 'interested-party-exempt-from-disclosure', 'description': 'From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity.'} assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'} assert bods_data['personType'] == 'unknownPerson' statementID = bods_data['statementID'] elif bods_data['statementType'] == 'ownershipOrControlStatement': - assert bods_data['statementID'] == 'd897e79f-6f03-f800-6fd0-495298cbea6a' + assert bods_data['statementID'] == '46cc4dca-9374-420d-7be9-8736a1170da0' assert bods_data['subject'] == {'describedByEntityStatement': 'e579dc8d-8f7d-90bc-099a-38fa175bd494'} assert bods_data['interestedParty'] == {'describedByPersonStatement': statementID} assert bods_data['interests'] == [{'type': 'other-influence-or-control', @@ -222,13 +235,14 @@ def test_repex_transform_natural_persons(repex_data_natural_persons): if not 'bods_data' in locals(): raise Exception("No statements produced") -def test_repex_transform_non_public(repex_data_non_public): +@pytest.mark.asyncio +async def test_repex_transform_non_public(repex_data_non_public): """Test transformation of NON_PUBLIC Reporting Exception data to BODS statements""" transform = Gleif2Bods() - for bods_data in transform.process(repex_data_non_public, 'repex'): + async for bods_data in transform.process(repex_data_non_public, 'repex', {}): print(bods_data) if bods_data['statementType'] in ('entityStatement', 'personStatement'): - assert bods_data['statementID'] == '428aca2a-b344-78a8-4450-6cc726f26ecf' + assert bods_data['statementID'] == 'c63959b7-0528-1f24-018e-afb37ed15b9f' if bods_data['statementType'] == 'entityStatement': unspecified_type = 'unspecifiedEntityDetails' else: @@ -244,10 +258,10 @@ def test_repex_transform_non_public(repex_data_non_public): if bods_data['statementType'] == 'entityStatement': assert bods_data['entityType'] == 'unknownEntity' else: - assert bods_data['entityType'] == 'unknownPerson' + assert bods_data['personType'] == 'unknownPerson' statementID = bods_data['statementID'] elif bods_data['statementType'] == 'ownershipOrControlStatement': - assert bods_data['statementID'] == '984398d2-adf1-9111-f461-62f3101ff41d' + assert bods_data['statementID'] == 'd7c7f605-6ee0-e3b7-b46c-4c7a888bbdf0' assert bods_data['subject'] == {'describedByEntityStatement': 'fc0205b0-dcf7-cedd-c071-fbd4708149f0'} assert bods_data['interestedParty'] == {'describedByEntityStatement': statementID} assert bods_data['interests'] == [{'type': 'other-influence-or-control', @@ -264,3 +278,47 @@ def test_repex_transform_non_public(repex_data_non_public): raise Exception(f"Incorrect statement type produced: {bods_data['statementType']}") if not 'bods_data' in locals(): raise Exception("No statements produced") + +@pytest.mark.asyncio +async def test_repex_transform_no_known_person(repex_data_no_known_person): + """Test transformation of NO_KNOWN_PERSON Reporting Exception data to BODS statements""" + transform = Gleif2Bods() + async for bods_data in transform.process(repex_data_no_known_person, 'repex', {}): + print(bods_data) + if bods_data['statementType'] in ('entityStatement', 'personStatement'): + assert bods_data['statementID'] == 'b88c850f-16ca-414a-ee07-cbd842e9aa6d' + if bods_data['statementType'] == 'entityStatement': + unspecified_type = 'unspecifiedEntityDetails' + else: + unspecified_type = 'unspecifiedPersonDetails' + assert bods_data[unspecified_type] == {'reason': 'interested-party-exempt-from-disclosure', + 'description': 'From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity.'} + assert validate_date_now(bods_data['publicationDetails']['publicationDate']) + assert bods_data['publicationDetails']['bodsVersion'] == '0.2' + assert bods_data['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert bods_data['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', + 'url': 'https://register.openownership.org'} + assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'} + if bods_data['statementType'] == 'entityStatement': + assert bods_data['entityType'] == 'unknownEntity' + else: + assert bods_data['personType'] == 'unknownPerson' + statementID = bods_data['statementID'] + elif bods_data['statementType'] == 'ownershipOrControlStatement': + assert bods_data['statementID'] == '96681822-7e26-a3be-8e4d-9feb646d6ef8' + assert bods_data['subject'] == {'describedByEntityStatement': '2a251471-ed07-1c82-6bcf-4843fa880309'} + assert bods_data['interestedParty'] == {'describedByPersonStatement': statementID} + assert bods_data['interests'] == [{'type': 'other-influence-or-control', + 'interestLevel': 'direct', + 'beneficialOwnershipOrControl': False, + 'details': 'A controlling interest.'}] + assert validate_date_now(bods_data['publicationDetails']['publicationDate']) + assert bods_data['publicationDetails']['bodsVersion'] == '0.2' + assert bods_data['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert bods_data['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', + 'url': 'https://register.openownership.org'} + assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'} + else: + raise Exception(f"Incorrect statement type produced: {bods_data['statementType']}") + if not 'bods_data' in locals(): + raise Exception("No statements produced") diff --git a/tests/test_kinesis.py b/tests/test_kinesis.py index eb26507..e630064 100644 --- a/tests/test_kinesis.py +++ b/tests/test_kinesis.py @@ -4,13 +4,15 @@ import datetime from pathlib import Path import json -from unittest.mock import patch, Mock +from unittest.mock import patch, Mock, MagicMock +import asyncio import pytest from bodspipelines.infrastructure.inputs import KinesisInput from bodspipelines.infrastructure.outputs import KinesisOutput def validate_datetime(d): + """Test is valid datetime""" try: datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') return True @@ -19,51 +21,89 @@ def validate_datetime(d): def validate_date_now(d): + """Test is today's date""" return d == datetime.date.today().strftime('%Y-%m-%d') def set_environment_variables(): - os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' - os.environ['ELASTICSEARCH_HOST'] = 'localhost' - os.environ['ELASTICSEARCH_PORT'] = '9876' - os.environ['ELASTICSEARCH_PASSWORD'] = '********' + """Set environmant variables""" + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" + + +class AsyncIterator: + """Dummy async iterator""" + def __init__(self, seq): + self.iter = iter(seq) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration @pytest.fixture def json_data_file(): + """LEI JSON data""" with open("tests/fixtures/lei-data.json", "r") as read_file: return json.load(read_file) -def test_kinesis_output(json_data_file): +@pytest.fixture +def lei_list(): + """List of enitity LEIs""" + return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216', + '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53', + '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11'] + + +@pytest.mark.asyncio +async def test_kinesis_output(json_data_file): """Test Kinesis output stream""" - with patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn: - mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []} + with (patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future set_environment_variables() kinesis_output = KinesisOutput(stream_name="gleif-test") + await kinesis_output.setup() + for item in json_data_file: - kinesis_output.process(item, "lei") - kinesis_output.finish() + await kinesis_output.process(item, "lei") + await kinesis_output.finish() - #assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10 - count = 0 - for item in mock_kn.return_value.put_records.call_args.kwargs['Records']: - assert json.loads(item['Data']) == json_data_file[count] - count += 1 - assert count == 10 + assert mock_kno.return_value.put.call_count == 13 -def test_kinesis_input(json_data_file): +@pytest.mark.asyncio +async def test_kinesis_input(json_data_file, lei_list): """Test Kinesis input stream""" - with patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn: - mock_kn.return_value.get_records.return_value = {'MillisBehindLatest': 0, 'Records': [{'Data': json.dumps(js).encode('utf-8')} for js in json_data_file]} + with (patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + async def result(): + for item in json_data_file: + yield item + mock_kni.return_value = result() + #mock_kni.return_value = AsyncIterator(lei_list) set_environment_variables() kinesis_input = KinesisInput(stream_name="gleif-test") + await kinesis_input.setup() + count = 0 - for item in kinesis_input.process(): + async for item in kinesis_input.process(): + print(item) assert item == json_data_file[count] count += 1 - assert count == 10 + assert count == 13 diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c87e208..84cb262 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,18 +5,20 @@ from pathlib import Path import json from unittest.mock import patch, Mock +import asyncio import pytest from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline from bodspipelines.infrastructure.inputs import KinesisInput -from bodspipelines.infrastructure.storage import ElasticStorage +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput from bodspipelines.infrastructure.processing.bulk_data import BulkData from bodspipelines.infrastructure.processing.xml_data import XMLData from bodspipelines.infrastructure.processing.json_data import JSONData -from bodspipelines.pipelines.gleif.utils import gleif_download_link +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData from bodspipelines.pipelines.gleif.transforms import Gleif2Bods -from bodspipelines.pipelines.gleif.transforms import generate_statement_id +from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, match_lei, match_rr, match_repex, id_lei, id_rr, id_repex) @@ -27,6 +29,7 @@ def validate_datetime(d): + """Test is valid datetime""" try: datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') return True @@ -35,9 +38,10 @@ def validate_datetime(d): def validate_date_now(d): + """Test is today's date""" return d == datetime.date.today().strftime('%Y-%m-%d') - +# Elasticsearch indexes for GLEIF data index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} @@ -49,6 +53,7 @@ def validate_date_now(d): # Identify type of GLEIF data def identify_gleif(item): + print("Identifying:", item) if 'Entity' in item: return 'lei' elif 'Relationship' in item: @@ -66,47 +71,101 @@ def identify_bods(item): return 'ownership' def set_environment_variables(): + """Set environment variables""" os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' os.environ['ELASTICSEARCH_HOST'] = 'localhost' os.environ['ELASTICSEARCH_PORT'] = '9876' os.environ['ELASTICSEARCH_PASSWORD'] = '********' + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" @pytest.fixture def lei_list(): + """List entity LEIs""" return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216', - '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53'] + '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53', + '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11'] +@pytest.fixture +def last_update_list(): + """List update datetimes""" + return ['2023-05-18T15:41:20.212Z', '2020-07-17T12:40:00.000Z', '2022-07-22T09:32:00.000Z', '2022-10-24T21:31:00.000Z', + '2023-05-18T17:24:00.540Z', '2023-05-03T07:03:05.620Z', '2019-04-22T21:31:00.000Z', '2023-05-10T04:42:18.790Z', + '2020-07-17T12:40:00.000Z', '2020-07-24T19:29:00.000Z', '2023-03-10T13:08:56+01:00', '2023-02-02T09:07:52.390Z', + '2023-04-25T13:18:00Z'] @pytest.fixture def xml_data_file(): + """GLEIF LEI XML data""" return Path("tests/fixtures/lei-data.xml") +@pytest.fixture +def rr_xml_data_file(): + """GLEIF RR XML data""" + return Path("tests/fixtures/rr-data.xml") + @pytest.fixture def json_data_file(): + """GLEIF LEI JSON data""" with open("tests/fixtures/lei-data.json", "r") as read_file: return json.load(read_file) +@pytest.fixture +def rr_json_data_file(): + """GLEIF RR JSON data""" + with open("tests/fixtures/rr-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def repex_json_data_file(): + """GLEIF Repex JSON data""" + with open("tests/fixtures/repex-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def ooc_json_data_file(): + """BODS Data craeted from GLEIF RR data""" + with open("tests/fixtures/rr-data-out.json", "r") as read_file: + return json.load(read_file) -def test_lei_ingest_stage(lei_list, xml_data_file): +def test_lei_ingest_stage(lei_list, last_update_list, xml_data_file): """Test ingest pipeline stage on LEI-CDF v3.1 records""" with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd, patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, - patch('bodspipelines.infrastructure.clients.elasticsearch_client.streaming_bulk') as mock_sb, - patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn): + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): mock_bd.return_value = xml_data_file mock_pdr.return_value = None mock_sdr.return_value = None - mock_sb.return_value = iter([(True, {'create': {'_id': lei}}) for lei in lei_list]) - mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []} + async def result(): + for lei, last in zip(lei_list, last_update_list): + yield (True, {'create': {'_id': f"{lei}_{last}"}}) + mock_sb.return_value = result() + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + consumer_close_future = asyncio.Future() + consumer_close_future.set_result(None) + mock_kni.return_value.close.return_value = consumer_close_future + set_environment_variables() # Defintion of LEI-CDF v3.1 XML date source lei_source = Source(name="lei", origin=BulkData(display="LEI-CDF v3.1", - url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), + #url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"), + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"), size=41491, directory="lei-cdf"), datatype=XMLData(item_tag="LEIRecord", @@ -114,7 +173,7 @@ def test_lei_ingest_stage(lei_list, xml_data_file): filter=['NextVersion', 'Extension'])) # GLEIF data: Store in Easticsearch and output new to Kinesis stream - output_new = NewOutput(storage=ElasticStorage(indexes=index_properties), + output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)), output=KinesisOutput(stream_name="gleif-test")) # Definition of GLEIF data pipeline ingest stage ingest_stage = Stage(name="ingest-test", @@ -127,40 +186,172 @@ def test_lei_ingest_stage(lei_list, xml_data_file): # Run pipelne pipeline.process("ingest-test") - assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10 - item = json.loads(mock_kn.return_value.put_records.call_args.kwargs['Records'][0]['Data']) - assert item["LEI"] == "001GPB6A9XPE8XJICC14" - assert item["Entity"]["LegalName"] == "Fidelity Advisor Leveraged Company Stock Fund" - assert item["Entity"]["OtherEntityNames"] == ["FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"] - assert item["Entity"]["LegalAddress"] == {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"} - assert item["Entity"]["HeadquartersAddress"] == {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC", - "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"} - assert item["Entity"]["RegistrationAuthority"] == {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"} - assert item["Entity"]["LegalJurisdiction"] == "US-MA" - assert item["Entity"]["EntityCategory"] == "FUND" - assert item["Entity"]["LegalForm"] == {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"} - assert item["Entity"]["EntityStatus"] == "ACTIVE" - assert item["Entity"]["EntityCreationDate"] == "2012-11-29T00:00:00.000Z" - assert item["Registration"]["InitialRegistrationDate"] == "2012-11-29T16:33:00.000Z" - assert item["Registration"]["LastUpdateDate"] == "2023-05-18T15:41:20.212Z" - assert item["Registration"]["RegistrationStatus"] == "ISSUED" - assert item["Registration"]["NextRenewalDate"] == "2024-05-18T15:48:53.604Z" - assert item["Registration"]["ManagingLOU"] == "EVK05KS7XY1DEII3R011" - assert item["Registration"]["ValidationSources"] == "FULLY_CORROBORATED" - assert item["Registration"]["ValidationAuthority"] == {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"} - - -def test_lei_transform_stage(lei_list, json_data_file): + assert mock_kno.return_value.put.call_count == 13 + + item = mock_kno.return_value.put.call_args.args[0] + print(item) + + assert item['LEI'] == '213800BJPX8V9HVY1Y11' + assert item['Entity']['LegalName'] == 'Swedeit Italian Aktiebolag' + assert item['Entity']['LegalAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', + 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O', + 'Country': 'SE', 'PostalCode': '426 76'} + assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', + 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O', + 'Country': 'SE', 'PostalCode': '426 76'} + assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11', + 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76', + 'type': 'AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses'] + assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11', + 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76', + 'type': 'AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses'] + assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000544', 'RegistrationAuthorityEntityID': '556543-1193'} + assert item['Entity']['LegalJurisdiction'] == 'SE' + assert item['Entity']['EntityCategory'] == 'GENERAL' + assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'XJHM'} + assert item['Entity']['EntityStatus'] == 'ACTIVE' + assert item['Entity']['EntityCreationDate'] == '1997-06-05T02:00:00+02:00' + assert item['Registration']['InitialRegistrationDate'] == '2014-04-09T00:00:00Z' + assert item['Registration']['LastUpdateDate'] == '2023-04-25T13:18:00Z' + assert item['Registration']['RegistrationStatus'] == 'ISSUED' + assert item['Registration']['NextRenewalDate'] == '2024-05-12T06:59:39Z' + assert item['Registration']['ManagingLOU'] == '549300O897ZC5H7CY412' + assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' + assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000544', 'ValidationAuthorityEntityID': '556543-1193'} + + +def test_rr_ingest_stage(rr_xml_data_file, rr_json_data_file): + """Test ingest pipeline stage on RR-CDF v2.1 records""" + with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd, + patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock bulk data input + mock_bd.return_value = rr_xml_data_file + + # Mock directories + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES stream bulk output + async def result(): + for item in rr_json_data_file: + yield (True, {'create': {'_id': id_rr(item)}}) + mock_sb.return_value = result() + + # Mock Kinesis output + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Mock Kinesis input + consumer_close_future = asyncio.Future() + consumer_close_future.set_result(None) + mock_kni.return_value.close.return_value = consumer_close_future + + # Set environment variables + set_environment_variables() + + # Defintion of RR-CDF v2.1 XML date source + rr_source = Source(name="rr", + origin=BulkData(display="RR-CDF v2.1", + data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/rr/latest"), + size=2823, + directory="rr-cdf"), + datatype=XMLData(item_tag="RelationshipRecord", + namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"}, + filter=['Extension'])) + + # GLEIF data: Store in Easticsearch and output new to Kinesis stream + output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)), + output=KinesisOutput(stream_name="gleif-test")) + + # Definition of GLEIF data pipeline ingest stage + ingest_stage = Stage(name="ingest-test", + sources=[rr_source], + processors=[], + outputs=[output_new] + ) + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif-test", stages=[ingest_stage]) + + # Run pipelne + pipeline.process("ingest-test") + + assert mock_kno.return_value.put.call_count == 10 + + item = mock_kno.return_value.put.call_args.args[0] + + print(item) + + assert item['Relationship']['StartNode'] == {'NodeID': '010G7UHBHEI87EKP0Q97', 'NodeIDType': 'LEI'} + assert item['Relationship']['EndNode'] == {'NodeID': '549300GC5MDF1KXYMP06', 'NodeIDType': 'LEI'} + assert item['Relationship']['RelationshipType'] == 'IS_FUND-MANAGED_BY' + assert item['Relationship']['RelationshipPeriods'] == [{'StartDate': '2023-05-09T06:14:06.846Z', + 'PeriodType': 'RELATIONSHIP_PERIOD'}] + assert item['Relationship']['RelationshipStatus'] == 'ACTIVE' + assert item['Registration'] == {'InitialRegistrationDate': '2012-06-06T15:57:00.000Z', + 'LastUpdateDate': '2023-05-16T14:21:08.886Z', + 'RegistrationStatus': 'PUBLISHED', + 'NextRenewalDate': '2024-06-27T00:00:00.000Z', + 'ManagingLOU': 'EVK05KS7XY1DEII3R011', + 'ValidationSources': 'FULLY_CORROBORATED', + 'ValidationDocuments': 'SUPPORTING_DOCUMENTS', + 'ValidationReference': 'https://www.sec.gov/Archives/edgar/data/1458460/000140508622000264/xslFormDX01/primary_doc.xml'} + + +def test_lei_transform_stage(lei_list, last_update_list, json_data_file): """Test transform pipeline stage on LEI-CDF v3.1 records""" with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, - patch('bodspipelines.infrastructure.clients.elasticsearch_client.streaming_bulk') as mock_sb, - patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn): - mock_kn.return_value.get_records.return_value = {'MillisBehindLatest': 0, 'Records': [{'Data': json.dumps(js).encode('utf-8')} for js in json_data_file]} + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + async def result(): + for item in json_data_file: + yield item + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(json_data_file) mock_pdr.return_value = None mock_sdr.return_value = None - mock_sb.return_value = iter([(True, {'create': {'_id': generate_statement_id(lei, 'entityStatement')}}) for lei in lei_list]) - mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []} + + async def result(): + for lei, last in zip(lei_list, last_update_list): + data = {"LEI": lei, 'Registration': {'LastUpdateDate': last}} + yield (True, {'create': {'_id': generate_statement_id(entity_id(data), 'entityStatement')}}) + mock_sb.return_value = result() + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + set_environment_variables() # Kinesis stream of GLEIF data from ingest stage @@ -169,7 +360,7 @@ def test_lei_transform_stage(lei_list, json_data_file): datatype=JSONData()) # BODS data: Store in Easticsearch and output new to Kinesis stream - bods_output_new = NewOutput(storage=ElasticStorage(indexes=bods_index_properties), + bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)), output=KinesisOutput(stream_name="bods-gleif-test"), identify=identify_bods) @@ -185,21 +376,234 @@ def test_lei_transform_stage(lei_list, json_data_file): # Run pipelne pipeline.process("transform-test") - assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10 - item = json.loads(mock_kn.return_value.put_records.call_args.kwargs['Records'][0]['Data']) - assert item['statementID'] == '7239ce6e-d0c6-731a-30b5-b2157eb12419' + assert mock_kno.return_value.put.call_count == 13 + + item = mock_kno.return_value.put.call_args.args[0] + print(item) + + assert item['statementID'] == 'e2d096a9-23d5-ab26-0943-44c62c6a6a98' assert item['statementType'] == 'entityStatement' - assert item['statementDate'] == '2023-05-18' + assert item['statementDate'] == '2023-04-25' assert item['entityType'] == 'registeredEntity' - assert item['name'] == 'Fidelity Advisor Leveraged Company Stock Fund' - assert item['incorporatedInJurisdiction'] == {'name': 'US-MA', 'code': 'US-MA'} - assert {'id': '001GPB6A9XPE8XJICC14', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers'] - assert {'id': 'S000005113', 'schemeName': 'RA000665'} in item['identifiers'] - assert {'type': 'registered', 'address': '245 SUMMER STREET, BOSTON', 'country': 'US', 'postCode': '02210'} in item['addresses'] - assert {'type': 'business', 'address': 'C/O Fidelity Management & Research Company LLC, Boston', 'country': 'US', 'postCode': '02210'} in item['addresses'] + assert item['name'] == 'Swedeit Italian Aktiebolag' + assert item['incorporatedInJurisdiction'] == {'name': 'Sweden', 'code': 'SE'} + assert {'id': '213800BJPX8V9HVY1Y11', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers'] + assert {'id': '556543-1193', 'schemeName': 'RA000544'} in item['identifiers'] + assert {'type': 'registered', 'address': 'C/O Anita Lindberg, Västra Frölunda', 'country': 'SE', 'postCode': '426 76'} in item['addresses'] + assert {'type': 'business', 'address': 'C/O Anita Lindberg, Västra Frölunda', 'country': 'SE', 'postCode': '426 76'} in item['addresses'] assert validate_date_now(item['publicationDetails']['publicationDate']) assert item['publicationDetails']['bodsVersion'] == '0.2' assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'} assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'} - assert item['foundingDate'] == '2012-11-29T00:00:00.000Z' + + +def test_rr_transform_stage(rr_json_data_file, ooc_json_data_file): + """Test transform pipeline stage on RR-CDF v2.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock Kinesis input + async def result(): + for item in rr_json_data_file: + yield item + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(rr_json_data_file) + + # Mock directories + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES stream bulk output + async def result(): + for item in rr_json_data_file: + yield (True, {'create': {'_id': generate_statement_id(rr_id(item), 'ownershipOrControlStatement')}}) + mock_sb.return_value = result() + + # Mock Kinesis output + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + + # Mock Kinesis input + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Set environment variables + set_environment_variables() + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test", + sources=[gleif_source], + processors=[Gleif2Bods(identify=identify_gleif)], + outputs=[bods_output_new] + ) + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + pipeline.process("transform-test") + + # Check item count + assert mock_kno.return_value.put.call_count == 10 + + item = mock_kno.return_value.put.call_args.args[0] + + print(item) + + assert item['statementID'] == '208d41ba-977c-6760-801e-77fdc9d47307' + assert item['statementType'] == 'ownershipOrControlStatement' + assert item['statementDate'] == '2023-05-16' + assert item['subject'] == {'describedByEntityStatement': '06999bda-3717-8acb-719e-d6dd6d145c2a'} + assert item['interestedParty'] == {'describedByEntityStatement': '5bd23434-6df3-b2cf-4f28-83f16e6fea7a'} + assert item['interests'] == [{'type': 'other-influence-or-control', + 'interestLevel': 'direct', + 'beneficialOwnershipOrControl': False, + 'startDate': '2023-05-09T06:14:06.846Z'}] + assert validate_date_now(item['publicationDetails']['publicationDate']) + assert item['publicationDetails']['bodsVersion'] == '0.2' + assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', + 'url': 'https://register.openownership.org'} + assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'} + + +def test_rr_transform_stage(repex_json_data_file, ooc_json_data_file): + """Test transform pipeline stage on RR-CDF v2.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock Kinesis input + async def result(): + for item in repex_json_data_file: + yield item + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(repex_json_data_file) + + # Mock directories + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES stream bulk output + async def result(): + for item in repex_json_data_file: + if item["ExceptionReason"] == "NO_KNOWN_PERSON": + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), + 'personStatement')}}) + else: + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), + 'entityStatement')}}) + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), + 'ownershipOrControlStatement')}}) + mock_sb.return_value = result() + + # Mock Kinesis output + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + + # Mock Kinesis input + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Set environment variables + set_environment_variables() + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test", + sources=[gleif_source], + processors=[Gleif2Bods(identify=identify_gleif)], + outputs=[bods_output_new] + ) + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + pipeline.process("transform-test") + + # Check item count + assert mock_kno.return_value.put.call_count == 20 + + item = mock_kno.return_value.put.call_args.args[0] + + print(item) + + assert item['statementID'] == '48c88421-b1ae-671d-82fa-00c3adc063a8' + assert item['statementType'] == 'ownershipOrControlStatement' + assert item['statementDate'] == '2023-06-09' + assert item['subject'] == {'describedByEntityStatement': '34db1616-4997-534b-9b93-a9da9c8edda9'} + assert item['interestedParty'] == {'describedByEntityStatement': '639d58eb-7bcc-de1d-da9f-74748d9c5cad'} + assert item['interests'] == [{'type': 'other-influence-or-control', + 'interestLevel': 'direct', + 'beneficialOwnershipOrControl': False, + 'details': 'A controlling interest.'}] + assert validate_date_now(item['publicationDetails']['publicationDate']) + assert item['publicationDetails']['bodsVersion'] == '0.2' + assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', + 'url': 'https://register.openownership.org'} + assert item['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'} + assert item['annotations'][0]['motivation'] == 'commenting' + assert item['annotations'][0]['description'] == 'The nature of this interest is unknown' + assert item['annotations'][0]['statementPointerTarget'] == '/interests/0/type' + assert validate_date_now(item['annotations'][0]['creationDate']) + assert item['annotations'][0]['createdBy'] == {'name': 'Open Ownership', + 'uri': 'https://www.openownership.org'} diff --git a/tests/test_pipeline_updates.py b/tests/test_pipeline_updates.py new file mode 100644 index 0000000..5a658bb --- /dev/null +++ b/tests/test_pipeline_updates.py @@ -0,0 +1,480 @@ +import os +import sys +import time +import datetime +from pathlib import Path +import json +from unittest.mock import patch, Mock +import asyncio +import pytest + +from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline +from bodspipelines.infrastructure.inputs import KinesisInput +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput +from bodspipelines.infrastructure.processing.bulk_data import BulkData +from bodspipelines.infrastructure.processing.xml_data import XMLData +from bodspipelines.infrastructure.processing.json_data import JSONData +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods +from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id +from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, + match_lei, match_rr, match_repex, + id_lei, id_rr, id_repex) +from bodspipelines.infrastructure.updates import ProcessUpdates +from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties, + ownership_statement_properties, + match_entity, match_person, match_ownership, + id_entity, id_person, id_ownership, + latest_properties, match_latest, id_latest, + references_properties, match_references, id_references, + updates_properties, match_updates, id_updates) + + +def validate_datetime(d): + """Test is valid datetime""" + try: + datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') + return True + except ValueError: + return False + + +def validate_date_now(d): + """Test is today's date""" + return d == datetime.date.today().strftime('%Y-%m-%d') + +# Elasticsearch indexes for GLEIF data +index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, + "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, + "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} + +# Elasticsearch indexes for BODS data +bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity}, + "person": {"properties": person_statement_properties, "match": match_person, "id": id_person}, + "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership}, + "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest}, + "references": {"properties": references_properties, "match": match_references, "id": id_references}, + "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates}} + +# Identify type of GLEIF data +def identify_gleif(item): + if 'Entity' in item: + return 'lei' + elif 'Relationship' in item: + return 'rr' + elif 'ExceptionCategory' in item: + return 'repex' + +# Identify type of BODS data +def identify_bods(item): + if item['statementType'] == 'entityStatement': + return 'entity' + elif item['statementType'] == 'personStatement': + return 'person' + elif item['statementType'] == 'ownershipOrControlStatement': + return 'ownership' + +def set_environment_variables(): + """Set environment variable""" + os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' + os.environ['ELASTICSEARCH_HOST'] = 'localhost' + os.environ['ELASTICSEARCH_PORT'] = '9876' + os.environ['ELASTICSEARCH_PASSWORD'] = '********' + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" + + +@pytest.fixture +def lei_list(): + """GLEIF LEI list""" + return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216', + '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53', + '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11'] + +@pytest.fixture +def lei_list_updates(): + """GLEIF LEI updates list""" + return ["001GPB6A9XPE8XJICC14", "00EHHQ2ZHDCFXJCPCL46", "00QDBXDXLLF3W3JJJO36", "00W0SLGGVF0QQ5Q36N03", "213800FERQ5LE3H7WJ58"] + +@pytest.fixture +def latest_lei_ids(): + """Latest GLEIF LEI statement ids""" + return { + '001GPB6A9XPE8XJICC14': ('6057835c-9fbe-44ce-bdb0-0a8d82f2259d', None), + '004L5FPTUREIWK9T2N63': ('6e58bac7-a2c8-d233-fa74-e9738bc056d6', None), + '00EHHQ2ZHDCFXJCPCL46': ('9d22180c-dff9-2e93-b7a7-f6aacdbf4d02', None), + '00GBW0Z2GYIER7DHDS71': ('5f6448a2-b1fe-5e4f-8b1c-db0b1c627d98', None), + '00KLB2PFTM3060S2N216': ('c786d0f1-87fc-d6cf-5149-75b76e853289', None), + '00QDBXDXLLF3W3JJJO36': ('04357a16-7425-ad4e-8ed0-2e96a03dcdd9', None), + '00TR8NKAEL48RGTZEW89': ('359d7ade-acf0-aa10-3422-55e6ae1ca187', None), + '00TV1D5YIV5IDUGWBW29': ('1553f654-adde-1be3-9d6d-72c53b5af260', None), + '00W0SLGGVF0QQ5Q36N03': ('90eebf21-d507-a242-cd90-98410834aaf0', None), + '00X5RQKJQQJFFX0WPA53': ('6a3d293d-cc39-5f6c-850b-7855862c360c', None), + '1595D0QCK7Y15293JK84': ('c1dec12f-c6b2-1f69-d2fd-75166e7df72f', None), + '213800FERQ5LE3H7WJ58': ('6c7e8e94-e375-8d6e-408f-31bba4f5969a', None), + '213800BJPX8V9HVY1Y11': ('e2d096a9-23d5-ab26-0943-44c62c6a6a98', None), + } + +@pytest.fixture +def last_update_list(): + """GLEIF LEI last update datetimes list""" + return ['2023-05-18T15:41:20.212Z', '2020-07-17T12:40:00.000Z', '2022-07-22T09:32:00.000Z', '2022-10-24T21:31:00.000Z', + '2023-05-18T17:24:00.540Z', '2023-05-03T07:03:05.620Z', '2019-04-22T21:31:00.000Z', '2023-05-10T04:42:18.790Z', + '2020-07-17T12:40:00.000Z', '2020-07-24T19:29:00.000Z', '2023-03-10T13:08:56+01:00', '2023-02-02T09:07:52.390Z', + '2023-04-25T13:18:00Z'] + +@pytest.fixture +def last_update_list_updates(): + """GLEIF LEI updates last update datetimes list""" + return ["2023-06-18T15:41:20.212Z", "2022-08-22T09:32:00.000Z", "2023-06-03T07:03:05.620Z", "2020-08-17T12:40:00.000Z", + "2023-03-02T09:07:52.390Z"] + +@pytest.fixture +def xml_data_file(): + """GLEIF LEI XML data""" + return Path("tests/fixtures/lei-data.xml") + +@pytest.fixture +def json_data_file(): + """GLEIF LEI updates JSON data""" + with open("tests/fixtures/lei-updates-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def json_data_out(): + """BODS statements built from GLEIF LEI updates data""" + with open("tests/fixtures/lei-updates-data-out.json", "r") as read_file: + return json.load(read_file) + +def test_lei_transform_stage_updates(lei_list_updates, last_update_list_updates, latest_lei_ids, json_data_file, json_data_out): + """Test transform pipeline stage on LEI-CDF v3.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock Kinesis input + async def result(): + for item in json_data_file: + yield item + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(json_data_file) + + # Mock directory methods + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES async_scan + updates_stream = {} + async def as_result(): + for item in updates_stream: + yield updates_stream[item] + mock_as.return_value = as_result() + + # Mock ES async_streaming_bulk + async def sb_result(): + for lei, last in zip(lei_list_updates, last_update_list_updates): + data = {"LEI": lei, 'Registration': {'LastUpdateDate': last}} + yield (True, {'create': {'_id': generate_statement_id(entity_id(data), 'entityStatement')}}) + mock_sb.return_value = sb_result() + + # Mock Kinesis output + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Mock ES index method + index_future = asyncio.Future() + index_future.set_result(None) + mock_es.return_value.index.return_value = index_future + + # Mock ES search method + async def async_search(index=None, query=None): + id = query["query"]["match"]["_id"] + if index == "latest": + result = [{'latest_id': id, + 'statement_id': latest_lei_ids[id][0], + 'reason': latest_lei_ids[id][1]}] + elif index == 'references': + result = [] + elif index == 'exceptions': + result = [] + out = {'hits': {'hits': result}} + return out + mock_es.return_value.search.side_effect = async_search + + # Setup environment variables + set_environment_variables() + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage))], + outputs=[bods_output_new] + ) + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + pipeline.process("transform-test-updates", updates=True) + assert mock_kno.return_value.put.call_count == 5 + #assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10 + item = mock_kno.return_value.put.call_args.args[0] + print(item) + + + assert item['statementID'] == '11739b48-9df9-2d05-34aa-79c04d5cca06' + assert item['statementType'] == 'entityStatement' + assert item['statementDate'] == '2023-03-02' + assert item['entityType'] == 'registeredEntity' + assert item['name'] == 'DENTSU INTERNATIONAL LIMITED' + assert item['incorporatedInJurisdiction'] == {'name': 'United Kingdom', 'code': 'GB'} + assert {'id': '213800FERQ5LE3H7WJ58', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers'] + assert {'id': '01403668', 'schemeName': 'RA000585'} in item['identifiers'] + assert {'type': 'registered', 'address': '10 TRITON STREET, LONDON', 'country': 'GB', 'postCode': 'NW1 3BF'} in item['addresses'] + assert {'type': 'business', 'address': '10 TRITON STREET, LONDON', 'country': 'GB', 'postCode': 'NW1 3BF'} in item['addresses'] + assert validate_date_now(item['publicationDetails']['publicationDate']) + assert item['publicationDetails']['bodsVersion'] == '0.2' + assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'} + assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'} + assert item['foundingDate'] == '1978-12-05T00:00:00Z' + assert item['replacesStatements'] == ['6c7e8e94-e375-8d6e-408f-31bba4f5969a'] + + +@pytest.fixture +def rr_json_data_file(): + """GLEIF RR updates JSON data""" + with open("tests/fixtures/rr-updates-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def ooc_json_data_file(): + """BODS statement produced from GLEIF RR updates data""" + with open("tests/fixtures/rr-updates-data-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def rr_json_data_file_gleif(): + """GLEIF RR JSON data""" + with open("tests/fixtures/rr-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def rr_json_data_file_stored(): + """BODS statements produced from GLEIF RR data""" + with open("tests/fixtures/rr-data-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def latest_ids(rr_json_data_file_stored, rr_json_data_file_gleif): + """Latest statementIDs""" + latest_index = {} + for item, statement in zip(rr_json_data_file_gleif, rr_json_data_file_stored): + start = item["Relationship"]["StartNode"]["NodeID"] + end = item["Relationship"]["EndNode"]["NodeID"] + rel_type = item["Relationship"]["RelationshipType"] + latest_id = f"{start}_{end}_{rel_type}" + latest = {'latest_id': latest_id, + 'statement_id': statement["statementID"], + 'reason': None} + latest_index[latest_id] = latest + return latest_index + + +def test_rr_transform_stage_updates(rr_json_data_file, ooc_json_data_file, latest_ids): + """Test transform pipeline stage on RR-CDF v2.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock Kinesis input + async def result(): + for item in rr_json_data_file: + yield item + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(rr_json_data_file) + + # Mock directories + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES async_scan + updates_stream = {} + async def as_result(): + for item in updates_stream: + yield updates_stream[item] + mock_as.return_value = as_result() + + # Mock ES async_streaming_bulk + async def sb_result(): + for item in rr_json_data_file: + yield (True, {'create': {'_id': generate_statement_id(rr_id(item), 'ownershipOrControlStatement')}}) + mock_sb.return_value = sb_result() + + # Mock Kinesis output + put_future = asyncio.Future() + put_future.set_result(None) + mock_kno.return_value.put.return_value = put_future + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Mock Latest Index + latest_index_data = latest_ids + + # Mock ES index method + async def async_index(index, document): + #print("async_index:", index, document) + if index == "updates": + updates_stream[document['_source']['referencing_id']] = document['_source'] + elif index == "latest": + latest_index_data[document['_source']['latest_id']] = document['_source'] + return None + mock_es.return_value.index.side_effect = async_index + + # Mock ES update method + async def async_update(index, id, document): + print("async_update:", index, id, document) + if index == "latest": + latest_index_data[document['latest_id']] = document + elif index == "updates": + updates_stream[document['referencing_id']] = document + return None + mock_es.return_value.update.side_effect = async_update + + # Mock ES delete method + async def async_delete(index, id): + print("async_delete:", index, id) + if index == "latest": + del latest_rr_ids[id] + elif index == "updates": + print(updates_stream) + if id in updates_stream: del updates_stream[id] # IS THIS RIGHT? + return None + mock_es.return_value.delete.side_effect = async_delete + + # Mock ES search method + async def async_search(index=None, query=None): + id = query["query"]["match"]["_id"] + if index == "latest": + if id in latest_index_data: + results = [latest_index_data[id]] + else: + results = [] + elif index == 'references': + results = [] + elif index == 'exceptions': + results = [] + out = {'hits': {'hits': results}} + return out + mock_es.return_value.search.side_effect = async_search + + # Setup environment variables + set_environment_variables() + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage))], + outputs=[bods_output_new] + ) + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + pipeline.process("transform-test-updates", updates=True) + assert mock_kno.return_value.put.call_count == 3 + + item = mock_kno.return_value.put.call_args.args[0] + print(item) + + assert item['statementID'] == '8fec0d1c-b31b-7632-7c47-901e10bcd367' + assert item['statementType'] == 'ownershipOrControlStatement' + assert item['statementDate'] == '2023-03-24' + assert item['subject'] == {'describedByEntityStatement': '9f94abe2-349c-8e96-edaf-cf832eab1ac8'} + assert item['interestedParty'] == {'describedByEntityStatement': '20815b26-efdc-a516-a905-7abdfa63d128'} + assert {'type': 'other-influence-or-control', + 'details': 'LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY', + 'interestLevel': 'unknown', + 'beneficialOwnershipOrControl': False, + 'startDate': '2018-02-06T00:00:00.000Z'} in item['interests'] + assert validate_date_now(item['publicationDetails']['publicationDate']) + assert item['publicationDetails']['bodsVersion'] == '0.2' + assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions' + assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'} + assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'} + assert item['replacesStatements'] == ['3cd06b78-3c1c-0cab-881b-cbbc36af1741'] diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 0000000..fa040cb --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,122 @@ +import os +import sys +import time +import json +from unittest.mock import patch, Mock +import asyncio +import pytest + +from redis import RedisError + +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.redis_client import RedisClient +from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, + match_lei, match_rr, match_repex, + id_lei, id_rr, id_repex) + +index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, + "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, + "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} + +def set_environment_variables(): + """Set environment variables""" + os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' + os.environ['ELASTICSEARCH_HOST'] = 'localhost' + os.environ['ELASTICSEARCH_PORT'] = '9876' + os.environ['ELASTICSEARCH_PASSWORD'] = '********' + +@pytest.fixture +def lei_item(): + """Example LEI-CDF v3.1 data""" + return {'LEI': '097900BICQ0000135514', + 'Entity': {'LegalName': 'Ing. Magdaléna Beňo Frackowiak ZARIA TRAVEL', + 'TransliteratedOtherEntityNames': {'TransliteratedOtherEntityName': 'ING MAGDALENA BENO FRACKOWIAK ZARIA TRAVEL'}, + 'LegalAddress': {'FirstAddressLine': 'Partizánska Ľupča 708', 'City': 'Partizánska Ľupča', 'Country': 'SK', 'PostalCode': '032 15'}, + 'HeadquartersAddress': {'FirstAddressLine': 'Partizánska Ľupča 708', 'City': 'Partizánska Ľupča', 'Country': 'SK', 'PostalCode': '032 15'}, + 'RegistrationAuthority': {'RegistrationAuthorityID': 'RA000670', 'RegistrationAuthorityEntityID': '43846696'}, + 'LegalJurisdiction': 'SK', + 'EntityCategory': 'SOLE_PROPRIETOR', + 'LegalForm': {'EntityLegalFormCode': 'C4PZ'}, + 'EntityStatus': 'ACTIVE', + 'EntityCreationDate': '2007-11-15T08:00:00+01:00'}, + 'Registration': {'InitialRegistrationDate': '2018-02-16T00:00:00+01:00', + 'LastUpdateDate': '2023-01-10T08:30:56.044+01:00', + 'RegistrationStatus': 'ISSUED', + 'NextRenewalDate': '2024-02-16T00:00:00+01:00', + 'ManagingLOU': '097900BEFH0000000217', + 'ValidationSources': 'FULLY_CORROBORATED', + 'ValidationAuthority': {'ValidationAuthorityID': 'RA000670', 'ValidationAuthorityEntityID': '43846696'}}} + +@pytest.fixture +def lei_list(): + """List entity LEIs""" + return ['097900BICQ0000135514', '097900BICQ0000135515', '097900BICQ0000135516', '097900BICQ0000135517', '097900BICQ0000135518', + '097900BICQ0000135519', '097900BICQ0000135520', '097900BICQ0000135521', '097900BICQ0000135522', '097900BICQ0000135523'] + +@pytest.mark.asyncio +async def test_lei_storage_get(lei_item): + """Test getting a new LEI-CDF v3.1 record in redis""" + with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd: + get_future = asyncio.Future() + get_future.set_result(json.dumps(lei_item).encode("utf-8")) + mock_rd.return_value.get.return_value = get_future + set_environment_variables() + storage = Storage(storage=RedisClient(indexes=index_properties)) + assert await storage.get_item(lei_item["LEI"], 'lei') == lei_item + + +@pytest.mark.asyncio +async def test_lei_storage_new(lei_item): + """Test storing a new LEI-CDF v3.1 record in redis""" + with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd: + set_future = asyncio.Future() + set_future.set_result(None) + mock_rd.return_value.set.return_value = set_future + mock_rd.return_value.get.side_effect = RedisError() + set_environment_variables() + storage = Storage(storage=RedisClient(indexes=index_properties)) + assert await storage.process(lei_item, 'lei') == lei_item + + +@pytest.mark.asyncio +async def test_lei_storage_existing(lei_item): + """Test trying to store LEI-CDF v3.1 record which is already in redis""" + with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd: + get_future = asyncio.Future() + get_future.set_result(json.dumps(lei_item).encode("utf-8")) + mock_rd.return_value.get.return_value = get_future + set_environment_variables() + storage = Storage(storage=RedisClient(indexes=index_properties)) + assert await storage.process(lei_item, 'lei') == False + + +@pytest.mark.asyncio +async def test_lei_storage_batch_new(lei_item, lei_list): + """Test storing a batch of new LEI-CDF v3.1 records in redis""" + with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd: + async def build_stream(lei_item, lei_list): + for lei in lei_list: + item = lei_item.copy() + item['LEI'] = lei + yield item + mock_rd.return_value.pipeline.return_value.execute.return_value = [True async for result in build_stream(lei_item, lei_list)] + set_environment_variables() + storage = Storage(storage=RedisClient(indexes=index_properties)) + async for item in storage.process_batch(build_stream(lei_item, lei_list), "lei"): + assert item["LEI"] in lei_list + + +@pytest.mark.asyncio +async def test_lei_storage_batch_existing(lei_item, lei_list): + """Test storing a batch of existing LEI-CDF v3.1 records in redis""" + with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd: + async def build_stream(lei_item, lei_list): + for lei in lei_list: + item = lei_item.copy() + item['LEI'] = lei + yield item + mock_rd.return_value.pipeline.return_value.execute.return_value = [False async for result in build_stream(lei_item, lei_list)] + set_environment_variables() + storage = Storage(storage=RedisClient(indexes=index_properties)) + async for item in storage.process_batch(build_stream(lei_item, lei_list), "lei"): + assert False, "Error: New record found" diff --git a/tests/test_transforms_updates_es.py b/tests/test_transforms_updates_es.py new file mode 100644 index 0000000..71a9107 --- /dev/null +++ b/tests/test_transforms_updates_es.py @@ -0,0 +1,296 @@ +import pytest +import os +import json +import requests +from unittest.mock import patch, Mock, AsyncMock + +from .utils import AsyncIterator, list_index_contents +from .config import set_environment_variables + +pytest_plugins = ["docker_compose"] + +from bodspipelines.pipelines.gleif import config +from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline +from bodspipelines.infrastructure.inputs import KinesisInput +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput +from bodspipelines.infrastructure.processing.bulk_data import BulkData +from bodspipelines.infrastructure.processing.xml_data import XMLData +from bodspipelines.infrastructure.processing.json_data import JSONData +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData, identify_gleif +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods +from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id +from bodspipelines.pipelines.gleif.updates import GleifUpdates +from bodspipelines.pipelines.gleif.indexes import gleif_index_properties +from bodspipelines.infrastructure.updates import ProcessUpdates, referenced_ids +from bodspipelines.infrastructure.indexes import bods_index_properties +from bodspipelines.infrastructure.utils import identify_bods + +# Setup environment variables +set_environment_variables() + +#@pytest.fixture(scope="function") +#def wait_for_es(function_scoped_container_getter): +@pytest.fixture(scope="module") +def wait_for_es(module_scoped_container_getter): + service = module_scoped_container_getter.get("bods_ingester_gleif_es").network_info[0] + os.environ['ELASTICSEARCH_HOST'] = service.hostname + os.environ['ELASTICSEARCH_PORT'] = service.host_port + #config.setup() + return service + +@pytest.fixture +def lei_json_data_file_gleif(): + """GLEIF LEI Record update""" + with open("tests/fixtures/lei-updates-replaces-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def relationship_update_data_file_gleif(): + """GLEIF LEI and RR Records update""" + with open("tests/fixtures/relationship-update-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def reporting_exceptions_data_file_gleif(): + """GLEIF LEI and RR Records update""" + with open("tests/fixtures/reporting-exceptions-data.json", "r") as read_file: + return json.load(read_file) + +@pytest.mark.asyncio +async def test_lei_replaces(wait_for_es, lei_json_data_file_gleif): + #print(requests.get('http://localhost:9200/_cat/indices?v').text) + #assert False + """Test transform pipeline stage on LEI-CDF v3.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni, + patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno, + #patch('bodspipelines.infrastructure.outputs.KinesisOutput') as mock_kno, + #patch('bodspipelines.infrastructure.inputs.KinesisInput') as mock_kni + ): + + # Mock setup/finish_write functions + async def async_func(): + return None + mock_kni.return_value.setup.return_value = async_func() + mock_kno.return_value.setup.return_value = async_func() + mock_kno.return_value.finish_write.return_value = async_func() + mock_kni.return_value.close.return_value = async_func() + mock_kno.return_value.close.return_value = async_func() + + # Mock Kinesis stream input + mock_kni.return_value.read_stream.return_value = AsyncIterator(lei_json_data_file_gleif) + + # Mock Kinesis output stream (save results) + kinesis_output_stream = [] + async def async_put(record): + kinesis_output_stream.append(record) + return None + mock_kno.return_value.add_record.side_effect = async_put + + # Mock directory methods + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # Setup indexes + await bods_storage.setup_indexes() + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage), + updates=GleifUpdates())], + outputs=[bods_output_new] + ) + + assert hasattr(transform_stage.processors[0], "finish_updates") + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + #pipeline.process("transform-test-updates", updates=True) + await pipeline.process_stage("transform-test-updates", updates=True) + + print(kinesis_output_stream) + + assert kinesis_output_stream[0]['statementDate'] == '2023-12-29' + assert kinesis_output_stream[-1]['statementDate'] == '2024-01-03' + assert kinesis_output_stream[-1]['replacesStatements'][0] == kinesis_output_stream[0]['statementID'] + +@pytest.mark.asyncio +async def test_relationship_replaces(wait_for_es, relationship_update_data_file_gleif): + """Test transform pipeline stage on relationship update""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni, + patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno, + ): + + # Mock setup/finish_write functions + async def async_func(): + return None + mock_kni.return_value.setup.return_value = async_func() + mock_kno.return_value.setup.return_value = async_func() + mock_kno.return_value.finish_write.return_value = async_func() + mock_kni.return_value.close.return_value = async_func() + mock_kno.return_value.close.return_value = async_func() + + # Mock Kinesis stream input + mock_kni.return_value.read_stream.return_value = AsyncIterator(relationship_update_data_file_gleif) + + # Mock Kinesis output stream (save results) + kinesis_output_stream = [] + async def async_put(record): + kinesis_output_stream.append(record) + return None + mock_kno.return_value.add_record.side_effect = async_put + + # Mock directory methods + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # Setup indexes + await bods_storage.setup_indexes() + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage), + updates=GleifUpdates())], + outputs=[bods_output_new] + ) + + assert hasattr(transform_stage.processors[0], "finish_updates") + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + #pipeline.process("transform-test-updates", updates=True) + await pipeline.process_stage("transform-test-updates", updates=True) + + print(kinesis_output_stream) + + #list_index_contents("updates") + + assert kinesis_output_stream[0]['statementDate'] == '2023-02-22' + assert kinesis_output_stream[1]['statementDate'] == '2023-02-22' + assert kinesis_output_stream[2]['statementDate'] == '2023-02-22' + assert kinesis_output_stream[3]['statementDate'] == '2024-01-30' + assert kinesis_output_stream[4]['statementDate'] == '2024-01-30' + assert kinesis_output_stream[5]['statementDate'] == '2024-01-30' + assert kinesis_output_stream[-2]['replacesStatements'][0] == kinesis_output_stream[1]['statementID'] + assert kinesis_output_stream[-1]['replacesStatements'][0] == kinesis_output_stream[2]['statementID'] + +@pytest.mark.asyncio +async def test_reporting_exceptions(wait_for_es, reporting_exceptions_data_file_gleif): + """Test transform pipeline stage on relationship update""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni, + patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno, + ): + + # Mock setup/finish_write functions + async def async_func(): + return None + mock_kni.return_value.setup.return_value = async_func() + mock_kno.return_value.setup.return_value = async_func() + mock_kno.return_value.finish_write.return_value = async_func() + mock_kni.return_value.close.return_value = async_func() + mock_kno.return_value.close.return_value = async_func() + + # Mock Kinesis stream input + mock_kni.return_value.read_stream.return_value = AsyncIterator(reporting_exceptions_data_file_gleif) + + # Mock Kinesis output stream (save results) + kinesis_output_stream = [] + async def async_put(record): + kinesis_output_stream.append(record) + return None + mock_kno.return_value.add_record.side_effect = async_put + + # Mock directory methods + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # Setup indexes + await bods_storage.setup_indexes() + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage), + updates=GleifUpdates())], + outputs=[bods_output_new] + ) + + assert hasattr(transform_stage.processors[0], "finish_updates") + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + #pipeline.process("transform-test-updates", updates=True) + await pipeline.process_stage("transform-test-updates", updates=True) + + print(kinesis_output_stream) + + assert kinesis_output_stream[0]['statementDate'] == '2022-01-25' + assert kinesis_output_stream[1]['statementDate'] == '2024-01-01' + assert kinesis_output_stream[2]['statementDate'] == '2024-01-01' + assert kinesis_output_stream[2]['subject']['describedByEntityStatement'] == kinesis_output_stream[0]['statementID'] + assert kinesis_output_stream[2]['interestedParty']['describedByEntityStatement'] == kinesis_output_stream[1]['statementID'] + assert kinesis_output_stream[3]['statementDate'] == '2024-01-01' + assert kinesis_output_stream[4]['statementDate'] == '2024-01-01' + assert kinesis_output_stream[4]['subject']['describedByEntityStatement'] == kinesis_output_stream[0]['statementID'] + assert kinesis_output_stream[4]['interestedParty']['describedByEntityStatement'] == kinesis_output_stream[3]['statementID'] + diff --git a/tests/test_updates.py b/tests/test_updates.py new file mode 100644 index 0000000..957b002 --- /dev/null +++ b/tests/test_updates.py @@ -0,0 +1,684 @@ +import os +import sys +import time +import datetime +from pathlib import Path +import json +import itertools +from unittest.mock import patch, Mock +import asyncio +import pytest + +from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline +from bodspipelines.infrastructure.inputs import KinesisInput +from bodspipelines.infrastructure.storage import Storage +from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient +from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput +from bodspipelines.infrastructure.processing.bulk_data import BulkData +from bodspipelines.infrastructure.processing.xml_data import XMLData +from bodspipelines.infrastructure.processing.json_data import JSONData +from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData +from bodspipelines.pipelines.gleif.transforms import Gleif2Bods +from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id +from bodspipelines.pipelines.gleif.updates import GleifUpdates +from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties, + match_lei, match_rr, match_repex, + id_lei, id_rr, id_repex) +from bodspipelines.infrastructure.updates import ProcessUpdates, referenced_ids +from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties, + ownership_statement_properties, + match_entity, match_person, match_ownership, + id_entity, id_person, id_ownership, + latest_properties, match_latest, id_latest, + references_properties, match_references, id_references, + updates_properties, match_updates, id_updates, + exceptions_properties, match_exceptions, id_exceptions) + +def validate_datetime(d): + """Test is valid datetime""" + try: + datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') + return True + except ValueError: + return False + + +def validate_date_now(d): + """Test is today's date""" + return d == datetime.date.today().strftime('%Y-%m-%d') + +# Elasticsearch indexes for GLEIF data +index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei}, + "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr}, + "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}} + +# Elasticsearch indexes for BODS data +bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity}, + "person": {"properties": person_statement_properties, "match": match_person, "id": id_person}, + "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership}, + "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest}, + "references": {"properties": references_properties, "match": match_references, "id": id_references}, + "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates}, + "exceptions": {"properties": exceptions_properties, "match": match_exceptions, "id": id_exceptions}} + +# Identify type of GLEIF data +def identify_gleif(item): + if 'Entity' in item: + return 'lei' + elif 'Relationship' in item: + return 'rr' + elif 'ExceptionCategory' in item: + return 'repex' + +# Identify type of BODS data +def identify_bods(item): + if item['statementType'] == 'entityStatement': + return 'entity' + elif item['statementType'] == 'personStatement': + return 'person' + elif item['statementType'] == 'ownershipOrControlStatement': + return 'ownership' + +def set_environment_variables(): + """Set environment variables""" + os.environ['ELASTICSEARCH_PROTOCOL'] = 'http' + os.environ['ELASTICSEARCH_HOST'] = 'localhost' + os.environ['ELASTICSEARCH_PORT'] = '9876' + os.environ['ELASTICSEARCH_PASSWORD'] = '********' + os.environ['BODS_AWS_REGION'] = "eu-west-1" + os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************" + os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************" + + +@pytest.fixture +def updates_json_data_file(): + """GLEIF Updates Records""" + out = [] + with (open("tests/fixtures/lei-updates-data2-new.json", "r") as lei_file, + open("tests/fixtures/rr-updates-data2-new.json", "r") as rr_file, + open("tests/fixtures/repex-updates-data2-new.json", "r") as repex_file): + out.extend(json.load(lei_file)) + out.extend(json.load(rr_file)) + out.extend(json.load(repex_file)) + return out + +@pytest.fixture +def rr_json_data_file_gleif(): + """GLEIF Relationship Records""" + with open("tests/fixtures/rr-updates-data2.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def rr_json_data_file_stored(): + """Stored Relationship Record Statements""" + with open("tests/fixtures/rr-updates-data2-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def lei_json_data_file_gleif(): + """GLEIF LEI Records""" + with open("tests/fixtures/lei-updates-data2.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def lei_json_data_file_stored(): + """LEI Stored Statements""" + with open("tests/fixtures/lei-updates-data2-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def repex_json_data_file_gleif(): + """GLEIF Reporting Exception Records""" + with open("tests/fixtures/repex-updates-data2.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def repex_json_data_file_gleif_new(): + """GLEIF Reporting Exception Records""" + with open("tests/fixtures/repex-updates-data2-new.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def repex_json_data_file_stored(): + """Reporting Exception Stored Statements""" + with open("tests/fixtures/repex-updates-data2-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def lei_json_data_file_output(): + """LEI Output Statements""" + with open("tests/fixtures/lei-updates-data2-new-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def rr_json_data_file_output(): + """Relationship Record Output Statements""" + with open("tests/fixtures/rr-updates-data2-new-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def repex_json_data_file_output(): + """Reporting Exception Output Statements""" + with open("tests/fixtures/repex-updates-data2-new-out.json", "r") as read_file: + return json.load(read_file) + +@pytest.fixture +def stored_references(lei_json_data_file_stored, rr_json_data_file_stored, rr_json_data_file_gleif): + """Stored References""" + references = {} + for lei_statement in lei_json_data_file_stored: + statement_id = lei_statement['statementID'] + ref_statement_ids = {} + for rr_statement, rr_record in zip(rr_json_data_file_stored, rr_json_data_file_gleif): + start = rr_record["Relationship"]["StartNode"]["NodeID"] + end = rr_record["Relationship"]["EndNode"]["NodeID"] + rel_type = rr_record["Relationship"]["RelationshipType"] + if statement_id == rr_statement["subject"]["describedByEntityStatement"]: + ref_statement_ids[rr_statement['statementID']] = f"{start}_{end}_{rel_type}" + if (("describedByEntityStatement" in rr_statement["interestedParty"] and + statement_id == rr_statement["interestedParty"]["describedByEntityStatement"]) or + ("describedByPersonStatement" in rr_statement["interestedParty"] and + statement_id == rr_statement["interestedParty"]["describedByPersonStatement"])): + ref_statement_ids[rr_statement['statementID']] = f"{start}_{end}_{rel_type}" + if ref_statement_ids: + references[statement_id] = {'statement_id': statement_id, 'references_id': ref_statement_ids} + return references + +@pytest.fixture +def latest_ids(lei_json_data_file_stored, lei_json_data_file_gleif, + rr_json_data_file_stored, rr_json_data_file_gleif, + repex_json_data_file_stored, repex_json_data_file_gleif): + """Latest statementIDs""" + latest_index = {} + for item, statement in zip(lei_json_data_file_gleif, lei_json_data_file_stored): + latest = {'latest_id': item["LEI"], + 'statement_id': statement["statementID"], + 'reason': None} + latest_index[item["LEI"]] = latest + for item, statement in zip(rr_json_data_file_gleif, rr_json_data_file_stored): + start = item["Relationship"]["StartNode"]["NodeID"] + end = item["Relationship"]["EndNode"]["NodeID"] + rel_type = item["Relationship"]["RelationshipType"] + latest_id = f"{start}_{end}_{rel_type}" + latest = {'latest_id': latest_id, + 'statement_id': statement["statementID"], + 'reason': None} + latest_index[latest_id] = latest + entity = True + repex_json_data_double_gleif = list(itertools.chain.from_iterable(itertools.repeat(x, 2) + for x in repex_json_data_file_gleif)) + for item, statement in zip(repex_json_data_double_gleif, repex_json_data_file_stored): + lei = item['LEI'] + category = item['ExceptionCategory'] + reason = item['ExceptionReason'] + if entity: + latest_id = f"{lei}_{category}_{reason}_entity" + else: + latest_id = f"{lei}_{category}_{reason}_ownership" + latest = {'latest_id': latest_id, + 'statement_id': statement["statementID"], + 'reason': reason} + latest_index[latest_id] = latest + entity = not entity + return latest_index + + +@pytest.fixture +def exception_ids(repex_json_data_file_gleif, repex_json_data_file_stored): + """Latest data for GLEIF Reporting Exceptions""" + exception_data = {} + repex_json_data_double_gleif = list(itertools.chain.from_iterable(itertools.repeat(x, 2) + for x in repex_json_data_file_gleif)) + for item, statement in zip(repex_json_data_double_gleif, repex_json_data_file_stored): + lei = item['LEI'] + category = item['ExceptionCategory'] + reason = item['ExceptionReason'] + reference = item['ExceptionReference'] if 'ExceptionReference' in item else None + latest_id = f"{lei}_{category}" + if statement["statementType"] in ("entityStatement", "personStatement"): + exception_data[latest_id] = [None, statement["statementID"], reason, reference, statement["statementType"]] + else: + exception_data[latest_id][0] = statement["statementID"] + return exception_data + + +@pytest.fixture +def voided_ids(): + """Statement ids of voiding statements""" + return ["8f08b038-e0c2-0748-2a45-acb62b17f25b", + "1408ba29-4792-1d02-472a-661e3962778d", + "bcae847f-988a-2f75-ac09-6138864fd5a0", + "2d6ce004-492d-34ff-652a-b525b5fe4ca8", + "4243104a-bfef-7d92-a151-0e412a7a5269", + "c67bb160-25b6-c5b9-5767-b636b9846c9f"] + + +@pytest.fixture +def updated_ids(): + """Statement ids for final update statements""" + return ["89788883-ff51-a2c6-1be3-bcb7797abbae", + "996c02d8-cbbd-8b95-7702-34c295fa183f"] + + +def assert_annotations(statement, annotation_list): + """Asset statement contains listed annotations""" + for annotation_desc in annotation_list: + count = 0 + for annotation in statement["annotations"]: + if annotation["description"] == annotation_desc: count += 1 + assert count == 1, f"Annotation '{annotation_desc}' not found in {statement['statementID']}" + +def assert_lei_data(statement, lei_out, update=False, deleted=False): + """Assert statement matches expected output for LEI""" + print("Checking:", statement['statementID']) + assert statement['statementID'] == lei_out['statementID'] + assert statement['statementType'] == lei_out['statementType'] + assert statement['statementDate'] == lei_out['statementDate'] + assert validate_date_now(statement['publicationDetails']['publicationDate']) + if update: + assert statement["replacesStatements"] == update + else: + assert not "replacesStatements" in statement # New LEI + if deleted: + assert_annotations(statement, ["GLEIF data for this entity - LEI: 00TR8NKAEL48RGTZEW89; Registration Status: RETIRED"]) + else: + assert statement['name'] == lei_out['name'] + +def assert_rr_data(statement, rr_out, update=False, deleted=False): + """Assert statement matches expected output for RR""" + print("Checking:", statement['statementID']) + assert statement['statementID'] == rr_out['statementID'] + assert statement['statementType'] == rr_out['statementType'] + assert statement['statementDate'] == rr_out['statementDate'] + if update: + assert statement["replacesStatements"] == update + else: + assert not "replacesStatements" in statement # New LEI + if deleted: + assert statement["subject"]["describedByEntityStatement"] == "" + assert statement["interestedParty"]["describedByEntityStatement"] == "" + assert_annotations(statement, ["GLEIF relationship deleted on this statementDate."]) + else: + assert (statement["subject"]["describedByEntityStatement"] + == rr_out["subject"]["describedByEntityStatement"]) + assert (statement["interestedParty"]["describedByEntityStatement"] + == rr_out["interestedParty"]["describedByEntityStatement"]) + +def assert_repex_data(statement, repex_out, stype, update=False, deleted=False): + """Assert statement matches expected ouput for Repex""" + print("Checking:", statement['statementID'], statement) + assert statement['statementID'] == repex_out['statementID'] + if update: + assert statement["replacesStatements"] == update + else: + assert not "replacesStatements" in statement + if stype == 0: + assert statement['statementType'] == repex_out['statementType'] + assert statement["annotations"] == repex_out["annotations"] + elif stype == 1: + assert statement['statementType'] == repex_out['statementType'] + if repex_out['statementType'] == "entityStatement": + assert statement["entityType"] == repex_out["entityType"] + assert (statement["unspecifiedEntityDetails"]["reason"] == + repex_out["unspecifiedEntityDetails"]["reason"]) + assert (statement["unspecifiedEntityDetails"]["description"] == + repex_out["unspecifiedEntityDetails"]["description"]) + elif repex_out['statementType'] == "personStatement": + assert statement["personType"] == repex_out["personType"] + assert (statement["unspecifiedPersonDetails"]["reason"] == + repex_out["unspecifiedPersonDetails"]["reason"]) + assert (statement["unspecifiedPersonDetails"]["description"] == + repex_out["unspecifiedPersonDetails"]["description"]) + else: + assert statement["statementType"] == "ownershipOrControlStatement" + assert (statement["subject"]["describedByEntityStatement"] + == repex_out["subject"]["describedByEntityStatement"]) + if "describedByPersonStatement" in repex_out["interestedParty"]: + assert (statement["interestedParty"]["describedByPersonStatement"] + == repex_out["interestedParty"]["describedByPersonStatement"]) + else: + assert (statement["interestedParty"]["describedByEntityStatement"] + == repex_out["interestedParty"]["describedByEntityStatement"]) + if deleted: + assert {"reason": "interested-party-exempt-from-disclosure", + "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control." + } in statement["annotations"] + +def test_transform_stage_updates(updates_json_data_file, rr_json_data_file_stored, lei_json_data_file_stored, + lei_json_data_file_gleif, stored_references, latest_ids, exception_ids, + voided_ids, updated_ids, lei_json_data_file_output, rr_json_data_file_output, + repex_json_data_file_output, repex_json_data_file_gleif, + repex_json_data_file_stored, repex_json_data_file_gleif_new): + """Test transform pipeline stage on LEI-CDF v3.1 records""" + with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr, + patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as, + patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es, + patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno, + patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni): + + # Mock Kinesis input + class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + mock_kni.return_value = AsyncIterator(updates_json_data_file) + + # Mock directory methods + mock_pdr.return_value = None + mock_sdr.return_value = None + + # Mock ES async_scan + updates_stream = {} + async def as_result(): + print("Updates stream:", json.dumps(updates_stream)) + for item in updates_stream: + yield updates_stream[item] + mock_as.return_value = as_result() + + # Mock ES async_streaming_bulk + async def sb_result(): + voided = iter(voided_ids) + first = True + for item in updates_json_data_file: + if "Entity" in item: + id = generate_statement_id(entity_id(item), 'entityStatement') + if item["LEI"] == "00TR8NKAEL48RGTZEW89": id = "c9c4f816-ec2e-5292-4a0c-5b1680ff46a0" + yield (True, {'create': {'_id': id}}) + elif "Relationship" in item: + print(item) + id = generate_statement_id(rr_id(item), 'ownershipOrControlStatement') + if (item['Relationship']['StartNode']['NodeID'] == '98450051BS9C610A8T78' and + item['Relationship']['EndNode']['NodeID'] == '984500501A1B1045PB30'): + id = "f16a374a-09f1-a640-8303-a52e12957c30" + yield (True, {'create': {'_id': id}}) + elif "ExceptionCategory" in item: + print(item) + if "Extension" in item: + yield (True, {'create': {'_id': next(voided)}}) + yield (True, {'create': {'_id': next(voided)}}) + else: + if not item["LEI"] in ("159516LKTEGWITQUIE78", "159515PKYKQYJLT0KF16"): + print("Voiding:") + yield (True, {'create': {'_id': next(voided)}}) + if item['ExceptionReason'] in ("NATURAL_PERSONS", "NO_KNOWN_PERSON"): + print("Person:") + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'personStatement')}}) + else: + print("Entity:") + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'entityStatement')}}) + if item["LEI"] != "2549007VY3IWUVGW7A82": + print("Ownership:") + yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'ownershipOrControlStatement')}}) + for id in updated_ids: + yield (True, {'create': {'_id': id}}) + mock_sb.return_value = sb_result() + + # Mock Kinesis output stream (save results) + kinesis_output_stream = [] + async def async_put(record): + kinesis_output_stream.append(record) + return None + mock_kno.return_value.put.side_effect = async_put + + # Mock Kinesis output + flush_future = asyncio.Future() + flush_future.set_result(None) + mock_kno.return_value.flush.return_value = flush_future + producer_close_future = asyncio.Future() + producer_close_future.set_result(None) + mock_kno.return_value.close.return_value = producer_close_future + + # Mock Latest Index + latest_index_data = latest_ids + + print("Initial latest IDs:", json.dumps(latest_index_data)) + + print("Initial exception IDs:", json.dumps(exception_ids)) + + # Mock ES index method + async def async_index(index, document): + print("async_index:", index, document) + #if index == "references": + # stored_references.append(document['_source']) + if index == "updates": + updates_stream[document['_source']['referencing_id']] = document['_source'] + elif index == "latest": + latest_index_data[document['_source']['latest_id']] = document['_source'] + return None + mock_es.return_value.index.side_effect = async_index + + # Mock ES update method + async def async_update(index, id, document): + print("async_update:", index, id, document) + if index == "latest": + latest_index_data[document['latest_id']] = document + elif index == "updates": + updates_stream[document['referencing_id']] = document + return None + mock_es.return_value.update.side_effect = async_update + + # Mock ES delete method + async def async_delete(index, id): + print("async_delete:", index, id) + if index == "latest": + del latest_index_data[id] + elif index == "updates": + del updates_stream[id] + return None + mock_es.return_value.delete.side_effect = async_delete + + print("stored_references:", json.dumps(stored_references)) + + # Mock ES search method + async def async_search(index=None, query=None): + id = query["query"]["match"]["_id"] + print(f"Searching for {id} in {index}") + if index == "latest": + if id in latest_index_data: + results = [latest_index_data[id]] + + else: + results = [] + elif index == 'references': + if id in stored_references: + results = [stored_references[id]] + else: + results = [] + elif index == 'exceptions': + if id in exception_ids: + results = [{'latest_id': id, + 'statement_id': exception_ids[id][0], + 'other_id': exception_ids[id][1], + 'reason': exception_ids[id][2], + 'reference': exception_ids[id][3], + 'entity_type': exception_ids[id][4]}] + print("Stored exception:", results) + else: + results = [] + elif index == "ownership": + match = [item for item in rr_json_data_file_stored if item ['statementID'] == id] + if match: + results = [match[0]] + else: + results = [] + elif index == "updates": + if id in updates_stream: + results = [updates_stream[id]] + else: + results = [] + out = {'hits': {'hits': results}} + return out + mock_es.return_value.search.side_effect = async_search + + # Setup environment variables + set_environment_variables() + + # Kinesis stream of GLEIF data from ingest stage + gleif_source = Source(name="gleif", + origin=KinesisInput(stream_name="gleif-test"), + datatype=JSONData()) + + # Storage + bods_storage = ElasticsearchClient(indexes=bods_index_properties) + + # BODS data: Store in Easticsearch and output new to Kinesis stream + bods_output_new = NewOutput(storage=Storage(storage=bods_storage), + output=KinesisOutput(stream_name="bods-gleif-test"), + identify=identify_bods) + + # Definition of GLEIF data pipeline transform stage + transform_stage = Stage(name="transform-test-updates", + sources=[gleif_source], + processors=[ProcessUpdates(id_name='XI-LEI', + transform=Gleif2Bods(identify=identify_gleif), + storage=Storage(storage=bods_storage), + updates=GleifUpdates())], + outputs=[bods_output_new] + ) + + assert hasattr(transform_stage.processors[0], "finish_updates") + + # Definition of GLEIF data pipeline + pipeline = Pipeline(name="gleif", stages=[transform_stage]) + + # Run pipelne + pipeline.process("transform-test-updates", updates=True) + + print(json.dumps(kinesis_output_stream)) + + print("Final latest IDs:", json.dumps(latest_index_data)) + + # Check updates empty + assert len(updates_stream) == 0 + + # Check LEI updates + stored_offset = -1 + for i, item in enumerate(lei_json_data_file_output): + print(i) + if i == 0: + assert_lei_data(kinesis_output_stream[i], item) + elif i == 1: + result = {"statementID": "c9c4f816-ec2e-5292-4a0c-5b1680ff46a0", + "statementDate": "2023-09-03", + "statementType": "entityStatement"} + assert_lei_data(kinesis_output_stream[i], result, deleted=True, + update=[lei_json_data_file_stored[i+stored_offset]["statementID"]]) + else: + assert_lei_data(kinesis_output_stream[i], item, + update=[lei_json_data_file_stored[i+stored_offset]["statementID"]]) + if i == 7: stored_offset = 0 + + # Offset for start of RR statements + stream_offset = i + 1 + + # Check RR updates + stored_offset = -1 + for i, item in enumerate(rr_json_data_file_output): + print(i+stream_offset) + if i == 0: + assert_rr_data(kinesis_output_stream[i+stream_offset], item) + elif i == 9: + result = {"statementID": "f16a374a-09f1-a640-8303-a52e12957c30", + "statementDate": "2023-11-01", + "statementType": "ownershipOrControlStatement"} + assert_rr_data(kinesis_output_stream[i+stream_offset], result, + update=[rr_json_data_file_stored[i+stored_offset]["statementID"]], deleted=True) + else: + assert_rr_data(kinesis_output_stream[i+stream_offset], item, + update=[rr_json_data_file_stored[i+stored_offset]["statementID"]]) + if i == 5: stored_offset = 1 + + # Offset for start of Repex statements + stream_offset = stream_offset + i + 1 + + # Check Repex updates + stored_offset = -2 + cycle = 0 + count = 0 + ent_offset = 0 + for i, item in enumerate(repex_json_data_file_output): + print(i) + if i == 11: + stream_offset -= 1 + continue + if i < 2 or i > 10: + if i == 0 or i == 12: + entity = 1 + else: + entity = 2 + if i == 13: + repex_update = [repex_json_data_file_stored[-1]["statementID"]] + assert_repex_data(kinesis_output_stream[i+stream_offset], item, entity, update=repex_update) + else: + assert_repex_data(kinesis_output_stream[i+stream_offset], item, entity) + else: + if cycle == 0: + print("Current item:", item) + item_lei = item['annotations'][0]['description'].split()[-1] + repex_item = [ritem for ritem in repex_json_data_file_gleif + if ritem["LEI"] == item_lei][0] + repex_item_new = [ritem for ritem in repex_json_data_file_gleif_new + if ritem["LEI"] == item_lei][0] + item_reason = repex_item["ExceptionReason"] + annotation_desc_del = f'Statement series retired due to deletion of a {item_reason} GLEIF Reporting Exception for {item_lei}' + annotation_desc_chg = f'Statement retired due to change in a {item_reason} GLEIF Reporting Exception for {item_lei}' + result = {"statementID": voided_ids[count], + "statementDate": datetime.date.today().strftime('%Y-%m-%d'), + "statementType": repex_json_data_file_stored[i + stored_offset + ent_offset]["statementType"], + } + if "Extension" in repex_item_new: + result["annotations"] = [{'motivation': 'commenting', + 'description': annotation_desc_del, + 'statementPointerTarget': '/', + 'creationDate': datetime.date.today().strftime('%Y-%m-%d'), + 'createdBy': {'name': 'Open Ownership', + 'uri': 'https://www.openownership.org'}}] + else: + result["annotations"] = [{'motivation': 'commenting', + 'description': annotation_desc_chg, + 'statementPointerTarget': '/', + 'creationDate': datetime.date.today().strftime('%Y-%m-%d'), + 'createdBy': {'name': 'Open Ownership', + 'uri': 'https://www.openownership.org'}}] + repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]] + assert_repex_data(kinesis_output_stream[i+stream_offset], result, cycle, update=repex_update) + cycle = 1 + if i == 10: + count += 1 + stream_offset += 1 + ent_offset += 1 + result = {"statementID": voided_ids[count], + "statementDate": datetime.date.today().strftime('%Y-%m-%d'), + "statementType": "ownershipOrControlStatement", + "interestedParty": { + "describedByEntityStatement": "" + }, + "subject": { + "describedByEntityStatement": "" + }, + "annotations": [{"statementPointerTarget": "/", + "motivation": "commenting", + "description": "GLEIF RegistrationStatus set to RETIRED on this statementDate."}], + } + repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]] + assert_repex_data(kinesis_output_stream[i+stream_offset], result, cycle, update=repex_update) + else: + stream_offset += 1 + assert_repex_data(kinesis_output_stream[i+stream_offset], item, cycle) + cycle = 2 + elif cycle == 2: + repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]] + assert_repex_data(kinesis_output_stream[i+stream_offset], item, cycle, update=repex_update) + cycle = 0 + count += 1 + if i == 9: ent_offset += 2 diff --git a/tests/test_xml_parser.py b/tests/test_xml_parser.py index a36ecdf..647af15 100644 --- a/tests/test_xml_parser.py +++ b/tests/test_xml_parser.py @@ -2,14 +2,17 @@ import sys import time import datetime +import json from pathlib import Path import json from unittest.mock import patch, Mock +import asyncio import pytest from bodspipelines.infrastructure.processing.xml_data import XMLData def validate_datetime(d): + """Test is valid datetime""" try: datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z') return True @@ -18,41 +21,49 @@ def validate_datetime(d): def validate_date_now(d): + """Test is today's date""" return d == datetime.date.today().strftime('%Y-%m-%d') @pytest.fixture def lei_list(): + """List entity LEIs""" return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216', '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53'] @pytest.fixture def lei_xml_data_file(): + """GLEIF LEI XML data""" return Path("tests/fixtures/lei-data.xml") @pytest.fixture def rr_xml_data_file(): + """GLEIF RR XML data""" return Path("tests/fixtures/rr-data.xml") @pytest.fixture def repex_xml_data_file(): + """GLEIF Repex XML data""" return Path("tests/fixtures/repex-data.xml") -def test_lei_xml_parser(lei_xml_data_file): +@pytest.mark.asyncio +async def test_lei_xml_parser(lei_xml_data_file): + """Test XML parser on GLEIF LEI data""" xml_parser = XMLData(item_tag="LEIRecord", namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016"}, filter=['NextVersion', 'Extension']) data = xml_parser.process(lei_xml_data_file) count = 0 - for item in data: + async for header, item in data: if count == 0: assert item['LEI'] == '001GPB6A9XPE8XJICC14' assert item['Entity']['LegalName'] == 'Fidelity Advisor Leveraged Company Stock Fund' - assert item['Entity']['OtherEntityNames'] == ['FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund'] + assert item['Entity']['OtherEntityNames'] == [{'type': 'PREVIOUS_LEGAL_NAME', + 'OtherEntityName': 'FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund'}] assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '245 SUMMER STREET', 'City': 'BOSTON', 'Region': 'US-MA', 'Country': 'US', 'PostalCode': '02210'} assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Fidelity Management & Research Company LLC', @@ -70,19 +81,90 @@ def test_lei_xml_parser(lei_xml_data_file): assert item['Registration']['ManagingLOU'] == 'EVK05KS7XY1DEII3R011' assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000665', 'ValidationAuthorityEntityID': 'S000005113'} + elif count == 10: + assert item['LEI'] == '1595D0QCK7Y15293JK84' + assert item['Entity']['LegalName'] == 'GALAPAGOS CONSERVATION TRUST' + assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '7-14 Great Dover Street', 'City': 'London', 'Country': 'GB', 'PostalCode': 'SE1 4YR'} + assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': '7-14 Great Dover Street', 'City': 'London', 'Country': 'GB', + 'PostalCode': 'SE1 4YR'} + assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000585', 'RegistrationAuthorityEntityID': '03004112'} + assert item['Entity']['LegalJurisdiction'] == 'GB' + assert item['Entity']['EntityCategory'] == 'GENERAL' + assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'G12F'} + assert item['Entity']['EntityStatus'] == 'ACTIVE' + assert item['Entity']['EntityCreationDate'] == '1994-12-21T00:00:00+01:00' + assert item['Registration']['InitialRegistrationDate'] == '2023-02-13T22:13:11+01:00' + assert item['Registration']['LastUpdateDate'] == '2023-03-10T13:08:56+01:00' + assert item['Registration']['RegistrationStatus'] == 'ISSUED' + assert item['Registration']['NextRenewalDate'] == '2024-02-13T22:13:11+01:00' + assert item['Registration']['ManagingLOU'] == '98450045AN5EB5FDC780' + assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' + assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000585', 'ValidationAuthorityEntityID': '03004112'} + assert item['Registration']['OtherValidationAuthorities'] == [{'ValidationAuthorityID': 'RA000589', 'ValidationAuthorityEntityID': '1043470'}] + elif count == 11: + assert item['LEI'] == '213800FERQ5LE3H7WJ58' + assert item['Entity']['LegalName'] == 'DENTSU INTERNATIONAL LIMITED' + assert item['Entity']['OtherEntityNames'] == [{'type': 'PREVIOUS_LEGAL_NAME', 'OtherEntityName': 'DENTSU AEGIS NETWORK LTD.'}, + {'type': 'PREVIOUS_LEGAL_NAME', 'OtherEntityName': 'AEGIS GROUP PLC'}] + assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '10 TRITON STREET', 'AdditionalAddressLine': "REGENT'S PLACE", + 'City': 'LONDON', 'Region': 'GB-LND', 'Country': 'GB', 'PostalCode': 'NW1 3BF'} + assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': '10 TRITON STREET', 'AdditionalAddressLine': "REGENT'S PLACE", + 'City': 'LONDON', 'Region': 'GB-LND', 'Country': 'GB', 'PostalCode': 'NW1 3BF'} + assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000585', 'RegistrationAuthorityEntityID': '01403668'} + assert item['Entity']['LegalJurisdiction'] == 'GB' + assert item['Entity']['EntityCategory'] == 'GENERAL' + assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'H0PO'} + assert item['Entity']['EntityStatus'] == 'ACTIVE' + assert item['Entity']['EntityCreationDate'] == '1978-12-05T00:00:00Z' + assert item['Registration']['InitialRegistrationDate'] == '2014-02-10T00:00:00Z' + assert item['Registration']['LastUpdateDate'] == '2023-02-02T09:07:52.390Z' + assert item['Registration']['RegistrationStatus'] == 'ISSUED' + assert item['Registration']['NextRenewalDate'] == '2024-02-17T00:00:00Z' + assert item['Registration']['ManagingLOU'] == '213800WAVVOPS85N2205' + assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' + assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000585', 'ValidationAuthorityEntityID': '01403668'} + elif count == 12: + assert item['LEI'] == '213800BJPX8V9HVY1Y11' + assert item['Entity']['LegalName'] == 'Swedeit Italian Aktiebolag' + assert item['Entity']['LegalAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', + 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O', + 'Country': 'SE', 'PostalCode': '426 76'} + assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', + 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O', + 'Country': 'SE', 'PostalCode': '426 76'} + assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11', + 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76', + 'type': 'AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses'] + assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11', + 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76', + 'type': 'AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses'] + assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000544', 'RegistrationAuthorityEntityID': '556543-1193'} + assert item['Entity']['LegalJurisdiction'] == 'SE' + assert item['Entity']['EntityCategory'] == 'GENERAL' + assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'XJHM'} + assert item['Entity']['EntityStatus'] == 'ACTIVE' + assert item['Entity']['EntityCreationDate'] == '1997-06-05T02:00:00+02:00' + assert item['Registration']['InitialRegistrationDate'] == '2014-04-09T00:00:00Z' + assert item['Registration']['LastUpdateDate'] == '2023-04-25T13:18:00Z' + assert item['Registration']['RegistrationStatus'] == 'ISSUED' + assert item['Registration']['NextRenewalDate'] == '2024-05-12T06:59:39Z' + assert item['Registration']['ManagingLOU'] == '549300O897ZC5H7CY412' + assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' + assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000544', 'ValidationAuthorityEntityID': '556543-1193'} count += 1 - assert count == 10 + assert count == 13 -def test_rr_xml_parser(rr_xml_data_file): +@pytest.mark.asyncio +async def test_rr_xml_parser(rr_xml_data_file): + """Test XML parser on GLEIF RR data""" xml_parser = XMLData(item_tag="RelationshipRecord", namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"}, filter=['Extension']) data = xml_parser.process(rr_xml_data_file) count = 0 - for item in data: + async for header, item in data: if count == 0: - print(item) assert item['Relationship']['StartNode'] == {'NodeID': '001GPB6A9XPE8XJICC14', 'NodeIDType': 'LEI'} assert item['Relationship']['EndNode'] == {'NodeID': '5493001Z012YSB2A0K51', 'NodeIDType': 'LEI'} assert item['Relationship']['RelationshipType'] == 'IS_FUND-MANAGED_BY' @@ -100,13 +182,15 @@ def test_rr_xml_parser(rr_xml_data_file): assert count == 10 -def test_repex_xml_parser(repex_xml_data_file): +@pytest.mark.asyncio +async def test_repex_xml_parser(repex_xml_data_file): + """Test XML parser on GLEIF Repex data""" xml_parser = XMLData(item_tag="Exception", namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"}, filter=['NextVersion', 'Extension']) data = xml_parser.process(repex_xml_data_file) count = 0 - for item in data: + async for header, item in data: if count == 0: print(item) assert item['LEI'] == '001GPB6A9XPE8XJICC14' diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..b1f801b --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,22 @@ +import requests + +# Mock Kinesis input +class AsyncIterator: + def __init__(self, seq): + self.iter = iter(seq) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self.iter) + except StopIteration: + raise StopAsyncIteration + async def close(self): + return None + +def list_index_contents(index): + url = f"http://localhost:9200/{index}/_search" + query = '{"query": {"match_all": {}}, "size": "25"}' + print(requests.get(requestURL, + json=query, + headers={'content-type': 'application/json'}).text)