diff --git a/bodspipelines/infrastructure/caching.py b/bodspipelines/infrastructure/caching.py
new file mode 100644
index 0000000..bac2460
--- /dev/null
+++ b/bodspipelines/infrastructure/caching.py
@@ -0,0 +1,288 @@
+#from functools import wraps
+import inspect
+
+def get_id(storage, item_type, item):
+ """Get item id given item and item_type"""
+ return storage.storage.indexes[item_type]['id'](item)
+
+class Caching():
+ """Caching for updates"""
+ def __init__(self):
+ """Setup cache"""
+ self.initialised = False
+ self.cache = {"latest": {}, "references": {}, "exceptions": {}}
+ self.batch = {"latest": {}, "references": {}, "exceptions": {}}
+
+ async def load(self, storage):
+ """Load data into cache"""
+ for item_type in self.cache:
+ print(f"Loading cache for {item_type}")
+ async for item in storage.stream_items(item_type):
+ #print(item_type, item)
+ item_id = get_id(storage, item_type, item)
+ self.cache[item_type][item_id] = item
+ self.initialised = True
+
+ def save(self, item_type, item, item_id):
+ """Save item in cache"""
+ self.cache[item_type][item_id] = item
+
+ def batch_item(self, item_type, item, item_id, overwrite=False):
+ """Batch to be written later"""
+ if overwrite:
+ self.batch[item_type][item_id] = ('index', item)
+ else:
+ self.batch[item_type][item_id] = ('create', item)
+
+ def check_batch_item(self, item_type, item_id):
+ """Check batch for item"""
+ return self.batch[item_type].get(item_id)
+
+ def unbatch_item(self, item_type, item_id):
+ """Remove item from batch"""
+ del self.batch[item_type][item_id]
+
+ async def write_batch(self, storage, item_type):
+ """Write batch to storage"""
+ items = [self.batch[item_type][item_id] for item_id in self.batch[item_type]]
+ await storage.add_batch(item_type, items)
+ self.batch[item_type] = {}
+
+ async def check_batch(self, storage):
+ """Check if any batches need writing"""
+ for item_type in self.batch:
+ if len(self.batch[item_type]) > 485:
+ await self.write_batch(storage, item_type)
+
+ async def flush_batch(self, storage):
+ """Check if any batches need writing"""
+ for item_type in self.batch:
+ if len(self.batch[item_type]) > 0:
+ await self.write_batch(storage, item_type)
+
+ def read(self, item_type, item_id):
+ """Read item from cache"""
+ return self.cache[item_type].get(item_id)
+
+ def delete(self, item_type, item_id):
+ """Delete item from cache"""
+ del self.cache[item_type][item_id]
+
+#cache = Caching()
+
+#def cached(func):
+# """Apply caching to function"""
+# @wraps(func)
+# def wrapper(*args, **kwargs):
+# storage = func.__self__
+# if not cache.initialised:
+# cache.load(storage)
+# if func.__name__ == "add_item":
+# item = args[0]
+# item_type = args[1]
+# item_id = get_id(storage, item_type, item)
+# cache.save(item_type, item, item_id)
+# return func(*args, **kwargs)
+# elif func.__name__ == "get_item":
+# item_id = args[0]
+# item_type = args[1]
+# item = cache.read(item_type, item_id)
+# if item: return item
+# return func(*args, **kwargs)
+# return wrapper
+
+#async def load_cache(storage):
+# await cache.load(storage)
+
+#async def flush_cache(storage):
+# await cache.flush_batch(storage)
+
+#async def cached(func, *args, **kwargs):
+# """Apply caching to function call"""
+# #if inspect.isclass(func):
+# # storage = func
+# #else:
+# storage = func.__self__
+# #if "initialise" in kwargs and kwargs["initialise"]:
+# # cache.load(storage)
+# if "batch" in kwargs:
+# batch = kwargs["batch"]
+# del kwargs["batch"]
+# else:
+# batch = False
+# if "overwrite" in kwargs:
+# overwrite = kwargs["overwrite"]
+# del kwargs["overwrite"]
+# else:
+# overwrite = False
+# #if batch == "finished":
+# # cache.flush_batch(storage)
+# # return
+# #if not cache.initialised:
+# # cache.load(storage)
+# out = None
+# if func.__name__ == "add_item":
+# item = args[0]
+# item_type = args[1]
+# item_id = get_id(storage, item_type, item)
+# cache.save(item_type, item, item_id)
+# if batch:
+# cache.batch_item(item_type, item, item_id, overwrite=overwrite)
+# else:
+# out = await func(*args, **kwargs)
+# elif func.__name__ == "get_item":
+# item_id = args[0]
+# item_type = args[1]
+# item = cache.read(item_type, item_id)
+# if item:
+# out = item
+# else:
+# out = await func(*args, **kwargs)
+# elif func.__name__ == "delete_item":
+# item_id = args[0]
+# item_type = args[1]
+# cache.delete(item_type, item_id)
+# if cache.check_batch_item(item_type, item_id):
+# cache.unbatch_item(item_type, item_id)
+# else:
+# out = await func(*args, **kwargs)
+# if batch: await cache.check_batch(storage)
+# return out
+
+class Caching():
+ """Caching for updates"""
+ def __init__(self, storage, batching=False):
+ """Setup cache"""
+ self.initialised = False
+ self.cache = {"latest": {}, "references": {}, "exceptions": {}, "updates": {}}
+ self.batch = {"latest": {}, "references": {}, "exceptions": {}, "updates": {}} if batching else None
+ self.batch_size = batching if batching else None
+ self.memory_only = ["updates"]
+ self.storage = storage
+
+ async def load(self):
+ """Load data into cache"""
+ for item_type in self.cache:
+ if not item_type in self.memory_only:
+ print(f"Loading cache for {item_type}")
+ async for item in self.storage.stream_items(item_type):
+ #print(item_type, item)
+ item_id = get_id(self.storage, item_type, item)
+ self.cache[item_type][item_id] = item
+ self.initialised = True
+
+ def _save(self, item_type, item, item_id, overwrite=False):
+ """Save item in cache"""
+ if item_id in self.cache[item_type]:
+ if overwrite:
+ self.cache[item_type][item_id] = item
+ if item_id in self.batch[item_type]:
+ # ???
+ self._batch_item(item_type, item, item_id, overwrite=True)
+ else:
+ self._batch_item(item_type, item, item_id, overwrite=True)
+ else:
+ raise Exception("Cannot overwrite item {item_id} of type {item_type}")
+ else:
+ self.cache[item_type][item_id] = item
+ self._batch_item(item_type, item, item_id, overwrite=overwrite)
+
+ def _batch_item(self, item_type, item, item_id, overwrite=False):
+ """Batch to be written later"""
+ if overwrite:
+ self.batch[item_type][item_id] = ('index', item)
+ else:
+ self.batch[item_type][item_id] = ('index', item)
+
+ def _check_batch_item(self, item_type, item_id):
+ """Check batch for item"""
+ return self.batch[item_type].get(item_id)
+
+ def _unbatch_item(self, item_type, item_id):
+ """Remove item from batch"""
+ del self.batch[item_type][item_id]
+
+ async def _generate_items(self, items):
+ for item in items:
+ yield item[1]
+
+ async def _write_batch(self, item_type):
+ """Write batch to storage"""
+ for action in ('index', 'update', 'delete'):
+ items = [self.batch[item_type][item_id] for item_id in self.batch[item_type]
+ if self.batch[item_type][item_id][0] == action]
+ #print(f"{action}: {items}")
+ if items:
+ print(f"Flushing {action}: {len(items)} items")
+ await self.storage.dump_stream(item_type, action, self._generate_items(items))
+ self.batch[item_type] = {}
+
+ async def _check_batch(self):
+ """Check if any batches need writing"""
+ for item_type in self.batch:
+ if not item_type in self.memory_only:
+ if len(self.batch[item_type]) > self.batch_size:
+ await self._write_batch(item_type)
+
+ async def flush(self):
+ """Check if any item need writing"""
+ print("Flushing cache")
+ for item_type in self.batch:
+ if not item_type in self.memory_only:
+ #print(f"{item_type}: {len(self.batch[item_type])} items in batch")
+ if len(self.batch[item_type]) > 0:
+ await self._write_batch(item_type)
+
+ def _read(self, item_type, item_id):
+ """Read item from cache"""
+ return self.cache[item_type].get(item_id)
+
+ def _delete(self, item_type, item_id, if_exists=False):
+ """Delete item from cache"""
+ if if_exists and not item_id in self.cache[item_type]:
+ return
+ item = self.cache[item_type][item_id]
+ del self.cache[item_type][item_id]
+ if not item_type in self.memory_only:
+ if item_id in self.batch[item_type] and self.batch[item_type][item_id][0] == 'index':
+ del self.batch[item_type][item_id]
+ else:
+ self.batch[item_type][item_id] = ('delete', item)
+
+ #async def flush_cache(storage):
+ # await self._flush_batch()
+
+ async def add(self, item, item_type, overwrite=False):
+ """Apply caching to function call"""
+ item_id = get_id(self.storage, item_type, item)
+ self._save(item_type, item, item_id, overwrite=overwrite)
+ if not self.batch is None:
+ self._batch_item(item_type, item, item_id, overwrite=overwrite)
+ else:
+ out = await self.storage.add_item(*args, **kwargs)
+
+ async def get(self, item_id, item_type):
+ """Get cached item"""
+ #print(item_id, item_type)
+ item = self._read(item_type, item_id)
+ #if item:
+ # out = item
+ #else:
+ # out = await self.storage.get_item(item, item_type)
+ return item
+
+ async def delete(self, item_id, item_type, if_exists=False):
+ """Delete acched item"""
+ self._delete(item_type, item_id, if_exists=if_exists)
+ #if self._check_batch_item(item_type, item_id):
+ # self._unbatch_item(item_type, item_id)
+ #else:
+ # out = await self.storage.delete_item(item_id, item_type)
+
+ async def stream(self, item_type):
+ """Get cached items"""
+ for item_id in self.cache[item_type]:
+ yield self.cache[item_type].get(item_id)
+
+ def count(self, item_type):
+ return len(self.cache[item_type])
diff --git a/bodspipelines/infrastructure/clients/elasticsearch_client.py b/bodspipelines/infrastructure/clients/elasticsearch_client.py
index 020b157..89c72f7 100644
--- a/bodspipelines/infrastructure/clients/elasticsearch_client.py
+++ b/bodspipelines/infrastructure/clients/elasticsearch_client.py
@@ -1,18 +1,27 @@
import os
import json
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk, streaming_bulk
+import asyncio
+import elastic_transport
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.helpers import async_streaming_bulk, async_scan, async_bulk
-def create_client():
+async def create_client():
"""Create Elasticsearch client"""
protocol = os.getenv('ELASTICSEARCH_PROTOCOL')
host = os.getenv('ELASTICSEARCH_HOST')
port = os.getenv('ELASTICSEARCH_PORT')
password = os.getenv('ELASTICSEARCH_PASSWORD')
if password:
- return Elasticsearch(f"{protocol}://{host}:{port}", basic_auth=('elastic', password), timeout=30, max_retries=10, retry_on_timeout=True)
+ return AsyncElasticsearch(f"{protocol}://{host}:{port}",
+ basic_auth=('elastic', password),
+ timeout=30,
+ max_retries=10,
+ retry_on_timeout=True)
else:
- return Elasticsearch(f"{protocol}://{host}:{port}", timeout=30, max_retries=10, retry_on_timeout=True) #, basic_auth=('elastic', password))
+ return AsyncElasticsearch(f"{protocol}://{host}:{port}",
+ timeout=30,
+ max_retries=10,
+ retry_on_timeout=True)
def index_definition(record, out):
"""Create index definition from record"""
@@ -26,16 +35,20 @@ def index_definition(record, out):
class ElasticsearchClient:
"""ElasticsearchClient class"""
- def __init__(self):
+ def __init__(self, indexes):
"""Initial setup"""
- self.client = create_client()
+ self.client = None
+ self.indexes = indexes
self.index_name = None
+ async def create_client(self):
+ self.client = await create_client()
+
def set_index(self, index_name):
"""Set index name"""
self.index_name = index_name
- def create_index(self, index_name, properties):
+ async def create_index(self, index_name, properties):
"""Create index"""
self.set_index(index_name)
# index settings
@@ -43,31 +56,68 @@ def create_index(self, index_name, properties):
"number_of_replicas": 0}
mappings = {"dynamic": "strict",
"properties": properties}
- if not self.client.indices.exists(index=self.index_name):
+ if not await self.client.indices.exists(index=self.index_name):
# Ignore 400 means to ignore "Index Already Exist" error.
- self.client.options(ignore_status=400).indices.create(index=self.index_name, settings=settings, mappings=mappings)
+ await self.client.options(ignore_status=400).indices.create(index=self.index_name,
+ settings=settings,
+ mappings=mappings)
print('Elasticserach created Index')
+ async def setup_indexes(self):
+ """Setup indexes"""
+ done = False
+ if not self.client: await self.setup()
+ while not done:
+ try:
+ await self.create_indexes()
+ done = True
+ except elastic_transport.ConnectionError:
+ print("Waiting for Elasticsearch to start ...")
+ await asyncio.sleep(5)
+ await self.close()
+
def delete_index(self):
"""Delete index"""
self.client.options(ignore_status=[400, 404]).indices.delete(index=self.index_name)
- def stats(self, index_name):
- """Get index statistics"""
- return self.client.indices.stats(index=index_name)
+ async def create_indexes(self):
+ """Moved from storage"""
+ for index_name in self.indexes:
+ await self.create_index(index_name, self.indexes[index_name]['properties'])
- def store_data(self, data):
+ async def statistics(self, index_name):
+ """Get index statistics"""
+ #count = 0
+ stats = {}
+ #for index_name in self.indexes:
+ await self.client.indices.refresh(index=index_name)
+ result = await self.client.cat.count(index=index_name, params={"format": "json"})
+ #result = await self.client.indices.stats(index=index_name)
+ #print(result)
+ #stats[index_name] = result['_all']['primaries']['docs']['count']
+ #count += result['_all']['primaries']['docs']['count']
+ #stats['total'] = result['_all']['primaries']['indexing']['index_total']
+ stats['total'] = int(result[0]['count'])
+ return stats
+
+ async def store_data(self, data, id=None):
"""Store data in index"""
if isinstance(data, list):
for d in data:
- self.client.index(index=self.index_name, document=d)
+ await self.client.index(index=self.index_name, document=d)
else:
- self.client.index(index=self.index_name, document=data)
+ #print(f"Storing in {self.index_name}: {data}")
+ await self.client.index(index=self.index_name, document=data, id=id)
+
+ async def update_data(self, data, id):
+ """Update data in index"""
+ #print(f"Updating {self.index_name} ({id}): {data}")
+ await self.client.update(index=self.index_name, id=id, doc=data)
def bulk_store_data(self, actions, index_name):
"""Store bulk data in index"""
for ok, item in streaming_bulk(client=self.client, index=index_name, actions=actions):
- print(ok, item)
+ #print(ok, item)
if not ok:
yield False
else:
@@ -75,38 +125,89 @@ def bulk_store_data(self, actions, index_name):
def batch_store_data(self, actions, index_name):
"""Store bulk data in index"""
- #ok, errors = bulk(client=self.client, index=index_name, actions=actions)
errors = self.client.bulk(index=index_name, operations=actions)
print("Bulk:", errors)
return errors
- def batch_store_data(self, actions, batch, index_name):
+ async def batch_store_data(self, actions, batch, index_name):
"""Store bulk data in index"""
+ #await self.create_client()
record_count = 0
new_records = 0
- for ok, result in streaming_bulk(client=self.client, actions=actions, raise_on_error=False): #index=index_name,
+ #for b in batch:
+ # print(b['_id'], b['_index'])
+ async for ok, result in async_streaming_bulk(client=self.client, actions=actions, raise_on_error=False):
record_count += 1
#print(ok, result)
- #print(batch[0])
if ok:
new_records += 1
- match = [i for i in batch if i['_id'] == result['create']['_id']]
- yield match[0]['_source']
+ #match = [i for i in batch if i['_id'] == result['create']['_id']]
+ match = [i for i in batch if i['_id'] == result[i['_op_type']]['_id']]
+ if match[0]['_op_type'] == 'delete':
+ yield True
+ else:
+ yield match[0]['_source']
else:
print(ok, result)
- #
- #if not ok:
- # yield False
- #else:
- # yield item
if callable(index_name):
- print(f"Storing in {index_name(batch[0]['_source'])}: {record_count} records; {new_records} new records")
+ index_name = index_name(batch[0]['_source'])
+ #print(f"Storing in {index_name(batch[0]['_source'])}: {record_count} records; {new_records} new records")
+ if batch[0]['_op_type'] == 'delete':
+ print(f"Deleting in {index_name}: {record_count} records; {new_records} records deleted")
else:
print(f"Storing in {index_name}: {record_count} records; {new_records} new records")
- def search(self, search):
+ async def search(self, search):
"""Search index"""
- return self.client.search(index=self.index_name, query=search)
+ #print(f"ES search ({self.index_name}): {search}")
+ return await self.client.search(index=self.index_name, query=search)
+
+ async def get(self, id):
+ """Get by id"""
+ #match = await self.search({"query": {"match": {"_id": id}}})
+ match = await self.search({"match": {"_id": id}})
+ result = match['hits']['hits']
+ if result:
+ return result[0]['_source']
+ else:
+ return None
+
+ async def delete(self, id):
+ """Delete by id"""
+ return await self.client.delete(index=self.index_name, id=id)
+
+ async def delete_all(self, index):
+ """Delete all documents in index"""
+ await self.client.delete_by_query(index=index, query={"query":{"match_all":{}}})
+
+ async def scan_index(self, index):
+ """Scan index"""
+ async for doc in async_scan(client=self.client,
+ query={"query": {"match_all": {}}},
+ index=index):
+ yield doc
+
+ async def _generate_actions(self, index_name, action_type, items):
+ #if action_type == 'update':
+ # metadata = {'_op_type': action_type,
+ # '_type': 'document',
+ # "_index": index_name}
+ #else:
+ metadata = {'_op_type': action_type,
+ "_index": index_name}
+ async for item in items:
+ if action_type == 'delete':
+ action = metadata | {'_id': self.indexes[index_name]["id"](item)}
+ elif action_type == 'update':
+ action = metadata | {'_id': self.indexes[index_name]["id"](item)} | item
+ else:
+ action = metadata | {'_id': self.indexes[index_name]["id"](item)} | item
+ #print(action)
+ yield action
+
+ async def dump_stream(self, index_name, action_type, items):
+ await async_bulk(client=self.client,
+ actions=self._generate_actions(index_name, action_type, items))
def list_indexes(self):
"""List indexes"""
@@ -117,5 +218,13 @@ def get_mapping(self, index_name):
return self.client.indices.get_mapping(index=index_name)
def check_new(self, data):
+ """Dummy method"""
pass
+ async def setup(self):
+ """Setup Elasticsearch client"""
+ self.client = await create_client()
+
+ async def close(self):
+ """Close Elasticsearch client"""
+ await self.client.transport.close()
diff --git a/bodspipelines/infrastructure/clients/kinesis_client.py b/bodspipelines/infrastructure/clients/kinesis_client.py
index c94125d..5fb8642 100644
--- a/bodspipelines/infrastructure/clients/kinesis_client.py
+++ b/bodspipelines/infrastructure/clients/kinesis_client.py
@@ -2,19 +2,28 @@
import time
import json
import gzip
-import boto3
+#import boto3
-def create_client(service):
+from pathlib import Path
+from aiobotocore.session import get_session
+
+async def create_client(service):
"""Create AWS client for specified service"""
- return boto3.client(service, region_name=os.getenv('BODS_AWS_REGION'), aws_access_key_id=os.environ.get('BODS_AWS_ACCESS_KEY_ID'),
- aws_secret_access_key=os.environ.get('BODS_AWS_SECRET_ACCESS_KEY'))
+ #return boto3.client(service, region_name=os.getenv('BODS_AWS_REGION'), aws_access_key_id=os.environ.get('BODS_AWS_ACCESS_KEY_ID'),
+ # aws_secret_access_key=os.environ.get('BODS_AWS_SECRET_ACCESS_KEY'))
+ session = get_session()
+ return await session.create_client(service, region_name=os.getenv('BODS_AWS_REGION'),
+ aws_secret_access_key=os.environ["BODS_AWS_SECRET_ACCESS_KEY"],
+ aws_access_key_id=os.environ["BODS_AWS_ACCESS_KEY_ID"]).__aenter__()
+
-def get_stream_arn(client, stream_name):
- return client.describe_stream(StreamName=stream_name)["StreamDescription"]["StreamARN"]
+async def get_stream_arn(client, stream_name):
+ data = await client.describe_stream(StreamName=stream_name)
+ return data["StreamDescription"]["StreamARN"]
-def shard_id(client, stream_arn):
+async def shard_id(client, stream_arn):
"""Generate shard id"""
- response = client.describe_stream(StreamARN=stream_arn)
+ response = await client.describe_stream(StreamARN=stream_arn)
return response['StreamDescription']['Shards'][0]['ShardId']
def unpack_records(record_response):
@@ -25,23 +34,50 @@ def unpack_records(record_response):
records.append(json.loads(record['Data']))
return records
+def save_last_seqno(stream_name, last_seqno):
+ status_dir = os.getenv('KINESIS_STATUS_DIRECTORY')
+ path = Path(f"{status_dir}/{stream_name}")
+ if last_seqno:
+ with open(path, 'w') as file:
+ file.write(last_seqno)
+
+def load_last_seqno(stream_name):
+ status_dir = os.getenv('KINESIS_STATUS_DIRECTORY')
+ path = Path(f"{status_dir}/{stream_name}")
+ if path.is_file():
+ with open(path, 'r') as file:
+ return file.read()
+ else:
+ return None
+
class KinesisStream:
"""Kinesis Stream class"""
def __init__(self, stream_name=None, shard_count=1):
"""Initial setup"""
- self.client = create_client('kinesis')
- self.stream_arn = get_stream_arn(self.client, stream_name)
- self.shard_id = shard_id(self.client, self.stream_arn)
+ self.stream_name = stream_name
+ #self.stream_arn = get_stream_arn(self.client, stream_name)
+ #self.shard_id = shard_id(self.client, self.stream_arn)
self.shard_count = shard_count
self.records = []
self.waiting_bytes = 0
+ self.last_seqno = load_last_seqno(self.stream_name)
- def send_records(self):
+ async def setup(self):
+ """Setup Kinesis client"""
+ self.client = await create_client('kinesis')
+ self.stream_arn = await get_stream_arn(self.client, self.stream_name)
+ self.shard_id = await shard_id(self.client, self.stream_arn)
+
+ def save_last_seqno(self, response):
+ if response and 'Records' in response and len(response['Records']) > 0:
+ self.last_seqno = response['Records'][-1]['SequenceNumber']
+
+ async def send_records(self):
"""Send accumulated records"""
print(f"Sending {len(self.records)} records to {self.stream_arn}")
failed = len(self.records)
while failed == len(self.records):
- response = self.client.put_records(Records=self.records, StreamARN=self.stream_arn) #, StreamARN='string')
+ response = await self.client.put_records(Records=self.records, StreamARN=self.stream_arn) #, StreamARN='string')
failed = response['FailedRecordCount']
if failed == len(self.records):
time.sleep(1)
@@ -59,7 +95,7 @@ def send_records(self):
self.waiting_bytes = 0
break
- def add_record(self, record):
+ async def add_record(self, record):
"""Add record to stream"""
json_data = json.dumps(record) + "\n"
#encoded_data = bytes(json_data, 'utf-8')
@@ -69,28 +105,36 @@ def add_record(self, record):
self.waiting_bytes += num_bytes
#print(f"Added {num_bytes} byte record ...")
#print(f"Batched records {len(self.records)}")
- if self.waiting_bytes > 500000 or len(self.records) > 485: self.send_records()
+ if self.waiting_bytes > 500000 or len(self.records) > 485: await self.send_records()
- def finish_write(self):
+ async def finish_write(self):
"""Write any remaining records"""
- if len(self.records) > 0: self.send_records()
+ if len(self.records) > 0: await self.send_records()
- def read_stream(self):
+ async def read_stream(self):
"""Read records from stream"""
- shard_iterator = self.client.get_shard_iterator(StreamARN=self.stream_arn,
+ if self.last_seqno:
+ shard_iterator = await self.client.get_shard_iterator(StreamARN=self.stream_arn,
+ ShardId=self.shard_id,
+ ShardIteratorType='AFTER_SEQUENCE_NUMBER',
+ StartingSequenceNumber=self.last_seqno)
+ else:
+ shard_iterator = await self.client.get_shard_iterator(StreamARN=self.stream_arn,
ShardId=self.shard_id,
ShardIteratorType='TRIM_HORIZON')
- #ShardIteratorType='LATEST')
shard_iterator = shard_iterator['ShardIterator']
empty = 0
while True:
- record_response = self.client.get_records(ShardIterator=shard_iterator, Limit=100)
+ record_response = await self.client.get_records(ShardIterator=shard_iterator, Limit=500)
+ self.save_last_seqno(record_response)
#print(record_response)
if len(record_response['Records']) == 0 and record_response['MillisBehindLatest'] == 0:
empty += 1
else:
if len(record_response['Records']) > 0: empty = 0
- yield unpack_records(record_response)
+ print(f"Read {len(record_response['Records'])} records from {self.stream_arn}")
+ for item in unpack_records(record_response):
+ yield item
if empty > 250:
print(f"No records found in {self.stream_arn} after {empty} retries")
break
@@ -99,6 +143,12 @@ def read_stream(self):
else:
break
+ async def close(self):
+ """Close Kinesis client"""
+ if self.client:
+ await self.client.__aexit__(None, None, None)
+ save_last_seqno(self.stream_name, self.last_seqno)
+
# def read_stream(self):
# """Read records from stream"""
# stream = self.client.describe_stream(StreamName=self.stream_name)
diff --git a/bodspipelines/infrastructure/clients/redis_client.py b/bodspipelines/infrastructure/clients/redis_client.py
new file mode 100644
index 0000000..1bf372f
--- /dev/null
+++ b/bodspipelines/infrastructure/clients/redis_client.py
@@ -0,0 +1,87 @@
+import os
+import json
+
+from redis.asyncio import Redis, RedisError
+
+def create_client():
+ """Create redis client"""
+ host = os.getenv('REDIS_HOST')
+ port = os.getenv('REDIS_PORT')
+ return Redis(host=host, port=port)
+
+def get_key(index, id):
+ return f"{index}-{id}"
+
+class RedisClient:
+ """RedisClient class"""
+ def __init__(self, indexes):
+ """Initial setup"""
+ self.client = create_client()
+ self.indexes = indexes
+ self.index_name = None
+
+ def set_index(self, index_name):
+ """Set index name"""
+ self.index_name = index_name
+
+ async def batch_store_data(self, actions, batch, index_name, output_new=True):
+ """Store bulk data in index"""
+ record_count = 0
+ new_records = 0
+ async with self.client.pipeline() as pipe:
+ async for item in actions:
+ key = get_key(index_name, item['_id'])
+ await pipe.setnx(key, json.dumps(item['_source']))
+ results = await pipe.execute()
+ for i, result in enumerate(results):
+ if result is True and output_new:
+ new_records += 1
+ yield batch[i]['_source']
+ #else:
+ # print(result, batch[i]['_id'])
+
+ async def get(self, id):
+ """Search index"""
+ key = get_key(self.index_name, id)
+ try:
+ value = await self.client.get(key)
+ except RedisError:
+ return None
+ return json.loads(value)
+
+ async def store_data(self, data):
+ """Store data in index"""
+ if isinstance(data, list):
+ for d in data:
+ key = get_key(self.index_name, d['_id'])
+ await self.client.set(key, d['_source'])
+ else:
+ key = get_key(self.index_name, data['_id'])
+ await self.client.set(key, data['_source'])
+
+ async def count_keys(self, pattern: str) -> int:
+ """Counts the number of keys matching a pattern."""
+ keys_count = self.client.register_script(
+ """
+ return #redis.call('KEYS', ARGV[1])
+ """
+ )
+ return await keys_count(args=[pattern])
+
+ async def statistics(self) -> dict:
+ """Calculate storage statistics"""
+ stats = {}
+ for index in self.indexes:
+ keys = await self.count_keys(f"{index}*")
+ stats[index] = keys
+ total_keys = await self.client.dbsize()
+ stats['total'] = total_keys
+ return stats
+
+ async def setup(self):
+ """Dummy setup method"""
+ pass
+
+ async def setup_indexes(self):
+ """Dummy setup indexes"""
+ pass
diff --git a/bodspipelines/infrastructure/indexes.py b/bodspipelines/infrastructure/indexes.py
index e1e00e1..1efc617 100644
--- a/bodspipelines/infrastructure/indexes.py
+++ b/bodspipelines/infrastructure/indexes.py
@@ -4,6 +4,7 @@
'statementDate': {'type': 'text'},
'entityType': {'type': 'text'},
'name': {'type': 'text'},
+ 'isComponent': {"type": "boolean"},
'incorporatedInJurisdiction': {'type': 'object',
'properties': {'name': {'type': 'text'},
'code': {'type': 'text'}}},
@@ -29,7 +30,17 @@
'url': {'type': 'text'}}}}},
'source': {'type': 'object',
'properties': {'type': {'type': 'text'},
- 'description': {'type': 'text'}}}}
+ 'description': {'type': 'text'}}},
+ 'annotations': {'type': 'object',
+ 'properties': {'motivation': {'type': 'text'},
+ 'description': {'type': 'text'},
+ 'statementPointerTarget': {'type': 'text'},
+ 'creationDate': {'type': 'text'},
+ 'createdBy': {'type': 'object',
+ 'properties': {'name': {'type': 'text'},
+ 'uri': {'type': 'text'}}}}},
+ 'replacesStatements': {'type': 'text'}
+ }
# BODS Entity Statement Elasticsearch Properties
@@ -37,6 +48,7 @@
'statementType': {'type': 'text'},
'statementDate': {'type': 'text'},
'personType': {'type': 'text'},
+ 'isComponent': {"type": "boolean"},
'unspecifiedPersonDetails': {'type': 'object',
'properties': {'reason': {'type': 'text'},
'description': {'type': 'text'}}},
@@ -49,12 +61,23 @@
'url': {'type': 'text'}}}}},
'source': {'type': 'object',
'properties': {'type': {'type': 'text'},
- 'description': {'type': 'text'}}}}
+ 'description': {'type': 'text'}}},
+ 'annotations': {'type': 'object',
+ 'properties': {'motivation': {'type': 'text'},
+ 'description': {'type': 'text'},
+ 'statementPointerTarget': {'type': 'text'},
+ 'creationDate': {'type': 'text'},
+ 'createdBy': {'type': 'object',
+ 'properties': {'name': {'type': 'text'},
+ 'uri': {'type': 'text'}}}}},
+ 'replacesStatements': {'type': 'text'}
+ }
# BODS Ownership Or Control Statement
ownership_statement_properties = {'statementID': {'type': 'text'},
'statementType': {'type': 'text'},
'statementDate': {'type': 'text'},
+ 'isComponent': {"type": "boolean"},
'subject': {'type': 'object',
'properties': {'describedByEntityStatement': {'type': 'text'}}},
'interestedParty': {'type': 'object',
@@ -85,8 +108,42 @@
'creationDate': {'type': 'text'},
'createdBy': {'type': 'object',
'properties': {'name': {'type': 'text'},
- 'uri': {'type': 'text'}}}}}}
+ 'uri': {'type': 'text'}}}}},
+ 'replacesStatements': {'type': 'text'}
+ }
+
+
+# Additional indexes for managing updates
+latest_properties = {'latest_id': {'type': 'text'},
+ 'statement_id': {'type': 'text'},
+ 'reason': {'type': 'text'}}
+references_properties = {'statement_id': {'type': 'text'},
+ 'references_id': {'type': 'object',
+ 'properties': {'statement_id': {'type': 'text'},
+ 'latest_id': {'type': 'text'}}}
+ }
+#updates_properties = {'referencing_id': {'type': 'text'},
+# 'old_statement_id': {'type': 'text'},
+# 'new_statement_id': {'type': 'text'}}
+updates_properties = {'referencing_id': {'type': 'text'},
+ 'latest_id': {'type': 'text'},
+ 'updates': {'type': 'object',
+ 'properties': {'old_statement_id': {'type': 'text'},
+ 'new_statement_id': {'type': 'text'}}}
+ #{'type': 'text'}
+ }
+exceptions_properties = {'latest_id': {'type': 'text'},
+ 'statement_id': {'type': 'text'},
+ 'other_id': {'type': 'text'},
+ 'reason': {'type': 'text'},
+ 'reference': {'type': 'text'},
+ 'entity_type': {'type': 'text'}}
+
+# Properties for logging pipeline runs
+pipeline_run_properties = {'stage_name': {'type': 'text'},
+ 'start_timestamp': {"type": "text"},
+ 'end_timestamp': {"type": "text"}}
def match_entity(item):
return {"match": {"statementID": item["statementID"]}}
@@ -97,6 +154,21 @@ def match_person(item):
def match_ownership(item):
return {"match": {"statementID": item["statementID"]}}
+def match_latest(item):
+ return {"match": {"latest_id": item["latest_id"]}}
+
+def match_references(item):
+ return {"match": {"statement_id": item["statement_id"]}}
+
+def match_updates(item):
+ return {"match": {"old_statement_id": item["old_statement_id"]}}
+
+def match_exceptions(item):
+ return {"match": {"latest_id": item["latest_id"]}}
+
+def match_run(item):
+ return {"match": {"end_timestamp": item["end_timestamp"]}}
+
def id_entity(item):
return item["statementID"]
@@ -105,3 +177,28 @@ def id_person(item):
def id_ownership(item):
return item["statementID"]
+
+def id_latest(item):
+ return item["latest_id"]
+
+def id_references(item):
+ return item["statement_id"]
+
+def id_updates(item):
+ return item["referencing_id"]
+
+def id_exceptions(item):
+ return item["latest_id"]
+
+def id_run(item):
+ return item["end_timestamp"]
+
+# Elasticsearch indexes for BODS data
+bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity},
+ "person": {"properties": person_statement_properties, "match": match_person, "id": id_person},
+ "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership},
+ "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest},
+ "references": {"properties": references_properties, "match": match_references, "id": id_references},
+ "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates},
+ "exceptions": {"properties": exceptions_properties, "match": match_exceptions, "id": id_exceptions},
+ "runs": {"properties": pipeline_run_properties, "match": match_run, "id": id_run}}
diff --git a/bodspipelines/infrastructure/inputs.py b/bodspipelines/infrastructure/inputs.py
index 62f4895..d403312 100644
--- a/bodspipelines/infrastructure/inputs.py
+++ b/bodspipelines/infrastructure/inputs.py
@@ -6,8 +6,16 @@ def __init__(self, stream_name=None):
self.stream_name = stream_name
self.stream = KinesisStream(self.stream_name)
- def process(self):
- for records in self.stream.read_stream():
- for record in records:
- if isinstance(record, dict):
- yield record
+ async def process(self):
+ async for record in self.stream.read_stream():
+ #for record in records:
+ # if isinstance(record, dict):
+ yield record
+
+ async def setup(self):
+ if hasattr(self.stream, 'setup'):
+ await self.stream.setup()
+
+ async def close(self):
+ if hasattr(self.stream, 'close'):
+ await self.stream.close()
diff --git a/bodspipelines/infrastructure/memory_debugging.py b/bodspipelines/infrastructure/memory_debugging.py
new file mode 100644
index 0000000..37fa533
--- /dev/null
+++ b/bodspipelines/infrastructure/memory_debugging.py
@@ -0,0 +1,18 @@
+import tracemalloc
+import os, psutil
+from loguru import logger
+
+# Configure log file
+logger.add("memory.log")
+
+# Start tracing
+tracemalloc.start()
+
+def log_memory():
+ """Log current memory usage and largest objects"""
+ snapshot = tracemalloc.take_snapshot()
+ top_stats = snapshot.statistics('lineno')
+
+ logger.info(f"Process size: {psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2} MB")
+ for stat in top_stats[:10]:
+ logger.info(stat)
diff --git a/bodspipelines/infrastructure/outputs.py b/bodspipelines/infrastructure/outputs.py
index a3f4154..fa9e1f8 100644
--- a/bodspipelines/infrastructure/outputs.py
+++ b/bodspipelines/infrastructure/outputs.py
@@ -47,12 +47,25 @@ def process(self, item, item_type):
self.new_count += 1
#print(f"Processed: {self.processed_count}, New: {self.new_count}")
- def process_stream(self, stream, item_type):
+ async def process_stream(self, stream, item_type):
if self.identify: item_type = self.identify
- for item in self.storage.process_batch(stream, item_type):
+ async for item in self.storage.process_batch(stream, item_type):
if item:
- self.output.process(item, item_type)
- self.output.finish()
+ await self.output.process(item, item_type)
+ await self.output.finish()
+
+ async def setup(self):
+ if hasattr(self.storage, 'setup'):
+ await self.storage.setup()
+ if hasattr(self.output, 'setup'):
+ await self.output.setup()
+
+ async def close(self):
+ if hasattr(self.storage, 'close'):
+ await self.storage.close()
+ if hasattr(self.output, 'close'):
+ await self.output.close()
+
class KinesisOutput:
"""Output to Kinesis Stream"""
@@ -61,8 +74,16 @@ def __init__(self, stream_name=None):
self.stream_name = stream_name
self.stream = KinesisStream(self.stream_name)
- def process(self, item, item_type):
- self.stream.add_record(item)
+ async def process(self, item, item_type):
+ await self.stream.add_record(item)
+
+ async def finish(self):
+ await self.stream.finish_write()
+
+ async def setup(self):
+ if hasattr(self.stream, 'setup'):
+ await self.stream.setup()
- def finish(self):
- self.stream.finish_write()
+ async def close(self):
+ if hasattr(self.stream, 'close'):
+ await self.stream.close()
diff --git a/bodspipelines/infrastructure/pipeline.py b/bodspipelines/infrastructure/pipeline.py
index 28d3b4f..60811c9 100644
--- a/bodspipelines/infrastructure/pipeline.py
+++ b/bodspipelines/infrastructure/pipeline.py
@@ -1,10 +1,14 @@
+import time
+import asyncio
from typing import List, Union
from pathlib import Path
-from bodspipelines.infrastructure.processing.bulk_data import BulkData
-from bodspipelines.infrastructure.processing.xml_data import XMLData
+#from bodspipelines.infrastructure.processing.bulk_data import BulkData
+#from bodspipelines.infrastructure.processing.xml_data import XMLData
-from bodspipelines.infrastructure.storage import ElasticStorage
+#from bodspipelines.infrastructure.storage import ElasticStorage
+
+#from .memory_debugging import log_memory
class Source:
"""Data source definition class"""
@@ -14,15 +18,27 @@ def __init__(self, name=None, origin=None, datatype=None):
self.origin = origin
self.datatype = datatype
- def process(self, stage_dir):
+ async def process(self, stage_dir, updates=False):
"""Iterate over source items"""
if hasattr(self.origin, "prepare"):
- data = self.origin.prepare(stage_dir, self.name)
- for item in self.datatype.process(data):
- yield item
+ for data in self.origin.prepare(stage_dir, self.name, updates=updates):
+ async for header, item in self.datatype.process(data):
+ yield header, item
else:
- for item in self.origin.process():
- yield self.datatype.process(item)
+ async for item in self.origin.process():
+ header, item = self.datatype.process(item)
+ yield header, item
+
+ async def setup(self):
+ """Run origin setup"""
+ if hasattr(self.origin, 'setup'):
+ await self.origin.setup()
+
+ async def close(self):
+ """Close origin"""
+ if hasattr(self.origin, 'close'):
+ await self.origin.close()
+
class Stage:
"""Pipeline stage definition class"""
@@ -32,7 +48,7 @@ def __init__(self, name=None, sources=None, processors=None, outputs=None):
self.name = name
self.sources = sources
self.processors = processors
- self.outputs = outputs
+ self.outputs = outputs
def directory(self, parent_dir) -> Path:
"""Return subdirectory path after ensuring exists"""
@@ -40,41 +56,74 @@ def directory(self, parent_dir) -> Path:
path.mkdir(exist_ok=True)
return path
- def source_processing(self, source, stage_dir):
+ async def source_processing(self, source, stage_dir, updates=False):
"""Iterate over items from source, with processing"""
- for item in source.process(stage_dir):
+ #count = 0
+ async for header, item in source.process(stage_dir, updates=updates):
+ #print(header, item)
if self.processors:
+ items = [item]
for processor in self.processors:
- for out in processor.process(item, source.name):
- #print(out)
- yield out
+ new_items = []
+ for current_item in items:
+ #print("Processor:", processor)
+ async for out in processor.process(current_item, source.name, header, updates=updates):
+ #print(out)
+ #yield out
+ new_items.append(out)
+ items = new_items
+ for current_item in items:
+ yield current_item
else:
yield item
+ #count += 1
+ #if count % 100000 == 0:
+ # log_memory()
+ yield {"flush": True}
+ await asyncio.sleep(5)
+ for processor in self.processors:
+ print("Processor:", hasattr(processor, "finish_updates"), updates)
+ if hasattr(processor, "finish_updates") and updates:
+ async for out in processor.finish_updates(updates=updates):
+ yield out
- #def process_source(self, source, stage_dir):
- # """Iterate over items from source, with processing and output"""
- # for item in source.process(stage_dir):
- # for processor in self.processors:
- # item = processor.process(item, source.name)
-
- def process_source(self, source, stage_dir):
+ async def process_source(self, source, stage_dir, updates=False):
"""Iterate over items from source, and output"""
+ print("Process source:", len(self.outputs) > 1, not self.outputs[0].streaming)
if len(self.outputs) > 1 or not self.outputs[0].streaming:
- for item in self.source_processing(source, stage_dir):
+ print("Interating:")
+ async for item in self.source_processing(source, stage_dir, updates=updates):
for output in self.outputs:
output.process(item, source.name)
else:
- self.outputs[0].process_stream(self.source_processing(source, stage_dir), source.name)
+ print("Streaming:")
+ await self.outputs[0].process_stream(self.source_processing(source, stage_dir, updates=updates),
+ source.name)
- def process(self, pipeline_dir):
+ async def process(self, pipeline_dir, updates=False):
"""Process all sources for stage"""
print(f"Running {self.name} pipeline stage")
stage_dir = self.directory(pipeline_dir)
for source in self.sources:
print(f"Processing {source.name} source")
- self.process_source(source, stage_dir)
+ await self.process_source(source, stage_dir, updates=updates)
print(f"Finished {self.name} pipeline stage")
+ async def setup(self):
+ """Setup stage components"""
+ for components in (self.sources, self.processors, self.outputs):
+ for component in components:
+ if hasattr(component, 'setup'):
+ await component.setup()
+
+ async def close(self):
+ """Shutdown stage components"""
+ for components in (self.sources, self.processors, self.outputs):
+ for component in components:
+ if hasattr(component, 'close'):
+ await component.close()
+
+
class Pipeline:
"""Pipeline definition class"""
def __init__(self, name=None, stages=None):
@@ -95,8 +144,16 @@ def get_stage(self, name):
return stage
return None
- def process(self, stage_name):
+ async def process_stage(self, stage_name, updates=False):
"""Process specified pipeline stage"""
stage = self.get_stage(stage_name)
pipeline_dir = self.directory()
- stage.process(pipeline_dir)
+ await stage.setup()
+ await stage.process(pipeline_dir, updates=updates)
+ await stage.close()
+
+ def process(self, stage_name, updates=False):
+ """Process specified pipeline stage"""
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(self.process_stage(stage_name, updates=updates))
diff --git a/bodspipelines/infrastructure/processing/bulk_data.py b/bodspipelines/infrastructure/processing/bulk_data.py
index 2309ff9..7f03be2 100644
--- a/bodspipelines/infrastructure/processing/bulk_data.py
+++ b/bodspipelines/infrastructure/processing/bulk_data.py
@@ -8,10 +8,10 @@
class BulkData:
"""Bulk data definition class"""
- def __init__(self, display=None, url=None, size=None, directory=None):
+ def __init__(self, display=None, data=None, size=None, directory=None):
"""Initial setup"""
self.display = display
- self.url = url
+ self.data = data
self.size = size
self.directory = directory
@@ -26,39 +26,73 @@ def manifest_file(self, path) -> Path:
def create_manifest(self, path, name):
"""Create manifest file"""
manifest_file = self.manifest_file(path)
- if callable(self.url):
- url = self.url(name)
- else:
- url = self.url
+ manifest = []
with open(manifest_file, "w") as outfile:
- json.dump({"url": url, "timestamp": time.time()}, outfile)
+ for data in self.data.sources():
+ if callable(data):
+ url = data(name)
+ else:
+ url = data
+ manifest.append({"url": url, "timestamp": time.time()})
+ json.dump(manifest, outfile)
- def check_manifest(self, path, name):
- """Check manifest file exists and up-to-date"""
+ def read_manifest(self, path):
+ """Read manifest file if exists"""
manifest_file = self.manifest_file(path)
if manifest_file.exists():
with open(manifest_file, 'r') as openfile:
try:
- manifest = json.load(openfile)
+ return json.load(openfile)
except json.decoder.JSONDecodeError:
return False
- if callable(self.url):
- url = self.url(name)
+ else:
+ return None
+
+ def source_data(self, name, last_update=None, delta_type=False):
+ """Yield urls for source"""
+ for data in self.data.sources(last_update=last_update, delta_type=delta_type):
+ if callable(data):
+ url = data(name)
else:
- url = self.url
- if manifest["url"] == url and abs(manifest["timestamp"] - time.time()) < 24*60*60:
- return True
+ url = data
+ yield url
+
+ def check_manifest(self, path, name, updates=False):
+ """Check manifest file exists and up-to-date"""
+ #manifest_file = self.manifest_file(path)
+ manifest = self.read_manifest(path)
+ if updates and manifest:
+ d, t, *_ = manifest['url'].split('/')[-1].split('-')
+ last_update = f"{d[:4]}-{d[4:6]}-{d[6:8]}"
+ else:
+ last_update = False
+ if last_update:
+ if updates in ("month", "week", "day"):
+ # Special case for testing (usually True/False)
+ delta_type = updates
else:
- return False
+ delta_type = None
+ yield from self.source_data(name, last_update=last_update, delta_type=delta_type)
else:
- return False
+ if manifest: #manifest_file.exists():
+ #with open(manifest_file, 'r') as openfile:
+ # try:
+ # manifest = json.load(openfile)
+ # except json.decoder.JSONDecodeError:
+ # return False
+ for data in self.data.sources():
+ if callable(data):
+ url = data(name)
+ else:
+ url = data
+ match = [m for m in manifest if m["url"] == url]
+ if not match or abs(m["timestamp"] - time.time()) > 24*60*60:
+ yield url
+ else:
+ yield from self.source_data(name, last_update=last_update)
- def download_large(self, directory, name):
+ def download_large(self, directory, name, url):
"""Download file to specified directory"""
- if callable(self.url):
- url = self.url(name)
- else:
- url = self.url
with requests.get(url, stream=True) as r:
r.raise_for_status()
if 'content-disposition' in r.headers:
@@ -78,28 +112,83 @@ def download_large(self, directory, name):
def unzip_data(self, filename, directory):
"""Unzip specified file to directory"""
with zipfile.ZipFile(filename, 'r') as zip_ref:
- zip_ref.extractall(directory)
+ for fn in zip_ref.namelist():
+ zip_ref.extract(fn, path=directory)
+ yield fn
- def delete_old_data(self, directory):
+ def delete_old_data_all(self, directory):
+ """Delete all data files"""
for file in directory.glob('*'):
print(f"Deleting {file.name} ...")
file.unlink()
+ def delete_old_data(self, directory, url):
+ """Delete filename for specified url"""
+ fn = url.rsplit('/', 1)[-1]
+ for file in directory.glob('*'):
+ print(file.name, fn)
+ if file.name == fn:
+ print(f"Deleting {file.name} ...")
+ file.unlink()
+
+ def delete_unused_data(self, directory, files):
+ """Delete files not in list"""
+ for file in directory.glob('*'):
+ if not file.name in files:
+ print(f"Deleting {file.name} ...")
+ file.unlink()
+
+ def delete_zip_data(self, directory, url):
+ """Delete filename for specified url"""
+ fn = url.rsplit('/', 1)[-1]
+ for file in directory.glob('*'):
+ if file.name == fn:
+ print(f"Deleting {file.name} ...")
+ file.unlink()
+
+ def download_data(self, directory, name):
+ """Download data files"""
+ for data in self.data.sources():
+ if callable(data):
+ url = data(name)
+ else:
+ url = data
+ zip = self.download_large(directory, name, url)
+ for fn in self.unzip_data(zip, directory):
+ yield fn
+
def download_extract_data(self, path, name):
"""Download and extract data"""
- directory = self.data_dir(path)
- directory.mkdir(exist_ok=True)
+ #directory = self.data_dir(path)
+ #directory.mkdir(exist_ok=True)
self.delete_old_data(directory)
- zip = self.download_large(directory, name)
- self.unzip_data(zip, directory)
+ #zip = self.download_large(directory, name)
+ #self.unzip_data(zip, directory)
+ for fn in self.download_data(directory, name):
+ yield fn
+
+ def download_extract_data(self, directory, name, url):
+ """Download and unzip data files"""
+ self.delete_old_data(directory, url)
+ zip = self.download_large(directory, name, url)
+ for fn in self.unzip_data(zip, directory):
+ self.delete_zip_data(directory, url)
+ yield fn
- def prepare(self, path, name) -> Path:
+ def prepare(self, path, name, updates=False) -> Path:
"""Prepare data for use"""
- if not self.check_manifest(path, name):
- self.download_extract_data(path, name)
- self.create_manifest(path, name)
+ directory = self.data_dir(path)
+ directory.mkdir(exist_ok=True)
+ files = []
+ if list(directory.glob("*.xml")): # and not list(directory.glob("*golden-copy.xml")):
+ for f in directory.glob("*.xml"):
+ fn = f.name
+ files.append(fn)
+ yield directory / fn
else:
- print(f"{self.display} data up-to-date ...")
- for file in self.data_dir(path).glob('*.xml'):
- return file
-
+ for url in self.check_manifest(path, name, updates=updates):
+ for fn in self.download_extract_data(directory, name, url):
+ files.append(fn)
+ yield directory / fn
+ print("Files:", files)
+ self.create_manifest(path, name)
diff --git a/bodspipelines/infrastructure/processing/json_data.py b/bodspipelines/infrastructure/processing/json_data.py
index cff85aa..bd587d3 100644
--- a/bodspipelines/infrastructure/processing/json_data.py
+++ b/bodspipelines/infrastructure/processing/json_data.py
@@ -4,4 +4,4 @@ class JSONData:
def process(self, item):
"""Return item"""
#print(item)
- return item
+ return None, item
diff --git a/bodspipelines/infrastructure/processing/xml_data.py b/bodspipelines/infrastructure/processing/xml_data.py
index 23db3a6..37ef168 100644
--- a/bodspipelines/infrastructure/processing/xml_data.py
+++ b/bodspipelines/infrastructure/processing/xml_data.py
@@ -1,6 +1,21 @@
-from lxml import etree
+import aiofiles
+#from lxml import etree
+import xml.etree.ElementTree as etree
+
+
+async def stream_file(filename):
+ async with aiofiles.open(filename, mode='r') as f:
+ while True:
+ data = await f.read(65536)
+ if data:
+ yield data
+ else:
+ yield ""
+ break
+
def is_plural(tag, child_tag):
+ """Is tag name plural"""
if tag == child_tag + "s":
return True
elif tag == child_tag + "es":
@@ -9,69 +24,118 @@ def is_plural(tag, child_tag):
return True
return False
+
+def is_plural(tag, child_tag):
+ """Is tag name plural"""
+ if tag.endswith(child_tag + "s"):
+ return True
+ elif tag.endswith(child_tag + "es"):
+ return True
+ elif tag.endswith(child_tag[:-1] + 'ies'):
+ return True
+ return False
+
+
+def get_tag(element, pos):
+ """Return tag name without namespace"""
+ for ns in pos:
+ if ns in element.tag:
+ return element.tag[pos[ns]:]
+
+
+def handle_event(event, element, tag_name, skip, stack, pos, filter):
+ out = None
+ tag = get_tag(element, pos)
+ if event == 'start':
+ if skip:
+ pass
+ elif tag in filter:
+ skip = True
+ elif element.tag == tag_name or stack:
+ stack.append([element.tag, {}])
+ elif event == 'end':
+ if skip:
+ if tag in filter:
+ skip = False
+ elif element.tag == tag_name:
+ element.clear()
+ # Also eliminate now-empty references from the root node to elem
+ #for ancestor in element.xpath('ancestor-or-self::*'):
+ # while ancestor.getprevious() is not None:
+ # del ancestor.getparent()[0]
+ elem = stack.pop()
+ out = elem[1]
+ elif stack:
+ elem = stack.pop()
+ if elem[1]:
+ val = elem[1]
+ else:
+ val = element.text
+ if stack[-1][1]:
+ if isinstance(stack[-1][1], list):
+ if 'type' in element.attrib:
+ if isinstance(val, dict):
+ val['type'] = element.attrib['type']
+ stack[-1][1].append(val)
+ else:
+ stack[-1][1].append({'type': element.attrib['type'], tag: val})
+ else:
+ stack[-1][1].append(val)
+ else:
+ stack[-1][1][tag] = val
+ else:
+ if is_plural(stack[-1][0], tag):
+ if 'type' in element.attrib:
+ if isinstance(val, dict):
+ val['type'] = element.attrib['type']
+ stack[-1][1] = [val]
+ else:
+ stack[-1][1] = [{'type': element.attrib['type'], tag: val}]
+ else:
+ if isinstance(stack[-1][1], list):
+ stack[-1][1].append(val)
+ else:
+ stack[-1][1] = [val]
+ else:
+ stack[-1][1][tag] = val
+ return out, skip
+
+
+async def data_stream(filename, tag_name, namespaces, filter=[]):
+ """Stream items from XML file"""
+ skip = False
+ stack = []
+ pos = {namespaces[ns]: len(namespaces[ns])+2 for ns in namespaces}
+ parser = etree.XMLPullParser(('start', 'end',))
+ async for chunk in stream_file(filename):
+ parser.feed(chunk)
+ for event, element in parser.read_events():
+ out, skip = handle_event(event, element, tag_name, skip, stack, pos, filter)
+ if out: yield out
+
+
class XMLData:
- """XML data definition class"""
+ """XML data parser configuration"""
- def __init__(self, item_tag=None, namespace=None, filter=[]):
+ def __init__(self, item_tag=None, header_tag=None, namespace=None, filter=[]):
"""Initial setup"""
self.item_tag = item_tag
+ self.header_tag = header_tag
self.namespace = namespace
self.filter = filter
- def data_stream(self, filename):
- """Stream parsed XML elements from file"""
- print(f"Parsing {filename}")
- ns = self.namespace[next(iter(self.namespace))]
- tag_name = f"{{{ns}}}{self.item_tag}"
- for event, element in etree.iterparse(filename, events=('end',), tag=tag_name):
- yield element
-
- def is_array(self, tag, child):
- """Check if is array """
- child_tag = etree.QName(child[0]).localname
- if is_plural(tag, child_tag):
- #print("Array!!!!")
- return True
+ async def extract_header(self, filename):
+ """Extract header"""
+ if self.header_tag:
+ tag_name = f"{{{self.namespace[next(iter(self.namespace))]}}}{self.header_tag}"
+ async for item in data_stream(filename, tag_name, self.namespace, filter=self.filter):
+ return item
else:
- return False
-
- def add_element(self, out, tag, value):
- if not tag in self.filter and value:
- out[tag] = value
-
- def process_item(self, item, out):
- """Process XML item to dict"""
- for child in item:
- tag = etree.QName(child).localname
- #print(tag, len(child))
- if len(child) > 0:
- if self.is_array(tag, child):
- parent_data = []
- else:
- parent_data = {}
- child_data = self.process_item(child, parent_data)
- if isinstance(out, list):
- out.append(child_data)
- else:
- self.add_element(out, tag, child_data)
- #out[tag] = child_data
- else:
- try:
- child_value = child.xpath("./text()", namespaces=self.namespace)[0]
- except IndexError:
- child_value = False
- if isinstance(out, list):
- out.append(child_value)
- else:
- self.add_element(out, tag, child_value)
- #out[tag] = child_value
- return out
+ return None
- def process(self, filename):
+ async def process(self, filename):
"""Iterate over processed items from file"""
- for element in self.data_stream(filename):
- item = self.process_item(element, {})
- element.clear()
- #print(item)
- yield item
- #break # Remove
+ header = await self.extract_header(filename)
+ tag_name = f"{{{self.namespace[next(iter(self.namespace))]}}}{self.item_tag}"
+ async for item in data_stream(filename, tag_name, self.namespace, filter=self.filter):
+ yield header, item
diff --git a/bodspipelines/infrastructure/storage.py b/bodspipelines/infrastructure/storage.py
index 6538d52..46a7786 100644
--- a/bodspipelines/infrastructure/storage.py
+++ b/bodspipelines/infrastructure/storage.py
@@ -1,116 +1,145 @@
from typing import List, Union, Optional
from dataclasses import dataclass
-from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+class Storage:
+ """Storage definition class"""
-class ElasticStorage:
- """Elasticsearch storage definition class"""
- def __init__(self, indexes):
- self.indexes = indexes
- self.storage = ElasticsearchClient()
- self.current_index = None
+ def __init__(self, storage):
+ """Initialise storage"""
+ self.storage = storage
- def setup_indexes(self):
- for index_name in self.indexes:
- self.storage.create_index(index_name, self.indexes[index_name]['properties'])
-
- def create_action(self, index_name, item):
- #print(index_name, item)
- if callable(index_name): index_name = index_name(item)
- #return {"create": { "_index" : index_name, "_id" : self.indexes[index_name]["id"](item)}}
- return {"_id": self.indexes[index_name]["id"](item), '_index': index_name, '_op_type': 'create', "_source": item}
-
- def action_stream(self, stream, index_name):
- for item in stream:
- yield self.create_action(index_name, item)
-
- def batch_stream(self, stream, index_name):
- for item in stream:
- yield self.create_action(index_name, item), item
-
- def create_batch(self, batch):
- def func():
- for i in batch:
- yield i
- return func(), batch
-
- def batch_stream(self, stream, index_name):
- batch = []
- for item in stream:
- batch.append(self.create_action(index_name, item))
- if len(batch) > 485:
- yield self.create_batch(batch)
- batch = []
- if len(batch) > 0:
- yield self.create_batch(batch)
+ async def setup(self):
+ """Setup storage"""
+ await self.storage.setup()
def list_indexes(self):
+ """List indexes"""
return self.storage.list_indexes()
def list_index_details(self, index_name):
+ """List details for specified index"""
return self.storage.get_mapping(index_name)
- def stats(self):
- for index_name in self.indexes:
- print(f"Index {index_name}:", self.storage.stats(index_name))
+ async def statistics(self):
+ """Print storage statistics"""
+ print("Storage:")
+ print("")
+ statistics = await self.storage.statistics()
+ for index_name in statistics:
+ if index_name != "total":
+ print(f"{index_name} items:", statistics[index_name])
+ print("")
+ print("Total items:", statistics["total"])
def set_index(self, index_name):
+ """Set current index"""
self.current_index = index_name
self.storage.set_index(index_name)
def delete_index(self, index_name):
+ """Delete index"""
self.current_index = index_name
self.storage.set_index(index_name)
self.storage.delete_index()
def delete_all(self, index_name):
+ """Delete index"""
self.current_index = index_name
self.storage.set_index(index_name)
self.storage.delete_index()
self.storage.create_index(index_name, self.indexes[index_name]['properties'])
- def add_item(self, item, item_type):
- #print(item_type, self.indexes[item_type])
- query = self.indexes[item_type]['match'](item)
- #print("Query:", query)
- match = self.storage.search(query)
- #print(match)
- if not match['hits']['hits']:
- out = self.storage.store_data(item)
- #print(out)
- return item
+ def create_action(self, index_name, item, action_type='create'):
+ """Build create action for item"""
+ if callable(index_name): index_name = index_name(item)
+ if action_type == 'delete':
+ return {"_id": self.storage.indexes[index_name]["id"](item),
+ '_index': index_name,
+ '_op_type': action_type}
+ else:
+ return {"_id": self.storage.indexes[index_name]["id"](item),
+ '_index': index_name,
+ '_op_type': action_type,
+ "_source": item}
+
+ async def get_item(self, id, item_type):
+ """Get item from index"""
+ self.storage.set_index(item_type)
+ return await self.storage.get(id)
+
+ async def add_item(self, item, item_type, overwrite=False):
+ """Add item to index"""
+ self.storage.set_index(item_type)
+ id = self.storage.indexes[item_type]['id'](item)
+ result = await self.storage.get(id)
+ #print(result)
+ if overwrite or not result:
+ if overwrite and result:
+ #print(f"Updating: {item}")
+ out = await self.storage.update_data(item, id)
+ return item
+ else:
+ #print(f"Creating: {item}")
+ #action = self.create_action(item_type, item)
+ out = await self.storage.store_data(item, id=id)
+ return item
else:
return False
- def process(self, item, item_type):
- if item_type != self.current_index:
- self.set_index(item_type)
- return self.add_item(item, item_type)
-
- def process_stream(self, stream, item_type):
- for item in self.storage.bulk_store_data(self.action_stream(stream, item_type), item_type):
- yield item
+ async def delete_item(self, id, item_type):
+ """Delete item with id in index"""
+ self.storage.set_index(item_type)
+ await self.storage.delete(id)
+
+ async def stream_items(self, index):
+ """Stream items in index"""
+ async for item in self.storage.scan_index(index):
+ yield item['_source']
+
+ async def process(self, item, item_type):
+ """Add item to index"""
+ if item_type != self.storage.index_name:
+ self.storage.set_index(item_type)
+ return await self.add_item(item, item_type)
+
+ async def create_batch(self, batch):
+ """Create iterator that yields batch"""
+ async def func():
+ for i in batch:
+ yield i
+ return func(), batch
- def process_batch(self, stream, item_type):
+ async def batch_stream(self, stream, index_name):
+ """Create stream of batched actions"""
batch = []
- for action, item in self.batch_stream(stream, item_type):
- batch.append(action)
- batch.append(item)
- if len(batch) > 499:
- for item in self.storage.batch_store_data(batch, item_type):
- yield item
- batch = []
-
- def process_batch(self, stream, item_type):
- for actions, items in self.batch_stream(stream, item_type):
- for item in self.storage.batch_store_data(actions, items, item_type):
+ async for item in stream:
+ if "flush" in item and item["flush"] is True:
+ if len(batch) > 0:
+ yield await self.create_batch(batch)
+ else:
+ batch.append(self.create_action(index_name, item))
+ if len(batch) > 485:
+ yield await self.create_batch(batch)
+ batch = []
+ if len(batch) > 0:
+ yield await self.create_batch(batch)
+
+ async def process_batch(self, stream, item_type):
+ """Store items from stream in batches"""
+ async for actions, items in self.batch_stream(stream, item_type):
+ async for item in self.storage.batch_store_data(actions, items, item_type):
yield item
- #top_mem()
- def query(self, index_name, query):
- self.storage.set_index(index_name)
- return self.storage.search(query)
+ async def setup_indexes(self):
+ """Setup indexes"""
+ await self.storage.setup_indexes()
- def get_all(self, index_name):
- self.storage.set_index(index_name)
- return self.storage.search({'match_all': {}})
+ async def add_batch(self, item_type, items):
+ #print(f"Write batch of {len(items)} item for {item_type}")
+ actions = [self.create_action(item_type, current_item[1], action_type=current_item[0])
+ for current_item in items]
+ async for current_item in self.storage.batch_store_data(actions, actions, item_type):
+ pass
+
+ async def dump_stream(self, index_name, action_type, items):
+ await self.storage.dump_stream(index_name, action_type, items)
diff --git a/bodspipelines/infrastructure/updates.py b/bodspipelines/infrastructure/updates.py
new file mode 100644
index 0000000..2781e50
--- /dev/null
+++ b/bodspipelines/infrastructure/updates.py
@@ -0,0 +1,452 @@
+from bodspipelines.pipelines.gleif.indexes import id_rr as rr_id
+from bodspipelines.infrastructure.utils import (current_date_iso, generate_statement_id,
+ random_string, format_date)
+from bodspipelines.infrastructure.caching import Caching
+
+def convert_rel_type(rel_type):
+ """Convert Relationship Type To Exception Type"""
+ conv = {"IS_DIRECTLY_CONSOLIDATED_BY": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "IS_ULTIMATELY_CONSOLIDATED_BY": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT"}
+ return conv[rel_type]
+
+
+def convert_except_type(except_type):
+ """Convert Exception Type To Relationship Type"""
+ conv = {"DIRECT_ACCOUNTING_CONSOLIDATION_PARENT": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT": "IS_ULTIMATELY_CONSOLIDATED_BY"}
+ return conv[except_type]
+
+
+def build_latest(lei, bods_id, reason=False):
+ """Build latest object"""
+ return {'latest_id': lei, # Source id (e.g. LEI)
+ 'statement_id': bods_id, # Latest statement id
+ 'reason': reason} # Unused
+
+
+def build_references(statement_id, referencing_ids):
+ """Build references object"""
+ return {'statement_id': statement_id, # Statement id
+ 'references_id': referencing_ids} # Referencing statement ids
+
+
+def build_update(referencing_id, latest_id, updates):
+ """Build updates object"""
+ return {'referencing_id': referencing_id,
+ 'latest_id': latest_id,
+ 'updates': [{'old_statement_id': old, 'new_statement_id': updates[old]} for old in updates]}
+
+
+def build_exception(latest, occ_id, other_id, reason, reference, entity_type):
+ """Build exception object"""
+ return {'latest_id': latest, 'statement_id': occ_id, 'other_id': other_id, 'reason': reason,
+ 'reference': reference, 'entity_type': entity_type}
+
+
+async def latest_save(cache, lei, bods_id, reason=False, updates=False):
+ """Save latest statement id for LEI/RR/Repex"""
+ await cache.add(build_latest(lei, bods_id, reason=reason),
+ "latest", overwrite=True)
+
+async def latest_lookup(cache, lei, updates=False):
+ """Lookup latest statement id for LEI/RR/Repex"""
+ data = await cache.get(lei, "latest")
+ if data:
+ return data['statement_id'], data['reason']
+ else:
+ return None, None
+
+
+async def latest_delete(cache, old_statement_id, updates=False):
+ """Delete latest statement id for LEI/RR/Repex"""
+ await cache.delete(old_statement_id, "updates")
+
+
+async def exception_save(cache, latest, ooc_id, other_id, reason, reference, entity_type, updates=False):
+ """Save latest exception"""
+ await cache.add(build_exception(latest, ooc_id, other_id, reason, reference, entity_type),
+ "exceptions",
+ overwrite=True)
+
+async def exception_lookup(cache, latest, updates=False):
+ """Lookup latest exception"""
+ data = await cache.get(latest, "exceptions")
+ if data:
+ return data['statement_id'], data['other_id'], data['reason'], data['reference'], data['entity_type']
+ else:
+ return None, None, None, None, None
+
+async def exception_delete(cache, old_statement_id, updates=False):
+ """Delete exception id"""
+ await cache.delete(old_statement_id, "exceptions")
+
+def build_referencing(referencing_ids):
+ """Build referencing object"""
+ out = []
+ for statement_id in referencing_ids:
+ out.append({'statement_id': statement_id, 'latest_id': referencing_ids[statement_id]})
+ return out
+
+async def references_save(cache, statement_id, referencing_ids, updates=False, overwrite=False):
+ """Save list of statement ids referencing statement"""
+ await cache.add(build_references(statement_id, build_referencing(referencing_ids)),
+ "references",
+ overwrite=overwrite)
+
+def translate_references(references):
+ """Translate references"""
+ out = {}
+ if isinstance(references, list):
+ for ref in references:
+ out[ref['statement_id']] = ref['latest_id']
+ else:
+ out[references['statement_id']] = references['latest_id']
+ return out
+
+async def lookup_references(cache, statement_id, updates=False):
+ """Lookup list of statement ids referencing statement"""
+ data = await cache.get(statement_id, "references")
+ if data:
+ return translate_references(data['references_id'])
+ else:
+ return {}
+
+async def references_update(cache, referenced_id, statement_id, latest_id, updates=False):
+ """Update list of statement ids referencing statement"""
+ referencing_ids = await lookup_references(cache, referenced_id, updates=updates)
+ referencing_ids[statement_id] = latest_id
+ await references_save(cache, referenced_id, referencing_ids, updates=updates, overwrite=True)
+
+async def references_remove(cache, referenced_id, statement_id, updates=False):
+ """Update list of statement ids referencing statement"""
+ referencing_ids = await lookup_references(cache, referenced_id, updates=updates)
+ if statement_id in referencing_ids: del referencing_ids[statement_id]
+ await references_save(cache, referenced_id, referencing_ids, updates=updates, overwrite=True)
+
+async def updates_save(cache, referencing_id, latest_id, updates):
+ """Save statement to updates"""
+ await cache.add(build_update(referencing_id, latest_id, updates), "updates", overwrite=True)
+
+async def updates_delete(cache, old_statement_id, if_exists=False):
+ """Delete statement to updates"""
+ await cache.delete(old_statement_id, "updates", if_exists=if_exists)
+
+async def lookup_updates(cache, old_statement_id):
+ """Lookup statement to updates"""
+ data = await cache.get(old_statement_id, "updates")
+ if data:
+ return {update['old_statement_id']: update['new_statement_id'] for update in data['updates']}
+ else:
+ return {}
+
+async def updates_update(cache, referencing_id, latest_id, old_statement_id, new_statement_id):
+ """Save statement to update"""
+ updates = await lookup_updates(cache, referencing_id)
+ updates[old_statement_id] = new_statement_id
+ await updates_save(cache, referencing_id, latest_id, updates)
+
+async def process_updates(cache):
+ """Stream updates from index"""
+ async for update in cache.stream("updates"):
+ updates = {data['old_statement_id']: data['new_statement_id'] for data in update['updates']}
+ yield update['referencing_id'], update['latest_id'], updates
+
+async def retrieve_statement(storage, statement_type, statement_id):
+ """Retrive statement using statement_id"""
+ data = await storage.get_item(statement_id, statement_type)
+ return data
+
+def fix_statement_reference(statement, updates, latest_id):
+ """Update ownershipOrControlStatement with new_id"""
+ for old_id in updates:
+ new_id = updates[old_id]
+ if "subject" in statement and "describedByEntityStatement" in statement["subject"]:
+ if statement["subject"]["describedByEntityStatement"] == old_id:
+ statement["subject"]["describedByEntityStatement"] = new_id
+ if "interestedParty" in statement and "describedByEntityStatement" in statement["interestedParty"]:
+ if statement["interestedParty"]["describedByEntityStatement"] == old_id:
+ statement["interestedParty"]["describedByEntityStatement"] = new_id
+ old_statement_id = statement['statementID']
+ subject = statement["subject"]["describedByEntityStatement"]
+ interested = statement["interestedParty"]["describedByEntityStatement"]
+ statement_id = generate_statement_id(f"{latest_id}_{subject}_{interested}", 'ownershipOrControlStatement')
+ statement['statementID'] = statement_id
+ return old_statement_id
+
+def source_id(data, id_name):
+ """Extract identifier for specified type"""
+ for id in data['identifiers']:
+ if id['scheme'] == id_name:
+ return id['id']
+ return None
+
+def referenced_ids(statement, item=None):
+ """Collect statement ids referenced by statement"""
+ out = []
+ if "subject" in statement and "describedByEntityStatement" in statement["subject"]:
+ out.append(statement["subject"]["describedByEntityStatement"])
+ if ("interestedParty" in statement and "describedByEntityStatement" in statement["interestedParty"] and
+ (not item or item["Registration"]["RegistrationStatus"] == "PUBLISHED")):
+ out.append(statement["interestedParty"]["describedByEntityStatement"])
+ return out
+
+async def calculate_mapping(cache, item, updates=False):
+ """Calculate mapping lei and latest statementID"""
+ out = {}
+ if "Relationship" in item:
+ for lei in (item["Relationship"]["StartNode"]["NodeID"], item["Relationship"]["EndNode"]["NodeID"]):
+ latest_id, _ = await latest_lookup(cache, lei, updates=updates)
+ if latest_id: out[lei] = latest_id
+ else:
+ latest_id, _ = await latest_lookup(cache, item["LEI"], updates=updates)
+ if latest_id: out[item["LEI"]] = latest_id
+ return out
+
+async def item_setup(cache, item, updates=False):
+ """Setup exception and relationship mapping"""
+ if "ExceptionCategory" in item:
+ except_lei = item["LEI"]
+ except_type = item["ExceptionCategory"]
+ except_reason = item["ExceptionReason"]
+ except_reference = item["ExceptionReference"] if "ExceptionReference" in item else None
+ old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type = \
+ await exception_lookup(cache, f"{except_lei}_{except_type}", updates=updates)
+ else:
+ old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, except_type, \
+ except_reason, except_reference = None, None, None, None, None, None, None, None, None
+ if "Relationship" in item or "ExceptionCategory" in item:
+ mapping = await calculate_mapping(cache, item, updates=updates)
+ else:
+ mapping = {}
+ return mapping, old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, \
+ except_type, except_reason, except_reference
+
+async def process_entity_lei(statement_id, statement, item, lei, updates, mapping, data_type, cache):
+ """Process entity statement from LEI"""
+ if updates:
+ latest_id, _ = await latest_lookup(cache, lei, updates=updates) # Latest statement
+ if latest_id:
+ if item["Registration"]["RegistrationStatus"] == 'RETIRED':
+ update_date = item['Registration']['LastUpdateDate']
+ statement = data_type.void_entity_retired(latest_id,
+ update_date,
+ item["LEI"],
+ item["Registration"]["RegistrationStatus"])
+ statement_id = statement['statementID'] if statement else None
+ else:
+ data_type.add_replaces(statement, latest_id) # Add replaces statement
+ referencing_ids = await lookup_references(cache, latest_id, updates=updates)
+ else:
+ referencing_ids = []
+ for ref_id in referencing_ids:
+ await updates_update(cache,
+ ref_id,
+ referencing_ids[ref_id],
+ latest_id,
+ statement_id) # Save statements to update
+ if statement_id:
+ await latest_save(cache, lei, statement_id, updates=updates) # Save new latest
+ return statement_id, statement
+
+async def process_entity_repex(statement_id, statement, item, except_lei, except_type, except_reason,
+ old_reason, old_other_id, old_entity_type, data_type, mapping,
+ cache, updates=False):
+ """Process entity statement from reporting exception"""
+ void_statement = None
+ if "Extension" in item and "Deletion" in item["Extension"]:
+ latest_id, _ = await latest_lookup(cache, f"{except_lei}_{except_type}_{except_reason}_entity", updates=updates)
+ if latest_id:
+ statement = data_type.void_entity_deletion(latest_id,
+ item["Extension"]["Deletion"]["DeletedAt"],
+ item["LEI"],
+ item["ExceptionReason"])
+ statement_id = statement['statementID'] if statement else None
+ else:
+ statement, statement_id = None, None
+ elif (old_reason and except_reason != old_reason):
+ update_date = current_date_iso()
+ void_statement = data_type.void_entity_changed(old_other_id,
+ update_date,
+ old_entity_type,
+ except_lei,
+ old_reason)
+ if statement_id:
+ await latest_save(cache, f"{except_lei}_{except_type}_{except_reason}_entity", statement_id, updates=updates)
+ return statement_id, statement, void_statement
+
+async def process_ooc_rr(statement_id, statement, item, start, end, rel_type, entity_voided,
+ updates, data_type, cache):
+ """Process ownership or control statement for relationship"""
+ void_statement = None
+ if updates:
+ latest_id, _ = await latest_lookup(cache, f"{start}_{end}_{rel_type}", updates=updates)
+ if latest_id:
+ # Check if deleted
+ if "Extension" in item and "Deletion" in item["Extension"]:
+ statement = data_type.void_ooc_relationship_deletion(latest_id,
+ item["Extension"]["Deletion"]["DeletedAt"],
+ start,
+ end)
+ statement_id = statement['statementID'] if statement else None
+ elif item["Registration"]["RegistrationStatus"] == 'RETIRED':
+ update_date = item['Registration']['LastUpdateDate']
+ statement = data_type.void_ooc_relationship_retired(latest_id,
+ update_date,
+ start,
+ end)
+ statement_id = statement['statementID'] if statement else None
+ else:
+ data_type.add_replaces(statement, latest_id) # Add replaces statement
+ await updates_delete(cache, latest_id, if_exists=True)
+ # Check if replacing exception
+ if rel_type in ("IS_DIRECTLY_CONSOLIDATED_BY", "IS_ULTIMATELY_CONSOLIDATED_BY"):
+ except_type = convert_rel_type(rel_type)
+ except_ooc_id, except_other_id, except_reason, except_ref, except_entity_type \
+ = await exception_lookup(cache, f"{start}_{except_type}", updates=updates)
+ if except_ooc_id and not entity_voided:
+ update_date = current_date_iso()
+ void_statement = data_type.void_entity_replaced(except_other_id,
+ update_date,
+ except_entity_type,
+ start,
+ except_reason)
+ await exception_delete(cache, f"{start}_{except_type}", updates=updates)
+ await latest_save(cache, f"{start}_{except_type}_{except_reason}_entity", None, updates=updates)
+ # Update references
+ if statement_id:
+ ref_statement_ids = referenced_ids(statement, item=item) # Statements Referenced By OOC
+ for ref_id in ref_statement_ids:
+ if ref_id:
+ if latest_id:
+ await references_remove(cache, ref_id, latest_id, updates=updates)
+ await references_update(cache, ref_id, statement_id, f"{start}_{end}_{rel_type}", updates=updates)
+ # Save statementID in latest
+ if statement_id:
+ await latest_save(cache, f"{start}_{end}_{rel_type}", statement_id, updates=updates)
+ return statement_id, statement, void_statement
+
+async def process_ooc_repex(statement_id, statement, item, except_lei, except_type, except_reason,
+ except_reference, old_reason, old_reference, old_ooc_id, old_other_id,
+ data_type, cache, updates=False):
+ """Process ownership or control statement for reporting exception"""
+ if "Extension" in item and "Deletion" in item["Extension"]:
+ latest_id, _ = await latest_lookup(cache, f"{except_lei}_{except_type}_{except_reason}_ownership", updates=updates)
+ statement = data_type.void_ooc_exception_deletion(latest_id,
+ item["Extension"]["Deletion"]["DeletedAt"],
+ item["LEI"],
+ item["ExceptionReason"])
+ statement_id = statement['statementID'] if statement else None
+ await updates_delete(cache, latest_id, if_exists=True)
+ elif (old_reason and except_reason != old_reason):
+ data_type.add_replaces(statement, old_ooc_id) # Add replaces statement
+ await updates_delete(cache, old_ooc_id, if_exists=True)
+ elif (old_reference and except_reference != old_reference):
+ data_type.add_replaces(statement, old_ooc_id) # Add replaces statement
+ if statement['statementID'] == old_other_id:
+ statement['statementID'] = generate_statement_id(statement['statementID'], "ownership")
+ statement_id = statement['statementID']
+ await updates_delete(cache, old_ooc_id, if_exists=True)
+ if statement_id:
+ await latest_save(cache, f"{except_lei}_{except_type}_{except_reason}_ownership", statement_id, updates=updates)
+ return statement_id, statement
+
+class ProcessUpdates:
+ """Data processor definition class"""
+ def __init__(self, id_name=None, transform=None, updates=None, storage=None):
+ """Initial setup"""
+ self.transform = transform
+ self.updates = updates
+ self.id_name = id_name
+ self.storage = storage
+ self.cache = Caching(self.storage, batching=-1)
+
+ async def setup(self):
+ """Load data into cache"""
+ await self.storage.setup()
+ await self.cache.load()
+
+ async def process(self, item, item_type, header, updates=False):
+ """Process updates if applicable"""
+ print(f"Processing - updates: {updates}")
+ entity_voided = False
+ entity_type = None
+ mapping, old_ooc_id, old_other_id, old_reason, old_reference, old_entity_type, except_lei, \
+ except_type, except_reason, except_reference = await item_setup(self.cache, item, updates=updates)
+ async for statement in self.transform.process(item, item_type, header, mapping=mapping):
+ statement_id = statement['statementID']
+ if statement['statementType'] in ('entityStatement', 'personStatement'):
+ entity_type = statement['statementType']
+ if not "ExceptionCategory" in item:
+ lei = source_id(statement, self.id_name) # LEI for statement
+ statement_id, statement = await process_entity_lei(statement_id,
+ statement,
+ item,
+ lei,
+ updates,
+ mapping,
+ self.updates,
+ self.cache)
+ else:
+ statement_id, statement, void_statement = await process_entity_repex(statement_id,
+ statement, item, except_lei, except_type, except_reason,
+ old_reason, old_other_id, old_entity_type, self.updates,
+ mapping, self.cache, updates=updates)
+ if void_statement:
+ yield void_statement
+ entity_voided = True
+ other_id = statement_id
+ elif statement['statementType'] == 'ownershipOrControlStatement':
+ # Get identifying features
+ if "ExceptionCategory" in item:
+ start = except_lei
+ end = "None"
+ rel_type = convert_except_type(except_type)
+ else:
+ start = item["Relationship"]["StartNode"]["NodeID"]
+ end = item["Relationship"]["EndNode"]["NodeID"]
+ rel_type = item["Relationship"]["RelationshipType"]
+ # Save references
+ if not "ExceptionCategory" in item:
+ statement_id, statement, void_statement = await process_ooc_rr(statement_id, statement, item,
+ start, end, rel_type, entity_voided,
+ updates, self.updates, self.cache)
+ if void_statement: yield void_statement
+ else:
+ statement_id, statement = await process_ooc_repex(statement_id, statement, item,
+ except_lei, except_type, except_reason,
+ except_reference, old_reason, old_reference,
+ old_ooc_id, old_other_id, self.updates,
+ self.cache, updates=updates)
+ ooc_id = statement_id
+ if statement: yield statement
+ if "ExceptionCategory" in item:
+ except_lei = item["LEI"]
+ except_type = item["ExceptionCategory"]
+ except_reason = item["ExceptionReason"]
+ except_reference = item["ExceptionReference"] if "ExceptionReference" in item else None
+ await exception_save(self.cache, f"{except_lei}_{except_type}", ooc_id,
+ other_id, except_reason, except_reference, entity_type, updates=updates)
+
+ async def finish_updates(self, updates=False):
+ """Process updates to referencing statements"""
+ print("In finish_updates")
+ data_type = self.updates
+ if updates:
+ done_updates = []
+ async for ref_id, latest_id, todo_updates in process_updates(self.cache):
+ statement = await retrieve_statement(self.storage, "ownership", ref_id)
+ old_statement_id = fix_statement_reference(statement, todo_updates, latest_id)
+ statement_id = statement["statementID"]
+ data_type.add_replaces(statement, old_statement_id)
+ await latest_save(self.cache, latest_id, statement_id, updates=updates)
+ # Update references
+ ref_statement_ids = referenced_ids(statement) # Statements Referenced By OOC
+ for new_ref_id in ref_statement_ids:
+ if new_ref_id:
+ await references_remove(self.cache, new_ref_id, old_statement_id, updates=updates)
+ await references_update(self.cache, new_ref_id, statement_id, latest_id, updates=updates)
+ done_updates.append(old_statement_id)
+ yield statement
+ for statement_id in done_updates:
+ await updates_delete(self.cache, statement_id)
+ await self.cache.flush()
diff --git a/bodspipelines/infrastructure/utils.py b/bodspipelines/infrastructure/utils.py
index 0216fa6..e5ede00 100644
--- a/bodspipelines/infrastructure/utils.py
+++ b/bodspipelines/infrastructure/utils.py
@@ -1,8 +1,40 @@
+import datetime
+import dateutil
+import pytz
+import string
+import random
+import hashlib
+import uuid
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from functools import partial
+def random_string(length):
+ """Generate random string of specified length"""
+ characters = string.ascii_letters + string.digits
+ return ''.join(random.choice(characters) for _ in range(length))
+
+def format_date(d):
+ """Format date in ISO 8601"""
+ return dateutil.parser.isoparse(d).strftime("%Y-%m-%d")
+
+
+def current_date_iso():
+ """Generate current date in ISO 8601"""
+ return datetime.datetime.now(pytz.timezone('Europe/London')).strftime("%Y-%m-%d")
+
+
+def generate_statement_id(name, role, version=None):
+ """Generate statement ID deterministically"""
+ if version:
+ seed = '-'.join([name, role, version])
+ else:
+ seed = '-'.join([name, role])
+ m = hashlib.md5()
+ m.update(seed.encode('utf-8'))
+ return str(uuid.UUID(m.hexdigest()))
+
def requests_retry_session(
retries=3,
@@ -10,6 +42,7 @@ def requests_retry_session(
status_forcelist=(500, 502, 504),
session=None,
):
+ """Requests session with automatic retries"""
session = session or requests.Session()
retry = Retry(
total=retries,
@@ -24,10 +57,40 @@ def requests_retry_session(
return session
+def download(url):
+ """Download url"""
+ with requests_retry_session().get(url) as r:
+ r.raise_for_status()
+ return r
+
+
def download_delayed(url, func):
+ """Return partial download function"""
def download(url, func, param):
with requests_retry_session().get(url) as r:
r.raise_for_status()
out = func(r, param)
return out
return partial(download, url, func)
+
+def identify_bods(item):
+ """Identify type of BODS data"""
+ if item['statementType'] == 'entityStatement':
+ return 'entity'
+ elif item['statementType'] == 'personStatement':
+ return 'person'
+ elif item['statementType'] == 'ownershipOrControlStatement':
+ return 'ownership'
+
+async def load_last_run(storage, name=None):
+ """Load data about last pipeline run"""
+ runs = []
+ async for run in storage.stream_items("runs"):
+ runs.append(run)
+ if name:
+ runs = [run for run in runs if run['stage_name'] == name]
+ return sorted(runs, key=lambda x: float(x["end_timestamp"]))[-1]
+
+async def save_run(storage, data):
+ """Save data about last pipeline run"""
+ await storage.add_item(data, "runs")
diff --git a/bodspipelines/pipelines/gleif/annotations.py b/bodspipelines/pipelines/gleif/annotations.py
new file mode 100644
index 0000000..8612f2e
--- /dev/null
+++ b/bodspipelines/pipelines/gleif/annotations.py
@@ -0,0 +1,66 @@
+from bodspipelines.infrastructure.utils import current_date_iso
+
+def add_annotation(annotations, description, pointer):
+ """Add commenting annotation to statement"""
+ annotation = {'motivation': 'commenting',
+ 'description': description,
+ 'statementPointerTarget': pointer,
+ 'creationDate': current_date_iso(),
+ 'createdBy': {'name': 'Open Ownership',
+ 'uri': "https://www.openownership.org"}}
+ annotations.append(annotation)
+
+def add_repex_ooc_annotation(annotations):
+ """Annotation for all reporting exception ownership or control statements"""
+ add_annotation(annotations,
+ "The nature of this interest is unknown",
+ "/interests/0/type")
+
+def add_lei_annotation(annotations, lei, registration_status):
+ """Annotation of status for all entity statements (not generated as a result
+ of a reporting exception)"""
+ add_annotation(annotations,
+ f"GLEIF data for this entity - LEI: {lei}; Registration Status: {registration_status}",
+ "/")
+
+def add_rr_annotation_status(annotations, subject, interested):
+ """Annotation for all ownership or control statements"""
+ add_annotation(annotations,
+ f"Describes GLEIF relationship: {subject} is subject, {interested} is interested party",
+ "/")
+
+def add_annotation_retired(annotations):
+ """Annotation for retired statements"""
+ add_annotation(annotations,
+ "GLEIF RegistrationStatus set to RETIRED on this statementDate.",
+ "/")
+
+def add_rr_annotation_deleted(annotations):
+ """Annotation for deleted ownership or control statements"""
+ add_annotation(annotations,
+ "GLEIF relationship deleted on this statementDate.",
+ "/")
+
+def add_repex_annotation_reason(annotations, reason, lei):
+ """Annotation for all statements created as a reasult of a reporting exception"""
+ add_annotation(annotations,
+ f"This statement was created due to a {reason} GLEIF Reporting Exception for {lei}",
+ "/")
+
+def add_repex_annotation_changed(annotations, reason, lei):
+ """Annotation for all statements when reporting exception changed"""
+ add_annotation(annotations,
+ f"Statement retired due to change in a {reason} GLEIF Reporting Exception for {lei}",
+ "/")
+
+def add_repex_annotation_replaced(annotations, reason, lei):
+ """Annotation for all statements when reporting exception replaced with relationship"""
+ add_annotation(annotations,
+ f"Statement series retired due to replacement of a {reason} GLEIF Reporting Exception for {lei}",
+ "/")
+
+def add_repex_annotation_deleted(annotations, reason, lei):
+ """Annotation for ownership or control statement when reporting exception deleted"""
+ add_annotation(annotations,
+ f"Statement series retired due to deletion of a {reason} GLEIF Reporting Exception for {lei}",
+ "/")
diff --git a/bodspipelines/pipelines/gleif/config.py b/bodspipelines/pipelines/gleif/config.py
index c1ddf86..4d7fda6 100644
--- a/bodspipelines/pipelines/gleif/config.py
+++ b/bodspipelines/pipelines/gleif/config.py
@@ -1,129 +1,132 @@
+import os
import time
import elastic_transport
+import asyncio
+from datetime import datetime
from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
from bodspipelines.infrastructure.inputs import KinesisInput
-from bodspipelines.infrastructure.storage import ElasticStorage
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+from bodspipelines.infrastructure.clients.redis_client import RedisClient
from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
from bodspipelines.infrastructure.processing.bulk_data import BulkData
from bodspipelines.infrastructure.processing.xml_data import XMLData
from bodspipelines.infrastructure.processing.json_data import JSONData
+from bodspipelines.infrastructure.updates import ProcessUpdates
-from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties, ownership_statement_properties,
- match_entity, match_person, match_ownership,
- id_entity, id_person, id_ownership)
+from bodspipelines.pipelines.gleif.indexes import gleif_index_properties
+from bodspipelines.infrastructure.indexes import bods_index_properties
-from bodspipelines.pipelines.gleif.transforms import Gleif2Bods
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, AddContentDate, RemoveEmptyExtension
from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
match_lei, match_rr, match_repex,
id_lei, id_rr, id_repex)
-from bodspipelines.pipelines.gleif.utils import gleif_download_link
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData, identify_gleif
+from bodspipelines.pipelines.gleif.updates import GleifUpdates
+from bodspipelines.infrastructure.utils import identify_bods, load_last_run, save_run
# Defintion of LEI-CDF v3.1 XML date source
lei_source = Source(name="lei",
- origin=BulkData(display="LEI-CDF v3.1",
- #url='https://leidata.gleif.org/api/v1/concatenated-files/lei2/get/30447/zip',
- url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
- size=41491,
- directory="lei-cdf"),
- datatype=XMLData(item_tag="LEIRecord",
- namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016"},
- filter=['NextVersion', 'Extension']))
+ origin=BulkData(display="LEI-CDF v3.1",
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest",
+ data_date="2024-01-01"),
+ size=41491,
+ directory="lei-cdf"),
+ datatype=XMLData(item_tag="LEIRecord",
+ namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016",
+ "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"},
+ filter=['NextVersion', 'Extension']))
# Defintion of RR-CDF v2.1 XML date source
rr_source = Source(name="rr",
origin=BulkData(display="RR-CDF v2.1",
- #url='https://leidata.gleif.org/api/v1/concatenated-files/rr/get/30450/zip',
- url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
- size=2823,
- directory="rr-cdf"),
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/rr/latest",
+ data_date="2024-01-01"),
+ size=2823,
+ directory="rr-cdf"),
datatype=XMLData(item_tag="RelationshipRecord",
- namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"},
- filter=['Extension']))
+ namespace={"rr": "http://www.gleif.org/data/schema/rr/2016",
+ "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"},
+ filter=['NextVersion', ]))
# Defintion of Reporting Exceptions v2.1 XML date source
repex_source = Source(name="repex",
origin=BulkData(display="Reporting Exceptions v2.1",
- #url='https://leidata.gleif.org/api/v1/concatenated-files/repex/get/30453/zip',
- url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
- size=3954,
- directory="rep-ex"),
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/repex/latest",
+ data_date="2024-01-01"),
+ size=3954,
+ directory="rep-ex"),
datatype=XMLData(item_tag="Exception",
- namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"},
- filter=['NextVersion', 'Extension']))
+ header_tag="Header",
+ namespace={"repex": "http://www.gleif.org/data/schema/repex/2016",
+ "gleif": "http://www.gleif.org/data/schema/golden-copy/extensions/1.0"},
+ filter=['NextVersion', ]))
-# Console Output
-#output_console = Output(name="console", target=OutputConsole(name="gleif-ingest"))
-
-# Elasticsearch indexes for GLEIF data
-index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
- "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
- "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
+# Easticsearch storage for GLEIF data
+gleif_storage = ElasticsearchClient(indexes=gleif_index_properties)
# GLEIF data: Store in Easticsearch and output new to Kinesis stream
-output_new = NewOutput(storage=ElasticStorage(indexes=index_properties),
- output=KinesisOutput(stream_name="gleif-prod"))
+output_new = NewOutput(storage=Storage(storage=gleif_storage),
+ output=KinesisOutput(stream_name=os.environ.get('GLEIF_KINESIS_STREAM')))
# Definition of GLEIF data pipeline ingest stage
ingest_stage = Stage(name="ingest",
sources=[lei_source, rr_source, repex_source],
- processors=[],
- outputs=[output_new]
-)
+ processors=[AddContentDate(identify=identify_gleif),
+ RemoveEmptyExtension(identify=identify_gleif)],
+ outputs=[output_new])
# Kinesis stream of GLEIF data from ingest stage
gleif_source = Source(name="gleif",
- origin=KinesisInput(stream_name="gleif-prod"),
+ origin=KinesisInput(stream_name=os.environ.get('GLEIF_KINESIS_STREAM')),
datatype=JSONData())
-# Elasticsearch indexes for BODS data
-bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity},
- "person": {"properties": person_statement_properties, "match": match_person, "id": id_person},
- "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership}}
-
-# Identify type of GLEIF data
-def identify_gleif(item):
- if 'Entity' in item:
- return 'lei'
- elif 'Relationship' in item:
- return 'rr'
- elif 'ExceptionCategory' in item:
- return 'repex'
-
-# Identify type of BODS data
-def identify_bods(item):
- if item['statementType'] == 'entityStatement':
- return 'entity'
- elif item['statementType'] == 'personStatement':
- return 'person'
- elif item['statementType'] == 'ownershipOrControlStatement':
- return 'ownership'
+# Easticsearch storage for BODS data
+bods_storage = ElasticsearchClient(indexes=bods_index_properties)
# BODS data: Store in Easticsearch and output new to Kinesis stream
-bods_output_new = NewOutput(storage=ElasticStorage(indexes=bods_index_properties),
- output=KinesisOutput(stream_name="bods-gleif-prod"),
+bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name=os.environ.get('BODS_KINESIS_STREAM')),
identify=identify_bods)
# Definition of GLEIF data pipeline transform stage
transform_stage = Stage(name="transform",
sources=[gleif_source],
- processors=[Gleif2Bods(identify=identify_gleif)],
- outputs=[bods_output_new]
-)
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage),
+ updates=GleifUpdates())],
+ outputs=[bods_output_new])
# Definition of GLEIF data pipeline
pipeline = Pipeline(name="gleif", stages=[ingest_stage, transform_stage])
-# Setup Elasticsearch indexes
+# Setup storage indexes
+async def setup_indexes():
+ await gleif_storage.setup_indexes()
+ await bods_storage.setup_indexes()
+
+# Load run
+async def load_previous(name):
+ bods_storage_run = ElasticsearchClient(indexes=bods_index_properties)
+ await bods_storage_run.setup()
+ storage_run = Storage(storage=bods_storage_run)
+ return await load_last_run(storage_run, name=name)
+
+# Save data on current pipeline run
+async def save_current_run(name, start_timestamp):
+ bods_storage_run = ElasticsearchClient(indexes=bods_index_properties)
+ await bods_storage_run.setup()
+ storage_run = Storage(storage=bods_storage_run)
+ run_data = {'stage_name': name,
+ 'start_timestamp': str(start_timestamp),
+ 'end_timestamp': datetime.now().timestamp()}
+ await save_run(storage_run, run_data)
+
+# Setup pipeline storage
def setup():
- done = False
- while not done:
- try:
- ElasticStorage(indexes=index_properties).setup_indexes()
- ElasticStorage(indexes=bods_index_properties).setup_indexes()
- done = True
- except elastic_transport.ConnectionError:
- print("Waiting for Elasticsearch to start ...")
- time.sleep(5)
-
-#stats = ElasticStorage(indexes=index_properties).stats
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(setup_indexes())
+
diff --git a/bodspipelines/pipelines/gleif/indexes.py b/bodspipelines/pipelines/gleif/indexes.py
index 76a2759..6972463 100644
--- a/bodspipelines/pipelines/gleif/indexes.py
+++ b/bodspipelines/pipelines/gleif/indexes.py
@@ -1,11 +1,16 @@
+import hashlib
+
# GLEIF LEI Elasticsearch Properties
lei_properties = {'LEI': {'type': 'text'},
'Entity': {'type': 'object',
'properties': {'LegalName': {'type': 'text'},
- 'OtherEntityNames': {'type': 'text'},
-# 'TransliteratedOtherEntityNames': {'type': 'object',
-# 'properties': {'TransliteratedOtherEntityName': {'type': 'text'}}},
- 'TransliteratedOtherEntityNames': {'type': 'text'},
+ 'OtherEntityNames': #{'type': 'text'},
+ {'type': 'object',
+ 'properties': {'OtherEntityName': {'type': 'text'},
+ 'type': {'type': 'text'}}},
+ 'TransliteratedOtherEntityNames': {'type': 'object',
+ 'properties': {'TransliteratedOtherEntityName': {'type': 'text'},
+ 'type': {'type': 'text'}}},
'LegalAddress': {'type': 'object',
'properties': {'FirstAddressLine': {'type': 'text'},
'AdditionalAddressLine': {'type': 'text'},
@@ -35,7 +40,8 @@
'City': {'type': 'text'},
'Region': {'type': 'text'},
'Country': {'type': 'text'},
- 'PostalCode': {'type': 'text'}}},
+ 'PostalCode': {'type': 'text'},
+ 'type': {'type': 'text'}}},
'TransliteratedOtherAddresses': {'type': 'object',
'properties': {'FirstAddressLine': {'type': 'text'},
'AdditionalAddressLine': {'type': 'text'},
@@ -45,7 +51,8 @@
'City': {'type': 'text'},
'Region': {'type': 'text'},
'Country': {'type': 'text'},
- 'PostalCode': {'type': 'text'}}},
+ 'PostalCode': {'type': 'text'},
+ 'type': {'type': 'text'}}},
'RegistrationAuthority': {'type': 'object',
'properties': {'RegistrationAuthorityID': {'type': 'text'},
'RegistrationAuthorityEntityID': {'type': 'text'},
@@ -117,12 +124,21 @@
'properties': {'ValidationAuthorityID': {'type': 'text'},
'OtherValidationAuthorityID': {'type': 'text'},
'ValidationAuthorityEntityID': {'type': 'text'}}}
- }}}
+ }},
+ 'Extension': {'type': 'object',
+ 'properties': {'Deletion': {'type': 'object',
+ 'properties': {'DeletedAt': {'type': 'text'}}}}}
+ }
repex_properties = {'LEI': {'type': 'text'},
'ExceptionCategory': {'type': 'text'},
'ExceptionReason': {'type': 'text'},
- 'ExceptionReference': {'type': 'text'}}
+ 'ExceptionReference': {'type': 'text'},
+ 'ContentDate': {'type': 'text'},
+ 'Extension': {'type': 'object',
+ 'properties': {'Deletion': {'type': 'object',
+ 'properties': {'DeletedAt': {'type': 'text'}}}}}
+ }
def match_lei(item):
return {"match": {"LEI": item["LEI"]}}
@@ -131,24 +147,28 @@ def match_rr(item):
return {'bool': {'must': [{"match": {'Relationship.StartNode.NodeID': item['Relationship']['StartNode']['NodeID']}},
{"match": {'Relationship.EndNode.NodeID': item['Relationship']['EndNode']['NodeID']}},
{"match": {'Relationship.RelationshipType': item['Relationship']['RelationshipType']}}]}}
-#{"bool": {"must": [{"term": {'Relationship.StartNode.NodeID': item['Relationship']['StartNode']['NodeID']}},
-# {"term": {'Relationship.EndNode.NodeID': item['Relationship']['EndNode']['NodeID']}},
-# {"term": {'Relationship.RelationshipType': item['Relationship']['RelationshipType']}}]}}
def match_repex(item):
return {'bool': {'must': [{"match": {'LEI': item["LEI"]}},
{"match": {'ExceptionCategory': item["ExceptionCategory"]}},
{"match": {'ExceptionReason': item["ExceptionReason"]}}]}}
-#{"bool": {"must": [{"term": {'ExceptionCategory': item["ExceptionCategory"]}},
-# {"term": {'ExceptionReason': item["ExceptionReason"]}},
-# {"term": {'LEI': item["LEI"]}}]}}
def id_lei(item):
- return item["LEI"]
+ return f"{item['LEI']}_{item['Registration']['LastUpdateDate']}"
def id_rr(item):
- return f"{item['Relationship']['StartNode']['NodeID']}_{item['Relationship']['EndNode']['NodeID']}_{item['Relationship']['RelationshipType']}"
+ return f"{item['Relationship']['StartNode']['NodeID']}_{item['Relationship']['EndNode']['NodeID']}_{item['Relationship']['RelationshipType']}_{item['Registration']['LastUpdateDate']}"
def id_repex(item):
- return f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}"
+ if "ExceptionReference" in item:
+ ref_hash = hashlib.sha256(bytes(item['ExceptionReference'], 'utf8')).hexdigest()
+ item_id = f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}_{ref_hash}_{item['ContentDate']}"
+ else:
+ item_id = f"{item['LEI']}_{item['ExceptionCategory']}_{item['ExceptionReason']}_None_{item['ContentDate']}"
+ #print(item_id, len(item_id), item)
+ return item_id
+# Elasticsearch indexes for GLEIF data
+gleif_index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
+ "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
+ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
diff --git a/bodspipelines/pipelines/gleif/transforms.py b/bodspipelines/pipelines/gleif/transforms.py
index 6b5f33f..50d5eec 100644
--- a/bodspipelines/pipelines/gleif/transforms.py
+++ b/bodspipelines/pipelines/gleif/transforms.py
@@ -7,49 +7,75 @@
from typing import List, Union
import pycountry
+#from bodspipelines.infrastructure.caching import cached
+from bodspipelines.infrastructure.utils import format_date, current_date_iso, generate_statement_id
+from .annotations import (add_lei_annotation, add_rr_annotation_status,
+ add_repex_annotation_reason, add_repex_ooc_annotation)
+from .indexes import id_lei as entity_id
+from .indexes import id_rr as rr_id
+from .indexes import id_repex as repex_id
+
+#def entity_id(data):
+# """Create ID for entity"""
+# return f"{data['LEI']}_{data['Registration']['LastUpdateDate']}"
+
+#def rr_id(data):
+# """Create ID for relationship"""
+# return f"{data['Relationship']['StartNode']['NodeID']}_{data['Relationship']['EndNode']['NodeID']}_{data['Relationship']['RelationshipType']}_{data['Registration']['LastUpdateDate']}"
+
+#def repex_id(data):
+# """Create ID for reporting exception"""
+# if 'ExceptionReference' in data:
+# return f"{data['LEI']}_{data['ExceptionCategory']}_{data['ExceptionReason']}_{data['ExceptionReference']}"
+# else:
+# return f"{data['LEI']}_{data['ExceptionCategory']}_{data['ExceptionReason']}_None"
+
+#async def subject_id(lei):
+# async def get_item(x):
+# return x
+# get_item.__self__ = None
+# data = await cached(get_item, lei, "latest", batch=False)
+# return data['statement_id']
+
def format_address(address_type, address):
"""Format address structure"""
address_string = ", ".join([address['FirstAddressLine'], address['City']])
out = {'type': address_type,'address': address_string, 'country': address['Country']}
if 'PostalCode' in address: out['postCode'] = address['PostalCode']
return out
-def generate_statement_id(name, role, version=None):
- """Generate statement ID deterministically"""
- if version:
- seed = '-'.join([name, role, version])
- else:
- seed = '-'.join([name, role])
- m = hashlib.md5()
- m.update(seed.encode('utf-8'))
- return str(uuid.UUID(m.hexdigest()))
-
-def format_date(d):
- """Format date in ISO 8601"""
- return dateutil.parser.isoparse(d).strftime("%Y-%m-%d") #.isoformat(timespec='seconds')
-
-def current_date_iso():
- """Generate current date in ISO 8601"""
- return datetime.datetime.now(pytz.timezone('Europe/London')).strftime("%Y-%m-%d") #.isoformat(timespec='seconds')
def publication_details():
"""Generate publication details"""
return {'publicationDate': current_date_iso(), # TODO: fix publication date
'bodsVersion': "0.2",
- 'license': "https://register.openownership.org/terms-and-conditions",
- 'publisher': {"name": "OpenOwnership Register",
- "url": "https://register.openownership.org"}}
+ 'license': "https://creativecommons.org/publicdomain/zero/1.0/",
+ 'publisher': {"name": "Open Ownership",
+ "url": "https://www.openownership.org"}}
+
+def jurisdiction_name(data):
+ try:
+ if "-" in data['Entity']['LegalJurisdiction']:
+ subdivision = pycountry.subdivisions.get(code=data['Entity']['LegalJurisdiction'])
+ name = f"{subdivision.name}, {subdivision.country.name}"
+ else:
+ name = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name
+ except AttributeError:
+ name = data['Entity']['LegalJurisdiction']
+ return name
def transform_lei(data):
"""Transform LEI-CDF v3.1 data to BODS statement"""
- statementID = generate_statement_id(data['LEI'], 'entityStatement')
+ #print("Transforming LEI:")
+ statementID = generate_statement_id(entity_id(data), 'entityStatement')
statementType = 'entityStatement'
statementDate = format_date(data['Registration']['LastUpdateDate'])
entityType = 'registeredEntity'
name = data['Entity']['LegalName']
- try:
- country = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name
- except AttributeError:
- country = data['Entity']['LegalJurisdiction']
+ country = jurisdiction_name(data)
+ #try:
+ # country = pycountry.countries.get(alpha_2=data['Entity']['LegalJurisdiction']).name
+ #except AttributeError:
+ # country = data['Entity']['LegalJurisdiction']
jurisdiction = {'name': country, 'code': data['Entity']['LegalJurisdiction']}
identifiers = [{'id': data['LEI'], 'scheme':'XI-LEI', 'schemeName':'Global Legal Entity Identifier Index'}]
if 'RegistrationAuthority' in data['Entity']:
@@ -59,9 +85,6 @@ def transform_lei(data):
if 'RegistrationAuthorityID' in data['Entity']['RegistrationAuthority']:
authority['schemeName'] = data['Entity']['RegistrationAuthority']['RegistrationAuthorityID']
if authority: identifiers.append(authority)
- #identifiers.append({'id': data['Entity']['RegistrationAuthority']['RegistrationAuthorityEntityID'],
- # 'schemeName': data['Entity']['RegistrationAuthority']['RegistrationAuthorityID']})
- #foundingDate = data['Entity']['EntityCreationDate']
registeredAddress = format_address('registered', data['Entity']['LegalAddress'])
businessAddress = format_address('business', data['Entity']['HeadquartersAddress'])
if 'ValidationSources' in data['Registration']:
@@ -69,6 +92,8 @@ def transform_lei(data):
else:
sourceType = ['officialRegister']
sourceDescription = 'GLEIF'
+ annotations = []
+ add_lei_annotation(annotations, data['LEI'], data["Registration"]["RegistrationStatus"])
out = {'statementID': statementID,
'statementType': statementType,
'statementDate': statementDate,
@@ -76,8 +101,8 @@ def transform_lei(data):
'name': name,
'incorporatedInJurisdiction': jurisdiction,
'identifiers': identifiers,
- #'foundingDate': foundingDate,
'addresses': [registeredAddress,businessAddress],
+ 'annotations': annotations,
'publicationDetails': publication_details(),
'source': {'type':sourceType,'description':sourceDescription}}
if 'EntityCreationDate' in data['Entity']: out['foundingDate'] = data['Entity']['EntityCreationDate']
@@ -92,20 +117,27 @@ def interest_level(relationship_type, default):
else:
return default # Other options in data
-def transform_rr(data):
+def calc_statement_id(lei, mapping):
+ """Calculate statementID for lei using mapping if available"""
+ #print("calc_statement_id:", lei, mapping)
+ if lei in mapping:
+ return mapping[lei]
+ else:
+ return generate_statement_id(lei, 'entityStatement')
+
+def transform_rr(data, mapping):
"""Transform RR-CDF v2.1 data to BODS statement"""
- statementID = generate_statement_id(data['Relationship']['StartNode']['NodeID'] +
- data['Relationship']['EndNode']['NodeID'] +
- data['Relationship']['RelationshipType'], 'ownershipOrControlStatement')
+ statementID = generate_statement_id(rr_id(data), 'ownershipOrControlStatement')
statementType = 'ownershipOrControlStatement'
statementDate = format_date(data['Registration']['LastUpdateDate'])
- subjectDescribedByEntityStatement = generate_statement_id(data['Relationship']['StartNode']['NodeID'], 'entityStatement')
- interestedPartyDescribedByEntityStatement = generate_statement_id(data['Relationship']['EndNode']['NodeID'], 'entityStatement')
- #interestType = 'otherInfluenceOrControl'
+ subjectDescribedByEntityStatement = calc_statement_id(data['Relationship']['StartNode']['NodeID'], mapping)
+ interestedPartyDescribedByEntityStatement = calc_statement_id(data['Relationship']['EndNode']['NodeID'], mapping)
interestType = 'other-influence-or-control'
- interestLevel = interest_level(data['Relationship']['RelationshipType'], 'unknown')
+ #interestLevel = interest_level(data['Relationship']['RelationshipType'], 'unknown')
+ interestLevel = "unknown"
#periods = data['Relationship']['RelationshipPeriods']
interestStartDate = False
+ interestDetails = f"LEI RelationshipType: {data['Relationship']['RelationshipType']}"
start_date = False
if 'RelationshipPeriods' in data['Relationship']:
periods = data['Relationship']['RelationshipPeriods']
@@ -122,6 +154,10 @@ def transform_rr(data):
beneficialOwnershipOrControl = False
sourceType = ['officialRegister'] if not data['Registration']['ValidationSources'] == 'FULLY_CORROBORATED' else ['officialRegister', 'verified']
sourceDescription = 'GLEIF'
+ annotations = []
+ add_rr_annotation_status(annotations,
+ data['Relationship']['StartNode']['NodeID'],
+ data['Relationship']['EndNode']['NodeID'])
out = {'statementID': statementID,
'statementType':statementType,
'statementDate':statementDate,
@@ -130,7 +166,9 @@ def transform_rr(data):
'interests':[{'type': interestType,
'interestLevel': interestLevel,
'beneficialOwnershipOrControl': beneficialOwnershipOrControl,
- 'startDate': interestStartDate}],
+ 'startDate': interestStartDate,
+ 'details': interestDetails}],
+ 'annotations': annotations,
'publicationDetails': publication_details(),
'source':{'type': sourceType, 'description': sourceDescription}}
return out
@@ -143,14 +181,22 @@ def transform_repex_entity(data, description, person=False):
else:
statementType = 'entityStatement'
entityType = "unknownEntity"
- statementID = generate_statement_id(data['LEI'] + data['ExceptionCategory'] + data['ExceptionReason'], statementType)
+ statementID = generate_statement_id(repex_id(data), statementType)
+ statementDate = format_date(data['ContentDate'])
#isComponent = 'false' Required?
unspecified_reason = 'interested-party-exempt-from-disclosure'
- unspecified_description = description
+ if "ExceptionReference" in data:
+ unspecified_description = f"{description} ExemptionReference provided: {data['ExceptionReference']}"
+ else:
+ unspecified_description = description
sourceType = ['officialRegister']
sourceDescription = 'GLEIF'
+ annotations = []
+ add_repex_annotation_reason(annotations, data["ExceptionReason"], data["LEI"])
out = {'statementID': statementID,
'statementType': statementType,
+ 'statementDate': statementDate,
+ 'annotations': annotations,
'publicationDetails': publication_details(),
'source':{'type':sourceType,'description':sourceDescription}}
if person:
@@ -161,29 +207,28 @@ def transform_repex_entity(data, description, person=False):
out['unspecifiedEntityDetails'] = {'reason': unspecified_reason, 'description': unspecified_description}
return out, statementID
-def transform_repex_ooc(data, interested=None, person=False):
+def transform_repex_ooc(data, mapping, interested=None, person=False):
"""Transform Reporting Exception to Ownership or Control statement"""
- statementID = generate_statement_id(data['LEI'] + data['ExceptionCategory'] + data['ExceptionReason'], 'ownershipOrControlStatement')
+ statementID = generate_statement_id(repex_id(data), 'ownershipOrControlStatement')
statementType = 'ownershipOrControlStatement'
- subjectDescribedByEntityStatement = generate_statement_id(data['LEI'], 'entityStatement')
+ statementDate = format_date(data['ContentDate'])
+ #subjectDescribedByEntityStatement = generate_statement_id(data['LEI'], 'entityStatement')
+ subjectDescribedByEntityStatement = calc_statement_id(data['LEI'], mapping)
if interested:
interestedParty = interested
else:
interestedParty = data['ExceptionReason']
- #interestType = 'unknownInterest'
interestType = 'other-influence-or-control'
- annotation = {'motivation': 'commenting',
- 'description': "The nature of this interest is unknown",
- 'statementPointerTarget': "/interests/0/type",
- 'creationDate': current_date_iso(),
- 'createdBy': {'name': 'Open Ownership',
- 'uri': "https://www.openownership.org"}}
+ annotations = []
+ add_repex_ooc_annotation(annotations)
+ add_repex_annotation_reason(annotations, data["ExceptionReason"], data["LEI"])
if data['ExceptionCategory'] == "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT":
interestLevel = 'indirect'
elif data['ExceptionCategory'] == "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT":
interestLevel = 'direct'
else:
interestLevel = 'unknown'
+ interestDetails = f"LEI ExceptionCategory: {data['ExceptionCategory']}"
sourceType = ['officialRegister']
sourceDescription = 'GLEIF'
if interested:
@@ -195,69 +240,80 @@ def transform_repex_ooc(data, interested=None, person=False):
interestedParty = {'unspecified': {'reason': interestedParty}}
out = {'statementID': statementID,
'statementType':statementType,
+ 'statementDate': statementDate,
'subject': {'describedByEntityStatement': subjectDescribedByEntityStatement},
'interestedParty': interestedParty,
'interests':[{'type': interestType,
'interestLevel': interestLevel,
'beneficialOwnershipOrControl': False,
'details': "A controlling interest."}],
+ 'annotations': annotations,
'publicationDetails': publication_details(),
- 'source':{'type': sourceType, 'description': sourceDescription},
- 'annotations': [annotation]}
+ 'source':{'type': sourceType, 'description': sourceDescription}}
return out, statementID
-def transform_repex_no_lei(data):
+def transform_repex_no_lei(data, mapping):
"""Transform NO_LEI Reporting Exception"""
for func in (transform_repex_entity, transform_repex_ooc):
if func == transform_repex_ooc:
- statement, statement_id = func(data, interested=statement_id)
+ statement, statement_id = func(data, mapping, interested=statement_id)
else:
statement, statement_id = func(data, "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf.")
yield statement
-def transform_repex_natural_persons(data):
+def transform_repex_natural_persons(data, mapping):
"""Transform NATURAL_PERSONS Reporting Exception"""
for func in (transform_repex_entity, transform_repex_ooc):
if func == transform_repex_ooc:
- statement, statement_id = func(data, interested=statement_id, person=True)
+ statement, statement_id = func(data, mapping, interested=statement_id, person=True)
else:
statement, statement_id = func(data, "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity.", person=True)
yield statement
-def transform_repex_non_consolidating(data):
+def transform_repex_non_consolidating(data, mapping):
"""Transform NON_CONSOLIDATING Reporting Exception"""
for func in (transform_repex_entity, transform_repex_ooc):
if func == transform_repex_ooc:
- statement, statement_id = func(data, interested=statement_id)
+ statement, statement_id = func(data, mapping, interested=statement_id)
else:
statement, statement_id = func(data, "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control.")
yield statement
-def transform_repex_non_public(data):
+def transform_repex_non_public(data, mapping):
"""Transform NON_PUBLIC Reporting Exception"""
for func in (transform_repex_entity, transform_repex_ooc):
if func == transform_repex_ooc:
- statement, statement_id = func(data, interested=statement_id)
+ statement, statement_id = func(data, mapping, interested=statement_id)
else:
statement, statement_id = func(data, "From LEI ExemptionReason `NON_PUBLIC` or related deprecated values. The legal entity’s relationship information with an entity it controls is non-public. There are therefore obstacles to releasing this information.")
yield statement
-def transform_repex(data):
+def transform_repex_no_known(data, mapping):
+ """Transform NO_KNOWN_PERSON Reporting Exception"""
+ for func in (transform_repex_entity, transform_repex_ooc):
+ if func == transform_repex_ooc:
+ statement, statement_id = func(data, mapping, interested=statement_id, person=True)
+ else:
+ statement, statement_id = func(data, "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity.", person=True)
+ yield statement
+
+def transform_repex(data, mapping):
"""Transform Reporting Exceptions to BODS statements"""
- #print(data['ExceptionReason'])
if data['ExceptionReason'] == "NO_LEI":
- #print("Got here!")
- for statement in transform_repex_no_lei(data):
+ for statement in transform_repex_no_lei(data, mapping):
yield statement
elif data['ExceptionReason'] == "NATURAL_PERSONS":
- for statement in transform_repex_natural_persons(data):
+ for statement in transform_repex_natural_persons(data, mapping):
yield statement
elif data['ExceptionReason'] == "NON_CONSOLIDATING":
- for statement in transform_repex_non_consolidating(data):
+ for statement in transform_repex_non_consolidating(data, mapping):
yield statement
elif data['ExceptionReason'] in ('NON_PUBLIC', 'BINDING_LEGAL_COMMITMENTS', 'LEGAL_OBSTACLES',
- 'DISCLOSURE_DETRIMENTAL', 'DETRIMENT_NOT_EXCLUDED', 'DETRIMENT_NOT_EXCLUDED'):
- for statement in transform_repex_non_public(data):
+ 'DISCLOSURE_DETRIMENTAL', 'DETRIMENT_NOT_EXCLUDED', 'CONSENT_NOT_OBTAINED'):
+ for statement in transform_repex_non_public(data, mapping):
+ yield statement
+ elif data['ExceptionReason'] == "NO_KNOWN_PERSON":
+ for statement in transform_repex_no_known(data, mapping):
yield statement
class Gleif2Bods:
@@ -266,12 +322,41 @@ def __init__(self, identify=None):
"""Initial setup"""
self.identify = identify
- def process(self, item, item_type):
+ async def process(self, item, item_type, header, mapping={}, updates=False):
+ """Process item"""
if self.identify: item_type = self.identify(item)
+ #print("Gleif2Bods:", item_type)
if item_type == 'lei':
yield transform_lei(item)
elif item_type == 'rr':
- yield transform_rr(item)
+ yield transform_rr(item, mapping)
elif item_type == 'repex':
- yield from transform_repex(item)
+ for statement in transform_repex(item, mapping):
+ yield statement
+class AddContentDate:
+ """Data processor to add ContentDate"""
+ def __init__(self, identify=None):
+ """Initial setup"""
+ self.identify = identify
+
+ async def process(self, item, item_type, header, mapping={}, updates=False):
+ """Process item"""
+ if self.identify: item_type = self.identify(item)
+ if item_type == 'repex':
+ item["ContentDate"] = header["ContentDate"]
+ yield item
+
+class RemoveEmptyExtension:
+ """Data processor to remove empty Extension"""
+ def __init__(self, identify=None):
+ """Initial setup"""
+ self.identify = identify
+
+ async def process(self, item, item_type, header, mapping={}, updates=False):
+ """Process item"""
+ if self.identify: item_type = self.identify(item)
+ if item_type == 'repex':
+ if "Extension" in item and not isinstance(item["Extension"], dict):
+ del item["Extension"]
+ yield item
diff --git a/bodspipelines/pipelines/gleif/updates.py b/bodspipelines/pipelines/gleif/updates.py
new file mode 100644
index 0000000..eb12d17
--- /dev/null
+++ b/bodspipelines/pipelines/gleif/updates.py
@@ -0,0 +1,149 @@
+from bodspipelines.infrastructure.utils import (current_date_iso, generate_statement_id,
+ random_string, format_date)
+from .annotations import (add_lei_annotation, add_repex_annotation_changed,
+ add_repex_annotation_replaced, add_repex_ooc_annotation, add_rr_annotation_deleted,
+ add_annotation_retired, add_rr_annotation_status, add_repex_annotation_deleted)
+
+def void_entity_statement(reason, statement_id, status, update_date, entity_type, lei, unknown=False):
+ """Create BODS statemnt to void entity/person"""
+ id = generate_statement_id(f"{statement_id}", "voided")
+ #print(f"Voiding entity ({statement_id}):", id)
+ statement = {"statementID": id,
+ "statementDate": format_date(update_date),
+ "publicationDetails": {
+ "publicationDate": current_date_iso(),
+ "bodsVersion": "0.2",
+ "publisher": {
+ "name": "GLEIF"
+ }
+ },
+ "statementType": entity_type,
+ #"entityType": "registeredEntity",
+ "isComponent": False,
+ "replacesStatements": [statement_id]
+ }
+ if entity_type == "entityStatement":
+ if unknown:
+ statement["entityType"] = "unknownEntity"
+ else:
+ statement["entityType"] = "registeredEntity"
+ else:
+ statement["personType"] = "unknownPerson"
+ annotations = []
+ if reason == "RETIRED":
+ add_lei_annotation(annotations, lei, status)
+ elif reason == "DELETION":
+ add_repex_annotation_deleted(annotations, status, lei)
+ elif reason == "CHANGED":
+ add_repex_annotation_changed(annotations, status, lei)
+ elif reason == "REPLACED":
+ add_repex_annotation_replaced(annotations, status, lei)
+ statement["annotations"] = annotations
+ return statement
+
+def void_ooc_statement(reason, statement_id, status, update_date, lei, lei2):
+ """Create BODS statement to void ownership or control statement"""
+ id = generate_statement_id(f"{statement_id}", "voided_ownershipOrControlStatement")
+ #print(f"Voiding OOC ({statement_id}):", id)
+ statement = {"statementID": id,
+ "statementDate": format_date(update_date),
+ "publicationDetails": {
+ "publicationDate": current_date_iso(),
+ "bodsVersion": "0.2",
+ "publisher": {
+ "name": "GLEIF"
+ }
+ },
+ "statementType": "ownershipOrControlStatement",
+ "interestedParty":{
+ "describedByEntityStatement": ""
+ },
+ "isComponent": False,
+ "subject": {
+ "describedByEntityStatement": ""
+ },
+ "replacesStatements": [statement_id]
+ }
+ annotations = []
+ if reason == "RR_DELETION":
+ add_rr_annotation_deleted(annotations)
+ add_rr_annotation_status(annotations, lei, lei2)
+ elif reason == "RETIRED":
+ add_annotation_retired(annotations)
+ add_rr_annotation_status(annotations, lei, lei2)
+ elif reason == "REPEX_DELETION":
+ add_repex_annotation_deleted(annotations, status, lei)
+ statement["annotations"] = annotations
+ return statement
+
+class GleifUpdates():
+ """GLEIF specific updates"""
+
+ def __init__(self):
+ self.already_voided = []
+ self.already_replaced = []
+
+ def void_entity_retired(self, latest_id, update_date, lei, status):
+ """Void entity statement for retired LEI"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_entity_statement("RETIRED", latest_id, status, update_date, "entityStatement", lei)
+
+ def void_entity_deletion(self, latest_id, update_date, lei, status):
+ """Void entity statement for deleted reporting exception"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_entity_statement("DELETION", latest_id, status, update_date, "entityStatement", lei, unknown=True)
+
+ def void_entity_changed(self, latest_id, update_date, entity_type, lei, status):
+ """Void entity statement for changed reporting exception"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_entity_statement("CHANGED", latest_id, status, update_date, entity_type, lei, unknown=True)
+
+ def void_ooc_relationship_deletion(self, latest_id, update_date, start, end):
+ """Void ownership or control statement for deleted relationship record"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_ooc_statement("RR_DELETION", latest_id, None, update_date, start, end)
+
+ def void_ooc_relationship_retired(self, latest_id, update_date, start, end):
+ """Void ownership or control statement for retired relationship record """
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_ooc_statement("RETIRED", latest_id, None, update_date, start, end)
+
+ def void_entity_replaced(self, latest_id, update_date, entity_type, lei, status):
+ """Void entity statement for replaced (by relationship) reporting exception"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_entity_statement("REPLACED", latest_id, status, update_date, entity_type, lei, unknown=True)
+
+ def void_ooc_exception_deletion(self, latest_id, update_date, lei, status):
+ """Void ownership or control statement for deleted reporting exception"""
+ if latest_id in self.already_voided or latest_id in self.already_replaced:
+ return None
+ else:
+ self.already_voided.append(latest_id)
+ return void_ooc_statement("REPEX_DELETION", latest_id, status, update_date, lei, None)
+
+ def add_replaces(self, statement, old_statement_id):
+ """Add replacesStatements to statement object"""
+ if old_statement_id in self.already_replaced:
+ return None
+ else:
+ self.already_replaced.append(old_statement_id)
+ statement["replacesStatements"] = [old_statement_id]
+ return statement
diff --git a/bodspipelines/pipelines/gleif/utils.py b/bodspipelines/pipelines/gleif/utils.py
index 70bf062..8f34cb8 100644
--- a/bodspipelines/pipelines/gleif/utils.py
+++ b/bodspipelines/pipelines/gleif/utils.py
@@ -1,15 +1,240 @@
-from bodspipelines.infrastructure.utils import download_delayed
+from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
-def get_source(r, name):
+from bodspipelines.infrastructure.utils import download_delayed, download
+
+def source_metadata(r):
+ """Get metadata from request"""
data = r.json()
- if name == 'lei':
- url = data['data']['lei2']['full_file']['xml']['url']
- elif name == 'rr':
- url = data['data']['rr']['full_file']['xml']['url']
- elif name == 'repex':
- url = data['data']['repex']['full_file']['xml']['url']
+ return data['data']
+
+
+def get_source(r, name):
+ """Extract source url from metadata"""
+ data = source_metadata(r)
+ url = data['full_file']['xml']['url']
print(f"Using: {url}")
return url
+def step_date(date, time, period_name, count):
+ """Step date forward based on delta file type"""
+ if period_name == 'IntraDay':
+ d = datetime.strptime(f"{date} {time}", "%Y%m%d %H%M")
+ delta = relativedelta(hours=-8*count)
+ return (d + delta).strftime("%Y%m%d"), (d + delta).strftime("%H%M")
+ else:
+ d = datetime.strptime(date, "%Y%m%d")
+ if period_name == "LastMonth":
+ delta = relativedelta(months=-1)
+ elif period_name == "LastWeek":
+ delta = relativedelta(days=-7)
+ elif period_name == "LastDay":
+ delta = relativedelta(days=-1)
+ return (d + delta).strftime("%Y%m%d"), time
+
+
+def delta_days(date1, date2):
+ """Days between two dates"""
+ delta = datetime.strptime(date2, "%Y%m%d") - datetime.strptime(date1, "%Y%m%d")
+ return delta.days
+
+
+def step_period(data, base, period_name, count):
+ """Step through count periods"""
+ for _ in range(count):
+ url = data['delta_files'][period_name]['xml']['url']
+ date, time, *_ = url.split('/')[-1].split('-')
+ date, time = step_date(date, time, period_name, 1)
+ request = download(f"{base}/{date}-{time}")
+ data = source_metadata(request)
+ yield url, data
+
+
+def get_sources_int(data, base, last_update):
+ """Get urls for sources (internal)"""
+ current_date = data['publish_date']
+ print(current_date, last_update)
+ delta = relativedelta(datetime.strptime(current_date, "%Y-%m-%d %H:%M:%S"),
+ datetime.strptime(last_update, "%Y-%m-%d %H:%M:%S"))
+ print(current_date, delta)
+ periods = {'months': 'LastMonth', 'days': 'LastDay', 'hours': 'IntraDay'}
+ done = 0
+ if getattr(delta, 'months') > 0:
+ for url, data in step_period(data, base, 'LastMonth', 1):
+ yield url
+ done = -1
+ for period in periods:
+ if done < 0:
+ if period == 'days':
+ if getattr(delta, period) > 0:
+ for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)*3):
+ yield url
+ elif period == 'hours':
+ if getattr(delta, period) > 0:
+ for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)//8):
+ yield url
+ else:
+ if getattr(delta, period) - done > 0:
+ date1, time, *_ = url.split('/')[-1].split('-')
+ date2, time = step_date(date1, time, 'LastMonth', getattr(delta, period) - done)
+ month_days = delta_days(date1, date2)
+ for url, data in step_period(data, base, 'IntraDay', month_days*3):
+ yield url
+ else:
+ if period == 'days':
+ if getattr(delta, period) > 6:
+ for url, data in step_period(data, base, 'LastWeek', getattr(delta, period)//7):
+ yield url
+ extra_days = getattr(delta, period)%7
+ else:
+ extra_days = getattr(delta, period)
+ if extra_days > 0:
+ for url, data in step_period(data, base, 'LastDay', extra_days):
+ yield url
+ elif period == 'hours':
+ if getattr(delta, period) > 0:
+ for url, data in step_period(data, base, 'IntraDay', getattr(delta, period)//8):
+ yield url
+ else:
+ if getattr(delta, period) > 0:
+ for url, data in step_period(data, base, periods[period], getattr(delta, period)):
+ yield url
+
+def get_sources(url, last_update):
+ """Get urls for sources"""
+ base = url.rsplit('/', 1)[0]
+ request = download(url)
+ data = source_metadata(request)
+ yield from get_sources_int(data, base, last_update)
+
+def get_data_type(url):
+ """Get type of data"""
+ if "/lei" in url:
+ data_type = "lei2"
+ elif "/rr" in url:
+ data_type = "rr"
+ elif "/repex" in url:
+ data_type = "repex"
+ return data_type
+
+def get_url_for_target_date(url, data, target_date, target_time, delta_type=None):
+ """Get url for specified date"""
+ for item in data:
+ item_date = datetime.strptime(item["publish_date"].split()[0], "%Y-%m-%d")
+ item_time = item["publish_date"].split()[1]
+ if item_date == target_date and item_time == target_time:
+ data_type = get_data_type(url)
+ if delta_type:
+ if delta_type == "month":
+ return item[data_type]["delta_files"]["LastMonth"]["xml"]["url"]
+ elif delta_type == "week":
+ return item[data_type]["delta_files"]["LastWeek"]["xml"]["url"]
+ elif delta_type == "day":
+ return item[data_type]["delta_files"]["LastDay"]["xml"]["url"]
+ else:
+ return item[data_type]["full_file"]["xml"]["url"]
+
+def find_urls_for_target_date(target_date):
+ """Find urls for specified date"""
+ page = 1
+ while True:
+ page_url = f"https://goldencopy.gleif.org/api/v2/golden-copies/publishes?page={page}&per_page=10"
+ response = download(page_url)
+ data = source_metadata(response)
+ end_date = datetime.strptime(data[0]["publish_date"].split()[0], "%Y-%m-%d")
+ start_date = datetime.strptime(data[9]["publish_date"].split()[0], "%Y-%m-%d")
+ if target_date < start_date:
+ delta = start_date - target_date
+ print(delta.days)
+ page = page + max(1, int(delta.days/3))
+ elif target_date > end_date:
+ delta = target_date - end_date
+ print(delta.days)
+ page = page - max(1, int(delta.days/3))
+ else:
+ return data, page
+
+def get_source_by_date(url, data_date, delta_type=None):
+ """Get source data url for specified date"""
+ target_date = datetime.strptime(data_date.split()[0], "%Y-%m-%d")
+ if len(data_date.split()) > 1 and ":" in data_date.split()[1]:
+ target_time = data_date.split()[1]
+ else:
+ target_time = "00:00:00"
+ if delta_type == "month":
+ target_date = target_date + timedelta(days=31)
+ elif delta_type == "week":
+ target_date = target_date + timedelta(days=7)
+ elif delta_type == "day":
+ target_date = target_date + timedelta(days=1)
+ data, page = find_urls_for_target_date(target_date)
+ target_url = get_url_for_target_date(url, data, target_date, target_time, delta_type=delta_type)
+ return target_url
+
+def stream_delta_file(item, target_date, target_time, data_type):
+ """Yield delta file"""
+ item_date = datetime.strptime(item["publish_date"].split()[0], "%Y-%m-%d")
+ item_time = item["publish_date"].split()[1]
+ if item_date > target_date or (item_date == target_date and
+ int(item_time.split(":")[0]) > int(target_time.split(":")[0])):
+ yield item[data_type]["delta_files"]["IntraDay"]["xml"]["url"]
+
+def stream_delta_files(url, data, target_date, target_time, page):
+ """Yield delta files"""
+ data_type = get_data_type(url)
+ for item in data:
+ yield from stream_delta_file(item, target_date, target_time, data_type)
+ page = page - 1
+ while page > 0:
+ page_url = f"https://goldencopy.gleif.org/api/v2/golden-copies/publishes?page={page}&per_page=10"
+ response = download(page_url)
+ data = source_metadata(response)
+ for item in data:
+ yield from stream_delta_file(item, target_date, target_time, data_type)
+ page = page - 1
+
+def get_source_from_date(url, data_date, delta_type=None):
+ """Get source data url for specified date"""
+ target_date = datetime.strptime(data_date.split()[0], "%Y-%m-%d")
+ if len(data_date.split()) > 1 and ":" in data_date.split()[1]:
+ target_time = data_date.split()[1]
+ else:
+ target_time = "00:00:00"
+ data, page = data, page = find_urls_for_target_date(target_date)
+ yield from stream_delta_files(url, data, target_date, target_time, page)
+
def gleif_download_link(url):
+ """Returns callable to download data"""
return download_delayed(url, get_source)
+
+class GLEIFData:
+ def __init__(self, url=None, data_date=None):
+ """Initialise with url"""
+ self.url = url
+ self.data_date = data_date
+
+ def sources(self, last_update=False, delta_type=None):
+ """Yield data sources"""
+ if last_update:
+ print("Updating ...")
+ if delta_type:
+ if delta_type == "stream":
+ yield from get_source_from_date(self.url, self.data_date, delta_type=delta_type)
+ else:
+ yield get_source_by_date(self.url, self.data_date, delta_type=delta_type)
+ else:
+ yield from get_sources(self.url, last_update)
+ else:
+ if self.data_date:
+ yield get_source_by_date(self.url, self.data_date)
+ else:
+ yield gleif_download_link(self.url)
+
+def identify_gleif(item):
+ """Identify type of GLEIF data"""
+ if 'Entity' in item:
+ return 'lei'
+ elif 'Relationship' in item:
+ return 'rr'
+ elif 'ExceptionCategory' in item:
+ return 'repex'
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..0170318
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+addopts = --docker-compose=tests/docker-compose.yml
diff --git a/requirements.txt b/requirements.txt
index 7c2b518..9fd8bcc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,20 @@
-boto3==1.26.66
-elasticsearch==7.17.9
-elastic_transport==7.16.0
+aiobotocore==2.13.1
+elasticsearch==8.12.1
+elastic_transport==8.12.0
+aiohttp==3.9.3
lxml==4.9.2
parsel==1.7.0
progress==1.6
-pytest==7.2.1
+pytest==7.2.2
+pytest-docker-compose-v2==0.1.1
+pytest-asyncio==0.21.1
python-dateutil==2.8.2
requests==2.28.2
tqdm==4.64.1
pytz==2022.7.1
pycountry==22.3.5
+redis==4.6.0
+aiofiles==24.1.0
+# Debugging
+loguru==0.7.2
+psutil==5.9.8
diff --git a/setup.py b/setup.py
index 2586564..34957c6 100644
--- a/setup.py
+++ b/setup.py
@@ -3,6 +3,7 @@
from setuptools.command.install import install
install_requires = [
+ "async-kinesis",
"boto3",
"elasticsearch",
"elastic_transport",
@@ -14,7 +15,12 @@
"requests",
"tqdm",
"pytz",
- "pycountry"
+ "pycountry",
+ "aiohttp",
+ "aiofiles",
+ "redis",
+ "psutil",
+ "loguru"
]
setup(
diff --git a/tests/config.py b/tests/config.py
new file mode 100644
index 0000000..b05ff28
--- /dev/null
+++ b/tests/config.py
@@ -0,0 +1,11 @@
+import os
+
+def set_environment_variables():
+ """Set environment variables"""
+ os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
+ os.environ['ELASTICSEARCH_HOST'] = 'localhost'
+ os.environ['ELASTICSEARCH_PORT'] = '9876'
+ os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
new file mode 100644
index 0000000..5d23eb3
--- /dev/null
+++ b/tests/docker-compose.yml
@@ -0,0 +1,13 @@
+version: '3.9'
+
+services:
+ bods_ingester_gleif_es:
+ image: docker.elastic.co/elasticsearch/elasticsearch:8.14.1
+ environment:
+ - 'discovery.type=single-node'
+ - 'cluster.name=gleif-elasticsearch'
+ - 'bootstrap.memory_lock=true'
+ - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
+ - 'xpack.security.enabled=false'
+ ports:
+ - 9200:9200
diff --git a/tests/fixtures/lei-data-out.json b/tests/fixtures/lei-data-out.json
new file mode 100644
index 0000000..637fff5
--- /dev/null
+++ b/tests/fixtures/lei-data-out.json
@@ -0,0 +1,262 @@
+[
+ {
+ "statementID": "04c98917-93f2-bd6d-1c9b-40411dc9b8b3",
+ "statementType": "entityStatement",
+ "statementDate": "2023-06-18",
+ "entityType": "registeredEntity",
+ "name": "Fidelity Advisor Leveraged Company Stock Fund",
+ "incorporatedInJurisdiction": {
+ "name": "US-MA",
+ "code": "US-MA"
+ },
+ "identifiers": [
+ {
+ "id": "001GPB6A9XPE8XJICC14",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "S000005113",
+ "schemeName": "RA000665"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "245 SUMMER STREET, BOSTON",
+ "country": "US",
+ "postCode": "02210"
+ },
+ {
+ "type": "business",
+ "address": "C/O Fidelity Management & Research Company LLC, Boston",
+ "country": "US",
+ "postCode": "02210"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-11-29T00:00:00.000Z"
+ },
+ {
+ "statementID": "41c4522b-f5e2-4f19-5f6b-ec5e54258401",
+ "statementType": "entityStatement",
+ "statementDate": "2022-08-22",
+ "entityType": "registeredEntity",
+ "name": "Vanguard Russell 1000 Growth Index Trust",
+ "incorporatedInJurisdiction": {
+ "name": "US-PA",
+ "code": "US-PA"
+ },
+ "identifiers": [
+ {
+ "id": "00EHHQ2ZHDCFXJCPCL46",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "schemeName": "RA999999"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "C/O VANGUARD FIDUCIARY TRUST COMPANY, MALVERN",
+ "country": "US",
+ "postCode": "19355"
+ },
+ {
+ "type": "business",
+ "address": "C/O Vanguard Fiduciary Trust Company, Valley Forge",
+ "country": "US",
+ "postCode": "19482"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-10-05T00:00:00.000Z"
+ },
+ {
+ "statementID": "ede4fef6-2aa2-8caa-b69a-330187c1750f",
+ "statementType": "entityStatement",
+ "statementDate": "2023-06-03",
+ "entityType": "registeredEntity",
+ "name": "PRUDENTIAL INVESTMENT PORTFOLIOS 18",
+ "incorporatedInJurisdiction": {
+ "name": "US-DE",
+ "code": "US-DE"
+ },
+ "identifiers": [
+ {
+ "id": "00QDBXDXLLF3W3JJJO36",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "2835527",
+ "schemeName": "RA000602"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "CORPORATION TRUST CENTER 1209 ORANGE ST, WILMINGTON",
+ "country": "US",
+ "postCode": "19801"
+ },
+ {
+ "type": "business",
+ "address": "655 Broad Street, Newark",
+ "country": "US",
+ "postCode": "07102"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1997-12-18T05:00:00.000Z"
+ },
+ {
+ "statementID": "96a93ace-30e2-016f-5ac1-6e88f320a1b6",
+ "statementType": "entityStatement",
+ "statementDate": "2020-08-17",
+ "entityType": "registeredEntity",
+ "name": "GE Pacific-3 Holdings, Inc.",
+ "incorporatedInJurisdiction": {
+ "name": "US-DE",
+ "code": "US-DE"
+ },
+ "identifiers": [
+ {
+ "id": "00W0SLGGVF0QQ5Q36N03",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "schemeName": "RA999999"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "C/O The Corporation Trust Company, Wilmington",
+ "country": "US",
+ "postCode": "19801"
+ },
+ {
+ "type": "business",
+ "address": "C/O The Corporation Trust Company, Wilmington",
+ "country": "US",
+ "postCode": "19801"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "11739b48-9df9-2d05-34aa-79c04d5cca06",
+ "statementType": "entityStatement",
+ "statementDate": "2023-03-02",
+ "entityType": "registeredEntity",
+ "name": "DENTSU INTERNATIONAL LIMITED",
+ "incorporatedInJurisdiction": {
+ "name": "United Kingdom",
+ "code": "GB"
+ },
+ "identifiers": [
+ {
+ "id": "213800FERQ5LE3H7WJ58",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "01403668",
+ "schemeName": "RA000585"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "10 TRITON STREET, LONDON",
+ "country": "GB",
+ "postCode": "NW1 3BF"
+ },
+ {
+ "type": "business",
+ "address": "10 TRITON STREET, LONDON",
+ "country": "GB",
+ "postCode": "NW1 3BF"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1978-12-05T00:00:00Z"
+ }
+]
diff --git a/tests/fixtures/lei-data.json b/tests/fixtures/lei-data.json
index 6779727..6b45e5c 100644
--- a/tests/fixtures/lei-data.json
+++ b/tests/fixtures/lei-data.json
@@ -1 +1 @@
-[{"LEI": "001GPB6A9XPE8XJICC14", "Entity": {"LegalName": "Fidelity Advisor Leveraged Company Stock Fund", "OtherEntityNames": ["FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"], "LegalAddress": {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC", "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-29T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-29T16:33:00.000Z", "LastUpdateDate": "2023-05-18T15:41:20.212Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-18T15:48:53.604Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"}}}, {"LEI": "004L5FPTUREIWK9T2N63", "Entity": {"LegalName": "Hutchin Hill Capital, LP", "LegalAddress": {"FirstAddressLine": "C/O Corporation Service Company", "AdditionalAddressLine": "Suite 400", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19808"}, "HeadquartersAddress": {"FirstAddressLine": "22nd Floor", "AdditionalAddressLine": "888 7th Avenue", "City": "New York", "Region": "US-NY", "Country": "US", "PostalCode": "10106"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "4386463"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "T91T"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2018-05-08T13:46:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "4386463"}}}, {"LEI": "00EHHQ2ZHDCFXJCPCL46", "Entity": {"LegalName": "Vanguard Russell 1000 Growth Index Trust", "LegalAddress": {"FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY", "AdditionalAddressLine": "100 VANGUARD BLVD", "City": "MALVERN", "Region": "US-PA", "Country": "US", "PostalCode": "19355"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Vanguard Fiduciary Trust Company", "AdditionalAddressLine": "PO Box 2600", "City": "Valley Forge", "Region": "US-PA", "Country": "US", "PostalCode": "19482"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-PA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-10-05T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-10-05T20:30:00.000Z", "LastUpdateDate": "2022-07-22T09:32:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-07-22T07:35:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00GBW0Z2GYIER7DHDS71", "Entity": {"LegalName": "ARISTEIA CAPITAL, L.L.C.", "LegalAddress": {"FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY", "AdditionalAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "3rd Floor", "AdditionalAddressLine": "One Greenwich Plaza", "City": "Greenwich", "Region": "US-CT", "Country": "US", "PostalCode": "06830"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2758229"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "HZEH"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-03T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:55:00.000Z", "LastUpdateDate": "2022-10-24T21:31:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-09-23T19:58:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "2758229"}}}, {"LEI": "00KLB2PFTM3060S2N216", "Entity": {"LegalName": "Oakmark International Fund", "OtherEntityNames": ["HARRIS ASSOCIATES INVESTMENT TRUST - Oakmark International Fund"], "LegalAddress": {"FirstAddressLine": "155 FEDERAL ST., SUITE 700", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02110"}, "HeadquartersAddress": {"FirstAddressLine": "111 South Wacker Drive", "City": "Chicago", "Region": "US-IL", "Country": "US", "PostalCode": "60606-4319"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000002762"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-04T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-05T18:17:00.000Z", "LastUpdateDate": "2023-05-18T17:24:00.540Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-11T18:37:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000002762"}}}, {"LEI": "00QDBXDXLLF3W3JJJO36", "Entity": {"LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", "OtherEntityNames": ["Jennison 20/20 Focus Fund", "Prudential 20/20 Focus Fund", "Prudential 20/20 Fund", "Prudential Jennison 20/20 Focus Fund"], "LegalAddress": {"FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "655 Broad Street", "City": "Newark", "Region": "US-NJ", "Country": "US", "PostalCode": "07102"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2835527"}, "LegalJurisdiction": "US-DE", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "4FSX"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-12-18T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:58:00.000Z", "LastUpdateDate": "2023-05-03T07:03:05.620Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-04-21T15:23:48.020Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "0001052118"}}}, {"LEI": "00TR8NKAEL48RGTZEW89", "Entity": {"LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", "LegalAddress": {"FirstAddressLine": "89 Nexus Way", "City": "Camana Bay", "Country": "KY", "PostalCode": "KY1-9009"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", "AdditionalAddressLine": "Kaya Flamboyan 9", "City": "Willemstad", "Country": "CW"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000087", "RegistrationAuthorityEntityID": "574206"}, "LegalJurisdiction": "KY", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"}, "EntityStatus": "INACTIVE", "LegalEntityEvents": [{"LegalEntityEventType": "DISSOLUTION", "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000Z", "LegalEntityEventRecordedDate": "2022-03-24T19:41:39.000Z", "ValidationDocuments": "SUPPORTING_DOCUMENTS"}]}, "Registration": {"InitialRegistrationDate": "2012-08-24T18:53:00.000Z", "LastUpdateDate": "2019-04-22T21:31:00.000Z", "RegistrationStatus": "RETIRED", "NextRenewalDate": "2019-02-14T00:32:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000087", "ValidationAuthorityEntityID": "574206"}}}, {"LEI": "00TV1D5YIV5IDUGWBW29", "Entity": {"LegalName": "Ceridian Corporation Retirement Plan Trust", "LegalAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "TRUST"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-13T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-14T19:43:00.000Z", "LastUpdateDate": "2023-05-10T04:42:18.790Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-20T00:00:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00W0SLGGVF0QQ5Q36N03", "Entity": {"LegalName": "GE Pacific-3 Holdings, Inc.", "LegalAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XTIQ"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2016-01-12T15:45:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00X5RQKJQQJFFX0WPA53", "Entity": {"LegalName": "The Greater Morristown Young Men's Christian Association, Inc.", "OtherEntityNames": ["The Greater Morristown YMCA Inc"], "LegalAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "HeadquartersAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-NJ", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XSNP"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-11-22T10:55:00.000Z", "LastUpdateDate": "2020-07-24T19:29:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2013-11-22T10:38:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}]
+[{"LEI": "001GPB6A9XPE8XJICC14", "Entity": {"LegalName": "Fidelity Advisor Leveraged Company Stock Fund", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"}], "LegalAddress": {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC", "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-29T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-29T16:33:00.000Z", "LastUpdateDate": "2023-05-18T15:41:20.212Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-18T15:48:53.604Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"}}}, {"LEI": "004L5FPTUREIWK9T2N63", "Entity": {"LegalName": "Hutchin Hill Capital, LP", "LegalAddress": {"FirstAddressLine": "C/O Corporation Service Company", "AdditionalAddressLine": "Suite 400", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19808"}, "HeadquartersAddress": {"FirstAddressLine": "22nd Floor", "AdditionalAddressLine": "888 7th Avenue", "City": "New York", "Region": "US-NY", "Country": "US", "PostalCode": "10106"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "4386463"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "T91T"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2018-05-08T13:46:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "4386463"}}}, {"LEI": "00EHHQ2ZHDCFXJCPCL46", "Entity": {"LegalName": "Vanguard Russell 1000 Growth Index Trust", "LegalAddress": {"FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY", "AdditionalAddressLine": "100 VANGUARD BLVD", "City": "MALVERN", "Region": "US-PA", "Country": "US", "PostalCode": "19355"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Vanguard Fiduciary Trust Company", "AdditionalAddressLine": "PO Box 2600", "City": "Valley Forge", "Region": "US-PA", "Country": "US", "PostalCode": "19482"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-PA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-10-05T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-10-05T20:30:00.000Z", "LastUpdateDate": "2022-07-22T09:32:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-07-22T07:35:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00GBW0Z2GYIER7DHDS71", "Entity": {"LegalName": "ARISTEIA CAPITAL, L.L.C.", "LegalAddress": {"FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY", "AdditionalAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "3rd Floor", "AdditionalAddressLine": "One Greenwich Plaza", "City": "Greenwich", "Region": "US-CT", "Country": "US", "PostalCode": "06830"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2758229"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "HZEH"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-03T00:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:55:00.000Z", "LastUpdateDate": "2022-10-24T21:31:00.000Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2023-09-23T19:58:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000602", "ValidationAuthorityEntityID": "2758229"}}}, {"LEI": "00KLB2PFTM3060S2N216", "Entity": {"LegalName": "Oakmark International Fund", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "HARRIS ASSOCIATES INVESTMENT TRUST - Oakmark International Fund"}], "LegalAddress": {"FirstAddressLine": "155 FEDERAL ST., SUITE 700", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02110"}, "HeadquartersAddress": {"FirstAddressLine": "111 South Wacker Drive", "City": "Chicago", "Region": "US-IL", "Country": "US", "PostalCode": "60606-4319"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000002762"}, "LegalJurisdiction": "US-MA", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-04T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-05T18:17:00.000Z", "LastUpdateDate": "2023-05-18T17:24:00.540Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-11T18:37:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000002762"}}}, {"LEI": "00QDBXDXLLF3W3JJJO36", "Entity": {"LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Jennison 20/20 Focus Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential 20/20 Focus Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential 20/20 Fund"}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "Prudential Jennison 20/20 Focus Fund"}], "LegalAddress": {"FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST", "City": "WILMINGTON", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "655 Broad Street", "City": "Newark", "Region": "US-NJ", "Country": "US", "PostalCode": "07102"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000602", "RegistrationAuthorityEntityID": "2835527"}, "LegalJurisdiction": "US-DE", "EntityCategory": "FUND", "LegalForm": {"EntityLegalFormCode": "4FSX"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-12-18T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:58:00.000Z", "LastUpdateDate": "2023-05-03T07:03:05.620Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-04-21T15:23:48.020Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "0001052118"}}}, {"LEI": "00TR8NKAEL48RGTZEW89", "Entity": {"LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.", "LegalAddress": {"FirstAddressLine": "89 Nexus Way", "City": "Camana Bay", "Country": "KY", "PostalCode": "KY1-9009"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.", "AdditionalAddressLine": "Kaya Flamboyan 9", "City": "Willemstad", "Country": "CW"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000087", "RegistrationAuthorityEntityID": "574206"}, "LegalJurisdiction": "KY", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"}, "EntityStatus": "INACTIVE", "LegalEntityEvents": [{"LegalEntityEventType": "DISSOLUTION", "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000Z", "LegalEntityEventRecordedDate": "2022-03-24T19:41:39.000Z", "ValidationDocuments": "SUPPORTING_DOCUMENTS"}]}, "Registration": {"InitialRegistrationDate": "2012-08-24T18:53:00.000Z", "LastUpdateDate": "2019-04-22T21:31:00.000Z", "RegistrationStatus": "RETIRED", "NextRenewalDate": "2019-02-14T00:32:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000087", "ValidationAuthorityEntityID": "574206"}}}, {"LEI": "00TV1D5YIV5IDUGWBW29", "Entity": {"LegalName": "Ceridian Corporation Retirement Plan Trust", "LegalAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "RL&F Service Corp.", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "TRUST"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2012-11-13T05:00:00.000Z"}, "Registration": {"InitialRegistrationDate": "2012-11-14T19:43:00.000Z", "LastUpdateDate": "2023-05-10T04:42:18.790Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-20T00:00:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00W0SLGGVF0QQ5Q36N03", "Entity": {"LegalName": "GE Pacific-3 Holdings, Inc.", "LegalAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "HeadquartersAddress": {"FirstAddressLine": "C/O The Corporation Trust Company", "AdditionalAddressLine": "1209 Orange Street", "City": "Wilmington", "Region": "US-DE", "Country": "US", "PostalCode": "19801"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-DE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XTIQ"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-06-06T15:56:00.000Z", "LastUpdateDate": "2020-07-17T12:40:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2016-01-12T15:45:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "00X5RQKJQQJFFX0WPA53", "Entity": {"LegalName": "The Greater Morristown Young Men's Christian Association, Inc.", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "The Greater Morristown YMCA Inc"}], "LegalAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "HeadquartersAddress": {"FirstAddressLine": "79 Horsehill Road", "City": "Cedar Knolls", "Region": "US-NJ", "Country": "US", "PostalCode": "07927-2003"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA999999"}, "LegalJurisdiction": "US-NJ", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XSNP"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2012-11-22T10:55:00.000Z", "LastUpdateDate": "2020-07-24T19:29:00.000Z", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2013-11-22T10:38:00.000Z", "ManagingLOU": "EVK05KS7XY1DEII3R011", "ValidationSources": "ENTITY_SUPPLIED_ONLY", "ValidationAuthority": {"ValidationAuthorityID": "RA999999"}}}, {"LEI": "1595D0QCK7Y15293JK84", "Entity": {"LegalName": "GALAPAGOS CONSERVATION TRUST", "LegalAddress": {"FirstAddressLine": "7-14 Great Dover Street", "City": "London", "Country": "GB", "PostalCode": "SE1 4YR"}, "HeadquartersAddress": {"FirstAddressLine": "7-14 Great Dover Street", "City": "London", "Country": "GB", "PostalCode": "SE1 4YR"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000585", "RegistrationAuthorityEntityID": "03004112"}, "LegalJurisdiction": "GB", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "G12F"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1994-12-21T00:00:00+01:00"}, "Registration": {"InitialRegistrationDate": "2023-02-13T22:13:11+01:00", "LastUpdateDate": "2023-03-10T13:08:56+01:00", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-02-13T22:13:11+01:00", "ManagingLOU": "98450045AN5EB5FDC780", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000585", "ValidationAuthorityEntityID": "03004112"}, "OtherValidationAuthorities": [{"ValidationAuthorityID": "RA000589", "ValidationAuthorityEntityID": "1043470"}]}}, {"LEI": "213800FERQ5LE3H7WJ58", "Entity": {"LegalName": "DENTSU INTERNATIONAL LIMITED", "OtherEntityNames": [{"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "DENTSU AEGIS NETWORK LTD."}, {"type": "PREVIOUS_LEGAL_NAME", "OtherEntityName": "AEGIS GROUP PLC"}], "LegalAddress": {"FirstAddressLine": "10 TRITON STREET", "AdditionalAddressLine": "REGENT'S PLACE", "City": "LONDON", "Region": "GB-LND", "Country": "GB", "PostalCode": "NW1 3BF"}, "HeadquartersAddress": {"FirstAddressLine": "10 TRITON STREET", "AdditionalAddressLine": "REGENT'S PLACE", "City": "LONDON", "Region": "GB-LND", "Country": "GB", "PostalCode": "NW1 3BF"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000585", "RegistrationAuthorityEntityID": "01403668"}, "LegalJurisdiction": "GB", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "H0PO"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1978-12-05T00:00:00Z"}, "Registration": {"InitialRegistrationDate": "2014-02-10T00:00:00Z", "LastUpdateDate": "2023-02-02T09:07:52.390Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-02-17T00:00:00Z", "ManagingLOU": "213800WAVVOPS85N2205", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000585", "ValidationAuthorityEntityID": "01403668"}}}, {"LEI": "213800BJPX8V9HVY1Y11", "Entity": {"LegalName": "Swedeit Italian Aktiebolag", "LegalAddress": {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "V\u00e4stra Fr\u00f6lunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76"}, "HeadquartersAddress": {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "V\u00e4stra Fr\u00f6lunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76"}, "TransliteratedOtherAddresses": [{"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "Vastra Frolunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76", "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS"}, {"FirstAddressLine": "C/O Anita Lindberg", "MailRouting": "C/O Anita Lindberg", "AdditionalAddressLine": "Fortgatan 11", "City": "Vastra Frolunda", "Region": "SE-O", "Country": "SE", "PostalCode": "426 76", "type": "AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS"}], "RegistrationAuthority": {"RegistrationAuthorityID": "RA000544", "RegistrationAuthorityEntityID": "556543-1193"}, "LegalJurisdiction": "SE", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "XJHM"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1997-06-05T02:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2014-04-09T00:00:00Z", "LastUpdateDate": "2023-04-25T13:18:00Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-05-12T06:59:39Z", "ManagingLOU": "549300O897ZC5H7CY412", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000544", "ValidationAuthorityEntityID": "556543-1193"}}}]
diff --git a/tests/fixtures/lei-data.xml b/tests/fixtures/lei-data.xml
index 0f62498..bc84d35 100644
--- a/tests/fixtures/lei-data.xml
+++ b/tests/fixtures/lei-data.xml
@@ -5,7 +5,6 @@
2394586
-
@@ -762,6 +761,178 @@
USA
+
+
+ 1595D0QCK7Y15293JK84
+
+ GALAPAGOS CONSERVATION TRUST
+
+ 7-14 Great Dover Street
+ London
+ GB
+ SE1 4YR
+
+
+ 7-14 Great Dover Street
+ London
+ GB
+ SE1 4YR
+
+
+ RA000585
+ 03004112
+
+ GB
+ GENERAL
+
+ G12F
+
+ ACTIVE
+ 1994-12-21T00:00:00+01:00
+
+
+ 2023-02-13T22:13:11+01:00
+ 2023-03-10T13:08:56+01:00
+ ISSUED
+ 2024-02-13T22:13:11+01:00
+ 98450045AN5EB5FDC780
+ FULLY_CORROBORATED
+
+ RA000585
+ 03004112
+
+
+
+ RA000589
+ 1043470
+
+
+
+
+
+
+ 213800FERQ5LE3H7WJ58
+
+ DENTSU INTERNATIONAL LIMITED
+
+ DENTSU AEGIS NETWORK LTD.
+ AEGIS GROUP PLC
+
+
+ 10 TRITON STREET
+ REGENT'S PLACE
+ LONDON
+ GB-LND
+ GB
+ NW1 3BF
+
+
+ 10 TRITON STREET
+ REGENT'S PLACE
+ LONDON
+ GB-LND
+ GB
+ NW1 3BF
+
+
+ RA000585
+ 01403668
+
+ GB
+ GENERAL
+
+ H0PO
+
+ ACTIVE
+ 1978-12-05T00:00:00Z
+
+
+ 2014-02-10T00:00:00Z
+ 2023-02-02T09:07:52.390Z
+ ISSUED
+ 2024-02-17T00:00:00Z
+ 213800WAVVOPS85N2205
+ FULLY_CORROBORATED
+
+ RA000585
+ 01403668
+
+
+
+
+
+ 213800BJPX8V9HVY1Y11
+
+ Swedeit Italian Aktiebolag
+
+ C/O Anita Lindberg
+ C/O Anita Lindberg
+ Fortgatan 11
+ Västra Frölunda
+ SE-O
+ SE
+ 426 76
+
+
+ C/O Anita Lindberg
+ C/O Anita Lindberg
+ Fortgatan 11
+ Västra Frölunda
+ SE-O
+ SE
+ 426 76
+
+
+
+ C/O Anita Lindberg
+ C/O Anita Lindberg
+ Fortgatan 11
+ Vastra Frolunda
+ SE-O
+ SE
+ 426 76
+
+
+ C/O Anita Lindberg
+ C/O Anita Lindberg
+ Fortgatan 11
+ Vastra Frolunda
+ SE-O
+ SE
+ 426 76
+
+
+
+ RA000544
+ 556543-1193
+
+ SE
+ GENERAL
+
+ XJHM
+
+ ACTIVE
+ 1997-06-05T02:00:00+02:00
+
+
+ 2014-04-09T00:00:00Z
+ 2023-04-25T13:18:00Z
+ ISSUED
+ 2024-05-12T06:59:39Z
+ 549300O897ZC5H7CY412
+ FULLY_CORROBORATED
+
+ RA000544
+ 556543-1193
+
+
+
diff --git a/tests/fixtures/lei-updates-data-out.json b/tests/fixtures/lei-updates-data-out.json
new file mode 100644
index 0000000..637fff5
--- /dev/null
+++ b/tests/fixtures/lei-updates-data-out.json
@@ -0,0 +1,262 @@
+[
+ {
+ "statementID": "04c98917-93f2-bd6d-1c9b-40411dc9b8b3",
+ "statementType": "entityStatement",
+ "statementDate": "2023-06-18",
+ "entityType": "registeredEntity",
+ "name": "Fidelity Advisor Leveraged Company Stock Fund",
+ "incorporatedInJurisdiction": {
+ "name": "US-MA",
+ "code": "US-MA"
+ },
+ "identifiers": [
+ {
+ "id": "001GPB6A9XPE8XJICC14",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "S000005113",
+ "schemeName": "RA000665"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "245 SUMMER STREET, BOSTON",
+ "country": "US",
+ "postCode": "02210"
+ },
+ {
+ "type": "business",
+ "address": "C/O Fidelity Management & Research Company LLC, Boston",
+ "country": "US",
+ "postCode": "02210"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-11-29T00:00:00.000Z"
+ },
+ {
+ "statementID": "41c4522b-f5e2-4f19-5f6b-ec5e54258401",
+ "statementType": "entityStatement",
+ "statementDate": "2022-08-22",
+ "entityType": "registeredEntity",
+ "name": "Vanguard Russell 1000 Growth Index Trust",
+ "incorporatedInJurisdiction": {
+ "name": "US-PA",
+ "code": "US-PA"
+ },
+ "identifiers": [
+ {
+ "id": "00EHHQ2ZHDCFXJCPCL46",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "schemeName": "RA999999"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "C/O VANGUARD FIDUCIARY TRUST COMPANY, MALVERN",
+ "country": "US",
+ "postCode": "19355"
+ },
+ {
+ "type": "business",
+ "address": "C/O Vanguard Fiduciary Trust Company, Valley Forge",
+ "country": "US",
+ "postCode": "19482"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-10-05T00:00:00.000Z"
+ },
+ {
+ "statementID": "ede4fef6-2aa2-8caa-b69a-330187c1750f",
+ "statementType": "entityStatement",
+ "statementDate": "2023-06-03",
+ "entityType": "registeredEntity",
+ "name": "PRUDENTIAL INVESTMENT PORTFOLIOS 18",
+ "incorporatedInJurisdiction": {
+ "name": "US-DE",
+ "code": "US-DE"
+ },
+ "identifiers": [
+ {
+ "id": "00QDBXDXLLF3W3JJJO36",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "2835527",
+ "schemeName": "RA000602"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "CORPORATION TRUST CENTER 1209 ORANGE ST, WILMINGTON",
+ "country": "US",
+ "postCode": "19801"
+ },
+ {
+ "type": "business",
+ "address": "655 Broad Street, Newark",
+ "country": "US",
+ "postCode": "07102"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1997-12-18T05:00:00.000Z"
+ },
+ {
+ "statementID": "96a93ace-30e2-016f-5ac1-6e88f320a1b6",
+ "statementType": "entityStatement",
+ "statementDate": "2020-08-17",
+ "entityType": "registeredEntity",
+ "name": "GE Pacific-3 Holdings, Inc.",
+ "incorporatedInJurisdiction": {
+ "name": "US-DE",
+ "code": "US-DE"
+ },
+ "identifiers": [
+ {
+ "id": "00W0SLGGVF0QQ5Q36N03",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "schemeName": "RA999999"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "C/O The Corporation Trust Company, Wilmington",
+ "country": "US",
+ "postCode": "19801"
+ },
+ {
+ "type": "business",
+ "address": "C/O The Corporation Trust Company, Wilmington",
+ "country": "US",
+ "postCode": "19801"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "11739b48-9df9-2d05-34aa-79c04d5cca06",
+ "statementType": "entityStatement",
+ "statementDate": "2023-03-02",
+ "entityType": "registeredEntity",
+ "name": "DENTSU INTERNATIONAL LIMITED",
+ "incorporatedInJurisdiction": {
+ "name": "United Kingdom",
+ "code": "GB"
+ },
+ "identifiers": [
+ {
+ "id": "213800FERQ5LE3H7WJ58",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "01403668",
+ "schemeName": "RA000585"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "10 TRITON STREET, LONDON",
+ "country": "GB",
+ "postCode": "NW1 3BF"
+ },
+ {
+ "type": "business",
+ "address": "10 TRITON STREET, LONDON",
+ "country": "GB",
+ "postCode": "NW1 3BF"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-06",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1978-12-05T00:00:00Z"
+ }
+]
diff --git a/tests/fixtures/lei-updates-data.json b/tests/fixtures/lei-updates-data.json
new file mode 100644
index 0000000..0f43884
--- /dev/null
+++ b/tests/fixtures/lei-updates-data.json
@@ -0,0 +1,254 @@
+[
+ {
+ "LEI": "001GPB6A9XPE8XJICC14",
+ "Entity": {
+ "LegalName": "Fidelity Advisor Leveraged Company Stock Fund",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "245 SUMMER STREET",
+ "City": "BOSTON",
+ "Region": "US-MA",
+ "Country": "US",
+ "PostalCode": "02210"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O Fidelity Management & Research Company LLC",
+ "City": "Boston",
+ "Region": "US-MA",
+ "Country": "US",
+ "PostalCode": "02210"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000665",
+ "RegistrationAuthorityEntityID": "S000005113"
+ },
+ "LegalJurisdiction": "US-MA",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "8888",
+ "OtherLegalForm": "FUND"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2012-11-29T00:00:00.000Z"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-11-29T16:33:00.000Z",
+ "LastUpdateDate": "2023-06-18T15:41:20.212Z",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-05-18T15:48:53.604Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000665",
+ "ValidationAuthorityEntityID": "S000005113"
+ }
+ }
+ },
+ {
+ "LEI": "00EHHQ2ZHDCFXJCPCL46",
+ "Entity": {
+ "LegalName": "Vanguard Russell 1000 Growth Index Trust",
+ "LegalAddress": {
+ "FirstAddressLine": "C/O VANGUARD FIDUCIARY TRUST COMPANY",
+ "AdditionalAddressLine": "100 VANGUARD BLVD",
+ "City": "MALVERN",
+ "Region": "US-PA",
+ "Country": "US",
+ "PostalCode": "19355"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O Vanguard Fiduciary Trust Company",
+ "AdditionalAddressLine": "PO Box 2600",
+ "City": "Valley Forge",
+ "Region": "US-PA",
+ "Country": "US",
+ "PostalCode": "19482"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA999999"
+ },
+ "LegalJurisdiction": "US-PA",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "8888",
+ "OtherLegalForm": "FUND"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2012-10-05T00:00:00.000Z"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-10-05T20:30:00.000Z",
+ "LastUpdateDate": "2022-08-22T09:32:00.000Z",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2023-07-22T07:35:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "ENTITY_SUPPLIED_ONLY",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA999999"
+ }
+ }
+ },
+ {
+ "LEI": "00QDBXDXLLF3W3JJJO36",
+ "Entity": {
+ "LegalName": "PRUDENTIAL INVESTMENT PORTFOLIOS 18",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "Jennison 20/20 Focus Fund"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "Prudential 20/20 Focus Fund"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "Prudential 20/20 Fund"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "Prudential Jennison 20/20 Focus Fund"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "CORPORATION TRUST CENTER 1209 ORANGE ST",
+ "City": "WILMINGTON",
+ "Region": "US-DE",
+ "Country": "US",
+ "PostalCode": "19801"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "655 Broad Street",
+ "City": "Newark",
+ "Region": "US-NJ",
+ "Country": "US",
+ "PostalCode": "07102"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000602",
+ "RegistrationAuthorityEntityID": "2835527"
+ },
+ "LegalJurisdiction": "US-DE",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "4FSX"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "1997-12-18T05:00:00.000Z"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-06-06T15:58:00.000Z",
+ "LastUpdateDate": "2023-06-03T07:03:05.620Z",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-04-21T15:23:48.020Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000665",
+ "ValidationAuthorityEntityID": "0001052118"
+ }
+ }
+ },
+ {
+ "LEI": "00W0SLGGVF0QQ5Q36N03",
+ "Entity": {
+ "LegalName": "GE Pacific-3 Holdings, Inc.",
+ "LegalAddress": {
+ "FirstAddressLine": "C/O The Corporation Trust Company",
+ "AdditionalAddressLine": "1209 Orange Street",
+ "City": "Wilmington",
+ "Region": "US-DE",
+ "Country": "US",
+ "PostalCode": "19801"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O The Corporation Trust Company",
+ "AdditionalAddressLine": "1209 Orange Street",
+ "City": "Wilmington",
+ "Region": "US-DE",
+ "Country": "US",
+ "PostalCode": "19801"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA999999"
+ },
+ "LegalJurisdiction": "US-DE",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "XTIQ"
+ },
+ "EntityStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-06-06T15:56:00.000Z",
+ "LastUpdateDate": "2020-08-17T12:40:00.000Z",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2016-01-12T15:45:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "ENTITY_SUPPLIED_ONLY",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA999999"
+ }
+ }
+ },
+ {
+ "LEI": "213800FERQ5LE3H7WJ58",
+ "Entity": {
+ "LegalName": "DENTSU INTERNATIONAL LIMITED",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "DENTSU AEGIS NETWORK LTD."
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "AEGIS GROUP PLC"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "10 TRITON STREET",
+ "AdditionalAddressLine": "REGENT'S PLACE",
+ "City": "LONDON",
+ "Region": "GB-LND",
+ "Country": "GB",
+ "PostalCode": "NW1 3BF"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "10 TRITON STREET",
+ "AdditionalAddressLine": "REGENT'S PLACE",
+ "City": "LONDON",
+ "Region": "GB-LND",
+ "Country": "GB",
+ "PostalCode": "NW1 3BF"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000585",
+ "RegistrationAuthorityEntityID": "01403668"
+ },
+ "LegalJurisdiction": "GB",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "H0PO"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "1978-12-05T00:00:00Z"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-02-10T00:00:00Z",
+ "LastUpdateDate": "2023-03-02T09:07:52.390Z",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-02-17T00:00:00Z",
+ "ManagingLOU": "213800WAVVOPS85N2205",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000585",
+ "ValidationAuthorityEntityID": "01403668"
+ }
+ }
+ }
+]
diff --git a/tests/fixtures/lei-updates-data2-new-out.json b/tests/fixtures/lei-updates-data2-new-out.json
new file mode 100644
index 0000000..24ef4d2
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2-new-out.json
@@ -0,0 +1,636 @@
+[
+ {
+ "statementID": "179355f9-6d62-3ccc-1779-f09db959a042",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-12",
+ "entityType": "registeredEntity",
+ "name": "Delaware Ivy Value Fund",
+ "incorporatedInJurisdiction": {
+ "name": "United States",
+ "code": "US"
+ },
+ "identifiers": [
+ {
+ "id": "ZZZWS8Y6XVKM0Q9RJQ79",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "S000024832",
+ "schemeName": "RA000665"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "C/O THE CORPORATION TRUST COMPANY, WILMINGTON",
+ "country": "US",
+ "postCode": "19801"
+ },
+ {
+ "type": "business",
+ "address": "C/O Delaware Management Company, Inc., WILMINGTON",
+ "country": "US",
+ "postCode": "19808"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-06-06T00:00:00.000Z"
+ },
+ {
+ "statementID": "68b6a4dc-cc3d-3c8d-d44c-47b29ab0935c",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-03",
+ "entityType": "registeredEntity",
+ "name": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.",
+ "incorporatedInJurisdiction": {
+ "name": "Cayman Islands",
+ "code": "KY"
+ },
+ "identifiers": [
+ {
+ "id": "00TR8NKAEL48RGTZEW89",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "574206",
+ "schemeName": "RA000087"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "89 Nexus Way, Camana Bay",
+ "country": "KY",
+ "postCode": "KY1-9009"
+ },
+ {
+ "type": "business",
+ "address": "C/O Citco Fund Services (Curacao) B.V., Willemstad",
+ "country": "CW"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "296865b4-00a0-2f0e-47ac-c20037da1f30",
+ "statementType": "entityStatement",
+ "statementDate": "2023-06-25",
+ "entityType": "registeredEntity",
+ "name": "VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BHF50000076475",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "35903058",
+ "schemeName": "RA000526"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2004-10-09T08:00:00+02:00"
+ },
+ {
+ "statementID": "d400beea-3cdc-a894-ce0d-9ce8fa67c30b",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-17",
+ "entityType": "registeredEntity",
+ "name": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002337",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB04",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-23T08:00:00+01:00"
+ },
+ {
+ "statementID": "19fb3b00-c801-7ff9-fae7-a1955c772e71",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-17",
+ "entityType": "registeredEntity",
+ "name": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002143",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB01",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-04-02T08:00:00+02:00"
+ },
+ {
+ "statementID": "1ff23b39-62f3-4744-da92-b11c2c3f8ff3",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-17",
+ "entityType": "registeredEntity",
+ "name": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002240",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB02",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-22T08:00:00+01:00"
+ },
+ {
+ "statementID": "a8866f73-572f-5ec5-6b42-2fe78b72f128",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-17",
+ "entityType": "registeredEntity",
+ "name": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002434",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB03",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-22T08:00:00+01:00"
+ },
+ {
+ "statementID": "426b5ddd-267f-0020-ae05-5dbc1e0f3341",
+ "statementType": "entityStatement",
+ "statementDate": "2023-10-12",
+ "entityType": "registeredEntity",
+ "name": "Všeobecná úverová banka, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "549300JB1P61FUTPEZ75",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "31320155",
+ "schemeName": "RA000526"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "829 90"
+ },
+ {
+ "type": "business",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "829 90"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1992-04-01T08:00:00+02:00"
+ },
+ {
+ "statementID": "42b2a5f0-c7ff-a8c8-fd77-9b0292530830",
+ "statementType": "entityStatement",
+ "statementDate": "2023-02-25",
+ "entityType": "registeredEntity",
+ "name": "Corporate Business Solutions Partners S.à r.l.",
+ "incorporatedInJurisdiction": {
+ "name": "Luxembourg",
+ "code": "LU"
+ },
+ "identifiers": [
+ {
+ "id": "9845002B9874D50A4531",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "B191847",
+ "schemeName": "RA000432"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "23 Am Scheerleck, Wecker",
+ "country": "LU",
+ "postCode": "6868"
+ },
+ {
+ "type": "business",
+ "address": "23 Am Scheerleck, Wecker",
+ "country": "LU",
+ "postCode": "6868"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2014-11-17T00:00:00+00:00"
+ },
+ {
+ "statementID": "657a9791-1cf0-e35c-a65f-0785e4685749",
+ "statementType": "entityStatement",
+ "statementDate": "2023-02-11",
+ "entityType": "registeredEntity",
+ "name": "MONOOVA GLOBAL PAYMENTS PTY LTD",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "2138002YAB7Z9KO21397",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "106 249 852",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "UNIT L7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2003-09-09T00:00:00+00:00"
+ },
+ {
+ "statementID": "8304570e-5bbf-d8df-1674-51f9d2a1f354",
+ "statementType": "entityStatement",
+ "statementDate": "2022-03-23",
+ "entityType": "registeredEntity",
+ "name": "MONEYTECH GROUP LIMITED",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "984500501A1B1045PB30",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "611 393 554",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2016-03-18T00:00:00+00:00"
+ },
+ {
+ "statementID": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81",
+ "statementType": "entityStatement",
+ "statementDate": "2023-04-30",
+ "entityType": "registeredEntity",
+ "name": "MONOOVA FX PTY LTD",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "98450051BS9C610A8T78",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "151 337 852",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "LEVEL 7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "LEVEL 7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2011-06-07T00:00:00+00:00"
+ }
+]
diff --git a/tests/fixtures/lei-updates-data2-new.json b/tests/fixtures/lei-updates-data2-new.json
new file mode 100644
index 0000000..b60903e
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2-new.json
@@ -0,0 +1,629 @@
+[
+ {
+ "LEI": "ZZZWS8Y6XVKM0Q9RJQ79",
+ "Entity": {
+ "LegalName": "Delaware Ivy Value Fund",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "IVY FUNDS - Delaware Ivy Value Fund"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "IVY FUNDS - Ivy Value Fund"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "C/O THE CORPORATION TRUST COMPANY",
+ "AdditionalAddressLine": "1209 ORANGE ST",
+ "City": "WILMINGTON",
+ "Region": "US-DE",
+ "Country": "US",
+ "PostalCode": "19801"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O Delaware Management Company, Inc.",
+ "AdditionalAddressLine": "251 LITTLE FALLS DRIVE",
+ "City": "WILMINGTON",
+ "Region": "US-DE",
+ "Country": "US",
+ "PostalCode": "19808"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000665",
+ "RegistrationAuthorityEntityID": "S000024832"
+ },
+ "LegalJurisdiction": "US",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "8888",
+ "OtherLegalForm": "FUND"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2012-06-06T00:00:00.000Z"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-06-06T15:56:00.000Z",
+ "LastUpdateDate": "2023-09-12T20:51:51.031Z",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-12T20:51:51.032Z",
+ "ManagingLOU": "5493001KJTIIGC8Y1R12",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000665",
+ "ValidationAuthorityEntityID": "S000024832"
+ }
+ }
+ },
+ {
+ "LEI": "00TR8NKAEL48RGTZEW89",
+ "Entity": {
+ "LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.",
+ "LegalAddress": {
+ "FirstAddressLine": "89 Nexus Way",
+ "City": "Camana Bay",
+ "Country": "KY",
+ "PostalCode": "KY1-9009"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.",
+ "AdditionalAddressLine": "Kaya Flamboyan 9",
+ "City": "Willemstad",
+ "Country": "CW"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000087",
+ "RegistrationAuthorityEntityID": "574206"
+ },
+ "LegalJurisdiction": "KY",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "9999",
+ "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"
+ },
+ "EntityStatus": "INACTIVE",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "DISSOLUTION",
+ "LegalEntityEventEffectiveDate": "2018-11-09T00:00:00.000+01:00",
+ "LegalEntityEventRecordedDate": "2022-03-24T20:41:39.000+01:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-08-24T20:53:00.000+02:00",
+ "LastUpdateDate": "2023-09-03T08:48:03.080+02:00",
+ "RegistrationStatus": "RETIRED",
+ "NextRenewalDate": "2019-02-14T01:32:00.000+01:00",
+ "ManagingLOU": "39120001KULK7200U106",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000087",
+ "ValidationAuthorityEntityID": "574206"
+ }
+ }
+ },
+ {
+ "LEI": "097900BHF50000076475",
+ "Entity": {
+ "LegalName": "VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000526",
+ "RegistrationAuthorityEntityID": "35903058"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "2EEG"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2004-10-09T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2017-05-30T00:00:00+02:00",
+ "LastUpdateDate": "2023-06-25T08:30:56.059+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-05-30T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000526",
+ "ValidationAuthorityEntityID": "35903058"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002337",
+ "Entity": {
+ "LegalName": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s."
+ }
+ ],
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "PREFERRED_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s."
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB04"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-23T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:04.159+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB04"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002143",
+ "Entity": {
+ "LegalName": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB01"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2012-04-02T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:02.482+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB01"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002240",
+ "Entity": {
+ "LegalName": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB02"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-22T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:03.502+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB02"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002434",
+ "Entity": {
+ "LegalName": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB03"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-22T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:04.659+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB03"
+ }
+ }
+ },
+ {
+ "LEI": "549300JB1P61FUTPEZ75",
+ "Entity": {
+ "LegalName": "Všeobecná úverová banka, a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "Vseobecna uverova banka, as"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "829 90"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "829 90"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000526",
+ "RegistrationAuthorityEntityID": "31320155"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "2EEG"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "1992-04-01T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2013-10-02T00:00:00+02:00",
+ "LastUpdateDate": "2023-10-12T08:30:46.285+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-10-03T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000526",
+ "ValidationAuthorityEntityID": "31320155"
+ }
+ }
+ },
+ {
+ "LEI": "9845002B9874D50A4531",
+ "Entity": {
+ "LegalName": "Corporate Business Solutions Partners S.à r.l.",
+ "LegalAddress": {
+ "FirstAddressLine": "23 Am Scheerleck",
+ "City": "Wecker",
+ "Country": "LU",
+ "PostalCode": "6868"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "23 Am Scheerleck",
+ "City": "Wecker",
+ "Country": "LU",
+ "PostalCode": "6868"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000432",
+ "RegistrationAuthorityEntityID": "B191847"
+ },
+ "LegalJurisdiction": "LU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "DVXS"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2014-11-17T00:00:00+00:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2020-03-17T08:22:48+00:00",
+ "LastUpdateDate": "2023-02-25T09:13:32+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-03-16T08:36:11+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000432",
+ "ValidationAuthorityEntityID": "B191847"
+ }
+ }
+ },
+ {
+ "LEI": "2138002YAB7Z9KO21397",
+ "Entity": {
+ "LegalName": "MONOOVA GLOBAL PAYMENTS PTY LTD",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONOOVA GLOBAL PAYMENTS LIMITED"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONEYTECH LIMITED"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "SQUADRON FINANCIAL SERVICES LIMITED"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "UNIT L7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "106 249 852"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "TXVC"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2003-09-09T00:00:00+00:00",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_FORM",
+ "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ },
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_NAME",
+ "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-01-14T00:00:00+00:00",
+ "LastUpdateDate": "2023-02-11T11:21:01+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "106 249 852"
+ }
+ }
+ },
+ {
+ "LEI": "984500501A1B1045PB30",
+ "Entity": {
+ "LegalName": "MONEYTECH GROUP LIMITED",
+ "LegalAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "611 393 554"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "R4KK"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2016-03-18T00:00:00+00:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2021-07-27T07:07:14+00:00",
+ "LastUpdateDate": "2022-03-23T15:28:31+00:00",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-07-27T07:07:14+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "611 393 554"
+ }
+ }
+ },
+ {
+ "LEI": "98450051BS9C610A8T78",
+ "Entity": {
+ "LegalName": "MONOOVA FX PTY LTD",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONEYTECH FX PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "360 MARKETS PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "360FX PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "CAPITAL AND FINANCE FOREX PTY LTD"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "LEVEL 7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "LEVEL 7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "151 337 852"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "TXVC"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2011-06-07T00:00:00+00:00",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_ADDRESS",
+ "LegalEntityEventEffectiveDate": "2022-09-13T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-03-30T14:18:04+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T12:21:06+00:00",
+ "LastUpdateDate": "2023-04-30T14:18:04+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "151 337 852"
+ }
+ }
+ }
+]
diff --git a/tests/fixtures/lei-updates-data2-new.xml b/tests/fixtures/lei-updates-data2-new.xml
new file mode 100644
index 0000000..65bf57c
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2-new.xml
@@ -0,0 +1,649 @@
+
+
+ 2023-11-06T08:20:40Z
+ GLEIF_FULL_PUBLISHED
+ 2495059
+
+
+
+ ZZZWS8Y6XVKM0Q9RJQ79
+
+ Delaware Ivy Value Fund
+
+ IVY FUNDS - Delaware Ivy Value Fund
+ IVY FUNDS - Ivy Value Fund
+
+
+ C/O THE CORPORATION TRUST COMPANY
+ 1209 ORANGE ST
+ WILMINGTON
+ US-DE
+ US
+ 19801
+
+
+ C/O Delaware Management Company, Inc.
+ CORPORATION SERVICE COMPANY
+ 251 LITTLE FALLS DRIVE
+ WILMINGTON
+ US-DE
+ US
+ 19808
+
+
+ RA000665
+ S000024832
+
+ US
+ FUND
+
+ 8888
+ FUND
+
+ ACTIVE
+ 2012-06-06T00:00:00.000Z
+
+
+ 2012-06-06T15:56:00.000Z
+ 2023-09-12T20:51:51.031Z
+ ISSUED
+ 2024-09-12T20:51:51.032Z
+ 5493001KJTIIGC8Y1R12
+ FULLY_CORROBORATED
+
+ RA000665
+ S000024832
+
+
+
+
+
+ 00TR8NKAEL48RGTZEW89
+
+ ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.
+
+ 89 Nexus Way
+ Camana Bay
+ KY
+ KY1-9009
+
+
+ C/O Citco Fund Services (Curacao) B.V.
+ Kaya Flamboyan 9
+ Willemstad
+ CW
+
+
+ RA000087
+ 574206
+
+ KY
+ GENERAL
+
+ 9999
+ CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY
+
+ INACTIVE
+
+
+ DISSOLUTION
+ 2018-11-09T00:00:00.000+01:00
+ 2022-03-24T20:41:39.000+01:00
+ SUPPORTING_DOCUMENTS
+
+
+
+
+ 2012-08-24T20:53:00.000+02:00
+ 2023-09-03T08:48:03.080+02:00
+ RETIRED
+ 2019-02-14T01:32:00.000+01:00
+ 39120001KULK7200U106
+ FULLY_CORROBORATED
+
+ RA000087
+ 574206
+
+
+
+
+
+ 097900BHF50000076475
+
+ VÚB Generali dôchodková správcovská spoločnosť, a.s.
+
+ VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000526
+ 35903058
+
+ SK
+ GENERAL
+
+ 2EEG
+
+ ACTIVE
+ 2004-10-09T08:00:00+02:00
+
+
+ 2017-05-30T00:00:00+02:00
+ 2023-06-25T08:30:56.059+02:00
+ ISSUED
+ 2024-05-30T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000526
+ 35903058
+
+
+
+
+ Mlynské nivy 1, null, null, null, SK, null, 820 04, Bratislava
+ 0.94
+ pointAddress
+ 48.14687
+ 17.12492
+ 2016-05-17T10:50:50
+ TopLeft.Latitude: 48.1479942, TopLeft.Longitude: 17.1232352, BottomRight.Latitude: 48.1457458, BottomRight.Longitude: 17.1266048
+ houseNumber
+ Mlynské nivy 170/1, 821 09 Bratislava, Slovenská Republika
+ NT_kzljYRG5OuCE2R6Jrp7U1D_xcDMvED
+ Mlynské nivy
+ 170/1
+ 821 09
+ Bratislava
+ Bratislava I
+ Bratislavský kraj
+ SVK
+
+
+
+
+ 097900BEJX0000002337
+
+ SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.
+
+ MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+
+ SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s.
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB04
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-23T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:04.159+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB04
+
+
+
+
+
+ 097900BEJX0000002143
+
+ INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.
+
+ INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB01
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2012-04-02T08:00:00+02:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:02.482+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB01
+
+
+
+
+
+ 097900BEJX0000002240
+
+ KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+ KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB02
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-22T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:03.502+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB02
+
+
+
+
+
+ 097900BEJX0000002434
+
+ PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+ PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB03
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-22T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:04.659+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB03
+
+
+
+
+
+ 549300JB1P61FUTPEZ75
+
+ Všeobecná úverová banka, a.s.
+
+ Vseobecna uverova banka, as
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 829 90
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 829 90
+
+
+ RA000526
+ 31320155
+
+ SK
+ GENERAL
+
+ 2EEG
+
+ ACTIVE
+ 1992-04-01T08:00:00+02:00
+
+
+ 2013-10-02T00:00:00+02:00
+ 2023-10-12T08:30:46.285+02:00
+ ISSUED
+ 2024-10-03T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000526
+ 31320155
+
+
+
+
+
+ 9845002B9874D50A4531
+
+ Corporate Business Solutions Partners S.à r.l.
+
+ 23 Am Scheerleck
+ Wecker
+ LU
+ 6868
+
+
+ 23 Am Scheerleck
+ Wecker
+ LU
+ 6868
+
+
+ RA000432
+ B191847
+
+ LU
+ GENERAL
+
+ DVXS
+
+ ACTIVE
+ 2014-11-17T00:00:00+00:00
+
+
+ 2020-03-17T08:22:48+00:00
+ 2023-02-25T09:13:32+00:00
+ ISSUED
+ 2024-03-16T08:36:11+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000432
+ B191847
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+
+ MONOOVA GLOBAL PAYMENTS PTY LTD
+
+ MONOOVA GLOBAL PAYMENTS LIMITED
+ MONEYTECH LIMITED
+ SQUADRON FINANCIAL SERVICES LIMITED
+
+
+ UNIT L7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 106 249 852
+
+ AU
+ GENERAL
+
+ TXVC
+
+ ACTIVE
+ 2003-09-09T00:00:00+00:00
+
+
+ CHANGE_LEGAL_FORM
+ 2022-09-16T00:00:00+00:00
+ 2023-01-11T09:59:14+00:00
+ SUPPORTING_DOCUMENTS
+
+
+ CHANGE_LEGAL_NAME
+ 2022-09-16T00:00:00+00:00
+ 2023-01-11T09:59:14+00:00
+ SUPPORTING_DOCUMENTS
+
+
+
+
+ 2019-01-14T00:00:00+00:00
+ 2023-02-11T11:21:01+00:00
+ ISSUED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 106 249 852
+
+
+
+
+
+ 984500501A1B1045PB30
+
+ MONEYTECH GROUP LIMITED
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 611 393 554
+
+ AU
+ GENERAL
+
+ R4KK
+
+ ACTIVE
+ 2016-03-18T00:00:00+00:00
+
+
+ 2021-07-27T07:07:14+00:00
+ 2022-03-23T15:28:31+00:00
+ LAPSED
+ 2022-07-27T07:07:14+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 611 393 554
+
+
+
+
+
+ 98450051BS9C610A8T78
+
+ MONOOVA FX PTY LTD
+
+ MONEYTECH FX PTY LTD
+ 360 MARKETS PTY LTD
+ 360FX PTY LTD
+ CAPITAL AND FINANCE FOREX PTY LTD
+
+
+ LEVEL 7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ LEVEL 7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 151 337 852
+
+ AU
+ GENERAL
+
+ TXVC
+
+ ACTIVE
+ 2011-06-07T00:00:00+00:00
+
+
+ CHANGE_LEGAL_ADDRESS
+ 2022-09-13T00:00:00+00:00
+ 2023-03-30T14:18:04+00:00
+ SUPPORTING_DOCUMENTS
+
+
+
+
+ 2019-05-15T12:21:06+00:00
+ 2023-04-30T14:18:04+00:00
+ ISSUED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 151 337 852
+
+
+
+
+
+
diff --git a/tests/fixtures/lei-updates-data2-out.json b/tests/fixtures/lei-updates-data2-out.json
new file mode 100644
index 0000000..d6b602d
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2-out.json
@@ -0,0 +1,636 @@
+[
+ {
+ "statementID": "790b76d5-0db6-9d26-bb84-1d57963c4638",
+ "statementType": "entityStatement",
+ "statementDate": "2023-08-03",
+ "entityType": "registeredEntity",
+ "name": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.",
+ "incorporatedInJurisdiction": {
+ "name": "Cayman Islands",
+ "code": "KY"
+ },
+ "identifiers": [
+ {
+ "id": "00TR8NKAEL48RGTZEW89",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "574206",
+ "schemeName": "RA000087"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "89 Nexus Way, Camana Bay",
+ "country": "KY",
+ "postCode": "KY1-9009"
+ },
+ {
+ "type": "business",
+ "address": "C/O Citco Fund Services (Curacao) B.V., Willemstad",
+ "country": "CW"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "1865157c-c635-bb1b-cebf-9edcf88c7938",
+ "statementType": "entityStatement",
+ "statementDate": "2023-05-25",
+ "entityType": "registeredEntity",
+ "name": "VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BHF50000076475",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "35903058",
+ "schemeName": "RA000526"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2004-10-09T08:00:00+02:00"
+ },
+ {
+ "statementID": "8a8e8ee2-5bb4-7582-9b79-d531b6b59c12",
+ "statementType": "entityStatement",
+ "statementDate": "2023-08-17",
+ "entityType": "registeredEntity",
+ "name": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002337",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB04",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-23T08:00:00+01:00"
+ },
+ {
+ "statementID": "d60fa665-032c-6992-895f-66cdc7509909",
+ "statementType": "entityStatement",
+ "statementDate": "2023-08-17",
+ "entityType": "registeredEntity",
+ "name": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002143",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB01",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2012-04-02T08:00:00+02:00"
+ },
+ {
+ "statementID": "64b41c35-f336-0010-ae81-785d33e65ffd",
+ "statementType": "entityStatement",
+ "statementDate": "2023-08-17",
+ "entityType": "registeredEntity",
+ "name": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002240",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB02",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-22T08:00:00+01:00"
+ },
+ {
+ "statementID": "4f32f254-ec84-a32a-79d1-a036dfa789e6",
+ "statementType": "entityStatement",
+ "statementDate": "2023-08-17",
+ "entityType": "registeredEntity",
+ "name": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BEJX0000002434",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "SK35903058VUB03",
+ "schemeName": "RA000706"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ },
+ {
+ "type": "business",
+ "address": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s., Bratislava",
+ "country": "SK",
+ "postCode": "820 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2005-03-22T08:00:00+01:00"
+ },
+ {
+ "statementID": "eca994a4-c50a-62b4-1308-9077410ee7b2",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-12",
+ "entityType": "registeredEntity",
+ "name": "Všeobecná úverová banka, a.s.",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "549300JB1P61FUTPEZ75",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "31320155",
+ "schemeName": "RA000526"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "829 90"
+ },
+ {
+ "type": "business",
+ "address": "Mlynské nivy 1, Bratislava",
+ "country": "SK",
+ "postCode": "829 90"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "1992-04-01T08:00:00+02:00"
+ },
+ {
+ "statementID": "71da391b-9385-4a1c-c5fc-e040a762b172",
+ "statementType": "entityStatement",
+ "statementDate": "2023-09-27",
+ "entityType": "registeredEntity",
+ "name": "Aquila Real Asset Finance III a. s. v likvidácii",
+ "incorporatedInJurisdiction": {
+ "name": "Slovakia",
+ "code": "SK"
+ },
+ "identifiers": [
+ {
+ "id": "097900BJGO0000201513",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "52143058",
+ "schemeName": "RA000526"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "Dúbravská cesta 14, Bratislava - mestská časť Karlova Ves",
+ "country": "SK",
+ "postCode": "841 04"
+ },
+ {
+ "type": "business",
+ "address": "Dúbravská cesta 14, Bratislava - mestská časť Karlova Ves",
+ "country": "SK",
+ "postCode": "841 04"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2019-02-02T08:00:00+01:00"
+ },
+ {
+ "statementID": "fd9f3a92-ed00-95f2-b005-45772e4f40b7",
+ "statementType": "entityStatement",
+ "statementDate": "2023-01-25",
+ "entityType": "registeredEntity",
+ "name": "Corporate Business Solutions Partners S.à r.l.",
+ "incorporatedInJurisdiction": {
+ "name": "Luxembourg",
+ "code": "LU"
+ },
+ "identifiers": [
+ {
+ "id": "9845002B9874D50A4531",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "B191847",
+ "schemeName": "RA000432"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "23 Am Scheerleck, Wecker",
+ "country": "LU",
+ "postCode": "6868"
+ },
+ {
+ "type": "business",
+ "address": "23 Am Scheerleck, Wecker",
+ "country": "LU",
+ "postCode": "6868"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2014-11-17T00:00:00+00:00"
+ },
+ {
+ "statementID": "63a68db5-2ba9-2d26-0acc-2303ca07035e",
+ "statementType": "entityStatement",
+ "statementDate": "2023-01-11",
+ "entityType": "registeredEntity",
+ "name": "MONOOVA GLOBAL PAYMENTS PTY LTD",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "2138002YAB7Z9KO21397",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "106 249 852",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "UNIT L7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2003-09-09T00:00:00+00:00"
+ },
+ {
+ "statementID": "6c029f06-1c31-3864-3508-8c393bd6f61e",
+ "statementType": "entityStatement",
+ "statementDate": "2022-02-23",
+ "entityType": "registeredEntity",
+ "name": "MONEYTECH GROUP LIMITED",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "984500501A1B1045PB30",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "611 393 554",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "'L7', NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2016-03-18T00:00:00+00:00"
+ },
+ {
+ "statementID": "78205d04-b700-2c35-ea8c-af8cb534bc11",
+ "statementType": "entityStatement",
+ "statementDate": "2023-03-30",
+ "entityType": "registeredEntity",
+ "name": "MONOOVA FX PTY LTD",
+ "incorporatedInJurisdiction": {
+ "name": "Australia",
+ "code": "AU"
+ },
+ "identifiers": [
+ {
+ "id": "98450051BS9C610A8T78",
+ "scheme": "XI-LEI",
+ "schemeName": "Global Legal Entity Identifier Index"
+ },
+ {
+ "id": "151 337 852",
+ "schemeName": "RA000014"
+ }
+ ],
+ "addresses": [
+ {
+ "type": "registered",
+ "address": "LEVEL 7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ },
+ {
+ "type": "business",
+ "address": "LEVEL 7, NORTH SYDNEY",
+ "country": "AU",
+ "postCode": "2060"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-18",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ },
+ "foundingDate": "2011-06-07T00:00:00+00:00"
+ }
+]
diff --git a/tests/fixtures/lei-updates-data2.json b/tests/fixtures/lei-updates-data2.json
new file mode 100644
index 0000000..fcb36bb
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2.json
@@ -0,0 +1,623 @@
+[
+ {
+ "LEI": "00TR8NKAEL48RGTZEW89",
+ "Entity": {
+ "LegalName": "ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.",
+ "LegalAddress": {
+ "FirstAddressLine": "89 Nexus Way",
+ "City": "Camana Bay",
+ "Country": "KY",
+ "PostalCode": "KY1-9009"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "C/O Citco Fund Services (Curacao) B.V.",
+ "AdditionalAddressLine": "Kaya Flamboyan 9",
+ "City": "Willemstad",
+ "Country": "CW"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000087",
+ "RegistrationAuthorityEntityID": "574206"
+ },
+ "LegalJurisdiction": "KY",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "9999",
+ "OtherLegalForm": "CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY"
+ },
+ "EntityStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-08-24T20:53:00.000+02:00",
+ "LastUpdateDate": "2023-08-03T08:48:03.080+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2019-02-14T01:32:00.000+01:00",
+ "ManagingLOU": "39120001KULK7200U106",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000087",
+ "ValidationAuthorityEntityID": "574206"
+ }
+ }
+ },
+ {
+ "LEI": "097900BHF50000076475",
+ "Entity": {
+ "LegalName": "VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000526",
+ "RegistrationAuthorityEntityID": "35903058"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "2EEG"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2004-10-09T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2017-05-30T00:00:00+02:00",
+ "LastUpdateDate": "2023-05-25T08:30:56.059+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-05-30T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000526",
+ "ValidationAuthorityEntityID": "35903058"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002337",
+ "Entity": {
+ "LegalName": "SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s."
+ }
+ ],
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "PREFERRED_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s."
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB04"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-23T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:04.159+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB04"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002143",
+ "Entity": {
+ "LegalName": "INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB01"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2012-04-02T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:02.482+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB01"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002240",
+ "Entity": {
+ "LegalName": "KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB02"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-22T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:03.502+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB02"
+ }
+ }
+ },
+ {
+ "LEI": "097900BEJX0000002434",
+ "Entity": {
+ "LegalName": "PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.",
+ "AdditionalAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "820 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000706",
+ "RegistrationAuthorityEntityID": "SK35903058VUB03"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "FUND",
+ "LegalForm": {
+ "EntityLegalFormCode": "7EG2"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2005-03-22T08:00:00+01:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:04.659+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000706",
+ "ValidationAuthorityEntityID": "SK35903058VUB03"
+ }
+ }
+ },
+ {
+ "LEI": "549300JB1P61FUTPEZ75",
+ "Entity": {
+ "LegalName": "Všeobecná úverová banka, a.s.",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "Vseobecna uverova banka, as"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "829 90"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "Mlynské nivy 1",
+ "City": "Bratislava",
+ "Country": "SK",
+ "PostalCode": "829 90"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000526",
+ "RegistrationAuthorityEntityID": "31320155"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "2EEG"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "1992-04-01T08:00:00+02:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2013-10-02T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-12T08:30:46.285+02:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-10-03T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000526",
+ "ValidationAuthorityEntityID": "31320155"
+ }
+ }
+ },
+ {
+ "LEI": "097900BJGO0000201513",
+ "Entity": {
+ "LegalName": "Aquila Real Asset Finance III a. s. v likvidácii",
+ "TransliteratedOtherEntityNames": [
+ {
+ "type": "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME",
+ "TransliteratedOtherEntityName": "AQUILA REAL ASSET FINANCE III A S V LIKVIDACII"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "Dúbravská cesta 14",
+ "City": "Bratislava - mestská časť Karlova Ves",
+ "Country": "SK",
+ "PostalCode": "841 04"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "Dúbravská cesta 14",
+ "City": "Bratislava - mestská časť Karlova Ves",
+ "Country": "SK",
+ "PostalCode": "841 04"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000526",
+ "RegistrationAuthorityEntityID": "52143058"
+ },
+ "LegalJurisdiction": "SK",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "2EEG"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2019-02-02T08:00:00+01:00",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "LIQUIDATION",
+ "LegalEntityEventEffectiveDate": "2023-04-13T08:00:00+02:00",
+ "LegalEntityEventRecordedDate": "2023-09-27T12:34:16.118+02:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "AffectedFields": [
+ "Aquila Real Asset Finance III a. s. v likvidácii"
+ ]
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-06-14T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-27T12:34:16.118+02:00",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2023-06-18T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000526",
+ "ValidationAuthorityEntityID": "52143058"
+ }
+ }
+ },
+ {
+ "LEI": "9845002B9874D50A4531",
+ "Entity": {
+ "LegalName": "Corporate Business Solutions Partners S.à r.l.",
+ "LegalAddress": {
+ "FirstAddressLine": "23 Am Scheerleck",
+ "City": "Wecker",
+ "Country": "LU",
+ "PostalCode": "6868"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "23 Am Scheerleck",
+ "City": "Wecker",
+ "Country": "LU",
+ "PostalCode": "6868"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000432",
+ "RegistrationAuthorityEntityID": "B191847"
+ },
+ "LegalJurisdiction": "LU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "DVXS"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2014-11-17T00:00:00+00:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2020-03-17T08:22:48+00:00",
+ "LastUpdateDate": "2023-01-25T09:13:32+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-03-16T08:36:11+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000432",
+ "ValidationAuthorityEntityID": "B191847"
+ }
+ }
+ },
+ {
+ "LEI": "2138002YAB7Z9KO21397",
+ "Entity": {
+ "LegalName": "MONOOVA GLOBAL PAYMENTS PTY LTD",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONOOVA GLOBAL PAYMENTS LIMITED"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONEYTECH LIMITED"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "SQUADRON FINANCIAL SERVICES LIMITED"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "UNIT L7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "106 249 852"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "TXVC"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2003-09-09T00:00:00+00:00",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_FORM",
+ "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ },
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_NAME",
+ "LegalEntityEventEffectiveDate": "2022-09-16T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-01-11T09:59:14+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-01-14T00:00:00+00:00",
+ "LastUpdateDate": "2023-01-11T11:21:01+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "106 249 852"
+ }
+ }
+ },
+ {
+ "LEI": "984500501A1B1045PB30",
+ "Entity": {
+ "LegalName": "MONEYTECH GROUP LIMITED",
+ "LegalAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "'L7'",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "611 393 554"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "R4KK"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2016-03-18T00:00:00+00:00"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2021-07-27T07:07:14+00:00",
+ "LastUpdateDate": "2022-02-23T15:28:31+00:00",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-07-27T07:07:14+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "611 393 554"
+ }
+ }
+ },
+ {
+ "LEI": "98450051BS9C610A8T78",
+ "Entity": {
+ "LegalName": "MONOOVA FX PTY LTD",
+ "OtherEntityNames": [
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "MONEYTECH FX PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "360 MARKETS PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "360FX PTY LTD"
+ },
+ {
+ "type": "PREVIOUS_LEGAL_NAME",
+ "OtherEntityName": "CAPITAL AND FINANCE FOREX PTY LTD"
+ }
+ ],
+ "LegalAddress": {
+ "FirstAddressLine": "LEVEL 7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "HeadquartersAddress": {
+ "FirstAddressLine": "LEVEL 7",
+ "AdditionalAddressLine": "80 PACIFIC HIGHWAY",
+ "City": "NORTH SYDNEY",
+ "Region": "AU-NSW",
+ "Country": "AU",
+ "PostalCode": "2060"
+ },
+ "RegistrationAuthority": {
+ "RegistrationAuthorityID": "RA000014",
+ "RegistrationAuthorityEntityID": "151 337 852"
+ },
+ "LegalJurisdiction": "AU",
+ "EntityCategory": "GENERAL",
+ "LegalForm": {
+ "EntityLegalFormCode": "TXVC"
+ },
+ "EntityStatus": "ACTIVE",
+ "EntityCreationDate": "2011-06-07T00:00:00+00:00",
+ "LegalEntityEvents": [
+ {
+ "LegalEntityEventType": "CHANGE_LEGAL_ADDRESS",
+ "LegalEntityEventEffectiveDate": "2022-09-13T00:00:00+00:00",
+ "LegalEntityEventRecordedDate": "2023-03-30T14:18:04+00:00",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T12:21:06+00:00",
+ "LastUpdateDate": "2023-03-30T14:18:04+00:00",
+ "RegistrationStatus": "ISSUED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationAuthority": {
+ "ValidationAuthorityID": "RA000014",
+ "ValidationAuthorityEntityID": "151 337 852"
+ }
+ }
+ }
+]
diff --git a/tests/fixtures/lei-updates-data2.xml b/tests/fixtures/lei-updates-data2.xml
new file mode 100644
index 0000000..4924cb8
--- /dev/null
+++ b/tests/fixtures/lei-updates-data2.xml
@@ -0,0 +1,645 @@
+
+
+ 2023-11-06T08:20:40Z
+ GLEIF_FULL_PUBLISHED
+ 2495059
+
+
+
+ 00TR8NKAEL48RGTZEW89
+
+ ESG DOMESTIC OPPORTUNITY OFFSHORE FUND LTD.
+
+ 89 Nexus Way
+ Camana Bay
+ KY
+ KY1-9009
+
+
+ C/O Citco Fund Services (Curacao) B.V.
+ Kaya Flamboyan 9
+ Willemstad
+ CW
+
+
+ RA000087
+ 574206
+
+ KY
+ GENERAL
+
+ 9999
+ CAYMAN ISLANDS ORDINARY NON-RESIDENT COMPANY
+
+ ACTIVE
+
+
+ 2012-08-24T20:53:00.000+02:00
+ 2023-08-03T08:48:03.080+02:00
+ ISSUED
+ 2019-02-14T01:32:00.000+01:00
+ 39120001KULK7200U106
+ FULLY_CORROBORATED
+
+ RA000087
+ 574206
+
+
+
+
+
+ 097900BHF50000076475
+
+ VÚB Generali dôchodková správcovská spoločnosť, a.s.
+
+ VUB GENERALI DOCHODKOVA SPRAVCOVSKA SPOLOCNOST, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000526
+ 35903058
+
+ SK
+ GENERAL
+
+ 2EEG
+
+ ACTIVE
+ 2004-10-09T08:00:00+02:00
+
+
+ 2017-05-30T00:00:00+02:00
+ 2023-05-25T08:30:56.059+02:00
+ ISSUED
+ 2024-05-30T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000526
+ 35903058
+
+
+
+
+ Mlynské nivy 1, null, null, null, SK, null, 820 04, Bratislava
+ 0.94
+ pointAddress
+ 48.14687
+ 17.12492
+ 2016-05-17T10:50:50
+ TopLeft.Latitude: 48.1479942, TopLeft.Longitude: 17.1232352, BottomRight.Latitude: 48.1457458, BottomRight.Longitude: 17.1266048
+ houseNumber
+ Mlynské nivy 170/1, 821 09 Bratislava, Slovenská Republika
+ NT_kzljYRG5OuCE2R6Jrp7U1D_xcDMvED
+ Mlynské nivy
+ 170/1
+ 821 09
+ Bratislava
+ Bratislava I
+ Bratislavský kraj
+ SVK
+
+
+
+
+ 097900BEJX0000002337
+
+ SMART, zelený inovatívny negarantovaný dôchodkový fond VÚB Generali dôchodková správcovská spoločnosť, a.s.
+
+ MIX, zmiešaný negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+
+ SMART, zeleny inovativny negarantovany dochodkovy fond VUB Generali dochodkova spravcovska spolocnost, a.s.
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB04
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-23T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:04.159+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB04
+
+
+
+
+
+ 097900BEJX0000002143
+
+ INDEX, indexový negarantovaný dôchodkový fond VUB Generali d.s.s., a.s.
+
+ INDEX, INDEXOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB01
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2012-04-02T08:00:00+02:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:02.482+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB01
+
+
+
+
+
+ 097900BEJX0000002240
+
+ KLASIK, dlhopisový garantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+ KLASIK, DLHOPISOVY GARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB02
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-22T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:03.502+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB02
+
+
+
+
+
+ 097900BEJX0000002434
+
+ PROFIT, akciový negarantovaný dôchodkový fond VÚB Generali d.s.s., a.s.
+
+ PROFIT, AKCIOVY NEGARANTOVANY DOCHODKOVY FOND VUB GENERALI DSS, AS
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ c/o VÚB Generali dôchodková správcovská spoločnosť, a.s.
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 820 04
+
+
+ RA000706
+ SK35903058VUB03
+
+ SK
+ FUND
+
+ 7EG2
+
+ ACTIVE
+ 2005-03-22T08:00:00+01:00
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:04.659+02:00
+ ISSUED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000706
+ SK35903058VUB03
+
+
+
+
+
+ 549300JB1P61FUTPEZ75
+
+ Všeobecná úverová banka, a.s.
+
+ Vseobecna uverova banka, as
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 829 90
+
+
+ Mlynské nivy 1
+ Bratislava
+ SK
+ 829 90
+
+
+ RA000526
+ 31320155
+
+ SK
+ GENERAL
+
+ 2EEG
+
+ ACTIVE
+ 1992-04-01T08:00:00+02:00
+
+
+ 2013-10-02T00:00:00+02:00
+ 2023-09-12T08:30:46.285+02:00
+ ISSUED
+ 2024-10-03T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000526
+ 31320155
+
+
+
+
+
+ 097900BJGO0000201513
+
+ Aquila Real Asset Finance III a. s. v likvidácii
+
+ AQUILA REAL ASSET FINANCE III A S V LIKVIDACII
+
+
+ Dúbravská cesta 14
+ Bratislava - mestská časť Karlova Ves
+ SK
+ 841 04
+
+
+ Dúbravská cesta 14
+ Bratislava - mestská časť Karlova Ves
+ SK
+ 841 04
+
+
+ RA000526
+ 52143058
+
+ SK
+ GENERAL
+
+ 2EEG
+
+ ACTIVE
+ 2019-02-02T08:00:00+01:00
+
+
+ LIQUIDATION
+ 2023-04-13T08:00:00+02:00
+ 2023-09-27T12:34:16.118+02:00
+ SUPPORTING_DOCUMENTS
+
+ Aquila Real Asset Finance III a. s. v likvidácii
+
+
+
+
+
+ 2019-06-14T00:00:00+02:00
+ 2023-09-27T12:34:16.118+02:00
+ LAPSED
+ 2023-06-18T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+
+ RA000526
+ 52143058
+
+
+
+
+
+ 9845002B9874D50A4531
+
+ Corporate Business Solutions Partners S.à r.l.
+
+ 23 Am Scheerleck
+ Wecker
+ LU
+ 6868
+
+
+ 23 Am Scheerleck
+ Wecker
+ LU
+ 6868
+
+
+ RA000432
+ B191847
+
+ LU
+ GENERAL
+
+ DVXS
+
+ ACTIVE
+ 2014-11-17T00:00:00+00:00
+
+
+ 2020-03-17T08:22:48+00:00
+ 2023-01-25T09:13:32+00:00
+ ISSUED
+ 2024-03-16T08:36:11+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000432
+ B191847
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+
+ MONOOVA GLOBAL PAYMENTS PTY LTD
+
+ MONOOVA GLOBAL PAYMENTS LIMITED
+ MONEYTECH LIMITED
+ SQUADRON FINANCIAL SERVICES LIMITED
+
+
+ UNIT L7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 106 249 852
+
+ AU
+ GENERAL
+
+ TXVC
+
+ ACTIVE
+ 2003-09-09T00:00:00+00:00
+
+
+ CHANGE_LEGAL_FORM
+ 2022-09-16T00:00:00+00:00
+ 2023-01-11T09:59:14+00:00
+ SUPPORTING_DOCUMENTS
+
+
+ CHANGE_LEGAL_NAME
+ 2022-09-16T00:00:00+00:00
+ 2023-01-11T09:59:14+00:00
+ SUPPORTING_DOCUMENTS
+
+
+
+
+ 2019-01-14T00:00:00+00:00
+ 2023-01-11T11:21:01+00:00
+ ISSUED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 106 249 852
+
+
+
+
+
+ 984500501A1B1045PB30
+
+ MONEYTECH GROUP LIMITED
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ 'L7'
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 611 393 554
+
+ AU
+ GENERAL
+
+ R4KK
+
+ ACTIVE
+ 2016-03-18T00:00:00+00:00
+
+
+ 2021-07-27T07:07:14+00:00
+ 2022-02-23T15:28:31+00:00
+ LAPSED
+ 2022-07-27T07:07:14+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 611 393 554
+
+
+
+
+
+ 98450051BS9C610A8T78
+
+ MONOOVA FX PTY LTD
+
+ MONEYTECH FX PTY LTD
+ 360 MARKETS PTY LTD
+ 360FX PTY LTD
+ CAPITAL AND FINANCE FOREX PTY LTD
+
+
+ LEVEL 7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ LEVEL 7
+ 80 PACIFIC HIGHWAY
+ NORTH SYDNEY
+ AU-NSW
+ AU
+ 2060
+
+
+ RA000014
+ 151 337 852
+
+ AU
+ GENERAL
+
+ TXVC
+
+ ACTIVE
+ 2011-06-07T00:00:00+00:00
+
+
+ CHANGE_LEGAL_ADDRESS
+ 2022-09-13T00:00:00+00:00
+ 2023-03-30T14:18:04+00:00
+ SUPPORTING_DOCUMENTS
+
+
+
+
+ 2019-05-15T12:21:06+00:00
+ 2023-03-30T14:18:04+00:00
+ ISSUED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+
+ RA000014
+ 151 337 852
+
+
+
+
+
+
diff --git a/tests/fixtures/lei-updates-replaces-data.json b/tests/fixtures/lei-updates-replaces-data.json
new file mode 100644
index 0000000..a243b90
--- /dev/null
+++ b/tests/fixtures/lei-updates-replaces-data.json
@@ -0,0 +1,4 @@
+[
+{"LEI": "549300JV7BZB002LHI61", "Entity": {"LegalName": "Helsingin Hammaslaboratorio Oy", "LegalAddress": {"FirstAddressLine": "c/o Eskola", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "Kulosaarentie 4-8 C 20", "City": "Helsinki", "Region": "FI-18", "Country": "FI", "PostalCode": "00570"}, "HeadquartersAddress": {"FirstAddressLine": "c/o Eskola", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "Kulosaarentie 4-8 C 20", "City": "Helsinki", "Region": "FI-18", "Country": "FI", "PostalCode": "00570"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000188", "RegistrationAuthorityEntityID": "0535671-9"}, "LegalJurisdiction": "FI", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "DKUW"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1983-09-15T02:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2017-10-17T03:14:00Z", "LastUpdateDate": "2023-12-29T09:43:20Z", "RegistrationStatus": "PENDING_TRANSFER", "NextRenewalDate": "2024-01-23T15:15:02Z", "ManagingLOU": "549300O897ZC5H7CY412", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000188", "ValidationAuthorityEntityID": "0535671-9"}}},
+{"LEI": "549300JV7BZB002LHI61", "Entity": {"LegalName": "Helsingin Hammaslaboratorio Oy", "LegalAddress": {"FirstAddressLine": "Kulosaarentie 4-8 C 20", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "00570 HELSINKI", "City": "HELSINKI", "Country": "FI", "PostalCode": "FI-00570"}, "HeadquartersAddress": {"FirstAddressLine": "Kulosaarentie 4-8 C 20", "MailRouting": "c/o Eskola", "AdditionalAddressLine": "00570 HELSINKI", "City": "HELSINKI", "Country": "FI", "PostalCode": "FI-00570"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000188", "RegistrationAuthorityEntityID": "0535671-9"}, "LegalJurisdiction": "FI", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "DKUW"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "1983-12-19T00:00:00+02:00"}, "Registration": {"InitialRegistrationDate": "2017-10-17T03:14:00+03:00", "LastUpdateDate": "2024-01-03T21:10:02+02:00", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2025-01-03T12:00:00+02:00", "ManagingLOU": "743700OO8O2N3TQKJC81", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000188", "ValidationAuthorityEntityID": "0535671-9"}}}
+]
diff --git a/tests/fixtures/relationship-update-data.json b/tests/fixtures/relationship-update-data.json
new file mode 100644
index 0000000..4c9502a
--- /dev/null
+++ b/tests/fixtures/relationship-update-data.json
@@ -0,0 +1,8 @@
+[
+{"LEI": "0292003540H0S4VA7A50", "Entity": {"LegalName": "FBNQUEST ASSET MANAGEMENT LIMITED", "LegalAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "HeadquartersAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI, LAGOS", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000469", "RegistrationAuthorityEntityID": "978831"}, "LegalJurisdiction": "NG", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "LIMITED"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2006-01-01T12:30:00Z"}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000469", "ValidationAuthorityEntityID": "978831"}}},
+{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}},
+{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2023-02-22T09:00:21Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2024-01-30T11:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}},
+{"LEI": "0292003540H0S4VA7A50", "Entity": {"LegalName": "FBNQUEST ASSET MANAGEMENT LIMITED", "LegalAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "HeadquartersAddress": {"FirstAddressLine": "NO. 16, KEFFI STREET", "AdditionalAddressLine": "IKOYI, LAGOS", "City": "LAGOS", "Region": "NG-LA", "Country": "NG", "PostalCode": "101233"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000469", "RegistrationAuthorityEntityID": "978831"}, "LegalJurisdiction": "NG", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "9999", "OtherLegalForm": "LIMITED"}, "EntityStatus": "ACTIVE", "EntityCreationDate": "2006-01-01T12:30:00Z"}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "ISSUED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000469", "ValidationAuthorityEntityID": "978831"}}},
+{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}},
+{"Relationship": {"StartNode": {"NodeID": "0292003540H0S4VA7A50", "NodeIDType": "LEI"}, "EndNode": {"NodeID": "0292002717G4T0CH6E65", "NodeIDType": "LEI"}, "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY", "RelationshipPeriods": [{"StartDate": "2018-01-01T00:00:00Z", "EndDate": "2018-12-31T00:00:00Z", "PeriodType": "ACCOUNTING_PERIOD"}, {"StartDate": "2017-08-14T00:00:00Z", "PeriodType": "RELATIONSHIP_PERIOD"}], "RelationshipStatus": "ACTIVE", "RelationshipQualifiers": [{"QualifierDimension": "ACCOUNTING_STANDARD", "QualifierCategory": "IFRS"}], "RelationshipQuantifiers": [{"MeasurementMethod": "ACCOUNTING_CONSOLIDATION", "QuantifierAmount": "100.00", "QuantifierUnits": "PERCENTAGE"}]}, "Registration": {"InitialRegistrationDate": "2019-01-30T16:19:49Z", "LastUpdateDate": "2024-01-30T08:28:47Z", "RegistrationStatus": "PUBLISHED", "NextRenewalDate": "2025-01-30T10:19:49Z", "ManagingLOU": "029200067A7K6CH0H586", "ValidationSources": "FULLY_CORROBORATED", "ValidationDocuments": "SUPPORTING_DOCUMENTS", "ValidationReference": "https://www.cbn.gov.ng"}}
+]
diff --git a/tests/fixtures/repex-data.json b/tests/fixtures/repex-data.json
new file mode 100644
index 0000000..91afde7
--- /dev/null
+++ b/tests/fixtures/repex-data.json
@@ -0,0 +1,62 @@
+[
+ {
+ "LEI": "001GPB6A9XPE8XJICC14",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_KNOWN_PERSON",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "004L5FPTUREIWK9T2N63",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00EHHQ2ZHDCFXJCPCL46",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00GBW0Z2GYIER7DHDS71",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00KLB2PFTM3060S2N216",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_KNOWN_PERSON",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00QDBXDXLLF3W3JJJO36",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_KNOWN_PERSON",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00TV1D5YIV5IDUGWBW29",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "00X8DSV26QKJPKUT5B34",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "010G7UHBHEI87EKP0Q97",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ },
+ {
+ "LEI": "01370W6ZIY66KQ4J3570",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_PUBLIC",
+ "ContentDate": "2023-06-09T09:03:29Z"
+ }
+]
diff --git a/tests/fixtures/repex-updates-data2-new-out.json b/tests/fixtures/repex-updates-data2-new-out.json
new file mode 100644
index 0000000..0b6e2be
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2-new-out.json
@@ -0,0 +1,653 @@
+[
+ {
+ "statementID": "efc817ce-d25e-cae0-80f2-1405326c0cf6",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 159516LKTEGWITQUIE78",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "e8b77ad5-7ab6-629f-09a5-dcd040ba87ae",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "078879e7-147c-30de-163a-b13d7d5002f8"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "efc817ce-d25e-cae0-80f2-1405326c0cf6"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 159516LKTEGWITQUIE78",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "f9b8a63c-8071-5d72-032f-8f790313a8e2",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 0292001398F7K0YJI588",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "a535fc20-3f6b-1502-e066-650a7eb73006",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "24d30d0b-3f58-7118-ec6a-0772ddb7dfa3"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "f9b8a63c-8071-5d72-032f-8f790313a8e2"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 0292001398F7K0YJI588",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "cbb59de1-4612-07d8-59c9-78eb080dc028",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity."
+ }
+ },
+ {
+ "statementID": "65706612-e115-032d-daca-059196e046c8",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "a12cd374-be9e-e6f3-eb97-2e9b143a049d"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "cbb59de1-4612-07d8-59c9-78eb080dc028"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "960d8dad-3fb8-270c-7396-7f1800d3181d",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity."
+ }
+ },
+ {
+ "statementID": "e521936e-a800-f55b-1cf4-c0e85d16d810",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "30332285-0d9a-7256-449c-d917de1685c2"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "960d8dad-3fb8-270c-7396-7f1800d3181d"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "58a27735-dda3-7175-8521-d0ea1b4d1fba",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 0292001018B2N4DBI146",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf."
+ }
+ },
+ {
+ "statementID": "13325e39-b4a4-8ce2-bd34-b12e1e22ce7d",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "187cceb8-30de-1aed-0723-d006c6f47260"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "58a27735-dda3-7175-8521-d0ea1b4d1fba"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 0292001018B2N4DBI146",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "db0e963d-8d2d-3afb-e836-28b76dab3c6e",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "282cd39f-4c8d-3ca4-dd39-85919c649f6a"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "f4869861-cb38-f5a8-66a3-82ddefb10f83",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity. ExemptionReference provided: Datenschutz!"
+ }
+ },
+ {
+ "statementID": "15c1486d-9cdb-eae6-b1fb-3fa0456d070b",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "6d2a451e-8449-37c7-1cb2-44a1633fd8d8"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "f4869861-cb38-f5a8-66a3-82ddefb10f83"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/repex-updates-data2-new.json b/tests/fixtures/repex-updates-data2-new.json
new file mode 100644
index 0000000..4e39b50
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2-new.json
@@ -0,0 +1,50 @@
+[
+ {
+ "LEI": "159516LKTEGWITQUIE78",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "0292001398F7K0YJI588",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "004L5FPTUREIWK9T2N63",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_KNOWN_PERSON",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "020BQJXAXCZNLKIN7326",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NATURAL_PERSONS",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "0292001018B2N4DBI146",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_LEI",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "2549007VY3IWUVGW7A82",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "Extension": {
+ "Deletion": {
+ "DeletedAt": "2023-11-24T02:07:59Z"
+ }
+ },
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "159515PKYKQYJLT0KF16",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NATURAL_PERSONS",
+ "ExceptionReference": "Datenschutz!",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ }
+]
diff --git a/tests/fixtures/repex-updates-data2-new.xml b/tests/fixtures/repex-updates-data2-new.xml
new file mode 100644
index 0000000..ac15677
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2-new.xml
@@ -0,0 +1,58 @@
+
+
+ 2023-11-06T09:13:21Z
+ GLEIF_FULL_PUBLISHED
+ 4458620
+
+
+
+
+ 159516LKTEGWITQUIE78
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+
+ 0292001398F7K0YJI588
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+
+ 004L5FPTUREIWK9T2N63
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NO_KNOWN_PERSON
+
+
+
+ 020BQJXAXCZNLKIN7326
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NATURAL_PERSONS
+
+
+
+ 0292001018B2N4DBI146
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NO_LEI
+
+
+
+ 2549007VY3IWUVGW7A82
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+ 2023-11-24T02:07:59Z
+
+
+
+
+
+ 159515PKYKQYJLT0KF16
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NATURAL_PERSONS
+ Datenschutz!
+
+
+
+
diff --git a/tests/fixtures/repex-updates-data2-out.json b/tests/fixtures/repex-updates-data2-out.json
new file mode 100644
index 0000000..38296cc
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2-out.json
@@ -0,0 +1,653 @@
+[
+ {
+ "statementID": "854a1c24-5b4e-1587-9dea-222bc503163b",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 0292001398F7K0YJI588",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity."
+ }
+ },
+ {
+ "statementID": "c33246ac-e412-9e13-6f17-410c94ca303e",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "24d30d0b-3f58-7118-ec6a-0772ddb7dfa3"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "854a1c24-5b4e-1587-9dea-222bc503163b"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_KNOWN_PERSON GLEIF Reporting Exception for 0292001398F7K0YJI588",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "b964a2b7-9a2b-9d5e-4f1d-7802fb8bba82",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "0fd27404-0f10-f9e7-1cb1-46c7d6e1cb4e",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "a12cd374-be9e-e6f3-eb97-2e9b143a049d"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "b964a2b7-9a2b-9d5e-4f1d-7802fb8bba82"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 004L5FPTUREIWK9T2N63",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "faaa96d7-b2bb-73e6-5d77-a0b623f9c2c5",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NO_LEI`. This parent legal entity does not consent to obtain an LEI or to authorize its “child entity” to obtain an LEI on its behalf."
+ }
+ },
+ {
+ "statementID": "8638ee52-50f6-b3d9-bb81-a7e5f5b9c070",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "30332285-0d9a-7256-449c-d917de1685c2"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "faaa96d7-b2bb-73e6-5d77-a0b623f9c2c5"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NO_LEI GLEIF Reporting Exception for 020BQJXAXCZNLKIN7326",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "c635057b-a7a5-1bf7-e323-0b8aef4e3347",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 0292001018B2N4DBI146",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity."
+ }
+ },
+ {
+ "statementID": "25effa14-462b-7559-361c-1e62bb33fe5d",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "187cceb8-30de-1aed-0723-d006c6f47260"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "c635057b-a7a5-1bf7-e323-0b8aef4e3347"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 0292001018B2N4DBI146",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "894e8dc1-432c-d922-6517-e84923ee7b50",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for ZZZWS8Y6XVKM0Q9RJQ79",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "ae1f191a-8a87-e58c-7a60-4fa7d727cbbb",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "6844aee7-92e6-9b58-111f-c0685b52d81e"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "894e8dc1-432c-d922-6517-e84923ee7b50"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "indirect",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for ZZZWS8Y6XVKM0Q9RJQ79",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708",
+ "statementType": "entityStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "entityType": "unknownEntity",
+ "unspecifiedEntityDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ }
+ },
+ {
+ "statementID": "db0e963d-8d2d-3afb-e836-28b76dab3c6e",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "282cd39f-4c8d-3ca4-dd39-85919c649f6a"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "4a8fc4a0-9811-33b9-2a25-78b8b9df9708"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NON_CONSOLIDATING GLEIF Reporting Exception for 2549007VY3IWUVGW7A82",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "9f8997ec-809e-2ed1-b56b-93269498eecf",
+ "statementType": "personStatement",
+ "statementDate": "2023-11-06",
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ },
+ "personType": "unknownPerson",
+ "unspecifiedPersonDetails": {
+ "reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity. ExemptionReference provided: Datenschutz"
+ }
+ },
+ {
+ "statementID": "3fadbdb5-dd32-800e-c2d1-5dc1cd28754f",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-11-06",
+ "subject": {
+ "describedByEntityStatement": "6d2a451e-8449-37c7-1cb2-44a1633fd8d8"
+ },
+ "interestedParty": {
+ "describedByPersonStatement": "9f8997ec-809e-2ed1-b56b-93269498eecf"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "details": "A controlling interest."
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "The nature of this interest is unknown",
+ "statementPointerTarget": "/interests/0/type",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ },
+ {
+ "motivation": "commenting",
+ "description": "This statement was created due to a NATURAL_PERSONS GLEIF Reporting Exception for 159515PKYKQYJLT0KF16",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-15",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-15",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/repex-updates-data2.json b/tests/fixtures/repex-updates-data2.json
new file mode 100644
index 0000000..06d2df2
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2.json
@@ -0,0 +1,45 @@
+[
+ {
+ "LEI": "0292001398F7K0YJI588",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_KNOWN_PERSON",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "004L5FPTUREIWK9T2N63",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "020BQJXAXCZNLKIN7326",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NO_LEI",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "0292001018B2N4DBI146",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NATURAL_PERSONS",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "ZZZWS8Y6XVKM0Q9RJQ79",
+ "ExceptionCategory": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "2549007VY3IWUVGW7A82",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NON_CONSOLIDATING",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ },
+ {
+ "LEI": "159515PKYKQYJLT0KF16",
+ "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT",
+ "ExceptionReason": "NATURAL_PERSONS",
+ "ExceptionReference": "Datenschutz",
+ "ContentDate": "2023-11-06T09:13:21Z"
+ }
+]
diff --git a/tests/fixtures/repex-updates-data2.xml b/tests/fixtures/repex-updates-data2.xml
new file mode 100644
index 0000000..10732cd
--- /dev/null
+++ b/tests/fixtures/repex-updates-data2.xml
@@ -0,0 +1,54 @@
+
+
+ 2023-11-06T09:13:21Z
+ GLEIF_FULL_PUBLISHED
+ 4458620
+
+
+
+
+ 0292001398F7K0YJI588
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NO_KNOWN_PERSON
+
+
+
+ 004L5FPTUREIWK9T2N63
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+
+ 020BQJXAXCZNLKIN7326
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NO_LEI
+
+
+
+ 0292001018B2N4DBI146
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NATURAL_PERSONS
+
+
+
+
+ ZZZWS8Y6XVKM0Q9RJQ79
+ ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+
+ 2549007VY3IWUVGW7A82
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NON_CONSOLIDATING
+
+
+
+ 159515PKYKQYJLT0KF16
+ DIRECT_ACCOUNTING_CONSOLIDATION_PARENT
+ NATURAL_PERSONS
+ Datenschutz
+
+
+
+
diff --git a/tests/fixtures/reporting-exceptions-data.json b/tests/fixtures/reporting-exceptions-data.json
new file mode 100644
index 0000000..fa32627
--- /dev/null
+++ b/tests/fixtures/reporting-exceptions-data.json
@@ -0,0 +1,5 @@
+[
+{"LEI": "959800W87BKGBUDPF915", "Entity": {"LegalName": "COMUNIDAD HEREDITARIA FORAL DE MARIA BEGO\u00d1A BENITO PE\u00d1A", "LegalAddress": {"FirstAddressLine": "JUAN BAUTISTA URIARTE 9 4-B", "AddressNumber": "9", "AddressNumberWithinBuilding": "4-B", "City": "Galdakao", "Country": "ES", "PostalCode": "48960"}, "HeadquartersAddress": {"FirstAddressLine": "JUAN BAUTISTA URIARTE 9 4-B", "AddressNumber": "9", "AddressNumberWithinBuilding": "4-B", "City": "Galdakao", "Country": "ES", "PostalCode": "48960"}, "RegistrationAuthority": {"RegistrationAuthorityID": "RA000535", "RegistrationAuthorityEntityID": "V95881967"}, "LegalJurisdiction": "ES", "EntityCategory": "GENERAL", "LegalForm": {"EntityLegalFormCode": "8888", "OtherLegalForm": "COMUNIDAD HEREDITARIA FORAL"}, "EntityStatus": "ACTIVE"}, "Registration": {"InitialRegistrationDate": "2018-01-26T13:04:07.145+01:00", "LastUpdateDate": "2022-01-25T23:30:01.829+01:00", "RegistrationStatus": "LAPSED", "NextRenewalDate": "2022-01-25T16:24:15.251+01:00", "ManagingLOU": "959800R2X69K6Y6MX775", "ValidationSources": "PARTIALLY_CORROBORATED", "ValidationAuthority": {"ValidationAuthorityID": "RA000535", "ValidationAuthorityEntityID": "V95881967"}}},
+{"LEI": "959800W87BKGBUDPF915", "ExceptionCategory": "DIRECT_ACCOUNTING_CONSOLIDATION_PARENT", "ExceptionReason": "NON_CONSOLIDATING", "ContentDate": "2024-01-01T01:18:56Z"},
+{"LEI": "959800W87BKGBUDPF915", "ExceptionCategory": "ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT", "ExceptionReason": "NON_CONSOLIDATING", "ContentDate": "2024-01-01T01:18:56Z"}
+]
diff --git a/tests/fixtures/rr-data-out.json b/tests/fixtures/rr-data-out.json
new file mode 100644
index 0000000..9bcf3f7
--- /dev/null
+++ b/tests/fixtures/rr-data-out.json
@@ -0,0 +1,351 @@
+[
+ {
+ "statementID": "d459ada3-ee9d-5ee9-f40a-ac5aadecdd64",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-18",
+ "subject": {
+ "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-11-29T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "ce4b78b2-f797-cf87-c853-49f972f516d0",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-18",
+ "subject": {
+ "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "9687e19c-1d19-5cdf-f4c6-954aa05ea842"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-11-29T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "bc2c8a8f-a32d-92f0-0d6c-43b4d2980287",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-18",
+ "subject": {
+ "describedByEntityStatement": "17b29ec4-1a80-0acf-7207-50148bef2e43"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "ac13a555-2a3e-4bf8-e71d-f968e65f30ef"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-18T08:47:10.475Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "060c7f33-7ea5-769e-164a-d2683eebf983",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-18",
+ "subject": {
+ "describedByEntityStatement": "17b29ec4-1a80-0acf-7207-50148bef2e43"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "81e07925-b7f6-9d2f-0b6e-7b6889561510"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-18T08:47:10.570Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "95002ea4-69a1-da3f-4d34-4fad484a6fb7",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-02",
+ "subject": {
+ "describedByEntityStatement": "8ca194d4-ee72-56c4-e39d-0447dbcf0d17"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "7f9cc65e-8195-cb0a-08d3-6b99cce86690"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-02T06:05:38.072Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "ecc27092-3fda-2b48-4500-6cfcb61964f8",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-01",
+ "subject": {
+ "describedByEntityStatement": "099fd796-e9c1-d2a4-4efe-8843d421d827"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "95bec03f-6cf6-ba02-cda7-75ef614f9798"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-01T05:07:23.076Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "8729e7c2-9037-f289-057f-cb9a6e42f8e9",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-01",
+ "subject": {
+ "describedByEntityStatement": "099fd796-e9c1-d2a4-4efe-8843d421d827"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "2335891d-0bd6-e0ba-e0b2-27c58e383a38"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-01T05:07:23.152Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "4138bbc8-dc3c-e8ee-dfe8-f9876cb83500",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2022-06-04",
+ "subject": {
+ "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "f910bb52-6772-f634-698e-392402bd3dca"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-02-06T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "3cd06b78-3c1c-0cab-881b-cbbc36af1741",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-01-24",
+ "subject": {
+ "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "20815b26-efdc-a516-a905-7abdfa63d128"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "indirect",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-02-06T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "208d41ba-977c-6760-801e-77fdc9d47307",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-16",
+ "subject": {
+ "describedByEntityStatement": "06999bda-3717-8acb-719e-d6dd6d145c2a"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "5bd23434-6df3-b2cf-4f28-83f16e6fea7a"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2023-05-09T06:14:06.846Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-13",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-data.json b/tests/fixtures/rr-data.json
new file mode 100644
index 0000000..e30e317
--- /dev/null
+++ b/tests/fixtures/rr-data.json
@@ -0,0 +1,327 @@
+[
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "001GPB6A9XPE8XJICC14",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "5493001Z012YSB2A0K51",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-11-29T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2022-11-14T09:59:48.000Z",
+ "LastUpdateDate": "2023-05-18T15:41:20.212Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-18T15:48:53.604Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "001GPB6A9XPE8XJICC14",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "C7J4FOV6ELAVE39B7M82",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_SUBFUND_OF",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-11-29T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2022-11-14T09:59:48.000Z",
+ "LastUpdateDate": "2023-05-18T15:41:20.212Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-18T15:48:53.604Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/cgi-bin/browse-edgar?scd=series&CIK=0000722574&action=getcompany"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "00KLB2PFTM3060S2N216",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "YVC19L6NOWRA51XIKW06",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-18T08:47:10.475Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-11-05T18:17:00.000Z",
+ "LastUpdateDate": "2023-05-18T08:47:10.462Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-11T18:37:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/872323/000110465923007476/tm231669d1_485bpos.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "00KLB2PFTM3060S2N216",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "54930037R1JTV3ZYFY28",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_SUBFUND_OF",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-18T08:47:10.570Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-11-05T18:17:00.000Z",
+ "LastUpdateDate": "2023-05-18T08:47:10.559Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-11T18:37:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/cgi-bin/series?company=HARRIS+ASSOCIATES+INVESTMENT+TRUST"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "00QDBXDXLLF3W3JJJO36",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "549300FF2KD1XH54I851",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-02T06:05:38.072Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2021-03-23T21:33:00.000Z",
+ "LastUpdateDate": "2023-05-02T06:08:13.031Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-04-21T15:23:48.020Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/0001052118/000168386323000377/f24169d0.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "00X8DSV26QKJPKUT5B34",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "7HTL8AEQSEDX602FBU63",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-01T05:07:23.076Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-06-06T15:51:00.000Z",
+ "LastUpdateDate": "2023-05-01T05:07:23.062Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2023-12-22T00:00:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/Archives/edgar/data/912029/000174177323000891/c497k.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "00X8DSV26QKJPKUT5B34",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "549300S84WGKQCATYO76",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_SUBFUND_OF",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-01T05:07:23.152Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2017-08-25T01:37:00.000Z",
+ "LastUpdateDate": "2023-05-01T05:07:23.140Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2023-12-22T00:00:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/cgi-bin/browse-edgar?CIK=0000912029&action=getcompany&scd=series"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "010CMKZ3VON21WF2ZD45",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "ZZG38T0MDR3QY1ETUA76",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-01-01T00:00:00.000Z",
+ "EndDate": "2018-12-31T00:00:00.000Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-02-06T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-10-01T11:09:00.000Z",
+ "LastUpdateDate": "2022-06-04T01:00:00.000Z",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-06-03T13:04:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "ENTITY_SUPPLIED_ONLY",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "010CMKZ3VON21WF2ZD45",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "3C7474T6CDKPR9K6YT90",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2020-01-01T00:00:00.000Z",
+ "EndDate": "2020-12-31T00:00:00.000Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-02-06T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2020-01-01T00:00:00.000Z",
+ "EndDate": "2020-12-31T00:00:00.000Z",
+ "PeriodType": "DOCUMENT_FILING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "US_GAAP"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-10-01T11:09:00.000Z",
+ "LastUpdateDate": "2023-01-24T15:32:00.000Z",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-06-03T13:04:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "REGULATORY_FILING",
+ "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/40545/000004054521000011/ge-20201231.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "010G7UHBHEI87EKP0Q97",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "549300GC5MDF1KXYMP06",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2023-05-09T06:14:06.846Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-06-06T15:57:00.000Z",
+ "LastUpdateDate": "2023-05-16T14:21:08.886Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-06-27T00:00:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/Archives/edgar/data/1458460/000140508622000264/xslFormDX01/primary_doc.xml"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data-out.json b/tests/fixtures/rr-updates-data-out.json
new file mode 100644
index 0000000..4fdc133
--- /dev/null
+++ b/tests/fixtures/rr-updates-data-out.json
@@ -0,0 +1,106 @@
+[
+ {
+ "statementID": "00420996-34d4-e085-1366-d9bae2178613",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-07-18",
+ "subject": {
+ "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-11-29T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-20",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "502d54ca-ca55-843b-9db2-e516643563d6",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2022-08-04",
+ "subject": {
+ "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "f910bb52-6772-f634-698e-392402bd3dca"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-02-06T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-20",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "8fec0d1c-b31b-7632-7c47-901e10bcd367",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-03-24",
+ "subject": {
+ "describedByEntityStatement": "9f94abe2-349c-8e96-edaf-cf832eab1ac8"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "20815b26-efdc-a516-a905-7abdfa63d128"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "indirect",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-02-06T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-11-20",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data.json b/tests/fixtures/rr-updates-data.json
new file mode 100644
index 0000000..78023d7
--- /dev/null
+++ b/tests/fixtures/rr-updates-data.json
@@ -0,0 +1,117 @@
+[
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "001GPB6A9XPE8XJICC14",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "5493001Z012YSB2A0K51",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-11-29T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2022-11-14T09:59:48.000Z",
+ "LastUpdateDate": "2023-07-18T15:41:20.212Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-18T15:48:53.604Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "010CMKZ3VON21WF2ZD45",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "ZZG38T0MDR3QY1ETUA76",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-01-01T00:00:00.000Z",
+ "EndDate": "2018-12-31T00:00:00.000Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-02-06T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-10-01T11:09:00.000Z",
+ "LastUpdateDate": "2022-08-04T01:00:00.000Z",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-06-03T13:04:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "ENTITY_SUPPLIED_ONLY",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "010CMKZ3VON21WF2ZD45",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "3C7474T6CDKPR9K6YT90",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2020-01-01T00:00:00.000Z",
+ "EndDate": "2020-12-31T00:00:00.000Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-02-06T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2020-01-01T00:00:00.000Z",
+ "EndDate": "2020-12-31T00:00:00.000Z",
+ "PeriodType": "DOCUMENT_FILING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "US_GAAP"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2012-10-01T11:09:00.000Z",
+ "LastUpdateDate": "2023-03-24T15:32:00.000Z",
+ "RegistrationStatus": "LAPSED",
+ "NextRenewalDate": "2022-06-03T13:04:00.000Z",
+ "ManagingLOU": "EVK05KS7XY1DEII3R011",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "REGULATORY_FILING",
+ "ValidationReference": "https://www.sec.gov/ix?doc=/Archives/edgar/data/40545/000004054521000011/ge-20201231.htm"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data2-new-out.json b/tests/fixtures/rr-updates-data2-new-out.json
new file mode 100644
index 0000000..7f43921
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2-new-out.json
@@ -0,0 +1,352 @@
+[
+ {
+ "statementID": "9b2197d7-2d52-5b77-1356-c9d62b5ce670",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-08-23",
+ "subject": {
+ "describedByEntityStatement": "7239ce6e-d0c6-731a-30b5-b2157eb12419"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "97e2090a-9a29-3d48-3229-6cf390a8cf55"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-11-29T00:00:00.000Z"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "724871b2-e240-68c9-030c-c85370efa7c4",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-09-17",
+ "subject": {
+ "describedByEntityStatement": "d400beea-3cdc-a894-ce0d-9ce8fa67c30b"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-23T08:00:00+01:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "46af8fa4-1b3d-00db-c440-7b53bee06464",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-09-17",
+ "subject": {
+ "describedByEntityStatement": "19fb3b00-c801-7ff9-fae7-a1955c772e71"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-04-02T08:00:00+02:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "90ccd99f-086c-1bfe-211e-330266d50cb1",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-09-17",
+ "subject": {
+ "describedByEntityStatement": "1ff23b39-62f3-4744-da92-b11c2c3f8ff3"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-22T08:00:00+01:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "5729d9a2-f80d-b945-930f-0ed3c210f222",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-09-17",
+ "subject": {
+ "describedByEntityStatement": "a8866f73-572f-5ec5-6b42-2fe78b72f128"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-22T08:00:00+01:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "41e51ae9-3a31-e7ed-10a2-7bb13375ccf6",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-06-25",
+ "subject": {
+ "describedByEntityStatement": "296865b4-00a0-2f0e-47ac-c20037da1f30"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "426b5ddd-267f-0020-ae05-5dbc1e0f3341"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2022-10-24T08:00:00+02:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "c1aa1715-1cd3-b105-842a-1da48c0b5e30",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-02-11",
+ "subject": {
+ "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2019-06-30T00:00:00+00:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "82d4a9f1-5ee6-d620-1a85-e45323563dde",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-02-11",
+ "subject": {
+ "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "indirect",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2019-06-30T00:00:00+00:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "e6afa469-a581-82cd-e9b8-63763fae365d",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-04-30",
+ "subject": {
+ "describedByEntityStatement": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "657a9791-1cf0-e35c-a65f-0785e4685749"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "direct",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-06-30T00:00:00+00:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "5c126e1c-83f9-fece-2f82-70af13007c69",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-04-30",
+ "subject": {
+ "describedByEntityStatement": "64ced3b2-7c19-bd8f-3ff9-6eeb1ebe1d81"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "8304570e-5bbf-d8df-1674-51f9d2a1f354"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "indirect",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-06-30T00:00:00+00:00"
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2023-12-11",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data2-new.json b/tests/fixtures/rr-updates-data2-new.json
new file mode 100644
index 0000000..1d7fa9f
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2-new.json
@@ -0,0 +1,374 @@
+[
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "001GPB6A9XPE8XJICC14",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "5493001Z012YSB2A0K51",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-11-29T00:00:00.000Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2022-11-14T09:59:48.000Z",
+ "LastUpdateDate": "2023-08-23T18:46:01.213Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-18T15:48:53.604Z",
+ "ManagingLOU": "5493001KJTIIGC8Y1R12",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS",
+ "ValidationReference": "https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002337",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-23T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:04.159+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002143",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-04-02T08:00:00+02:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:02.482+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002240",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-22T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:03.502+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002434",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-22T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-09-17T08:31:04.659+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "549300JB1P61FUTPEZ75",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2022-10-24T08:00:00+02:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2022-01-01T08:00:00+01:00",
+ "EndDate": "2022-12-31T08:00:00+01:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "IFRS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-05-22T12:27:52.655+02:00",
+ "LastUpdateDate": "2023-06-25T08:30:56.059+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-30T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "EndDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-01-10T17:29:10+00:00",
+ "LastUpdateDate": "2023-02-11T11:21:01+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "EndDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-01-10T17:29:10+00:00",
+ "LastUpdateDate": "2023-02-11T11:21:01+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "98450051BS9C610A8T78",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "EndDate": "2020-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T00:00:00+00:00",
+ "LastUpdateDate": "2023-04-30T14:18:04+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "98450051BS9C610A8T78",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "EndDate": "2020-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T00:00:00+00:00",
+ "LastUpdateDate": "2023-04-30T14:18:04+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ },
+ "Extension": {
+ "Deletion": {
+ "DeletedAt": "2023-11-01T01:39:19Z"
+ }
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data2-new.xml b/tests/fixtures/rr-updates-data2-new.xml
new file mode 100644
index 0000000..19ea704
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2-new.xml
@@ -0,0 +1,411 @@
+
+
+ 2023-11-06T09:29:01Z
+ GLEIF_FULL_PUBLISHED
+ 407916
+
+
+
+
+
+ 001GPB6A9XPE8XJICC14
+ LEI
+
+
+ 5493001Z012YSB2A0K51
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2012-11-29T00:00:00.000Z
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2022-11-14T09:59:48.000Z
+ 2023-08-23T18:46:01.213Z
+ PUBLISHED
+ 2024-05-18T15:48:53.604Z
+ 5493001KJTIIGC8Y1R12
+ FULLY_CORROBORATED
+ SUPPORTING_DOCUMENTS
+ https://www.sec.gov/Archives/edgar/data/722574/000072257422000127/filing4699.htm
+
+
+
+
+
+ 097900BEJX0000002337
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-23T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:04.159+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002143
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2012-04-02T08:00:00+02:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:02.482+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002240
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-22T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:03.502+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002434
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-22T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-09-17T08:31:04.659+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BHF50000076475
+ LEI
+
+
+ 549300JB1P61FUTPEZ75
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2022-10-24T08:00:00+02:00
+ RELATIONSHIP_PERIOD
+
+
+ 2022-01-01T08:00:00+01:00
+ 2022-12-31T08:00:00+01:00
+ ACCOUNTING_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ IFRS
+
+
+
+
+ 2023-05-22T12:27:52.655+02:00
+ 2023-06-25T08:30:56.059+02:00
+ PUBLISHED
+ 2024-05-30T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ SUPPORTING_DOCUMENTS
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2018-06-30T00:00:00+00:00
+ 2019-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2019-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2023-01-10T17:29:10+00:00
+ 2023-02-11T11:21:01+00:00
+ PUBLISHED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_ULTIMATELY_CONSOLIDATED_BY
+
+
+ 2018-06-30T00:00:00+00:00
+ 2019-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2019-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2023-01-10T17:29:10+00:00
+ 2023-02-11T11:21:01+00:00
+ PUBLISHED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 98450051BS9C610A8T78
+ LEI
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2019-06-30T00:00:00+00:00
+ 2020-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2018-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ 2019-05-15T00:00:00+00:00
+ 2023-04-30T14:18:04+00:00
+ PUBLISHED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 98450051BS9C610A8T78
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_ULTIMATELY_CONSOLIDATED_BY
+
+
+ 2019-06-30T00:00:00+00:00
+ 2020-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2018-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2019-05-15T00:00:00+00:00
+ 2023-04-30T14:18:04+00:00
+ PUBLISHED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+ 2023-11-01T01:39:19Z
+
+
+
+
+
diff --git a/tests/fixtures/rr-updates-data2-out.json b/tests/fixtures/rr-updates-data2-out.json
new file mode 100644
index 0000000..c7a275f
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2-out.json
@@ -0,0 +1,530 @@
+[
+ {
+ "statementID": "d4e3f1bd-05dc-6a36-6341-ed769a2672a4",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-08-17",
+ "subject": {
+ "describedByEntityStatement": "8a8e8ee2-5bb4-7582-9b79-d531b6b59c12"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-23T08:00:00+01:00",
+ "details": "LEI RelationshipType: IS_FUND-MANAGED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BEJX0000002337 is subject, 097900BHF50000076475 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "5ebf3d86-dcba-7749-b419-73c3e91cb497",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-08-17",
+ "subject": {
+ "describedByEntityStatement": "d60fa665-032c-6992-895f-66cdc7509909"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2012-04-02T08:00:00+02:00",
+ "details": "LEI RelationshipType: IS_FUND-MANAGED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BEJX0000002143 is subject, 097900BHF50000076475 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "7ed9c1c0-5f2d-acb9-a351-0dce16a4ca67",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-08-17",
+ "subject": {
+ "describedByEntityStatement": "64b41c35-f336-0010-ae81-785d33e65ffd"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-22T08:00:00+01:00",
+ "details": "LEI RelationshipType: IS_FUND-MANAGED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BEJX0000002240 is subject, 097900BHF50000076475 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "b08a654d-6a50-16c2-f93f-8dcb54621f55",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-08-17",
+ "subject": {
+ "describedByEntityStatement": "4f32f254-ec84-a32a-79d1-a036dfa789e6"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2005-03-22T08:00:00+01:00",
+ "details": "LEI RelationshipType: IS_FUND-MANAGED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BEJX0000002434 is subject, 097900BHF50000076475 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "458cea07-b906-bcf2-1257-ab270a8010b0",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-05-25",
+ "subject": {
+ "describedByEntityStatement": "1865157c-c635-bb1b-cebf-9edcf88c7938"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "eca994a4-c50a-62b4-1308-9077410ee7b2"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2022-10-24T08:00:00+02:00",
+ "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BHF50000076475 is subject, 549300JB1P61FUTPEZ75 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "e313cd90-2781-30bd-cfd3-39243fbf401d",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2020-07-16",
+ "subject": {
+ "describedByEntityStatement": "71da391b-9385-4a1c-c5fc-e040a762b172"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "fd9f3a92-ed00-95f2-b005-45772e4f40b7"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2014-11-06T00:00:00Z",
+ "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BJGO0000201513 is subject, 9845002B9874D50A4531 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "f3af408c-b46e-491c-0ccc-8928fcca3f98",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2020-07-16",
+ "subject": {
+ "describedByEntityStatement": "71da391b-9385-4a1c-c5fc-e040a762b172"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "fd9f3a92-ed00-95f2-b005-45772e4f40b7"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2014-11-06T00:00:00Z",
+ "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 097900BJGO0000201513 is subject, 9845002B9874D50A4531 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "23f5dca9-1954-91e8-787e-8e80460d5aeb",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-01-11",
+ "subject": {
+ "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2019-06-30T00:00:00+00:00",
+ "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 2138002YAB7Z9KO21397 is subject, 984500501A1B1045PB30 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "10b394b0-8403-9183-1b28-26ea4ccf4567",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-01-11",
+ "subject": {
+ "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2019-06-30T00:00:00+00:00",
+ "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 2138002YAB7Z9KO21397 is subject, 984500501A1B1045PB30 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "65665354-d0c9-457d-f8b7-11eff538a891",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-03-30",
+ "subject": {
+ "describedByEntityStatement": "78205d04-b700-2c35-ea8c-af8cb534bc11"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "63a68db5-2ba9-2d26-0acc-2303ca07035e"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-06-30T00:00:00+00:00",
+ "details": "LEI RelationshipType: IS_DIRECTLY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 98450051BS9C610A8T78 is subject, 2138002YAB7Z9KO21397 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ },
+ {
+ "statementID": "77e6d2fb-f1e9-af58-3d47-45f261609ba0",
+ "statementType": "ownershipOrControlStatement",
+ "statementDate": "2023-03-30",
+ "subject": {
+ "describedByEntityStatement": "78205d04-b700-2c35-ea8c-af8cb534bc11"
+ },
+ "interestedParty": {
+ "describedByEntityStatement": "6c029f06-1c31-3864-3508-8c393bd6f61e"
+ },
+ "interests": [
+ {
+ "type": "other-influence-or-control",
+ "interestLevel": "unknown",
+ "beneficialOwnershipOrControl": false,
+ "startDate": "2018-06-30T00:00:00+00:00",
+ "details": "LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY"
+ }
+ ],
+ "annotations": [
+ {
+ "motivation": "commenting",
+ "description": "Describes GLEIF relationship: 98450051BS9C610A8T78 is subject, 984500501A1B1045PB30 is interested party",
+ "statementPointerTarget": "/",
+ "creationDate": "2024-02-16",
+ "createdBy": {
+ "name": "Open Ownership",
+ "uri": "https://www.openownership.org"
+ }
+ }
+ ],
+ "publicationDetails": {
+ "publicationDate": "2024-02-16",
+ "bodsVersion": "0.2",
+ "license": "https://register.openownership.org/terms-and-conditions",
+ "publisher": {
+ "name": "OpenOwnership Register",
+ "url": "https://register.openownership.org"
+ }
+ },
+ "source": {
+ "type": [
+ "officialRegister",
+ "verified"
+ ],
+ "description": "GLEIF"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data2.json b/tests/fixtures/rr-updates-data2.json
new file mode 100644
index 0000000..c37d4b5
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2.json
@@ -0,0 +1,439 @@
+[
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002337",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-23T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:04.159+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002143",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2012-04-02T08:00:00+02:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:02.482+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002240",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-22T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:03.502+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BEJX0000002434",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_FUND-MANAGED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2005-03-22T08:00:00+01:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE"
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2014-09-23T00:00:00+02:00",
+ "LastUpdateDate": "2023-08-17T08:31:04.659+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-09-26T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "OTHER_OFFICIAL_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BHF50000076475",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "549300JB1P61FUTPEZ75",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2022-10-24T08:00:00+02:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2022-01-01T08:00:00+01:00",
+ "EndDate": "2022-12-31T08:00:00+01:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "IFRS"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-05-22T12:27:52.655+02:00",
+ "LastUpdateDate": "2023-05-25T08:30:56.059+02:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-30T00:00:00+02:00",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "SUPPORTING_DOCUMENTS"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BJGO0000201513",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "9845002B9874D50A4531",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2020-07-16T00:00:00Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2014-11-06T00:00:00Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2020-07-16T00:00:00Z",
+ "PeriodType": "DOCUMENT_FILING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "IFRS"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "1.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2020-07-16T00:00:00Z",
+ "LastUpdateDate": "2020-07-16T00:00:00Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2021-06-18T00:00:00Z",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "097900BJGO0000201513",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "9845002B9874D50A4531",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2020-07-16T00:00:00Z",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2014-11-06T00:00:00Z",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ },
+ {
+ "StartDate": "2020-07-16T00:00:00Z",
+ "PeriodType": "DOCUMENT_FILING_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "IFRS"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "1.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2020-07-16T00:00:00Z",
+ "LastUpdateDate": "2020-07-16T00:00:00Z",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2021-06-18T00:00:00Z",
+ "ManagingLOU": "097900BEFH0000000217",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "EndDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-01-10T17:29:10+00:00",
+ "LastUpdateDate": "2023-01-11T11:21:01+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "EndDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2023-01-10T17:29:10+00:00",
+ "LastUpdateDate": "2023-01-11T11:21:01+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-01-13T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "98450051BS9C610A8T78",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "2138002YAB7Z9KO21397",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_DIRECTLY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "EndDate": "2020-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T00:00:00+00:00",
+ "LastUpdateDate": "2023-03-30T14:18:04+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ },
+ {
+ "Relationship": {
+ "StartNode": {
+ "NodeID": "98450051BS9C610A8T78",
+ "NodeIDType": "LEI"
+ },
+ "EndNode": {
+ "NodeID": "984500501A1B1045PB30",
+ "NodeIDType": "LEI"
+ },
+ "RelationshipType": "IS_ULTIMATELY_CONSOLIDATED_BY",
+ "RelationshipPeriods": [
+ {
+ "StartDate": "2019-06-30T00:00:00+00:00",
+ "EndDate": "2020-06-30T00:00:00+00:00",
+ "PeriodType": "ACCOUNTING_PERIOD"
+ },
+ {
+ "StartDate": "2018-06-30T00:00:00+00:00",
+ "PeriodType": "RELATIONSHIP_PERIOD"
+ }
+ ],
+ "RelationshipStatus": "ACTIVE",
+ "RelationshipQualifiers": [
+ {
+ "QualifierDimension": "ACCOUNTING_STANDARD",
+ "QualifierCategory": "OTHER_ACCOUNTING_STANDARD"
+ }
+ ],
+ "RelationshipQuantifiers": [
+ {
+ "MeasurementMethod": "ACCOUNTING_CONSOLIDATION",
+ "QuantifierAmount": "100.00",
+ "QuantifierUnits": "PERCENTAGE"
+ }
+ ]
+ },
+ "Registration": {
+ "InitialRegistrationDate": "2019-05-15T00:00:00+00:00",
+ "LastUpdateDate": "2023-03-30T14:18:04+00:00",
+ "RegistrationStatus": "PUBLISHED",
+ "NextRenewalDate": "2024-05-15T00:00:00+00:00",
+ "ManagingLOU": "529900T8BM49AURSDO55",
+ "ValidationSources": "FULLY_CORROBORATED",
+ "ValidationDocuments": "ACCOUNTS_FILING"
+ }
+ }
+]
diff --git a/tests/fixtures/rr-updates-data2.xml b/tests/fixtures/rr-updates-data2.xml
new file mode 100644
index 0000000..c72b60d
--- /dev/null
+++ b/tests/fixtures/rr-updates-data2.xml
@@ -0,0 +1,479 @@
+
+
+ 2023-11-06T09:29:01Z
+ GLEIF_FULL_PUBLISHED
+ 407916
+
+
+
+
+
+ 097900BEJX0000002337
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-23T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:04.159+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002143
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2012-04-02T08:00:00+02:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:02.482+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002240
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-22T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:03.502+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BEJX0000002434
+ LEI
+
+
+ 097900BHF50000076475
+ LEI
+
+ IS_FUND-MANAGED_BY
+
+
+ 2005-03-22T08:00:00+01:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ 2014-09-23T00:00:00+02:00
+ 2023-08-17T08:31:04.659+02:00
+ PUBLISHED
+ 2024-09-26T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ OTHER_OFFICIAL_DOCUMENTS
+
+
+
+
+
+ 097900BHF50000076475
+ LEI
+
+
+ 549300JB1P61FUTPEZ75
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2022-10-24T08:00:00+02:00
+ RELATIONSHIP_PERIOD
+
+
+ 2022-01-01T08:00:00+01:00
+ 2022-12-31T08:00:00+01:00
+ ACCOUNTING_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ IFRS
+
+
+
+
+ 2023-05-22T12:27:52.655+02:00
+ 2023-05-25T08:30:56.059+02:00
+ PUBLISHED
+ 2024-05-30T00:00:00+02:00
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ SUPPORTING_DOCUMENTS
+
+
+
+
+
+ 097900BJGO0000201513
+ LEI
+
+
+ 9845002B9874D50A4531
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2020-07-16T00:00:00Z
+ ACCOUNTING_PERIOD
+
+
+ 2014-11-06T00:00:00Z
+ RELATIONSHIP_PERIOD
+
+
+ 2020-07-16T00:00:00Z
+ DOCUMENT_FILING_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ IFRS
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 1.00
+ PERCENTAGE
+
+
+
+
+ 2020-07-16T00:00:00Z
+ 2020-07-16T00:00:00Z
+ PUBLISHED
+ 2021-06-18T00:00:00Z
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 097900BJGO0000201513
+ LEI
+
+
+ 9845002B9874D50A4531
+ LEI
+
+ IS_ULTIMATELY_CONSOLIDATED_BY
+
+
+ 2020-07-16T00:00:00Z
+ ACCOUNTING_PERIOD
+
+
+ 2014-11-06T00:00:00Z
+ RELATIONSHIP_PERIOD
+
+
+ 2020-07-16T00:00:00Z
+ DOCUMENT_FILING_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ IFRS
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 1.00
+ PERCENTAGE
+
+
+
+
+ 2020-07-16T00:00:00Z
+ 2020-07-16T00:00:00Z
+ PUBLISHED
+ 2021-06-18T00:00:00Z
+ 097900BEFH0000000217
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2018-06-30T00:00:00+00:00
+ 2019-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2019-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2023-01-10T17:29:10+00:00
+ 2023-01-11T11:21:01+00:00
+ PUBLISHED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_ULTIMATELY_CONSOLIDATED_BY
+
+
+ 2018-06-30T00:00:00+00:00
+ 2019-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2019-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2023-01-10T17:29:10+00:00
+ 2023-01-11T11:21:01+00:00
+ PUBLISHED
+ 2024-01-13T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 98450051BS9C610A8T78
+ LEI
+
+
+ 2138002YAB7Z9KO21397
+ LEI
+
+ IS_DIRECTLY_CONSOLIDATED_BY
+
+
+ 2019-06-30T00:00:00+00:00
+ 2020-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2018-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ 2019-05-15T00:00:00+00:00
+ 2023-03-30T14:18:04+00:00
+ PUBLISHED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
+
+ 98450051BS9C610A8T78
+ LEI
+
+
+ 984500501A1B1045PB30
+ LEI
+
+ IS_ULTIMATELY_CONSOLIDATED_BY
+
+
+ 2019-06-30T00:00:00+00:00
+ 2020-06-30T00:00:00+00:00
+ ACCOUNTING_PERIOD
+
+
+ 2018-06-30T00:00:00+00:00
+ RELATIONSHIP_PERIOD
+
+
+ ACTIVE
+
+
+ ACCOUNTING_STANDARD
+ OTHER_ACCOUNTING_STANDARD
+
+
+
+
+ ACCOUNTING_CONSOLIDATION
+ 100.00
+ PERCENTAGE
+
+
+
+
+ 2019-05-15T00:00:00+00:00
+ 2023-03-30T14:18:04+00:00
+ PUBLISHED
+ 2024-05-15T00:00:00+00:00
+ 529900T8BM49AURSDO55
+ FULLY_CORROBORATED
+ ACCOUNTS_FILING
+
+
+
+
diff --git a/tests/test_elasticsearch.py b/tests/test_elasticsearch.py
index 8924c24..0a54cd9 100644
--- a/tests/test_elasticsearch.py
+++ b/tests/test_elasticsearch.py
@@ -1,10 +1,13 @@
import os
import sys
import time
+import json
from unittest.mock import patch, Mock
+import asyncio
import pytest
-from bodspipelines.infrastructure.storage import ElasticStorage
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
match_lei, match_rr, match_repex,
id_lei, id_rr, id_repex)
@@ -14,6 +17,7 @@
"repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
def set_environment_variables():
+ """Setup environment variables"""
os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
os.environ['ELASTICSEARCH_HOST'] = 'localhost'
os.environ['ELASTICSEARCH_PORT'] = '9876'
@@ -41,18 +45,98 @@ def lei_item():
'ValidationSources': 'FULLY_CORROBORATED',
'ValidationAuthority': {'ValidationAuthorityID': 'RA000670', 'ValidationAuthorityEntityID': '43846696'}}}
-def test_lei_storage_new(lei_item):
+
+@pytest.fixture
+def lei_list():
+ """List of entity LEIs"""
+ return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216',
+ '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53',
+ '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11']
+
+@pytest.fixture
+def last_update_list():
+ """List of last update datetimes"""
+ return ["2023-05-18T15:41:20.212Z", "2020-07-17T12:40:00.000Z", "2022-07-22T09:32:00.000Z", "2022-10-24T21:31:00.000Z",
+ "2023-05-18T17:24:00.540Z", "2023-05-03T07:03:05.620Z", "2019-04-22T21:31:00.000Z", "2023-05-10T04:42:18.790Z",
+ "2020-07-17T12:40:00.000Z", "2020-07-24T19:29:00.000Z", "2023-03-10T13:08:56+01:00", "2023-02-02T09:07:52.390Z",
+ "2023-04-25T13:18:00Z"]
+
+
+@pytest.fixture
+def json_data():
+ """LEI JSON data"""
+ with open("tests/fixtures/lei-data.json", "r") as read_file:
+ return json.load(read_file)
+
+
+@pytest.mark.asyncio
+async def test_lei_storage_new(lei_item):
"""Test storing a new LEI-CDF v3.1 record in elasticsearch"""
- with patch('bodspipelines.infrastructure.clients.elasticsearch_client.Elasticsearch') as mock_es:
- mock_es.return_value.search.return_value = {"hits": {"total": {"value": 1, "relation": "eq"}, "hits": []}}
+ with patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es:
+ search_future = asyncio.Future()
+ search_future.set_result({"hits": {"total": {"value": 1, "relation": "eq"}, "hits": []}})
+ mock_es.return_value.search.return_value = search_future
+ index_future = asyncio.Future()
+ index_future.set_result(None)
+ mock_es.return_value.index.return_value = index_future
set_environment_variables()
- storage = ElasticStorage(indexes=index_properties)
- assert storage.process(lei_item, 'lei') == lei_item
+ storage = Storage(storage=ElasticsearchClient(indexes=index_properties))
+ await storage.setup()
+ assert await storage.process(lei_item, 'lei') == lei_item
+
-def test_lei_storage_existing(lei_item):
+@pytest.mark.asyncio
+async def test_lei_storage_existing(lei_item):
"""Test trying to store LEI-CDF v3.1 record which is already in elasticsearch"""
- with patch('bodspipelines.infrastructure.clients.elasticsearch_client.Elasticsearch') as mock_es:
- mock_es.return_value.search.return_value = {"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [lei_item]}}
+ with patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es:
+ search_future = asyncio.Future()
+ search_future.set_result({"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [lei_item]}})
+ mock_es.return_value.search.return_value = search_future
+ index_future = asyncio.Future()
+ index_future.set_result(None)
+ mock_es.return_value.index.return_value = index_future
+ set_environment_variables()
+ storage = Storage(storage=ElasticsearchClient(indexes=index_properties))
+ await storage.setup()
+ assert await storage.process(lei_item, 'lei') == False
+
+
+@pytest.mark.asyncio
+async def test_lei_bulk_storage_new(lei_list, last_update_list, json_data):
+ """Test ingest pipeline stage on LEI-CDF v3.1 records"""
+ with patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb:
+ async def result():
+ for lei, last in zip(lei_list, last_update_list):
+ yield (True, {'create': {'_id': f"{lei}_{last}"}})
+ mock_sb.return_value = result()
+ set_environment_variables()
+ storage = Storage(storage=ElasticsearchClient(indexes=index_properties))
+ await storage.setup()
+ async def json_stream():
+ for d in json_data:
+ yield d
+ count = 0
+ async for result in storage.process_batch(json_stream(), 'lei'):
+ assert result == json_data[count]
+ count += 1
+ assert count == 13
+
+
+@pytest.mark.asyncio
+async def test_lei_bulk_storage_existing(lei_list, last_update_list, json_data):
+ """Test ingest pipeline stage on LEI-CDF v3.1 records"""
+ with patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb:
+ async def result():
+ for lei, last in zip(lei_list, last_update_list):
+ yield (False, {'create': {'_id': f"{lei}_{last}"}})
+ mock_sb.return_value = result()
set_environment_variables()
- storage = ElasticStorage(indexes=index_properties)
- assert storage.process(lei_item, 'lei') == False
+ storage = Storage(storage=ElasticsearchClient(indexes=index_properties))
+ await storage.setup()
+ async def json_stream():
+ for d in json_data:
+ yield d
+ count = 0
+ async for result in storage.process_batch(json_stream(), 'lei'):
+ count += 1
+ assert count == 0
diff --git a/tests/test_gleif_data.py b/tests/test_gleif_data.py
new file mode 100644
index 0000000..1fb8f03
--- /dev/null
+++ b/tests/test_gleif_data.py
@@ -0,0 +1,225 @@
+from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
+from pathlib import Path
+import random
+import zipfile
+import io
+import pytest
+
+from bodspipelines.infrastructure.processing.bulk_data import BulkData
+
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData
+
+from unittest.mock import patch, Mock, MagicMock
+
+class TestGLEIFData:
+ """Test download of single file"""
+ source = 'lei'
+
+ @pytest.fixture(scope="class")
+ def temp_dir(self, tmp_path_factory):
+ """Fixture to create temporary directory"""
+ return tmp_path_factory.getbasetemp()
+
+ @pytest.fixture(scope="class")
+ def stage_dir(self, temp_dir):
+ """Fixture to create subdirectory"""
+ output_dir = Path(temp_dir) / "data" / self.source
+ output_dir.mkdir(parents=True)
+ return output_dir
+
+ def test_gleif_data(self, stage_dir):
+ """Test downloading GLEIF data"""
+ with (patch('bodspipelines.infrastructure.utils.download_delayed') as mock_download,
+ patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests):
+ def download_delayed(url):
+ def download(param):
+ fn = param.rsplit('/', 1)[-1]
+ print(fn)
+ return fn
+ return partial(download, url)
+ mock_download.side_effect = download_delayed
+ def zip_file(fn):
+ file = io.BytesIO()
+ zf = zipfile.ZipFile(file, mode='w')
+ zf.writestr(fn, '')
+ zf.close()
+ return bytes(file.getbuffer())
+ def requests_get(url, stream=True):
+ fn = url.rsplit('/', 1)[-1]
+ xfn = fn.split('.zip')[0]
+ mock_return = Mock()
+ mock_return.headers = {'content-disposition': f"filename={fn}",
+ 'content-length': 10000}
+ mock_return.iter_content.return_value = iter([zip_file(xfn)])
+ mock_context = Mock()
+ mock_context.__enter__ = Mock(return_value=mock_return)
+ mock_context.__exit__ = Mock(return_value=False)
+ return mock_context
+ mock_requests.side_effect = requests_get
+
+ origin=BulkData(display="LEI-CDF v3.1",
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"),
+ size=41491,
+ directory="lei-cdf")
+ count = 0
+ for fn in origin.prepare(stage_dir, self.source):
+ assert fn.endswith("-gleif-goldencopy-lei2-golden-copy.xml")
+ print([x for x in (stage_dir/'lei-cdf').iterdir()])
+ assert (stage_dir/'lei-cdf'/fn).exists()
+ count += 1
+ assert count == 1
+
+class TestGLEIFDataUpdateMonths:
+ """Test download of updates after over a month"""
+ source = 'lei'
+
+ @pytest.fixture(scope="class")
+ def temp_dir(self, tmp_path_factory):
+ """Fixture to create temporary directory"""
+ return tmp_path_factory.getbasetemp()
+
+ @pytest.fixture(scope="class")
+ def stage_dir(self, temp_dir):
+ """Fixture to create subdirectory"""
+ output_dir = Path(temp_dir) / "data2" / self.source
+ output_dir.mkdir(parents=True)
+ return output_dir
+
+ def test_gleif_data(self, stage_dir):
+ """Test downloading GLEIF data"""
+ with (patch('bodspipelines.infrastructure.utils.download') as mock_download,
+ patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests):
+ def download(url):
+ url_base = "https://goldencopy.gleif.org/storage/golden-copy-files/2023/06/13/795336/"
+ d = datetime.now().strftime("%Y%m%d")
+ t = f'{[x for x in (0, 800, 1600) if x < int(datetime.now().strftime("%H%M"))][-1]:04d}'
+ return {'data': {'publish_date': f"{d} {t}",
+ 'delta_files': {
+ 'LastMonth': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-month.xml.zip"}},
+ 'LastWeek': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-week.xml.zip"}},
+ 'LastDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-day.xml.zip"}},
+ 'IntraDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-intra-day.xml.zip"}},
+ }}}
+ mock_download.side_effect = download
+ def zip_file(fn):
+ file = io.BytesIO()
+ zf = zipfile.ZipFile(file, mode='w')
+ zf.writestr(fn, '')
+ zf.close()
+ return bytes(file.getbuffer())
+ def requests_get(url, stream=True):
+ fn = url.rsplit('/', 1)[-1]
+ xfn = fn.split('.zip')[0]
+ mock_return = Mock()
+ mock_return.headers = {'content-disposition': f"filename={fn}",
+ 'content-length': 10000}
+ mock_return.iter_content.return_value = iter([zip_file(xfn)])
+ mock_context = Mock()
+ mock_context.__enter__ = Mock(return_value=mock_return)
+ mock_context.__exit__ = Mock(return_value=False)
+ return mock_context
+ mock_requests.side_effect = requests_get
+ last_update_date = (datetime.now() + relativedelta(months=-1) - timedelta(days=5)).strftime("%Y-%m-%d")
+ last_update_time = f"{[x for x in (0, 8, 16) if int(datetime.now().strftime('%H00')) > x][-1]:02d}:00:00"
+
+ origin=BulkData(display="LEI-CDF v3.1",
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"),
+ size=41491,
+ directory="lei-cdf")
+ print(f'{last_update_date} {last_update_time}')
+ count = 0
+ for fn in origin.prepare(stage_dir, self.source, last_update=f'{last_update_date} {last_update_time}'):
+ print(fn)
+ if count == 0:
+ assert fn.endswith("-gleif-goldencopy-lei2-last-month.xml")
+ first = fn
+ else:
+ assert fn.endswith("-gleif-goldencopy-lei2-intra-day.xml")
+ print([x for x in (stage_dir/'lei-cdf').iterdir()])
+ assert (stage_dir/'lei-cdf'/fn).exists()
+ count += 1
+ first_date = datetime.strptime(first.split("-gleif")[0], "%Y%m%d-%H%M")
+ last_date = datetime.strptime(f'{last_update_date} {last_update_time}', "%Y-%m-%d %H:%M:%S")
+ delta = relativedelta(first_date, last_date)
+ assert count == 1 + 3*delta.days + delta.hours/8
+ assert len([x for x in (stage_dir/'lei-cdf').iterdir() if x.suffix == ".xml"]) == 1 + 3*delta.days + delta.hours/8
+
+
+class TestGLEIFDataUpdateWeeks:
+ """Test download of updates after less than a month"""
+ source = 'lei'
+
+ @pytest.fixture(scope="class")
+ def temp_dir(self, tmp_path_factory):
+ """Fixture to create temporary directory"""
+ return tmp_path_factory.getbasetemp()
+
+ @pytest.fixture(scope="class")
+ def stage_dir(self, temp_dir):
+ """Fixture to create subdirectory"""
+ output_dir = Path(temp_dir) / "data3" / self.source
+ output_dir.mkdir(parents=True)
+ return output_dir
+
+ def test_gleif_data(self, stage_dir):
+ """Test downloading GLEIF data"""
+ with (patch('bodspipelines.infrastructure.utils.download') as mock_download,
+ patch('bodspipelines.infrastructure.processing.bulk_data.requests.get') as mock_requests):
+ def download(url):
+ url_base = "https://goldencopy.gleif.org/storage/golden-copy-files/2023/06/13/795336/"
+ d = datetime.now().strftime("%Y%m%d")
+ t = f'{[x for x in (0, 800, 1600) if x < int(datetime.now().strftime("%H%M"))][-1]:04d}'
+ return {'data': {'publish_date': f"{d} {t}",
+ 'delta_files': {
+ 'LastMonth': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-month.xml.zip"}},
+ 'LastWeek': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-week.xml.zip"}},
+ 'LastDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-last-day.xml.zip"}},
+ 'IntraDay': {'xml': {'url': f"{url_base}{d}-{t}-gleif-goldencopy-lei2-intra-day.xml.zip"}},
+ }}}
+ mock_download.side_effect = download
+ def zip_file(fn):
+ file = io.BytesIO()
+ zf = zipfile.ZipFile(file, mode='w')
+ zf.writestr(fn, '')
+ zf.close()
+ return bytes(file.getbuffer())
+ def requests_get(url, stream=True):
+ fn = url.rsplit('/', 1)[-1]
+ xfn = fn.split('.zip')[0]
+ mock_return = Mock()
+ mock_return.headers = {'content-disposition': f"filename={fn}",
+ 'content-length': 10000}
+ mock_return.iter_content.return_value = iter([zip_file(xfn)])
+ mock_context = Mock()
+ mock_context.__enter__ = Mock(return_value=mock_return)
+ mock_context.__exit__ = Mock(return_value=False)
+ return mock_context
+ mock_requests.side_effect = requests_get
+ last_update_date = (datetime.now() - timedelta(days=19)).strftime("%Y-%m-%d")
+ last_update_time = f"{[x for x in (0, 8, 16) if int(datetime.now().strftime('%H00')) > x][-1]:02d}:00:00"
+
+ origin=BulkData(display="LEI-CDF v3.1",
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"),
+ size=41491,
+ directory="lei-cdf")
+ print(f'{last_update_date} {last_update_time}')
+ count = 0
+ for fn in origin.prepare(stage_dir, self.source, last_update=f'{last_update_date} {last_update_time}'):
+ print(fn)
+ if count < 2:
+ assert fn.endswith("-gleif-goldencopy-lei2-last-week.xml")
+ if count == 0: first = fn
+ #elif count < 7:
+ # assert fn.endswith("-gleif-goldencopy-lei2-last-day.xml")
+ else:
+ assert fn.endswith("-gleif-goldencopy-lei2-intra-day.xml") or fn.endswith("-gleif-goldencopy-lei2-last-day.xml")
+ print([x for x in (stage_dir/'lei-cdf').iterdir()])
+ assert (stage_dir/'lei-cdf'/fn).exists()
+ count += 1
+ first_date = datetime.strptime(first.split("-gleif")[0], "%Y%m%d-%H%M")
+ last_date = datetime.strptime(f'{last_update_date} {last_update_time}', "%Y-%m-%d %H:%M:%S")
+ delta = relativedelta(first_date, last_date)
+ print(delta)
+ assert count == delta.days//7 + delta.days%7 + delta.hours/8
+ assert len([x for x in (stage_dir/'lei-cdf').iterdir() if x.suffix == ".xml"]) == delta.days//7 + delta.days%7 + delta.hours/8
diff --git a/tests/test_gleif_source.py b/tests/test_gleif_source.py
new file mode 100644
index 0000000..6339d85
--- /dev/null
+++ b/tests/test_gleif_source.py
@@ -0,0 +1,137 @@
+import os
+import sys
+import time
+import datetime
+from pathlib import Path
+import json
+from unittest.mock import patch, Mock
+import asyncio
+import pytest
+
+from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
+from bodspipelines.infrastructure.inputs import KinesisInput
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
+from bodspipelines.infrastructure.processing.bulk_data import BulkData
+from bodspipelines.infrastructure.processing.xml_data import XMLData
+from bodspipelines.infrastructure.processing.json_data import JSONData
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, AddContentDate
+
+from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
+ match_lei, match_rr, match_repex,
+ id_lei, id_rr, id_repex)
+
+def set_environment_variables():
+ """Set environment variables"""
+ os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
+ os.environ['ELASTICSEARCH_HOST'] = 'localhost'
+ os.environ['ELASTICSEARCH_PORT'] = '9876'
+ os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
+
+index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
+ "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
+ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
+
+# Identify type of GLEIF data
+def identify_gleif(item):
+ if 'Entity' in item:
+ return 'lei'
+ elif 'Relationship' in item:
+ return 'rr'
+ elif 'ExceptionCategory' in item:
+ return 'repex'
+
+@pytest.fixture
+def repex_xml_data_file():
+ """GLEIF Repex XML data"""
+ return Path("tests/fixtures/repex-data.xml")
+
+@pytest.fixture
+def repex_list():
+ """GLEIF Repex id list"""
+ return ["001GPB6A9XPE8XJICC14_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None",
+ "004L5FPTUREIWK9T2N63_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "00EHHQ2ZHDCFXJCPCL46_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "00GBW0Z2GYIER7DHDS71_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "00KLB2PFTM3060S2N216_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None",
+ "00QDBXDXLLF3W3JJJO36_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NO_KNOWN_PERSON_None",
+ "00TV1D5YIV5IDUGWBW29_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "00X8DSV26QKJPKUT5B34_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "010G7UHBHEI87EKP0Q97_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_CONSOLIDATING_None",
+ "01370W6ZIY66KQ4J3570_DIRECT_ACCOUNTING_CONSOLIDATION_PARENT_NON_PUBLIC_None"]
+
+def test_repex_ingest_stage(repex_list, repex_xml_data_file):
+ """Test ingest pipeline stage on Reporting Exceptions v2.1 XML records"""
+ with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd,
+ patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ mock_bd.return_value = repex_xml_data_file
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+ async def result():
+ for repex_id in repex_list:
+ yield (True, {'create': {'_id': f"{repex_id}"}})
+ mock_sb.return_value = result()
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+ consumer_close_future = asyncio.Future()
+ consumer_close_future.set_result(None)
+ mock_kni.return_value.close.return_value = consumer_close_future
+
+ set_environment_variables()
+
+ # Defintion of Reporting Exceptions v2.1 XML date source
+ repex_source = Source(name="repex",
+ origin=BulkData(display="Reporting Exceptions v2.1",
+ #url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/repex/latest"),
+ size=3954,
+ directory="rep-ex"),
+ datatype=XMLData(item_tag="Exception",
+ header_tag="Header",
+ namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"},
+ filter=['NextVersion', 'Extension']))
+
+ # GLEIF data: Store in Easticsearch and output new to Kinesis stream
+ output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)),
+ output=KinesisOutput(stream_name="gleif-test"))
+
+ # Definition of GLEIF data pipeline ingest stage
+ ingest_stage = Stage(name="ingest-test",
+ sources=[repex_source],
+ processors=[AddContentDate(identify=identify_gleif)],
+ outputs=[output_new]
+ )
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif-test", stages=[ingest_stage])
+
+ # Run pipelne
+ pipeline.process("ingest-test")
+
+ # Check item count
+ assert mock_kno.return_value.put.call_count == 10
+
+ item = mock_kno.return_value.put.call_args.args[0]
+
+ print(item)
+
+ assert item['LEI'] == '01370W6ZIY66KQ4J3570'
+ assert item['ExceptionCategory'] == 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT'
+ assert item['ExceptionReason'] == 'NON_PUBLIC'
+ assert item['ContentDate'] == '2023-06-09T09:03:29Z'
diff --git a/tests/test_gleif_transforms.py b/tests/test_gleif_transforms.py
index 4c22940..f606f06 100644
--- a/tests/test_gleif_transforms.py
+++ b/tests/test_gleif_transforms.py
@@ -2,9 +2,10 @@
import datetime
-from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, generate_statement_id
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods, generate_statement_id, entity_id
def validate_datetime(d):
+ """Test is valid datetime"""
try:
datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
return True
@@ -12,11 +13,13 @@ def validate_datetime(d):
return False
def validate_date_now(d):
+ """Test is today's date"""
return d == datetime.date.today().strftime('%Y-%m-%d')
def test_generate_statement_id():
"""Test the deterministic generation of statement IDs"""
- assert generate_statement_id('2138008UKA1QH5L5XM10','entity') == '9edfc53c-34c7-4e3f-b427-bbbf65806dd3'
+ data = {"LEI": '2138008UKA1QH5L5XM10', 'Registration': {'LastUpdateDate': '2022-10-25T08:00:54.268+02:00'}}
+ assert generate_statement_id(entity_id(data),'entity') == '9d35c198-74f2-92bb-9205-a7bf73aff594'
@pytest.fixture
def lei_data():
@@ -81,43 +84,49 @@ def repex_data_no_known_person():
return {'LEI': '029200013A5N6ZD0F605',
'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT',
'ExceptionReason': 'NO_KNOWN_PERSON',
- 'NextVersion': '',
- 'Extension': ''}
+ #'NextVersion': '',
+ #'Extension': '',
+ 'ContentDate': "2023-06-09T09:03:29Z"}
@pytest.fixture
def repex_data_non_consolidating():
"""Example NON_CONSOLIDATING Reporting Exception"""
return {'LEI': '2138008UKA1QH5L5XM10',
'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT',
- 'ExceptionReason': 'NON_CONSOLIDATING'}
+ 'ExceptionReason': 'NON_CONSOLIDATING',
+ 'ContentDate': "2023-06-09T09:03:29Z"}
@pytest.fixture
def repex_data_natural_persons():
"""Example NATURAL_PERSONS Reporting Exception"""
return {'LEI': '213800RJPV1SI7G2HW19',
'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT',
- 'ExceptionReason': 'NATURAL_PERSONS'}
+ 'ExceptionReason': 'NATURAL_PERSONS',
+ 'ContentDate': "2023-06-09T09:03:29Z"}
@pytest.fixture
def repex_data_non_public():
"""Example NON_PUBLIC Reporting Exception"""
return {'LEI': '213800ZSKA23GF6L3F24',
'ExceptionCategory': 'ULTIMATE_ACCOUNTING_CONSOLIDATION_PARENT',
- 'ExceptionReason': 'NON_PUBLIC'}
+ 'ExceptionReason': 'NON_PUBLIC',
+ 'ContentDate': "2023-06-09T09:03:29Z"}
@pytest.fixture
def repex_data_no_lei():
"""Example NO_LEI Reporting Exception"""
return {'LEI': '213800QTG16IUUZ6WD32',
'ExceptionCategory': 'DIRECT_ACCOUNTING_CONSOLIDATION_PARENT',
- 'ExceptionReason': 'NO_LEI'}
+ 'ExceptionReason': 'NO_LEI',
+ 'ContentDate': "2023-06-09T09:03:29Z"}
-def test_lei_transform(lei_data):
+@pytest.mark.asyncio
+async def test_lei_transform(lei_data):
"""Test transformation of LEI-CDF v3.1 data to BODS entity statement"""
transform = Gleif2Bods()
- for bods_data in transform.process(lei_data, 'lei'):
+ async for bods_data in transform.process(lei_data, 'lei', {}):
print(bods_data)
- assert bods_data['statementID'] == '659b3c3a-3ac8-b5bd-6480-e56dff141fa9'
+ assert bods_data['statementID'] == 'ca229e51-cb6b-11b8-4ed7-48935aed30ad'
assert bods_data['statementType'] == 'entityStatement'
assert bods_data['statementDate'] == '2022-10-25'
assert bods_data['entityType'] == 'registeredEntity'
@@ -139,17 +148,19 @@ def test_lei_transform(lei_data):
if not 'bods_data' in locals():
raise Exception("No statements produced")
-def test_rr_transform(rr_data):
+@pytest.mark.asyncio
+async def test_rr_transform(rr_data):
"""Test transformation of RR-CDF v2.1 data to BODS ownership or control statement"""
transform = Gleif2Bods()
- for bods_data in transform.process(rr_data, 'rr'):
- assert bods_data['statementID'] == '7dc21488-bc9e-407c-293b-05578875b12f'
+ async for bods_data in transform.process(rr_data, 'rr', {}):
+ assert bods_data['statementID'] == '3f31f4cb-16dd-a251-144f-b871b330d5eb'
assert bods_data['statementType'] == 'ownershipOrControlStatement'
assert bods_data['statementDate'] == '2018-03-23'
assert bods_data['subject'] == {'describedByEntityStatement': 'c892c662-2904-0c7d-5978-07108d102c33'}
assert bods_data['interestedParty'] == {'describedByEntityStatement': '7d72f25e-910f-1f28-714d-7a761cbbc5de'}
assert bods_data['interests'] == [{'type': 'other-influence-or-control',
- 'interestLevel': 'indirect',
+ 'interestLevel': 'unknown',
+ 'details': 'LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY',
'beneficialOwnershipOrControl': False,
'startDate': '2018-03-23T00:00:00Z'}]
assert validate_date_now(bods_data['publicationDetails']['publicationDate'])
@@ -161,20 +172,21 @@ def test_rr_transform(rr_data):
if not 'bods_data' in locals():
raise Exception("No statements produced")
-def test_repex_transform_non_consolidating(repex_data_non_consolidating):
+@pytest.mark.asyncio
+async def test_repex_transform_non_consolidating(repex_data_non_consolidating):
"""Test transformation of NON_CONSOLIDATING Reporting Exception data to BODS statements"""
transform = Gleif2Bods()
- for bods_data in transform.process(repex_data_non_consolidating, 'repex'):
+ async for bods_data in transform.process(repex_data_non_consolidating, 'repex', {}):
print(bods_data)
if bods_data['statementType'] == 'entityStatement':
- assert bods_data['statementID'] == '27b6f7b1-4e16-618c-9a19-3a7ad016ee91'
+ assert bods_data['statementID'] == '35694dcd-31ba-23dd-65d6-5f97839ef3aa'
assert bods_data['unspecifiedEntityDetails'] == {'reason': 'interested-party-exempt-from-disclosure',
'description': 'From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control.'}
assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'}
assert bods_data['entityType'] == 'unknownEntity'
statementID = bods_data['statementID']
elif bods_data['statementType'] == 'ownershipOrControlStatement':
- assert bods_data['statementID'] == '47ab1442-5eb7-4d09-5745-e6330b89e6ed'
+ assert bods_data['statementID'] == '15ff987f-9e7b-2fb3-893c-f89acaa4ef6b'
assert bods_data['subject'] == {'describedByEntityStatement': '5cc33d9b-ee78-a7e1-de3a-8082f06c4798'}
assert bods_data['interestedParty'] == {'describedByEntityStatement': statementID}
assert bods_data['interests'] == [{'type': 'other-influence-or-control',
@@ -191,20 +203,21 @@ def test_repex_transform_non_consolidating(repex_data_non_consolidating):
if not 'bods_data' in locals():
raise Exception("No statements produced")
-def test_repex_transform_natural_persons(repex_data_natural_persons):
+@pytest.mark.asyncio
+async def test_repex_transform_natural_persons(repex_data_natural_persons):
"""Test transformation of NATURAL_PERSONS Reporting Exception data to BODS statements"""
transform = Gleif2Bods()
- for bods_data in transform.process(repex_data_natural_persons, 'repex'):
+ async for bods_data in transform.process(repex_data_natural_persons, 'repex', {}):
print(bods_data)
if bods_data['statementType'] == 'personStatement':
- assert bods_data['statementID'] == '12186e42-9384-8a9e-f7b3-279a399770fe'
+ assert bods_data['statementID'] == '4df7870a-3215-d80e-6bd4-a9c5e8c5b1f9'
assert bods_data['unspecifiedPersonDetails'] == {'reason': 'interested-party-exempt-from-disclosure',
'description': 'From LEI ExemptionReason `NATURAL_PERSONS`. An unknown natural person or persons controls an entity.'}
assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'}
assert bods_data['personType'] == 'unknownPerson'
statementID = bods_data['statementID']
elif bods_data['statementType'] == 'ownershipOrControlStatement':
- assert bods_data['statementID'] == 'd897e79f-6f03-f800-6fd0-495298cbea6a'
+ assert bods_data['statementID'] == '46cc4dca-9374-420d-7be9-8736a1170da0'
assert bods_data['subject'] == {'describedByEntityStatement': 'e579dc8d-8f7d-90bc-099a-38fa175bd494'}
assert bods_data['interestedParty'] == {'describedByPersonStatement': statementID}
assert bods_data['interests'] == [{'type': 'other-influence-or-control',
@@ -222,13 +235,14 @@ def test_repex_transform_natural_persons(repex_data_natural_persons):
if not 'bods_data' in locals():
raise Exception("No statements produced")
-def test_repex_transform_non_public(repex_data_non_public):
+@pytest.mark.asyncio
+async def test_repex_transform_non_public(repex_data_non_public):
"""Test transformation of NON_PUBLIC Reporting Exception data to BODS statements"""
transform = Gleif2Bods()
- for bods_data in transform.process(repex_data_non_public, 'repex'):
+ async for bods_data in transform.process(repex_data_non_public, 'repex', {}):
print(bods_data)
if bods_data['statementType'] in ('entityStatement', 'personStatement'):
- assert bods_data['statementID'] == '428aca2a-b344-78a8-4450-6cc726f26ecf'
+ assert bods_data['statementID'] == 'c63959b7-0528-1f24-018e-afb37ed15b9f'
if bods_data['statementType'] == 'entityStatement':
unspecified_type = 'unspecifiedEntityDetails'
else:
@@ -244,10 +258,10 @@ def test_repex_transform_non_public(repex_data_non_public):
if bods_data['statementType'] == 'entityStatement':
assert bods_data['entityType'] == 'unknownEntity'
else:
- assert bods_data['entityType'] == 'unknownPerson'
+ assert bods_data['personType'] == 'unknownPerson'
statementID = bods_data['statementID']
elif bods_data['statementType'] == 'ownershipOrControlStatement':
- assert bods_data['statementID'] == '984398d2-adf1-9111-f461-62f3101ff41d'
+ assert bods_data['statementID'] == 'd7c7f605-6ee0-e3b7-b46c-4c7a888bbdf0'
assert bods_data['subject'] == {'describedByEntityStatement': 'fc0205b0-dcf7-cedd-c071-fbd4708149f0'}
assert bods_data['interestedParty'] == {'describedByEntityStatement': statementID}
assert bods_data['interests'] == [{'type': 'other-influence-or-control',
@@ -264,3 +278,47 @@ def test_repex_transform_non_public(repex_data_non_public):
raise Exception(f"Incorrect statement type produced: {bods_data['statementType']}")
if not 'bods_data' in locals():
raise Exception("No statements produced")
+
+@pytest.mark.asyncio
+async def test_repex_transform_no_known_person(repex_data_no_known_person):
+ """Test transformation of NO_KNOWN_PERSON Reporting Exception data to BODS statements"""
+ transform = Gleif2Bods()
+ async for bods_data in transform.process(repex_data_no_known_person, 'repex', {}):
+ print(bods_data)
+ if bods_data['statementType'] in ('entityStatement', 'personStatement'):
+ assert bods_data['statementID'] == 'b88c850f-16ca-414a-ee07-cbd842e9aa6d'
+ if bods_data['statementType'] == 'entityStatement':
+ unspecified_type = 'unspecifiedEntityDetails'
+ else:
+ unspecified_type = 'unspecifiedPersonDetails'
+ assert bods_data[unspecified_type] == {'reason': 'interested-party-exempt-from-disclosure',
+ 'description': 'From LEI ExemptionReason `NO_KNOWN_PERSON`. There is no known person(s) controlling the entity.'}
+ assert validate_date_now(bods_data['publicationDetails']['publicationDate'])
+ assert bods_data['publicationDetails']['bodsVersion'] == '0.2'
+ assert bods_data['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert bods_data['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register',
+ 'url': 'https://register.openownership.org'}
+ assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'}
+ if bods_data['statementType'] == 'entityStatement':
+ assert bods_data['entityType'] == 'unknownEntity'
+ else:
+ assert bods_data['personType'] == 'unknownPerson'
+ statementID = bods_data['statementID']
+ elif bods_data['statementType'] == 'ownershipOrControlStatement':
+ assert bods_data['statementID'] == '96681822-7e26-a3be-8e4d-9feb646d6ef8'
+ assert bods_data['subject'] == {'describedByEntityStatement': '2a251471-ed07-1c82-6bcf-4843fa880309'}
+ assert bods_data['interestedParty'] == {'describedByPersonStatement': statementID}
+ assert bods_data['interests'] == [{'type': 'other-influence-or-control',
+ 'interestLevel': 'direct',
+ 'beneficialOwnershipOrControl': False,
+ 'details': 'A controlling interest.'}]
+ assert validate_date_now(bods_data['publicationDetails']['publicationDate'])
+ assert bods_data['publicationDetails']['bodsVersion'] == '0.2'
+ assert bods_data['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert bods_data['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register',
+ 'url': 'https://register.openownership.org'}
+ assert bods_data['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'}
+ else:
+ raise Exception(f"Incorrect statement type produced: {bods_data['statementType']}")
+ if not 'bods_data' in locals():
+ raise Exception("No statements produced")
diff --git a/tests/test_kinesis.py b/tests/test_kinesis.py
index eb26507..e630064 100644
--- a/tests/test_kinesis.py
+++ b/tests/test_kinesis.py
@@ -4,13 +4,15 @@
import datetime
from pathlib import Path
import json
-from unittest.mock import patch, Mock
+from unittest.mock import patch, Mock, MagicMock
+import asyncio
import pytest
from bodspipelines.infrastructure.inputs import KinesisInput
from bodspipelines.infrastructure.outputs import KinesisOutput
def validate_datetime(d):
+ """Test is valid datetime"""
try:
datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
return True
@@ -19,51 +21,89 @@ def validate_datetime(d):
def validate_date_now(d):
+ """Test is today's date"""
return d == datetime.date.today().strftime('%Y-%m-%d')
def set_environment_variables():
- os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
- os.environ['ELASTICSEARCH_HOST'] = 'localhost'
- os.environ['ELASTICSEARCH_PORT'] = '9876'
- os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ """Set environmant variables"""
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
+
+
+class AsyncIterator:
+ """Dummy async iterator"""
+ def __init__(self, seq):
+ self.iter = iter(seq)
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
@pytest.fixture
def json_data_file():
+ """LEI JSON data"""
with open("tests/fixtures/lei-data.json", "r") as read_file:
return json.load(read_file)
-def test_kinesis_output(json_data_file):
+@pytest.fixture
+def lei_list():
+ """List of enitity LEIs"""
+ return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216',
+ '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53',
+ '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11']
+
+
+@pytest.mark.asyncio
+async def test_kinesis_output(json_data_file):
"""Test Kinesis output stream"""
- with patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn:
- mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []}
+ with (patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
set_environment_variables()
kinesis_output = KinesisOutput(stream_name="gleif-test")
+ await kinesis_output.setup()
+
for item in json_data_file:
- kinesis_output.process(item, "lei")
- kinesis_output.finish()
+ await kinesis_output.process(item, "lei")
+ await kinesis_output.finish()
- #assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10
- count = 0
- for item in mock_kn.return_value.put_records.call_args.kwargs['Records']:
- assert json.loads(item['Data']) == json_data_file[count]
- count += 1
- assert count == 10
+ assert mock_kno.return_value.put.call_count == 13
-def test_kinesis_input(json_data_file):
+@pytest.mark.asyncio
+async def test_kinesis_input(json_data_file, lei_list):
"""Test Kinesis input stream"""
- with patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn:
- mock_kn.return_value.get_records.return_value = {'MillisBehindLatest': 0, 'Records': [{'Data': json.dumps(js).encode('utf-8')} for js in json_data_file]}
+ with (patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+ async def result():
+ for item in json_data_file:
+ yield item
+ mock_kni.return_value = result()
+ #mock_kni.return_value = AsyncIterator(lei_list)
set_environment_variables()
kinesis_input = KinesisInput(stream_name="gleif-test")
+ await kinesis_input.setup()
+
count = 0
- for item in kinesis_input.process():
+ async for item in kinesis_input.process():
+ print(item)
assert item == json_data_file[count]
count += 1
- assert count == 10
+ assert count == 13
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index c87e208..84cb262 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -5,18 +5,20 @@
from pathlib import Path
import json
from unittest.mock import patch, Mock
+import asyncio
import pytest
from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
from bodspipelines.infrastructure.inputs import KinesisInput
-from bodspipelines.infrastructure.storage import ElasticStorage
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
from bodspipelines.infrastructure.processing.bulk_data import BulkData
from bodspipelines.infrastructure.processing.xml_data import XMLData
from bodspipelines.infrastructure.processing.json_data import JSONData
-from bodspipelines.pipelines.gleif.utils import gleif_download_link
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData
from bodspipelines.pipelines.gleif.transforms import Gleif2Bods
-from bodspipelines.pipelines.gleif.transforms import generate_statement_id
+from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id
from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
match_lei, match_rr, match_repex,
id_lei, id_rr, id_repex)
@@ -27,6 +29,7 @@
def validate_datetime(d):
+ """Test is valid datetime"""
try:
datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
return True
@@ -35,9 +38,10 @@ def validate_datetime(d):
def validate_date_now(d):
+ """Test is today's date"""
return d == datetime.date.today().strftime('%Y-%m-%d')
-
+# Elasticsearch indexes for GLEIF data
index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
"rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
"repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
@@ -49,6 +53,7 @@ def validate_date_now(d):
# Identify type of GLEIF data
def identify_gleif(item):
+ print("Identifying:", item)
if 'Entity' in item:
return 'lei'
elif 'Relationship' in item:
@@ -66,47 +71,101 @@ def identify_bods(item):
return 'ownership'
def set_environment_variables():
+ """Set environment variables"""
os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
os.environ['ELASTICSEARCH_HOST'] = 'localhost'
os.environ['ELASTICSEARCH_PORT'] = '9876'
os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
@pytest.fixture
def lei_list():
+ """List entity LEIs"""
return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216',
- '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53']
+ '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53',
+ '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11']
+@pytest.fixture
+def last_update_list():
+ """List update datetimes"""
+ return ['2023-05-18T15:41:20.212Z', '2020-07-17T12:40:00.000Z', '2022-07-22T09:32:00.000Z', '2022-10-24T21:31:00.000Z',
+ '2023-05-18T17:24:00.540Z', '2023-05-03T07:03:05.620Z', '2019-04-22T21:31:00.000Z', '2023-05-10T04:42:18.790Z',
+ '2020-07-17T12:40:00.000Z', '2020-07-24T19:29:00.000Z', '2023-03-10T13:08:56+01:00', '2023-02-02T09:07:52.390Z',
+ '2023-04-25T13:18:00Z']
@pytest.fixture
def xml_data_file():
+ """GLEIF LEI XML data"""
return Path("tests/fixtures/lei-data.xml")
+@pytest.fixture
+def rr_xml_data_file():
+ """GLEIF RR XML data"""
+ return Path("tests/fixtures/rr-data.xml")
+
@pytest.fixture
def json_data_file():
+ """GLEIF LEI JSON data"""
with open("tests/fixtures/lei-data.json", "r") as read_file:
return json.load(read_file)
+@pytest.fixture
+def rr_json_data_file():
+ """GLEIF RR JSON data"""
+ with open("tests/fixtures/rr-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def repex_json_data_file():
+ """GLEIF Repex JSON data"""
+ with open("tests/fixtures/repex-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def ooc_json_data_file():
+ """BODS Data craeted from GLEIF RR data"""
+ with open("tests/fixtures/rr-data-out.json", "r") as read_file:
+ return json.load(read_file)
-def test_lei_ingest_stage(lei_list, xml_data_file):
+def test_lei_ingest_stage(lei_list, last_update_list, xml_data_file):
"""Test ingest pipeline stage on LEI-CDF v3.1 records"""
with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd,
patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
- patch('bodspipelines.infrastructure.clients.elasticsearch_client.streaming_bulk') as mock_sb,
- patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn):
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
mock_bd.return_value = xml_data_file
mock_pdr.return_value = None
mock_sdr.return_value = None
- mock_sb.return_value = iter([(True, {'create': {'_id': lei}}) for lei in lei_list])
- mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []}
+ async def result():
+ for lei, last in zip(lei_list, last_update_list):
+ yield (True, {'create': {'_id': f"{lei}_{last}"}})
+ mock_sb.return_value = result()
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+ consumer_close_future = asyncio.Future()
+ consumer_close_future.set_result(None)
+ mock_kni.return_value.close.return_value = consumer_close_future
+
set_environment_variables()
# Defintion of LEI-CDF v3.1 XML date source
lei_source = Source(name="lei",
origin=BulkData(display="LEI-CDF v3.1",
- url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
+ #url=gleif_download_link("https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"),
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/lei2/latest"),
size=41491,
directory="lei-cdf"),
datatype=XMLData(item_tag="LEIRecord",
@@ -114,7 +173,7 @@ def test_lei_ingest_stage(lei_list, xml_data_file):
filter=['NextVersion', 'Extension']))
# GLEIF data: Store in Easticsearch and output new to Kinesis stream
- output_new = NewOutput(storage=ElasticStorage(indexes=index_properties),
+ output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)),
output=KinesisOutput(stream_name="gleif-test"))
# Definition of GLEIF data pipeline ingest stage
ingest_stage = Stage(name="ingest-test",
@@ -127,40 +186,172 @@ def test_lei_ingest_stage(lei_list, xml_data_file):
# Run pipelne
pipeline.process("ingest-test")
- assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10
- item = json.loads(mock_kn.return_value.put_records.call_args.kwargs['Records'][0]['Data'])
- assert item["LEI"] == "001GPB6A9XPE8XJICC14"
- assert item["Entity"]["LegalName"] == "Fidelity Advisor Leveraged Company Stock Fund"
- assert item["Entity"]["OtherEntityNames"] == ["FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund"]
- assert item["Entity"]["LegalAddress"] == {"FirstAddressLine": "245 SUMMER STREET", "City": "BOSTON", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}
- assert item["Entity"]["HeadquartersAddress"] == {"FirstAddressLine": "C/O Fidelity Management & Research Company LLC",
- "City": "Boston", "Region": "US-MA", "Country": "US", "PostalCode": "02210"}
- assert item["Entity"]["RegistrationAuthority"] == {"RegistrationAuthorityID": "RA000665", "RegistrationAuthorityEntityID": "S000005113"}
- assert item["Entity"]["LegalJurisdiction"] == "US-MA"
- assert item["Entity"]["EntityCategory"] == "FUND"
- assert item["Entity"]["LegalForm"] == {"EntityLegalFormCode": "8888", "OtherLegalForm": "FUND"}
- assert item["Entity"]["EntityStatus"] == "ACTIVE"
- assert item["Entity"]["EntityCreationDate"] == "2012-11-29T00:00:00.000Z"
- assert item["Registration"]["InitialRegistrationDate"] == "2012-11-29T16:33:00.000Z"
- assert item["Registration"]["LastUpdateDate"] == "2023-05-18T15:41:20.212Z"
- assert item["Registration"]["RegistrationStatus"] == "ISSUED"
- assert item["Registration"]["NextRenewalDate"] == "2024-05-18T15:48:53.604Z"
- assert item["Registration"]["ManagingLOU"] == "EVK05KS7XY1DEII3R011"
- assert item["Registration"]["ValidationSources"] == "FULLY_CORROBORATED"
- assert item["Registration"]["ValidationAuthority"] == {"ValidationAuthorityID": "RA000665", "ValidationAuthorityEntityID": "S000005113"}
-
-
-def test_lei_transform_stage(lei_list, json_data_file):
+ assert mock_kno.return_value.put.call_count == 13
+
+ item = mock_kno.return_value.put.call_args.args[0]
+ print(item)
+
+ assert item['LEI'] == '213800BJPX8V9HVY1Y11'
+ assert item['Entity']['LegalName'] == 'Swedeit Italian Aktiebolag'
+ assert item['Entity']['LegalAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg',
+ 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O',
+ 'Country': 'SE', 'PostalCode': '426 76'}
+ assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg',
+ 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O',
+ 'Country': 'SE', 'PostalCode': '426 76'}
+ assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11',
+ 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76',
+ 'type': 'AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses']
+ assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11',
+ 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76',
+ 'type': 'AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses']
+ assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000544', 'RegistrationAuthorityEntityID': '556543-1193'}
+ assert item['Entity']['LegalJurisdiction'] == 'SE'
+ assert item['Entity']['EntityCategory'] == 'GENERAL'
+ assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'XJHM'}
+ assert item['Entity']['EntityStatus'] == 'ACTIVE'
+ assert item['Entity']['EntityCreationDate'] == '1997-06-05T02:00:00+02:00'
+ assert item['Registration']['InitialRegistrationDate'] == '2014-04-09T00:00:00Z'
+ assert item['Registration']['LastUpdateDate'] == '2023-04-25T13:18:00Z'
+ assert item['Registration']['RegistrationStatus'] == 'ISSUED'
+ assert item['Registration']['NextRenewalDate'] == '2024-05-12T06:59:39Z'
+ assert item['Registration']['ManagingLOU'] == '549300O897ZC5H7CY412'
+ assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED'
+ assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000544', 'ValidationAuthorityEntityID': '556543-1193'}
+
+
+def test_rr_ingest_stage(rr_xml_data_file, rr_json_data_file):
+ """Test ingest pipeline stage on RR-CDF v2.1 records"""
+ with (patch('bodspipelines.infrastructure.processing.bulk_data.BulkData.prepare') as mock_bd,
+ patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock bulk data input
+ mock_bd.return_value = rr_xml_data_file
+
+ # Mock directories
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES stream bulk output
+ async def result():
+ for item in rr_json_data_file:
+ yield (True, {'create': {'_id': id_rr(item)}})
+ mock_sb.return_value = result()
+
+ # Mock Kinesis output
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Mock Kinesis input
+ consumer_close_future = asyncio.Future()
+ consumer_close_future.set_result(None)
+ mock_kni.return_value.close.return_value = consumer_close_future
+
+ # Set environment variables
+ set_environment_variables()
+
+ # Defintion of RR-CDF v2.1 XML date source
+ rr_source = Source(name="rr",
+ origin=BulkData(display="RR-CDF v2.1",
+ data=GLEIFData(url="https://goldencopy.gleif.org/api/v2/golden-copies/publishes/rr/latest"),
+ size=2823,
+ directory="rr-cdf"),
+ datatype=XMLData(item_tag="RelationshipRecord",
+ namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"},
+ filter=['Extension']))
+
+ # GLEIF data: Store in Easticsearch and output new to Kinesis stream
+ output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=index_properties)),
+ output=KinesisOutput(stream_name="gleif-test"))
+
+ # Definition of GLEIF data pipeline ingest stage
+ ingest_stage = Stage(name="ingest-test",
+ sources=[rr_source],
+ processors=[],
+ outputs=[output_new]
+ )
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif-test", stages=[ingest_stage])
+
+ # Run pipelne
+ pipeline.process("ingest-test")
+
+ assert mock_kno.return_value.put.call_count == 10
+
+ item = mock_kno.return_value.put.call_args.args[0]
+
+ print(item)
+
+ assert item['Relationship']['StartNode'] == {'NodeID': '010G7UHBHEI87EKP0Q97', 'NodeIDType': 'LEI'}
+ assert item['Relationship']['EndNode'] == {'NodeID': '549300GC5MDF1KXYMP06', 'NodeIDType': 'LEI'}
+ assert item['Relationship']['RelationshipType'] == 'IS_FUND-MANAGED_BY'
+ assert item['Relationship']['RelationshipPeriods'] == [{'StartDate': '2023-05-09T06:14:06.846Z',
+ 'PeriodType': 'RELATIONSHIP_PERIOD'}]
+ assert item['Relationship']['RelationshipStatus'] == 'ACTIVE'
+ assert item['Registration'] == {'InitialRegistrationDate': '2012-06-06T15:57:00.000Z',
+ 'LastUpdateDate': '2023-05-16T14:21:08.886Z',
+ 'RegistrationStatus': 'PUBLISHED',
+ 'NextRenewalDate': '2024-06-27T00:00:00.000Z',
+ 'ManagingLOU': 'EVK05KS7XY1DEII3R011',
+ 'ValidationSources': 'FULLY_CORROBORATED',
+ 'ValidationDocuments': 'SUPPORTING_DOCUMENTS',
+ 'ValidationReference': 'https://www.sec.gov/Archives/edgar/data/1458460/000140508622000264/xslFormDX01/primary_doc.xml'}
+
+
+def test_lei_transform_stage(lei_list, last_update_list, json_data_file):
"""Test transform pipeline stage on LEI-CDF v3.1 records"""
with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
- patch('bodspipelines.infrastructure.clients.elasticsearch_client.streaming_bulk') as mock_sb,
- patch('bodspipelines.infrastructure.clients.kinesis_client.create_client') as mock_kn):
- mock_kn.return_value.get_records.return_value = {'MillisBehindLatest': 0, 'Records': [{'Data': json.dumps(js).encode('utf-8')} for js in json_data_file]}
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ async def result():
+ for item in json_data_file:
+ yield item
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(json_data_file)
mock_pdr.return_value = None
mock_sdr.return_value = None
- mock_sb.return_value = iter([(True, {'create': {'_id': generate_statement_id(lei, 'entityStatement')}}) for lei in lei_list])
- mock_kn.return_value.put_records.return_value = {'FailedRecordCount': 0, 'Records': []}
+
+ async def result():
+ for lei, last in zip(lei_list, last_update_list):
+ data = {"LEI": lei, 'Registration': {'LastUpdateDate': last}}
+ yield (True, {'create': {'_id': generate_statement_id(entity_id(data), 'entityStatement')}})
+ mock_sb.return_value = result()
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
set_environment_variables()
# Kinesis stream of GLEIF data from ingest stage
@@ -169,7 +360,7 @@ def test_lei_transform_stage(lei_list, json_data_file):
datatype=JSONData())
# BODS data: Store in Easticsearch and output new to Kinesis stream
- bods_output_new = NewOutput(storage=ElasticStorage(indexes=bods_index_properties),
+ bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)),
output=KinesisOutput(stream_name="bods-gleif-test"),
identify=identify_bods)
@@ -185,21 +376,234 @@ def test_lei_transform_stage(lei_list, json_data_file):
# Run pipelne
pipeline.process("transform-test")
- assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10
- item = json.loads(mock_kn.return_value.put_records.call_args.kwargs['Records'][0]['Data'])
- assert item['statementID'] == '7239ce6e-d0c6-731a-30b5-b2157eb12419'
+ assert mock_kno.return_value.put.call_count == 13
+
+ item = mock_kno.return_value.put.call_args.args[0]
+ print(item)
+
+ assert item['statementID'] == 'e2d096a9-23d5-ab26-0943-44c62c6a6a98'
assert item['statementType'] == 'entityStatement'
- assert item['statementDate'] == '2023-05-18'
+ assert item['statementDate'] == '2023-04-25'
assert item['entityType'] == 'registeredEntity'
- assert item['name'] == 'Fidelity Advisor Leveraged Company Stock Fund'
- assert item['incorporatedInJurisdiction'] == {'name': 'US-MA', 'code': 'US-MA'}
- assert {'id': '001GPB6A9XPE8XJICC14', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers']
- assert {'id': 'S000005113', 'schemeName': 'RA000665'} in item['identifiers']
- assert {'type': 'registered', 'address': '245 SUMMER STREET, BOSTON', 'country': 'US', 'postCode': '02210'} in item['addresses']
- assert {'type': 'business', 'address': 'C/O Fidelity Management & Research Company LLC, Boston', 'country': 'US', 'postCode': '02210'} in item['addresses']
+ assert item['name'] == 'Swedeit Italian Aktiebolag'
+ assert item['incorporatedInJurisdiction'] == {'name': 'Sweden', 'code': 'SE'}
+ assert {'id': '213800BJPX8V9HVY1Y11', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers']
+ assert {'id': '556543-1193', 'schemeName': 'RA000544'} in item['identifiers']
+ assert {'type': 'registered', 'address': 'C/O Anita Lindberg, Västra Frölunda', 'country': 'SE', 'postCode': '426 76'} in item['addresses']
+ assert {'type': 'business', 'address': 'C/O Anita Lindberg, Västra Frölunda', 'country': 'SE', 'postCode': '426 76'} in item['addresses']
assert validate_date_now(item['publicationDetails']['publicationDate'])
assert item['publicationDetails']['bodsVersion'] == '0.2'
assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'}
assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'}
- assert item['foundingDate'] == '2012-11-29T00:00:00.000Z'
+
+
+def test_rr_transform_stage(rr_json_data_file, ooc_json_data_file):
+ """Test transform pipeline stage on RR-CDF v2.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock Kinesis input
+ async def result():
+ for item in rr_json_data_file:
+ yield item
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(rr_json_data_file)
+
+ # Mock directories
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES stream bulk output
+ async def result():
+ for item in rr_json_data_file:
+ yield (True, {'create': {'_id': generate_statement_id(rr_id(item), 'ownershipOrControlStatement')}})
+ mock_sb.return_value = result()
+
+ # Mock Kinesis output
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+
+ # Mock Kinesis input
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Set environment variables
+ set_environment_variables()
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test",
+ sources=[gleif_source],
+ processors=[Gleif2Bods(identify=identify_gleif)],
+ outputs=[bods_output_new]
+ )
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ pipeline.process("transform-test")
+
+ # Check item count
+ assert mock_kno.return_value.put.call_count == 10
+
+ item = mock_kno.return_value.put.call_args.args[0]
+
+ print(item)
+
+ assert item['statementID'] == '208d41ba-977c-6760-801e-77fdc9d47307'
+ assert item['statementType'] == 'ownershipOrControlStatement'
+ assert item['statementDate'] == '2023-05-16'
+ assert item['subject'] == {'describedByEntityStatement': '06999bda-3717-8acb-719e-d6dd6d145c2a'}
+ assert item['interestedParty'] == {'describedByEntityStatement': '5bd23434-6df3-b2cf-4f28-83f16e6fea7a'}
+ assert item['interests'] == [{'type': 'other-influence-or-control',
+ 'interestLevel': 'direct',
+ 'beneficialOwnershipOrControl': False,
+ 'startDate': '2023-05-09T06:14:06.846Z'}]
+ assert validate_date_now(item['publicationDetails']['publicationDate'])
+ assert item['publicationDetails']['bodsVersion'] == '0.2'
+ assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register',
+ 'url': 'https://register.openownership.org'}
+ assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'}
+
+
+def test_rr_transform_stage(repex_json_data_file, ooc_json_data_file):
+ """Test transform pipeline stage on RR-CDF v2.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock Kinesis input
+ async def result():
+ for item in repex_json_data_file:
+ yield item
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(repex_json_data_file)
+
+ # Mock directories
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES stream bulk output
+ async def result():
+ for item in repex_json_data_file:
+ if item["ExceptionReason"] == "NO_KNOWN_PERSON":
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item),
+ 'personStatement')}})
+ else:
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item),
+ 'entityStatement')}})
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item),
+ 'ownershipOrControlStatement')}})
+ mock_sb.return_value = result()
+
+ # Mock Kinesis output
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+
+ # Mock Kinesis input
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Set environment variables
+ set_environment_variables()
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=ElasticsearchClient(indexes=bods_index_properties)),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test",
+ sources=[gleif_source],
+ processors=[Gleif2Bods(identify=identify_gleif)],
+ outputs=[bods_output_new]
+ )
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ pipeline.process("transform-test")
+
+ # Check item count
+ assert mock_kno.return_value.put.call_count == 20
+
+ item = mock_kno.return_value.put.call_args.args[0]
+
+ print(item)
+
+ assert item['statementID'] == '48c88421-b1ae-671d-82fa-00c3adc063a8'
+ assert item['statementType'] == 'ownershipOrControlStatement'
+ assert item['statementDate'] == '2023-06-09'
+ assert item['subject'] == {'describedByEntityStatement': '34db1616-4997-534b-9b93-a9da9c8edda9'}
+ assert item['interestedParty'] == {'describedByEntityStatement': '639d58eb-7bcc-de1d-da9f-74748d9c5cad'}
+ assert item['interests'] == [{'type': 'other-influence-or-control',
+ 'interestLevel': 'direct',
+ 'beneficialOwnershipOrControl': False,
+ 'details': 'A controlling interest.'}]
+ assert validate_date_now(item['publicationDetails']['publicationDate'])
+ assert item['publicationDetails']['bodsVersion'] == '0.2'
+ assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register',
+ 'url': 'https://register.openownership.org'}
+ assert item['source'] == {'type': ['officialRegister'], 'description': 'GLEIF'}
+ assert item['annotations'][0]['motivation'] == 'commenting'
+ assert item['annotations'][0]['description'] == 'The nature of this interest is unknown'
+ assert item['annotations'][0]['statementPointerTarget'] == '/interests/0/type'
+ assert validate_date_now(item['annotations'][0]['creationDate'])
+ assert item['annotations'][0]['createdBy'] == {'name': 'Open Ownership',
+ 'uri': 'https://www.openownership.org'}
diff --git a/tests/test_pipeline_updates.py b/tests/test_pipeline_updates.py
new file mode 100644
index 0000000..5a658bb
--- /dev/null
+++ b/tests/test_pipeline_updates.py
@@ -0,0 +1,480 @@
+import os
+import sys
+import time
+import datetime
+from pathlib import Path
+import json
+from unittest.mock import patch, Mock
+import asyncio
+import pytest
+
+from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
+from bodspipelines.infrastructure.inputs import KinesisInput
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
+from bodspipelines.infrastructure.processing.bulk_data import BulkData
+from bodspipelines.infrastructure.processing.xml_data import XMLData
+from bodspipelines.infrastructure.processing.json_data import JSONData
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods
+from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id
+from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
+ match_lei, match_rr, match_repex,
+ id_lei, id_rr, id_repex)
+from bodspipelines.infrastructure.updates import ProcessUpdates
+from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties,
+ ownership_statement_properties,
+ match_entity, match_person, match_ownership,
+ id_entity, id_person, id_ownership,
+ latest_properties, match_latest, id_latest,
+ references_properties, match_references, id_references,
+ updates_properties, match_updates, id_updates)
+
+
+def validate_datetime(d):
+ """Test is valid datetime"""
+ try:
+ datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
+ return True
+ except ValueError:
+ return False
+
+
+def validate_date_now(d):
+ """Test is today's date"""
+ return d == datetime.date.today().strftime('%Y-%m-%d')
+
+# Elasticsearch indexes for GLEIF data
+index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
+ "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
+ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
+
+# Elasticsearch indexes for BODS data
+bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity},
+ "person": {"properties": person_statement_properties, "match": match_person, "id": id_person},
+ "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership},
+ "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest},
+ "references": {"properties": references_properties, "match": match_references, "id": id_references},
+ "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates}}
+
+# Identify type of GLEIF data
+def identify_gleif(item):
+ if 'Entity' in item:
+ return 'lei'
+ elif 'Relationship' in item:
+ return 'rr'
+ elif 'ExceptionCategory' in item:
+ return 'repex'
+
+# Identify type of BODS data
+def identify_bods(item):
+ if item['statementType'] == 'entityStatement':
+ return 'entity'
+ elif item['statementType'] == 'personStatement':
+ return 'person'
+ elif item['statementType'] == 'ownershipOrControlStatement':
+ return 'ownership'
+
+def set_environment_variables():
+ """Set environment variable"""
+ os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
+ os.environ['ELASTICSEARCH_HOST'] = 'localhost'
+ os.environ['ELASTICSEARCH_PORT'] = '9876'
+ os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
+
+
+@pytest.fixture
+def lei_list():
+ """GLEIF LEI list"""
+ return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216',
+ '00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53',
+ '1595D0QCK7Y15293JK84', '213800FERQ5LE3H7WJ58', '213800BJPX8V9HVY1Y11']
+
+@pytest.fixture
+def lei_list_updates():
+ """GLEIF LEI updates list"""
+ return ["001GPB6A9XPE8XJICC14", "00EHHQ2ZHDCFXJCPCL46", "00QDBXDXLLF3W3JJJO36", "00W0SLGGVF0QQ5Q36N03", "213800FERQ5LE3H7WJ58"]
+
+@pytest.fixture
+def latest_lei_ids():
+ """Latest GLEIF LEI statement ids"""
+ return {
+ '001GPB6A9XPE8XJICC14': ('6057835c-9fbe-44ce-bdb0-0a8d82f2259d', None),
+ '004L5FPTUREIWK9T2N63': ('6e58bac7-a2c8-d233-fa74-e9738bc056d6', None),
+ '00EHHQ2ZHDCFXJCPCL46': ('9d22180c-dff9-2e93-b7a7-f6aacdbf4d02', None),
+ '00GBW0Z2GYIER7DHDS71': ('5f6448a2-b1fe-5e4f-8b1c-db0b1c627d98', None),
+ '00KLB2PFTM3060S2N216': ('c786d0f1-87fc-d6cf-5149-75b76e853289', None),
+ '00QDBXDXLLF3W3JJJO36': ('04357a16-7425-ad4e-8ed0-2e96a03dcdd9', None),
+ '00TR8NKAEL48RGTZEW89': ('359d7ade-acf0-aa10-3422-55e6ae1ca187', None),
+ '00TV1D5YIV5IDUGWBW29': ('1553f654-adde-1be3-9d6d-72c53b5af260', None),
+ '00W0SLGGVF0QQ5Q36N03': ('90eebf21-d507-a242-cd90-98410834aaf0', None),
+ '00X5RQKJQQJFFX0WPA53': ('6a3d293d-cc39-5f6c-850b-7855862c360c', None),
+ '1595D0QCK7Y15293JK84': ('c1dec12f-c6b2-1f69-d2fd-75166e7df72f', None),
+ '213800FERQ5LE3H7WJ58': ('6c7e8e94-e375-8d6e-408f-31bba4f5969a', None),
+ '213800BJPX8V9HVY1Y11': ('e2d096a9-23d5-ab26-0943-44c62c6a6a98', None),
+ }
+
+@pytest.fixture
+def last_update_list():
+ """GLEIF LEI last update datetimes list"""
+ return ['2023-05-18T15:41:20.212Z', '2020-07-17T12:40:00.000Z', '2022-07-22T09:32:00.000Z', '2022-10-24T21:31:00.000Z',
+ '2023-05-18T17:24:00.540Z', '2023-05-03T07:03:05.620Z', '2019-04-22T21:31:00.000Z', '2023-05-10T04:42:18.790Z',
+ '2020-07-17T12:40:00.000Z', '2020-07-24T19:29:00.000Z', '2023-03-10T13:08:56+01:00', '2023-02-02T09:07:52.390Z',
+ '2023-04-25T13:18:00Z']
+
+@pytest.fixture
+def last_update_list_updates():
+ """GLEIF LEI updates last update datetimes list"""
+ return ["2023-06-18T15:41:20.212Z", "2022-08-22T09:32:00.000Z", "2023-06-03T07:03:05.620Z", "2020-08-17T12:40:00.000Z",
+ "2023-03-02T09:07:52.390Z"]
+
+@pytest.fixture
+def xml_data_file():
+ """GLEIF LEI XML data"""
+ return Path("tests/fixtures/lei-data.xml")
+
+@pytest.fixture
+def json_data_file():
+ """GLEIF LEI updates JSON data"""
+ with open("tests/fixtures/lei-updates-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def json_data_out():
+ """BODS statements built from GLEIF LEI updates data"""
+ with open("tests/fixtures/lei-updates-data-out.json", "r") as read_file:
+ return json.load(read_file)
+
+def test_lei_transform_stage_updates(lei_list_updates, last_update_list_updates, latest_lei_ids, json_data_file, json_data_out):
+ """Test transform pipeline stage on LEI-CDF v3.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock Kinesis input
+ async def result():
+ for item in json_data_file:
+ yield item
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(json_data_file)
+
+ # Mock directory methods
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES async_scan
+ updates_stream = {}
+ async def as_result():
+ for item in updates_stream:
+ yield updates_stream[item]
+ mock_as.return_value = as_result()
+
+ # Mock ES async_streaming_bulk
+ async def sb_result():
+ for lei, last in zip(lei_list_updates, last_update_list_updates):
+ data = {"LEI": lei, 'Registration': {'LastUpdateDate': last}}
+ yield (True, {'create': {'_id': generate_statement_id(entity_id(data), 'entityStatement')}})
+ mock_sb.return_value = sb_result()
+
+ # Mock Kinesis output
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Mock ES index method
+ index_future = asyncio.Future()
+ index_future.set_result(None)
+ mock_es.return_value.index.return_value = index_future
+
+ # Mock ES search method
+ async def async_search(index=None, query=None):
+ id = query["query"]["match"]["_id"]
+ if index == "latest":
+ result = [{'latest_id': id,
+ 'statement_id': latest_lei_ids[id][0],
+ 'reason': latest_lei_ids[id][1]}]
+ elif index == 'references':
+ result = []
+ elif index == 'exceptions':
+ result = []
+ out = {'hits': {'hits': result}}
+ return out
+ mock_es.return_value.search.side_effect = async_search
+
+ # Setup environment variables
+ set_environment_variables()
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage))],
+ outputs=[bods_output_new]
+ )
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ pipeline.process("transform-test-updates", updates=True)
+ assert mock_kno.return_value.put.call_count == 5
+ #assert len(mock_kn.return_value.put_records.call_args.kwargs['Records']) == 10
+ item = mock_kno.return_value.put.call_args.args[0]
+ print(item)
+
+
+ assert item['statementID'] == '11739b48-9df9-2d05-34aa-79c04d5cca06'
+ assert item['statementType'] == 'entityStatement'
+ assert item['statementDate'] == '2023-03-02'
+ assert item['entityType'] == 'registeredEntity'
+ assert item['name'] == 'DENTSU INTERNATIONAL LIMITED'
+ assert item['incorporatedInJurisdiction'] == {'name': 'United Kingdom', 'code': 'GB'}
+ assert {'id': '213800FERQ5LE3H7WJ58', 'scheme': 'XI-LEI', 'schemeName': 'Global Legal Entity Identifier Index'} in item['identifiers']
+ assert {'id': '01403668', 'schemeName': 'RA000585'} in item['identifiers']
+ assert {'type': 'registered', 'address': '10 TRITON STREET, LONDON', 'country': 'GB', 'postCode': 'NW1 3BF'} in item['addresses']
+ assert {'type': 'business', 'address': '10 TRITON STREET, LONDON', 'country': 'GB', 'postCode': 'NW1 3BF'} in item['addresses']
+ assert validate_date_now(item['publicationDetails']['publicationDate'])
+ assert item['publicationDetails']['bodsVersion'] == '0.2'
+ assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'}
+ assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'}
+ assert item['foundingDate'] == '1978-12-05T00:00:00Z'
+ assert item['replacesStatements'] == ['6c7e8e94-e375-8d6e-408f-31bba4f5969a']
+
+
+@pytest.fixture
+def rr_json_data_file():
+ """GLEIF RR updates JSON data"""
+ with open("tests/fixtures/rr-updates-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def ooc_json_data_file():
+ """BODS statement produced from GLEIF RR updates data"""
+ with open("tests/fixtures/rr-updates-data-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def rr_json_data_file_gleif():
+ """GLEIF RR JSON data"""
+ with open("tests/fixtures/rr-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def rr_json_data_file_stored():
+ """BODS statements produced from GLEIF RR data"""
+ with open("tests/fixtures/rr-data-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def latest_ids(rr_json_data_file_stored, rr_json_data_file_gleif):
+ """Latest statementIDs"""
+ latest_index = {}
+ for item, statement in zip(rr_json_data_file_gleif, rr_json_data_file_stored):
+ start = item["Relationship"]["StartNode"]["NodeID"]
+ end = item["Relationship"]["EndNode"]["NodeID"]
+ rel_type = item["Relationship"]["RelationshipType"]
+ latest_id = f"{start}_{end}_{rel_type}"
+ latest = {'latest_id': latest_id,
+ 'statement_id': statement["statementID"],
+ 'reason': None}
+ latest_index[latest_id] = latest
+ return latest_index
+
+
+def test_rr_transform_stage_updates(rr_json_data_file, ooc_json_data_file, latest_ids):
+ """Test transform pipeline stage on RR-CDF v2.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock Kinesis input
+ async def result():
+ for item in rr_json_data_file:
+ yield item
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(rr_json_data_file)
+
+ # Mock directories
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES async_scan
+ updates_stream = {}
+ async def as_result():
+ for item in updates_stream:
+ yield updates_stream[item]
+ mock_as.return_value = as_result()
+
+ # Mock ES async_streaming_bulk
+ async def sb_result():
+ for item in rr_json_data_file:
+ yield (True, {'create': {'_id': generate_statement_id(rr_id(item), 'ownershipOrControlStatement')}})
+ mock_sb.return_value = sb_result()
+
+ # Mock Kinesis output
+ put_future = asyncio.Future()
+ put_future.set_result(None)
+ mock_kno.return_value.put.return_value = put_future
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Mock Latest Index
+ latest_index_data = latest_ids
+
+ # Mock ES index method
+ async def async_index(index, document):
+ #print("async_index:", index, document)
+ if index == "updates":
+ updates_stream[document['_source']['referencing_id']] = document['_source']
+ elif index == "latest":
+ latest_index_data[document['_source']['latest_id']] = document['_source']
+ return None
+ mock_es.return_value.index.side_effect = async_index
+
+ # Mock ES update method
+ async def async_update(index, id, document):
+ print("async_update:", index, id, document)
+ if index == "latest":
+ latest_index_data[document['latest_id']] = document
+ elif index == "updates":
+ updates_stream[document['referencing_id']] = document
+ return None
+ mock_es.return_value.update.side_effect = async_update
+
+ # Mock ES delete method
+ async def async_delete(index, id):
+ print("async_delete:", index, id)
+ if index == "latest":
+ del latest_rr_ids[id]
+ elif index == "updates":
+ print(updates_stream)
+ if id in updates_stream: del updates_stream[id] # IS THIS RIGHT?
+ return None
+ mock_es.return_value.delete.side_effect = async_delete
+
+ # Mock ES search method
+ async def async_search(index=None, query=None):
+ id = query["query"]["match"]["_id"]
+ if index == "latest":
+ if id in latest_index_data:
+ results = [latest_index_data[id]]
+ else:
+ results = []
+ elif index == 'references':
+ results = []
+ elif index == 'exceptions':
+ results = []
+ out = {'hits': {'hits': results}}
+ return out
+ mock_es.return_value.search.side_effect = async_search
+
+ # Setup environment variables
+ set_environment_variables()
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage))],
+ outputs=[bods_output_new]
+ )
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ pipeline.process("transform-test-updates", updates=True)
+ assert mock_kno.return_value.put.call_count == 3
+
+ item = mock_kno.return_value.put.call_args.args[0]
+ print(item)
+
+ assert item['statementID'] == '8fec0d1c-b31b-7632-7c47-901e10bcd367'
+ assert item['statementType'] == 'ownershipOrControlStatement'
+ assert item['statementDate'] == '2023-03-24'
+ assert item['subject'] == {'describedByEntityStatement': '9f94abe2-349c-8e96-edaf-cf832eab1ac8'}
+ assert item['interestedParty'] == {'describedByEntityStatement': '20815b26-efdc-a516-a905-7abdfa63d128'}
+ assert {'type': 'other-influence-or-control',
+ 'details': 'LEI RelationshipType: IS_ULTIMATELY_CONSOLIDATED_BY',
+ 'interestLevel': 'unknown',
+ 'beneficialOwnershipOrControl': False,
+ 'startDate': '2018-02-06T00:00:00.000Z'} in item['interests']
+ assert validate_date_now(item['publicationDetails']['publicationDate'])
+ assert item['publicationDetails']['bodsVersion'] == '0.2'
+ assert item['publicationDetails']['license'] == 'https://register.openownership.org/terms-and-conditions'
+ assert item['publicationDetails']['publisher'] == {'name': 'OpenOwnership Register', 'url': 'https://register.openownership.org'}
+ assert item['source'] == {'type': ['officialRegister', 'verified'], 'description': 'GLEIF'}
+ assert item['replacesStatements'] == ['3cd06b78-3c1c-0cab-881b-cbbc36af1741']
diff --git a/tests/test_redis.py b/tests/test_redis.py
new file mode 100644
index 0000000..fa040cb
--- /dev/null
+++ b/tests/test_redis.py
@@ -0,0 +1,122 @@
+import os
+import sys
+import time
+import json
+from unittest.mock import patch, Mock
+import asyncio
+import pytest
+
+from redis import RedisError
+
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.redis_client import RedisClient
+from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
+ match_lei, match_rr, match_repex,
+ id_lei, id_rr, id_repex)
+
+index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
+ "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
+ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
+
+def set_environment_variables():
+ """Set environment variables"""
+ os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
+ os.environ['ELASTICSEARCH_HOST'] = 'localhost'
+ os.environ['ELASTICSEARCH_PORT'] = '9876'
+ os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+
+@pytest.fixture
+def lei_item():
+ """Example LEI-CDF v3.1 data"""
+ return {'LEI': '097900BICQ0000135514',
+ 'Entity': {'LegalName': 'Ing. Magdaléna Beňo Frackowiak ZARIA TRAVEL',
+ 'TransliteratedOtherEntityNames': {'TransliteratedOtherEntityName': 'ING MAGDALENA BENO FRACKOWIAK ZARIA TRAVEL'},
+ 'LegalAddress': {'FirstAddressLine': 'Partizánska Ľupča 708', 'City': 'Partizánska Ľupča', 'Country': 'SK', 'PostalCode': '032 15'},
+ 'HeadquartersAddress': {'FirstAddressLine': 'Partizánska Ľupča 708', 'City': 'Partizánska Ľupča', 'Country': 'SK', 'PostalCode': '032 15'},
+ 'RegistrationAuthority': {'RegistrationAuthorityID': 'RA000670', 'RegistrationAuthorityEntityID': '43846696'},
+ 'LegalJurisdiction': 'SK',
+ 'EntityCategory': 'SOLE_PROPRIETOR',
+ 'LegalForm': {'EntityLegalFormCode': 'C4PZ'},
+ 'EntityStatus': 'ACTIVE',
+ 'EntityCreationDate': '2007-11-15T08:00:00+01:00'},
+ 'Registration': {'InitialRegistrationDate': '2018-02-16T00:00:00+01:00',
+ 'LastUpdateDate': '2023-01-10T08:30:56.044+01:00',
+ 'RegistrationStatus': 'ISSUED',
+ 'NextRenewalDate': '2024-02-16T00:00:00+01:00',
+ 'ManagingLOU': '097900BEFH0000000217',
+ 'ValidationSources': 'FULLY_CORROBORATED',
+ 'ValidationAuthority': {'ValidationAuthorityID': 'RA000670', 'ValidationAuthorityEntityID': '43846696'}}}
+
+@pytest.fixture
+def lei_list():
+ """List entity LEIs"""
+ return ['097900BICQ0000135514', '097900BICQ0000135515', '097900BICQ0000135516', '097900BICQ0000135517', '097900BICQ0000135518',
+ '097900BICQ0000135519', '097900BICQ0000135520', '097900BICQ0000135521', '097900BICQ0000135522', '097900BICQ0000135523']
+
+@pytest.mark.asyncio
+async def test_lei_storage_get(lei_item):
+ """Test getting a new LEI-CDF v3.1 record in redis"""
+ with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd:
+ get_future = asyncio.Future()
+ get_future.set_result(json.dumps(lei_item).encode("utf-8"))
+ mock_rd.return_value.get.return_value = get_future
+ set_environment_variables()
+ storage = Storage(storage=RedisClient(indexes=index_properties))
+ assert await storage.get_item(lei_item["LEI"], 'lei') == lei_item
+
+
+@pytest.mark.asyncio
+async def test_lei_storage_new(lei_item):
+ """Test storing a new LEI-CDF v3.1 record in redis"""
+ with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd:
+ set_future = asyncio.Future()
+ set_future.set_result(None)
+ mock_rd.return_value.set.return_value = set_future
+ mock_rd.return_value.get.side_effect = RedisError()
+ set_environment_variables()
+ storage = Storage(storage=RedisClient(indexes=index_properties))
+ assert await storage.process(lei_item, 'lei') == lei_item
+
+
+@pytest.mark.asyncio
+async def test_lei_storage_existing(lei_item):
+ """Test trying to store LEI-CDF v3.1 record which is already in redis"""
+ with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd:
+ get_future = asyncio.Future()
+ get_future.set_result(json.dumps(lei_item).encode("utf-8"))
+ mock_rd.return_value.get.return_value = get_future
+ set_environment_variables()
+ storage = Storage(storage=RedisClient(indexes=index_properties))
+ assert await storage.process(lei_item, 'lei') == False
+
+
+@pytest.mark.asyncio
+async def test_lei_storage_batch_new(lei_item, lei_list):
+ """Test storing a batch of new LEI-CDF v3.1 records in redis"""
+ with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd:
+ async def build_stream(lei_item, lei_list):
+ for lei in lei_list:
+ item = lei_item.copy()
+ item['LEI'] = lei
+ yield item
+ mock_rd.return_value.pipeline.return_value.execute.return_value = [True async for result in build_stream(lei_item, lei_list)]
+ set_environment_variables()
+ storage = Storage(storage=RedisClient(indexes=index_properties))
+ async for item in storage.process_batch(build_stream(lei_item, lei_list), "lei"):
+ assert item["LEI"] in lei_list
+
+
+@pytest.mark.asyncio
+async def test_lei_storage_batch_existing(lei_item, lei_list):
+ """Test storing a batch of existing LEI-CDF v3.1 records in redis"""
+ with patch('bodspipelines.infrastructure.clients.redis_client.Redis') as mock_rd:
+ async def build_stream(lei_item, lei_list):
+ for lei in lei_list:
+ item = lei_item.copy()
+ item['LEI'] = lei
+ yield item
+ mock_rd.return_value.pipeline.return_value.execute.return_value = [False async for result in build_stream(lei_item, lei_list)]
+ set_environment_variables()
+ storage = Storage(storage=RedisClient(indexes=index_properties))
+ async for item in storage.process_batch(build_stream(lei_item, lei_list), "lei"):
+ assert False, "Error: New record found"
diff --git a/tests/test_transforms_updates_es.py b/tests/test_transforms_updates_es.py
new file mode 100644
index 0000000..71a9107
--- /dev/null
+++ b/tests/test_transforms_updates_es.py
@@ -0,0 +1,296 @@
+import pytest
+import os
+import json
+import requests
+from unittest.mock import patch, Mock, AsyncMock
+
+from .utils import AsyncIterator, list_index_contents
+from .config import set_environment_variables
+
+pytest_plugins = ["docker_compose"]
+
+from bodspipelines.pipelines.gleif import config
+from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
+from bodspipelines.infrastructure.inputs import KinesisInput
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
+from bodspipelines.infrastructure.processing.bulk_data import BulkData
+from bodspipelines.infrastructure.processing.xml_data import XMLData
+from bodspipelines.infrastructure.processing.json_data import JSONData
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData, identify_gleif
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods
+from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id
+from bodspipelines.pipelines.gleif.updates import GleifUpdates
+from bodspipelines.pipelines.gleif.indexes import gleif_index_properties
+from bodspipelines.infrastructure.updates import ProcessUpdates, referenced_ids
+from bodspipelines.infrastructure.indexes import bods_index_properties
+from bodspipelines.infrastructure.utils import identify_bods
+
+# Setup environment variables
+set_environment_variables()
+
+#@pytest.fixture(scope="function")
+#def wait_for_es(function_scoped_container_getter):
+@pytest.fixture(scope="module")
+def wait_for_es(module_scoped_container_getter):
+ service = module_scoped_container_getter.get("bods_ingester_gleif_es").network_info[0]
+ os.environ['ELASTICSEARCH_HOST'] = service.hostname
+ os.environ['ELASTICSEARCH_PORT'] = service.host_port
+ #config.setup()
+ return service
+
+@pytest.fixture
+def lei_json_data_file_gleif():
+ """GLEIF LEI Record update"""
+ with open("tests/fixtures/lei-updates-replaces-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def relationship_update_data_file_gleif():
+ """GLEIF LEI and RR Records update"""
+ with open("tests/fixtures/relationship-update-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def reporting_exceptions_data_file_gleif():
+ """GLEIF LEI and RR Records update"""
+ with open("tests/fixtures/reporting-exceptions-data.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.mark.asyncio
+async def test_lei_replaces(wait_for_es, lei_json_data_file_gleif):
+ #print(requests.get('http://localhost:9200/_cat/indices?v').text)
+ #assert False
+ """Test transform pipeline stage on LEI-CDF v3.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni,
+ patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno,
+ #patch('bodspipelines.infrastructure.outputs.KinesisOutput') as mock_kno,
+ #patch('bodspipelines.infrastructure.inputs.KinesisInput') as mock_kni
+ ):
+
+ # Mock setup/finish_write functions
+ async def async_func():
+ return None
+ mock_kni.return_value.setup.return_value = async_func()
+ mock_kno.return_value.setup.return_value = async_func()
+ mock_kno.return_value.finish_write.return_value = async_func()
+ mock_kni.return_value.close.return_value = async_func()
+ mock_kno.return_value.close.return_value = async_func()
+
+ # Mock Kinesis stream input
+ mock_kni.return_value.read_stream.return_value = AsyncIterator(lei_json_data_file_gleif)
+
+ # Mock Kinesis output stream (save results)
+ kinesis_output_stream = []
+ async def async_put(record):
+ kinesis_output_stream.append(record)
+ return None
+ mock_kno.return_value.add_record.side_effect = async_put
+
+ # Mock directory methods
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # Setup indexes
+ await bods_storage.setup_indexes()
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage),
+ updates=GleifUpdates())],
+ outputs=[bods_output_new]
+ )
+
+ assert hasattr(transform_stage.processors[0], "finish_updates")
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ #pipeline.process("transform-test-updates", updates=True)
+ await pipeline.process_stage("transform-test-updates", updates=True)
+
+ print(kinesis_output_stream)
+
+ assert kinesis_output_stream[0]['statementDate'] == '2023-12-29'
+ assert kinesis_output_stream[-1]['statementDate'] == '2024-01-03'
+ assert kinesis_output_stream[-1]['replacesStatements'][0] == kinesis_output_stream[0]['statementID']
+
+@pytest.mark.asyncio
+async def test_relationship_replaces(wait_for_es, relationship_update_data_file_gleif):
+ """Test transform pipeline stage on relationship update"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni,
+ patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno,
+ ):
+
+ # Mock setup/finish_write functions
+ async def async_func():
+ return None
+ mock_kni.return_value.setup.return_value = async_func()
+ mock_kno.return_value.setup.return_value = async_func()
+ mock_kno.return_value.finish_write.return_value = async_func()
+ mock_kni.return_value.close.return_value = async_func()
+ mock_kno.return_value.close.return_value = async_func()
+
+ # Mock Kinesis stream input
+ mock_kni.return_value.read_stream.return_value = AsyncIterator(relationship_update_data_file_gleif)
+
+ # Mock Kinesis output stream (save results)
+ kinesis_output_stream = []
+ async def async_put(record):
+ kinesis_output_stream.append(record)
+ return None
+ mock_kno.return_value.add_record.side_effect = async_put
+
+ # Mock directory methods
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # Setup indexes
+ await bods_storage.setup_indexes()
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage),
+ updates=GleifUpdates())],
+ outputs=[bods_output_new]
+ )
+
+ assert hasattr(transform_stage.processors[0], "finish_updates")
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ #pipeline.process("transform-test-updates", updates=True)
+ await pipeline.process_stage("transform-test-updates", updates=True)
+
+ print(kinesis_output_stream)
+
+ #list_index_contents("updates")
+
+ assert kinesis_output_stream[0]['statementDate'] == '2023-02-22'
+ assert kinesis_output_stream[1]['statementDate'] == '2023-02-22'
+ assert kinesis_output_stream[2]['statementDate'] == '2023-02-22'
+ assert kinesis_output_stream[3]['statementDate'] == '2024-01-30'
+ assert kinesis_output_stream[4]['statementDate'] == '2024-01-30'
+ assert kinesis_output_stream[5]['statementDate'] == '2024-01-30'
+ assert kinesis_output_stream[-2]['replacesStatements'][0] == kinesis_output_stream[1]['statementID']
+ assert kinesis_output_stream[-1]['replacesStatements'][0] == kinesis_output_stream[2]['statementID']
+
+@pytest.mark.asyncio
+async def test_reporting_exceptions(wait_for_es, reporting_exceptions_data_file_gleif):
+ """Test transform pipeline stage on relationship update"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.inputs.KinesisStream') as mock_kni,
+ patch('bodspipelines.infrastructure.outputs.KinesisStream') as mock_kno,
+ ):
+
+ # Mock setup/finish_write functions
+ async def async_func():
+ return None
+ mock_kni.return_value.setup.return_value = async_func()
+ mock_kno.return_value.setup.return_value = async_func()
+ mock_kno.return_value.finish_write.return_value = async_func()
+ mock_kni.return_value.close.return_value = async_func()
+ mock_kno.return_value.close.return_value = async_func()
+
+ # Mock Kinesis stream input
+ mock_kni.return_value.read_stream.return_value = AsyncIterator(reporting_exceptions_data_file_gleif)
+
+ # Mock Kinesis output stream (save results)
+ kinesis_output_stream = []
+ async def async_put(record):
+ kinesis_output_stream.append(record)
+ return None
+ mock_kno.return_value.add_record.side_effect = async_put
+
+ # Mock directory methods
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # Setup indexes
+ await bods_storage.setup_indexes()
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage),
+ updates=GleifUpdates())],
+ outputs=[bods_output_new]
+ )
+
+ assert hasattr(transform_stage.processors[0], "finish_updates")
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ #pipeline.process("transform-test-updates", updates=True)
+ await pipeline.process_stage("transform-test-updates", updates=True)
+
+ print(kinesis_output_stream)
+
+ assert kinesis_output_stream[0]['statementDate'] == '2022-01-25'
+ assert kinesis_output_stream[1]['statementDate'] == '2024-01-01'
+ assert kinesis_output_stream[2]['statementDate'] == '2024-01-01'
+ assert kinesis_output_stream[2]['subject']['describedByEntityStatement'] == kinesis_output_stream[0]['statementID']
+ assert kinesis_output_stream[2]['interestedParty']['describedByEntityStatement'] == kinesis_output_stream[1]['statementID']
+ assert kinesis_output_stream[3]['statementDate'] == '2024-01-01'
+ assert kinesis_output_stream[4]['statementDate'] == '2024-01-01'
+ assert kinesis_output_stream[4]['subject']['describedByEntityStatement'] == kinesis_output_stream[0]['statementID']
+ assert kinesis_output_stream[4]['interestedParty']['describedByEntityStatement'] == kinesis_output_stream[3]['statementID']
+
diff --git a/tests/test_updates.py b/tests/test_updates.py
new file mode 100644
index 0000000..957b002
--- /dev/null
+++ b/tests/test_updates.py
@@ -0,0 +1,684 @@
+import os
+import sys
+import time
+import datetime
+from pathlib import Path
+import json
+import itertools
+from unittest.mock import patch, Mock
+import asyncio
+import pytest
+
+from bodspipelines.infrastructure.pipeline import Source, Stage, Pipeline
+from bodspipelines.infrastructure.inputs import KinesisInput
+from bodspipelines.infrastructure.storage import Storage
+from bodspipelines.infrastructure.clients.elasticsearch_client import ElasticsearchClient
+from bodspipelines.infrastructure.outputs import Output, OutputConsole, NewOutput, KinesisOutput
+from bodspipelines.infrastructure.processing.bulk_data import BulkData
+from bodspipelines.infrastructure.processing.xml_data import XMLData
+from bodspipelines.infrastructure.processing.json_data import JSONData
+from bodspipelines.pipelines.gleif.utils import gleif_download_link, GLEIFData
+from bodspipelines.pipelines.gleif.transforms import Gleif2Bods
+from bodspipelines.pipelines.gleif.transforms import generate_statement_id, entity_id, rr_id, repex_id
+from bodspipelines.pipelines.gleif.updates import GleifUpdates
+from bodspipelines.pipelines.gleif.indexes import (lei_properties, rr_properties, repex_properties,
+ match_lei, match_rr, match_repex,
+ id_lei, id_rr, id_repex)
+from bodspipelines.infrastructure.updates import ProcessUpdates, referenced_ids
+from bodspipelines.infrastructure.indexes import (entity_statement_properties, person_statement_properties,
+ ownership_statement_properties,
+ match_entity, match_person, match_ownership,
+ id_entity, id_person, id_ownership,
+ latest_properties, match_latest, id_latest,
+ references_properties, match_references, id_references,
+ updates_properties, match_updates, id_updates,
+ exceptions_properties, match_exceptions, id_exceptions)
+
+def validate_datetime(d):
+ """Test is valid datetime"""
+ try:
+ datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
+ return True
+ except ValueError:
+ return False
+
+
+def validate_date_now(d):
+ """Test is today's date"""
+ return d == datetime.date.today().strftime('%Y-%m-%d')
+
+# Elasticsearch indexes for GLEIF data
+index_properties = {"lei": {"properties": lei_properties, "match": match_lei, "id": id_lei},
+ "rr": {"properties": rr_properties, "match": match_rr, "id": id_rr},
+ "repex": {"properties": repex_properties, "match": match_repex, "id": id_repex}}
+
+# Elasticsearch indexes for BODS data
+bods_index_properties = {"entity": {"properties": entity_statement_properties, "match": match_entity, "id": id_entity},
+ "person": {"properties": person_statement_properties, "match": match_person, "id": id_person},
+ "ownership": {"properties": ownership_statement_properties, "match": match_ownership, "id": id_ownership},
+ "latest": {"properties": latest_properties, "match": match_latest, "id": id_latest},
+ "references": {"properties": references_properties, "match": match_references, "id": id_references},
+ "updates": {"properties": updates_properties, "match": match_updates, "id": id_updates},
+ "exceptions": {"properties": exceptions_properties, "match": match_exceptions, "id": id_exceptions}}
+
+# Identify type of GLEIF data
+def identify_gleif(item):
+ if 'Entity' in item:
+ return 'lei'
+ elif 'Relationship' in item:
+ return 'rr'
+ elif 'ExceptionCategory' in item:
+ return 'repex'
+
+# Identify type of BODS data
+def identify_bods(item):
+ if item['statementType'] == 'entityStatement':
+ return 'entity'
+ elif item['statementType'] == 'personStatement':
+ return 'person'
+ elif item['statementType'] == 'ownershipOrControlStatement':
+ return 'ownership'
+
+def set_environment_variables():
+ """Set environment variables"""
+ os.environ['ELASTICSEARCH_PROTOCOL'] = 'http'
+ os.environ['ELASTICSEARCH_HOST'] = 'localhost'
+ os.environ['ELASTICSEARCH_PORT'] = '9876'
+ os.environ['ELASTICSEARCH_PASSWORD'] = '********'
+ os.environ['BODS_AWS_REGION'] = "eu-west-1"
+ os.environ['BODS_AWS_ACCESS_KEY_ID'] ="********************"
+ os.environ['BODS_AWS_SECRET_ACCESS_KEY'] = "****************************************"
+
+
+@pytest.fixture
+def updates_json_data_file():
+ """GLEIF Updates Records"""
+ out = []
+ with (open("tests/fixtures/lei-updates-data2-new.json", "r") as lei_file,
+ open("tests/fixtures/rr-updates-data2-new.json", "r") as rr_file,
+ open("tests/fixtures/repex-updates-data2-new.json", "r") as repex_file):
+ out.extend(json.load(lei_file))
+ out.extend(json.load(rr_file))
+ out.extend(json.load(repex_file))
+ return out
+
+@pytest.fixture
+def rr_json_data_file_gleif():
+ """GLEIF Relationship Records"""
+ with open("tests/fixtures/rr-updates-data2.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def rr_json_data_file_stored():
+ """Stored Relationship Record Statements"""
+ with open("tests/fixtures/rr-updates-data2-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def lei_json_data_file_gleif():
+ """GLEIF LEI Records"""
+ with open("tests/fixtures/lei-updates-data2.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def lei_json_data_file_stored():
+ """LEI Stored Statements"""
+ with open("tests/fixtures/lei-updates-data2-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def repex_json_data_file_gleif():
+ """GLEIF Reporting Exception Records"""
+ with open("tests/fixtures/repex-updates-data2.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def repex_json_data_file_gleif_new():
+ """GLEIF Reporting Exception Records"""
+ with open("tests/fixtures/repex-updates-data2-new.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def repex_json_data_file_stored():
+ """Reporting Exception Stored Statements"""
+ with open("tests/fixtures/repex-updates-data2-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def lei_json_data_file_output():
+ """LEI Output Statements"""
+ with open("tests/fixtures/lei-updates-data2-new-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def rr_json_data_file_output():
+ """Relationship Record Output Statements"""
+ with open("tests/fixtures/rr-updates-data2-new-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def repex_json_data_file_output():
+ """Reporting Exception Output Statements"""
+ with open("tests/fixtures/repex-updates-data2-new-out.json", "r") as read_file:
+ return json.load(read_file)
+
+@pytest.fixture
+def stored_references(lei_json_data_file_stored, rr_json_data_file_stored, rr_json_data_file_gleif):
+ """Stored References"""
+ references = {}
+ for lei_statement in lei_json_data_file_stored:
+ statement_id = lei_statement['statementID']
+ ref_statement_ids = {}
+ for rr_statement, rr_record in zip(rr_json_data_file_stored, rr_json_data_file_gleif):
+ start = rr_record["Relationship"]["StartNode"]["NodeID"]
+ end = rr_record["Relationship"]["EndNode"]["NodeID"]
+ rel_type = rr_record["Relationship"]["RelationshipType"]
+ if statement_id == rr_statement["subject"]["describedByEntityStatement"]:
+ ref_statement_ids[rr_statement['statementID']] = f"{start}_{end}_{rel_type}"
+ if (("describedByEntityStatement" in rr_statement["interestedParty"] and
+ statement_id == rr_statement["interestedParty"]["describedByEntityStatement"]) or
+ ("describedByPersonStatement" in rr_statement["interestedParty"] and
+ statement_id == rr_statement["interestedParty"]["describedByPersonStatement"])):
+ ref_statement_ids[rr_statement['statementID']] = f"{start}_{end}_{rel_type}"
+ if ref_statement_ids:
+ references[statement_id] = {'statement_id': statement_id, 'references_id': ref_statement_ids}
+ return references
+
+@pytest.fixture
+def latest_ids(lei_json_data_file_stored, lei_json_data_file_gleif,
+ rr_json_data_file_stored, rr_json_data_file_gleif,
+ repex_json_data_file_stored, repex_json_data_file_gleif):
+ """Latest statementIDs"""
+ latest_index = {}
+ for item, statement in zip(lei_json_data_file_gleif, lei_json_data_file_stored):
+ latest = {'latest_id': item["LEI"],
+ 'statement_id': statement["statementID"],
+ 'reason': None}
+ latest_index[item["LEI"]] = latest
+ for item, statement in zip(rr_json_data_file_gleif, rr_json_data_file_stored):
+ start = item["Relationship"]["StartNode"]["NodeID"]
+ end = item["Relationship"]["EndNode"]["NodeID"]
+ rel_type = item["Relationship"]["RelationshipType"]
+ latest_id = f"{start}_{end}_{rel_type}"
+ latest = {'latest_id': latest_id,
+ 'statement_id': statement["statementID"],
+ 'reason': None}
+ latest_index[latest_id] = latest
+ entity = True
+ repex_json_data_double_gleif = list(itertools.chain.from_iterable(itertools.repeat(x, 2)
+ for x in repex_json_data_file_gleif))
+ for item, statement in zip(repex_json_data_double_gleif, repex_json_data_file_stored):
+ lei = item['LEI']
+ category = item['ExceptionCategory']
+ reason = item['ExceptionReason']
+ if entity:
+ latest_id = f"{lei}_{category}_{reason}_entity"
+ else:
+ latest_id = f"{lei}_{category}_{reason}_ownership"
+ latest = {'latest_id': latest_id,
+ 'statement_id': statement["statementID"],
+ 'reason': reason}
+ latest_index[latest_id] = latest
+ entity = not entity
+ return latest_index
+
+
+@pytest.fixture
+def exception_ids(repex_json_data_file_gleif, repex_json_data_file_stored):
+ """Latest data for GLEIF Reporting Exceptions"""
+ exception_data = {}
+ repex_json_data_double_gleif = list(itertools.chain.from_iterable(itertools.repeat(x, 2)
+ for x in repex_json_data_file_gleif))
+ for item, statement in zip(repex_json_data_double_gleif, repex_json_data_file_stored):
+ lei = item['LEI']
+ category = item['ExceptionCategory']
+ reason = item['ExceptionReason']
+ reference = item['ExceptionReference'] if 'ExceptionReference' in item else None
+ latest_id = f"{lei}_{category}"
+ if statement["statementType"] in ("entityStatement", "personStatement"):
+ exception_data[latest_id] = [None, statement["statementID"], reason, reference, statement["statementType"]]
+ else:
+ exception_data[latest_id][0] = statement["statementID"]
+ return exception_data
+
+
+@pytest.fixture
+def voided_ids():
+ """Statement ids of voiding statements"""
+ return ["8f08b038-e0c2-0748-2a45-acb62b17f25b",
+ "1408ba29-4792-1d02-472a-661e3962778d",
+ "bcae847f-988a-2f75-ac09-6138864fd5a0",
+ "2d6ce004-492d-34ff-652a-b525b5fe4ca8",
+ "4243104a-bfef-7d92-a151-0e412a7a5269",
+ "c67bb160-25b6-c5b9-5767-b636b9846c9f"]
+
+
+@pytest.fixture
+def updated_ids():
+ """Statement ids for final update statements"""
+ return ["89788883-ff51-a2c6-1be3-bcb7797abbae",
+ "996c02d8-cbbd-8b95-7702-34c295fa183f"]
+
+
+def assert_annotations(statement, annotation_list):
+ """Asset statement contains listed annotations"""
+ for annotation_desc in annotation_list:
+ count = 0
+ for annotation in statement["annotations"]:
+ if annotation["description"] == annotation_desc: count += 1
+ assert count == 1, f"Annotation '{annotation_desc}' not found in {statement['statementID']}"
+
+def assert_lei_data(statement, lei_out, update=False, deleted=False):
+ """Assert statement matches expected output for LEI"""
+ print("Checking:", statement['statementID'])
+ assert statement['statementID'] == lei_out['statementID']
+ assert statement['statementType'] == lei_out['statementType']
+ assert statement['statementDate'] == lei_out['statementDate']
+ assert validate_date_now(statement['publicationDetails']['publicationDate'])
+ if update:
+ assert statement["replacesStatements"] == update
+ else:
+ assert not "replacesStatements" in statement # New LEI
+ if deleted:
+ assert_annotations(statement, ["GLEIF data for this entity - LEI: 00TR8NKAEL48RGTZEW89; Registration Status: RETIRED"])
+ else:
+ assert statement['name'] == lei_out['name']
+
+def assert_rr_data(statement, rr_out, update=False, deleted=False):
+ """Assert statement matches expected output for RR"""
+ print("Checking:", statement['statementID'])
+ assert statement['statementID'] == rr_out['statementID']
+ assert statement['statementType'] == rr_out['statementType']
+ assert statement['statementDate'] == rr_out['statementDate']
+ if update:
+ assert statement["replacesStatements"] == update
+ else:
+ assert not "replacesStatements" in statement # New LEI
+ if deleted:
+ assert statement["subject"]["describedByEntityStatement"] == ""
+ assert statement["interestedParty"]["describedByEntityStatement"] == ""
+ assert_annotations(statement, ["GLEIF relationship deleted on this statementDate."])
+ else:
+ assert (statement["subject"]["describedByEntityStatement"]
+ == rr_out["subject"]["describedByEntityStatement"])
+ assert (statement["interestedParty"]["describedByEntityStatement"]
+ == rr_out["interestedParty"]["describedByEntityStatement"])
+
+def assert_repex_data(statement, repex_out, stype, update=False, deleted=False):
+ """Assert statement matches expected ouput for Repex"""
+ print("Checking:", statement['statementID'], statement)
+ assert statement['statementID'] == repex_out['statementID']
+ if update:
+ assert statement["replacesStatements"] == update
+ else:
+ assert not "replacesStatements" in statement
+ if stype == 0:
+ assert statement['statementType'] == repex_out['statementType']
+ assert statement["annotations"] == repex_out["annotations"]
+ elif stype == 1:
+ assert statement['statementType'] == repex_out['statementType']
+ if repex_out['statementType'] == "entityStatement":
+ assert statement["entityType"] == repex_out["entityType"]
+ assert (statement["unspecifiedEntityDetails"]["reason"] ==
+ repex_out["unspecifiedEntityDetails"]["reason"])
+ assert (statement["unspecifiedEntityDetails"]["description"] ==
+ repex_out["unspecifiedEntityDetails"]["description"])
+ elif repex_out['statementType'] == "personStatement":
+ assert statement["personType"] == repex_out["personType"]
+ assert (statement["unspecifiedPersonDetails"]["reason"] ==
+ repex_out["unspecifiedPersonDetails"]["reason"])
+ assert (statement["unspecifiedPersonDetails"]["description"] ==
+ repex_out["unspecifiedPersonDetails"]["description"])
+ else:
+ assert statement["statementType"] == "ownershipOrControlStatement"
+ assert (statement["subject"]["describedByEntityStatement"]
+ == repex_out["subject"]["describedByEntityStatement"])
+ if "describedByPersonStatement" in repex_out["interestedParty"]:
+ assert (statement["interestedParty"]["describedByPersonStatement"]
+ == repex_out["interestedParty"]["describedByPersonStatement"])
+ else:
+ assert (statement["interestedParty"]["describedByEntityStatement"]
+ == repex_out["interestedParty"]["describedByEntityStatement"])
+ if deleted:
+ assert {"reason": "interested-party-exempt-from-disclosure",
+ "description": "From LEI ExemptionReason `NON_CONSOLIDATING`. The legal entity or entities are not obliged to provide consolidated accounts in relation to the entity they control."
+ } in statement["annotations"]
+
+def test_transform_stage_updates(updates_json_data_file, rr_json_data_file_stored, lei_json_data_file_stored,
+ lei_json_data_file_gleif, stored_references, latest_ids, exception_ids,
+ voided_ids, updated_ids, lei_json_data_file_output, rr_json_data_file_output,
+ repex_json_data_file_output, repex_json_data_file_gleif,
+ repex_json_data_file_stored, repex_json_data_file_gleif_new):
+ """Test transform pipeline stage on LEI-CDF v3.1 records"""
+ with (patch('bodspipelines.infrastructure.pipeline.Pipeline.directory') as mock_pdr,
+ patch('bodspipelines.infrastructure.pipeline.Stage.directory') as mock_sdr,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_streaming_bulk') as mock_sb,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.async_scan') as mock_as,
+ patch('bodspipelines.infrastructure.clients.elasticsearch_client.AsyncElasticsearch') as mock_es,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Producer') as mock_kno,
+ patch('bodspipelines.infrastructure.clients.kinesis_client.Consumer') as mock_kni):
+
+ # Mock Kinesis input
+ class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+ mock_kni.return_value = AsyncIterator(updates_json_data_file)
+
+ # Mock directory methods
+ mock_pdr.return_value = None
+ mock_sdr.return_value = None
+
+ # Mock ES async_scan
+ updates_stream = {}
+ async def as_result():
+ print("Updates stream:", json.dumps(updates_stream))
+ for item in updates_stream:
+ yield updates_stream[item]
+ mock_as.return_value = as_result()
+
+ # Mock ES async_streaming_bulk
+ async def sb_result():
+ voided = iter(voided_ids)
+ first = True
+ for item in updates_json_data_file:
+ if "Entity" in item:
+ id = generate_statement_id(entity_id(item), 'entityStatement')
+ if item["LEI"] == "00TR8NKAEL48RGTZEW89": id = "c9c4f816-ec2e-5292-4a0c-5b1680ff46a0"
+ yield (True, {'create': {'_id': id}})
+ elif "Relationship" in item:
+ print(item)
+ id = generate_statement_id(rr_id(item), 'ownershipOrControlStatement')
+ if (item['Relationship']['StartNode']['NodeID'] == '98450051BS9C610A8T78' and
+ item['Relationship']['EndNode']['NodeID'] == '984500501A1B1045PB30'):
+ id = "f16a374a-09f1-a640-8303-a52e12957c30"
+ yield (True, {'create': {'_id': id}})
+ elif "ExceptionCategory" in item:
+ print(item)
+ if "Extension" in item:
+ yield (True, {'create': {'_id': next(voided)}})
+ yield (True, {'create': {'_id': next(voided)}})
+ else:
+ if not item["LEI"] in ("159516LKTEGWITQUIE78", "159515PKYKQYJLT0KF16"):
+ print("Voiding:")
+ yield (True, {'create': {'_id': next(voided)}})
+ if item['ExceptionReason'] in ("NATURAL_PERSONS", "NO_KNOWN_PERSON"):
+ print("Person:")
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'personStatement')}})
+ else:
+ print("Entity:")
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'entityStatement')}})
+ if item["LEI"] != "2549007VY3IWUVGW7A82":
+ print("Ownership:")
+ yield (True, {'create': {'_id': generate_statement_id(repex_id(item), 'ownershipOrControlStatement')}})
+ for id in updated_ids:
+ yield (True, {'create': {'_id': id}})
+ mock_sb.return_value = sb_result()
+
+ # Mock Kinesis output stream (save results)
+ kinesis_output_stream = []
+ async def async_put(record):
+ kinesis_output_stream.append(record)
+ return None
+ mock_kno.return_value.put.side_effect = async_put
+
+ # Mock Kinesis output
+ flush_future = asyncio.Future()
+ flush_future.set_result(None)
+ mock_kno.return_value.flush.return_value = flush_future
+ producer_close_future = asyncio.Future()
+ producer_close_future.set_result(None)
+ mock_kno.return_value.close.return_value = producer_close_future
+
+ # Mock Latest Index
+ latest_index_data = latest_ids
+
+ print("Initial latest IDs:", json.dumps(latest_index_data))
+
+ print("Initial exception IDs:", json.dumps(exception_ids))
+
+ # Mock ES index method
+ async def async_index(index, document):
+ print("async_index:", index, document)
+ #if index == "references":
+ # stored_references.append(document['_source'])
+ if index == "updates":
+ updates_stream[document['_source']['referencing_id']] = document['_source']
+ elif index == "latest":
+ latest_index_data[document['_source']['latest_id']] = document['_source']
+ return None
+ mock_es.return_value.index.side_effect = async_index
+
+ # Mock ES update method
+ async def async_update(index, id, document):
+ print("async_update:", index, id, document)
+ if index == "latest":
+ latest_index_data[document['latest_id']] = document
+ elif index == "updates":
+ updates_stream[document['referencing_id']] = document
+ return None
+ mock_es.return_value.update.side_effect = async_update
+
+ # Mock ES delete method
+ async def async_delete(index, id):
+ print("async_delete:", index, id)
+ if index == "latest":
+ del latest_index_data[id]
+ elif index == "updates":
+ del updates_stream[id]
+ return None
+ mock_es.return_value.delete.side_effect = async_delete
+
+ print("stored_references:", json.dumps(stored_references))
+
+ # Mock ES search method
+ async def async_search(index=None, query=None):
+ id = query["query"]["match"]["_id"]
+ print(f"Searching for {id} in {index}")
+ if index == "latest":
+ if id in latest_index_data:
+ results = [latest_index_data[id]]
+
+ else:
+ results = []
+ elif index == 'references':
+ if id in stored_references:
+ results = [stored_references[id]]
+ else:
+ results = []
+ elif index == 'exceptions':
+ if id in exception_ids:
+ results = [{'latest_id': id,
+ 'statement_id': exception_ids[id][0],
+ 'other_id': exception_ids[id][1],
+ 'reason': exception_ids[id][2],
+ 'reference': exception_ids[id][3],
+ 'entity_type': exception_ids[id][4]}]
+ print("Stored exception:", results)
+ else:
+ results = []
+ elif index == "ownership":
+ match = [item for item in rr_json_data_file_stored if item ['statementID'] == id]
+ if match:
+ results = [match[0]]
+ else:
+ results = []
+ elif index == "updates":
+ if id in updates_stream:
+ results = [updates_stream[id]]
+ else:
+ results = []
+ out = {'hits': {'hits': results}}
+ return out
+ mock_es.return_value.search.side_effect = async_search
+
+ # Setup environment variables
+ set_environment_variables()
+
+ # Kinesis stream of GLEIF data from ingest stage
+ gleif_source = Source(name="gleif",
+ origin=KinesisInput(stream_name="gleif-test"),
+ datatype=JSONData())
+
+ # Storage
+ bods_storage = ElasticsearchClient(indexes=bods_index_properties)
+
+ # BODS data: Store in Easticsearch and output new to Kinesis stream
+ bods_output_new = NewOutput(storage=Storage(storage=bods_storage),
+ output=KinesisOutput(stream_name="bods-gleif-test"),
+ identify=identify_bods)
+
+ # Definition of GLEIF data pipeline transform stage
+ transform_stage = Stage(name="transform-test-updates",
+ sources=[gleif_source],
+ processors=[ProcessUpdates(id_name='XI-LEI',
+ transform=Gleif2Bods(identify=identify_gleif),
+ storage=Storage(storage=bods_storage),
+ updates=GleifUpdates())],
+ outputs=[bods_output_new]
+ )
+
+ assert hasattr(transform_stage.processors[0], "finish_updates")
+
+ # Definition of GLEIF data pipeline
+ pipeline = Pipeline(name="gleif", stages=[transform_stage])
+
+ # Run pipelne
+ pipeline.process("transform-test-updates", updates=True)
+
+ print(json.dumps(kinesis_output_stream))
+
+ print("Final latest IDs:", json.dumps(latest_index_data))
+
+ # Check updates empty
+ assert len(updates_stream) == 0
+
+ # Check LEI updates
+ stored_offset = -1
+ for i, item in enumerate(lei_json_data_file_output):
+ print(i)
+ if i == 0:
+ assert_lei_data(kinesis_output_stream[i], item)
+ elif i == 1:
+ result = {"statementID": "c9c4f816-ec2e-5292-4a0c-5b1680ff46a0",
+ "statementDate": "2023-09-03",
+ "statementType": "entityStatement"}
+ assert_lei_data(kinesis_output_stream[i], result, deleted=True,
+ update=[lei_json_data_file_stored[i+stored_offset]["statementID"]])
+ else:
+ assert_lei_data(kinesis_output_stream[i], item,
+ update=[lei_json_data_file_stored[i+stored_offset]["statementID"]])
+ if i == 7: stored_offset = 0
+
+ # Offset for start of RR statements
+ stream_offset = i + 1
+
+ # Check RR updates
+ stored_offset = -1
+ for i, item in enumerate(rr_json_data_file_output):
+ print(i+stream_offset)
+ if i == 0:
+ assert_rr_data(kinesis_output_stream[i+stream_offset], item)
+ elif i == 9:
+ result = {"statementID": "f16a374a-09f1-a640-8303-a52e12957c30",
+ "statementDate": "2023-11-01",
+ "statementType": "ownershipOrControlStatement"}
+ assert_rr_data(kinesis_output_stream[i+stream_offset], result,
+ update=[rr_json_data_file_stored[i+stored_offset]["statementID"]], deleted=True)
+ else:
+ assert_rr_data(kinesis_output_stream[i+stream_offset], item,
+ update=[rr_json_data_file_stored[i+stored_offset]["statementID"]])
+ if i == 5: stored_offset = 1
+
+ # Offset for start of Repex statements
+ stream_offset = stream_offset + i + 1
+
+ # Check Repex updates
+ stored_offset = -2
+ cycle = 0
+ count = 0
+ ent_offset = 0
+ for i, item in enumerate(repex_json_data_file_output):
+ print(i)
+ if i == 11:
+ stream_offset -= 1
+ continue
+ if i < 2 or i > 10:
+ if i == 0 or i == 12:
+ entity = 1
+ else:
+ entity = 2
+ if i == 13:
+ repex_update = [repex_json_data_file_stored[-1]["statementID"]]
+ assert_repex_data(kinesis_output_stream[i+stream_offset], item, entity, update=repex_update)
+ else:
+ assert_repex_data(kinesis_output_stream[i+stream_offset], item, entity)
+ else:
+ if cycle == 0:
+ print("Current item:", item)
+ item_lei = item['annotations'][0]['description'].split()[-1]
+ repex_item = [ritem for ritem in repex_json_data_file_gleif
+ if ritem["LEI"] == item_lei][0]
+ repex_item_new = [ritem for ritem in repex_json_data_file_gleif_new
+ if ritem["LEI"] == item_lei][0]
+ item_reason = repex_item["ExceptionReason"]
+ annotation_desc_del = f'Statement series retired due to deletion of a {item_reason} GLEIF Reporting Exception for {item_lei}'
+ annotation_desc_chg = f'Statement retired due to change in a {item_reason} GLEIF Reporting Exception for {item_lei}'
+ result = {"statementID": voided_ids[count],
+ "statementDate": datetime.date.today().strftime('%Y-%m-%d'),
+ "statementType": repex_json_data_file_stored[i + stored_offset + ent_offset]["statementType"],
+ }
+ if "Extension" in repex_item_new:
+ result["annotations"] = [{'motivation': 'commenting',
+ 'description': annotation_desc_del,
+ 'statementPointerTarget': '/',
+ 'creationDate': datetime.date.today().strftime('%Y-%m-%d'),
+ 'createdBy': {'name': 'Open Ownership',
+ 'uri': 'https://www.openownership.org'}}]
+ else:
+ result["annotations"] = [{'motivation': 'commenting',
+ 'description': annotation_desc_chg,
+ 'statementPointerTarget': '/',
+ 'creationDate': datetime.date.today().strftime('%Y-%m-%d'),
+ 'createdBy': {'name': 'Open Ownership',
+ 'uri': 'https://www.openownership.org'}}]
+ repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]]
+ assert_repex_data(kinesis_output_stream[i+stream_offset], result, cycle, update=repex_update)
+ cycle = 1
+ if i == 10:
+ count += 1
+ stream_offset += 1
+ ent_offset += 1
+ result = {"statementID": voided_ids[count],
+ "statementDate": datetime.date.today().strftime('%Y-%m-%d'),
+ "statementType": "ownershipOrControlStatement",
+ "interestedParty": {
+ "describedByEntityStatement": ""
+ },
+ "subject": {
+ "describedByEntityStatement": ""
+ },
+ "annotations": [{"statementPointerTarget": "/",
+ "motivation": "commenting",
+ "description": "GLEIF RegistrationStatus set to RETIRED on this statementDate."}],
+ }
+ repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]]
+ assert_repex_data(kinesis_output_stream[i+stream_offset], result, cycle, update=repex_update)
+ else:
+ stream_offset += 1
+ assert_repex_data(kinesis_output_stream[i+stream_offset], item, cycle)
+ cycle = 2
+ elif cycle == 2:
+ repex_update = [repex_json_data_file_stored[i + stored_offset + ent_offset]["statementID"]]
+ assert_repex_data(kinesis_output_stream[i+stream_offset], item, cycle, update=repex_update)
+ cycle = 0
+ count += 1
+ if i == 9: ent_offset += 2
diff --git a/tests/test_xml_parser.py b/tests/test_xml_parser.py
index a36ecdf..647af15 100644
--- a/tests/test_xml_parser.py
+++ b/tests/test_xml_parser.py
@@ -2,14 +2,17 @@
import sys
import time
import datetime
+import json
from pathlib import Path
import json
from unittest.mock import patch, Mock
+import asyncio
import pytest
from bodspipelines.infrastructure.processing.xml_data import XMLData
def validate_datetime(d):
+ """Test is valid datetime"""
try:
datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S%z')
return True
@@ -18,41 +21,49 @@ def validate_datetime(d):
def validate_date_now(d):
+ """Test is today's date"""
return d == datetime.date.today().strftime('%Y-%m-%d')
@pytest.fixture
def lei_list():
+ """List entity LEIs"""
return ['001GPB6A9XPE8XJICC14', '004L5FPTUREIWK9T2N63', '00EHHQ2ZHDCFXJCPCL46', '00GBW0Z2GYIER7DHDS71', '00KLB2PFTM3060S2N216',
'00QDBXDXLLF3W3JJJO36', '00TR8NKAEL48RGTZEW89', '00TV1D5YIV5IDUGWBW29', '00W0SLGGVF0QQ5Q36N03', '00X5RQKJQQJFFX0WPA53']
@pytest.fixture
def lei_xml_data_file():
+ """GLEIF LEI XML data"""
return Path("tests/fixtures/lei-data.xml")
@pytest.fixture
def rr_xml_data_file():
+ """GLEIF RR XML data"""
return Path("tests/fixtures/rr-data.xml")
@pytest.fixture
def repex_xml_data_file():
+ """GLEIF Repex XML data"""
return Path("tests/fixtures/repex-data.xml")
-def test_lei_xml_parser(lei_xml_data_file):
+@pytest.mark.asyncio
+async def test_lei_xml_parser(lei_xml_data_file):
+ """Test XML parser on GLEIF LEI data"""
xml_parser = XMLData(item_tag="LEIRecord",
namespace={"lei": "http://www.gleif.org/data/schema/leidata/2016"},
filter=['NextVersion', 'Extension'])
data = xml_parser.process(lei_xml_data_file)
count = 0
- for item in data:
+ async for header, item in data:
if count == 0:
assert item['LEI'] == '001GPB6A9XPE8XJICC14'
assert item['Entity']['LegalName'] == 'Fidelity Advisor Leveraged Company Stock Fund'
- assert item['Entity']['OtherEntityNames'] == ['FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund']
+ assert item['Entity']['OtherEntityNames'] == [{'type': 'PREVIOUS_LEGAL_NAME',
+ 'OtherEntityName': 'FIDELITY ADVISOR SERIES I - Fidelity Advisor Leveraged Company Stock Fund'}]
assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '245 SUMMER STREET', 'City': 'BOSTON', 'Region': 'US-MA',
'Country': 'US', 'PostalCode': '02210'}
assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Fidelity Management & Research Company LLC',
@@ -70,19 +81,90 @@ def test_lei_xml_parser(lei_xml_data_file):
assert item['Registration']['ManagingLOU'] == 'EVK05KS7XY1DEII3R011'
assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED'
assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000665', 'ValidationAuthorityEntityID': 'S000005113'}
+ elif count == 10:
+ assert item['LEI'] == '1595D0QCK7Y15293JK84'
+ assert item['Entity']['LegalName'] == 'GALAPAGOS CONSERVATION TRUST'
+ assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '7-14 Great Dover Street', 'City': 'London', 'Country': 'GB', 'PostalCode': 'SE1 4YR'}
+ assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': '7-14 Great Dover Street', 'City': 'London', 'Country': 'GB',
+ 'PostalCode': 'SE1 4YR'}
+ assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000585', 'RegistrationAuthorityEntityID': '03004112'}
+ assert item['Entity']['LegalJurisdiction'] == 'GB'
+ assert item['Entity']['EntityCategory'] == 'GENERAL'
+ assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'G12F'}
+ assert item['Entity']['EntityStatus'] == 'ACTIVE'
+ assert item['Entity']['EntityCreationDate'] == '1994-12-21T00:00:00+01:00'
+ assert item['Registration']['InitialRegistrationDate'] == '2023-02-13T22:13:11+01:00'
+ assert item['Registration']['LastUpdateDate'] == '2023-03-10T13:08:56+01:00'
+ assert item['Registration']['RegistrationStatus'] == 'ISSUED'
+ assert item['Registration']['NextRenewalDate'] == '2024-02-13T22:13:11+01:00'
+ assert item['Registration']['ManagingLOU'] == '98450045AN5EB5FDC780'
+ assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED'
+ assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000585', 'ValidationAuthorityEntityID': '03004112'}
+ assert item['Registration']['OtherValidationAuthorities'] == [{'ValidationAuthorityID': 'RA000589', 'ValidationAuthorityEntityID': '1043470'}]
+ elif count == 11:
+ assert item['LEI'] == '213800FERQ5LE3H7WJ58'
+ assert item['Entity']['LegalName'] == 'DENTSU INTERNATIONAL LIMITED'
+ assert item['Entity']['OtherEntityNames'] == [{'type': 'PREVIOUS_LEGAL_NAME', 'OtherEntityName': 'DENTSU AEGIS NETWORK LTD.'},
+ {'type': 'PREVIOUS_LEGAL_NAME', 'OtherEntityName': 'AEGIS GROUP PLC'}]
+ assert item['Entity']['LegalAddress'] == {'FirstAddressLine': '10 TRITON STREET', 'AdditionalAddressLine': "REGENT'S PLACE",
+ 'City': 'LONDON', 'Region': 'GB-LND', 'Country': 'GB', 'PostalCode': 'NW1 3BF'}
+ assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': '10 TRITON STREET', 'AdditionalAddressLine': "REGENT'S PLACE",
+ 'City': 'LONDON', 'Region': 'GB-LND', 'Country': 'GB', 'PostalCode': 'NW1 3BF'}
+ assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000585', 'RegistrationAuthorityEntityID': '01403668'}
+ assert item['Entity']['LegalJurisdiction'] == 'GB'
+ assert item['Entity']['EntityCategory'] == 'GENERAL'
+ assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'H0PO'}
+ assert item['Entity']['EntityStatus'] == 'ACTIVE'
+ assert item['Entity']['EntityCreationDate'] == '1978-12-05T00:00:00Z'
+ assert item['Registration']['InitialRegistrationDate'] == '2014-02-10T00:00:00Z'
+ assert item['Registration']['LastUpdateDate'] == '2023-02-02T09:07:52.390Z'
+ assert item['Registration']['RegistrationStatus'] == 'ISSUED'
+ assert item['Registration']['NextRenewalDate'] == '2024-02-17T00:00:00Z'
+ assert item['Registration']['ManagingLOU'] == '213800WAVVOPS85N2205'
+ assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED'
+ assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000585', 'ValidationAuthorityEntityID': '01403668'}
+ elif count == 12:
+ assert item['LEI'] == '213800BJPX8V9HVY1Y11'
+ assert item['Entity']['LegalName'] == 'Swedeit Italian Aktiebolag'
+ assert item['Entity']['LegalAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg',
+ 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O',
+ 'Country': 'SE', 'PostalCode': '426 76'}
+ assert item['Entity']['HeadquartersAddress'] == {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg',
+ 'AdditionalAddressLine': 'Fortgatan 11', 'City': 'Västra Frölunda', 'Region': 'SE-O',
+ 'Country': 'SE', 'PostalCode': '426 76'}
+ assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11',
+ 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76',
+ 'type': 'AUTO_ASCII_TRANSLITERATED_LEGAL_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses']
+ assert {'FirstAddressLine': 'C/O Anita Lindberg', 'MailRouting': 'C/O Anita Lindberg', 'AdditionalAddressLine': 'Fortgatan 11',
+ 'City': 'Vastra Frolunda', 'Region': 'SE-O', 'Country': 'SE', 'PostalCode': '426 76',
+ 'type': 'AUTO_ASCII_TRANSLITERATED_HEADQUARTERS_ADDRESS'} in item['Entity']['TransliteratedOtherAddresses']
+ assert item['Entity']['RegistrationAuthority'] == {'RegistrationAuthorityID': 'RA000544', 'RegistrationAuthorityEntityID': '556543-1193'}
+ assert item['Entity']['LegalJurisdiction'] == 'SE'
+ assert item['Entity']['EntityCategory'] == 'GENERAL'
+ assert item['Entity']['LegalForm'] == {'EntityLegalFormCode': 'XJHM'}
+ assert item['Entity']['EntityStatus'] == 'ACTIVE'
+ assert item['Entity']['EntityCreationDate'] == '1997-06-05T02:00:00+02:00'
+ assert item['Registration']['InitialRegistrationDate'] == '2014-04-09T00:00:00Z'
+ assert item['Registration']['LastUpdateDate'] == '2023-04-25T13:18:00Z'
+ assert item['Registration']['RegistrationStatus'] == 'ISSUED'
+ assert item['Registration']['NextRenewalDate'] == '2024-05-12T06:59:39Z'
+ assert item['Registration']['ManagingLOU'] == '549300O897ZC5H7CY412'
+ assert item['Registration']['ValidationSources'] == 'FULLY_CORROBORATED'
+ assert item['Registration']['ValidationAuthority'] == {'ValidationAuthorityID': 'RA000544', 'ValidationAuthorityEntityID': '556543-1193'}
count += 1
- assert count == 10
+ assert count == 13
-def test_rr_xml_parser(rr_xml_data_file):
+@pytest.mark.asyncio
+async def test_rr_xml_parser(rr_xml_data_file):
+ """Test XML parser on GLEIF RR data"""
xml_parser = XMLData(item_tag="RelationshipRecord",
namespace={"rr": "http://www.gleif.org/data/schema/rr/2016"},
filter=['Extension'])
data = xml_parser.process(rr_xml_data_file)
count = 0
- for item in data:
+ async for header, item in data:
if count == 0:
- print(item)
assert item['Relationship']['StartNode'] == {'NodeID': '001GPB6A9XPE8XJICC14', 'NodeIDType': 'LEI'}
assert item['Relationship']['EndNode'] == {'NodeID': '5493001Z012YSB2A0K51', 'NodeIDType': 'LEI'}
assert item['Relationship']['RelationshipType'] == 'IS_FUND-MANAGED_BY'
@@ -100,13 +182,15 @@ def test_rr_xml_parser(rr_xml_data_file):
assert count == 10
-def test_repex_xml_parser(repex_xml_data_file):
+@pytest.mark.asyncio
+async def test_repex_xml_parser(repex_xml_data_file):
+ """Test XML parser on GLEIF Repex data"""
xml_parser = XMLData(item_tag="Exception",
namespace={"repex": "http://www.gleif.org/data/schema/repex/2016"},
filter=['NextVersion', 'Extension'])
data = xml_parser.process(repex_xml_data_file)
count = 0
- for item in data:
+ async for header, item in data:
if count == 0:
print(item)
assert item['LEI'] == '001GPB6A9XPE8XJICC14'
diff --git a/tests/utils.py b/tests/utils.py
new file mode 100644
index 0000000..b1f801b
--- /dev/null
+++ b/tests/utils.py
@@ -0,0 +1,22 @@
+import requests
+
+# Mock Kinesis input
+class AsyncIterator:
+ def __init__(self, seq):
+ self.iter = iter(seq)
+ def __aiter__(self):
+ return self
+ async def __anext__(self):
+ try:
+ return next(self.iter)
+ except StopIteration:
+ raise StopAsyncIteration
+ async def close(self):
+ return None
+
+def list_index_contents(index):
+ url = f"http://localhost:9200/{index}/_search"
+ query = '{"query": {"match_all": {}}, "size": "25"}'
+ print(requests.get(requestURL,
+ json=query,
+ headers={'content-type': 'application/json'}).text)