From 36ae325f2bc3d2842dc6649ddb41d3f2c919c1b2 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 11 Jun 2024 11:07:30 +0200 Subject: [PATCH 01/30] Clear readme --- README.md | 522 ++---------------------------------------------------- 1 file changed, 10 insertions(+), 512 deletions(-) diff --git a/README.md b/README.md index df0038d..d673925 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# clinlp + [![test](https://github.com/umcu/clinlp/actions/workflows/test.yml/badge.svg)](https://github.com/umcu/clinlp/actions/workflows/test.yml) [![docs](https://readthedocs.org/projects/clinlp/badge/?version=latest)](https://clinlp.readthedocs.io/en/latest/?badge=latest) [![pypi version](https://img.shields.io/pypi/v/clinlp?color=blue)](https://pypi.org/project/clinlp/) @@ -6,8 +8,6 @@ [![made with spaCy](https://img.shields.io/badge/made_with-spaCy-blue)](https://spacy.io/) [![ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -# clinlp - ![clinlp](media/clinlp.png) * :hospital: `clinical` + :netherlands: `nl` + :clipboard: `NLP` = :sparkles: `clinlp` @@ -15,18 +15,16 @@ * :rocket: Open source, created and maintained by the Dutch Clinical NLP community * :triangular_ruler: Useful out of the box, but customization highly recommended -If you are enthusiastic about using or contributing to `clinlp`, please don't hesitate to get in touch (via [e-mail](mailto:analytics@umcutrecht.nl) or by creating an [issue](https://github.com/umcu/clinlp/issues/new)). `clinlp` is intended as a community project, and we would love to hear from you. - -This readme contains information on [getting started](#getting-started), how to [cite](#citing) this work, some basic [documentation](#documentation), the [roadmap](#roadmap), the overarching [principles and goals](#principles-and-goals) and how to [contribute](#contributing) :arrow_down:. - ## Getting started ### Installation + ```bash pip install clinlp ``` ### Example + ```python import spacy from clinlp.ie import Term @@ -92,514 +90,14 @@ for ent in doc.spans["ents"]: * `partus prematurus` `set()` * `VI` `{'Temporality.Future'}` -## Citing - -If you use `clinlp`, please find the appropriate citation by clicking the Zenodo button below. This should always point you to the current latest release: - -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10528055.svg)](https://doi.org/10.5281/zenodo.10528055) - ## Documentation -### Introduction - -`clinlp` is built on top of spaCy, a widely used library for Natural Language Processing. Before getting started with `clinlp`, it may be useful to read [spaCy 101: Everything you need to know (~10 mins)](https://spacy.io/usage/spacy-101). Main things to know are that spaCy consists of a tokenizer (breaks a text up into small pieces, i.e. words), and various components that further process the text. - -Currently, `clinlp` offers the following components, tailored to Dutch Clinical text, further discussed below: - -1. [Tokenizer](#tokenizer) -2. [Normalizer](#normalizer) -3. [Sentence splitter](#sentence-splitter) -4. [Entity matcher](#entity-matcher) -5. [Qualifier detection (negation, historical, etc.)](#qualifier-detection) - - [Context Algorithm](#context-algorithm) - - [Transformer based detection](#transformer-based-detection) -6. [Metrics and statistics](#metrics-and-statistics) - -### Tokenizer - -The `clinlp` tokenizer is built into the blank model: - -```python -nlp = spacy.blank("clinlp") -``` - -It employs some custom rule based logic, including: -- Clinical text-specific logic for splitting punctuation, units, dosages (e.g. `20mg/dag` :arrow_right: `20` `mg` `/` `dag`) -- Custom lists of abbreviations, units (e.g. `pt.`, `zn.`, `mmHg`) -- Custom tokenizing rules (e.g. `xdd` :arrow_right: `x` `dd`) -- Regarding [DEDUCE](https://github.com/vmenger/deduce) tags as a single token (e.g. `[DATUM-1]`). - - Deidentification is not builtin `clinlp` and should be done as a preprocessing step. - -### Normalizer - -The normalizer sets the `Token.norm` attribute, which can be used by further components (entity matching, qualification). It currently has two options (enabled by default): -- Lowercasing -- Removing diacritics, where possible. For instance, it will map `ë` :arrow_right: `e`, but keeps most other non-ascii characters intact (e.g. `µ`, `²`). - -Note that this component only has effect when explicitly configuring successor components to match on the `Token.norm` attribute. - -### Sentence splitter - -The sentence splitter can be added as follows: - -```python -nlp.add_pipe("clinlp_sentencizer") -``` - -It is designed to detect sentence boundaries in clinical text, whenever a character that demarks a sentence ending is matched (e.g. newline, period, question mark). It also correctly detects items in an enumerations (e.g. starting with `-` or `*`). - -### Entity matcher - -`clinlp` includes a `clinlp_rule_based_entity_matcher` component that can be used for matching entities in text, based on a dictionary of known concepts and their terms/synonyms. It includes options for matching on different token attributes, proximity matching, fuzzy matching and matching pseudo/negative terms. - -The most basic example would be the following, with further options described below: - -```python -concepts = { - "sepsis": [ - "sepsis", - "lijnsepsis", - "systemische infectie", - "bacteriemie", - ], - "veneus_infarct": [ - "veneus infarct", - "VI", - ] -} - -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher") -entity_matcher.load_concepts(concepts) -``` - -> :bulb: `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. - -> :bulb: The `clinlp_rule_based_entity_matcher` component wraps the spaCy `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: -> ```python -> from clinlp.ie import SPAN_KEY -> ruler = nlp.add_pipe('span_ruler', config={'span_key': SPAN_KEY}) -> ``` - -#### Attribute - -Specify the token attribute the entity matcher should use as follows (by default `TEXT`): - -```python -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"attr": "NORM"}) -``` - -Any [Token attribute](https://spacy.io/api/token#attributes) can be used, but in the above example the `clinlp_normalizer` should be added before the entity matcher, or the `NORM` attribute is simply the literal text. `clinlp` does not include Part of Speech tags and dependency trees, at least not until a reliable model for Dutch clinical text is created, though it's always possible to add a relevant component from a trained (general) Dutch model if needed. - -#### Proximity matching - -The proxmity setting defines how many tokens can optionally be skipped between the tokens of a pattern. With `proxmity` set to `1`, the pattern `slaapt slecht` will also match `slaapt vaak slecht`, but not `slaapt al weken slecht`. - -```python -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"proximity": 1}) -``` - -#### Fuzzy matching - -Fuzzy matching enables finding misspelled variants of terms. For instance, with `fuzzy` set to `1`, the pattern `diabetes` will also match `diabets`, `ddiabetes`, or `diabetis`, but not `diabetse` or `ddiabetess`. The threshold is based on Levenshtein distance with insertions, deletions and replacements (but not swaps). - -```python -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"fuzzy": 1}) -``` - -Additionally, the `fuzzy_min_len` argument can be used to specify the minimum length of a phrase for fuzzy matching. This also works for multi-token phrases. For example, with `fuzzy` set to `1` and `fuzzy_min_len` set to `5`, the pattern `bloeding graad ii` would also match `bloedin graad ii`, but not `bloeding graad iii`. - -```python -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"fuzzy": 1, "fuzzy_min_len": 5}) -``` - -#### Terms -The settings above are described at the matcher level, but can all be overridden at the term level by adding a `Term` to a concept, rather than a literal phrase: - -```python -from clinlp.ie import Term - -concepts = { - "sepsis": [ - "sepsis", - "lijnsepsis", - Term("early onset", proximity=1), - Term("late onset", proximity=1), - Term("EOS", attr="TEXT", fuzzy=0), - Term("LOS", attr="TEXT", fuzzy=0) - ] -} - -entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"attr": "NORM", "fuzzy": 1}) -entity_matcher.load_concepts(concepts) -``` - -In the above example, by default the `NORM` attribute is used, and `fuzzy` is set to `1`. In addition, for the terms `early onset` and `late onset` proximity matching is set to `1`, in addition to matcher-level config of matching the `NORM` attribute and fuzzy matching. For the `EOS` and `LOS` abbreviations the `TEXT` attribute is used (so the matching is case sensitive), and fuzzy matching is disabled. - -#### Pseudo/negative phrases - -On the term level, it is possible to add pseudo or negative patterns, for those phrases that need to be excluded. For example: - -```python -concepts = { - "prematuriteit": [ - "prematuur", - Term("prematuur ademhalingspatroon", pseudo=True), - ] -} -``` - -In this case `prematuur` will be matched, but not in the context of `prematuur ademhalingspatroon` (which may indicate prematurity, but is not a definitive diagnosis). - -#### spaCy patterns - -Finally, if you need more control than literal phrases and terms as explained above, the entity matcher also accepts [spaCy patterns](https://spacy.io/usage/rule-based-matching#adding-patterns). These patterns do not respect any other configurations (like attribute, fuzzy, proximity, etc.): - -```python -concepts = { - "delier": [ - Term("delier", attr="NORM"), - Term("DOS", attr="TEXT"), - [ - {"NORM": {"IN": ["zag", "ziet", "hoort", "hoorde", "ruikt", "rook"]}}, - {"OP": "?"}, - {"OP": "?"}, - {"OP": "?"}, - {"NORM": {"FUZZY1": "dingen"}}, - {"OP": "?"}, - {"NORM": "die"}, - {"NORM": "er"}, - {"OP": "?"}, - {"NORM": "niet"}, - {"OP": "?"}, - {"NORM": {"IN": ["zijn", "waren"]}} - ], - ] -} -``` - -#### Concept dictionary from external source - -When matching entities, it is possible to load external lists of concepts (e.g. from a medical thesaurus such as UMLS) from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: - -| **concept** | **phrase** | **attr** | **proximity** | **fuzzy** | **fuzzy_min_len** | **pseudo** | **comment** | -|--|--|--|--|--|--|--|--| -| prematuriteit | prematuriteit | | | | | | some comment | -| prematuriteit | 3.0.0`) - * Therefore non-destructive -* Work towards some level of standardization of components (abstraction, protocols) -* Follows industry best practices (system design, code, documentation, testing, CI/CD) +## Links -Overarching goals: +Some useful links: -* Improve the quality of Dutch Clinical NLP pipelines -* Enable easier (re)use/valorization of efforts -* Help mature the field of Dutch Clinical NLP -* Help develop the Dutch Clinical NLP community +* [Contributing guidelines](https://clinlp.readthedocs.io/en/latest/contributing.html) +* [`clinlp` development roadmap](https://github.com/orgs/umcu/projects/3) +* [Create an issue](https://github.com/umcu/clinlp/issues/new/choose) \ No newline at end of file From 07107bd8319694576700ad1dd8f42ec4d2c9d0cd Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 11 Jun 2024 11:11:33 +0200 Subject: [PATCH 02/30] Update readme --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d673925..dafec5a 100644 --- a/README.md +++ b/README.md @@ -92,12 +92,10 @@ for ent in doc.spans["ents"]: ## Documentation -The full documentation can be found at [Full documentation at clinlp.readthedocs.io](https://clinlp.readthedocs.io). +The full documentation can be found at [clinlp.readthedocs.io](https://clinlp.readthedocs.io). ## Links -Some useful links: - * [Contributing guidelines](https://clinlp.readthedocs.io/en/latest/contributing.html) * [`clinlp` development roadmap](https://github.com/orgs/umcu/projects/3) -* [Create an issue](https://github.com/umcu/clinlp/issues/new/choose) \ No newline at end of file +* [Create an issue](https://github.com/umcu/clinlp/issues/new/choose) From 1a1b968b5a5dd1605b3f1993c33730ec65e63b6e Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 11 Jun 2024 11:18:57 +0200 Subject: [PATCH 03/30] Add small summary --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dafec5a..0a859dc 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ * :rocket: Open source, created and maintained by the Dutch Clinical NLP community * :triangular_ruler: Useful out of the box, but customization highly recommended +`clinlp` is a Python package that provides a set of tools for processing clinical text written in Dutch. It is built on top of [spaCy](https://spacy.io/) and is designed to be easy to use, fast, and flexible. The package is used, developed and maintained by researchers and developers in the field of Dutch clinical NLP. + +If you have questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html)! + ## Getting started ### Installation From beecc6ee68870c6e1d3f3682b23ffe85960c6acb Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 11 Jun 2024 13:35:35 +0200 Subject: [PATCH 04/30] Update create issue link --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b1b476..c2c6107 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ Please keep in mind that this page describes the ideal process and criteria for Our preferred way of communication is through [issues](https://github.com/umcu/clinlp/issues), GitHubs built-in issue tracker. We use it for most communication, including questions, bug reports, feature requests, help getting started, etc. This way, the entire community can benefit from the discussion. If this is not an option, you can also reach out to us by e-mail: [analytics@umcutrecht.nl](mailto:analytics@umcutrecht.nl). -To create an issue right now, you can use the following link: [Create an issue](https://github.com/umcu/clinlp/issues/new). +To create an issue right now, you can use the following link: [Create an issue](https://github.com/umcu/clinlp/issues/new/choose). We will try to respond to you as soon as possible. Please keep in mind that we are with a small group of maintainers, so we might not always be able to get back to you within a few days. @@ -75,7 +75,7 @@ If you have a feature request that you would like someone to pick up, please inc Keep in mind that a feature request might not be picked up immediately, or at all. We will try to keep the roadmap up to date, so you can see what is being worked on, and what is planned for the future. Furthermore, remember that `clinlp` is a collection of generic components that process clinical text written in Dutch. If the proposed addition does not meet those criteria, a separate release might be a better option. We typically also don't include preprocessing components (e.g. fixing encodings, de-identification, etc.), as those should preferably be handled at the source. -If you would like to contribute to the project yourself directly, it's recommended to [create an issue](https://github.com/umcu/clinlp/issues/new) to discuss your idea beforehand. This way, we can make sure that your contribution is in line with the project's goals and that it is not already being worked on by someone else. Of course, for small changes that only touch a couple of lines of code, you can also directly create a pull request. When you are ready to start working on your contribution, please follow the steps outlined in the [Pull requests](#pull-requests) section. +If you would like to contribute to the project yourself directly, it's recommended to [create an issue](https://github.com/umcu/clinlp/issues/new/choose) to discuss your idea beforehand. This way, we can make sure that your contribution is in line with the project's goals and that it is not already being worked on by someone else. Of course, for small changes that only touch a couple of lines of code, you can also directly create a pull request. When you are ready to start working on your contribution, please follow the steps outlined in the [Pull requests](#pull-requests) section. ## Pull requests From a7dbfbb6d71b6085412c7cc6e088fdb94063246d Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 11:50:13 +0200 Subject: [PATCH 05/30] Add introduction page --- docs/source/index.md | 17 +++++- docs/source/introduction.md | 104 ++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 docs/source/introduction.md diff --git a/docs/source/index.md b/docs/source/index.md index cac1092..4ef3a77 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,6 +1,19 @@ -# clinlp documentation +![clinlp logo](../../media/clinlp.png) -Welcome to the documentation pages for `clinlp`. +Welcome to the documentation pages for `clinlp`, a library for performing NLP on clinical text written in Dutch. In the menu to the left, you should be able to find the information you are looking for. If you have any questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html)! + +```{toctree} +:caption: Documentation +:hidden: +Introduction +Installation +Getting started +Components +Metrics +Qualifiers +Roadmap +Citing +``` ```{toctree} :caption: Search & index diff --git a/docs/source/introduction.md b/docs/source/introduction.md new file mode 100644 index 0000000..dc9d3c7 --- /dev/null +++ b/docs/source/introduction.md @@ -0,0 +1,104 @@ +# Introduction + +```{include} ../../README.md +:start-after: +:end-before: +``` + +`clinlp` is a library for performing NLP on clinical text written in Dutch. It is designed to be a standardized framework for building, maintaining and sharing solutions for NLP tasks in the clinical domain. The library is built on top of the [`spaCy`](https://spacy.io/) library, and extends it with components that are specifically tailored to Dutch clinical text. The library is open source and welcomes contributions from the community. + +`clinlp` was motivated by the lack of standardized tools for processing clinical text in Dutch. This makes it difficult for researchers, data scientists and developers working with Dutch clinical text to build, validate and maintain NLP solutions. With `clinlp`, we aim to fill this gap. + +We organized `clinlp` around four basic principles: useful NLP tools, a standardized framework, production-ready quality, and open source collaboration. + +## 1. Useful NLP tools + +```{include} ../../README.md +:start-after: +:end-before: +``` + +There are many interesting NLP tasks in the clinical domain, like normalization, entity recognition, qualifier detection, entity linking, summarization, reasoning, and many more. In addition to that, each task can often be solved in multiple ways: using rule-based methods, classical machine learning, deep learning, transformers, or a combination of these, with trade-offs between them. + +The main idea behind `clinlp` is to build, maintain and share solutions for these NLP tasks, specifically for clinical text written in Dutch. In `clinlp`, we typically call a specific implementation for a task a "component". For instance: a rule-based sentence boundary detector, or a transformer-based negation detector. + +When building new solutions, we preferably start with a component that implements a simple, rule-based solution, which can function as a baseline. Then subsequently, more sophisticated components can be added. If possible, we try to (re)use existing implementations, but if needed, building from scratch is also an option. + +```{admonition} Contributing +:class: note + +Components can be built by anyone from the Dutch clinical NLP field, typically a researcher, data scientist or developer who works with Dutch clinical text in daily practice. If you have a contribution in mind, please check out the [contributing](contributing) page. +``` + +Currently, `clinlp` mainly includes components used for information extraction, such as tokenizing, detecting sentence boundaries, normalizing text, detecting entities, and detecting qualifiers (e.g. negation, uncertainty). We are now extending the library with more components, both for different tasks (e.g. entity linking, summarization) and different methods for solving the tasks (e.g. a transformer-based entity recognizer). + +We prefer components to work out of the box, but to be highly customizable. For instance, our implementation of the [Context Algorithm](TODO) has a set of built in rules for for qualifying entities with Presence, Temporality and Experiencer properties. However, both the types of qualifiers and the rules can easily be modified or replaced by the user. This way, the components can be used in a wide variety of use cases, and no user is forced to use a one-size-fits-all solution. + +```{admonition} Important +:class: important + +Remember, there is no guarantee that components based on existing rules or pre-trained models also extend to your particular dataset and use case. It is always recommended to evaluate the performance of the components on your own data. +``` + +In addition to functional components, `clinlp` also implements some functionality for computing metrics. This is useful for evaluating the performance of the components, and for comparing different methods for solving the same task. + +An overview of all components included in `clinlp` can be found in the [components library](components). + +## 2. Standardized framework + +```{include} ../../README.md +:start-after: +:end-before: +``` + +Some of the real power from `clinlp` comes from the fact that the different components it implements are organized in a standardized framework. This framework ensures that the components can be easily combined and that they can be used in a consistent way. This makes it easy to build complex pipelines that can effectively process clinical text. + +We use the [`spaCy`](https://spacy.io/) library as the backbone of our framework. This allows us to leverage the power of `spaCy`'s NLP capabilities and to build on top of it. We have extended `spaCy` with our own domain-specific language defaults to make it easier to work with clinical text. In a pipeline, you can mix and match different `clinlp` components with existing `spaCy` components, and add your own custom components to that mix as well. For example, you could use the `clinlp` normalizer, the `spaCy` entity recognizer, and a custom built entity linker in the same pipeline without any issues. + +```{admonition} Getting familiar with spaCy +:class: note + +It's highly recommended to read [spaCy 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. +``` + +In addition to the `spaCy` framework, we have added some additional abstractions and interfaces that make building components easier. For instance, if you want to add a new component that detects qualifiers, it can make use of the `QualifierDetector` interface, and the `Qualifier` and `QualifierClass` classes. This way, the new component can easily be integrated in the framework, while the developer can focus on building a new solution. + +Finally, by adopting a framework, we can easily build components that wrap a specific pre-trained model. The transformer-based qualifier detectors included in `clinlp` are good examples of this. These components wrap around pre-trained transformer models, but fit seamlessly into the `clinlp` framework. This way, we can easily add new components that use the latest and greatest in NLP research. + +## 3. Production-ready quality + +```{include} ../../README.md +:start-after: +:end-before: +``` + +`clinlp` can potentially serve many types of users, from researchers, to developers, data scientists and clinicians. One thing they all have in common, is that they would like to rely on the library to work as expected. Our goal is to build a library that is of production grade quality, meaning that it is reliable, robust, and can be used in production environments. To ensure this, we employ various software development best practices, including: + +* Proper system design by using abstractions, interfaces and design patterns (where appropriate). +* Formatting, linting and type hints for a clean, consistent and readable codebase +* Versioning and a changelog to track changes over time +* Optimizations for speed and scalability +* Structural management of dependencies and packaging +* Extensive testing to ensure that the library works (and keeps working) as expected +* Documentation to explain the library's principles, functionality and how to use it +* Continuous deployment and frequent new releases + +```{admonition} Constant improvement +:class: note +We actively maintain the library, and are always looking for ways to improve it. If you have suggestions how to further increase the quality of the library, please let us know. +``` + +More detail on our best practices can be found in the [coding standards](https://clinlp.readthedocs.io/en/latest/contributing.html#coding-standards) section of the contributing page. + +## Open source collaboration + +```{include} ../../README.md +:start-after: +:end-before: +``` + +`clinlp` is being built as a free and open source library, but we cannot do it alone. As an open source project, we highly welcome contributions from the community. We believe that open source collaboration is the best way to build high quality software that can be used by everyone. We encourage you to contribute to the project by reporting issues, suggesting improvements, or even submitting your own code. + +In order to be transparent, we prefer to communicate through means that are open to everyone. This includes using GitHub for issue tracking, pull requests and discussions, and using the `clinlp` documentation for explaining the library's principles and functionality. We keep our [roadmap](https://clinlp.readthedocs.io/en/latest/roadmap.html) and [changelog](https://clinlp.readthedocs.io/en/latest/changelog.html) up to date, so you can see what we are working on and what has changed in the library. + +Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP. As an added benefit, we believe working together in one place is potentially also be a great way to improve quality of scientific research. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that the research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. From a96bc4a6bffe5d8d457d6483c559fae3e8e7febe Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 11:50:44 +0200 Subject: [PATCH 06/30] Add numbering --- docs/source/introduction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/introduction.md b/docs/source/introduction.md index dc9d3c7..ff8837d 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -90,7 +90,7 @@ We actively maintain the library, and are always looking for ways to improve it. More detail on our best practices can be found in the [coding standards](https://clinlp.readthedocs.io/en/latest/contributing.html#coding-standards) section of the contributing page. -## Open source collaboration +## 4. Open source collaboration ```{include} ../../README.md :start-after: From 71a63186d4eb24e192d704d98a99426e1ecaad8a Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 11:51:00 +0200 Subject: [PATCH 07/30] Add markers for including in docs --- README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0a859dc..9dab5a3 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,19 @@ ![clinlp](media/clinlp.png) -* :hospital: `clinical` + :netherlands: `nl` + :clipboard: `NLP` = :sparkles: `clinlp` -* :star: Performant and production-ready NLP pipelines for clinical text written in Dutch -* :rocket: Open source, created and maintained by the Dutch Clinical NLP community -* :triangular_ruler: Useful out of the box, but customization highly recommended - -`clinlp` is a Python package that provides a set of tools for processing clinical text written in Dutch. It is built on top of [spaCy](https://spacy.io/) and is designed to be easy to use, fast, and flexible. The package is used, developed and maintained by researchers and developers in the field of Dutch clinical NLP. + +* :hospital: `clinical` + :netherlands: `nl` + :clipboard: `NLP` = :sparkles: `clinlp` + +* :star: NLP tools and algorithms for clinical text written in Dutch + +* :triangular_ruler: Organization in a standardized, flexible and highly usable framework using spaCy + +* :rocket: Production-ready, performant, well-tested and easy to use + +* :bulb: Free, open source, created and maintained by the Dutch Clinical NLP community + + +## Contact If you have questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html)! From 089bff77a775fe20611dde05b54037f951dde3dd Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 12:00:52 +0200 Subject: [PATCH 08/30] Add information how to create a component --- CONTRIBUTING.md | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2c6107..ecd0441 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ - [Repository structure](#repository-structure) - [Coding standards](#coding-standards) - [General principles](#general-principles) + - [Creating a component](#creating-a-component) - [Formatting and linting](#formatting-and-linting) - [Dependencies](#dependencies) - [Tests](#tests) @@ -146,6 +147,47 @@ Please keep the following principles in mind when writing code: We fully acknowledge that writing production ready code is a skill that takes time to develop. We are happy to work together, so please don't hesitate to reach out to us. This is especially true for scientific researchers who are working on something cool, but are new to software development. +### Creating a component + +When creating a new component for `clinlp`, try to: + +- Use a class to define the component, and use `__init__` to set the arguments. +- Inherit from `Pipe` to make it compatible with `spaCy`. +- Use the `clinlp_component` decorator, to automatically register it in the component library. +- Use a dictionary to define any defaults, and pass this to `default_config` of `clinlp_component`. +- Use type hints for all arguments and return values. +- Use the `requires` and `assigns` arguments to specify which fields the component needs, and which it sets. +- Implement the actual behavior of the component in the `__call__` method + +The following code snippet shows an example of a new component: + +```python +from clinlp.utils import clinlp_component +from spacy.language import Pipe +from spacy.tokens import Doc + +_defaults = { + "arg_1": 1, + "arg_2": True +} + +@clinlp_component( + name="my_new_component", + requires=["input_spacy_field"], + assigns=["output_spacy_field"], + default_config=_defaults +) + +class MyNewComponent(Pipe): + + def __init__(self, arg_1: Type = _defaults['arg_1'], arg_2: Type = _defaults['arg_2']): + ... + + def __call__(doc: Doc) -> Doc: + ... + return doc +``` + ### Formatting and linting We use `ruff` for both formatting and linting. It is configured in `pyproject.toml`. @@ -260,7 +302,7 @@ We use type hints throughout the codebase, for both functions and classes. This ### Documentation -We like our code to be well documented. The documentation can be found in the `docs` directory. If you are making changes to the codebase, please make sure to update the documentation accordingly. +We like our code to be well documented. The documentation can be found in the `docs` directory. If you are making changes to the codebase, please make sure to update the documentation accordingly. If you are adding new components, please add them to the [component library](TODO), and following the existing structure. #### Docstrings From 1b977bf748c84552476c8d4df4fa92f08362435c Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 12:01:13 +0200 Subject: [PATCH 09/30] Add information how to install --- docs/source/installation.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/source/installation.md diff --git a/docs/source/installation.md b/docs/source/installation.md new file mode 100644 index 0000000..bbda180 --- /dev/null +++ b/docs/source/installation.md @@ -0,0 +1,23 @@ +# Installation + +Installing `clinlp` is as easy as running: + +```bash +pip install clinlp +``` + +As a good practice, we recommend installing `clinlp` in a virtual environment. If you are not familiar with virtual environments, you can find more information [here](https://docs.python.org/3/library/venv.html). + +## Optional dependencies + +To keep the base package lightweight, we use optional dependencies for some components. In the component library, each component will list the required optional dependencies, if any. They can be installed using: + +```bash +pip install clinlp[extra_name] +``` + +Or, if you want to install multiple extras at once: + +```bash +pip install clinlp[extra_name1,extra_name2] +``` From 15a417bd76e93af97d10ab02dc0694359cd29bcc Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 13:45:46 +0200 Subject: [PATCH 10/30] Add getting started example --- docs/source/getting_started.md | 165 +++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 docs/source/getting_started.md diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md new file mode 100644 index 0000000..0fd3ccb --- /dev/null +++ b/docs/source/getting_started.md @@ -0,0 +1,165 @@ +# Getting started + +This guide contains some code examples to get you started with `clinlp`. Since `clinlp` is built on top of the `spaCY` framework, it's highly recommended to read [spaCy 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. + +## Creating a blank model + +You can create a blank `clinlp` model using the following code: + +```python +import spacy +import clinlp + +nlp = spacy.blank('clinlp') +``` + +This instantiates a `Language` object, which is the central object in `spaCy`. It contains all default settings for a language, in our case Dutch clinical text, such as the tokenizer, abbreviations, stop words, and so on. Calling it on a piece of text creates a `Doc` object: + +```python +text = "De patient krijgt 2x daags 500 mg paracetamol." +doc = nlp(text) +``` + +In the `Doc` object, you can find the tokenized text: + +```python +print(list(token.text for token in doc)) + +> ['De', 'patient', 'krijgt', '2', 'x', 'daags', '500', 'mg', 'paracetamol', '.'] +``` + +Each token in the document is a `Token` object, which contains the text and some additional information. You can also access the tokens directly from the `Doc` object: + +```python +print(doc[8]) + +> 'paracetamol' +``` + +A span of multiple tokens, essentially a slice of the document, is called a `Span` object. This can be a sentence, a named entity, or any other contiguous part of the text. You can create a `Span` by slicing the `Doc`: + +```python +print(doc[6:8]) + +> '500 mg' +``` + +Even when using a blank model, the `Doc`, `Token` and `Span` objects already contain some information about the text and tokens, such as the token's text and its position in the document. In the next section, we will add more components to the model, that will add more interesting information. Then we can start using our model for more interesting things. + +## Adding components + + +The above model is a blank model, which means it does not contain any additional components yet. It's essentially an almost empty pipeline.Adding a component is done using: + +```python +nlp.add_pipe('component_name') +``` + +For example, let's add the `clinlp` normalizer and `clinlp` sentencizer to the model. They respectively normalize the text and detect sentence boundaries: + +```python +nlp.add_pipe('clinlp_normalizer') +nlp.add_pipe('clinlp_sentencizer') +``` + +If we now again process a piece of sample text, we can see that `clinlp` has added some additional information to the `Doc` and `Span` objects: + +```python +doc = nlp( + "Patiënt krijgt 2x daags 500 mg " + "paracetamol. De patiënt is allergisch " + "voor penicilline." +) + +print(token.norm_ for token in doc) +> ['patient', 'krijgt', '2', 'x', 'daags', '500', 'mg', 'paracetamol', '.', 'de', 'patient', 'is', 'allergisch', 'voor', 'penicilline', '.'] + +print(str(sent) for sent in doc.sents) +> ['Patiënt krijgt 2x daags 500 mg paracetamol.', 'De patiënt is allergisch voor penicilline.'] +``` +Other components can use these newly set properties `Token.norm_` and `Doc.sents`. For example, an entity recognizer can use the normalized text to recognize entities, and a negation detector can use the sentence boundaries to determine the range of a negation. + +## Information extraction example + +Now that we understand the basics of a blank model and adding components, let's add two more components to create a basic information extraction pipeline. + +First, we will add the `clinlp_rule_based_entity_matcher`, along with some sample concepts to match: + +```python +from clinlp.ie import Term + +concepts = { + "prematuriteit": [ + "preterm", " 'Preterme' 'prematuriteit' +> ' 'bd enigszins verlaagd' 'hypotensie' +> 'hypotensie' 'hypotensie' +> 'veneus infarkt' 'veneus_infarct' +> 'partus prematurus' 'prematuriteit' +> 'VI' 'veneus_infarct' + +``` + +As you can see, the `doc.spans['ents']` property now contains seven `Span` objects, each with the matched text, along with the concept label. + +Now, as a final step, let's add the `clinlp_context_algorithm` component to the pipeline, which implements the Context Algorithm. It can detect qualifiers, such as `Presence`, `Temporality` and `Expierencer`, based on triggers like `geen`, `uitgesloten` for matched entities. + +```python +nlp.add_pipe("clinlp_context_algorithm", config={"phrase_matcher_attr": "NORM"}) +``` + +We again configure it to match on the `NORM` attribute, set by the `clinlp_normalizer`. + +If we now process the same text, we can see that the Context Algorithm has added some additional information to the entities: + +```python +doc = nlp(text) + +for ent in doc.spans['ents']: + print(ent.text, ent._.qualifiers) + + +> 'Preterme' set() +> ' 'bd enigszins verlaagd' set() +> 'hypotensie' {'Experiencer.Family'} +> 'veneus infarkt' {'Presence.Absent'} +> 'partus prematurus' set() +> 'VI' {'Temporality.Future'} +``` + +In the above example, for readability all default qualifiers have been omitted. You can see that three out of seven entities have correctly been qualified, either as `Absent`, related to `Family`, or potentially occurring in the `Future`. Of course, your specific use case determines how the output of this pipeline will further be handled. + +## Conclusion + +In this guide, we have shown how to create a blank model, add components to it, and process a piece of text. It also shows how to configure individual components and organize them in a specific information extraction pipeline. Note that there are more components available than shown in this example, you can find them in the [components library](components). By now you understand the basics, and are ready to further explore everything `clinlp` can offer! From c2522391d106fe7c90cd525e570beb15ecac3c84 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 13:45:51 +0200 Subject: [PATCH 11/30] Typo --- src/clinlp/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clinlp/util.py b/src/clinlp/util.py index d2d6e81..3cd0cf6 100644 --- a/src/clinlp/util.py +++ b/src/clinlp/util.py @@ -26,7 +26,7 @@ def get_class_init_signature(cls: Type) -> Tuple[list, dict]: ``list`` The arguments of the class's ``__init__`` method. ``dict`` - and keyword arguments of the class's ``__init__`` method. + and keyword arguments of the class's ``__init__`` method. """ args = [] kwargs = {} From 37524420e51e25b176815bfd4641ece853093659 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 13:55:42 +0200 Subject: [PATCH 12/30] Update displacy doc render --- README.md | 7 ++++--- docs/source/getting_started.md | 2 +- media/example_doc_render.png | Bin 27872 -> 29781 bytes 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9dab5a3..bf8f764 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ![clinlp](media/clinlp.png) -* :hospital: `clinical` + :netherlands: `nl` + :clipboard: `NLP` = :sparkles: `clinlp` +* :hospital: `clinical` + :netherlands: `nl` + :clipboard: `NLP` = :sparkles: `clinlp` * :star: NLP tools and algorithms for clinical text written in Dutch @@ -68,7 +68,7 @@ entity_matcher.load_concepts(concepts) nlp.add_pipe("clinlp_context_algorithm", config={"phrase_matcher_attr": "NORM"}) text = ( - "Preterme neonaat (a2Para0m_o26ua0nI%7I-_) zS3b{r?>}&VxU<&KO_z05ovPaB>^(7RDsoupr08&Pa9Hy1q&48+5G;ZF7StEO-_F;P z1aNTZHMUYxYVuN2)M{?dR<;gcIJkE)X&ES5ntJ#lXa0vYG~$v-pTpNbx4>yg;&kz; zqsfP;Nn#?sI)Sz#Fvmu7%Z$C1>1%?6>PXiYE!VNZiVB`s2_1BwP{{j#KAk>ZO-=q; zxBNM==Ku4D5*+m$Kdm|`2t~Y|U6K6gbT~R82D~W?PnZOc#|Wp5&8Intj(vj?JuwxW z_S+kdn`eVN>+-sGg~DdHFmOk6U&e$A&WwBXB>Uqtz68u{cZA5*PwI9YsfBuW|M8!n%P6P{TPZ9lFM z`@yK+KmsL`vl4PvzRzQdF=5~%gjwnJW;f^WS;nXig50^jTUcj}T%4zTQO5X{zbpNL z$l!0@8D8pr-vYU&ZUJ6|>nUJy=AJ0U1bSfFQy79E6CoiT0Qr1y39DrtSqH%o+|( z@9~wUH$kg)v`#Ft{HPu#p$0hMg#nDmsJD*0rSXMnrIq!U!EZB%`r!u*`yM3kusAm8 zJfV&%8EWX_a-whwzxO=^JKzg`z|X8trbRbcduZ+=vBTKlk!M0U)lgP@;WDYSRuJSi zV=YfGLdE%`;T7SCUWa*5zZma(TSxgLOtS9P30zvii}^5F7dUbRicNlg#Hq0HO~F51mh6bb4n#i-0(0M?kkN*ZfYYKF*H0giR%a2om8P3CIQYH3CVW|HKDwc5o%~! zXbY0cGi);)2bdmsM^Qf|q-KbJy|~9LHhrlSwqPoOPW_0GCk~&A8JjRGomG&@=&gnOmM|^)W$|JGx*E7#P>Z<|V*%10 zLn_@R9Vlt>Rf`+ZG1N=?p@442@E7MVQV+cLm*1nEf@3d7E-ec58hrA4)Ef zu_=y9dydMCHWdCsl4y(6ESz0FmEj0KH+pN3VB61Jpgzqf;Wp`3M^;u=aoCt%T9&#& zrPOe*U;;UdCQJ1-@<+VUTmrd|NzF+&$to#7lh;3vT8coK1=4e5)+L>2^WTWk2d9>d zeHw#;eWCW?D$8$RXIOU;E@L|ZAsFpVl!mN;Qk$$VlU&kVihITmtR}hYqYynGH6M*? zT1yhnN7W3rkIv~mjAE4`nlqZ`TANHinC6*tw24cL%2&0#-)6KYuZoMvQ52fl-gEP^ zMCiN|5Mk zt;Dki_W6#=3*}JYt>kRC!X*1fiV^2c@|&#}RA)kMsMccI8*bUpIm&CxLBd&*eV^^b ze##yQF0~-EZ=S4PI>~N}_xc87?8j9~)7A~Yg&2E~Kg(zVlDs$n|DI`%b!HO2;H z23IxS1{+O1<}-1nnNOe_U7gAN_}&n|yKy z3Me_Yz~h$9cIMIY34gm-;b0R{bs4DU_>a@c2h_Sx0yfKUM&N}aU z+j?0byJ(Q;w}2PRb`Go^F(Nb zoUBl$OQ}>jE%|xVM!jE02jd)XIYKk1bPXLWPu7=XWbziDh$@*~1M=3xn!2OX(1-C4LE4f#tT!nbQ-QC;GzJ??^5wYM}fw<;1ELz{jQXq+EMrXnXS65$2MGD{!--kTc@SzC$24hltY zLr?>2)r$NY(~5xZPm^`MLe2W7ZAGx#?&;f>%4WNTqsav-uM4m5%ct!fO|mtmb$eHo zeR&;C>rJPgexBg#<@=NSGOT2*E-|iPr`0NGi_6(kV1Hm(d||wZvX`>*WaZ=#iwop4 z*dciB=@>;A+oSnY<6`sIZ|FFoVIRY$gJYg|r)YYlzDH$686+N1R){Ky=+9s`)jCj} zz57Xg5_fnaw&!2nFtmZq{e6f$xeP zR5|gY{#6X0%g%B(+A#LJrt(k0*Lm%m9$U9EQIHW<4mMB2=B~A%p}n!MRlM&LZ&q%a zOd2{CeZN2Xy?^e1O5rnf%4s`vy?SK1sTprUbpE{&)9UU`?PPk+diL|6tI@yq%z7g~ zpzbHlo>?DVn;bY%n`-Hg)vdv|g44|yn#4k8)&4wT;yvu1Tal~ZY;?Y=qpIV1w!-$L zP2?h?dpur$?uz#5*reI$P4=Fpo~RaTF^wti2CfsCcfRh>)ehD6^f38VJ$h{j9eaPr z@54{rDC^08R6AF{v)EzSD_fsd&(Bw8eGYxlcv?8OU*hX=#_1^werDZ{7MQDnLnm=D zr-gIAffrAJ5Amab+b+t=^V2WLdWN4vdcvRn^?lA9L;MdM+*TXhtqF2L(MwxfyysMY zxKO6(b-Z33omq}=&nmx!0H(*Ls&t;v@qG3$6i<@UxV(=n?FO1X{ z)Oi5>ssq-Qw^CMydkfs7!of$`!XW{7@W7W8_`<;p+yB-L6czsaR8Y;<2kf9HZR-dO5734v2S1naKl%TUl>fB&uadg| zDap&h`R|hdO8Nhn)OH8ENjWrod|9J|GvnaYS z+yBm)D0*G6t_&QUIGnt+gq9EdX)bc+Ynj&0{Vugd{1 zQsi;-r!MpVJo=j=)BB3k8tLE9+DYKw!`gk}I{p6r)R4xV^1o~P=TeZ3=Mqt!jnZ4? z{qJU8aqj=CO&6s9Ybq;XjW~HY>%pn*dGedw|MPnTNB$#!s0iwZmW?2xHMemVqt#;P z?mPN`+pVaG&R^F{CQ*v4ZJ*h@70JGhzkL7-%n5A344*H@*?2ZasDjs+qG|k-}76cS|r#m zncoeQIfr-MtTK~(&Z`@(UaV`a_@8x^5eHu{ypQ;9)%V?FPIF4j9W7o`J9P1!q3mz2I4G|g;G{e&&asEtu zZ~T>eKhNK(^>U0EW6s-4+6Y~5E!xOy<;~+&{xjxrb%V<7%{S+PFC9Ngrv&zM_~oUh zrJ}I0q#`lVtaDstrS61X|Dc*yz*!7eMYDGOnmvEKnvR;}+vsGJkBWHyHK%G;QId`h zbjMrQDhu3imF-BVM*7~svFZFrit6G#*P6>fgcObl_etwc!Uv=36uP>tNV3SjFqEh) zt4|ekeFtOhf?ZrJ8hyw_h6Qe0fT3VqO>n73HT$jm?rTH*n`es79R!2VR!|K+7qqv6 zpC3$GJfUV0C%_o%R4oq?nUFVc1o=Di3uzNRL=ft~tD*F;xj!t!sru4xx4DeSyz0;} zNjOYw5m)zkIbOx|DUMvmv1vg^f4Hi)yYh}hGh>(Lf3gIuo;4gK(Bca%lriuu%qTh zXWg2PpY8C>$kKvGC2OLA_Xkthk8zX%=Mtx4;43}%NuDOBF(c=$Ywi{3IBS)zZr&}C z{fqRmWI0L!Y}SBMlTGw)J;B4zDUB>~9oIw-j!RF#yd10dX=y5#$bGiX^DVu}b-Nrh za2NWq1V$1KV3{lCX;EyD!e%@~m#Z>1O?V zLH5{fb!=j_So{;zp1a#JDDYy4$QDnRo7m*O88AeYgvqgThoBL+@NQx&*hf{dKU$E$ z?HC3vo1n{4hWn{fY|8q2v4@j}CJ}mr%rl4Ajz2?~cKU7s^Hcnkeq$}RNxlSjkMo7C zm5?8;IktR9uq_1EDBn>3@O_qb-q+UKF;tMc&S>ESHDx#ex1h1nkVE?q-!`kjQ3 z;}f^w(*QRoy122S|7p{QCyvC5FkV`+KKzxDN1vM&Yo3e?Ax$giJ+gyM50?@IyRW?jThW6ijWA7XTwf zV>GBREOgpSERo9-rzeXpR$i3VL;Wa6>HAvQ`R?RB;kwqa`Vto3np&m~!J;&%)mFHZ z-(^!guuOC;R-SOP@q30zP@8OwWXa*EqF;*pd011H=p(SHwK+=Y3@)oN1}Fw^8fBw& z(&Zn;w6s}*iFqG6qrP!I>AUu2)aUfY$yl)7~Li^_MxG0SAT4roOgy|+@W}FQ+|x*pM?Q9EiPj{b%_*vo=wN9*$WM?Ni<2+ z;X-o!Mc~xw=te|(ZQY*ctE8sR3gHmXo&l>T<$`I2Ua~~BXD;KWgm|xT9CchvcyZmc zcCf^kVXj=*={^qiV-y#@6K5CMr6H((#%xSJiQ_S@7OjBF3ewSo?)rw2Amn8FjVcI7 z;e&o@02&ds&Do;w)?DePx$ZsjwUkVAyNn)!&vE=vIBELpEcjU{ z`THolo)Qfm)*tYidO2*wS;?)tc5ASZnxpR$mYUSbf7^%DI5bc^tl5kkBQsRwd8y8j zspcj8<@LuMO-G_>#>&sHtWAy6)Y`Yi2uC=(amzF&nF&e8KSeKSUO|1tQ!OlpIxaW^ zu+&km$5mQC{$r-qSK~gP+DYtTgarMYA_Ih-SzHrngmT+EqIzlCS3+M>l+ik^(OEE1 z379FGD-fjWBA3)`$~Oo&Q{~pPew$y&X9_ul>xP?i4&^m&#}=x)8r4gBjZk>om?&?V z1wrp<*r@ipn{wL+$MD~mOrPgALE!e-&tNpepR9TH7;|CdDgF*`x0`3!jvfH4VP7YK zU4(irF)UtcT8OGO1gQ#Z{@|6_Z~7=hZ1Mfz)}HQd_oBXijh5_yCZlGxgsge4*4;oy z6N+(oZ*2356@YMj7nu5TXmMgzY(!eTK(x}ai|Li4qi3i&wAW0hr}d?KeU z-3Sf7s&M`AhzW$X-G{+|0S9KN&Ot-MDOip zmViN~%;e8>xP8rz3i=d%i?kIA5r`zb8>mAgtk9eS%wvq%z<{2|-O5N_Zo zol-}-EFmM0>No=V{WY^S6&`_Yu)teFGh%6h>IwvRzSDiwci|dI6xo#U3$H&CCSP4nnMzycZ3{& zb|Dr7s{?q*T0=y%;XD*3aalfKQKGBFc9{2lYi$ts0o|IT^G|rZv_ykqtPB|L`Ir#M)FfEEPSB#!n@=V3`4=nH z^yX|stSe4?{>7MiH2~DpKbap771&j8OI{?){v2_S@D>n=P4PT-zl;4e*XaMVO-@RV z!c$8GWuI3my|oE03!fKb>3jgE)=Y_9bZus$SoYFoFQ-J9iZS~$e>9n%NGHC^!)=|5 z6trVu<0OM6Va0_WF8rJn0M)ZLU&lPnhKRyiGpj+Gq?fBshZkp(K6JQ${gT{wN{rF= zYmA(Xqp2{8hA=B7A)>?29Oyk=OiB(;QFw1w+)eLj4N3ot zi-Io3vbDxR1_nh`t7sW^diw^ZQ9|qGF%_+_I26$OE#3(U8S}=rdQhB$gYq)o6xXR0 zM@Y43U`+Rkpl5T?@ak~x4-V(cJY`j4OGl<{F2b*zqW2A{!#z8kj-b9tABF@rJE5l4 z&MG;2B>-KostxehF7P@v4#9l-*N5KLLJ+%oZ*K}x&-+Y}O(Y(nYN6D;D7Q(23O#8S zJutqVfmTQ1Vs;k^v=aDTO;E+XgzLl@^KF!?^al$kDaN0vh@!ZU7M+44e0YEXg#qJLa=0=_HvyevgUK9s}G{qy9&iSR4ui6*7wt)Sn0R(ML9d( zubgiz)*Cp-ho@%={;ZNSe>ww0m&YdT@m(1+7+DxW4w&cO7FWcYUC`c3f(7jb9M`rP z-WhsP2fLNHWh*Kf-rdZ$}&2ZI;=ble%@s+#r-Tg4P33ZDtytyx>bk13H+>$sSCL`bln&*X2 zh~t-cX?xXkl+gO`nvFTf5roc{L#^+a6sP7FDr5R%&4+&syzIyR68*FDN=2I-@5k?r z;2wI509$54%rE2?Z^X~Psg#uq3+QW=%n_nhrJxPzZXw0d+6@X}WT&Al#?>mnYfz!V z)Ma$tYm|xNBxa@qNhaNre%^Ql{)P*chjtp?1dY@5-02AoPaLKNjaBa!jSa$*drHq~GO}YL zk0*xfiY*x6E+D&MbU0qsR-X0WdB;MswyG{Yq_atIxwM~lsGesGyiW#O1;(}=5Oti5 z7?Dp{mr1C(1*ZCx#<6o`Lcb&T<3UIim>`7wYGr5}mF=)Z>b>Q}PF(&^JW(;d>ZBw1 zsO*F!AnD=r`K+?rKc~*YbI_A_(k4Z0glL!MmnFula4L+>sMoZTNzk3LX{&2IxdnboZLGUd&Qs3%Rrk-fD@E(DAO#=ayv%S%&{UMv(%PZ{Vr-c$Cp(zS3`Ic(?3!Y$QHMu^-zP5#bLjos@v*3&TZ~rX~pQ@ zY=>PhxZJ}H@=uOvKwMl?{1$#G=n=V31*Qe_1nMH3g^bR|{EmEuO8!Xl{1~V`0j>s_S81RIApt zqq^$no6sn`7P*@k&~NZKsZi}7AQ*O(btVICtrTHIY%Q#NL9K{s@F^!_)h>bSKf>)mX#9Ixa=>EoSsXmg#AeRZAk9|tcNH5ti z6#5;->hEn{uF#avRqXD1$!Sx%I}Oz@M3Y6N;tBoG%UUZ6Yn746Bu_aM@`RKyIJTgQ zd!h~t@FuztCgjSW($f+I+MRmntJy|9m|vnJf6PZK`Cb1Pv*77ck@3 zHVD4mB}6?f2_p9be^AMzOy-)hR{6uv zG$(4$r@M22856|H5;$zuFOsWT=)LAy$JpVwj^>__n)OK;J9Q*7)8JaF}h3yM0hp=~yvC43rvI zSmRPJ^j=X7fiE$|G`&s^>}=8^%TA)6XOt+Hfv_qorOb=~?m7r!7TLMmlBsf2sB04#EQ{$@}n$;2=B7YeX zs^&t#7pZb2{B9|E2D^qK&`TF3=keQ&3v$ox&!PvlkoWV>kJ4Qe7*1ywn8CXjLatDz zV#YV>(qwK&44NweY$-#A)kQ%aNm6=i0SuVtB6Sz^nB5X8m!RyaNH9Ej2#S~`tyiRG z6A1p3Wbr_HH*r6JED~M+3M?~TB)muIRuOkXs{s1;&Sde$?OWpp`TSGOG+_`km2a=0 zz4SNp$9SSsPKp4j;JfI4=au%87TtlPO?v8ch|`_=D+mLI2>x36+r$$4Gqx0CugMDN z%=)E`cd~}u3Y6^YH{Fh^onRfQX#GcXbN=CaZx6`j4EC__|=Q> z6qKCdYJ)L)k(r=waJjczvE9%qyuu!?Vl38h=#x>*Y{Y%(gDz_0f&KM~Gb?E+(t#L* z8iQ1l$k_Zu?+0$VUIZ?3;v~scEmsV0?;K;mu!g)eh}RfO-3cysTzzOaz<3M#9s*;MZt|3bxPT(PS#5< z=DC5`78yZ!jwQM@2>UH0u-8`WVCok-Oy>(lXDoK1WY1K zHk|qy)TmC*8$YqWTE67K;GEV?{iaYe2kfVpy^bS zXa8N>-%Fz)!k46`dsM#%ivJ~a3U>c;KD$vfLjN@SU%6`%LYOUUL_^nu{`2m@E8Z{6 zA>PS-k$74%+cYu$^_tYFb*tZ|IGzyTFgo+*Yp_otiGfl5D;g0^N6QbLPE5R4!-5D2 zy(XwyPsgvrIiH!oFw6m=@ht_tDY%E_{g7G3P``d4=g&72N8de={7@0n00rKbl_WSfND-XIWZ7K>g zC#{bFyRZfDF4ZT0$r{}5KR4gjJSNvi)?~dUOyhurM-8GWqnYZ)SSUx6v?~>8BKV)N zu+c#}x}-TC8!k3$P@F)-M7+j2#q4QKAE}Ui8}TdpoA- z9KWNAF)|>O>So&xZTE9u6ZyS{@Acht=^TjGc%foWEETosJ0tU!0>q8JZF-Tf(WeT0 zI^{fWgAT#^aaJiId_%SlIhvW@;)PJkA;Rdt#8Lnd{NLKkcGh8dhDUr64n)lj(+YvH z5&MT{0z%YipDj3YEkDZlYpRNrV9;N%b^X#Hw*8#k|0cum<+7(L#}L65&^?R{W50@Y zuBY3bDBuM-b9}aAJ~;tFSJc0JOQvmU76Qg~aiVDDR(RL*)58`JNz#dMfR55*?B}?v z_yBRr{y45F;(zD}fS3wLq4c3v+XMV#3i?krMM=sd{;!f-S~P@j#Qyx!Yz62M#B|x= z;CXd9ETGc%$Ui{%WebSkX(9^_=W@rfECNE29^<>PdVipM1M`}yH5T&S$6q>Bh-_{T zfzU3-dy_!VgN>e^OXAp@R819^3R(!ABYEC9D69Xqci0W?Rvrp<(gB+X*2r(CfC7QUclw(zyOX| zwGP)FzXiX?!AwXf`4x(LA{OC}KM-DGT!%$ystA<0d`aNnju!MH=j&ID`#OlPT79?$ z0r)hdk(cWh4qGIaQo`$j_rI8qbXuawYKLyU{EG$K;1~B*z6?XWf#v+c4PUy{ADFU$ zW$XMv1UZ>(=WgHD>qUd>ur!1Z@x6_=z{D7&fsigrvz;nq$z8uWw&bFp#q+K*2K@Re zXBN-1I1@p{jN+P#<+|3(dcPTlv*?}5;WdQ10nlzTHH@GcAZ(rg*2>Falud_|@Pqfd zFGF?fc5FNQe#uSHUtEXek*@s^OyP* zd7{pdc}kMPMl%Wi)1=>1XY)41%+?Aw={$<#rzb(KL;x~kRmZj zimozzhaRmbGNT;gUSN}#M+`lFxX#%b&GlMk`b%`00XvrFn@)!>=4MehtM;{U8|=X} zmLq5kIk>|V-fNAl{b%jRU-~cT*GK=JBp-An3QUTzhK>fO2n&UOuen#rgwY|}*L@hek`8fk8{5BxQ=F>nhi5ktvH7V1 zA_s$X*i~QYv%4K8{2sv$ESWaE1O6!+g~CHk*S@C7L-Zx(70O6#D+O+(=f@3zwN@Qz zxA~$15K9EbonS)cgwDTvhW6m!CEIHa5NS&uzB@H|!GR>xgJzTGYt^TAFC03t62zTl z3T6QpxeR`=9*+>60P4A9^d1n;_A7+u|7EDkg)BhSdeFx3hNOm_-Es)sn{aLnq1d&* z#O{;H;50xS9A(Sq@r#|w8p8ETwg9^&iq|C&dWoZazil#@oT;8O)ejA9JYRpxj>1?r z-_o@yj6hg?2OIv3`I}gg*?ZNtY(d5^+p$@_>TSBln|qF8}>fgAC=kw?6YFA`I9(yL0zuL ztUP?aNPv?1`nG^B&h+{3!6YGA21A~YB%i(xO?rFLMZ$yW;h&{{%*0Bjpim5RTpeFd zm7HOV(dc!#D~#t@UVCH(`7dRCL`_*Xl!1zYcEKOt@a{ccx)5qff;B7l%oJ$oF#1Z2 zI!oqY`uA;O*miZiOgIKvii)7^)@}M$-g^uSoS~iY>yJ3-1Q#G6JUGfc&>v^*!?x4M zpPSfK`2K+`G0<@$-}(00H9UF&!$d6gkoGBrp-#b^gDZC5v`biu!VrCIC!+J>rkcB3MHgyD7~F}`&@_jLp*h2XeWjqRqA^nV~=tvG<*+6fCT9Vo$fZs~@Muo`nPMb~7S8 z|ABTuIC803iM?+RzSk;L3!Tw8|EErZG5$1OFe1BZqo=5%dAAHYELDC8g zc;sKMoeomXWIy^Q0}~LF=qT$lcLSagZLt?1_X($wiVR;e_ODUY@7xT<_(u~29PrJW zN7(=*8UT=R5qM-eS3?g%5L37dHQXv<$IE{tK))gQw^Ok+Z1EvhR-P!_lQ59%c8D@Cp z1~mR#5kwxYlooLu{gN0g1gwBIgXI(erc2%lWpRK*6&a#-4@}Xo<)gX*Yz)8$>H4(C z2h7_q-Mx6ebyPr}^WIdhZP^@x%>1yMVbWYAsowRvV5$gO3xK*h;P{UML|c3lfWSuo z1~~P}y6^VLSjhHt{SHfba{K_e^XGiwy-CUSo(Lv|AUCLf@MWQDzUZox7aL#@O_1{$fz?3?nZzp_95Wb9O)`6-Af8FPX?(o%iswLv_5A#k6ToMX1El$TC@` z9}!2mK>jxX_UnLS_xK8Erv|Jday_sA6F__AR(7;hfWW9G;};6Ki+Co0t($5=_q;lQ z+0>R6s)%l00D>Qz6##qc+yb%-BICeo?ZADAYZ1%m5@%Bb*|_Jwf+5G^_>;$=`$LAw zP;*}Z{2p@!m`<-;I*?BbUroR{k)N@*tvKo#Y&QMHkCS6pfB>)d6iM;k*+Qy_Gp{HR zFlPILu3z{VFZf410DT>EFn$l94=NMt0URIsyHJzr%5xLC!Q>yGQ&cLcgod|aOa0|I zp;1|Kohd&7V69TMl^Nz26Q&ACvYzHE?R357bNB(@r!nBj+^KKwyiFZXWzd~lk;+DF zM}Z1(qi`l;J+Dok@85@bZ+&y1stI^tIp(spVF-YuOdWFieq_F0^y>=%DPu`?g8p1D z*?-lQ$!d_@jO^U@K??2h?b%H?@F=zvCcomklTZ2cS5mPBh#D~88+oRsYxD5Fwqstu zhUqzm2lPEplmQTZBlK6sod+OlI(A=pd<*FTAeF~jG7Ah>Q?#1%HsstQL0~v^{-ac8 zjR4r~0O;lHV-DjZ5JG=9l8RNMp0y@TQ^e{shq{I)bP1#5DOYmRr+kRtWlU!r`>;>! ze4E!6R^2-k*4%(iVzwJX1w|F#`8)d9h;n=T^ZVs z&KwRnojig2A9xT%qrt*gH0uRnuL9x~<-U@0uNN!vQaMOoMi(1dvIOo(2`2oCuiR## z=xGcD+?<8(zsuV-RW#cZx4QheAPjrRS`;(wwJF0#COtr8 z_cDuM6mb7*&j5Xe|0Q6GhgSgX%uR0qT!uHCu={SnEa=^uv;X?FqrUsO8L%uLD1a&L zb3Bx;Ub6Lr@Jine^3m%7d?j~xweic;G!}e~al(e$MP_`X4X|7Rc$#Zkc(l4<>XV`G zZrUh`bxxtkrEGt5FF$Y`7)<4CXZEl(^1y zV>Cz@`4xCv)3PWUT$T#c@}G(rvo{>dDo7$LUY`c7-zEcdaXkTj6P>>cI%0 zYAD%->?&+*iL7?bYbY|^PzBwZ2@>^2)u&7e}$5~R}XrrNASytS6n}IGCS9MUC%ehl`hWL@zh*Y z942=EE|_Ld=~9vHC?e5i@z7xrOX@CyH{NeT7J)ZZ0RzLNpd zFTo9jnxbxyYkU&fm5HuOpE0^qbvcR>TNz(%%&Z6BLH=3F*|0*hDdC8*U$K>cEk4bW zO6QZpDu59*TtFA3_SJ=9Z-kUjlK>Tvf|u^99&vx>5E3!kAVTp4qZ>UVF%li2(~B2x zt6C}@@Ld4x(D|wc!Vn@c-`88uFV4uskmC|6YTcuBA>qSG1NJTsG*tt)q%G4r5%&%b z_UYD3%I4EtFwgj`!Zp;Jexx)*;6O8b*AK(V(vbb0baKQ)S)xiVj_=aq6&h%G1~{69 z7;#L%j|G#2?|!6w$`>TYC#FJ1v5Gh}myiq`ZiEW$@(jTtXNy{nBB9{VM$4-Po&mJ# z7J`-S>C*6D+%QjluRv0|j2CTM5SXM)e>;E*&KE}g!I_8PJzH->9J8S1J4C?A&b^z! z4O)wPwAkgSX1i9gt3wxaId&tB-j3xD3g zWC`YvSyVaw6zLykLi)67$P0@zM4o9|q*XgYK;jku~r+nAYHr_uZ zj1pFPr~PVL7=BK7Z{&J>h%laE5{Q-LEn6)RKD`yNL??NqV|aC62oiO8d6bqdgRYmh z7WdHa4w7@6EMjpWP~N+A3Ls$C_+cF=(YGD;7|OfuV{wX;4QW+c#eDuaH3rW&5LeFu z^%ei_Kz|j@K}%V(yoV5AdKtlk=Bv+x!j$c1E<*ifda=^yYBYxq=h#SF82nWglN^ZT zluU8duMcT|o;Ht}ce6nLby$|a$WFjHz~y9L-H=*Cr8OIUqCoaaWZrZ9W3fQOm1QAb z+|%ocSt_J$*YK!0S!q-ikV)Q z7p0LFZ>bSpkR4$&)RlIm6Wz5jwBjWf+lWf0xqvZq@vHZS9C3n21RZf!Q?qs*!+3W`C}aJ8#sZ z8-)X~&mA`jaiw!J(;IxF@o6;Nx$yFOQyl^%KKVt9E+dEF$r4L63|fS&Y`mk-NO;pC z6;EuEKqG+8w!@?c#|_yU#TbdUDkbVGzKwxuyOWPz;O{s*Vn(iwoq+T`zRCG*$m8MpZEmtJ^+MA>Ow!Gf9YFAosdl9ZO+LQ|EL6w13pzeK(&Ncwd8-h|yK2J|N>OMI z!4_o>1H_E(M*tPU0aH;yCYHoPSiTr0W2+%Dp>;|oi3eixH@a5_I?mcFpf57J8ZjqM zX?<#Wx0_TSs9!pEmSSCC>`x|PKmzK(&{Z!LVD=N0w?Btb@c_Vr_jipkzAFU#kIBu>5*bNevr^O z2gBm|9YYpdM8H_=c&W1`~5NW6*@@(WE+ViG9Fg= z^yk4|w0j%Npv!G;k?3-ydq~?2zvr_b%rgAKU!8Ihg+PIKaPtMuSh9Ef*TkzKdpmNc zX>$>cFnO)_vX=ot>d|WN+6)KxAkAFXRePq*YbQe455nux`{Gs&L1{w< zzKv1MF%J^r0y5LEkMUK9tTGoef{H8mX5aj}BVYM0sM$m3P>(_Z!I z4~K>?uuZR~yiWjJy#~gHISG!4!IqJtYa)!;Oes&4=<8n8KALX$cEmnBD#ceApUAoF zKKc|g^rn(z-#%_@VX8-E`gtyvWDykhzLw6y zx3{0{C*3b5pUpaN`p&e-`YmSgP-vT^vlqK6lu2ii%BK{t1xi0a%cy;RKA~&s+TR|c z9m{C>Sn4_lNDBqURk544eD;k18|gH&RGUaHMklz$OD*Vk{1|D(fVU3iCbD4s#SG=40eexakF+U8xEaI=B9qrX_Mkf9c988Ysn2aB`Mi} zIo$Su)>kL^}Z08 zcX1*KlM;DU3PQrV+A$OuJ_GgBB8oLggif!3N79!&{m63hH!(J-H%SaGNalSl!5LeF zk~n7{qvG+&5WLP{QGPy4N72EM`w7{Xe8@f@+MAcnF(!xkGiBIV86M)m@y$@*Es{{V z1HN?L22ty)6`{xImk2nss)Qb*GU?9a8Gjsc)>oiMzH)fQ8?aXwP09^gleLw%jRhZ3 zQr&n1Wyza9xEt&yr42Zztu@MgYJ_R>@>U1R#>14y=kiRlabS(Wk9f4e&7%o#!(Lp6 zzHv)zBI(bS44AM>H3xlM@;v3y{qY7#wg zFE4ydHAb23nTIAtQ~m$7ch*r+u7A6iMi2q%E-C3AS`iqK-Ux!^&|T8fodZZqDo9BV zDIm>|3Q8!_jetrjAaU+z|Mq*f>#TL&zu&Xg`Ae71FvC1?fA8!1T-U^v-WcQ!cFGvc z`PIxuV zrA1@i3>Pk=g`1K(T-V%4zEKzz-}TX_qb2g5R1wO5WSBBaXTY2%+Km3PWd z*$+=+dV8-TC0js;Rg~y`B*{fnz!BZ8<EW3B@Lt{*Ru9`CB}zG3$SOzrFv{Syt<`9 zJ#K?mGF7H0T9oE$)e?@Ci|6R}8(Ko*8C2?D!zCZC((_;1*|)2^y=72Z-k5ZiNztd4 zjk)`=-GU)bAv%zp9X>JRd{+%)gSB#3FL;@czql*2Vd9G>?B+~+JnORs(>Rf4FU^=^ zX0N@H0KpBktt?EZb64BZmuleY488rt@ELV;-e!dyo1;^uJ!iWBrlC-ZNhdn?%Jwm@ zGjpE7!bicjp*#ADM5*%EU)U$JC<_tKicst5FTlmi#F}{zPh=Q#elD3LwK~GT)>7YPMP(OxXV2Z1&CGWiO=cTq zSlCI)Sxe&1C^9BNdAKu1TlJP9qGGFp_qecw%Y`oAUX1zO=ET0EmNKvG9jhVkNBd9=Pt~fOyauk%*f}a;Rn-_x_iB{1 zsKPh*EX7NCZm^Vh!y=g-F|gxCCY#HvG^$ZT`LN;W1nMp>HlgWi<@%NuwTV{$3K;=#@Nj|g=BN8-R|w1 zftSpiOJe&LM5^YfCX&=xB^B?xy=gg*7{B&6REDt%$O+s?Af;EbQYho6%b=PTUAYtEF|uL))>(Oo=kXO+(VdP8#x=SJw3hG!!zatS(=H+{1~hv9 zO3=D3dNy0DFl~qPmPNMhsV>?V{n#c=Z;*E`_)78aIq}ht#_XvvcY7l2&pP#OYh(E4 z?n_!QK|G6*zZ{Q>EsbUj&v$)&c42-%#K4%Qm7~_IEgaczd5j@#Vfoqg;qO=yF9%Lc z!(JU;$*01jdi4KObri zC>|S?lbZj~KmU2?d;|b-=CRu!{rU6|@a$}s<<9^5anM6O$-v8ak?omQ{rU7L@a(9F zt3O|5TmyoIG`gNP{`vI(%bUC!L1CT%ESWdp*SycYON6h|@J1ZtpVZHWLsm2TZ?_N= z`?1~vrDtiidFGT?_ax7)(<>A&jZFmB73@vAeS#3$?Nwzu-uUcy{G(gg{$cITW7=;j zU5Nc<*=m>!GZNS`gxGkk%mVh6PJo)oS2i0Q42+RIpm1k2v3Bc%`KQfBu0;0v@TF{acLzylBfM_#Mtd+SaQ1P6?z zUHt02k{t!n;AO~q_oJfZje|bFUj}ccKeKowkNhD^tu5; z>q)cN-SVOP*Iu4F3!Rz*C_Qi}`_PC=R9e4D4*HC>m&CIcy! z=TPLC7)ub9bFcuycu*oc3dCx-+b_>{5I}Axh75*kETBSIQ~Rx@G%}=#fples6U1xu z0&SZq9|*IMec*VZC3~DUt(4u6grYR@)JZ?p_j@$x@S5kxr*&V(J0?^clqlS~4v0Ac za0n%kM^^VfstH7bnD^^+M-Xo(DhN1Gg#iJ+7aTq^n5Gr0un@7#UYFZ2cWIeL69`Fd z-3EHQS)&c|t^hziM8^O%QPmUa@j35B#5X`xR4jpmZxgC|1$>}LfE5{W9s#O_E(na~ zwIENq^&6;Nq)p&7!F{?u^dEqX5j*N8LxnKuX$#2ymbDE6moMdx4o`K9#R>S`dV$SH zg_Qkike)qgg}gwaNZ5BFROkH*6~w6mKU+OSgw0RN?R8=GdNLZuwR2v9&-Nw3XK(>j zTeB>_06Fal5a%;mm!q>CZU+X;abTX;^mQlHjIpH#lw9it;s6!oA59*=$~HES!sx)gvH4R zI8!7*?zH(lCug=ZNh@9dzU93o#<1ef;1+*9_yyRZX6Zs}w46i{vcOCtQ0GmpQ?RyX z=`c4`@LRbZp7;YCLk4h;w>mvTEJWX+xit+gjH=*R_Ka~~E16P?bJhVCu!(l<41_I5 zkoB1O%)ct7bXxfF_Lr%~826x6ss?54H7QBVs1PPY+FM?i2c@^2d_+M|y~1qf9v%ZZ zl-oPXBH^KwX2QmkGtHVNB@(jrE!t=JBYh9)$ z?{QCK8wQ7xbq*ZHin%<%#?Zp0@Cj)*o_IgSr{nzFoPSIj8q1k)V7-HZM_&k!7!;9n z3eyRx1`N`}*f?upc#bwNVJtk$M8}sLEMxMl(!T;>97^Ru7OQt)#pAYL+OE z%53&++s)=FeIt8vdv(KJJWR7F0GFwZ^d-2u-}ded;eCGj9jsmNkk|A%=kAYo5)xAA zmjd5UBhUfp$32MFkg)Xh$ohBUM$|rY?!5rado5?or8j6Z+&f0tZ@1JH+M8yLp@dpo z;V!ToZ(i_M3*T)jUcWC%FKU;3Cwb6k8uR=E%wazG>S98dM7XU|AOGpM%qs@evlq5u z>GJ{IAIMTg^#uZMX@|Lq_h*I{sZ{ETbI&j!q?#G%r~VRW@-`(Q%bqu;(ZTgQ__(YV zrQ4Yfm)z|*i=TL%`uAe^F5zx^`>St*AAsVoWbJ5ZhC(GzT(7in3QOdQmp2MkN6&{R zdaCC*p;XTwpN$H!TMy7JJK?2sTN>8wi zeb(H0q2Y?1PCjp~b2)(9cEh1&5Uhn?Lc!}f!#|`O&aA!M>W`p!%`;@}_2}h%uGbp{ zeIb7Q>%5QDKkKu48yS263`*n2GCw1R(1paiuZL%UCOjY6{#p2`{-9N1x*k38boJl4 zfiEV+?1x?Vy+aP^e@BTi1+d`di{&rS{XvM_gHV$H|3HA?{J-(>6pm1!7K48CK`Z4l z2-6QK9RTN&$PBpPF2;IG5YQn9RFRQLmAS?oKQX>Jw-{>+vO z*I^6<1t!bjPBR++vGzyjm7F#sxuxa75tvJ+U>z#IK@Jo2Oq{R~?UWPe^^eF>2>f{T zgl&ACfabvjz(VNq!9-$gm0|2L={#l>5EAFmD)8Ut*6@LbHqeQ~H!6R1z6|c2gaAkYb(&T&ps23q z{mOcNO!}|#Q_?Mu$Vw5GnSaoag={_F8Fu`_WR~;OTD2eFB2YId!4mse7ep?F1|qXHovzzI_IB7sHqY4P z2!e&QWcg=Ed4BuV$$)$hSd^T)&qgpiK?#jr2bej^#)S}<&};=L1)jaO?TXh?fJc_D zC3_40D*6N1I4=-Tur>?KOAXF|&USgH^9~gI6iKUo(U>EJ)02Xi&U`MucL0VUN0VJx zBJ5>h5E7SpyQn4&$CS#%Gljh>p;WV02Ab9Y2E)V{c=X|Ud%O1HbI0m(gOF8FJ7buN zixB`#wm~Zp2?K2kri<$`0Z^zqFcguUXw)9#)*x&*UP%5M{K_)BomZE4whRNPH^F?( zdk&fnCZsYAEUue)Rw1FG8A9;8w^u&d0Z4fKe0PI|^2XUa>PL-vK{~$xEMI;y@e{JA zz{1V*m@=MqAHNXGwE+Lb}M5-k-I_Z_cUc3c%qEoxg@fY`}dhW|cfXj0J zF|k>a%z=t|v?jeX=K?w~MTzn{7(GKuBBSdP9wJhrdD@)=n+~WQ9cvgqNQANI=Rx%4 zEUwEm-dP>E{F`U2Lonfh7!w48%eF0`>vi)BFUFJPFUy9%f~&ad2mM>bhZ0jcbDL?! zGy^rb1@FCvMEB+14Z<13L$NDfhR&sEu|N<$nD+lF9~V#RKFE`5XwZ6jIu?v;g~))y z*n(3Jb?x`S44!>ttd2E5m|zdC6;ATHPOMMW@G85)3l z&yMQV(1phP*dy)~9B$Hkg=4R)njW*p8=}B|X9wKz7OTIi0;+d{oi&$?)FGCO(tZJ) zx-t+BD`tAmDJpB9B(=>Im@!ms4`$LmMQjb$`1X!?b9FG43k7mqfV)WMU|6wND(4l(O`*1xwDO_*q6HIH`+am5tMnGuk(Npo*M{370QghUalV5FWim62<$|;%MWv z1yPeZGaVEe>6(}y2iVd*_j%pBjgaZ{;nJuFa0h$Cr){#Uaw`DFsV~wZhSAND(R3L7wN}^#xcs1ZEP(%t*fW*UG9qysOS%#5|WFFkzf4|@XCD^TF%Y|hz zEnyFZRUZ$bReCijNvKLwl}rvr_(!F6K-<~7noeC+FE3*wRy%IP4hZC!`aFZ73nn!4_FVQ*xdBTPc^$VR)F%OUjK(R%-`*+4LDdrDq_O(qo};VteW1K^xuGn=*TMHGTso5k9NM0*~pDP9=%49|6tR>|I`J<;if8NUnG#PL7qYgbyi*^^eplvD`_&iiD;6 zlDQ~mv4(%NBz(gi(l>(i%#w+?UqUSywPsD<-&6MW)Vfj|`$W(*nL8V`qq$S0aYV$9 z^yIL*E`c+{1=4GY`Yq0IXTvJF5VPjA@% ztGuI6*wU8NJTPARWIldg)XKG*c(LOHQo_ieGyy7bzhrI?3;RhQ-C+aI!aKZ9^yO^m zsWao^WU)Ae%s>}*uG7sIcWbknBMq|br7edB(2-JowB<_ZVeBWFO(6oegI-m@L>S$r ztZ>#*7@`*7WE59B&v++4(pv-j_N?q1G3RW3pPA z$nn&dijY!uTYoU_?^8u1pH_TO6g{y8$ge5)+F!e_*$L^X#Oc`{1SvM*r5fK^)Rz6w z*HEWgS2{1dC&fPvQlP(Btw7vLR;)Ge-@Ie?8`nOW+wvYWG5*?f+9WgN_s}C2d{e?s z;ScD~`>0zKaA`HGLhi1oyb|;zSr23K;#hy=U5o(G*OVnKQm%!9yGJhxUs7t#PC)KU z%JoV()|yyP`+Fe_xQTE%U&{v76Bvg_wzh*m034?rSNyvBl5vNWR{it$}ob4&YMkbpcQr9Z8d>~%}P#{aNp5d2yLi?AXrK`gAx)(+N1fvRN zTGu|DF;+=9T>;8U^#$hm7!-oq;*(zs6-PJ3^+p!2--ttWFH&u z^U`UnjubA1y5`W%#U%D>aD|~yijpJS>e=%>e>2$Q(5+QPG7WO-RcKqAbZ>Y#MXm>)UEN<|8jR_?RcRr`i}yjW($GgUBoj;B2KRe+IaZg}6hQj7Luu8&>#) z`3H4=ojKC2Z^-@jp<*Q~#2tuDjg&kX=sHuq4bRx~2J)Ji+0;1-)44iE!S&u_@c{UR z@?JL>HC$`Tow}4*)n1a|&dk{f4rF*651rG3ipBaN!bQ6B`r2e%wqdJEuBm(Ep6Q$P zTIhj2{dX_p`DwxSTQP9Gg_b_qK7r7M85KJXo5n(t(6}OJ+$~>pS|*C=`*+8)>8F~Z z-aPEMw?GT1XO~c;^p>iwQlXia;Sy;g8_nkl>R1=uq;?kA2V&Xk;Rpw8-b+JujsVg= zf|7EB+?J_c#Rg)6T)w^A&n@D(h1qevVAewB%cLV?g8SHRmc)o?CamVHtZ_qyEUFJc zZ_A})mv$q9+x4aCiLb!OnaSaGf0fnHX4M+RTi;8~)<(&2`6ipl3dBitzCq>KHb~S6 zP;yAatKxsoQ18Vr5=wN%ahdW`-^VW?ul@?vOjK+Qx)@(bY0DgxRi4A%A~drM!aj1j z#0g){y0lfTjJ)K6tBPH6+OKS3EW27c58#c&^!I%n=Zx{imSozqxmQ>qGy}_i=K7;J zHd1={+JgiGYcILNLJDWzl>UccI#hJBqx|VdVqE59Uhz z^X-2DIqh`;Sj`2xrbuyzKwlL z0HrmQ#hs(nwB`VFYd`HSj_00c8Lrtu}!MvC8il=G^wLOzBjq17{e9~ z=d}{+&(3{){d@;!QL`@St4;hnf`G^#d+8rro;+r5{qtKs`XtA&{ zHeBnM^NEr83HEb_&{lF`snD)&MA~&08;eF65+oJ}B@unux9}+*Q*D&A>A{eY8c8P( zA7QkukyA`~t&ID9qEfAhR{CEz>V2QR$7UZpzSWc2Po^{$bM}T@Tan8(yl*Ivfelvs zyVQ*LGyby{-r*$%L|CeWp^UU*StXN!Es0&-tLRQuE6Qw;xKtkLaxk|Hu@uhW&ozpV zxdfFJN;DR8cGBx?GNFv$la16DK0h1BF6K@&kYuKO^ki;>!PaS??!&xjyzpm* z$q|>9nB6fhk|b7)e*H?{7cqV*XF%Yhyr?JUA~2AfRT#y>&+kGQBGm}v>y+9F;#)$f zAC1^fXMcWnKflQ$#v=Mq^n56il7F+X5OKlxJIBCXddJUZH4H~ojY|Ml^aSHRvi2wp zUqaMbhEH}_WO*-|RR5N5j-S!kIVzxOKUN~tsqf^FL@4PAqBl+AfOaEeak$=0*J4!z zgCLk~(heN8yv(Kt=$_T1=+j|#R^PXzk$A7=Bh)SK6Ea!mZ5)t1pdpa#^T(_s6*=vH z9&)Y_1Zq>C53l1?diq(d-ZHXCk=*xPTR#+p7iNC`tzSrn^)|I_hwgS4PE;$5e0E1-JUZ16Q^CYtF#hA9f`*c@l#|neoNv+*;)Zhmk zX&c@=A@{bpgRgXPIKzoL`PNe_uxGD2xgK(nXs39Hfz(*B^i%JXBZ_#+)sOn7!tAk; zv?D@?JtqT#!Xpd=vIe0QwY44A!5TUw!5ZsryB`9Lgo*=KR*Txh4%B|XlM(}26PuhD z@xt+C+_Sy6x$u{`XoVf?*e&^oly2r-9bueeSo1OO5k*Yp$;09g&@1oUxdQF1f)GRa~l&EbYdbOl_3B*Rk16cD8J{{;!+b&jZ*^Lp+g~`DL zbUV11%XQ_OM-wbcgIwriLmId)(pB@_=*WAxCtW!HYQneo8b>l*h#@xAI65*MHOKWS zn}aIz&OKghMZ|8DR0A{r?AuSBlGY=Nb~p1zDLYPsEHa*!k9hF|r<4OTGQyi4{E{KB z)?d6!pkv~O52Kn6*1PVNn+CX?&}& z-B-q{dt8mFFO_QLXTf?m%^QMvR%0M@u0MCTV0`>$FSv;Mqsx<~jhE`9Smpv5DGg)$ zSO(+rI+ZI+M>LmYueHxyh`ni|M{d^LZ{-U0zyGCnGgD~!nPM)|!>TB=DYawO0C`{f zp;)J=&QU)*>srW^n%~@!=u4-QXrX%1xxNOYtPK9&Bd@Nh74(n$=K7_<^}Nu&ePf2D zK|0uuxo6h(Ii})uYAo&-V9SVFcHAL5G!S<^dmmtV6MY!&vCKY`P@*>&Wi+wcLLB?- z)ek5g@;nN!&S`3tbB2p!FWSTuCEnM$7Vl?B-tTElqq|1G&%vh3q)Q<`V1F?~x4AW%v4rZq&(+e$|cGBBP-EMBSu!=rY&`Xg&-?~bAJ*244TMzSg=`!;dE9dZ;$E0 zgZ6Ic2EILwl!QnLZS!RWo1n6bCvK?NH~Ta@&bT|i%)v`IzBozS=9;A#FbQ= zQ9pB;_}cn4`gbEP^QOnm1o6-Ng5PRCqL6d$PF}IMG#ZeqsY;A;pUBjbC0`5}Ephb> zTDGPBO=~l<`c~SlyjfGnVnV*pVPM%Wu#8~jeXIexhi+eo*E)TurJ}WBC7$2);d)Wt zN!Gm+jzc69pYtZMfwUwhn7E_wSXGT99NFHFoIF~ZOn4&Fp)rTwvshM2mdQ`jswBag z^XBR#!)>BjmQY*d*Vydz1wJz&>e6L?u>xTfqD3u{(JxF6oc>${*%6B5lAcWB={u_A zoag8EwB(wc@Infbt#F!Z3fgVwzisA!8rBO-S+#%hjpP%A{2KYg?GOE~3`Fq?DjvJIkt`=OMl0(3*+B1ckmVqKV9w=f5+{R-njw_ zYuI_}Pv$G;7U`T1%U*|9J10!f^{#%@_wxM2d@6=1>m^4)R54c1Y9%vAYR80mS(ExD z+H7RqW`8Nz=cg4d5MhnEcZRU$6a#mN4$JXf3T%}IeOuVZ#j(!^QO+%mPW-%rD6KgFX;)nCBLoNoWMkqJ5B5(-ZVvxDA@da`m=2w!QRCQt177*&{0eugZoj{lh-{=HJ|D>%W zfT?dn8gdKh8EDJ}sZV(*M6nElk2fGl{BKVlSON?-&+SV>LV`i~O)R$-jMZi%N-V&* z8tMU>D;XnrpMt9iA`Y~}0U%`vY2gfzohbN0k<9+YzBG?=Z}lE`QBWy@)Xg}mzqm-j zNv_Th%1)-4K|}~9fcRRa)0?;U0wi4Vl;Vrr#)R^nHYJu zjdZU{vyH`R9uo=9ky@~k*&;=QW6B7+nDsAeM>@b>?BO+|bI8npKuItTS@w`A-{y`j zM)(0F=jdTK4Z#l1lf@U{2#{f($(ZEU9=azF4jzSIS{+EL=~2`}y%|1>AmZou889ZE z`?Z#8!^YO>dY%YC21xgS{T^M=<*BX4FK`Ar7^dajR^7Q7(cHdH8e#?v1&R5>T##>E z2LE{zQu(Dioh}78Rek|aCK_lBLhV4;bih%TgxreBk0Vjhejz>Rs94*TX(LzJ`H0%k zp5b(MaO<;hx`dRe$%{BX7f#*e9XBaAP zvQRccLA-S3?RvlLs@X+|^SFR+4M8`IS|fmOB()17r3k2>!Jxko03B?Q>e(7^^C|Hf z6WAK}g1}YzKJSgTA02ISMKR8`-MAPmav%s8egCH9?EnBwI9waLdr~KOlf$0Qii*D7Uf4HQsnV4`6}vjQU&)*+J5+TiCT{ z79SK0dkAnl4|64g#X)I*NFza5jShOrzOs*gz%u>)M^G#AGNT4$m{}5}zqcTOjXZO^ zxBUk&A6j)@s#kg$EY>FTwJe1pAbW@2-J-nd)ao$*RM`v{bc#^Qk;Bihs#>=zdM8&g z0O&L>gx^AJII#2?^{V#J=F^CUAb@OkGy!kIIeiH%Cs=DC6ioJ49I9hh$TjgUkP8$g zf@g2&_P8)PQVp=vKQY%3vG}AZM~TOafJ8f`a@0WBePea{PYH5oTeJBl>5_e{dI1i_aL48P%;;9K{fCcrsyTr+TEk^!^ zv;+M&%SF-=S9`xr{D^*h_&Ers8$`+y691>dLfUf$oSX}j1SFfy!l&z)*Mj&VkLNS^ zMvY!&I_F~eok>JI%~u-$pg=@T1Td}gAH8y7;+qR*2t_J0Z-^mJnaM?ZcCzJ*nS`xW zU>jMMM=j4R9|Ux56a)vY{{|m;`0a{5p-0?@KwHvY*I@JLh}%eG?|&SHYJMj^TXswr zFK%_3@z{a8s^E=M8&I0v8}*A~vMF+o+!y_Pz_*};{$yDSrfv#6X*B+RLTrc%Su*Pp z!X8NENUqPb@Okgj#%KXYE@m1w@e*J1&@YlC5gW+1IUPQ_hf$<{U}-&a%NhmyjJX_K zD;%HP=|hRRA8@ClDE}0z2KqLTuXS>ie8^jG49Lia$*8Qd@Ic@_cfbGu literal 27872 zcmagFbyytDx2}!5J7I8w1oy$+JrE>#7%aHELvSa!69P$a4ess|U~mf#gS&jgyZ71q zci!{I$8`Iza)RO-$S7S^`rFfj6wU(=AaH1+TUciS!#aHy!^-b*?JyTZw%iV#Q$Mkva| zh2ni;eCKZNgg74@fuimDLAt9Rrbe4ZSHpIJ7=)`4K#b0(^^6SYK6yTVyci#QS_Peq zF8iEhb7KdN;|+o`O3}nqd2}dn*T*8G)z%ZdgD`%I!@E$ShYD~Rd3vIlaPhr-5M4$D zdA6JL9vvFLyi^vx!`nrHO>m4!mJ#S?iHSGas#m9d6-F_I!*`(F(=xe**)`6sJ-{%qKPVR&D1XypX!+TabO1B+uEu+C|IEtRKor zL_EqtM%v%=+%yz$o_tT7H-wf4C-E~_?uLs7WULU!qR9by5srId)ywcQ<5aGaP)bIY zHQ^=`s`ViSccG=yt6`FvMOyXxht8bC6%FX7ZzXf|(*!wGeX`q`P2f~byuBdwcfQjw zV;Mh-2AFxdhtg!`*ok1+_K9@N0D#&DvZu@jt@m(P&A%t?EUq^ zje6{xhJl>@evWFh{QGF#f!F@Y&idLj>Ds|mPsfXne zZ$~h67NWV~3GTBW<@**5jwItMF?F&{o2eLmW4efY6CysZ2}s(V3clh~9dI1)H` zl5*)0^o8Lhw5bu~-7>;BVsr?R;+u(dbcBgCUW0HTf%m~FG+zg?z9HQq3Vg|yV52@B z^!tXsFSSZztPW6=(ksRK9IPta#E6Y4TQo^(^-hqCRUCclj}_v_*b_OjN$x$uORUAn zc4=hCK+W!Ir+XkXfOt0qz31)vdq?MG8UxM>fKKeM9^S2wPK}N^zjc03c+%uX?yi@g z+TV)%@)$+r#TthihgJ4M`uNu;HXJruPM>Ze2!aw>CJK&kPf?vhwYmY4xCPl&S&k`= zxsKtJQn%uCcwlMGLT=6k5;bLpafS*Sd+IrSHTm0no9$ZrZYbWO?K@^J6jeniwo z*5$cIxhBs_-AilJ93~zQK5rw`q_oBusIJP1Dzqz{yv2U6#n?}vqevnnogt?u1(vb? zZcuWmj`ZWFmgY|(-l$AfL&fs^IgJTTN{uxpCMHtGCPo#_Bt}+^pX!38t{Oj-_PJGJ zEhH`q-cFxe{j&D4DzUz?hU~dEqpn2ficb_*exIM5n9iFNW8(f={`GE{=5OEdhhfZN z{;$T|j2LD}@tnyTos57rMf7x1ds#La1!Uwo{juc2>NAy8$>^|acAx6@jZ6q595 zzg9i|wFocREC?&em7^kP>1MQFr4(`Gg&v8kUo}vLWNiP?mOc5`tR|hdltDCwPQcW?6dI4ns!Kc^hX_Z^NHe! ztcgAl&9iDKF(^g;B^tR_2vuN9{+9fc!A*Cn%v1YLyI$LH&h`-J(0nd-u9AUFMK!(E z-uZAF!WYMv!PjC-vG@~=1ZJzXYw)wNG-as!Rnuap)_BusY`X?}>F=7BII8-ykxM(n zIqMW=7QMyKV9C&=nWLFko9L2sVX=$F7ELy8*w?lv>3HU@cjUbMxAs@EO~oPgyx4Nt zGS~8TlStktK^bB5`4oE(c#oh5tQ%GAIJu+&_jpw`KeZss`dv+SDfYy9`%0r+0rs56 zyqm?Pe*F`_*@vR|UlWpfGwr}Nn6*!9d}6#ERvn5R49`f z=Mmjc-EJlQF+0Qt1*}Xbn@m1Bf*j3HXKwa4~6EV|W@>3%RYDa8rgg4)Rm!y0bR&!G` z%v(&`$B*4scn69OgNiC9Ao+ZNHVrap{ zoa8p{p3`RP-XW~pE`&UkkvNkw(w$lK%ge$`Y}@Ux&$iCrfJ}Vu-BW`LlI5xbQ+>Q^7jJ;o}TF4vv=RF~d5N2(}z4b6Y( z{*V?>^Yml0q~H(JucbL)Ik-9Xw|4XOiQt5rkJ1Mvbj6nW@CpaUtrWtvwlxl$o?_?l zb33vhWEXty+M!xJc3g{f?lbelb;Cj*RzF;2)$>;_KRQKQjf|BW=^r-i+cJZ%&09bZ zZRD^*Z+@!$9Pz;O*n#*qFk9#xPAuoyxZ51FfuHQ+UDbA1BieR9XWM+fwub0yM$hg1 zxjt4~4%iBW1br9$)l7bMbe|OB$N^c8Hcs+y@?pGHA2B`IXr}F#b41~4yXHeAfStT$v~Ah5Z(p(J!@w`ON&zFG#huwX z$CW$p8O$`Yv183ewbjd3ubos#IeOz=W94#&cgd6e<%$n^Bzew_^iA&CMC;3r#B)ej zxDo{}1)-R>m%+32gTj@=PRAb)LATt!^W)}wp7Z@4d()edH5dPvMa|oo=dxCv%Eq0h zm5VLEXFq-;<$$S+_t(y+AEv~#UaIfv!H2W~o=;BoVig^`v5#YuOCKOSYsX%Y({^(z z-}RyJGVBn_XfbR5#fOZCrYohRwAC&FV+CX8_KX0VXNt$CDfPL@1Z67Xm^wJ1Z9Wqd zjKdAg1|m9M%=(RY?~aG~5B zZDje^*HN5vFL9R#J%u%tCO$#s9!DK_^spqyTQ4a$YnCL}2QUG7FQ_WmNZ?$FuYJ%B zthTwXf`y6-%sc2g3Jh$xH4Fmu3>Nx%1N}hvx=CMPkfBFB=tnvq?*HV%qvXT?pK}<; z*Mbt7QVI&tqo$d&xw*ZIm4oY)*U}xdsX1#cT~}QdWkE9sJ2q30!zXh#PrHArzzBN^ zLQn0?T}`Px?QHE`1U*G){wW~{J%3GRr=j|%h^vhVjjoD1m6U_CITarp2O9^CC>j+N zm9R6&Lh!w`?0;8>eiNawa&>hSWM}vA@L==cW^-`1WaktR5MbxvV&~#wg_dA-@v?U{ z^<=eoq5ZE${!cs7<}PN=){d^$4)#>9?V5gaaB~%*p?Nj*KY#yqo#vj_|7*$K<-fZH z?I8PW4m&3s2mAlD4XrBtnkuMn?P+eSCv9y9bq~~rC?_YE@IU4MpPc`-_#ZWO|F(!v%mKIKRoeYZvH0~>Ss|jVfOzS zGf^}puRbCe7;zW{X$dV)*y9z%be6s={`D_7g}%}eWKJGB_RA|dznM8Ba5vd-_%}~Z zdRJB=@zcbpbA4}Efl4di?LaptCnCFE7M9;zE_DWedApaq*cH<=;mRZ61xZu==b;?s zD4UTEE>fH(0p&jrI4Gb%9QkdZeT;dBG zdQ`CgwjC!+1s_^eJ*f6C+nrePoouyKrvWcF0b|C`=V?26FOQbV9dUx_&Uy{lK{@V= zKfrGDhFLE+FVDucOZF{Sgd>M?)CpX_n{-WCZ^gVUT+WooEuN^Wh@4xWEpliVk5G2@2RaB%33ss|{R9AFMxcT>9D+Qrea9C%-AcgYMLM0UoYlzjnA z07F@FQCJXz6y#!P;OkQ;Y7?dNkzrdjJz4kDQNCeN;3to{n}Z+ogloY-y4_k_9Nh+h zL#V(m2|oi|G8_U%UJ6PXBHWP83L^vC%5U3-Un2&)<_+~=$MHKlxL=Xt!>gYQv+ODf zUkgtF=-$HF1c{%22cnAGI`-vbZ* zw$Kh8cZyPsJYNV(7V>c~ARl0q^`00-p7!cO>(Twkk68F4mr}8x0$^D_B{@7d+Pd zvX~7v(3mS(^&5E7TDRY34TwFTxx6$y?TX$s?c7ZSJP(^(NYPX!;@mC_h|US0MXCV+ zHC!I8TjW_Q(CSsoA6I<)n2n~gA-=o%Ud4~)gy$~t)| z&#^M!R9cdl+&qi(Xx$+w0m^vgT$NecRX`n_W*F+xevwn$L$BC#; zvvQdE%joK7tY{vz!#r#EbB&od3S zz9;C6w`KF)K*Ak^2IqsciA&l; z4I|)&O32u~iQ+OJnQ#E%g~G8D)~V(0FM9xZUFqqhZbmgMuj9JDvV1vUU&(ke`{~G% z{bu&B?;|ugSf_rwo9o5CBwTzxrj&>e9A)RiU#(uyqd&uIBd^wE?;gj{(qeZf<&5fwcc5 zU4aH5gjo9uzO>p{pw8R~O! zh;3}AtZve%dy<->w=mMs`Fg1i!H%AeUg zdd#Y05cu=qcOR?$RFxV=k^AV26y?1-({4P>8W3@4=X~|1-Q%y(QxJvMY!Sr=d`m9Z`FSOjTT&g)DsgyT4 zY&*95+wSJSyhWk2Q|_nNtM`)Cc4l`;tH4!qw0eE-HSN)MXedLePXPAi3cf#z_Bo<% zZ|cIdS3gR|Cp&s;L;wmGc~1o9L=ig}WR`m-OsSHQaPS(@N29Fv0_}|$KRYjf7QOnR zM)+`;OKkMGMfsw8!v0C=j6>+5khraM;j`I5+I{=-UpAfl?j>XBny6%~kvp3G^0>=> z;{XlQ%E1mvsS7H6j!!dpNDkgJS&A*s>DYO*B+Faap~B?^i}xGo#2MO0!5I(u-$|*@s=%o@ijlWiTya$_(&OUa* z12hbENfe2Rvf#t0UuS!2Z6%T`)FqA5>_j0n+scO9G{j7#wfNcV2r@87l0fEpo%UaT zFepHy^Q1Ha+7aqPVbYe1+?&=ur@DM63{ctAhhXaunS?0ym~$y{2q28ea`|z*B1P2L zsn=C8>93akYJ1DQNhk1akOG0#gy(}&z2Q8ck673+dvKQwjV=p_x^*Na@$Gz)9aWh;ITRQ?@ z|J?5?xkmwuw*&3BQzmC3d=pA?JXV`duX!t+Vhi~ncKQ;MNRI{H^abJMQ@Bn_6i1aG zl?eZ38oOz~Y1Ua~~4gdpt z9HP|lm;7ezu87VMU&Vj+F*5Bh*Nq)t$$1)rwj?NXk8LfPgQVx16(ian|FRno^>zoX zYFT37TlNSD-Z!FCZn1={+#|CqlsR>hIJjVezmb{t8aclkBc71O3&nI)o@^Ffhs9>} zj^ZXzB{v*!;*pC>0FgfDgn?S57{=c_%+V$>CB8TlFdl;5Lsk6vS zmDRl+>W4yr<(dYMFnt0cN8Vo+OQu6hhTAXvt)_`&ts#Tr#E_(+s&G6f&p}*C->&Lq zo60uD(`_)-GqGV{2@veb6r3Sr`vWGK-1~ke&*&1zNA3F+d8>-*Cw|f4WK!?IV`m-> zI9SXvsf0@M>qH~-kbU$a@tSwI>z+b}l4*&!Pj!(-g-WZ?#kpd6)1leo=}Z4QsTkKg zU=_Vi`}tR;1QB#3v-XMM{09F77}JNb4F7c>6r+UvjgRxq+IP0cu_9J!w^cD77)pV- zZ~Lny)8EOC;NPQpcwOp9BorV=sbHc}*Zlyzf_`$J6| zl{$58G%4XQdJ6)vyM^Ek(Wc*Mu=FcvbVp0jMgS7nHme{_ySavqJMH}>>M(h?y$p&~ zteQSa$=RS{k#YJyxuKrm8FL-#of0ZbRxY%dKh|nKi(l_G#W2n~d?*|-{q<5_&VKMA zL~f&v^@MCEg^)ao&8qs0^1jUIZGYiP4f~j)8gM_8Mh}D;XQ)JSQV&r>s1N@9WgaV; z4%3qQ5^d*GHDkBJbLXl?mJcN9CE=J+_%XR*>4*9 z+#YRhgaK3%0)=D(WCt?usSgH(1W=yDA&H2QwB87$Aw{Ib9k+d1nz`pGIjY~B7Esq` zhvt%fu+v&Z5tCoSW`D^G~$5f9x{%mk#(fy9f&($mv6FhI@a@kSl*NuG5& z?*H_ib^K*@&^y@7b3|IOI<5+<*0&O|sCO&gD0&9|JYfY0khutZ(Lm4URutt?#mLHcob$7BdYwu0T7NVqg-QwO^Uz5>;|`x*m#@#+S=` znYJ{hD^5`6Adeq}7u8nl{5%Gxzea(mQrEc8x2(DqtJHKcIL~JJwe-Ga(>Su&qi#qt zLd%>_mGx7AeM*$*;y@O3jlDa=wDEpeS~CLp&I87xq!=@DfW%s}{%?$esxf?p87wSSTZS=XgR*J5usiTyW<0FkAxX7k=io>))L1*l zh*MO~xJ9hKb-)$xb-Rh%Z)AR*oWwb_nA4-HEOsd=q-prqk$|^`GJ}zYXdw5;n z^}8=rN@4H_sZZB8Z`yzM(3@D`Q#$Z{@P_mm7dOp^ZiswxpbSwgcW^a`H8wFHUtEt; zKaPMG)+7bMy*|q&NYL5`>IH>;T1t4o0eBZalz2q?cB8C??|Su(EiN2b63p3^S+yOw(PJ9<#~mKAzCT+1$TDf?&Fi(^-3>&-YVwM*B_nmRg$y0{`vk^08K! z)GzD)45tC5h$PHHfb>wJsY^~UN{=|HIuVybOt_oNH_0ywZfz1hh3Huo^~-zetPkXRhl0?;TnRu(l^pL-)sg5tMcu^lLAQJ z^;_aTTXz15S4J6_5!{&1L=L~ro6#64_8N}c2!J(M!7}RCH#Ea~Zak*cv$Yu++^Cdc z1RVdl=cU!Z#RZTsFlYo)b^eXHr|6PcX1Fm#fUe5X6Rya9xETTCia`Ix;xz;VUZ zeervoc#;w7aKQT`7(?6vi*SyDwZ*O=_kAj?4sw+sVY1?Q5(fRw)dbiDm!#lsCv?kc zw+W)^NO?G{KaUD?hmhjVP_ek(sy0_7_6oLmXQAfjks?3sL_uG;S& z1sd#(EXb`~XDkBGrO@5xZ-pqUkP9h$L=n)NN_haaT#4|2nQ*^pYeana4|j}=v3h%d zaPhB|_okRH%GK=kh9FCUy?7%fB2Hc=CPX45Kkl ztt4aa+lq-urovdH=EUqsbMBVJDonK%juNzTh;7lEbjJ)gyF;IgI~cikR#3rBtGFyQEvs|URnw< zA;i+q+b(j8^;Vy+w3zAE=wP5DVzA$km~_X82%VmP$|b9X-ZGp4A_&S;x#d?b$lT_GAafaByy3K;jmDUov5p?bRzZ5aDo z?Arc_=EfXn7r>t0l3&LRoK!;?nzHn@Oh3Nepgwon#LxQ;6m;B9P%fZ;j`!i32Cy#G zMxTrA21^cvWaDtj(k}`FdtdCDdZoHvj@<_>*3v^>NXZmnq+c?23kmMutrFW+8j5*R zRfN;VGPEwV6S)l>p)tn@r^0qW)R*eD`q#y=WXZ1!lc-{m+8A z*4#LP9KIUH@%H6VNA;GUv&z7v9~`3doXlL#MdX%FX94E;A)vHc^!}ro)&l3JUMUpCps@>(pw{lSew;*2kNQ2!` zRDaNfDa)+|WjPr7Br<{bsNk6{w5wtgbehDQc8a$xA07LQ1lO0PSBP-_U3X>x~ff-^bW<- z=4Tm{%o?=F4Efr}vfPXf|E^GGg$tm5Kg(@OA{DxO%z3)(m9LJG0?k2@w2aA8fJG71gzmf85gAL)7_KubL>n$)}0;J5l~Mx1nuz zn9qCPZr@=TaP*LJJ+SC#OfOw0TgoUjAc7z|$X7QiF8#Cc;l7|_Q1_jUWgpG(oJL>@ zd$B62v%D?yh&FgS-X#NIoWY3oD~Dv<4|G)rmLsSoM^U6=3>HePE@HYm6gPt!C<-Vbs;by=2;M=J9Yc+WNQmhmeT;a0x-?WT7fDxCij1FI^p|zROd$Bd3#EHvq((! zlSgRuzxNNA&_u0PXaaa0W2@cihJPQUfGJ{~MsA1X-``gyVG=r`1X~oWPkC0}i4n|l z-L46j@iGJ#Zs}r7;k<+!VExJC+9|ldSasnE&&9D_XVvckberS)nS|p}_PpN=I>V)! zimVvWARme>$l((1^~xs(c?@BdfB}->D&&nA#Va~rmVVcfHjj+gVo)K+YwkAZ!zB8h z$Ib)uw?&LJAGB2q72Ol5-*z~P2Y0gJfm{U1^)~88SPIQeBsE=Hc2@B>?&L{r4d>7+r7hi?nyuMFXmnO zX@kjY*=w)Gid$Fk<+24@Z3EXw?c+?i;@%~`b#aPxj6*rQvSvx6u6t&`5xn{;s$i@?ZD&)(W47>3Cgyubm<@I`DS<}f(cNZ{6A*BcrW8{is@0$eZ=ODHpFrkok= z1~`BbT(EXg_g7aWzhY<@gj3k%M$(UCJQ?G;%72^)W&gZ?X2~7dQcY;`%h~cbm6zF5Pk4@>v zLyzV3zhvv^)+^{`d=)&LXXBpmxZqKh%}d5p4QW`Xg12TRPfH5=dMp!Dyu#X*x-Ka@ zc>|QyMpL=1)qbSRus!VV+NJxlrGBwa@C?}sY0g=2dy@=Kcw0k{qn`ru+?NZzInoW& z(66(*LWHmR-q62q?0cje=QTL@P>)Ay%eCb>l5jMMIG*Q2|HW){-8ma^-wI{-ef~Xd z%K$x!e9VaI+q)h0&*A=W#rmbNId1#M(v!uIa%maw@G5`ab;a+DpDeU-V08(%NT)7M z2+U%VGDL03h{l^1DpO9?)~vgO_i2LwUd2lS7p|InZ#+k4w+QdmCruj1?Za700@1aT zBsOq#^$#fz74r^V^axtSk^-_zP>|}_1x%m1Rl65%2RSWAbWd{vV^qGT|w9T_)WC#sn?cqKB5b?tjROIKg%!{po(BhYq<9aZZUJTCvf6rN0Ze zY=-lXLtoeu>fz%|JHV+%a?33*ZpA;BEKIjIPsu*CRr(rGW+D$PV&SdO?Qb@z)zE_9 zr4sTP+4JC~GeY`hezuleKz9WmGLxdpSqljwz2Ut6$>%^5vB3m-?vapr_Yl|1hk{=h)nn5{T=NfhbpI{W$|x`=nGgjNoYO)hKG zQuPUxhG7CoT&Mel69wiaC9quJETttnalsP!5`)_l9mI$bjMqU&WoD;*VycZLC^L29 z!&1A{pIbF1g+W7oyh)7|MjcVM%5uEaM?=qvjij6!#EAz3ew#1tyPH3(h{D(}Z;^v% zvBy+S{ZSeTF_r_D5fu#x1n6AsbCvFZ;!eV8Ie8j>^Y^~g5PFh047dQGJ3aruxA`Q$ zj!Mi3f{&RVvBN*IEj_O7n3JZmWt89L9igia@sdUt(VOSvKhHkKTbeiQ!PmqNaxti# z@=4d8$0*makgnR}KbPR2d8PT*k35W17j`s{|8f87V4MuXT^9Hl%y33CKt?$*K|WlQ zqz3UcIapZq!}KHWL{|l_;o4$-rEKPBr+13cJpJO!Pn13>u>G}mOUR43_ni%KLNY$s zbzT6eX3y#9gHMv^rA8fOQ424-*5wG!E6F#1;^><$F;@Cnb7PptAg-XS-%&)am6g9!5tI3qC=&7oH`@`fgmg2j0@+}E~AlGM0YTJs= zn%9StFJkoox%wY9WPgdP*{Ax2A)Zl&HZQ`BSKk;0UH4E0i34O{_-EG=Sjm6#L)zNC ztgtIL;Dsq&emrzJZ2!oCDIZb36XyBCbXAfAHnLp8i&6kO>ZGe{ox|m=Ys?OS=ct zT){cnUGA3Kxa!C{7Sn=WP8*vtjd+5H9_Av&Bl~I5AR_1(Lpck1_wS3lkS0i*G)mXP z#mcZ{u3#qs42b(0NFngE2REsnY{||5X~H$up}gYm?VE9mlBl9j<8XWst_+1O6ltV9 z6&lLhNcf{tiG{E;?H%SWCl6tze6~o?E_xU|#Guj4Hn{m(7s1@faN_hgTQzPMRY@{S z?@jeolBFse7?4z;w`6ytKT_>eJ80ih{+Nqmi;dt8rpP%#@pJwj4%q7qoX+J3PRa^D z5+477*!2nVelA-&ZOSpfZu2WQELrh*@b?$PgOx)76`Qw_j{Kq1JGrj(E!PgP>3fN3 z)gi9;ax{8&Xylf$_-ZXAVrT|vu^^t}3rHYmc&wpK;I6%uhhe?w8xrI`p3zz3 zq_FJU3Ks_yxq)m*TLt*gV1KT6vIUxn7!ph%*Fi{)*GQJ;TY@9peXcyt0YdMtuC8~X zeVBI$SW5&oirhT5bK(w?hrX@IvhxI^l1eEuT8knqW~AMH1RK=zN%8^j0tAE`S)SKW zglhr4HD9lyV0;!D!YZZ+D<+y-8jNBJkCA?PXo09Fj>OCBKxEZ98U|pTEeiXPEMwpUBGPlLOM#=IdicC$4YPmQOZa3*UF5`30uphC}m*)2Q6FYWisNR47ku(p)r8h|AQweOPEaK7BOL( zXZ*i%_wB3PeKyZw`G2Dh2`DU>K^QoxNalPg{}%(1fMV zxXAtuJn)Iawb(u~m5pJXm%z;Y+Vj7sabSQOk%BhtrE8%S*%=oFZJ6BcughwX_^>z? z=Zg;#%7s}P@4wjsm2jx%vg{RHB>&A8JVM8ZL(E-|`#%AdS5CndWm`;9zdNa4Z8&F# zr$kTE=CCmmdJQ+RE`E3VPc|*G9Z-=g3o1vijr~w{`#BzmE%tcY?~wcYB3tFDsrJ;p z{T9mVH-o))bFz|ijzCIeL(uMC>_8=K9|xaT2_|dpafiQnV#R%_ywPA)*9V~tJ;ymp zGrO=%!+^uV!z`Bk~s0+mvCpvnGg#~(VLzw@{K@PhcPwH!PFpc2rS6vcVM2MB}E z0#T6P%}Qg_fXGSpao@p`&+0|)4pgQ!YkD705mlBIe`{&Qc7bu(&)@FkGFO{he{=PG z)6vn?iO4qDdKT-DU2<1e8-*ea-x;RS^E>T9}`C_VXr{BW;2C|QZDy=M>E_)5Qbfq6`#0y0fC`YtK8MdKf`5m82OFwVLl<=;=1k-#w6g%O|{_t{g&z07seDNwt z6_?rWLABW19SLl2b#O&(y>BNha@jKEkRJ`B-tTV2xE-j--#itoxMnGOk&*REg=o}! zO(*AS-7j00cLaktDos5Z~MbqfV$ z3@ZGN%qVR@60hJ$T3T}rLA=+fS?22nGyjD$2oSzTs8^L*begBXW|oLO?f)H&g!_k3 zn1J#om<$SHuS`uA)Rx#-Ewou#J-hCYdGcTGpzhMic;Kz{8nf#^5FI#ZXl%1kwJwcP@_zZ7el10FEP{)pp{bloLqT#;%3k|M&#SeKy ze0tx}Db6C~%79yFm_tQp$F&opv1dQoH37aPUXkI3*l4MO~?ku2(8{Uw!eVK&GZHF;UYv<60@Wu!5U!SLT#L|0X@}h8+rdn3vTb zJ#G?rFwXB;eiQLUWOpXAF|vP@_}Ie8;c z|J?5dz2JI*(@uDu+aqW!IA1^~Q==DDkDfHRpH5>lU#=XgAV3`iea=uV>VG_yGI=M@y)C3IED|^4R>C#3@K{g(-2DA3Mfgy z_qyzmS$Z2o`%$4~>WR81!{(g01GV`9D)VQT2t1#~uR-Z7c55~$c0>gGo|4>quVBpo z;dmZOiL5ambCtcKCDF-;{>B|AvGq@V?EclJze$bIVzxSU&Fu3^Cl`y@|&#@RH-F?JAiuMi7Z+*jTyF=X@*Tu>`Z`Vz$m>SdRc(Y`WN*r za%`v5P6d?m_?l0X(!$F*A<~%pgNJe)$5NxJdy1bzV%YG+Ic=JvZz8c|@|AP5=hj99 zy5869^q)eRrc$qd?%ZlFA~XsCWEPZO>|2D~nCABHldtOjCj+CiC>>>{c6HJqoAr8i zsq$d}1=4vKyRYft_<0A~C=hXOuAzvx%Q8aq1`5u!`}T*QH5rsXKSGHp#;#@<{K}Qf z@BFMbt@B1cgy6T_Tz1HC+jS)#=U(&m9Go8kpsNsMpxalXLuyD(5<`>w+)(QJB6_!u<> zK<8DnNiQy+@qL);Avdc$dYAp8_79>E@(&TmPuMwPxe!#fAL!C(IqU0jnz&)GMvyLKk0bB*n8 zf4M{s-{iWiX#D2R*;oDtirX>R71T*AL22$kd4X~)RtU%(?MLS9EvHQ&x${$srHE6a zCg2iyTw8RDl~3V3%CO5}j-cy>NzF@9D!i7xf2&;@J}TdcqZf66eQvT=>)DHNY2XeG zPg%~;8_xdrqhZ|qt1!UWJSMb6a&0wOp@1i!O5H$kokVxpU_$ot59y%LcDwr#`inhO zS@1$;Ox~!_+)#x(c)j`2;?`DWp}D#<njnK+D$CbBI!CP}R08HFoP?BpP!G#DVRfjX^2? zvOJGnGMH|@oF+qn^+h6^C#qK}u^}Ib=?;ryInipK6bwXV5ExPz)^P)U7N7;}y%Ilv zp<5IW_M3RXwbo{XaR(VqtUsYzq$# zI^!&X3{K5?;@ry2cwbkTXw9pdw6-#nRW`B9m5|7qu^ry8Bsqf^^KK**TYTTCgm}4h z?MhVg41>#aeFj9Ys~fEc;)47UJ5VU`m#;%?q>fXgT%K{a`+M3vH=t0jj}w=ro$Bf5 zV;~Ehj0D2~kzRmJL>a3Q!SXA;z&3wM)4mNH%)J_-4M*i$#FWe8)-x-9YQHaF24-xR zrhes05O}`aN4gKJ;G#&ULdDpsj4Wctkqv%y{0tf&0s_t`1Wpu%QQ!~=F>+U^q)VpD zt&}0oy8DFpN3YDwj}T%3YuwqZ{W&FrO}t{?#|_;kS@ygwz%FCj{)Xe^KCZ;KhD3j1 zn-3F8Fu0M(X7RNv*&JX?RgtW~fU;VcAd!Mf1BlO_QNR_q;VmZRVr0eNj{>v&ma1zs zo|NGGL|E1N<^ey5E3P4nnLJ;Dp`ozc0LmtUNVQGi#3_6ZLUQ%P;#NX!GRQ`wa0{{2 zJ#p~O5>?ZBv)eRw+fbA2j(X*d0+cZgE)P6@g-cNbkG?6Opb_1oc*20qKRJNKg}zGB znOca%hEx=kEci7y+8i(0xq*4Rfd0<$*-7MaVo#*bgVzhzCWn%8QzEuks8GuD>xxqi z8!A)vkPhef22A^S>s5%9`?nF%ealkSdEg>n`L7t8N5*^g^TzMjz|&R}BJ5Bjvo*v* z&r%!AdI|p{Ei^6g>0-kT7~&1k6q_i_`r=B}o=Ctzd1Y@c689Pdly+zJN9{_h%U2a1 zu_HF;tE5youD#Z=$7>|9EFv3nFq<7kH9L;j$qwaOQK=X;`j$46_8o+wBR}1gGE$gR+^lVLu%Es%PQ?N|iRy za8NQ=NIhj7*G0p|Nw|05LNdEMN08Jgvb&CqBKS`^?(-u&g6~QchW0kX^vDs_C zs;%Wfcd7xAkuW@IEOKqvECJvz8u4px<`@kpQFXc~SnQ5bkx-eQHz(EUBP2|G&4Xx) z?1jK{{QjWc`Yw_|EvHmlv5}81i@+&IJX?=oSj|wOH(JQd!XNt|b?cPE zWL-huflF1v6D++`pIkMvr*?lM{E54L-O{w4@-8qzspboXp?Se5#x zbj4q#Z{7<-wi!$3Fk^^z!$Q_QDF^00%8|OQ6R8$LIAS`%XV^ko0MpxaGa$teX~EuNC~Mabq&qe6qjFZqtSK5NU`O*buyRO)7{iQTz5X1$X{)$c$vjVf03n@2gJADY=U>QuCZ zry&CgKrGxTfPqn|tU)}zNO~1hknG8<` zsU%tV!~Mw$GqPZzE(w*2E?5iAey#a(v5TAq20o=vL1KX(2>VqhHMD-$sM>xFatRi8 zVBR5gVFuFf{i7Od_n4N53ioUc$w>R7nq*JiPk7ZWzH$sWbV)S0HaZ#YJj6p5W*)(C zl$kyN!-43;E{7!uPm!VRb_|ykTCgX7@gOAru)f30tYP?Zx}9nhPWz6 z@z6#eOnQbXY8d-_IxUTXmdHLQlw&M&yYTy*X?J6yMIa-b9s*WzkVec%XTZPvJ_6a zpkvE8pc^t zS49&udC!+;P8&BBO*XT-0vY_jyD*3EU6dCyGTWOEARRByV+^8ak==4J)l>z3ED}u# zI$xTjH~sTgV;}162`{Hda6z-T?pL_;N0KOILujJ?OZBSGD?QnlU&#j(YD*!0DEQ-C zU^lv;BWHW_PJszmdvf5M-CW+j4OT{Hcjl!AbZS<5oz1OH0+zO%X8RWv*ap@0@m<2% z(b~wbJ}qyX?1>5mO2&U>SGw&X-6ff63WoEIF6{W&hz>eKSqfWpVIppPj%+%IJ-EN! zc%Rm+MXj*yqdt*pNfS}Qy<$k$h7!YirC z>r+3tD*gO14B?{dzmJn6z=a%PMWo|I-S*a3 z+*JD$TW3u-$Uv6c(v%yYvti3I&E|g-we^cTWyeb7HKf=KL?+wnvu2CRy~B~~gxA7& z_a!wgoQvDNjpB7J=Pcj{6I@H zo*^_!8!cW-8Mxx<*hTRmFCgoEg!22Q>Z^DYG`D&(#ae>&i}*W@aa=|Y+lU#QjA?bGxdvVF&y;!PQgP%9g94cPs1y^c%?&_i=J3D) z!)_wCO~0}=nMI;}j?I*8+yAez^NeS!ec*m=N@G-!Ha5j=QzUk4ms0ems2Qm}Lx??V zua>HcDysChcWdvx_ufK{+ACC_lmDCN^Xc>EdE?FHIOinSxsr3f-``Kz*1exdL3Pz6 zy3(D|L80h=1lMwh<>xS{PFN`Cc5M!IENrvYU)ul*{xEW%LO|rLmV8G?%h!IWASQCw z=;z0Z{szS-Xq(4z{9;n4qIW$-zww%@Hh3M#e?RkFbb@T#ZVG@`+RFtA4aRXm)<4KU>r{gfj|i z3xYl&^#Th&_3j_}Se02@uV@@dPtQ`Rlw&?)Q4ixvIQTMu5GKA!s4x9Ro1SW-q-&*O zK}yZ5-@i)zfT!-^_sYBX(u%Wg4vNaLuj42A;u*xeC01&nv$pbOJ2Z&S6V|pjF3feS zFQ7QiE7db4hU}s;uR&s~N@o4le#*pH1+nf)GX}#%MzBG-d^i_RYzmwv>h`r_(=c=V zJ{9d^#oA}{V~_bB{`h~WCgOZ!R+;zukF?F(4wplzg=%HCKaySjByssZ?1B+^gi@F_ z!pdX}YMffsH$y!{FBu+<#&~~xMb0(-R{c3uT&Q#M%W@g^77hv?i`Ri1iB&|JdE~Nd zE#6M668^AO^_!5T?8WCz6^mm{Vac3%j{mgoE46n{M{RcBe$An)Ket?}rnL9H>x}J! z6Xz7oY&Vz3=Ly|#EV#_(lPsc#T#T`()$s;~^&g|VvvX>lqAebwTHD^3T={I~n7SnG z1D~qw%(hauGFO9(C0O76EW5%o-NfsI9LX|e{mNBDJ{Z@CD?T@}=F!}BtR_!DYUaaZ z!W6^a(Nmv&v*|N|rE$>a3MBTp4eE+0Uh4g(h;YtdDyuHQR$q2Axv45!#0qo`)$_I% zSG3N(_;ipvG(Ca5_j1OHlLqtWqhFV;Nwp(iOXzNhc-?GO6@RF3X;wR|uljn2`TkQg zwyv^=wu6JZR`LE643z^>Wv`Xs5@x;k+vc{zYXahOGaAH%oO;`tQXLbTdE!4#9H*Zm ztv|1#38@)K(s;{Zu-um52Z;Yri*U$J~_zT|_FFgS)!{XUb}*^jt=qV%4*t?bue zbhapZoeOPCi@Fe<eERg3Pdtn|b3U1QAN{sWt>$@`48bHx~m z#8c$>V>b2b>)JO9R)Ye0NwIFzucBusm5zAP>g);$TcxXiEy5OOQ%zLNjDdrav4G4!t$hc01^%h!|BZ?d7L zGJ|lO4qNZ=2-)%5!(rIuBM;CIx^<>>(XFm?ko%8aWQsN+_b9F+gTdO!3>BK!5i!|EurNCA&4`NlOba;>l3=5;s8 z`<)g6lZ1h5UPn4jCHKnEj;uCf@z$p^N%B`*Zli{3V!=Z#lN7^SE9~i=551Cd1VdmN zncn%|op|CxGY~G!z8rz^{9}CTx7y`3pG?aYI29dI%&w+U@w~u%Y-V-3V#fV&!0b6F zRacr;?_7y_EPv2SSlGWmWo-K34$Y)J+fG#`g4bA0K)Eg38)QGQ>={&QCi-&6O_vTO z1YIv@DUVbSvZ#6TytyoTS6xoNg`w+P(K(@xVF^lZ`9 z0opA0wUoJlK7&^5FkYl%GXIh)U-vb`iv=kWP3zXz;kWdWjMSZGe z&HF0SnZaFvX{q71OMOxGb@6spP9NcZ#S>hL@5VnwOq*(uM@gg5vC#GWBFecl-@=UV z3MGPZI2qiPkc{*#2hqU6Vq|3iwipCD8Th#|%*`AZ^jYyM```kz|7N!iq@Z}Yd7*KE zu8s}3J(U`Nb7jgE`b$FjxKd_(QQ#c3fa`m7(G#yaic?nn3Ml1{uvGw&Iy`7Uh2bV1 zq}XUejz9Rc!?)Gx_P?nkew>B@*W;HG5LpvHRAj08*>Mi#9vp*$V7XB>CHuvaS0@&8 z0bUaW4t;8DUNovA16uv$iUqE7k|m7chVRT7Tu&@_yY)?NI0FU?yHsCQL697!0W|7$ zRRNP6YURIa)JIw&7AtVg3BU~)TlMxCQni8Im1I>PJ*b8SGZ zN6KIm3IJ4*lT6gR|34~=VedrWy2YA-?*iE5`Tcf8Rk3$Y^YN|BqDdWfAw&?c3jkB& z5wTMwqW5tz9iIFWIm&;LrB!Q6-K7Xr0KXsN(jpd&iQ`ztd?^}!VVmjQJ}HLP()wC{eD;!Wk! z*-G%Z;T+jH;_PF2h1Me=`5vhV;Plt8Chkb)0eH)6BT&n+)N<4HZm?%~(4Sl&?C)Dx zAnMfF@}+UsuK?KSa`KT)C6o}|ZtF%>5l%uAsWngd%z6Sio9XDFz;*RcZw;t{eH29O zojJk~Hon=J{w0R^6DZ-Y2#t-g(#Es@4YtIiN(ki2Si|osapqzQ=_GY?A4~N<2B6%1 z9pWbGw7~$b$`qgMUIlpfaD-y7KC4*Yk53muWL>AzPmA|_UZp_B&Xj>NguSSMvxRE_ zNo&#rJl{ft{EMt0Mfa!IbW1E2k}Vsr=AM;C!oqX$FRtE7f9Ky-4smh7E+6>*c0c(W ztn1-l+;6V0;eQWws#-))(gd7UWJ=%}lC2Dg+J>& z3Ns#m{;UI`ZQK4GAbmV1ilusRo_vV>CeKskLOlJQJ#!iO+YZM-Vl!VW8~YoF?0-Ik zN1ce@YqPKYmtxBsjHc{u&17B$x#selenTI|n9oP`DZb!02AG1n7&>@5>!>S1T=-Qb#f9_9hy51PMDRxgOc!Z~WYJE;$2qC@rMu7nb zAU6-Ey27WCf1Km#DjJAr{VR{W|87#4@ak9H{^tx|fWR(bfZ)B*-F$S@%`O&iPhi7% zWG#_Bnr7vffEcjoheyJnfM?akOHsnm@&VpC$|vRQ1^cc9t@cH9oR@fPQuT2Nl^Pk8 z!J&N=Z;QdVC!nN-HN&LRjcv&1v63}*EEmmWLQXGGmiYOWI+Y3p__2hH-xW0q{eaxI zy|&-#UB;W+d3O0@d2)Mp1Nca47RQ5Mi{L@mH&`J4YJLo)=hv9vjP#ga9(nxd32x%T zj^4!fizN!_)BFIv!v1I2U=O!h0kX=rO3ncFV|2WLoj??o-Wk~%(Jyw$=`CU6Mg z#Ix*i>#6^N_!5A>8XS1{nJns+tKB@i#OLar%2lg#Qc^9YfzbB?1PHYc;LQy^!xuL| z?!OT|9Ln-y)}ZsZQ0*G?{`5Sc>n~yrVmmsr0)7F%Cp82hrH^7>+dbKbUf4E_*6Iy5a039YwEy@Hf$?nqbuBS z1)s(zV)X~kf63_QqQTlkF{;Xa-#xYN)y%~3>+~Cd7Bj};Ye-y>F_sciT%0R)0fkeX z0ObCXcLCInU2ox@6wZp{uhN_ zXgFW2t8~b421bi3H{0;a0k$l`-|@$5${GJROFKX(hWY-lw#Q`5T*Fp$o=95<^gSB>K@Trqvs!hsZBJucrPCT zq5GQ3eV{zzqH~l%Ah|2mdZ;)G?RH>+!vj||aA@<-u@Vaa`?rg|+p-KmMFbMiMp7l0 z0MMCI3xq9&4gT{3ga#lHF5A2{bvkvt?*W7^XjII4(p)#tgePCM|sXk&F= z1XhyNhs@zzP3zM!zwmQiQr`#Gv-`bpfP@#2nDmeS8( z%^mT?EXpVMem@NaPu(l=?r_@eNd)oVsl1^|;n6=4HO*5v+a#6x{+Pnsc%c9EUlWlc zL_4lE#5ttnHIFn{L5sbs3>W`f`&-j2qR@02EuC^>b9e5=ae&_q-a2p;_x~s8YXZs6 zH7hDSvstn#O16~9`)<=#jkcmaC3OU3jlYrR{!z(6uDlZVQnB--TTR{_LB30Q74+ts zf3MQqk7E4&o~U{Gi(Z8tQ1IfdV0~xG-9fKLq#GU7786F}^YlGOWkxiZyx+`s?_Vk+ z8|BJ=_Pj8d71OYRl)FJ-)9*8fM!@W$hg-??`fDT8i5baQyqBL=8_qFZ05*$`o6j4Z zo0)-YGkZNiIp;9jFY=7-KIvnjv=LEjT*vx%3F-ks$2=_2ZG%l7SP=Q26l2O-v3Tx} z1g^+(^3hFTMvM}33M;ver7u?g0LmmxvAjX#UE|-0iLe|oY(zXX+aLb`eIoPN4!` zp^Mw(A#c3dd;71nb$fk~0=&u&rN@IZQZ+!ML$8s|p57Ni(rDU}WnFC;4p>B&Anx7l zDlM5OUw8y>-x=blM-C!CRY2E7iYS}`Ax5WTX1-@fUjj2Z^-DG)#1A)`iytPs|yYnLRqhu zNnMtmcI*dUc@5w`Efe#+G;%q%Znx)1wRFUAbDvuqVq+ecvv}U>{oa*C=MNS}2LMLK z^FiY!py{A6eTgQshqn=TlBXbd?0vYHEA0Q%9keh)xR*G@9+D>Qf#vY`I<<`B7NogE zw>?-Kmc{!XJ`VyHfF71#|4IJ5Pqu~-_BG=3jnaITeL!MXArlb+&{PKrZ_KCq9n<~U zUEm2ge=QL!iO^eE=j)o#!G|upQ5872bfMgE*^zZ(tj3rBIY)!{7oPD5b1LQ-(#X-S zt5eK6$~+WpI+LlB+W0Oc*ZaOg5V&dTix<_1PSlKIK#F0D4@aX~t`3b3l*r~_?kDj? zg1lW0b$it&E9bMq6jUc2iw9t0#@+D&z`JVA4598(K_xCMH`V`q=JF5D>^J+Lr;3iT zE$P-Y`Cwyu!k`8F3r}&B62o2o=h0-0?ZvSoQ1I2a5-9 z?yLo8sZ^O?v@+h;F4`$yg#Y#fR671W_pc$uWGE2%y*#{k<2`ST-h|*+Ho+$~JH>Sj zHgB6h-c`FN8c!TN*1lD(ap`)(q)AExD{;7~&Kk(9=*pka_F7=D`QYx_cxuffW8~Is zeF$}gJdGx0sFt85OVsCoCzvDWVUG4T-e)RscuWP|d=bQk4B-vDEo>Xfn5~_han4+l zcDME@rQ{W%{NsYg1ORs5>*94hlHty#4_7-7PcC8wd3w0RM!?z zF5*(w(7V@o^{XQ=k{Mf8dlIZn;Lp@UC!G?&_!&i8#}?Jw$oXL=n8O^~uM=_z1Bw~k zGyl+2Yu1UZo6l%$>XZ#!vq}dIj934dp$(XBW0Bl8pP5nh_>Z_SRoU#BzF$wRUx1l8 z`R=>~23o9GMn*xyYu7H~+OIP4VIZb0*iCdEk8Sk^4UFd*p(0!>u|1&hM0fzG1m`3T zEOF*c!kEqO$|%>o6Hb#?WMs39U%IQ2W{ydow0Rjv?x;jI55y$Aps}FQ;WUXo`0XWU zl4KkGv6moK%8--Q%Oowiknn{&PiaXpW#pjwguTaVGM&Hp6RaE#x?R-%+=(b=njCz5lt#2hp5lZ|;~$I&iV;0G zP4@sVgj+BBZPwcfw-n+x?i4=Ki|?xXrNc15YT&UZZA&CJu@BnV>}R6{hVU) zbMH=E5~~lx`xCb!&GB&d=vzxR(sM!($GmNk)8X->kX^RlZi`CTRb2hPN2#d<0loaj z{7Nud!`Fl5rfto&N_>Kcs3n!!yP4>ACWS0_5OmSs6B>2US))GMbB;IG5G>XvN7VbX zd$FdSdf!hYAYm*IYu3}4L&Z(C_rpOISHK~9957b8)MNY1BY%vGshU;hKwxyAQ9Vu$ zFOmKdpr{|8hrjKaZhmiy^-@dRV#0+w6{90SaN+*cLAc9oS2FO8BZ}jimqtHPnbbKND^j!yOvjsT5J1uo9_3~hn=^=!hS+p*B){|(p!&2NMz93f=?3M0JYnY% z`TRuLvLp+r538@74ShQu2F2{k-NTGYMP}{c;3bk*xaj>6lwU$nKkft(#!EYnV#|&b zHH(yI^mpNB0mi*swS$@!N!I~jI(|D)IWyk2YNxBA@!XYoVRO0h=Cnju=4mo7S+PS6 z_pJT-z3Gm_a39&=k~N*;u9(O7h;y22{b1WV>4B@C`V*OjhS78p*+b*=HeUJR^hFxc}|53cooiTgETb&Ff|-I}xMuGPo9A;_Jzq?Bp{ z%xDE>Lx5XG_%p+!o>xz4MC_|KVJ1K;09kK5X1U*38PzZC8hh|jGSs@Wp18Z%_r017 zmK6bO+vWh>-RvFA4H&p1B$fx^s@0P5cviQ+g#_;t`hpl=EklAo{Ltc_Q8{g}7Trsh zCbi@#f9Q=A`YA|2$P>>TX;YP5$a&XR-MkvX)))bbsjk-iZtZ!)#3O>X-^->taAR^0)C>xb#=CiGk~HnJCQb(5+*1qiA_8--XRQT0fA zrYFa@^ICqzI2o&|CGv)H!Z$k@AB60DE%LyyK49Dej<^m1Ta)QsU;p zOtPXi_#~Fs<_QChOB`4f4%p*2<)h(4he39)s2Y#t=>4|{r?xX_yD|?Td5bZ@@7%D3 zHR4^{XGwdWs=dy{WRJvjF8&dd|z^Dd5T5ihefhJ{^sNAKy&48LCx@{tOG zDFm%d+ow{!)aP>0x!JNi?l=$mktJ;kkLC)Ft=bi<1Eand_+8z<`o~U@q3GLkFi|Y@ zz2irCQl}Fpsi*kVu=@k22qBE%p9jyVZKs&pK53`Sok8mwEAm3Vl)lSy>E}DqGi*SY zlx=+CtJu+?=Gb9r2j>O$qs}pxAxEJK!d}Ep>+X6=A=f|W@l}X)nI<3ZRtAMoma^bl;KzcE>+!! zDhH30>GR6y78?>~Ck+>isI%iN!bFIK;a2DhGZ2@8$Tq||UrsrkZ5Gor=6C1J2K}Dq$ zVfQ0rJ-RI(Uw`Q1jELlDjz~G8O0|$yXZxlTU%GH>O;K04H?AQf_D!S3BifvM z=NvLB=IEY$(U*^tRAP=Sm)9Db@3{227O)b$p5OPG;(=HKrIACYRcD(Yh(3*GYUPU} z!=|F4h92OX&aO!sx6y+}X3~}o0s6QLQ6-__Q zrGXhh9Ked#Jr-^UP;7aSm=jL2YZ{=8j&snE-uVCoSBkE|Ahx%_){ltVO z0|@7f!t?>V@=nf_t(l43A~(Cdxi`^1|kvH!vE?B($Kd#lLA z!<#e4`V!L<_Ctn-zxe}|B#3(C_Ogna_a+x|6-JKnst??n}6Cq7c}Do+Cz=|@3P*_;RB3i)F- z)io%@92`jEz0SKGJ|Dj#y&*5xiSBH2r21j$l2c;K*~z%HbB;2qw~R=h*Jz&P3Rcnn zd7pqKLvU+@^#Y+BaH|WFNm*Z*LY%emk6H_^Q+H>Oym|UT`_1$UgNU-2O?OjB#4Xjt z)eE-R)qjUNHbu?#%99q2Bo(TDQ*ds|tj%w>d!2rZN3BJ^ zGJROyxj4{dQ0)mDOhggi!~E+YLw_;hiD+JafXv??IGT}m-DnE;6hN+zKLQcmGpin6;TP zprC6BCzGCNy9;?vaPP7!Gq!;(+A@!bcn$Ko)&(>ypsd{)3F#Rhb@KL*h+?@%PZ5dp zP&-M;qcPjfHu8LFh;E4tVp>|$ua}h&VHz_ug_})E)GLD$GpLzPdxF?Xu$tOmltC=( z!=62N>Q-u(S(Dg%*gSUwp2|Dlj+=){z3Pm`+)MKXw$wYzHDs;D(M9A=bB-o-SDOpz_K6N$wN@6 zo>w!hXYDdqRs*e#!|>kf&?Gd@c}g@MeGOg1Y-?6u_3zKI1`pn9DGm9S;nt3c*N9{Q z_fMeCY7_(tWC+3cqZiA-+HnHp8-jiFp7VB|Y8Cz_tm5B2gDkx48}ymk3eU@myAD-? zmLD4jGr;E=V>>QjDIVjpJ{kw@!>S_poH}JTb%WqkgY3vc&Nc=_=OS2xfMk;+wkE7^ zdVagH3zxsILqW5BBs04yt@y$77aV6ldG`L_02c680q54TH>&4uBeyuA9KrjXD5tm0 zKPe%zh`FUb5xb`Si`S-Re>%tRc{uCLSXRiU+mW%+Nt}N7svP>M={$dLDc5DIoLtP1 zWmkz|21#oAG^Zr4elad6gNPa_t?5F!$WXJ)CYHERh3YRTPcQHE2X~yliFP!?9Gbeo z=WN%0&K=qg!D$zlFQ{SHr_LBf%X~-QSBIttdNpgfa3MRs8ury#j{U%cK`=@e!?yFEBgEw zo8vs+Mz<6l;f@4nO_IpG7JS2_o2oN(J>AaPK_Yej@k_Zc`sPI})jL@m$r-1mw_$Hi zLlpNxtv+CjcsP=ijk29x>nSmt2nxfidta3;UgS<{Z{0CRBF^4yeKN2C*rrJp)>@@v zv01ez>}5?++7f^cKgR(i@C^zoJJk}k$7xAoNnnymdW`&q|?-Ix{X^?Oh9gWlIdXA*@XC7+2kLr zlW31iRI@C3V$MZR8+zL&!;M(aJPuw-tNCTuvHjmZ(Gp3*%fuslPRyRplV%0Gqq|-K zPsK7dJ`fu*JSFZO`YL@t2{bq>rnFs9)!{4QbKL*9chH_K-N~5&AH){mPdom9g4jWd zETYDQku%DFBV{8RfO=LGS`Tvmjg$!q0oF~>?#S`iz_b9KXd4-bZ6^H{1Q#H{C&X`A hN&Y?H$UuCf;5+~}pe2-S$N$3y4Ha$W;wKg%{|D_YqTv7l From 265c6893de25450db86303c82c3bfbf377581b80 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 14:56:20 +0200 Subject: [PATCH 13/30] Add citing info --- docs/source/citing.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/source/citing.md diff --git a/docs/source/citing.md b/docs/source/citing.md new file mode 100644 index 0000000..62834c9 --- /dev/null +++ b/docs/source/citing.md @@ -0,0 +1,7 @@ +# Citing + +If you use `clinlp` in your research, please cite our work. This helps making clinlp findable and accessible to others. You can find the appropriate citation (APA, BibTex, etc.) by clicking the Zenodo button below. This should always point you to the current latest release: + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10528055.svg)](https://doi.org/10.5281/zenodo.10528055) + +You can also find citations for other specific versions on the page above. From 3c08d74ff4ecef0125ed81e77fdfcd505c4519a6 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 14:58:02 +0200 Subject: [PATCH 14/30] Update roadmap docs page --- docs/source/roadmap.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/source/roadmap.md diff --git a/docs/source/roadmap.md b/docs/source/roadmap.md new file mode 100644 index 0000000..829219a --- /dev/null +++ b/docs/source/roadmap.md @@ -0,0 +1,3 @@ +# Roadmap + +We keep track of work being done on `clinlp` (now and in the future) using GitHub Projects, on this page: [clinlp development roadmap](https://github.com/orgs/umcu/projects/3/views/1). All issues created automatically appear on the roadmap, where they will be triaged and assigned. From b9bddb086feaa00f6b4a03a0b1587c2f7d3dd27a Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 15:24:38 +0200 Subject: [PATCH 15/30] Update metrics doc page --- docs/source/metrics.md | 317 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 docs/source/metrics.md diff --git a/docs/source/metrics.md b/docs/source/metrics.md new file mode 100644 index 0000000..a3fb747 --- /dev/null +++ b/docs/source/metrics.md @@ -0,0 +1,317 @@ +# Metrics and statistics + +`clinlp` contains calculators for some specific metrics and statistics for evaluating NLP tools. You can find some basic information on using them below. + +## Information extraction + +Information extraction related metrics and statistics for annotated datasets can be computed by using the `InfoExtractionDataset` and `InfoExtractionMetrics` classes. They require the following optional dependencies: + +```bash +pip install clinlp[metrics] +``` + +### Creating a `InfoExtractionDataset` + +An `InfoExtractionDataset` contains a collection of annotated documents, regardless of whether the annotations were collected manually, or from an NLP tool. + +#### From `MedCATTrainer` + +The `MedCATTrainer` interface allows exporting annotated data in a `JSON` format. It can be converted to a `InfoExtractionDataset` as follows: + +```python +from clinlp.metrics import InfoExtractionDataset +import json +from pathlib import Path + +with Path('medcattrainer_export.json').open('rb') as f: + mtrainer_data = json.load(f) + +mct_dataset = InfoExtractionDataset.from_medcattrainer(mctrainer_data) +``` + +#### From `clinlp` + +```python +from clinlp.metrics import InfoExtractionDataset +import clinlp +import spacy + +# assumes a model (nlp) and iterable of texts (texts) exists +nlp_docs = nlp.pipe(texts) + +clinlp_dataset = InfoExtractionDataset.from_clinlp_docs(nlp_docs) + +``` + +#### From other + +If your data is in a different format, you can manually convert it by creating `Annotation` and `Document` objects, and add those to a `InfoExtractionDataset`. Below are some pointers on how to create the appropriate objects: + +```python +from clinlp.metrics import Annotation, Document, InfoExtractionDataset + +annotation = Annotation( + text='prematuriteit', + start=0, + end=12, + label='C0151526_prematuriteit', + qualifiers={ + "Presence": "Present", + "Temporality": "Current", + "Experiencer": "Patient" + } +) + +document = Document( + identifier='doc_0001', + text='De patiënt heeft prematuriteit.', + annotations=[annotation1, annotation2, ...] +) + +dataset = InfoExtractionDataset( + documents=[document1, document2, ...] +) + +``` + +If you are writing code to convert data from a specific existing format, please consider contributing to `clinlp` by adding a `InfoExtractionDataset` method like `from_medcattrainer` and `from_clinlp_docs` that does the conversion. + +#### Displaying descriptive statistics + +Get descriptive statistics for an `InfoExtractionDataset` as follows: + +```python +dataset.stats() + +> { + "num_docs": 50, + "num_annotations": 513, + "span_counts": { + "prematuriteit": 43, + "infectie": 31, + "fototherapie": 25, + "dysmaturiteit": 24, + "IRDS": 20, + "prematuur": 15, + "sepsis": 15, + "hyperbilirubinemie": 14, + "Prematuriteit": 14, + "ROP": 13, + "necrotiserende enterocolitis": 12, + "Prematuur": 11, + "infektie": 11, + "ductus": 11, + "bloeding": 8, + "dysmatuur": 7, + "IUGR": 7, + "Hyperbilirubinemie": 7, + "transfusie": 6, + "hyperbilirubinaemie": 6, + "Dopamine": 6, + "wisseltransfusie": 5, + "premature partus": 5, + "retinopathy of prematurity": 5, + "bloedtransfusie": 5, + }, + "label_counts": { + "C0151526_prematuriteit": 94, + "C0020433_hyperbilirubinemie": 68, + "C0243026_sepsis": 63, + "C0015934_intrauterine_groeivertraging": 57, + "C0002871_anemie": 37, + "C0035220_infant_respiratory_distress_syndrome": 25, + "C0035344_retinopathie_van_de_prematuriteit": 21, + "C0520459_necrotiserende_enterocolitis": 18, + "C0013274_patent_ductus_arteriosus": 18, + "C0020649_hypotensie": 18, + "C0559477_perinatale_asfyxie": 18, + "C0270191_intraventriculaire_bloeding": 17, + "C0877064_post_hemorrhagische_ventrikeldilatatie": 13, + "C0014850_oesophagus_atresie": 12, + "C0006287_bronchopulmonale_dysplasie": 9, + "C0031190_persisterende_pulmonale_hypertensie": 7, + "C0015938_macrosomie": 6, + "C0751954_veneus_infarct": 5, + "C0025289_meningitis": 5, + "C0023529_periventriculaire_leucomalacie": 2, + }, + "qualifier_counts": { + "Presence": {"Present": 436, "Uncertain": 34, "Absent": 30}, + "Temporality": {"Current": 473, "Historical": 18, "Future": 9}, + "Experiencer": {"Patient": 489, "Family": 9, "Other": 2}, + } +} +``` + +You can also get the individual statistics, rather than all combined in a dictionary, i.e.: + +```python +dataset.num_docs() + +> 50 +``` + +### Comparison statistics + +To compare two `InfoExtractionDataset` objects, you need to create a `InfoExtractionMetrics` object that compares two datasets. The `InfoExtractionMetrics` object will then calculate the relevant metrics for the annotations the two datasets. + +```python +from clinlp.metrics import InfoExtractionMetrics + +nlp_metrics = InfoExtractionMetrics(dataset1, dataset2) +``` + +#### Entity metrics + +For comparison metrics on entities, use: + +```python +nlp_metrics.entity_metrics() + +> { + 'ent_type': { + 'correct': 480, + 'incorrect': 1, + 'partial': 0, + 'missed': 32, + 'spurious': 21, + 'possible': 513, + 'actual': 502, + 'precision': 0.9561752988047809, + 'recall': 0.935672514619883, + 'f1': 0.9458128078817734 + }, + 'partial': { + 'correct': 473, + 'incorrect': 0, + 'partial': 8, + 'missed': 32, + 'spurious': 21, + 'possible': 513, + 'actual': 502, + 'precision': 0.950199203187251, + 'recall': 0.9298245614035088, + 'f1': 0.9399014778325123 + }, + 'strict': { + 'correct': 473, + 'incorrect': 8, + 'partial': 0, + 'missed': 32, + 'spurious': 21, + 'possible': 513, + 'actual': 502, + 'precision': 0.9422310756972112, + 'recall': 0.9220272904483431, + 'f1': 0.9320197044334976 + }, + 'exact': { + 'correct': 473, + 'incorrect': 8, + 'partial': 0, + 'missed': 32, + 'spurious': 21, + 'possible': 513, + 'actual': 502, + 'precision': 0.9422310756972112, + 'recall': 0.9220272904483431, + 'f1': 0.9320197044334976 + } +} +``` + +The different metrics (`partial`, `exact`, `strict` and `ent_type`) are calculated using `Nervaluate`, based on the SemEval 2013 - 9.1 task. Check the [Nervaluate documentation](https://github.com/MantisAI/nervaluate) for more information. + +#### Qualifier metrics + +For comparison metrics on qualifiers, use: + +```python +nlp_metrics.qualifier_info() + +> { + "Experiencer": { + "metrics": { + "n": 460, + "precision": 0.3333333333333333, + "recall": 0.09090909090909091, + "f1": 0.14285714285714288, + }, + "misses": [ + { + "doc.identifier": "doc_0001", + "annotation": { + "text": "anemie", + "start": 1849, + "end": 1855, + "label": "C0002871_anemie", + }, + "true_qualifier": "Family", + "pred_qualifier": "Patient", + }, + ..., + ], + }, + "Temporality": { + "metrics": {"n": 460, "precision": 0.0, "recall": 0.0, "f1": 0.0}, + "misses": [ + { + "doc.identifier": "doc_0001", + "annotation": { + "text": "premature partus", + "start": 1611, + "end": 1627, + "label": "C0151526_prematuriteit", + }, + "true_qualifier": "Current", + "pred_qualifier": "Historical", + }, + ..., + ], + }, + "Plausibility": { + "metrics": { + "n": 460, + "precision": 0.6486486486486487, + "recall": 0.5217391304347826, + "f1": 0.5783132530120482, + }, + "misses": [ + { + "doc.identifier": "doc_0001", + "annotation": { + "text": "Groeivertraging", + "start": 1668, + "end": 1683, + "label": "C0015934_intrauterine_groeivertraging", + }, + "true_qualifier": "Current", + "pred_qualifier": "Future", + }, + ..., + ], + }, + "Negation": { + "metrics": { + "n": 460, + "precision": 0.7692307692307693, + "recall": 0.6122448979591837, + "f1": 0.6818181818181818, + }, + "misses": [ + { + "doc.identifier": "doc_0001", + "annotation": { + "text": "wisseltransfusie", + "start": 4095, + "end": 4111, + "label": "C0020433_hyperbilirubinemie", + }, + "true_qualifier": "Present", + "pred_qualifier": "Absent", + }, + ..., + ] + } +} +``` From e52690a2673c2299051bc32f88426fa5db7a728a Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 15:24:51 +0200 Subject: [PATCH 16/30] Restructure toc --- docs/source/index.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/source/index.md b/docs/source/index.md index 4ef3a77..00f1064 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -3,16 +3,26 @@ Welcome to the documentation pages for `clinlp`, a library for performing NLP on clinical text written in Dutch. In the menu to the left, you should be able to find the information you are looking for. If you have any questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html)! ```{toctree} -:caption: Documentation +:caption: clinlp :hidden: Introduction Installation Getting started +Roadmap +Citing +``` + +```{toctree} +:caption: Usage +:hidden: Components Metrics +``` + +```{toctree} +:caption: Standards +:hidden: Qualifiers -Roadmap -Citing ``` ```{toctree} From 18e246e4989d019b5eca76b7bd0e28d63630e72d Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 16:29:03 +0200 Subject: [PATCH 17/30] Add components library --- docs/source/components.md | 299 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 docs/source/components.md diff --git a/docs/source/components.md b/docs/source/components.md new file mode 100644 index 0000000..7d27a1e --- /dev/null +++ b/docs/source/components.md @@ -0,0 +1,299 @@ +# Components + +This page describes the various pipeline components that `clinlp` offers, along with how to configure and use them effectively. This page assumes you have made yourself familiar with the foundations of the `clinlp` and `spaCy` frameworks. If this is not the case, it might be a good idea to read the [Getting Started](getting_started.md) page first. + +## Basic components + +### `clinlp` (language) + +| property | value | +| --- | --- | +| name | `clinlp` | +| class | `clinlp.language.Clinlp` | +| example | `nlp = spacy.blank("clinlp")` | +| requires | `-` | +| assigns | `-` | +| config options | `-` | + +The `clinlp` language class is an instantiation of the `spaCy` `Language` class, with some customizations for clinical text. It contains the default settings for Dutch clinical text, such as rules for tokenizing, abbreviations and units. + +The tokenizer employs some custom rule based logic, including: + +- Clinical text-specific logic for splitting punctuation, units, dosages (e.g. `20mg/dag` :arrow_right: `20` `mg` `/` `dag`) +- Custom lists of abbreviations, units (e.g. `pt.`, `zn.`, `mmHg`) +- Custom tokenizing rules (e.g. `xdd` :arrow_right: `x` `dd`) +- Regarding [DEDUCE](https://github.com/vmenger/deduce) tags as a single token (e.g. `[DATUM-1]`). + - De-identification is not built into `clinlp` and should be done as a preprocessing step. + +### `clinlp_normalizer` + +| property | value | +| --- | --- | +| name | `clinlp_normalizer` | +| class | `clinlp.normalize.Normalizer` | +| example | `nlp.add_pipe("clinlp_normalizer")` | +| requires | `-` | +| assigns | `token.norm` | +| config options | `{lowercase=True, map_non_ascii=True}` | + +The normalizer sets the `Token.norm` attribute, which can be used by further components (entity matching, qualification). It currently has two options (enabled by default): + +- Lowercasing +- Mapping non-ascii characters to ascii-characters, for instance removing diacritics, where possible. For instance, it will map `ë` :arrow_right: `e`, but keeps most other non-ascii characters intact (e.g. `µ`, `²`). + +Note that this component only has effect when explicitly configuring successor components to match on the `Token.norm` attribute. + +### `clinlp_sentencizer` + +| property | value | +| --- | --- | +| name | `clinlp_sentencizer` | +| class | `clinlp.sentencize.Sentencizer` | +| example | `nlp.add_pipe("clinlp_sentencizer")` | +| requires | `-` | +| assigns | `token.is_sent_start`, `doc.sents` | +| config options | `{"sent_end_chars": [".", "!", "?", "\n", "\r"], "sent_start_punct": ["-", "*", "[", "("],}` | + +The sentencizer is a rule-based sentence boundary detector. It is designed to detect sentence boundaries in clinical text, whenever a character that demarks a sentence ending is matched (e.g. newline, period, question mark). The next sentence is started whenever an alpha character or a character in `sent_start_punct` is encountered. This prevents e.g. sentences ending in `...` to be classified as three separate sentences. The sentencizer correctly detects items in enumerations (e.g. starting with `-` or `*`). + +## Entity Matching + +### `clinlp_rule_based_entity_matcher` + +| property | value | +| --- | --- | +| name | `clinlp_rule_based_entity_matcher` | +| class | `clinlp.ie.entity.RuleBasedEntityMatcher` | +| example | `nlp.add_pipe("clinlp_rule_based_entity_matcher")` | +| requires | `-` | +| assigns | `doc.spans['ents']` | +| config options | `{"attr": "TEXT", "proximity": 0, "fuzzy": 0, "fuzzy_min_len": 0, "pseudo": False}` | + +The `clinlp_rule_based_entity_matcher` component can be used for matching entities in text, based on a dictionary of known concepts and their terms/synonyms. It includes options for matching on different token attributes, proximity matching, fuzzy matching and unmatching pseudo/negative terms. + +The most basic example would be the following, with further options described below: + +```python +concepts = { + "sepsis": [ + "sepsis", + "lijnsepsis", + "systemische infectie", + "bacteriemie", + ], + "veneus_infarct": [ + "veneus infarct", + "VI", + ] +} + +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher") +entity_matcher.load_concepts(concepts) +``` + +```{admonition} Spans vs ents +:tip: +`clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. +``` + +```{admonition} Using spaCy components directly +:tip: +The `clinlp_rule_based_entity_matcher` component wraps the spaCy `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: + + from clinlp.ie import SPAN_KEY + ruler = nlp.add_pipe('span_ruler', config={'span_key': SPAN_KEY}) + +``` + +#### Attribute + +Specify the token attribute the entity matcher should use as follows (by default `TEXT`): + +```python +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"attr": "NORM"}) +``` + +Any [Token attribute](https://spacy.io/api/token#attributes) can be used, but in the above example the `clinlp_normalizer` should be added before the entity matcher, or the `NORM` attribute is simply the literal text. `clinlp` does not include Part of Speech tags and dependency trees, at least not until a reliable model for Dutch clinical text is created, though it's always possible to add a relevant component from a trained (general) Dutch model if needed. + +#### Proximity matching + +The proximity setting defines how many tokens can optionally be skipped between the tokens of a pattern. With `proxmity` set to `1`, the pattern `slaapt slecht` will also match `slaapt vaak slecht`, but not `slaapt al weken slecht`. + +```python +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"proximity": 1}) +``` + +#### Fuzzy matching + +Fuzzy matching enables finding misspelled variants of terms. For instance, with `fuzzy` set to `1`, the pattern `diabetes` will also match `diabets`, `ddiabetes`, or `diabetis`, but not `diabetse` or `ddiabetess`. The threshold is based on Levenshtein distance with insertions, deletions and replacements (but not swaps). + +```python +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"fuzzy": 1}) +``` + +Additionally, the `fuzzy_min_len` argument can be used to specify the minimum length of a phrase for fuzzy matching. This also works for multi-token phrases. For example, with `fuzzy` set to `1` and `fuzzy_min_len` set to `5`, the pattern `bloeding graad ii` would also match `bloedin graad ii`, but not `bloeding graad iii`. + +```python +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"fuzzy": 1, "fuzzy_min_len": 5}) +``` + +#### Terms + +The settings above are described at the matcher level, but can all be overridden at the term level by adding a `Term` to a concept, rather than a literal phrase: + +```python +from clinlp.ie import Term + +concepts = { + "sepsis": [ + "sepsis", + "lijnsepsis", + Term("early onset", proximity=1), + Term("late onset", proximity=1), + Term("EOS", attr="TEXT", fuzzy=0), + Term("LOS", attr="TEXT", fuzzy=0) + ] +} + +entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"attr": "NORM", "fuzzy": 1}) +entity_matcher.load_concepts(concepts) +``` + +In the above example, by default the `NORM` attribute is used, and `fuzzy` is set to `1`. In addition, for the terms `early onset` and `late onset` proximity matching is set to `1`, in addition to matcher-level config of matching the `NORM` attribute and fuzzy matching. For the `EOS` and `LOS` abbreviations the `TEXT` attribute is used (so the matching is case sensitive), and fuzzy matching is disabled. + +#### Pseudo/negative phrases + +On the term level, it is possible to add pseudo or negative patterns, for those phrases that need to be excluded. For example: + +```python +concepts = { + "prematuriteit": [ + "prematuur", + Term("prematuur ademhalingspatroon", pseudo=True), + ] +} +``` + +In this case `prematuur` will be matched, but not in the context of `prematuur ademhalingspatroon` (which may indicate prematurity, but is not a definitive diagnosis). + +#### `spaCy` patterns + +Finally, if you need more control than literal phrases and terms as explained above, the entity matcher also accepts [spaCy patterns](https://spacy.io/usage/rule-based-matching#adding-patterns). These patterns do not respect any other configurations (like attribute, fuzzy, proximity, etc.): + +```python +concepts = { + "delier": [ + Term("delier", attr="NORM"), + Term("DOS", attr="TEXT"), + [ + {"NORM": {"IN": ["zag", "ziet", "hoort", "hoorde", "ruikt", "rook"]}}, + {"OP": "?"}, + {"OP": "?"}, + {"OP": "?"}, + {"NORM": {"FUZZY1": "dingen"}}, + {"OP": "?"}, + {"NORM": "die"}, + {"NORM": "er"}, + {"OP": "?"}, + {"NORM": "niet"}, + {"OP": "?"}, + {"NORM": {"IN": ["zijn", "waren"]}} + ], + ] +} +``` + +#### Concept dictionary from external source + +When matching entities, it is possible to load external lists of concepts (e.g. from a medical thesaurus such as UMLS) from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: + +| **concept** | **phrase** | **attr** | **proximity** | **fuzzy** | **fuzzy_min_len** | **pseudo** | **comment** | +|--|--|--|--|--|--|--|--| +| prematuriteit | prematuriteit | | | | | | some comment | +| prematuriteit | `presence_threshold`, entities will be qualified as `Presence.Present`. If the predicted probability is between these thresholds, the entity will be qualified as `Presence.Uncertain`. + +### `clinlp_experiencer_transformer` + +| property | value | +| --- | --- | +| name | `clinlp_experiencer_transformer` | +| class | `clinlp.ie.qualifier.transformer.ExperiencerTransformer` | +| example | `nlp.add_pipe('clinlp_experiencer_transformer')` | +| requires | `doc.spans['ents']` | +| assigns | `span._.qualifiers` | +| config options | `{"token_window": 32, "strip_entities": True, "placeholder": None, "prob_aggregator": statistics.mean, "family_threshold": 0.5}` | + +The `clinlp_experiencer_transformer` wraps a very similar model as the [`clinlp_negation_transformer`](#clinlp_negation_transformer) component, with which it shares most of its configuration. + +Additionally, it has a threshold for determining whether an entity is experienced by the patient or by a family member. If the predicted probability < `family_threshold`, the entity will be qualified as `Experiencer.Patient`. If the predicted probability > `family_threshold`, the entity will be qualified as `Experiencer.Family`. The `Experiencer.Other` qualifier is currently not implemented in this component. From 806d45ba57fa4bc9f490f584e042fa3b90d81967 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 17:12:52 +0200 Subject: [PATCH 18/30] Add qualifiers definitions to docs --- docs/qualifier_definitions.md | 3 -- docs/qualifiers.docx | Bin 27210 -> 0 bytes docs/source/qualifiers.md | 96 ++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 3 deletions(-) delete mode 100644 docs/qualifier_definitions.md delete mode 100644 docs/qualifiers.docx create mode 100644 docs/source/qualifiers.md diff --git a/docs/qualifier_definitions.md b/docs/qualifier_definitions.md deleted file mode 100644 index 396b3a8..0000000 --- a/docs/qualifier_definitions.md +++ /dev/null @@ -1,3 +0,0 @@ -# Qualifier operational definitions - -It's useful to have some operational definitions of a qualifier/context, i.e. what we mean exactly when we talk about negations, hypothetical situations, etc. For now the framework we use can be found here: [qualifiers.docx](qualifiers.docx). This information will be incorporated in a separate documentation page in the near future. diff --git a/docs/qualifiers.docx b/docs/qualifiers.docx deleted file mode 100644 index 78d4b09b1d718d7eddd744184993156236a56d56..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27210 zcmeFY(|0Dow=Md`PRF*{F*|nhg&o_rZQJPBw$-t1+qQZ7x9>RPo;&XT5B8~tdZ?;G^2AAAChN#hp%j7Xx-!C!&1%_~yvR7C@$`LV{hW^W)7tg<4H zLIs^)UHhC&MPy8@qeDc~8=lk9Oo6j&=~ghvO|InUn7k?ei8|&Sw5!Vp^u9*e6uU`` zTY{tPepalVf3c<6K^RG!`V%GuzcEUz5$)c}=TJ1fFER2{{Uls%WM)+= zXKbc@llti|&`78me#~jO3hAw3^^Bvkc5U9qp}t>S5BD8?^v#qbeTCBMYhJ$n|Y9JAW_r9GUgE>k2tuHFm&O+YIJ zTlMNN3g!P7S>NAa0Qvt9JPG1)TCV@i$o@kg+&_5gIv86yGW`5c|9@fmKNySu$JQ$o zdjG)^Uf??LGjR5=!dfp@fh>dJ%sSR86tt$KH0t__#q!rD&&mp@_Nl)3_}qNrjE7^U znDb7O_6<&oGD5^(n59>pe$DqzH$ZZ5XAzUzlHDGB#?h0>=NPFZ^;lq}I(o!34rKgG zSlVO{>LHED(_smmISKXjf+1N$LAJf}@^hLGOYR@0#U%>~O;4zDUeT@){PrnqA6OIq zx>OeWxY)j}dhG$Fv&2^BXbb8`IvguT2J*~!gGyMOo6gxYkAm)!!SG-@m^_$phIcO$ zEfkHftMw2|A55E z*3jAdACvuu!u~g4K>qRAzt#V40i6cDtt|!vQR%&K^Ij7IkLW6z^3>upmZpk-A^RK~)py~H*zaSY;$C_^Jv~V5 zo?D1as(%XRgkDz3NTp-9Z$r&4hm|&3W^bTLH27h4WzW=f0Mu{A0HA`V51ZtHKD4eY z){j%YUap*KUQ4`R-mv$?=&)5AAtw2#gJp}9OgOAiEtoShc~$RAiNwnlVkUZC7p8%V z6>UO2mem-Z^Nn?$D?BVw z--Yb2M3+(RMI+>S6pjS8wAW>kPmbl$xN5?M=-SXT=7==p*XO%*I>d&A|>3&^~{l5487~c;i zdZc6m!h*_>y@n3gefMW5C`9pZY#x4v4YGDe3-bf-o#Z#(%)L?(A@ioM2j!>Vu_bV|>=}8S_ zjWh@Oo(OSen!KXif1bHX!_eoC>f3`^Lu)i=VU@o6B@b>psS$RV>kRj%oe4>o5NJ+& z`ooq&{Vvbr`}4w^>{IyF=aI=XP;|0A_-f+UQ&rfV!JExekn7>Ojj# zVetMR=dg}uNZpv?XdWwr9$`Eg-)Bw?-10T<`J|3Sail0zAvoOb5T#ExzG!NenF^wU@yK%XdgSi z^|l{eQ*45`_eZ3;{IRSDsH)T4XP;&f$0JyJ@ zeq2F2kAS_>2JxS`Gm=V`KKJfd$k#WHL;OAr*(5}d{a_G=-{f>^4@ZdIFK2RuPl~m9 zAL=SysIx|#p9$pG-=D4@Pt|Y#PS=TF=6d$mmyVxi@ODA-ZWtdKE2PfnaY^CJwC?xb zY7QKR(xsgEE%rtz%>%e%J&>L4^z~Nw_SP{aWGm7My@A%TT>I{2!nrgh7$g0Wb`<4nW#A2-7Xx)-5 z@X)?iR6hu3h!+2&uI z@EpeYimAV68byM?GIPJp6}S?!8ApW|vtwF$W4qlOmrvzAA<1L_uJxC{Pj(*UR&l4- zE#@jNfP6e&?*??ABzl*DZ%+?&!_yVWM?NEwHXR~O5MM`JDvQM%50l1o0S1Yc==<^M zJ3Z7SwtjRO&}xK0#OMPTS!je{hgT9_P6_HD|U$Vw3Y`nWS}zl+8ngJf8P%>N>Gt~ z8VmTo`uN0xtNjm0&1nif~cg9kV0VrCW9oI z{I>|P{*Zw@gkdj_(V9-kg-X6;y*pYlNZ(izc5+)!X2!{huh?nuQCO*&I{q-8?*k7B zk_f%sMaL`6Qw9RG1rE|sb4VyW4|J7yz=)^%-Qe8}1q!I3Vv5y!^F}!fdRz$+h@`PT zAWi6?1`>M)5==t_qzQ@@W;GNBkavG2?L1lRie&H6jSmfdIvLxuKqX zr0@vpJvQ(d*8QN{N&?ZRd_00u^*@Q-_Fj01iRUq0`LEbi~P5KBGF-F_m{RDi8% zmVpl&|M0N!Jboik2_9i(0Lebg0}DX;T=6?wF{`fwNh6UD%z;=o_?fF^R4H(G(1O=c z^p}Rji8$VdmGp{mEBq*8DFl!=I$Q6}fA*Pde}?28O=*e(IPafIOb+pJ6w2|MCW;&? zgfJ`KjCA$QdRgkxk!pZ-|Kfk-gY|VYBuR2<5X0t|FOi%uQV z=I8NKTU=-9x-jPE+xqF$4XGl~Uu>55-Yub4I8t4OB^E038zvG>?y_uawTDFJ&Z@|? zCH;sX-T47J5DCxP7^+uS$Kc7oBbe{r9Q(GQAvlLzZxnL2Fd)?sw%w!Od*~q*E!PZ^ zH)!O4zHd-NG?+Z1(zCPjL0fS1QyB-eHs>{ z`+Ozy8sdz9FIhhk)S9SLkT09JJElzRX*_tr7_;g= zpyd`?r(EGk@)n@q25Q1F4WHkLsfoAy;OJbKU#*cc4&|zg5AcoGN7<+_M+1k#5Txd<6|Q@^Uw>#G zI4B)}@xsqv7qQjg>>X+0y=zfemZ10aVubGV`fUO2GpH+jA2ry^!A)NFwwoY7$~(=D zaB+BJ!7KjZlDcvHomO2s;h7An!82jjDI4xA2%-on(sBmh)ItYbDIQ&M_4dg3oOX?# z82%EUz!XuN*>bb@-?y$z{#{)^S*Y|(CRks!1(!H?C}~MK)NDs|#$6@q7ljy^2sD$qHm%>s6YUh66<*3y7*m1MUe+HPPTKE@1 z>ei#><}pBHNBs31Th!8c`NBLv=}(6#X(?>TPCg_)K1FH5VS97XTBGVkTe%DRv+D@+ zNy4Q^_EKI~!vU`dSFs_1F+dU9;s#bKSrMJ?XXFmaXMVn0HVeZJexPXf<1ik`JNuAC zou3=_In&;1_nA}Psa|pTtzS)@oslwzh>~dbIl6hGbPEW2^O7Qj`L%NoRJ^%xqLJ{n zC-2A9-Y*7^_i@F4nE(%1R9Sj?SS-(6vb3ExePO_UMn&+)X=(|PL^lmSl1d5a#~PM? zlZFE-?_;)QtU8hW9xU%gVXqo`icKZTs%bTIXg8yRx@xv6CA8MeF}UD2@Z)?whu?d3 zS%km#B0Q>Z*X&%YA~b6fc7mbGZ*d)Y5(8iGu@dekI+XD>fp#s1-dW9pt7{aKcd*W? zjU}o~uZIazUeZT#@=sqHB)|}aOxjw7CCQKQuz^Os?La|v(t)hbajzno;=-4{T*riL z^EZ#;dF~da8qkuM*;1;*zAWlm)T8nSRJSo33SKe3w>}xx;D}6SrSXVM_8#<*9s(6s z*FX{&vprir!%_&dJ^a9Sd5N#MhDYJU8MA_LD%W~tC(lN2czCcGhj%Uu#2kln zQ=n#fAuQFC2CHnPuKf+O`~>%OVQ=@RyWFSzEUfh^WjuUknr=~vXP4PG{J0+b-PC9Y zvN|dCS>@M+$->VIAx(2sQ%CvQ!a@fNvD)Xf+4@HBfdFDwFCZVf7ma%|TTPL7_A|-h zuSKcAhp@-7SYvmzhtbG#|JqfXI>D5o8860yvydR;d2+H`YzFI?Q(jWRG^TE~9V`B< zqPHrwX)K_+X_18hGQ z4m3;qs1oiY$)!FG+;;cjI{uC4qW(3nmZGz{0qdXm>y+7pk2jyoJbdSW<%S__PBPn~ zpDe#NCDJ^KxmtQJZ-w;r)PFgj)JMKu+~yTKX-;ksKq}$75?b&$Y8K5H$=PYbxZ*lP zuyjODE;Kvx_W}1}=<7TQQS}=5gu7ib?WGqlFX5rPEHfo`&O;o-gL-u-yFY(_NUkd? zZIi^;*8E(Sc*GlM6+Gx4V}9iGl=2Tk$+uU-E=>Hot6n3oe^R@S_DHUc(>yDq1#MN4 zDeO+{^M>c3n19xQ;@hkaejhJ4!q~oFPLHLh-)}$M5{`aotB;Oh!rDPZ(Av7cH&x9P z#B@6W7qX~^N+g55ju!>^;+{~1dN{U^cKP6rn7=fMDR5g$?y@GMSue_Z%VjM!G$B`F zvZ}fm$CxvqEWx!eosT*kKODq}h8)K&2wbREkx!e$Uv;b&b~WTpKX&WZcHj4$IS{Rf z)Dp^fu$*RSYvb=~N{ZGr`(hMu?7Alq8GG47?!iqc?ofP=sRs~EQp%5MsfOR#xWNXf zLfxm>)iCvcK8JyiMS*|Z`SZrtrin4~by(}AvJNL(O`d3VlaG}b?XDjiRkDt!IQ4S| z)M-=uWD^KEGOj<|j$5V51)eHfi_bfi^oQ#mK8z(8jBl2>{!lE#3891&=R~oxr}2g9 zK2i(LVeGPjkT*e5O=0lpDL444CKbGh|G9lt;A zC+d;$_c7kt26&+KhqB+a`pf=_i0-K|mrEa-{_F&pj_kRagsA_F#TMdDvjfHv19Ex% zVJ`HTq#RW=B$QhfgYPPfO@8@!wx5=E+8x4jjKQq0+Y4dwH`p!=v#Gdo_67dSU(gTl z_;K#cnX$SuMcjCXH@Y4`2>k5*P&vMOk`G_mw0wUT2KM41wBqvhGe+*!rF|*bl@uq(6x8FeLPHqDZe!wuqk|np*DJTF)9{*-lB} zLeHAED?Pv1F>HPmoL^IlQ0}ebslT}{`LX}Eb#FscwyPE89M@;3@b*G+}X~z0;UdSn)iP+pUG>hx#F$R8i2UFSQUmT#+@G!c-mh{C;9NNbd9* z&Mr(IQ#9kz%l~3+AjyGedGiVrCCS4DNNRKoeoul%7hl%l;PZ6D`L#&pVtDC zZvr2LK4NU6q!2PaGDI+*zD7MV{Y0-YF~FA(wBv${TOxS|DoB%|RvH1Uut!eaj<>Jc z@^=>3hoSoH zVp_p6q%a~R+Vis}Q}et@(}co>L+YInKD}b8q?-PO#wQ73GGraLbG-N}0*T%hqp9D| zRofA5{4)F5Q+r4IH$=fQIiU~4bxEJAuASqhQPIy_(EVa>%tKt?tR&TtSX_3WN~~hH#3w+pj0C zDP>ZppOM9vX>Vmj`<6n-5qhl=yUx{R-hWTDW{hX&nx-)4bY{6qno_7u+0y0<90ix1 zrW0F|8t_W5M&_Lb&wLiQ;pzsXGjB`+i7iqpH15j!UTK}SK{P&Z&w9wluL9!`6Ir+2 zv<}z`w$i4X^f9QmZus)QPNZ+&+F2m`YB1ZEFoS%O5Y>b}#U6C~gb1|*3b|sf5_sgqa`d=R9)L>Bd=a)6Y=N$ATA)+Qj5i|5~Ei z^YysV9@(1y79$TbXW5xeeW{`4f8@(dqsjG^1U_?B352 zg8;n)J@2>y58?^n|JgbKj8~qHmu)tFV;h9)F_xTxFj_)?Y<9RPKPpRK`#d_b?Cim9 z8-f!L$7B`7AeV9C0G{jbK};gkL1R`o$;eDZ*udsjYX?@K1M6LdXGEu$b$sre6M{rQ z%XOGPDiJ92SwTim_QNHPp*fca;9`?Zs8%pFN|SK$S5(fUt7K(|1ttMmG4j*VIS z)hsZ%wM>lUH-+mbr3tO>n-{=X&23 zUv6Feci5KXQ)MRCRRleQUp~Hph@FSo;0mb59q&a_HJE#FDP$QbjtDW;NWS`ZP-PG( z&Z^E6^co(y8bor}=G-mRsz&XTR8%4K<@EtpZOoQih!@(^9i{fXbRd>Nrppvb1Nvy4 zLPXfX>sMO86c9f`qSSg^o5aXsPtuHcYhk>UlvV4De3tjo`3TVX%8RMPvfX(hN_f3k zJ@S8Q)$q0U4G;EO`^CN}Y-P7Lt`L=%6FIlt+czvI#U2%!AnSf)#=LrmWmJ1tVjqrq zSdJF_dsYBb>IF8n+?xCJHt2MaDK}X;Ysz{N(zR{5scTzP!~J&N|G3+ilCU65rMGQ^ zcJ!$e=78T+dU84@FL58*t4dCCkV~{KbF3a{VpXuEF|U$#$@w<)P~GdSve5hOTKTVZ zvlxM+bd8+80dEViEx4kuJStW{rkSx?G8Uc{4W70wQFrCpjH^@I&28}My1$UCuk^pR z+D3J7$nowLFW>UuU?%8 zH+QevO2!*hk=yHHollT%c+ZcZk!)Vuj8jHkIHA}{IHvX3dvcPq|An~-9kH=biyunZ zv(Uxr264DRZR@MsEnv-b|bxGmPAN{WZt797>8CHZ`9X zZ;wL(@slD2Y40@Tff7RU_g5trQ@~0JS}b)`{N`~B8>E+dBN{O2b|o7QsY=+5NG8cy z^V$b`V>YxKbWTK(4ZX}!rZm^Axc^>fm=WRL+p@Savib6=noyCHxnygaMxzGj;1WwN zfO4s9lC10-!&vs8g1F0GKD1Ts5vLW52OaW2#+rfSAtd2`JYI z8>TRmeU0HQV4nLD)pdbw{Lut^;;)tLKgv(a^VK|{D7!r8YzfcEPnwG)@#b>4K7nWX zu)MuVR>rQ*ugq&&qpHjEMr$&N)zk)uX3gbFb?lp7xp~NhUbeDB$UM{G{LV{Sp(O)x zAOl$!|4Ki$5|Dp$Yw(TCrH3}GOOosRQ|ajtO?X~vy!G^To zOU2iSgM1=HPy~wJZ|mTNwn+7Klpu_#M;<6gqySwAy@msIA@*Lbq)6O5fVK3I zk;Rdtg$|X1CN;?g?jziuT!a&2+M{Q321{xNQj|nWU*^#DJ=48lkQbjs+Eyj_6=^zz z88#?8bRyCv_zooGO&4KakbNSH5sYvo*j!xT;7ian=thiq195J+B%WeaCHBaYA`^`u zzr-T62<#Sowjp?2JrUwPk2GokI*vIYfHV9ClPbOkKy<#NXlGT|P8?e>Lx`M15j4vK z4q4D^qTGsBBFQxN8Y}P|AQ=K*crZMDa2bof&%d9w%E(3iagXl~eP6qCF?48Qa}Tms zC(d{x$GNSa9-p>+*NN2Z(@D;Yg*rB^gyx+dXz2-lY8XXE&B>J>D4$1U_{UeF9%7qH zj1jp2ErLuyyT^d-{Zi7_vRzihj+S+J?M zgPOmNO9B({dX@44$s}4fKhl6{Ol+Es4w+d}gb6w*3t2FI=v#WU0n{Db8F;q>ufHCK=~NSa9|V_9fGK*Z;UdZ) zM9`}b#tJ#XQ&=Jh9PDaFz2mwqKVdcED3Xvnbw5LD5Em>_a9bH_(#C{*6Lp!{dgIBn zOivYO^`lVfti-YLfHSG)|jqw5Pok+|AyY??fajRvARgk z2j&m8R3VQ%C?b=M4_Bc&=lO+PULY-~`z|>~gUX_U7x=`xjhv(1LArvA1&sPU`d=@^ ze+H6AqX=pRqQ1+^DFUzgAm&Zy$HrFSesP9=R&}Z!#Piu_z9r3I%`yjp@x0K06`Hn2 z3^^nQ*A$!*#d-S%ZsX`(>q{f-?0-Qz2|Gxa+jlD+iZPA_wpwBcH8@aU2Se9J47x*W za}6{5L-|W%*9|&peL0;e+w)qt7>2h;JsBbo(djAF0eR}h+uX-(LXl!RF-K7gu)p$d z%$`F&8pgieR0C}J&;?-u1gS4jZU7pCm)~SZeiL-A5wEqB8>}dsC~%;DxS-bp(%8p0 z=ScYk8YU9OEXzF$=hc>KF^$j3H7s_agW}h?(Zk4^TF9FHP~zyKsuy0;gQwcr$K~~A zt~vqPms30lYMOIH*u0LHR-C_E!=p#~cJ0<(Y8PdHrTbZjm^lXP1=dZwbh6nq7dhMf zlR1B((l1~_=v~x~_(Qlm6E>l}Vp%BdcA2&}*)8*5o5405h}MoUI$U#UH{nH-wHg{m z>qWJ9#T(d>>zgVby{9J3Z>>2Dl@;cy7P;h<_e(jS*=suwo5k`dsmo{DNZym^uBSHn zW%3q0zW&{4=N`#r#(kXxt172?D9{69?w zb5o~8f!%xoAuM5q20nqBGLx&LX0{$!f_O*ISVQEd!mldT-CgSvLKB6k3&X_yn&h`p zA$CCC!*JQS)+&4B@nTsgyx!zS#JW!%X>D2^#}7BaOl$u18seWV(V+L#We(-Clz@GIb=Z>AG3|k5(58f z%^tJk=4WIVj$f61ETfmZ0-|$FIS6Qq>h-Azf*=3D>4M zs@9)oyWqN@fd)UQUR@6;{(<^bnozb@?g3E}9oLz!5mK$oAkRZc){S1~_C@t*juAv% zdJXuIL$rO0#P9*#4eHRFRm$q$-5miIa4sLa5%JbSOXnz1#$lL9cbU^Pu0O3Zte?lYPbL^2i+_+ zfylVc9MbcQ%a&RXooR(l)ITA-!ikPBplIs9jSgXLIXcJ}&;O)Txc-c3g#=k9(z_o_ z44|d$39j&4OymBbDlw#BwZvAyq>;>@NvztEY%UCoxPs1(@i~SyYx$$Dt%z2!{3$Vj z8TqA&Gmo0bc??G_)>C7qUJ&(4$}`XuAIkeHUhbA> z=3RWv77A&~UFhusL}<$nYa@YTc!egSR3gBECgZQ^fx}PzKQzF0_mDVHcI)G_4lDeeg4XVUmSN%Kso=}l3kn>b``<)_@5C>~JU3qW~vxeM*F z{E0uuPc7=^lc>(S?G0wDY7DDujza#j={vfu1~)y>c4Z?gaiZGn##(Pz*zmG`Om6+Z zM@GP8G#q!F)V{1FtZ}+xC>1g(l5{IrS{o5qxBF{Sm5jKjpOMGk=%5U)Ri|b=nTFXV z@~b7QT_1%*f9Rqw*Ii~WyTkRJL+gqT+gU4|YKb``=rXW*RsC%^x1xdbOqo|Kq{sdX zcB#!QqK=ARR3X(_eaJlC2(Qqy+ibKX`N|COIGJqwJ*%==6>lowyq|lpybzib)mH;# zKR|)x;)p@iA3vPXvi5WQVWDWn^YCI|8cptcxbFNwrivs0Tm59 zIb-7!o8#($&qz#ZM8e#vPiM^G(38M%9FWqtP+dOGNGMLyt(Xgt&IqlRlqmOfed&J+ zRC6p06bCrCoqnXS4d17a@romht2s_keqVpT=>vGt=r-KcAIdm4&D?<}<}TB-8>p*) zc}c?^doSt@pQTmh+Z`Sw#A?i(cyR_}?+iAJCc5-t|7Ne|)jBu$DaRgxTOn8fN6{#^ zIdjM|Sgb~5tEJ@Q-X&y@M4Yt8gq`aE@i+zw3JT7#H8Mp?wx7PAg3mw6(7$4KwWYUgKgBR++G(}dA>he}M zgeMG`ijgZC{Sk8bxSp6Gm^j)pywzR)FP`v~5AXY`&i;DNrj$&O=>8M)jGam_D9>~} zWi(#JV?9?J2eeU_BAq|pXGgv&$9cJ*oo|^At5Aa=x}e{9C0y0a(29O+pZ3D{^3w<2 zIFU;Ya0ku(Q1nOH?|`jI>HcMl*ndi@tS^iugMl_-$6PbTQTFPOh;V~-+x-x|a)OnQ z$*8Qt;!wtA35v?p)E$7VzivQwr5evKcZ4^dG&k_aj{U5v*?%dTyQB2RK!dA7K=?3H z;_NLatoIt!0dyVa2}oHc*{GTb9ELQZDsH1h+*=~nS8LU+DDBojit#84H=%^PNHIa5 z4lrVUp0^_)J(TWvE%2iwz|eCucB@9z_+0i*Uu#zSJaUeaeuV89S zeiX$gY1-o6B%k=vYWz%Izs_^an8Cp?yx6QweB}KSjT>M4;>>W5p>~^;pPbs~j$e+M1t_kEL?}*#PKF8P{uGLQ148=+kX7bdFgJ2=JE!03T z4SZJ^%Wp>2N@9+}i^z8M@+L)|aDiMJ7M44{yMr7tjj0pQ48vPsC&3?_H~N+uT7HdT z=NYLAzMhqW&cG+W=mr*>2}kf6TW1I3fa3sJ%`bo0L}`LGZc5`gQP7k%mTP>768V zng2^K+2m($qq6f{sie3TR;@FGG7q>sLGIl#GGGk;4x<%019x|R=)_b*@7WM9%T)C|Ly zo^6KzL}9S$8fvZ~YEHJ&o|SP0UL7*lRgpxNQ$X9OZnh3J-)PDt(aE{fzxvzg&ou0+ z*W#ukm3|)b07psdY5VYFs1lSlPalJ(;K-H-JGH?GHk+N67pob4f~RRS zQFvue!~YF)$!ddT-@~{m<&^+!YCJqP{KW5^B-sWndm19Wm5)yd0_`Tn|}yIP7-hYk(HQqt(m+uU#{X1VBau*G__ zxoz_W)ci6(=;c@!97Sf6#N@W(>8dWw+BGVGd0{gK8Na*UUDh*{c8945B!nJ=e$D2q zje1z+1qRfY-%mg-u`aZgjCzw=Rbgk7sJ-(qpJ2@#XwB5k%(Km+Jo(V<8~1d0wEYHG zeTbrl=OmNE$$5CO3eaUDu!h(<0!w_up^{M|7hUBUh2lmN=Rjy`xr|7#{H>RmV9}g6 z8snvnVm8oE-eHHlDs=iZXf&DiK&-VD<>QVjG-OVILF~DrRrSa`?OPg4E0)MOtf??R zn0)P#NPZSW#yM3+<1sassjB0pOSLoyAv43k>*yYYS5rC?K!5LanXbRaz zqA#|c7Jvp!#S=gE0)~{2Bs_ZkS~A(KAee;NCNc5s%BnYzX>oj}Nl$MMwEAYjt%d(l zY3pECM;8%6^~gO8`|GX}!S)O)S3LnS#uk|CTniE;3E&S~0qD5vZkZ?K=`wsfhrQM) zKAt5FQYWapah80XBN}dy{Qg+bNC78O?lt8$av3_|^|ePxXn@={cM|^c@D%1Nk@OMf zc{w*dISBb^MsOxH?xBzEH-*TXy~LjJ&b=>c$?S5*5bi%WmG4L$S6ahwcOGhZ@Sg!6 zuu3!0XNtH*`)2Gp2Z5Ax>*GiC79s`h$8V_Yar~MWbe<6sflV3Y%65?m|3&@63$n-t z?UNo&`ASZc^;sc&!HAG3m_FDD{YgK(qa6pD9dP&W>r?Rt=3?aG)8x15wS@<6u>Xnf zZCx@b7Nc(2f^ffzOp|1RbzwGftpF+`b4>ZyJ3j77Nd4^GfjspdS_s2>Y$)%vA@hin zw3D%3mezAvju2t*&fy9P`(%%T-u+0m=I!>AbIHNlJL~2Ihf^5!u`%0XW0(nzad@$z z^0KyOg;ReM4@7#6o`4Y3FC;T4EWG=&+I?KaUP?J1pAJQ{jqOeS0686u?tI0RZU#+sxr)W^8TD@LxUC zfBZ!k>T;2|9BAE`Z}we-37xp8pyXVSzo`m~ zQiW`Jlja1#5dJ>&(~&ivhL#rA@ses@QLG{$7%$fnNOnIz%zMhoO{br85Qu?i5)Cm$ zYs#Lcp99d|or3mr4>xcTVG*G0K=zTzSjFf~Ls}gT-{nhwiGoE6>$fVRbf(GM^B%G$qs z*i(xPK(1zP2z*Z-9)T8wmmI3uHKIAa>lFOfI9mpgEZKjCywN?H10KBF=8}K}n&nFy zAsoX;0;dI{t_?ag884Y%ckN#$Rr|{DosdPb$&YM^>7P1_J(lusYTErR+BNfZxNZni zgqD^sQr+6mZ;(2HLIL}u-0dp1tQL+BD9pu1!?3nX7B+u5LQP8zezyO9RkMm{f+kR8 zj{{t4QQ5%{&yaTd*N%0n*ava8M$7hU6|uduLZG{jYa4SCC-4g0=Wu_ z2iYh|a-wReHdmcYZs}u1yhv;?Cve5=LtWq4L}z#|36Ju$Qd~ud7rghANbLs4qygBF7X`pnm`Yi_W@ zL+VYNq;3qgYUK!*URS=xvf4;)X2KXlv|jl7GywnS3pp`k z7Nce$2dfN?rOEt~?CNGjZCk7gikP)es;P}v$8_Rmd;Nlm+VO*c`6SEO$iUZf9iEtp zVg%F#nrWq>NR!&2l^<1MT{2Rp%E8ca-o%Eg8jTk@X1dJpD`RY!#GZ>S7T+!2!c__SA{m?r!ncrEhalJXY`La9w6OJMf*{C_3QRBTgKD9GMl6Fq?R&^OZ*S{Nh64; zE=UcnCo}rlp^O|RiUaux=(YX!hh2y3Sjc)sj>eh0d}^4(eJ^e>Ovch2kb*r&y1srS z3WxNBY~mfqMGz>M!+Ni|O_faxKj{K*?)2^bGH&3uwG;?dy-!Q~MQTyHgs0`>1&LFc zP7A2_YCBP`(RQ$i4RfN%s!)E{2~TK{&T3JunT8r`(pxm$!}130ogJvWcBCpTH7T|Y zbD{W{T-iroN9xt7%n)nprFzP;k`}}8JY=|5>mM=)eXOy5!>0EH@68D@_)Z93J;?Wrzu{s$bS>6{((&(vf=7V!+rB+2KKd{2oe-C z@+MXD_kl7A*pV6XUPGdoZb<29SKX45FFWz`ji1(ZCeFG=dDZ1nT8`Bzg*!h4=q=_d zO1?4TWd)Ej8@uMQDmCf+hQ!nVnX-hZp8|I^5kY%u(S;t(A+U-OOa`<2Au>vNk5-+N z>f~@fBdw}x+TUslHR>%}hAD@Hlx89~MTOlQ)0QnY&A zNlNQOU_@b@f~Ayt2*X0XM)DE`E?~oHd}hN*OzX+K7N_aVPi-N&z3Iszg8+Yg97U^vLkXi!JJ~|Z{)?DydQNx{c*gewQ9RglcpW+qEFuM>egVfCheb%6^jmK zy7X;-W1iJ;M1vH{(x-Ui_@73tzOHnhYyNh}ViY~G<%o%vpYP;jEL~p%vv0rnsuFwU zqkjQ_<2>c=fhKLMW$5XLPr34^&N#s)Elbo77MmWnGv_L}lf8yenPzrH+E*$TpJI6` zXD)Eb=@lm&>A@a@_iM=>9kVr!gwxjas$DNSX{z^YUr_&@t&uA4ZW9h10O*7R{J*wF zM`I@^a~o5~|FB@A%DT-uCz7|W-gocJJ$JiWmq?Z3itJxWQN5@{8LP;^IStx$kfcWLPurgI(b|ad%m;-eZRTGn>hhk@hoBqyRzaJ9H&2ym=B*`+et$-niqn{yg z;mokR0;dlm z@N&i?m%Rft(wkN}b`+*GbXfyySgri@Gq$nc^fdBo#H}lwX{wwRGJmJj^9uZ#mpfs3uOSv*f&lpAzg4Ri+ENK2w zzQB!^=9_|Xbqmqo+Epa0Vu5;kN65v>cY=}I17ygipou8Av^JWy3pvc+Hsjn?q7^Td zkf#0#Q2U}5)n+@<$uFn2D?u5ux0G!4A(|Lf&MKOU=nvxax6tugmCPvl$@?xM+`A%p z>u0sAt}XkXL%7yg$jl}htXf*0zBa+WyHH@%u$V_piJ%MdVbDOhN?}nqVxSPX`XebL z!{5d8#3(*7>jWXwYG%}cGV<5z%xFl~|7!26qvFb%eIdBJySqbhNC?(Q6Wk%e-QBfu zcMt9m+!_gPL4qW>2iM>dZqIjTzM08<_r3Mr-?w|6)BCKwf3^2Ht5%&_yMA3|vu#2{ z6Pyu8R~0K`f#c45q4Jjby0!K2%$&MK&FKY4F1gnHagaE1G&Y>C?x)2m38AB$fnEV7 zNk99ShReeU(^s5?6PYC#VQwoHON;S~_O018p;7qGC4GaJLr&ZjY(q(aXCYGd2MUJs zB5@SEK0lH4&|qy&-3jr*=Vsq^Kex5V|7bZ*k(q!%hgM zXx+<+33Yb!sq4*1N69EIlg4}0A%nqpO9p%0={Yyc8?4^ELm~lAW7|@m`=Z8zkX%Z*4mlDuMKmk8J?h|(i>Si zzOSdoHJ_1O#NrSM(hA=Fj2*0+Q2L%G2bp5Y>%;UMb7oT={mG7iV(Dg2h&3*&eBlQ4 z)qG%_J{a|#wxm)@`FJEgMD{bCGyGeusERahvDHc&V|BHZf&;w} zKY$`|d9}%X+vU3PoN#urzP~p2tdEXtxmnMD$@R@Q*F#$*qfZns4lT0pg782O0cfw| zkL&C4v(k2Da)-sc&Cb#OljT;g{ZZ%(dHx8H=Z^)+k^uQH`dcS^2WJjrdneQ1Q~dmI zY6OU4MklVxLsHxXtie5pB{IM*6+KWDy5On#U?K!LN7{1VN{UF7ZVgMZ3dZf>e@1gpdxLXWjS_q;cSU6W^R-l$c zxoPHoryPg^26RZKGkMDJA{7I{Z$9a7BONE%7qbc6PqCM8Y$kC8q7w?2!cnG?3RZ{a z??;=ICKsq?rhFy|#Km=Dd{tQJh&uxl^$z96`gPKsG(K_*yr0y~NbIV>T%fwT)ZE9{ zzRz5~xhtZ)nrJCC=5zrj<+w=9<5cqd@AozBuI`~4@ zls?S!%UG?oR)WTv1OJsHh(Gbq%pPNt_bT2Hot=VYzQO#f&c1hW_+OQU=<4sEY?Uz} zL}kG==NQT-vrKU>=H?OcsL;Vh@LtVFibfmXQf)DxyysyhF!}8|8ak3PA~rTV`3Fcq zWjXpOKeW~HEs9kan=3EMyTk-oqS8audj+B@oxkr;%gxN#!yn{iEHhL-ga(*LP z$z<2m6Bag>Us|W4ByBA+tclsqM@PPr+Zu(bH@&;fonlr)@S|l#7~Ijl=O}mLK8%{Y zn!uq5BlfnJl<>>r_^i-23qzt@rjK1RFM9DNXb7HMDk^~^01eJY|82r#oRXz&+543U zN|OS_y9IDXfFe}u<*Ee}cJbJ0BPx44Jsw70Vw3`uP5TQIKExihr?ht$2z&F*WnJz+(Vak(eK0#e2C6EY263XJ z<2b_nA&>r_3frxj%=G|C_YqzX1%&|_@vp2O&MuxdrvGw|kMx(EmZY(L*D9V6aP!Z>H3E<|8K)T~f_aIZ#2zn(5b1=*kmkVg~IWjUot>D|AZDpi`i zT)%#bGu#^XF0jO}-?rg=Jh51x?l*s(E|QHVw$?2!K;)qfObn1@%{LDw$VBsd+{k;r z{duRD+AYcZL*Nx;@9U}I6ZfR1Jvs)DKfpNAcSwVGgM@%J1~5S|6V{hr6n8imIJt>8 z(&tGSdN;PQw`UGtXf2noq6bSu+jZ#{u2luBiOQxVPUM zZgJF8PnbL9&3KBl1Cfzl=rsvIYMwb&`ARFUxK+!owt==U7W~HgThjA{5~Q_9$%2MY zvel!09GVJ|d}#&Bx`)(>BDjZ%d+XoZ7h4;k7171kBGjGz${L^n7$+0O8x9xr$P}_& zKu;q~dFrBD>4hyk8ytJX0yBygm6HgCQnThr&xTN zfw!mUHq3Xi_3JB|c{P%w8@f2uV1F7^>be`DWm3J&=N2duD}=8L986fAUrA0RhQAk5 zO7~J%X4{H>_u5_FF(`wHxpDn4(uM8VvsM(jK z!Nk)7@OAl)s5%l`ghfGvT#V2$#)vMwu~f0s0w7MQ_3d$fsrPi>hCmkfgSMwMM-EFJ zu)@pNLvy1oPAzdlI;CTD-u`4T=&a(&L?>oq?vARNCPDp z*}O)tXqwbhu29e^UD|n@A1|-wv1FR`PO(t%B1`)4wB>;_&LpxnRtU7Rd%{jb&rWe^ zH>7_XKYTZ{D(eNIY|t@z%8pp=N8IgteLD~|j&uGLL~yz)m8U`7M*MXQ6RTn3ef`Pl z6dSMRCi6MzNM*k>?MLj|?tZb0_w|Gy^xMkKPv(DsH5#$YlPpeU8#Q1Se_GLaJ|iw6 zuKT8&O_;pJIHU}+?|0Cc+Nb_utFzGko}}t${o-diU{)y=StGa6&%!KTpR4);CXO+3 zZ<ioSfctVPJ4`Odh5pV z%4>z*VQ2oJ*7j$QBIb2G3Zgi-wQGBiYj(sQ6H)8i1Jet+!kOfW z@6z&vfzniUJy_TjW+1>`5k`uK_;i{ejpaK0@yp@^gS>1Z3CL?p4o!&EB5HTQJlCHz z8brHVON%xDj%QI`5r(c*ATc|n1Jh_jyP7*_BD15XN@spnTc9M-8y0$Jw{*DM~zMh&q*hK0z1(KVU*c3rfhD zDr(*_u}#D{V)rbBF8A<X~?fHiR;i{f( zh44Q9yg6Xxlz9_?oL(v5y!a&CKB@}sjAxELuAU9x)O4i(q&Ki|TP;I?B8KSpbwG_$ ze?SlJi@r?KHm^ZlauG)?<(kxGpg=mMxWi(;Iws54j}gieWTtEvNd{JV}!_mv+6jl0NMw%CA)W1T$v^OO2_YhWR`nARHFb_C=Tv>5P$&bPMvJW8x&1l?ZKm= ziTy#;X~ENUDj^0+9?4P#6JD8OS{}LGWfRa|cF{QIo(-HgtX^}_b*l~Bv28t`WZj5{ zs5G=$_(UHx53EiZ<_or2avC@uSh=#EWw|m;I5suJKz!zqS15SsJq{9~*8=$x-X!@F zxW65ZZ>jzp^2eAtCmVa-7Hnq8*C==rA3~9-rX*6f2FU+!Fv!>tB zRnn!aX7Gmk%wgktCY8w6>I5n87BM|&u)XFNu#{_8NgGgP0t>$wRJ1HH65&nfH$99L zba7OTts#`a;5YpqN#)|8Dpo^C?oBOtszc#^InxO4#--#9HHLd{CT)Q%Kdhb*o6p&v zXePZknoE?=5@QC7n3Fxf{*51 z`JlSIbOq#^X0Fw$Ji$t@$)to>lWk5prZV-QtXXNo43_18?1o@n)ozaBV<0Qvswn+pDgT$*e`t9zuVf+ z(j3xh5nU>NRV@CFZmFvF+vu9*;FwkCN)!=!qVSV}RcA!x$Z4E>#hiV5te%kzM#mo^ks_cK-lGVNipE zk}Z@uQmi1VJN9{Z)7AL#aY*$}3txMiSCk~%;x*5eMCoB^nMk5rp_uem+U9T?h+`lk z=JZmi&iByRQ8zU!ZXjZgU}tF4Cjn~Itut@Dgxh9&f?F^fEnERDYi`pI58dV5fXb$e zO;azd#!nYmUFma$Mj}LGytg;L&FmsgCSk6dyC#qcA}cH(#`3STAh!0aoGzNb3NNYS z%Dhyhs41{qrpkP|@O%ZYx#94}p(%7q?1gEOjDv1oWAdRF>}WqB<2^ZrW+1ocPN0)Y zOcHfppFqNeBLzeL{j*9_js5Ipc$l%Vh{alaWY|`>w1{j9xC=Z@^J13a-68cW&AJ-u zclV<-JqljRJON zO&$+hw7p9rQOS=ahn9>hw z2Cq0@klOJ}v%N%!zE>&os~=duetw%Nxsk?Nd*15vrJ}Y$c(9>+qSgJRAH747l|k5D zcbMM;lK;nAVIRdRpWSjFk)NK`8TsB@14pVP)31ugc{^G`wD8(r!7L9T1)E`YT^l=I z$HRRUV)`{-(%tU4Y!N)FnkOg1?`0u$F(DSqmsKjHo5OL_W#*#;ct>|Jrdij+adzOoUEg9qY=VLq=KYdZN~n zUq(-8u`TV!yr^Te?uuXXU|wVJF)uN|yG5gEXSaR@$=$h8`syHpCfO^=waPC93*1@0 zjL6G=VrrTEgMu3u`kZT)p<|8NJ*;L0YQe^hW>=22S#pqt0^nzSe~{us^gW~pV19)J z<&LcpW|?>@8|T3)ymsF)-U!TF^EpQ$Ri0N_K>n}_3f$Y4eLUu(TCM2|klsj>=@!GW^T#cI z)9*4KE<9u_7h{~&ZsLkH+G!_F6wcj?gUrd$HdWI z=1*1~TO!sKNmNmHC6(L6fQKtnx7mFk!o*#$o7j`x`dXKVRE692`P>F(uzPCR=^A&w zFl35_8T#3e7G)8*v;&F&Q^M?SY&{DB`(M2sWdNgL z(~TuvX>kaVHx;}O$}1^G7y5^s#a?|QaPpozloWP{$+VRjQB}?*J0I-$>*|j)c@L|j zNYtyM-;l3EcOIDktjxLFXnvS0P+aqMeLlZmkEly#{dN#I7$VY(u6MM(Qc@7L?lo#D zAMlA*(>=N5q`j4<^t)f_yg^o23$iRX$2AT#{e;&5cdY1aae-yn1|Z>;f_Jpp0L*ZS z=-^C&F7GjyAiJ5MqBp4$P*+DN2NiN6A-Jlsj7w5Jf3XZ7*3{_Y;lgX}d@#LlDX$yW zH&d88h%N(eW9x`EANVXjs82uRbOMx;A1kW&fKYSLEDu(u%%N2k?7}mq(#?#TZcw5k zd}g&K055RFmkj81wQ^ioPqmmeaQP2WPexGzL8{3|hld@OvWw zF+FR-Q*j4XlJZQ`!d8!Jhu2P*9Z0F^u9B{jlA+|;tQ+LpJ9$A_{OJ>E>|w={%`M+~ zFjG9or2Jn@x^o6kzx zg3O-rF-vkXt9ERAAZz91!=uIYL%W%Oo0cg4AaBGCtvo;c){wbgk%PV;d{&1v6 zpMI#$*EYUY-nNtiP_kWAu(pg1{KClH4N?p9sSptiIXx^orVVKKoabJAJ3Y^~H z#gjaqrVE)7t&}YrU*8%GF0+hB5qg=24Al&kjp+_%Wkx*>$PXJj&=4px)>aqTDQ6^g zqQ!SP3gig3ggTRz%rap7xT&t$CHnzw)3ty_a+!m%1GF>)FSD@a8G42y|KliJdg4uo zcyoV_3WMlJbhtTICqDzuJbuQcoJ3I4!w{{h1>RV=iD$G4C@vD`?Xzh?S-waAUN2Su zeu?L!3l%D_e)W_H{cC{?@?SGv4>1`J?Ac~OVNgURq21_Tl_Wc|f^g3tfhfH=VTA&HHU4OmH}HkTpPXIw(aw)`h;(Ny`%kby<=^-O6dSP zzb5-o?&0o_5T{HRG2i?~j3k+z}O9@gy-bv@hExZ?X!`>OFZL<0T zpZds+5U3mPTEL|wJA;yrU(!CKn{LeMHkA9m@6=c$88>}L>)>lgHzXAEPg}+ahkH8! z3DCqqLOB1nWoD2HB5LoAZ2lElY)l!kTjIn7ox)^ZHg9>k}@M2*nYjq#?ae(6-bSWTRbJm=2F-!E@jK1 zuEQ7+F(1!koCd`4wPCL=aPrg7zn761n~0W4T_R{-LurO)`F zRn=+)$8g)SbDR6#d~|=_=->w;eYP-1EoETijpX%e!?rB=4K2u)E66sXhXFK=Njid_$l%1$5{Ye^0e)kQ3|m8rCr?8p89bI;CPqpm z8h`164{&$S4jfSOxcLTg$N@zB*c_T15!)NjEiv>@Zq)XH*5-hPU850LUfQG}wydLr0FZ4^(XD4j`C^)uh-#a0w?!Vt*p#r^qVA7Qs`Bb| zCw}&G&uU=zMRR@+#Qpo{Ka|jUEB|)^e=qCtZzQCV2MN*trO3yxz`xf!_$#m*vey4o z^@Cp}{hFlyPg!b6{~=}lukc@K(SO3Jgnx(sN|OFn#IJm^KSi)e{1HC;3qj;p_^+gr zKjEg*zr%l_nEWc?@3e?N@la4ZGEh+eAW8fR|9gb`ukZ}HzrcTwSO1FsHC*^7UQY3k zyZ?9a@K^A!KIETZC&S;tfAc7R#s6xz|HO-#{Eq*d6@M!a2T?C5C{)N78ls^*=D%J2 E7aS_j9{>OV diff --git a/docs/source/qualifiers.md b/docs/source/qualifiers.md new file mode 100644 index 0000000..8c646ae --- /dev/null +++ b/docs/source/qualifiers.md @@ -0,0 +1,96 @@ +# Definition of qualifier classes used in `clinlp` + +This page describes the definitions of qualifier classes for Dutch clinical text we use in `clinlp`. + +## Introduction + +When processing clinical documents (e.g., doctor’s notes, discharge letters), detecting qualifiers (e.g., `absent`, `historical`, `non-patient`, `increasing`, `decreasing`, `severe`, `light`, etc.) follows the matching of concepts (e.g., diagnoses, symptoms, procedures). In this document we primarily use the term “qualifier”, although the terms “context”, “meta-annotation”, and “modifier” are also used to denote the same concept. + +A consensus on potential qualifier classes, along with **clear** definitions of the qualifiers they encompass, is needed to develop accurate algorithms for detecting them. Despite some shared intuitive understanding of recognizing, for instance, a _negation_ in a sentence, there are numerous cases where intuition simply falls short. In practice this impedes manual annotation of gold standards, used for training and evaluating algorithms. In turn the resulting annotations (with Kappa-Cohen as low as 0.5) lead to a difficult target for supervised machine learning models. A standardized classification of qualifiers, as proposed here, will hopefully advance both research and clinical implementation of NLP (Natural Language Processing) algorithms. This page is the result of some deliberation among various researchers and developers working with clinical text, but it is not necessarily definitive. We welcome feedback and suggestions for improvement. + +For the proposed we took the Context Algorithm (Harkema, 2009) as a starting point, both because this is an influential paper, and because a corresponding Dutch corpus is available (Afzal, 2014). There are already some trained models available that can for a large part be re-used. We will here describe three qualifier classes: `Presence`, `Temporality`, and `Experiencer`, including some definitions, issues to resolve, and illustrative examples. These classes can be further de-aggregated at a later stage, and other classes may follow later as well. + +**Qualifier classes** are denoted by boldface, with the _qualifiers_ (the mutually exclusive values a **qualifier class** can assume) italicized. + +## Presence + +| **absent** / **negated** | **uncertain** | **present** / **affirmed** (default) | +|--------------------------|---------------|--------------------------------------| +| Concepts that are explicitly or implicitly described as absent in a patient | Whether the concept was absent or present is not asserted with very high certainty | Concepts that are explicitly or implicitly described as present in a patient | + +Assessing whether some concept was present or absent is one of the most important parts of a clinician’s job. Whether something is present or occurred in the real world is knowable in principle, but in the clinical world, such assertions are rarely made with complete certainty. This is already implied by the uncertainty at the core of the clinical reasoning process, but in clinical text the uncertainty is often made explicit by means of hedging. It’s therefore important to note that when we are extracting concepts from medical text, it’s very hard to make direct assertions about the real world, but we are limited to recognizing probability statements made by clinicians. + +The **presence** class therefore captures whether a concept is present in three qualifiers. The _present_ and _absent_ qualifiers are used when the clinician assesses a concept as being present (or absent) with very high probability, extending beyond reasonable doubt. When neither presence nor absence is definitively asserted, the _uncertain_ qualifier applies. This qualifier therefore ranges from very unlikely to very likely. + +The default qualifier for **presence** is _present_. When the text does not indicate absence or uncertainty of a concept, we assume the writer intended to convey its presence. + +### To resolve + +- In future work, the uncertain qualifier may be further split up, for instance, into a negative uncertain (i.e., unlikely) qualifier and a positive uncertain (i.e., likely) qualifier. Or perhaps an ‘uncertain uncertain’ qualifier in addition to those, for 50/50 cases. +- The exact threshold for absent and present should be further defined. What probability cutoff should be regarded as ‘beyond a reasonable doubt’? We can set this threshold at two standard deviations (`<0.025`, `>0.975`), but it would be even better to do some small empiric study with clinicians, to find out where each trigger term should go. Consider for example edge cases such as: very insignificant, very likely, subclinical, etc. + +### Examples + +| Example | Qualifier | +| ------- | --------- | +| Rechtszijdig fraai de middelste neusgang te visualiseren, vrij van **poliepen**. | Absent | +| Tractus circulatorius: geen **pijn op de borst**. | Absent | +| Een **acuut coronair syndroom** werd uitgesloten. | Absent | +| Werkdiagnose **maagklachten** bij diclofenac gebruik en weinig intake. | Uncertain | +| Waarschijnlijk **hematurie** bij reeds gepasseerde niersteen. | Uncertain | +| Dat er toen **bradypacing** is geweest valt niet uit te sluiten. | Uncertain | +| In juni 2023 **longembolie** waarvoor rivaroxaban met nu asymptomatische progressie. | Present | +| **PTSS** en **recidiverende depressie** in VG. | Present | +| Status na mild **delier**, heden wel slaperig. | Present | + +## Temporality + +| **historical** | **current** (default) | **future** | +|----------------|-----------------------|------------| +| Concepts that were applicable at some point in history, but not in the last two weeks. | Concepts that were applicable in the last two weeks (potentially starting before that) up to and including the present moment. | Concepts that are potentially applicable in a future scenario. | + +The **temporality** class places concepts in a temporal framework, ranging from past to future, relative to the document date. The _historical_ and _current_ qualifiers distinguish between concepts that were applicable in the past, versus concepts that are applicable in the present. The exact cutoff between _historical_ and _current_ is problem-specific and therefore hard to definitively establish in a general sense. In a discharge summary, everything that happened before the admission period could be considered _historical_, which can easily range up to months, while during a GP (General Practitioner) visit, events from a few days prior might be considered _historical_. For the general case, we see no reason to divert from the threshold of two weeks in the original Context paper (Harkema et al., 2009). Note that the _current_ qualifier also applies when the concept is applicable in the last two weeks, but already started before that. + +The _future_ qualifier is applicable when a concept is described in a future scenario, for instance when describing the risk of developing a condition at a later stage, or when describing a procedure that will take place later. + +### To resolve + +- A way to dynamically define the threshold for _historical_ and _current_, so that a cutoff can be established for each problem. In future work, we might map each concept to a timedelta (e.g., -1 year, -14 days, +5 days), but that does not fit the current qualifier framework very well. Also, it seems quite a hard problem. + +### Examples + +| Example | Qualifier | +| ------- | --------- | +| Zwanger, meerdere **miskramen** in de voorgeschiedenis. | _historical_ | +| Progressieve autonome functiestoornissen bij eerdere **dermoidcyste**. | _historical_ | +| Als tiener een **osteotomiecorrectie** beiderzijds gehad. | _historical_ | +| Echocardiografisch zagen wij geen aanwijzingen voor een **hypertrofe cardiomyopathie**. | _current_ | +| Al langer bestaande **bloeddrukproblematiek**. | _current_ | +| CT thorax: **laesie** rechter onderkwab bevestigd. | _current_ | +| Conservatieve maatregelen ter preventie van **pulmonale infectie** zijn herbesproken. | _future_ | +| Mocht hij **koorts** en/of **tachycardie** ontwikkelen, dan contact opnemen met dienstdoende arts. | _future_ | +| Wordt nu opgenomen middels IBS ter afwending van **suïcide**. | _future_ | + +## Experiencer + +| **patient** (default) | **family** | **other** | +|-----------------------|------------|-----------| +| Concepts applicable to the patient related to the current document. | Concepts not applicable to the patient, but to someone with a genetic relationship to the patient. | Concepts not applicable to the patient, but to someone without a genetic relationship to the patient. | + +The **experiencer** qualifier distinguishes between concepts that apply to the _patient_, to those that apply to _family_ members with a genetic relationship to the patient, and _other_ individuals with no genetic relationship to the patient (e.g. acquaintances). Clinical documents are typically obtained from electronic health records, where the relation between a document and a patient is explicit. Since a patient is a well separated entity, there is usually little ambiguity which class applies. If a concept applies to both the patient and another person, the patient label should be selected. + +### Examples + +| Example | Qualifier | +| ------------------------------------------------------------ | --------- | +| Behandeling in WKZ ivm **diabetes** beeindigd. | _patient_ | +| Pte wil geen medicatie tegen **parkinson** ivm slechte ervaringen broer | _patient_ | +| X-enkel rechts: **schuine fractuur laterale malleolus** | _patient_ | +| Familieanamnese omvat: **ADD**/**ADHD**: broer | _family_ | +| Moederszijde: voor zover bekend geen **kanker** | _family_ | +| 2. **Covid** positieve huisgenoot | _other_ | + +## References + +- Afzal, Z., Pons, E., Kang, N. et al. ContextD: an algorithm to identify contextual properties of medical terms in a Dutch clinical corpus. BMC Bioinformatics 15, 373 (2014). [https://doi.org/10.1186/s12859-014-0373-3](https://doi.org/10.1186/s12859-014-0373-3) +- Harkema H, Dowling JN, Thornblade T, Chapman WW. ConText: an algorithm for determining negation, experiencer, and temporal status from clinical reports. J Biomed Inform. 2009 Oct;42(5):839-51. doi: 10.1016/j.jbi.2009.05.002. Epub 2009 May 10. PMID: 19435614; PMCID: PMC2757457. From ddaa71315541aa97b9d7fca22b4f01195b862d17 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 17:13:03 +0200 Subject: [PATCH 19/30] Fix admonitions --- docs/source/components.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/components.md b/docs/source/components.md index 7d27a1e..75e65b4 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -92,12 +92,12 @@ entity_matcher.load_concepts(concepts) ``` ```{admonition} Spans vs ents -:tip: +:class: tip `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. ``` ```{admonition} Using spaCy components directly -:tip: +:class: tip The `clinlp_rule_based_entity_matcher` component wraps the spaCy `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: from clinlp.ie import SPAN_KEY From 872843a52d791df59135395e6c64f4e9a78ed5df Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Thu, 13 Jun 2024 17:17:00 +0200 Subject: [PATCH 20/30] Fix broken links --- CONTRIBUTING.md | 2 +- docs/source/introduction.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecd0441..4781228 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -302,7 +302,7 @@ We use type hints throughout the codebase, for both functions and classes. This ### Documentation -We like our code to be well documented. The documentation can be found in the `docs` directory. If you are making changes to the codebase, please make sure to update the documentation accordingly. If you are adding new components, please add them to the [component library](TODO), and following the existing structure. +We like our code to be well documented. The documentation can be found in the `docs` directory. If you are making changes to the codebase, please make sure to update the documentation accordingly. If you are adding new components, please add them to the [component library](https://clinlp.readthedocs.io/en/latest/components.html), and following the existing structure. #### Docstrings diff --git a/docs/source/introduction.md b/docs/source/introduction.md index ff8837d..88e03fa 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -32,7 +32,7 @@ Components can be built by anyone from the Dutch clinical NLP field, typically a Currently, `clinlp` mainly includes components used for information extraction, such as tokenizing, detecting sentence boundaries, normalizing text, detecting entities, and detecting qualifiers (e.g. negation, uncertainty). We are now extending the library with more components, both for different tasks (e.g. entity linking, summarization) and different methods for solving the tasks (e.g. a transformer-based entity recognizer). -We prefer components to work out of the box, but to be highly customizable. For instance, our implementation of the [Context Algorithm](TODO) has a set of built in rules for for qualifying entities with Presence, Temporality and Experiencer properties. However, both the types of qualifiers and the rules can easily be modified or replaced by the user. This way, the components can be used in a wide variety of use cases, and no user is forced to use a one-size-fits-all solution. +We prefer components to work out of the box, but to be highly customizable. For instance, our implementation of the [Context Algorithm](components.md#clinlp_context_algorithm) has a set of built in rules for for qualifying entities with Presence, Temporality and Experiencer properties. However, both the types of qualifiers and the rules can easily be modified or replaced by the user. This way, the components can be used in a wide variety of use cases, and no user is forced to use a one-size-fits-all solution. ```{admonition} Important :class: important From 9711d8afff18cc694ebdc37a7ec7de31307ef98f Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Mon, 17 Jun 2024 13:04:57 +0200 Subject: [PATCH 21/30] Add pr template --- .github/PR_TEMPLATE/pull_request_template.md | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/PR_TEMPLATE/pull_request_template.md diff --git a/.github/PR_TEMPLATE/pull_request_template.md b/.github/PR_TEMPLATE/pull_request_template.md new file mode 100644 index 0000000..4edb0b2 --- /dev/null +++ b/.github/PR_TEMPLATE/pull_request_template.md @@ -0,0 +1,23 @@ +--- +name: Pull Request +about: Create a pull request to make a change to the code +title: '' +labels: bug +assignees: '' + +--- + +**Describe the change** +Please provide a clear and concise description and motivation of the proposed change. + +**Linked issue** +If this pull request is related to an issue, please provide a link to the issue here. + +**I have checked my changes are in line with the [Coding Standards](https://clinlp.readthedocs.io/en/latest/contributing.html#coding-standards)** +Yes/no + +**I have added my changes to the `CHANGELOG.md` file** +Yes/no + +**Any other relevant information** +Add any other context about the pull request here. From c4d361e2687200ce9936354a8e39aff04aaf36ef Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Mon, 17 Jun 2024 13:05:20 +0200 Subject: [PATCH 22/30] Text editing improvements --- CHANGELOG.md | 4 +- README.md | 11 +++-- docs/source/citing.md | 2 +- docs/source/components.md | 63 ++++++++++++++++++--------- docs/source/getting_started.md | 17 +++++--- docs/source/index.md | 2 +- docs/source/installation.md | 2 +- docs/source/introduction.md | 52 ++++++++++++++--------- docs/source/qualifiers.md | 78 +++++++++++++++++----------------- 9 files changed, 135 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c800ecc..a0d3614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * :exclamation: `clinlp` now stores entities in `doc.spans['ents']` rather than `doc.ents`, allowing for overlap * :exclamation: Overlap in entities found by the entity matcher is no longer resolved by default (replacing old behavior). To remove overlap, pass `resolve_overlap=True`. * Refactored tests to use `pytest` best practices -* Changed `clinlp_autocomponent` to `clinlp_component`, which automatically registers your component with spaCy +* Changed `clinlp_autocomponent` to `clinlp_component`, which automatically registers your component with `spaCy` * Codebase and linting improvements * Renamed the `other_threshold` config to `family_threshold` in the `clinlp_experiencer_transformer` component @@ -143,7 +143,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* Remove a default spacy abbreviation (`ts.`) +* Remove a default `spaCy` abbreviation (`ts.`) * Option for max scope on qualifier rules, limiting the number of tokens it applies to * A transformer based pipeline for negation detection (`clinlp_negation_transformer`) * A base class `QualifierDetector` for qualifier detection diff --git a/README.md b/README.md index bf8f764..6048a2b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ * :star: NLP tools and algorithms for clinical text written in Dutch -* :triangular_ruler: Organization in a standardized, flexible and highly usable framework using spaCy +* :triangular_ruler: Organized in a standardized but flexible framework using `spaCy` * :rocket: Production-ready, performant, well-tested and easy to use @@ -24,7 +24,7 @@ ## Contact -If you have questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html)! +If you have questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](https://clinlp.readthedocs.io/en/latest/contributing.html#contact)! ## Getting started @@ -103,11 +103,14 @@ for ent in doc.spans["ents"]: ## Documentation -The full documentation can be found at [clinlp.readthedocs.io](https://clinlp.readthedocs.io). +The full documentation can be found at [https://clinlp.readthedocs.io](https://clinlp.readthedocs.io). ## Links +* [Documentation](https://clinlp.readthedocs.io) * [Contributing guidelines](https://clinlp.readthedocs.io/en/latest/contributing.html) +* [API](https://clinlp.readthedocs.io/en/latest/api/api.html) * [`clinlp` development roadmap](https://github.com/orgs/umcu/projects/3) * [Create an issue](https://github.com/umcu/clinlp/issues/new/choose) -* TODO... \ No newline at end of file +* [Cite `clinlp`](https://clinlp.readthedocs.io/en/latest/citing.html) +* [Changelog](https://clinlp.readthedocs.io/en/latest/changelog.html) diff --git a/docs/source/citing.md b/docs/source/citing.md index 62834c9..eb42415 100644 --- a/docs/source/citing.md +++ b/docs/source/citing.md @@ -1,6 +1,6 @@ # Citing -If you use `clinlp` in your research, please cite our work. This helps making clinlp findable and accessible to others. You can find the appropriate citation (APA, BibTex, etc.) by clicking the Zenodo button below. This should always point you to the current latest release: +If you use `clinlp` in your research, please cite our work. This helps making `clinlp` findable and accessible to others. You can find the appropriate citation (APA, BibTex, etc.) by clicking the Zenodo button below. This should always point you to the current latest release: [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10528055.svg)](https://doi.org/10.5281/zenodo.10528055) diff --git a/docs/source/components.md b/docs/source/components.md index 75e65b4..8cfcbbf 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -9,15 +9,20 @@ This page describes the various pipeline components that `clinlp` offers, along | property | value | | --- | --- | | name | `clinlp` | -| class | `clinlp.language.Clinlp` | +| class | [clinlp.language.Clinlp](clinlp.language.Clinlp) | | example | `nlp = spacy.blank("clinlp")` | | requires | `-` | | assigns | `-` | | config options | `-` | -The `clinlp` language class is an instantiation of the `spaCy` `Language` class, with some customizations for clinical text. It contains the default settings for Dutch clinical text, such as rules for tokenizing, abbreviations and units. +The `clinlp` language class is an instantiation of the `spaCy` `Language` class, with some customizations for clinical text. It contains the default settings for Dutch clinical text, such as rules for tokenizing, abbreviations and units. Creating an instance of the `clinlp` language class is usually the first step in setting up a pipeline for clinical text processing. -The tokenizer employs some custom rule based logic, including: +```{admonition} Note +:class: tip +Note that `clinlp` does not start from a pre-trained `spaCy` model, but from a blank model. This is because `spaCy` only provides models and components pre-trained on general Dutch text, which typically perform poorly on the domain-specific language of clinical text. Although, you are always free to to add pre-trained components from a general Dutch model to the pipeline if needed. +``` + +The included tokenizer employs some custom rule based logic, including: - Clinical text-specific logic for splitting punctuation, units, dosages (e.g. `20mg/dag` :arrow_right: `20` `mg` `/` `dag`) - Custom lists of abbreviations, units (e.g. `pt.`, `zn.`, `mmHg`) @@ -30,11 +35,11 @@ The tokenizer employs some custom rule based logic, including: | property | value | | --- | --- | | name | `clinlp_normalizer` | -| class | `clinlp.normalize.Normalizer` | +| class | [clinlp.normalizer.Normalizer](clinlp.normalizer.Normalizer) | | example | `nlp.add_pipe("clinlp_normalizer")` | | requires | `-` | | assigns | `token.norm` | -| config options | `{lowercase=True, map_non_ascii=True}` | +| config options | `lowercase = True`
`map_non_ascii = True` | The normalizer sets the `Token.norm` attribute, which can be used by further components (entity matching, qualification). It currently has two options (enabled by default): @@ -48,11 +53,11 @@ Note that this component only has effect when explicitly configuring successor c | property | value | | --- | --- | | name | `clinlp_sentencizer` | -| class | `clinlp.sentencize.Sentencizer` | +| class | [clinlp.sentencizer.Sentencizer](clinlp.sentencizer.Sentencizer) | | example | `nlp.add_pipe("clinlp_sentencizer")` | | requires | `-` | | assigns | `token.is_sent_start`, `doc.sents` | -| config options | `{"sent_end_chars": [".", "!", "?", "\n", "\r"], "sent_start_punct": ["-", "*", "[", "("],}` | +| config options | `sent_end_chars = [".", "!", "?", "\n", "\r"]`
`sent_start_punct = ["-", "*", "[", "("]` | The sentencizer is a rule-based sentence boundary detector. It is designed to detect sentence boundaries in clinical text, whenever a character that demarks a sentence ending is matched (e.g. newline, period, question mark). The next sentence is started whenever an alpha character or a character in `sent_start_punct` is encountered. This prevents e.g. sentences ending in `...` to be classified as three separate sentences. The sentencizer correctly detects items in enumerations (e.g. starting with `-` or `*`). @@ -63,11 +68,11 @@ The sentencizer is a rule-based sentence boundary detector. It is designed to de | property | value | | --- | --- | | name | `clinlp_rule_based_entity_matcher` | -| class | `clinlp.ie.entity.RuleBasedEntityMatcher` | +| class | [clinlp.ie.entity.RuleBasedEntityMatcher](clinlp.ie.entity.RuleBasedEntityMatcher) | | example | `nlp.add_pipe("clinlp_rule_based_entity_matcher")` | | requires | `-` | | assigns | `doc.spans['ents']` | -| config options | `{"attr": "TEXT", "proximity": 0, "fuzzy": 0, "fuzzy_min_len": 0, "pseudo": False}` | +| config options | `attr = "TEXT"`
`proximity = 0`
`fuzzy = 0`
`fuzzy_min_len = 0`
`pseudo = False` | The `clinlp_rule_based_entity_matcher` component can be used for matching entities in text, based on a dictionary of known concepts and their terms/synonyms. It includes options for matching on different token attributes, proximity matching, fuzzy matching and unmatching pseudo/negative terms. @@ -96,9 +101,9 @@ entity_matcher.load_concepts(concepts) `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. ``` -```{admonition} Using spaCy components directly +```{admonition} Using `spaCy` components directly :class: tip -The `clinlp_rule_based_entity_matcher` component wraps the spaCy `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: +The `clinlp_rule_based_entity_matcher` component wraps the `spaCy` `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: from clinlp.ie import SPAN_KEY ruler = nlp.add_pipe('span_ruler', config={'span_key': SPAN_KEY}) @@ -178,7 +183,7 @@ In this case `prematuur` will be matched, but not in the context of `prematuur a #### `spaCy` patterns -Finally, if you need more control than literal phrases and terms as explained above, the entity matcher also accepts [spaCy patterns](https://spacy.io/usage/rule-based-matching#adding-patterns). These patterns do not respect any other configurations (like attribute, fuzzy, proximity, etc.): +Finally, if you need more control than literal phrases and terms as explained above, the entity matcher also accepts [`spaCy` patterns](https://spacy.io/usage/rule-based-matching#adding-patterns). These patterns do not respect any other configurations (like attribute, fuzzy, proximity, etc.): ```python concepts = { @@ -205,7 +210,7 @@ concepts = { #### Concept dictionary from external source -When matching entities, it is possible to load external lists of concepts (e.g. from a medical thesaurus such as UMLS) from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: +External lists of concepts (e.g. from a medical thesaurus such as UMLS) can also be loaded directly from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: | **concept** | **phrase** | **attr** | **proximity** | **fuzzy** | **fuzzy_min_len** | **pseudo** | **comment** | |--|--|--|--|--|--|--|--| @@ -242,13 +247,13 @@ Will result in the following concept dictionary: | property | value | | --- | --- | | name | `clinlp_context_algorithm` | -| class | `clinlp.ie.qualifier.context_algorithm.ContextAlgorithm` | +| class | [[clinlp.ie.qualifier.context_algorithm.ContextAlgorithm](clinlp.ie.qualifier.context_algorithm.ContextAlgorithm) | | example | `nlp.add_pipe('clinlp_context_algorithm')` | | requires | `doc.sents`, `doc.spans['ents']` | | assigns | `span._.qualifiers` | -| config options | `{'phrase_matcher_attr': "TEXT", "load_rules": True, "rules": "src/clinlp/resources/context_rules.json}` | +| config options | `phrase_matcher_attr = "TEXT"`
`load_rules = True`
`rules = "src/clinlp/resources/context_rules.json"` | -The rule-based [Context Algorithm](https://doi.org/10.1016%2Fj.jbi.2009.05.002) is fairly accurate, and quite transparent and fast. A set of rules, that checks for presence, temporality, and experiencer, is loaded by default: +The rule-based [Context Algorithm](https://doi.org/10.1016%2Fj.jbi.2009.05.002) is fairly accurate, and quite transparent and fast. A set of rules, that checks for `Presence`, `Temporality`, and `Experiencer`, is loaded by default: ```python nlp.add_pipe("clinlp_context_algorithm", config={"phrase_matcher_attr": "NORM"}) @@ -260,16 +265,21 @@ A custom set of rules, including different types of qualifiers, can easily be de cm = nlp.add_pipe("clinlp_context_algorithm", config={"rules": "/path/to/my_own_ruleset.json"}) ``` +```{admonition} Definitions of qualifiers +:class: tip +For more extensive documentation on the definitions of the qualifiers we use in `clinlp`, see the [Qualifiers](qualifiers.md) page. +``` + ### `clinlp_negation_transformer` | property | value | | --- | --- | | name | `clinlp_negation_transformer` | -| class | `clinlp.ie.qualifier.transformer.NegationTransformer` | +| class | [clinlp.ie.qualifier.transformer.NegationTransformer](clinlp.ie.qualifier.transformer.NegationTransformer) | | example | `nlp.add_pipe('clinlp_negation_transformer')` | | requires | `doc.spans['ents']` | | assigns | `span._.qualifiers` | -| config options | `{"token_window": 32, "strip_entities": True, "placeholder": None, "prob_aggregator": statistics.mean, "absence_threshold": 0.1, "presence_threshold": 0.9}` | +| config options | `token_window = 32`
`strip_entities = True`
`placeholder = None`
`prob_aggregator = statistics.mean`
`absence_threshold = 0.1`
`presence_threshold = 0.9` | The `clinlp_negation_transformer` wraps the the negation detector described in [van Es et al, 2022](https://doi.org/10.48550/arxiv.2209.00470). The underlying transformer can be found on [huggingface](https://huggingface.co/UMCU/). The negation detector is reported as more accurate than the rule-based version (see paper for details), at the cost of less transparency and additional computational cost. @@ -283,17 +293,28 @@ The component can be configured to consider a maximum number of tokens as contex The thresholds define where the cutoff for absence and presence are. If the predicted probability of presence < `absence_threshold`, entities will be qualified as `Presence.Absent`. If the predicted probability of presence > `presence_threshold`, entities will be qualified as `Presence.Present`. If the predicted probability is between these thresholds, the entity will be qualified as `Presence.Uncertain`. +```{admonition} Definitions of qualifiers +:class: tip +For more extensive documentation on the definitions of the qualifiers we use in `clinlp`, see the [Qualifiers](qualifiers.md) page. +``` + + ### `clinlp_experiencer_transformer` | property | value | | --- | --- | | name | `clinlp_experiencer_transformer` | -| class | `clinlp.ie.qualifier.transformer.ExperiencerTransformer` | +| class | [clinlp.ie.qualifier.transformer.ExperiencerTransformer](clinlp.ie.qualifier.transformer.ExperiencerTransformer) | | example | `nlp.add_pipe('clinlp_experiencer_transformer')` | | requires | `doc.spans['ents']` | | assigns | `span._.qualifiers` | -| config options | `{"token_window": 32, "strip_entities": True, "placeholder": None, "prob_aggregator": statistics.mean, "family_threshold": 0.5}` | +| config options | `token_window = 32`
`strip_entities = True`
`placeholder = None`
`prob_aggregator = statistics.mean`
`family_threshold = 0.5` | -The `clinlp_experiencer_transformer` wraps a very similar model as the [`clinlp_negation_transformer`](#clinlp_negation_transformer) component, with which it shares most of its configuration. +The `clinlp_experiencer_transformer` wraps a very similar model as the [`clinlp_negation_transformer`](#clinlp_negation_transformer) component, with which it shares most of its configuration. Additionally, it has a threshold for determining whether an entity is experienced by the patient or by a family member. If the predicted probability < `family_threshold`, the entity will be qualified as `Experiencer.Patient`. If the predicted probability > `family_threshold`, the entity will be qualified as `Experiencer.Family`. The `Experiencer.Other` qualifier is currently not implemented in this component. + +```{admonition} Definitions of qualifiers +:class: tip +For more extensive documentation on the definitions of the qualifiers we use in `clinlp`, see the [Qualifiers](qualifiers.md) page. +``` diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index 0a67073..32790fe 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -1,6 +1,6 @@ # Getting started -This guide contains some code examples to get you started with `clinlp`. Since `clinlp` is built on top of the `spaCY` framework, it's highly recommended to read [spaCy 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. +This guide contains some code examples to get you started with `clinlp`. Since `clinlp` is built on top of the `spaCy` framework, it's highly recommended to read [`spaCy` 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. ## Creating a blank model @@ -48,8 +48,7 @@ Even when using a blank model, the `Doc`, `Token` and `Span` objects already con ## Adding components - -The above model is a blank model, which means it does not contain any additional components yet. It's essentially an almost empty pipeline.Adding a component is done using: +The above model is a blank model, which means it does not contain any additional components yet. It's essentially an almost empty pipeline. Adding a component is done using: ```python nlp.add_pipe('component_name') @@ -113,7 +112,13 @@ The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, If we now process a piece of text, we can see that the entity recognizer has recognized some entities: ```python -text = "Preterme neonaat ( ``` -`clinlp` is a library for performing NLP on clinical text written in Dutch. It is designed to be a standardized framework for building, maintaining and sharing solutions for NLP tasks in the clinical domain. The library is built on top of the [`spaCy`](https://spacy.io/) library, and extends it with components that are specifically tailored to Dutch clinical text. The library is open source and welcomes contributions from the community. +`clinlp` is a Python library for performing NLP on clinical text written in Dutch. It is designed to be a standardized framework for building, maintaining and sharing solutions for NLP tasks in the clinical domain. The library is built on top of the [`spaCy`](https://spacy.io/) library, and extends it with components that are specifically tailored to Dutch clinical text. The library is open source and welcomes contributions from the community. + +## Motivation `clinlp` was motivated by the lack of standardized tools for processing clinical text in Dutch. This makes it difficult for researchers, data scientists and developers working with Dutch clinical text to build, validate and maintain NLP solutions. With `clinlp`, we aim to fill this gap. +## Principles + We organized `clinlp` around four basic principles: useful NLP tools, a standardized framework, production-ready quality, and open source collaboration. -## 1. Useful NLP tools +### 1. Useful NLP tools ```{include} ../../README.md :start-after: :end-before: ``` -There are many interesting NLP tasks in the clinical domain, like normalization, entity recognition, qualifier detection, entity linking, summarization, reasoning, and many more. In addition to that, each task can often be solved in multiple ways: using rule-based methods, classical machine learning, deep learning, transformers, or a combination of these, with trade-offs between them. +There are many interesting NLP tasks in the clinical domain, like normalization, entity recognition, qualifier detection, entity linking, summarization, reasoning, and many more. In addition to that, each task can often be solved using rule-based methods, classical machine learning, deep learning, transformers, or a combination of these, with trade-offs between them. The main idea behind `clinlp` is to build, maintain and share solutions for these NLP tasks, specifically for clinical text written in Dutch. In `clinlp`, we typically call a specific implementation for a task a "component". For instance: a rule-based sentence boundary detector, or a transformer-based negation detector. -When building new solutions, we preferably start with a component that implements a simple, rule-based solution, which can function as a baseline. Then subsequently, more sophisticated components can be added. If possible, we try to (re)use existing implementations, but if needed, building from scratch is also an option. +Currently, `clinlp` mainly includes components used for information extraction, such as tokenizing, detecting sentence boundaries, normalizing text, detecting entities, and detecting qualifiers (e.g. negation, uncertainty). The library is regularly being updated with new or improved components, both components for different tasks (e.g. entity linking, summarization) and components that use a different method for solving a task (e.g. a transformer-based entity recognizer). ```{admonition} Contributing :class: note -Components can be built by anyone from the Dutch clinical NLP field, typically a researcher, data scientist or developer who works with Dutch clinical text in daily practice. If you have a contribution in mind, please check out the [contributing](contributing) page. +Components can be built by anyone from the Dutch clinical NLP field, typically a researcher, data scientist, engineer or clinician who works with Dutch clinical text in daily practice. If you have a contribution in mind, please check out the [Contributing](contributing) page. ``` -Currently, `clinlp` mainly includes components used for information extraction, such as tokenizing, detecting sentence boundaries, normalizing text, detecting entities, and detecting qualifiers (e.g. negation, uncertainty). We are now extending the library with more components, both for different tasks (e.g. entity linking, summarization) and different methods for solving the tasks (e.g. a transformer-based entity recognizer). +When building new solutions, we preferably start with a component that implements a simple, rule-based solution, which can function as a baseline. Then subsequently, more sophisticated components can be built. If possible, we try to (re)use existing implementations, but if needed, building from scratch is also an option. We prefer components to work out of the box, but to be highly customizable. For instance, our implementation of the [Context Algorithm](components.md#clinlp_context_algorithm) has a set of built in rules for for qualifying entities with Presence, Temporality and Experiencer properties. However, both the types of qualifiers and the rules can easily be modified or replaced by the user. This way, the components can be used in a wide variety of use cases, and no user is forced to use a one-size-fits-all solution. -```{admonition} Important +```{admonition} Validating components :class: important Remember, there is no guarantee that components based on existing rules or pre-trained models also extend to your particular dataset and use case. It is always recommended to evaluate the performance of the components on your own data. @@ -42,9 +46,9 @@ Remember, there is no guarantee that components based on existing rules or pre-t In addition to functional components, `clinlp` also implements some functionality for computing metrics. This is useful for evaluating the performance of the components, and for comparing different methods for solving the same task. -An overview of all components included in `clinlp` can be found in the [components library](components). +An overview of all components included in `clinlp` can be found on the [Components](components) page. -## 2. Standardized framework +### 2. Standardized framework ```{include} ../../README.md :start-after: @@ -58,23 +62,23 @@ We use the [`spaCy`](https://spacy.io/) library as the backbone of our framework ```{admonition} Getting familiar with spaCy :class: note -It's highly recommended to read [spaCy 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. +It's highly recommended to read [`spaCy` 101: Everything you need to know (~15 minutes)](https://spacy.io/usage/spacy-101) before getting started with `clinlp`. Understanding the basic `spaCy` framework will make working with `clinlp` much easier. ``` -In addition to the `spaCy` framework, we have added some additional abstractions and interfaces that make building components easier. For instance, if you want to add a new component that detects qualifiers, it can make use of the `QualifierDetector` interface, and the `Qualifier` and `QualifierClass` classes. This way, the new component can easily be integrated in the framework, while the developer can focus on building a new solution. +In addition to the `spaCy` framework, we have added some additional abstractions and interfaces that make building components easier. For instance, if you want to add a new component that detects qualifiers, it can make use of the `QualifierDetector` abstraction, and the `Qualifier` and `QualifierClass` classes. This way, the new component can easily be integrated in the framework, while the developer can focus on building a new solution. Finally, by adopting a framework, we can easily build components that wrap a specific pre-trained model. The transformer-based qualifier detectors included in `clinlp` are good examples of this. These components wrap around pre-trained transformer models, but fit seamlessly into the `clinlp` framework. This way, we can easily add new components that use the latest and greatest in NLP research. -## 3. Production-ready quality +### 3. Production-ready quality ```{include} ../../README.md :start-after: :end-before: ``` -`clinlp` can potentially serve many types of users, from researchers, to developers, data scientists and clinicians. One thing they all have in common, is that they would like to rely on the library to work as expected. Our goal is to build a library that is of production grade quality, meaning that it is reliable, robust, and can be used in production environments. To ensure this, we employ various software development best practices, including: +`clinlp` can potentially serve many types of users, including researchers, data scientists, engineers and clinicians. One thing they all have in common, is that they would like to rely on the library to work as expected. Our goal is to build a library with the robustness and reliability required in production environments, i.e. real world environments. To ensure this, we employ various software development best practices, including: -* Proper system design by using abstractions, interfaces and design patterns (where appropriate). +* Proper system design by using abstractions, interfaces and design patterns (where appropriate) * Formatting, linting and type hints for a clean, consistent and readable codebase * Versioning and a changelog to track changes over time * Optimizations for speed and scalability @@ -83,14 +87,11 @@ Finally, by adopting a framework, we can easily build components that wrap a spe * Documentation to explain the library's principles, functionality and how to use it * Continuous deployment and frequent new releases -```{admonition} Constant improvement -:class: note We actively maintain the library, and are always looking for ways to improve it. If you have suggestions how to further increase the quality of the library, please let us know. -``` -More detail on our best practices can be found in the [coding standards](https://clinlp.readthedocs.io/en/latest/contributing.html#coding-standards) section of the contributing page. +More detail on the `clinlp` development practices can be found in the [Coding Standards](contributing.md#coding-standards) section of the contributing page. -## 4. Open source collaboration +### 4. Open source collaboration ```{include} ../../README.md :start-after: @@ -99,6 +100,15 @@ More detail on our best practices can be found in the [coding standards](https:/ `clinlp` is being built as a free and open source library, but we cannot do it alone. As an open source project, we highly welcome contributions from the community. We believe that open source collaboration is the best way to build high quality software that can be used by everyone. We encourage you to contribute to the project by reporting issues, suggesting improvements, or even submitting your own code. -In order to be transparent, we prefer to communicate through means that are open to everyone. This includes using GitHub for issue tracking, pull requests and discussions, and using the `clinlp` documentation for explaining the library's principles and functionality. We keep our [roadmap](https://clinlp.readthedocs.io/en/latest/roadmap.html) and [changelog](https://clinlp.readthedocs.io/en/latest/changelog.html) up to date, so you can see what we are working on and what has changed in the library. +In order to be transparent, we prefer to communicate through means that are open to everyone. This includes using GitHub for issue tracking, pull requests and discussions, and using the `clinlp` documentation for explaining the library's principles and functionality. We keep our [Roadmap](roadmap) and [Changelog](changelog) up to date, so you can see what we are working on and what has changed in the library. + +Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP across organizations and institutions. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that the research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. + +## About + +`clinlp` was initiated by a group of data scientists and engineers from the UMCU, who ran into practical issues working with clinical text and decided to build a library to solve them. + +The library is currently actively maintained by: -Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP. As an added benefit, we believe working together in one place is potentially also be a great way to improve quality of scientific research. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that the research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. +* [Vincent Menger, ML engineer, UMCU](https://github.com/vmenger) +* [Bram van Es, Assistant Professor, UMCU](https://github.com/bramiozo) diff --git a/docs/source/qualifiers.md b/docs/source/qualifiers.md index 8c646ae..d8d9dc5 100644 --- a/docs/source/qualifiers.md +++ b/docs/source/qualifiers.md @@ -1,28 +1,28 @@ # Definition of qualifier classes used in `clinlp` -This page describes the definitions of qualifier classes for Dutch clinical text we use in `clinlp`. +This page describes the definitions of qualifier classes for Dutch clinical text we use in `clinlp`. ## Introduction -When processing clinical documents (e.g., doctor’s notes, discharge letters), detecting qualifiers (e.g., `absent`, `historical`, `non-patient`, `increasing`, `decreasing`, `severe`, `light`, etc.) follows the matching of concepts (e.g., diagnoses, symptoms, procedures). In this document we primarily use the term “qualifier”, although the terms “context”, “meta-annotation”, and “modifier” are also used to denote the same concept. +When processing clinical documents (e.g., doctor’s notes, discharge letters), detecting qualifiers (e.g., `absent`, `historical`, `non-patient`, `increasing`, `decreasing`, `severe`, `light`, etc.) follows the matching of concepts (e.g., diagnoses, symptoms, procedures). In `clinlp` we primarily use the term “qualifier”, although the terms “context”, “meta-annotation”, and “modifier” have also been used to denote the same concept. -A consensus on potential qualifier classes, along with **clear** definitions of the qualifiers they encompass, is needed to develop accurate algorithms for detecting them. Despite some shared intuitive understanding of recognizing, for instance, a _negation_ in a sentence, there are numerous cases where intuition simply falls short. In practice this impedes manual annotation of gold standards, used for training and evaluating algorithms. In turn the resulting annotations (with Kappa-Cohen as low as 0.5) lead to a difficult target for supervised machine learning models. A standardized classification of qualifiers, as proposed here, will hopefully advance both research and clinical implementation of NLP (Natural Language Processing) algorithms. This page is the result of some deliberation among various researchers and developers working with clinical text, but it is not necessarily definitive. We welcome feedback and suggestions for improvement. +A consensus on potential qualifier classes, along with **clear** definitions of the qualifiers they encompass, is needed to develop accurate algorithms for detecting them. Despite some shared intuitive understanding of recognizing, for instance, a _negation_ in a sentence, there are numerous cases where intuition simply falls short. In practice this has impeded manual annotation of gold standards, used for training and evaluating algorithms. In turn the resulting annotations (with Kappa-Cohen as low as 0.5) lead to a difficult target for supervised machine learning models. A standardized classification of qualifiers, as proposed here, will hopefully advance both research and clinical implementation of NLP (Natural Language Processing) algorithms. This page is the result of some deliberation among various researchers and developers working with clinical text, but it is not necessarily definitive. We welcome feedback and suggestions for improvement. -For the proposed we took the Context Algorithm (Harkema, 2009) as a starting point, both because this is an influential paper, and because a corresponding Dutch corpus is available (Afzal, 2014). There are already some trained models available that can for a large part be re-used. We will here describe three qualifier classes: `Presence`, `Temporality`, and `Experiencer`, including some definitions, issues to resolve, and illustrative examples. These classes can be further de-aggregated at a later stage, and other classes may follow later as well. +For these definitions we took the Context Algorithm (Harkema, 2009) as a starting point, both because this is an influential paper, and because a corresponding Dutch corpus is available (Afzal, 2014). There are already some trained models available that can for a large part be re-used. We will here describe three qualifier classes: **Presence**, **Temporality**, and **Experiencer**, including some definitions, issues to resolve, and illustrative examples. These classes can be further de-aggregated at a later stage, and other classes may follow later as well. Note that choosing qualifiers is a trade-off between granularity and practicality. We aim for a balance that is useful for most clinical NLP tasks. -**Qualifier classes** are denoted by boldface, with the _qualifiers_ (the mutually exclusive values a **qualifier class** can assume) italicized. +**Qualifier classes** are denoted by boldface, with the `Qualifier` (a mutually exclusive value a **qualifier class** can assume) formatted as inline code. ## Presence -| **absent** / **negated** | **uncertain** | **present** / **affirmed** (default) | +| `Absent` / `Negated` | `Uncertain` | `Present` / `Affirmed` (default) | |--------------------------|---------------|--------------------------------------| | Concepts that are explicitly or implicitly described as absent in a patient | Whether the concept was absent or present is not asserted with very high certainty | Concepts that are explicitly or implicitly described as present in a patient | -Assessing whether some concept was present or absent is one of the most important parts of a clinician’s job. Whether something is present or occurred in the real world is knowable in principle, but in the clinical world, such assertions are rarely made with complete certainty. This is already implied by the uncertainty at the core of the clinical reasoning process, but in clinical text the uncertainty is often made explicit by means of hedging. It’s therefore important to note that when we are extracting concepts from medical text, it’s very hard to make direct assertions about the real world, but we are limited to recognizing probability statements made by clinicians. +Assessing whether some concept was present or absent is one of the most important parts of a clinician's job. Whether something is present or occurred in the real world is knowable in principle, but in the clinical world, such assertions are rarely made with complete certainty. This is already implied by the uncertainty at the core of the clinical reasoning process, but in clinical text the uncertainty is often made explicit by means of hedging. It’s therefore important to note that when we are extracting concepts from medical text, it’s very hard to make direct assertions about the real world, but we are limited to recognizing probability statements made by clinicians. -The **presence** class therefore captures whether a concept is present in three qualifiers. The _present_ and _absent_ qualifiers are used when the clinician assesses a concept as being present (or absent) with very high probability, extending beyond reasonable doubt. When neither presence nor absence is definitively asserted, the _uncertain_ qualifier applies. This qualifier therefore ranges from very unlikely to very likely. +The **presence** class therefore captures whether a concept is present in three qualifiers. The `Present` and `Absent` qualifiers are used when the clinician assesses a concept as being present (or absent) with very high probability, extending beyond reasonable doubt. When neither presence nor absence is definitively asserted, the `Uncertain` qualifier applies. This qualifier therefore ranges from very unlikely to very likely. -The default qualifier for **presence** is _present_. When the text does not indicate absence or uncertainty of a concept, we assume the writer intended to convey its presence. +The default qualifier for **presence** is `Present`. When the text does not indicate absence or uncertainty of a concept, we assume the writer intended to convey its presence. ### To resolve @@ -33,62 +33,62 @@ The default qualifier for **presence** is _present_. When the text does not indi | Example | Qualifier | | ------- | --------- | -| Rechtszijdig fraai de middelste neusgang te visualiseren, vrij van **poliepen**. | Absent | -| Tractus circulatorius: geen **pijn op de borst**. | Absent | -| Een **acuut coronair syndroom** werd uitgesloten. | Absent | -| Werkdiagnose **maagklachten** bij diclofenac gebruik en weinig intake. | Uncertain | -| Waarschijnlijk **hematurie** bij reeds gepasseerde niersteen. | Uncertain | -| Dat er toen **bradypacing** is geweest valt niet uit te sluiten. | Uncertain | -| In juni 2023 **longembolie** waarvoor rivaroxaban met nu asymptomatische progressie. | Present | -| **PTSS** en **recidiverende depressie** in VG. | Present | -| Status na mild **delier**, heden wel slaperig. | Present | +| Rechtszijdig fraai de middelste neusgang te visualiseren, vrij van poliepen. | `Absent` | +| Tractus circulatorius: geen pijn op de borst. | `Absent` | +| Een acuut coronair syndroom werd uitgesloten. | `Absent` | +| Werkdiagnose maagklachten bij diclofenac gebruik en weinig intake. | `Uncertain` | +| Waarschijnlijk hematurie bij reeds gepasseerde niersteen. | `Uncertain` | +| Dat er toen bradypacing is geweest valt niet uit te sluiten. | `Uncertain` | +| In juni 2023 longembolie waarvoor rivaroxaban met nu asymptomatische progressie. | `Present` | +| PTSS en recidiverende depressie in VG. | `Present` | +| Status na mild delier, heden wel slaperig. | `Present` | ## Temporality -| **historical** | **current** (default) | **future** | +| `Historical` | `Current` (default) | `Future` | |----------------|-----------------------|------------| | Concepts that were applicable at some point in history, but not in the last two weeks. | Concepts that were applicable in the last two weeks (potentially starting before that) up to and including the present moment. | Concepts that are potentially applicable in a future scenario. | -The **temporality** class places concepts in a temporal framework, ranging from past to future, relative to the document date. The _historical_ and _current_ qualifiers distinguish between concepts that were applicable in the past, versus concepts that are applicable in the present. The exact cutoff between _historical_ and _current_ is problem-specific and therefore hard to definitively establish in a general sense. In a discharge summary, everything that happened before the admission period could be considered _historical_, which can easily range up to months, while during a GP (General Practitioner) visit, events from a few days prior might be considered _historical_. For the general case, we see no reason to divert from the threshold of two weeks in the original Context paper (Harkema et al., 2009). Note that the _current_ qualifier also applies when the concept is applicable in the last two weeks, but already started before that. +The **temporality** class places concepts in a temporal framework, ranging from past to future, relative to the document date. The `Historical` and `Current` qualifiers distinguish between concepts that were applicable in the past, versus concepts that are applicable in the present. The exact cutoff between `Historical` and `Current` is problem-specific and therefore hard to definitively establish in a general sense. In a discharge summary, everything that happened before the admission period could be considered `Historical`, which can easily range up to months, while during a GP (General Practitioner) visit, events from a few days prior might be considered `Historical`. For the general case, we see no reason to divert from the threshold of two weeks in the original Context paper (Harkema et al., 2009). Note that the `Current` qualifier also applies when the concept is applicable in the last two weeks, but already started before that. -The _future_ qualifier is applicable when a concept is described in a future scenario, for instance when describing the risk of developing a condition at a later stage, or when describing a procedure that will take place later. +The `Future` qualifier is applicable when a concept is described in a future scenario, for instance when describing the risk of developing a condition at a later stage, or when describing a procedure that will take place later. ### To resolve -- A way to dynamically define the threshold for _historical_ and _current_, so that a cutoff can be established for each problem. In future work, we might map each concept to a timedelta (e.g., -1 year, -14 days, +5 days), but that does not fit the current qualifier framework very well. Also, it seems quite a hard problem. +- A way to dynamically define the threshold for `Historical` and `Current`, so that a cutoff can be established for each problem. In future work, we might map each concept to a timedelta (e.g., -1 year, -14 days, +5 days), but that does not fit the current qualifier framework very well. Also, it seems quite a hard problem. ### Examples | Example | Qualifier | | ------- | --------- | -| Zwanger, meerdere **miskramen** in de voorgeschiedenis. | _historical_ | -| Progressieve autonome functiestoornissen bij eerdere **dermoidcyste**. | _historical_ | -| Als tiener een **osteotomiecorrectie** beiderzijds gehad. | _historical_ | -| Echocardiografisch zagen wij geen aanwijzingen voor een **hypertrofe cardiomyopathie**. | _current_ | -| Al langer bestaande **bloeddrukproblematiek**. | _current_ | -| CT thorax: **laesie** rechter onderkwab bevestigd. | _current_ | -| Conservatieve maatregelen ter preventie van **pulmonale infectie** zijn herbesproken. | _future_ | -| Mocht hij **koorts** en/of **tachycardie** ontwikkelen, dan contact opnemen met dienstdoende arts. | _future_ | -| Wordt nu opgenomen middels IBS ter afwending van **suïcide**. | _future_ | +| Zwanger, meerdere miskramen in de voorgeschiedenis. | `Historical` | +| Progressieve autonome functiestoornissen bij eerdere dermoidcyste. | `Historical` | +| Als tiener een osteotomiecorrectie beiderzijds gehad. | `Historical` | +| Echocardiografisch zagen wij geen aanwijzingen voor een hypertrofe cardiomyopathie. | `Current` | +| Al langer bestaande bloeddrukproblematiek. | `Current` | +| CT thorax: laesie rechter onderkwab bevestigd. | `Current` | +| Conservatieve maatregelen ter preventie van pulmonale infectie zijn herbesproken. | `Future` | +| Mocht hij koorts en/of tachycardie ontwikkelen, dan contact opnemen met dienstdoende arts. | `Future` | +| Wordt nu opgenomen middels IBS ter afwending van suïcide. | `Future` | ## Experiencer -| **patient** (default) | **family** | **other** | +| `Patient` (default) | `Family` | `Other` | |-----------------------|------------|-----------| | Concepts applicable to the patient related to the current document. | Concepts not applicable to the patient, but to someone with a genetic relationship to the patient. | Concepts not applicable to the patient, but to someone without a genetic relationship to the patient. | -The **experiencer** qualifier distinguishes between concepts that apply to the _patient_, to those that apply to _family_ members with a genetic relationship to the patient, and _other_ individuals with no genetic relationship to the patient (e.g. acquaintances). Clinical documents are typically obtained from electronic health records, where the relation between a document and a patient is explicit. Since a patient is a well separated entity, there is usually little ambiguity which class applies. If a concept applies to both the patient and another person, the patient label should be selected. +The **experiencer** qualifier distinguishes between concepts that apply to the `Patient`, to those that apply to `Family` members with a genetic relationship to the patient, and `Other` individuals with no genetic relationship to the patient (e.g. acquaintances). Clinical documents are typically obtained from electronic health records, where the relation between a document and a patient is explicit. Since a patient is a well separated entity, there is usually little ambiguity which class applies. If a concept applies to both the patient and another person, the patient label should be selected. ### Examples | Example | Qualifier | | ------------------------------------------------------------ | --------- | -| Behandeling in WKZ ivm **diabetes** beeindigd. | _patient_ | -| Pte wil geen medicatie tegen **parkinson** ivm slechte ervaringen broer | _patient_ | -| X-enkel rechts: **schuine fractuur laterale malleolus** | _patient_ | -| Familieanamnese omvat: **ADD**/**ADHD**: broer | _family_ | -| Moederszijde: voor zover bekend geen **kanker** | _family_ | -| 2. **Covid** positieve huisgenoot | _other_ | +| Behandeling in WKZ ivm diabetes beeindigd. | `Patient` | +| Pte wil geen medicatie tegen parkinson ivm slechte ervaringen broer | `Patient` | +| X-enkel rechts: schuine fractuur laterale malleolus | `Patient` | +| Familieanamnese omvat: ADD/ADHD: broer | `Family` | +| Moederszijde: voor zover bekend geen kanker | `Family` | +| 2. Covid positieve huisgenoot | `Other` | ## References From 3d9a96cfcefaaa678b4a35d15a4517adad203541 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Mon, 17 Jun 2024 13:10:56 +0200 Subject: [PATCH 23/30] Small text edit --- docs/source/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.md b/docs/source/index.md index 5f0d973..c3d9433 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,6 +1,6 @@ ![clinlp logo](../../media/clinlp.png) -Welcome to the documentation pages for `clinlp`, a library for performing NLP on clinical text written in Dutch. In the menu to the left, you should be able to find the information you are looking for. If you have any questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](contributing.md#contact)! +Welcome to the documentation pages for `clinlp`, a Python library for performing NLP on clinical text written in Dutch. In the menu to the left, you should be able to find the information you are looking for. If you have any questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](contributing.md#contact)! ```{toctree} :caption: clinlp From c49a1aa30dc9e1898f6772d87bb2d465c14da4f3 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Mon, 17 Jun 2024 13:16:26 +0200 Subject: [PATCH 24/30] Minor formatting updates --- docs/source/components.md | 13 ++++++------- docs/source/metrics.md | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/source/components.md b/docs/source/components.md index 8cfcbbf..d083e71 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -59,7 +59,7 @@ Note that this component only has effect when explicitly configuring successor c | assigns | `token.is_sent_start`, `doc.sents` | | config options | `sent_end_chars = [".", "!", "?", "\n", "\r"]`
`sent_start_punct = ["-", "*", "[", "("]` | -The sentencizer is a rule-based sentence boundary detector. It is designed to detect sentence boundaries in clinical text, whenever a character that demarks a sentence ending is matched (e.g. newline, period, question mark). The next sentence is started whenever an alpha character or a character in `sent_start_punct` is encountered. This prevents e.g. sentences ending in `...` to be classified as three separate sentences. The sentencizer correctly detects items in enumerations (e.g. starting with `-` or `*`). +The sentencizer is a rule-based sentence boundary detector. It is designed to detect sentence boundaries in clinical text, whenever a character that marks a sentence ending is matched (e.g. newline, period, question mark). The next sentence is started whenever an alpha character or a character in `sent_start_punct` is encountered. This prevents e.g. sentences ending in `...` to be classified as three separate sentences. The sentencizer correctly detects items in enumerations (e.g. starting with `-` or `*`). ## Entity Matching @@ -74,7 +74,7 @@ The sentencizer is a rule-based sentence boundary detector. It is designed to de | assigns | `doc.spans['ents']` | | config options | `attr = "TEXT"`
`proximity = 0`
`fuzzy = 0`
`fuzzy_min_len = 0`
`pseudo = False` | -The `clinlp_rule_based_entity_matcher` component can be used for matching entities in text, based on a dictionary of known concepts and their terms/synonyms. It includes options for matching on different token attributes, proximity matching, fuzzy matching and unmatching pseudo/negative terms. +The `clinlp_rule_based_entity_matcher` component can be used for matching entities in text, based on a dictionary of known concepts and their terms/synonyms. It includes options for matching on different token attributes, proximity matching, fuzzy matching and non-matching pseudo/negative terms. The most basic example would be the following, with further options described below: @@ -164,7 +164,7 @@ entity_matcher = nlp.add_pipe("clinlp_rule_based_entity_matcher", config={"attr" entity_matcher.load_concepts(concepts) ``` -In the above example, by default the `NORM` attribute is used, and `fuzzy` is set to `1`. In addition, for the terms `early onset` and `late onset` proximity matching is set to `1`, in addition to matcher-level config of matching the `NORM` attribute and fuzzy matching. For the `EOS` and `LOS` abbreviations the `TEXT` attribute is used (so the matching is case sensitive), and fuzzy matching is disabled. +In the above example, by default the `NORM` attribute is used, and `fuzzy` is set to `1`. In addition, for the terms `early onset` and `late onset` proximity matching is set to `1`, in addition to matcher-level config of matching the `NORM` attribute and fuzzy matching. For the `EOS` and `LOS` abbreviations the `TEXT` attribute is used (so the matching is case sensitive), and fuzzy matching is disabled. #### Pseudo/negative phrases @@ -210,7 +210,7 @@ concepts = { #### Concept dictionary from external source -External lists of concepts (e.g. from a medical thesaurus such as UMLS) can also be loaded directly from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: +External lists of concepts (e.g. from a medical thesaurus such as `UMLS`) can also be loaded directly from `csv` through the `create_concept_dict` function. Your `csv` should contain a combination of concept and phrase on each line, with optional columns to configure the `Term`-options described above (e.g. `attribute`, `proximity`, `fuzzy`). You may present the columns in any order, but make sure the names match the `Term` attributes. Any other columns are ignored. For example: | **concept** | **phrase** | **attr** | **proximity** | **fuzzy** | **fuzzy_min_len** | **pseudo** | **comment** | |--|--|--|--|--|--|--|--| @@ -247,7 +247,7 @@ Will result in the following concept dictionary: | property | value | | --- | --- | | name | `clinlp_context_algorithm` | -| class | [[clinlp.ie.qualifier.context_algorithm.ContextAlgorithm](clinlp.ie.qualifier.context_algorithm.ContextAlgorithm) | +| class | [clinlp.ie.qualifier.context_algorithm.ContextAlgorithm](clinlp.ie.qualifier.context_algorithm.ContextAlgorithm) | | example | `nlp.add_pipe('clinlp_context_algorithm')` | | requires | `doc.sents`, `doc.spans['ents']` | | assigns | `span._.qualifiers` | @@ -281,7 +281,7 @@ For more extensive documentation on the definitions of the qualifiers we use in | assigns | `span._.qualifiers` | | config options | `token_window = 32`
`strip_entities = True`
`placeholder = None`
`prob_aggregator = statistics.mean`
`absence_threshold = 0.1`
`presence_threshold = 0.9` | -The `clinlp_negation_transformer` wraps the the negation detector described in [van Es et al, 2022](https://doi.org/10.48550/arxiv.2209.00470). The underlying transformer can be found on [huggingface](https://huggingface.co/UMCU/). The negation detector is reported as more accurate than the rule-based version (see paper for details), at the cost of less transparency and additional computational cost. +The `clinlp_negation_transformer` wraps the the negation detector described in [van Es et al, 2022](https://doi.org/10.48550/arxiv.2209.00470). The underlying transformer can be found on [HuggingFace](https://huggingface.co/UMCU/). The negation detector is reported as more accurate than the rule-based version (see paper for details), at the cost of less transparency and additional computational cost. This component requires the following optional dependencies: @@ -298,7 +298,6 @@ The thresholds define where the cutoff for absence and presence are. If the pred For more extensive documentation on the definitions of the qualifiers we use in `clinlp`, see the [Qualifiers](qualifiers.md) page. ``` - ### `clinlp_experiencer_transformer` | property | value | diff --git a/docs/source/metrics.md b/docs/source/metrics.md index a3fb747..2af740a 100644 --- a/docs/source/metrics.md +++ b/docs/source/metrics.md @@ -1,6 +1,6 @@ # Metrics and statistics -`clinlp` contains calculators for some specific metrics and statistics for evaluating NLP tools. You can find some basic information on using them below. +`clinlp` contains calculators for some specific metrics and statistics for evaluating NLP tools. You can find some basic information on using them below. ## Information extraction From d8ce3a6697b666c4f18abbbcd806985f455ff26d Mon Sep 17 00:00:00 2001 From: msnackey Date: Tue, 18 Jun 2024 11:58:14 +0200 Subject: [PATCH 25/30] Textual changes from review --- docs/source/components.md | 2 +- docs/source/getting_started.md | 2 +- docs/source/installation.md | 2 +- docs/source/introduction.md | 2 +- docs/source/qualifiers.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/components.md b/docs/source/components.md index d083e71..7ca43b5 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -101,7 +101,7 @@ entity_matcher.load_concepts(concepts) `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. ``` -```{admonition} Using `spaCy` components directly +```{admonition} Using spaCy components directly :class: tip The `clinlp_rule_based_entity_matcher` component wraps the `spaCy` `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index 32790fe..f50918e 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -107,7 +107,7 @@ entity_matcher = nlp.add_pipe( entity_matcher.load_concepts(concepts) ``` -The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to matches against the `NORM` attribute by default, which it finds in the `Token.norm_` property the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may skipped between the words `bd` and `verlaagd`. +The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to match against the `NORM` attribute by default, which it finds in the `Token.norm_` property that the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may be skipped between the words `bd` and `verlaagd`. If we now process a piece of text, we can see that the entity recognizer has recognized some entities: diff --git a/docs/source/installation.md b/docs/source/installation.md index 5221e1b..fb7fc3f 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -6,7 +6,7 @@ The easiest way to install `clinlp` is by using `pip`: pip install clinlp ``` -As a good practice, we recommend installing `clinlp` in a virtual environment. If you are not familiar with virtual environments, you can find more information [here](https://docs.python.org/3/library/venv.html). +As good practice, we recommend installing `clinlp` in a virtual environment. If you are not familiar with virtual environments, you can find more information [here](https://docs.python.org/3/library/venv.html). ## Optional dependencies diff --git a/docs/source/introduction.md b/docs/source/introduction.md index ffc35d6..29e6827 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -102,7 +102,7 @@ More detail on the `clinlp` development practices can be found in the [Coding St In order to be transparent, we prefer to communicate through means that are open to everyone. This includes using GitHub for issue tracking, pull requests and discussions, and using the `clinlp` documentation for explaining the library's principles and functionality. We keep our [Roadmap](roadmap) and [Changelog](changelog) up to date, so you can see what we are working on and what has changed in the library. -Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP across organizations and institutions. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that the research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. +Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP across organizations and institutions. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. ## About diff --git a/docs/source/qualifiers.md b/docs/source/qualifiers.md index d8d9dc5..ce64eed 100644 --- a/docs/source/qualifiers.md +++ b/docs/source/qualifiers.md @@ -1,6 +1,6 @@ # Definition of qualifier classes used in `clinlp` -This page describes the definitions of qualifier classes for Dutch clinical text we use in `clinlp`. +This page describes the definitions of qualifier classes for Dutch clinical text that are used in `clinlp`. ## Introduction From e2bcc843e74d46a0f55007ddc374b88fd15322e6 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 18 Jun 2024 14:08:39 +0200 Subject: [PATCH 26/30] Revert "Textual changes from review" This reverts commit d8ce3a6697b666c4f18abbbcd806985f455ff26d. --- docs/source/components.md | 2 +- docs/source/getting_started.md | 2 +- docs/source/installation.md | 2 +- docs/source/introduction.md | 2 +- docs/source/qualifiers.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/components.md b/docs/source/components.md index 7ca43b5..d083e71 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -101,7 +101,7 @@ entity_matcher.load_concepts(concepts) `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. ``` -```{admonition} Using spaCy components directly +```{admonition} Using `spaCy` components directly :class: tip The `clinlp_rule_based_entity_matcher` component wraps the `spaCy` `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index f50918e..32790fe 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -107,7 +107,7 @@ entity_matcher = nlp.add_pipe( entity_matcher.load_concepts(concepts) ``` -The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to match against the `NORM` attribute by default, which it finds in the `Token.norm_` property that the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may be skipped between the words `bd` and `verlaagd`. +The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to matches against the `NORM` attribute by default, which it finds in the `Token.norm_` property the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may skipped between the words `bd` and `verlaagd`. If we now process a piece of text, we can see that the entity recognizer has recognized some entities: diff --git a/docs/source/installation.md b/docs/source/installation.md index fb7fc3f..5221e1b 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -6,7 +6,7 @@ The easiest way to install `clinlp` is by using `pip`: pip install clinlp ``` -As good practice, we recommend installing `clinlp` in a virtual environment. If you are not familiar with virtual environments, you can find more information [here](https://docs.python.org/3/library/venv.html). +As a good practice, we recommend installing `clinlp` in a virtual environment. If you are not familiar with virtual environments, you can find more information [here](https://docs.python.org/3/library/venv.html). ## Optional dependencies diff --git a/docs/source/introduction.md b/docs/source/introduction.md index 29e6827..ffc35d6 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -102,7 +102,7 @@ More detail on the `clinlp` development practices can be found in the [Coding St In order to be transparent, we prefer to communicate through means that are open to everyone. This includes using GitHub for issue tracking, pull requests and discussions, and using the `clinlp` documentation for explaining the library's principles and functionality. We keep our [Roadmap](roadmap) and [Changelog](changelog) up to date, so you can see what we are working on and what has changed in the library. -Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP across organizations and institutions. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. +Finally, by working together in `clinlp`, we hope to strengthen the connections in our specific field of Dutch clinical NLP across organizations and institutions. By committing to making algorithms and implementations available in this package, and to collaboratively further standardize algorithms and protocols, we can ensure that the research is reproducible and that the algorithms can be used by others. This way, we can build on each other's work, and make the field of Dutch clinical NLP stronger. ## About diff --git a/docs/source/qualifiers.md b/docs/source/qualifiers.md index ce64eed..d8d9dc5 100644 --- a/docs/source/qualifiers.md +++ b/docs/source/qualifiers.md @@ -1,6 +1,6 @@ # Definition of qualifier classes used in `clinlp` -This page describes the definitions of qualifier classes for Dutch clinical text that are used in `clinlp`. +This page describes the definitions of qualifier classes for Dutch clinical text we use in `clinlp`. ## Introduction From a78df9f760eb64069b427ba6bac9c962deccfd5c Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 18 Jun 2024 14:17:02 +0200 Subject: [PATCH 27/30] Open external links in new tab --- docs/conf.py | 7 ++++++- poetry.lock | 39 ++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 05056f0..3351ded 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,12 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "myst_parser"] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "myst_parser", + "sphinx_new_tab_link", +] source_suffix = { ".rst": "restructuredtext", diff --git a/poetry.lock b/poetry.lock index ed7dc07..007137a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2257,6 +2257,26 @@ sphinx = ">=4.0" [package.extras] docs = ["furo", "ipython", "myst-parser", "sphinx-copybutton", "sphinx-inline-tabs"] +[[package]] +name = "sphinx-new-tab-link" +version = "0.4.0" +description = "Open external links in new tabs of the browser in Sphinx HTML documents" +optional = false +python-versions = "*" +files = [ + {file = "sphinx-new-tab-link-0.4.0.tar.gz", hash = "sha256:aaefd94d5aa75c60a6c1e94b80d75c4281c3b6f95669b8e606f212744818b916"}, + {file = "sphinx_new_tab_link-0.4.0-py3-none-any.whl", hash = "sha256:2353bfd3a171fdbd9dcdf33e5f26b4447607293ff2b57e0f6a2dc18d1507598e"}, +] + +[package.dependencies] +sphinxcontrib-extdevhelper-kasane = "*" + +[package.extras] +dev = ["build", "twine", "wheel"] +lint = ["black", "flake8", "isort"] +testing = ["beautifulsoup4", "pytest"] +typecheck = ["mypy", "types-beautifulsoup"] + [[package]] name = "sphinxcontrib-applehelp" version = "1.0.8" @@ -2289,6 +2309,23 @@ lint = ["docutils-stubs", "flake8", "mypy"] standalone = ["Sphinx (>=5)"] test = ["pytest"] +[[package]] +name = "sphinxcontrib-extdevhelper-kasane" +version = "0.2.0" +description = "襲 - Provide dynamic inheritance shortcuts to make Sphinx extension development easier" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-extdevhelper-kasane-0.2.0.tar.gz", hash = "sha256:4dc7b00327f33c7b421c27122b40278eeaca43f24601b572cee5616d31b206a9"}, + {file = "sphinxcontrib_extdevhelper_kasane-0.2.0-py3-none-any.whl", hash = "sha256:20f94e3b209cddec24596234458ea3887e7a7ad45b54a4d0a5bf169ff45a38f1"}, +] + +[package.dependencies] +Sphinx = "*" + +[package.extras] +dev = ["autoflake", "black", "flake8", "isort", "mypy", "pytest", "pytest-randomly", "pyupgrade", "taskipy"] + [[package]] name = "sphinxcontrib-htmlhelp" version = "2.0.5" @@ -2924,4 +2961,4 @@ transformers = ["transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "3653e00789b2bc9c172da1de3a89ae8c59db874813308fc751c2af49c8a606d8" +content-hash = "e102fd27aaa4be66c4cf9c2d5227976f25443f6e2342a91ba1e16ed188222398" diff --git a/pyproject.toml b/pyproject.toml index fd41a6b..9fe4e45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ myst-parser = "^3.0.1" emoji = "^2.12.1" toml = "^0.10.2" furo = "^2024.5.6" +sphinx-new-tab-link = "^0.4.0" [tool.pytest.ini_options] testpaths = ["tests"] From 87404576f181774a20429d0a55d8b0d0e6ff8db4 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 18 Jun 2024 14:17:16 +0200 Subject: [PATCH 28/30] Textual improvements --- docs/source/components.md | 2 +- docs/source/getting_started.md | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/source/components.md b/docs/source/components.md index d083e71..7ca43b5 100644 --- a/docs/source/components.md +++ b/docs/source/components.md @@ -101,7 +101,7 @@ entity_matcher.load_concepts(concepts) `clinlp` stores entities in `doc.spans`, specifically in `doc.spans["ents"]`. The reason for this is that spans can overlap, while the entities in `doc.ents` cannot. If you use other/custom components, make sure they read/write entities from/to the same span key if interoperability is needed. ``` -```{admonition} Using `spaCy` components directly +```{admonition} Using spaCy components directly :class: tip The `clinlp_rule_based_entity_matcher` component wraps the `spaCy` `Matcher` and `PhraseMatcher` components, adding some convenience and configurability. However, the `Matcher`, `PhraseMatcher` or `SpanRuler` can also be used directly with `clinlp` for those who prefer it. You can configure the `SpanRuler` to write to the same `SpanGroup` as follows: diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index 32790fe..5069899 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -78,6 +78,22 @@ print(str(sent) for sent in doc.sents) ``` Other components can use these newly set properties `Token.norm_` and `Doc.sents`. For example, an entity recognizer can use the normalized text to recognize entities, and a negation detector can use the sentence boundaries to determine the range of a negation. +You can always inspect the current model's pipeline using: + +```python +print(nlp.pipe_names) + +> ['clinlp_normalizer', 'clinlp_sentencizer'] +``` + +This shows the current components in the pipeline, in the order they are executed. The order of the components is important, as the output of one component is the input of the next component. The order of the components can be changed by using the `nlp.add_pipe` method with the `before` or `after` parameter. For example, to add a component before the `clinlp_sentencizer`: + +```python +nlp.add_pipe('component_name', before='clinlp_sentencizer') +``` + +This will add the component before the `clinlp_sentencizer` in the pipeline. + ## Information extraction example Now that we understand the basics of a blank model and adding components, let's add two more components to create a basic information extraction pipeline. @@ -107,7 +123,7 @@ entity_matcher = nlp.add_pipe( entity_matcher.load_concepts(concepts) ``` -The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to matches against the `NORM` attribute by default, which it finds in the `Token.norm_` property the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may skipped between the words `bd` and `verlaagd`. +The above code adds three concepts to be matched (`prematuriteit`, `hypotensie`, and `veneus_infarct`), along with synonyms to match. Additionally, it configures the entity matcher on how to perform the matching. We have here configured the entity matcher to match against the `NORM` attribute by default, which it finds in the `Token.norm_` property the `clinlp_normalizer` set earlier. The `fuzzy` parameter specifies how much the concept text and the real text can differ (based on the edit distance). Some settings are overruled at the `Term` level. For instance, the `proximity=1` parameter for `bd verlaagd` specifies that at most one token may skipped between the words `bd` and `verlaagd`. If we now process a piece of text, we can see that the entity recognizer has recognized some entities: @@ -163,7 +179,7 @@ for ent in doc.spans['ents']: > 'VI' {'Temporality.Future'} ``` -In the above example, for readability all default qualifiers have been omitted. You can see that three out of seven entities have correctly been qualified, either as `Absent`, related to `Family`, or potentially occurring in the `Future`. Of course, your specific use case determines how the output of this pipeline will further be handled. +In the above example, for readability all default qualifier values (`Presence.Present`, `Temporality.Current`, `Experiencer.Patient`) have been omitted. You can see that three out of seven entities have correctly been qualified, either as `Absent`, related to `Family`, or potentially occurring in the `Future`. Of course, your specific use case determines how the output of this pipeline will further be handled. ## Conclusion From 698d3b673c1b8075c230d8aa6e9b82f95fc4f792 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 18 Jun 2024 14:20:30 +0200 Subject: [PATCH 29/30] Add github repo link --- docs/source/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/index.md b/docs/source/index.md index c3d9433..71fdbf5 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,6 +2,10 @@ Welcome to the documentation pages for `clinlp`, a Python library for performing NLP on clinical text written in Dutch. In the menu to the left, you should be able to find the information you are looking for. If you have any questions, need help getting started, found a bug, or have a feature request, please don't hesitate to [contact us](contributing.md#contact)! +## Links + +- [GitHub repository](https://github.com/umcu/clinlp) + ```{toctree} :caption: clinlp :hidden: From ad81e77962e2a4bcb74add93e894b250304bafa3 Mon Sep 17 00:00:00 2001 From: Vincent Menger Date: Tue, 18 Jun 2024 14:26:14 +0200 Subject: [PATCH 30/30] Update pydantic --- poetry.lock | 168 ++++++++++++++++++++++++++-------------------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/poetry.lock b/poetry.lock index 007137a..857a10c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1425,18 +1425,18 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pydantic" -version = "2.7.2" +version = "2.7.4" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.7.2-py3-none-any.whl", hash = "sha256:834ab954175f94e6e68258537dc49402c4a5e9d0409b9f1b86b7e934a8372de7"}, - {file = "pydantic-2.7.2.tar.gz", hash = "sha256:71b2945998f9c9b7919a45bde9a50397b289937d215ae141c1d0903ba7149fd7"}, + {file = "pydantic-2.7.4-py3-none-any.whl", hash = "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"}, + {file = "pydantic-2.7.4.tar.gz", hash = "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.18.3" +pydantic-core = "2.18.4" typing-extensions = ">=4.6.1" [package.extras] @@ -1444,90 +1444,90 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.18.3" +version = "2.18.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.18.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:744697428fcdec6be5670460b578161d1ffe34743a5c15656be7ea82b008197c"}, - {file = "pydantic_core-2.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37b40c05ced1ba4218b14986fe6f283d22e1ae2ff4c8e28881a70fb81fbfcda7"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a9a75622357076efb6b311983ff190fbfb3c12fc3a853122b34d3d358126c"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2e253af04ceaebde8eb201eb3f3e3e7e390f2d275a88300d6a1959d710539e2"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:855ec66589c68aa367d989da5c4755bb74ee92ccad4fdb6af942c3612c067e34"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d3e42bb54e7e9d72c13ce112e02eb1b3b55681ee948d748842171201a03a98a"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6ac9ffccc9d2e69d9fba841441d4259cb668ac180e51b30d3632cd7abca2b9b"}, - {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c56eca1686539fa0c9bda992e7bd6a37583f20083c37590413381acfc5f192d6"}, - {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:17954d784bf8abfc0ec2a633108207ebc4fa2df1a0e4c0c3ccbaa9bb01d2c426"}, - {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:98ed737567d8f2ecd54f7c8d4f8572ca7c7921ede93a2e52939416170d357812"}, - {file = "pydantic_core-2.18.3-cp310-none-win32.whl", hash = "sha256:9f9e04afebd3ed8c15d67a564ed0a34b54e52136c6d40d14c5547b238390e779"}, - {file = "pydantic_core-2.18.3-cp310-none-win_amd64.whl", hash = "sha256:45e4ffbae34f7ae30d0047697e724e534a7ec0a82ef9994b7913a412c21462a0"}, - {file = "pydantic_core-2.18.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b9ebe8231726c49518b16b237b9fe0d7d361dd221302af511a83d4ada01183ab"}, - {file = "pydantic_core-2.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b8e20e15d18bf7dbb453be78a2d858f946f5cdf06c5072453dace00ab652e2b2"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0d9ff283cd3459fa0bf9b0256a2b6f01ac1ff9ffb034e24457b9035f75587cb"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f7ef5f0ebb77ba24c9970da18b771711edc5feaf00c10b18461e0f5f5949231"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73038d66614d2e5cde30435b5afdced2b473b4c77d4ca3a8624dd3e41a9c19be"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6afd5c867a74c4d314c557b5ea9520183fadfbd1df4c2d6e09fd0d990ce412cd"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd7df92f28d351bb9f12470f4c533cf03d1b52ec5a6e5c58c65b183055a60106"}, - {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:80aea0ffeb1049336043d07799eace1c9602519fb3192916ff525b0287b2b1e4"}, - {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaee40f25bba38132e655ffa3d1998a6d576ba7cf81deff8bfa189fb43fd2bbe"}, - {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9128089da8f4fe73f7a91973895ebf2502539d627891a14034e45fb9e707e26d"}, - {file = "pydantic_core-2.18.3-cp311-none-win32.whl", hash = "sha256:fec02527e1e03257aa25b1a4dcbe697b40a22f1229f5d026503e8b7ff6d2eda7"}, - {file = "pydantic_core-2.18.3-cp311-none-win_amd64.whl", hash = "sha256:58ff8631dbab6c7c982e6425da8347108449321f61fe427c52ddfadd66642af7"}, - {file = "pydantic_core-2.18.3-cp311-none-win_arm64.whl", hash = "sha256:3fc1c7f67f34c6c2ef9c213e0f2a351797cda98249d9ca56a70ce4ebcaba45f4"}, - {file = "pydantic_core-2.18.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f0928cde2ae416a2d1ebe6dee324709c6f73e93494d8c7aea92df99aab1fc40f"}, - {file = "pydantic_core-2.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bee9bb305a562f8b9271855afb6ce00223f545de3d68560b3c1649c7c5295e9"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e862823be114387257dacbfa7d78547165a85d7add33b446ca4f4fae92c7ff5c"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a36f78674cbddc165abab0df961b5f96b14461d05feec5e1f78da58808b97e7"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba905d184f62e7ddbb7a5a751d8a5c805463511c7b08d1aca4a3e8c11f2e5048"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fdd362f6a586e681ff86550b2379e532fee63c52def1c666887956748eaa326"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b214b7ee3bd3b865e963dbed0f8bc5375f49449d70e8d407b567af3222aae4"}, - {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:691018785779766127f531674fa82bb368df5b36b461622b12e176c18e119022"}, - {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60e4c625e6f7155d7d0dcac151edf5858102bc61bf959d04469ca6ee4e8381bd"}, - {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4e651e47d981c1b701dcc74ab8fec5a60a5b004650416b4abbef13db23bc7be"}, - {file = "pydantic_core-2.18.3-cp312-none-win32.whl", hash = "sha256:ffecbb5edb7f5ffae13599aec33b735e9e4c7676ca1633c60f2c606beb17efc5"}, - {file = "pydantic_core-2.18.3-cp312-none-win_amd64.whl", hash = "sha256:2c8333f6e934733483c7eddffdb094c143b9463d2af7e6bd85ebcb2d4a1b82c6"}, - {file = "pydantic_core-2.18.3-cp312-none-win_arm64.whl", hash = "sha256:7a20dded653e516a4655f4c98e97ccafb13753987434fe7cf044aa25f5b7d417"}, - {file = "pydantic_core-2.18.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:eecf63195be644b0396f972c82598cd15693550f0ff236dcf7ab92e2eb6d3522"}, - {file = "pydantic_core-2.18.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c44efdd3b6125419c28821590d7ec891c9cb0dff33a7a78d9d5c8b6f66b9702"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e59fca51ffbdd1638b3856779342ed69bcecb8484c1d4b8bdb237d0eb5a45e2"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70cf099197d6b98953468461d753563b28e73cf1eade2ffe069675d2657ed1d5"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63081a49dddc6124754b32a3774331467bfc3d2bd5ff8f10df36a95602560361"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:370059b7883485c9edb9655355ff46d912f4b03b009d929220d9294c7fd9fd60"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a64faeedfd8254f05f5cf6fc755023a7e1606af3959cfc1a9285744cc711044"}, - {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19d2e725de0f90d8671f89e420d36c3dd97639b98145e42fcc0e1f6d492a46dc"}, - {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:67bc078025d70ec5aefe6200ef094576c9d86bd36982df1301c758a9fff7d7f4"}, - {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:adf952c3f4100e203cbaf8e0c907c835d3e28f9041474e52b651761dc248a3c0"}, - {file = "pydantic_core-2.18.3-cp38-none-win32.whl", hash = "sha256:9a46795b1f3beb167eaee91736d5d17ac3a994bf2215a996aed825a45f897558"}, - {file = "pydantic_core-2.18.3-cp38-none-win_amd64.whl", hash = "sha256:200ad4e3133cb99ed82342a101a5abf3d924722e71cd581cc113fe828f727fbc"}, - {file = "pydantic_core-2.18.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:304378b7bf92206036c8ddd83a2ba7b7d1a5b425acafff637172a3aa72ad7083"}, - {file = "pydantic_core-2.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c826870b277143e701c9ccf34ebc33ddb4d072612683a044e7cce2d52f6c3fef"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e201935d282707394f3668380e41ccf25b5794d1b131cdd96b07f615a33ca4b1"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5560dda746c44b48bf82b3d191d74fe8efc5686a9ef18e69bdabccbbb9ad9442"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b32c2a1f8032570842257e4c19288eba9a2bba4712af542327de9a1204faff8"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:929c24e9dea3990bc8bcd27c5f2d3916c0c86f5511d2caa69e0d5290115344a9"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a8376fef60790152564b0eab376b3e23dd6e54f29d84aad46f7b264ecca943"}, - {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dccf3ef1400390ddd1fb55bf0632209d39140552d068ee5ac45553b556780e06"}, - {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:41dbdcb0c7252b58fa931fec47937edb422c9cb22528f41cb8963665c372caf6"}, - {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:666e45cf071669fde468886654742fa10b0e74cd0fa0430a46ba6056b24fb0af"}, - {file = "pydantic_core-2.18.3-cp39-none-win32.whl", hash = "sha256:f9c08cabff68704a1b4667d33f534d544b8a07b8e5d039c37067fceb18789e78"}, - {file = "pydantic_core-2.18.3-cp39-none-win_amd64.whl", hash = "sha256:4afa5f5973e8572b5c0dcb4e2d4fda7890e7cd63329bd5cc3263a25c92ef0026"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:77319771a026f7c7d29c6ebc623de889e9563b7087911b46fd06c044a12aa5e9"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:df11fa992e9f576473038510d66dd305bcd51d7dd508c163a8c8fe148454e059"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d531076bdfb65af593326ffd567e6ab3da145020dafb9187a1d131064a55f97c"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d33ce258e4e6e6038f2b9e8b8a631d17d017567db43483314993b3ca345dcbbb"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f9cd7f5635b719939019be9bda47ecb56e165e51dd26c9a217a433e3d0d59a9"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cd4a032bb65cc132cae1fe3e52877daecc2097965cd3914e44fbd12b00dae7c5"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f2718430098bcdf60402136c845e4126a189959d103900ebabb6774a5d9fdb"}, - {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c0037a92cf0c580ed14e10953cdd26528e8796307bb8bb312dc65f71547df04d"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b95a0972fac2b1ff3c94629fc9081b16371dad870959f1408cc33b2f78ad347a"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a62e437d687cc148381bdd5f51e3e81f5b20a735c55f690c5be94e05da2b0d5c"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b367a73a414bbb08507da102dc2cde0fa7afe57d09b3240ce82a16d608a7679c"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ecce4b2360aa3f008da3327d652e74a0e743908eac306198b47e1c58b03dd2b"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd4435b8d83f0c9561a2a9585b1de78f1abb17cb0cef5f39bf6a4b47d19bafe3"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:616221a6d473c5b9aa83fa8982745441f6a4a62a66436be9445c65f241b86c94"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7e6382ce89a92bc1d0c0c5edd51e931432202b9080dc921d8d003e616402efd1"}, - {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff58f379345603d940e461eae474b6bbb6dab66ed9a851ecd3cb3709bf4dcf6a"}, - {file = "pydantic_core-2.18.3.tar.gz", hash = "sha256:432e999088d85c8f36b9a3f769a8e2b57aabd817bbb729a90d1fe7f18f6f1f39"}, + {file = "pydantic_core-2.18.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4"}, + {file = "pydantic_core-2.18.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb"}, + {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c"}, + {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e"}, + {file = "pydantic_core-2.18.4-cp310-none-win32.whl", hash = "sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc"}, + {file = "pydantic_core-2.18.4-cp310-none-win_amd64.whl", hash = "sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0"}, + {file = "pydantic_core-2.18.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d"}, + {file = "pydantic_core-2.18.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951"}, + {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2"}, + {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9"}, + {file = "pydantic_core-2.18.4-cp311-none-win32.whl", hash = "sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558"}, + {file = "pydantic_core-2.18.4-cp311-none-win_amd64.whl", hash = "sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b"}, + {file = "pydantic_core-2.18.4-cp311-none-win_arm64.whl", hash = "sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8"}, + {file = "pydantic_core-2.18.4-cp312-none-win32.whl", hash = "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07"}, + {file = "pydantic_core-2.18.4-cp312-none-win_amd64.whl", hash = "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a"}, + {file = "pydantic_core-2.18.4-cp312-none-win_arm64.whl", hash = "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f"}, + {file = "pydantic_core-2.18.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2"}, + {file = "pydantic_core-2.18.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057"}, + {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b"}, + {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af"}, + {file = "pydantic_core-2.18.4-cp38-none-win32.whl", hash = "sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2"}, + {file = "pydantic_core-2.18.4-cp38-none-win_amd64.whl", hash = "sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443"}, + {file = "pydantic_core-2.18.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528"}, + {file = "pydantic_core-2.18.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23"}, + {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b"}, + {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a"}, + {file = "pydantic_core-2.18.4-cp39-none-win32.whl", hash = "sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d"}, + {file = "pydantic_core-2.18.4-cp39-none-win_amd64.whl", hash = "sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9"}, + {file = "pydantic_core-2.18.4.tar.gz", hash = "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864"}, ] [package.dependencies]