diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..6525aa6 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,127 @@ +version: 2.1 + +jobs: + test_gradient_model_py36: + docker: + - image: circleci/python:3.6.9 + working_directory: ~/project/packages/gradient_boosting_model + steps: + - checkout: + path: ~/project + - run: + name: Run tests with Python 3.6 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py36 + test_gradient_model_py37: + docker: + - image: circleci/python:3.7.6 + working_directory: ~/project/packages/gradient_boosting_model + steps: + - checkout: + path: ~/project + - run: + name: Run tests with Python 3.7 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py37 + test_gradient_model_py38: + docker: + - image: circleci/python:3.8.0 + working_directory: ~/project/packages/gradient_boosting_model + steps: + - checkout: + path: ~/project + - run: + name: Run tests with Python 3.8 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py38 + test_ml_api_py36: + docker: + - image: circleci/python:3.6.9 + - image: postgres + environment: + POSTGRES_USER: test_user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_test + environment: + DB_HOST: localhost + DB_PORT: 5432 + DB_USER: test_user + DB_PASSWORD: password + DB_NAME: ml_api_test + SHADOW_MODE_ACTIVE: true + working_directory: ~/project/packages/ml_api + steps: + - checkout: + path: ~/project + - run: + name: Run API tests with Python 3.6 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py36 + test_ml_api_py37: + docker: + - image: circleci/python:3.7.6 + - image: postgres + environment: + POSTGRES_USER: test_user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_test + environment: + DB_HOST: localhost + DB_PORT: 5432 + DB_USER: test_user + DB_PASSWORD: password + DB_NAME: ml_api_test + SHADOW_MODE_ACTIVE: true + working_directory: ~/project/packages/ml_api + steps: + - checkout: + path: ~/project + - run: + name: Run API tests with Python 3.7 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py37 + test_ml_api_py38: + docker: + - image: circleci/python:3.8.1 + - image: postgres + environment: + POSTGRES_USER: test_user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_test + environment: + DB_HOST: localhost + DB_PORT: 5432 + DB_USER: test_user + DB_PASSWORD: password + DB_NAME: ml_api_test + SHADOW_MODE_ACTIVE: true + working_directory: ~/project/packages/ml_api + steps: + - checkout: + path: ~/project + - run: + name: Run API tests with Python 3.8 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py38 +workflows: + version: 2 + test-all: + jobs: + - test_gradient_model_py36 + - test_gradient_model_py37 + - test_gradient_model_py38 + - test_ml_api_py36 + - test_ml_api_py37 + - test_ml_api_py38 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c853bc5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +exercise_notebooks/* +*/env* +*/venv* +.circleci* +packages/gradient_boosting_model +*.env +*.log +.git +.gitignore +.dockerignore +*.mypy_cache +*.pytest_cache + +### Python ### + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] diff --git a/.gitignore b/.gitignore index 5346ee9..f887dad 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.tox/ # Spyder project settings .spyderproject @@ -124,3 +125,7 @@ test.csv # trained models packages/gradient_boosting_model/gradient_boosting_model/trained_models/*.pkl *.h5 + +# differential test artifacts +packages/ml_api/differential_tests/expected_results/ +packages/ml_api/differential_tests/actual_results/ diff --git a/README.md b/README.md index 95280ac..304b6fb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1 @@ -# testing-and-monitoring-ml-deployments -WIP +Example project for the course "Testing & Monitoring Machine Learning Model Deployments". For setup instructions, see the course lectures. diff --git a/exercise_notebooks/elk_exercise/Dockerfile b/exercise_notebooks/elk_exercise/Dockerfile new file mode 100644 index 0000000..86647f9 --- /dev/null +++ b/exercise_notebooks/elk_exercise/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.7-alpine +WORKDIR /application + +COPY ./requirements.txt requirements.txt +RUN apk add --no-cache \ + gcc \ + libc-dev \ + linux-headers \ + bash; \ + pip install -r requirements.txt; + +COPY . /application + + +EXPOSE 5000 +VOLUME /application +CMD gunicorn --bind 0.0.0.0:5000 \ + --workers=1 \ + --log-config gunicorn_logging.conf \ + --log-level=DEBUG \ + --access-logfile=- \ + --error-logfile=- \ + application:application diff --git a/exercise_notebooks/elk_exercise/app/__init__.py b/exercise_notebooks/elk_exercise/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exercise_notebooks/elk_exercise/app/flask_app.py b/exercise_notebooks/elk_exercise/app/flask_app.py new file mode 100644 index 0000000..e688b45 --- /dev/null +++ b/exercise_notebooks/elk_exercise/app/flask_app.py @@ -0,0 +1,18 @@ +import logging + +from flask import Flask, current_app + + +def index(): + current_app.logger.info('home') + return 'home' + + +def create_app(): + main_app = Flask(__name__) + main_app.add_url_rule('/', 'index', index) + gunicorn_error_logger = logging.getLogger('gunicorn.error') + main_app.logger.addHandler(gunicorn_error_logger) + main_app.logger.setLevel(logging.DEBUG) + + return main_app diff --git a/exercise_notebooks/elk_exercise/application.py b/exercise_notebooks/elk_exercise/application.py new file mode 100644 index 0000000..e03e2a0 --- /dev/null +++ b/exercise_notebooks/elk_exercise/application.py @@ -0,0 +1,7 @@ +from app.flask_app import create_app + + +application = create_app() + +if __name__ == '__main__': + application.run() diff --git a/exercise_notebooks/elk_exercise/docker-compose.yml b/exercise_notebooks/elk_exercise/docker-compose.yml new file mode 100644 index 0000000..7c73164 --- /dev/null +++ b/exercise_notebooks/elk_exercise/docker-compose.yml @@ -0,0 +1,91 @@ +version: '3.2' + +services: + # The environment variable "ELK_VERSION" is used throughout this file to + # specify the version of the images to run. The default is set in the + # '.env' file in this folder. It can be overridden with any normal + # technique for setting environment variables, for example: + # + # ELK_VERSION=7.0.0-beta1 docker-compose up + # + # REF: https://docs.docker.com/compose/compose-file/#variable-substitution + webapp: + build: . + container_name: webapp + expose: + - 5000 + ports: + - 5000:5000 + links: + - logstash + networks: + - elk + depends_on: + - logstash + - kibana + - elasticsearch + volumes: + - ./:/application + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + volumes: + - type: bind + source: ./elasticsearch/config/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme + # Use single node discovery in order to disable production mode and avoid bootstrap checks + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html + discovery.type: single-node + networks: + - elk + + logstash: + image: docker.elastic.co/logstash/logstash:${ELK_VERSION} + volumes: + - type: bind + source: ./logstash/config/logstash.yml + target: /usr/share/logstash/config/logstash.yml + read_only: true + - type: bind + source: ./logstash/pipeline + target: /usr/share/logstash/pipeline + read_only: true + ports: + - "5001:5001" + - "9600:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + image: docker.elastic.co/kibana/kibana:${ELK_VERSION} + volumes: + - type: bind + source: ./kibana/config/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + elasticsearch: \ No newline at end of file diff --git a/exercise_notebooks/elk_exercise/elasticsearch/config/elasticsearch.yml b/exercise_notebooks/elk_exercise/elasticsearch/config/elasticsearch.yml new file mode 100644 index 0000000..cbed5c3 --- /dev/null +++ b/exercise_notebooks/elk_exercise/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,11 @@ +--- +## Default Elasticsearch configuration from Elasticsearch base image. +## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +## X-Pack settings +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +xpack.license.self_generated.type: basic +xpack.security.enabled: true +xpack.monitoring.collection.enabled: true diff --git a/exercise_notebooks/elk_exercise/gunicorn_logging.conf b/exercise_notebooks/elk_exercise/gunicorn_logging.conf new file mode 100644 index 0000000..7ec8e8c --- /dev/null +++ b/exercise_notebooks/elk_exercise/gunicorn_logging.conf @@ -0,0 +1,46 @@ +[loggers] +keys=root, logstash.error, logstash.access + +[handlers] +keys=console, logstash + +[formatters] +keys=generic, access, json + +[logger_root] +level=INFO +handlers=console + +[logger_logstash.error] +level=INFO +handlers=logstash +propagate=1 +qualname=gunicorn.error + +[logger_logstash.access] +level=INFO +handlers=logstash +propagate=0 +qualname=gunicorn.access + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[handler_logstash] +class=logstash.TCPLogstashHandler +formatter=json +args=('logstash', 5001) + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s +datefmt=%Y-%m-%d %H:%M:%S +class=logging.Formatter + +[formatter_access] +format=%(message)s +class=logging.Formatter + +[formatter_json] +class=pythonjsonlogger.jsonlogger.JsonFormatter \ No newline at end of file diff --git a/exercise_notebooks/elk_exercise/kibana/config/kibana.yml b/exercise_notebooks/elk_exercise/kibana/config/kibana.yml new file mode 100644 index 0000000..93380e9 --- /dev/null +++ b/exercise_notebooks/elk_exercise/kibana/config/kibana.yml @@ -0,0 +1,13 @@ +--- +## Default Kibana configuration from Kibana base image. +## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js +# +server.name: kibana +server.host: "0" +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +xpack.monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +# +elasticsearch.username: elastic +elasticsearch.password: changeme \ No newline at end of file diff --git a/exercise_notebooks/elk_exercise/logstash/config/logstash.yml b/exercise_notebooks/elk_exercise/logstash/config/logstash.yml new file mode 100644 index 0000000..a48c35f --- /dev/null +++ b/exercise_notebooks/elk_exercise/logstash/config/logstash.yml @@ -0,0 +1,12 @@ +--- +## Default Logstash configuration from Logstash base image. +## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml +# +http.host: "0.0.0.0" +xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] + +## X-Pack security credentials +# +xpack.monitoring.enabled: true +xpack.monitoring.elasticsearch.username: elastic +xpack.monitoring.elasticsearch.password: changeme diff --git a/exercise_notebooks/elk_exercise/logstash/pipeline/logstash.conf b/exercise_notebooks/elk_exercise/logstash/pipeline/logstash.conf new file mode 100644 index 0000000..7c273f0 --- /dev/null +++ b/exercise_notebooks/elk_exercise/logstash/pipeline/logstash.conf @@ -0,0 +1,17 @@ +input { + tcp { + port => 5001 + tags => ["webapp_logs"] + type => "webapp_logs" + codec => json + } +} + +output { + elasticsearch { + hosts => "elasticsearch:9200" + user => "elastic" + password => "changeme" + index => "webapp_logs-%{+YYYY.MM.dd}" + } +} \ No newline at end of file diff --git a/exercise_notebooks/elk_exercise/requirements.txt b/exercise_notebooks/elk_exercise/requirements.txt new file mode 100644 index 0000000..6607dd0 --- /dev/null +++ b/exercise_notebooks/elk_exercise/requirements.txt @@ -0,0 +1,5 @@ +Flask>=1.1.1,<1.2.0 +python3-logstash>=0.4.80,<0.5.0 +python-json-logger>=0.1.11,<0.2.0 +gunicorn>=20.0.4,<20.1.0 + diff --git a/exercise_notebooks/prometheus_exercise/Dockerfile b/exercise_notebooks/prometheus_exercise/Dockerfile new file mode 100644 index 0000000..4fc5705 --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.7-alpine +WORKDIR /application + +COPY ./requirements.txt requirements.txt +RUN apk add --no-cache \ + gcc \ + libc-dev \ + linux-headers \ + bash; \ + pip install -r requirements.txt; + +COPY . /application + + +EXPOSE 5000 +VOLUME /application +CMD gunicorn --workers=1 --bind 0.0.0.0:5000 application:application diff --git a/exercise_notebooks/prometheus_exercise/app/__init__.py b/exercise_notebooks/prometheus_exercise/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exercise_notebooks/prometheus_exercise/app/flask_app.py b/exercise_notebooks/prometheus_exercise/app/flask_app.py new file mode 100644 index 0000000..9d2357d --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/app/flask_app.py @@ -0,0 +1,45 @@ +import prometheus_client +from flask import Flask +from werkzeug.middleware.dispatcher import DispatcherMiddleware +from app.helpers.middleware import setup_metrics + + +def index(): + return 'home' + + +def cpu(): + # For older machines, you may want to lower + # this range to prevent timeouts. + for i in range(10000): + i**i + + return 'cpu intensive operation complete' + + +def memory(): + d = {} + # For older machines, you may want to lower + # this range to prevent timeouts. + for i in range(10000000): + i = str(i) + i += "xyz" + d[i] = i + + return 'memory intensive operation complete' + + +def create_app(): + main_app = Flask(__name__) + main_app.add_url_rule('/', 'index', index) + main_app.add_url_rule('/cpu', 'cpu', cpu) + main_app.add_url_rule('/memory', 'memory', memory) + setup_metrics(main_app) + + # Add prometheus wsgi middleware to route /metrics requests + app = DispatcherMiddleware( + app=main_app.wsgi_app, + mounts={'/metrics': prometheus_client.make_wsgi_app()} + ) + + return app diff --git a/exercise_notebooks/prometheus_exercise/app/helpers/__init__.py b/exercise_notebooks/prometheus_exercise/app/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exercise_notebooks/prometheus_exercise/app/helpers/middleware.py b/exercise_notebooks/prometheus_exercise/app/helpers/middleware.py new file mode 100644 index 0000000..f547ee3 --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/app/helpers/middleware.py @@ -0,0 +1,58 @@ +from flask import request, Flask +from flask.wrappers import Response +from prometheus_client import Counter, Histogram +import time + + +# Counter and Histogram are examples of default metrics +# available from the prometheus Python client. +REQUEST_COUNT = Counter( + name='http_request_count', + documentation='App Request Count', + labelnames=['app_name', 'method', 'endpoint', 'http_status'] +) +REQUEST_LATENCY = Histogram( + name='http_request_latency_seconds', + documentation='Request latency', + labelnames=['app_name', 'endpoint'] +) + + +def start_timer() -> None: + """Get start time of a request.""" + request._prometheus_metrics_request_start_time = time.time() + + +def stop_timer(response: Response) -> Response: + """Get stop time of a request..""" + request_latency = time.time() - request._prometheus_metrics_request_start_time + REQUEST_LATENCY.labels( + app_name='webapp', + endpoint=request.path).observe(request_latency) + return response + + +def record_request_data(response: Response) -> Response: + """Capture request data. + + Uses the flask request object to extract information such as + the HTTP request method, endpoint and HTTP status. + """ + REQUEST_COUNT.labels( + app_name='webapp', + method=request.method, + endpoint=request.path, + http_status=response.status_code).inc() + return response + + +def setup_metrics(app: Flask) -> None: + """Setup Prometheus metrics. + + This function uses the flask before_request + and after_request hooks to capture metrics + with each HTTP request to the application. + """ + app.before_request(start_timer) + app.after_request(record_request_data) + app.after_request(stop_timer) diff --git a/exercise_notebooks/prometheus_exercise/application.py b/exercise_notebooks/prometheus_exercise/application.py new file mode 100644 index 0000000..e03e2a0 --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/application.py @@ -0,0 +1,7 @@ +from app.flask_app import create_app + + +application = create_app() + +if __name__ == '__main__': + application.run() diff --git a/exercise_notebooks/prometheus_exercise/config/grafana/basic_cadvisor_dashboard.json b/exercise_notebooks/prometheus_exercise/config/grafana/basic_cadvisor_dashboard.json new file mode 100644 index 0000000..b621f02 --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/config/grafana/basic_cadvisor_dashboard.json @@ -0,0 +1,605 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Docker monitoring with Prometheus and cAdvisor with node selection", + "editable": true, + "gnetId": 8321, + "graphTooltip": 1, + "id": 6, + "iteration": 1578215128428, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 0 + }, + "height": "20", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_last_seen", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Running containers", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "mbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 0 + }, + "height": "20", + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})/1024/1024", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 0 + }, + "height": "20", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}[5m]) * 100)", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total CPU Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(container_cpu_user_seconds_total{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}[5m]) * 100", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_usage_bytes{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 21, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(container_cpu_user_seconds_total, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "/([^:]+):.*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "8080", + "value": "8080" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Port", + "multi": false, + "name": "port", + "options": [], + "query": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", + "refresh": 1, + "regex": "/[^:]+:(.*)/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Docker monitoring with node selection", + "uid": "pHUTSjLZk", + "version": 2 +} \ No newline at end of file diff --git a/exercise_notebooks/prometheus_exercise/config/grafana/grafana_flask_basic_dashboard.json b/exercise_notebooks/prometheus_exercise/config/grafana/grafana_flask_basic_dashboard.json new file mode 100644 index 0000000..b5a52ca --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/config/grafana/grafana_flask_basic_dashboard.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(http_request_count_total{job=\"webapp\"}[1m])", + "legendFormat": "{{app_name}} {{endpoint}} {{http_status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(http_request_latency_seconds_sum{job=\"webapp\"}[1m]) / rate(http_request_latency_seconds_count{job=\"webapp\"}[1m])", + "legendFormat": "{{endpoint}} (seconds)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Really Simple Flask Dashboard", + "uid": "q8vgEpLZk", + "version": 4 +} \ No newline at end of file diff --git a/exercise_notebooks/prometheus_exercise/config/prometheus/prometheus.yml b/exercise_notebooks/prometheus_exercise/config/prometheus/prometheus.yml new file mode 100644 index 0000000..19d7bd8 --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/config/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +# my global config +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'my-project' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['prometheus:9090'] + - job_name: 'webapp' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ['webapp:5000'] + + - job_name: 'cadvisor' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['cadvisor:8080'] diff --git a/exercise_notebooks/prometheus_exercise/docker-compose.yml b/exercise_notebooks/prometheus_exercise/docker-compose.yml new file mode 100644 index 0000000..522e59b --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/docker-compose.yml @@ -0,0 +1,51 @@ +version: '3' + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + webapp: + build: . + container_name: webapp + expose: + - 5000 + ports: + - 5000:5000 + volumes: + - ./:/application + prometheus: + image: prom/prometheus + container_name: prometheus + volumes: + - ./config/prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + expose: + - 9090 + ports: + - 9090:9090 + depends_on: + - cadvisor + grafana: + image: grafana/grafana + depends_on: + - prometheus + ports: + - 3000:3000 + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=foobar + - GF_USERS_ALLOW_SIGN_UP=false + + cadvisor: + image: google/cadvisor + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - 8080:8080 diff --git a/exercise_notebooks/prometheus_exercise/requirements.txt b/exercise_notebooks/prometheus_exercise/requirements.txt new file mode 100644 index 0000000..0fbe48f --- /dev/null +++ b/exercise_notebooks/prometheus_exercise/requirements.txt @@ -0,0 +1,4 @@ +Flask>=1.1.1,<1.2.0 +prometheus_client>=0.7.1,<0.8.0 +gunicorn>=20.0.4,<20.1.0 + diff --git a/exercise_notebooks/shadow_mode_exercise/assessing_model_results.ipynb b/exercise_notebooks/shadow_mode_exercise/assessing_model_results.ipynb new file mode 100755 index 0000000..1733a2d --- /dev/null +++ b/exercise_notebooks/shadow_mode_exercise/assessing_model_results.ipynb @@ -0,0 +1,1740 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup\n", + "\n", + "1. Checkout the code at commit: **\"Shadow Mode ML Code - Analyse Results\"**\n", + "\n", + "\n", + "2. Make sure your virtualenv is active, and you have installed the dependencies listed in the requirements.txt located in the same directory as this notebook.\n", + "\n", + "\n", + "3. Make sure you have copied the Kaggle houseprice.csv into the same directory as this notebook. By this point in the course, you should have this csv file here:\n", + "\n", + "testing-and-monitoring-ml-deployments/packages/gradient_boosting_model/gradient_boosting_model/datasets/\n", + "\n", + "\n", + "4. Before running any cells in the notebook, start your docker containers: Open the terminal/Command prompt and navigate to this directory:\n", + "\n", + "testing-and-monitoring-ml-deployments/packages/ml_api\n", + "\n", + "5. Then run: `docker-compose -f docker/docker-compose.yml up -d --build`\n", + "\n", + "(Old window version users, remember to run: `docker-machine start default` followed by `docker-machine env` before the docker compose command)\n", + "\n", + "6. Populate the DB with simulated shadow data by running\n", + "`tox -e generate_predictions` (also from the ml_api directory)\n", + "\n", + "Note: Feel free to run the populate DB command multiple times, however if you generate more than 10k requests then some of the tests below will fail.\n", + "\n", + "**Keep in mind that the populate_database.py script has some element of randomness, so based on that and on that you and I may run this script a different amount of times, the results shown in this notebook are indicative, and may be identical**\n", + "\n", + "Focus instead on the take-home messages derived from the tests." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import scipy.stats as stats\n", + "import seaborn as sns\n", + "from sqlalchemy import create_engine\n", + "\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A reminder that SQLAlchemy DB URIs look like this:\n", + "`postgres+psycop2://myuser:mypassword@hackersdb.example.com:5432/mydatabase`" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "postgres+psycopg2://user:password@localhost:6609/ml_api_dev\n" + ] + } + ], + "source": [ + "# note that this connection string can be found in the app DevelopmentConfig.\n", + "\n", + "# to save hassle with updating the PATH so we can import the config object,\n", + "# we write it out in full here:\n", + "\n", + "db_uri = \"postgres+psycopg2://user:password@localhost:6609/ml_api_dev\"\n", + "print(db_uri)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# connect to the database\n", + "\n", + "engine = create_engine(db_uri)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's load our live data, that is, the predictions we generated and are stored\n", + "# in our database (takes about 30 seconds to run)\n", + "\n", + "sql_df = pd.read_sql_table(\"gradient_boosting_model_predictions\", con=engine)\n", + "\n", + "# munge json array of inputs from postgres jsonb field.\n", + "inputs_df = sql_df.inputs.apply(\n", + " lambda row: pd.DataFrame(json.loads(row))).tolist()\n", + "\n", + "live_data = pd.concat(inputs_df, sort=False)\n", + "outputs_df = sql_df.outputs.apply(lambda row: pd.Series(json.loads(row)))\n", + "live_data['SalePrice'] = outputs_df.values\n", + "live_data.reset_index(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# rename live columns to match training data file\n", + "\n", + "SECONDARY_VARIABLES_TO_RENAME = {\n", + " \"FirstFlrSF\": \"1stFlrSF\",\n", + " \"SecondFlrSF\": \"2ndFlrSF\",\n", + " \"ThreeSsnPortch\": \"3SsnPorch\",\n", + "}\n", + "live_data.rename(columns=SECONDARY_VARIABLES_TO_RENAME, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# columns ==> inputs to the new model\n", + "\n", + "model_features = ['LotArea', 'OverallQual', 'YearRemodAdd',\n", + " 'BsmtQual', 'BsmtFinSF1','TotalBsmtSF',\n", + " '1stFlrSF', '2ndFlrSF', 'GrLivArea',\n", + " 'GarageCars', 'YrSold']\n", + "\n", + "# From the live data, we select only those variables that\n", + "# are actually used in the model\n", + "# and the predictions (SalePrice)\n", + "\n", + "live_data = live_data[model_features + ['SalePrice']]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
LotAreaOverallQualYearRemodAddBsmtQualBsmtFinSF1TotalBsmtSF1stFlrSF2ndFlrSFGrLivAreaGarageCarsYrSoldSalePrice
1228378151969TA280.0720.0212562713082.02008124359.797157
12291529961976TA0.01444.04860120414442.02008157332.471992
12302205051974Gd0.0896.02384174817922.02008167039.397270
1231805241969Gd252.0936.0459511639361.02008121015.582065
12322850351971Gd119.0864.014033698642.02008145224.114862
\n", + "
" + ], + "text/plain": [ + " LotArea OverallQual YearRemodAdd BsmtQual BsmtFinSF1 TotalBsmtSF \\\n", + "1228 3781 5 1969 TA 280.0 720.0 \n", + "1229 15299 6 1976 TA 0.0 1444.0 \n", + "1230 22050 5 1974 Gd 0.0 896.0 \n", + "1231 8052 4 1969 Gd 252.0 936.0 \n", + "1232 28503 5 1971 Gd 119.0 864.0 \n", + "\n", + " 1stFlrSF 2ndFlrSF GrLivArea GarageCars YrSold SalePrice \n", + "1228 2125 627 1308 2.0 2008 124359.797157 \n", + "1229 4860 1204 1444 2.0 2008 157332.471992 \n", + "1230 2384 1748 1792 2.0 2008 167039.397270 \n", + "1231 4595 1163 936 1.0 2008 121015.582065 \n", + "1232 1403 369 864 2.0 2008 145224.114862 " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "live_data.tail()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
LotAreaOverallQualYearRemodAddBsmtQualBsmtFinSF1TotalBsmtSF1stFlrSF2ndFlrSFGrLivAreaGarageCarsYrSoldSalePrice
0845072003Gd706856856854171022008208500
1960061976Gd978126212620126222007181500
21125072002Gd486920920866178622008223500
3955071970TA216756961756171732006140000
41426082000Gd655114511451053219832008250000
\n", + "
" + ], + "text/plain": [ + " LotArea OverallQual YearRemodAdd BsmtQual BsmtFinSF1 TotalBsmtSF \\\n", + "0 8450 7 2003 Gd 706 856 \n", + "1 9600 6 1976 Gd 978 1262 \n", + "2 11250 7 2002 Gd 486 920 \n", + "3 9550 7 1970 TA 216 756 \n", + "4 14260 8 2000 Gd 655 1145 \n", + "\n", + " 1stFlrSF 2ndFlrSF GrLivArea GarageCars YrSold SalePrice \n", + "0 856 854 1710 2 2008 208500 \n", + "1 1262 0 1262 2 2007 181500 \n", + "2 920 866 1786 2 2008 223500 \n", + "3 961 756 1717 3 2006 140000 \n", + "4 1145 1053 2198 3 2008 250000 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Now let's load the data we used to train the model\n", + "\n", + "# remember to copy the houseprice.csv data to the directory of this Jupyter notebook\n", + "# or alternatively change the path below to find the file\n", + "\n", + "\n", + "# load needed columns + target\n", + "train_data = pd.read_csv('houseprice.csv',\n", + " usecols=model_features + ['SalePrice'])\n", + "\n", + "\n", + "train_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((1460, 12), (1233, 12))" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's compare the shapes of both live and training data:\n", + "\n", + "train_data.shape, live_data.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Take this outputs as a demo of what to expect, and don't worry too much if the results are not identical. The more times you run the populate_database.py script, the larger the live_data will be." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Checks\n", + "\n", + "### Input checks - categorical variables\n", + "\n", + "We only have one categorical variable among our features:\n", + "\n", + "BsmtQual (Categorical): Evaluates the height of the basement\n", + "\n", + " Ex\tExcellent (100+ inches)\t\n", + " Gd\tGood (90-99 inches)\n", + " TA\tTypical (80-89 inches)\n", + " Fa\tFair (70-79 inches)\n", + " Po\tPoor (<70 inches)\n", + " NA\tNo Basement\n", + " \n", + "These are the values allowed according to how the variable was defined, and as we can see it can also take missing values.\n", + "\n", + "\n", + "You can find more details about the variable definitions and their permitted values here:\n", + "\n", + "[Source](http://bee-fore.s3-eu-west-1.amazonaws.com/datasets/62.txt)\n", + "\n", + "The first test aims to corroborate that live data takes only the permitted values. We can do as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Gd', 'TA', 'Ex', nan, 'Fa'], dtype=object)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's evaluate the unique values in our training data\n", + "\n", + "train_data['BsmtQual'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['TA', 'Gd', 'Ex', 'Fa'], dtype=object)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's now evaluate the unique values in our live data\n", + "\n", + "live_data['BsmtQual'].unique()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that both our training and live data take only the permitted values. We also see that the category Po is not present in either of the data sets, which is curious. Probably there were not that many basements in Poor conditions.\n", + "\n", + "If we wanted, we could write a short test as follows, and any number bigger than 0 would fail:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len([x for x in live_data['BsmtQual'] if x not in ['Gd', 'TA', 'Ex', np.nan, 'Fa']])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Value range checks - Numerical variables\n", + "\n", + "We have a numerical and discrete variable in our data set that can only take 1 of the permitted values:\n", + "\n", + "OverallQual (Ordinal): Rates the overall material and finish of the house\n", + "\n", + " 10\tVery Excellent\n", + " 9\tExcellent\n", + " 8\tVery Good\n", + " 7\tGood\n", + " 6\tAbove Average\n", + " 5\tAverage\n", + " 4\tBelow Average\n", + " 3\tFair\n", + " 2\tPoor\n", + " 1\tVery Poor\n", + " \n", + "Given that the number of different permitted values is small, we could do an input check as we did with BsmtQual, or, for the sake of the demo, we can check value ranges:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "min 1\n", + "max 10\n", + "Name: OverallQual, dtype: int64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# we know that the min and max values are 1 and 10 according to the\n", + "# variable definition, we can indeed check that as follows:\n", + "\n", + "train_data['OverallQual'].agg(['min', 'max'])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "min 2\n", + "max 10\n", + "Name: OverallQual, dtype: int64" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's repeat for the live variable \n", + "\n", + "live_data['OverallQual'].agg(['min', 'max'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The values are within the range, so we have no reason to worry.\n", + "\n", + "We could write a small test as follows and any return bigger than 0 should fail:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len([x for x in live_data['OverallQual'] if x >10 or x <1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Excellent, we see that the variables in both our train and live data take values within the permitted range.\n", + "\n", + "**Note** we could also check value ranges for the remaining of the numerical variables, we just need to be confident that the variables can take only values within the range we want to evaluate.\n", + "\n", + "\n", + "### Missing value checks\n", + "\n", + "We know from our research phase, that these variables should not take missing data:\n", + "\n", + "numerical_na_not_allowed:\n", + " - LotArea\n", + " - OverallQual\n", + " - YearRemodAdd\n", + " - BsmtFinSF1\n", + " - TotalBsmtSF\n", + " - FirstFlrSF\n", + " - SecondFlrSF\n", + " - GrLivArea\n", + " - GarageCars\n", + " - YrSold" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# let's capture the above in a list\n", + "\n", + "numerical_na_not_allowed = ['LotArea', 'OverallQual', 'YearRemodAdd',\n", + " 'BsmtFinSF1', 'TotalBsmtSF', 'GarageCars',\n", + " 'YrSold']" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LotArea 0.0\n", + "OverallQual 0.0\n", + "YearRemodAdd 0.0\n", + "BsmtFinSF1 0.0\n", + "TotalBsmtSF 0.0\n", + "GarageCars 0.0\n", + "YrSold 0.0\n", + "dtype: float64" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# they should not take missing values in our train data\n", + "# let's check that:\n", + "\n", + "train_data[numerical_na_not_allowed].isnull().mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LotArea 0.0\n", + "OverallQual 0.0\n", + "YearRemodAdd 0.0\n", + "BsmtFinSF1 0.0\n", + "TotalBsmtSF 0.0\n", + "GarageCars 0.0\n", + "YrSold 0.0\n", + "dtype: float64" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's check if that is the case in our live data\n", + "# as well\n", + "\n", + "live_data[numerical_na_not_allowed].isnull().mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perfect, as we can see, none of the variables that we are receiving live, take missing data where missing data is not expected. \n", + "\n", + "If we had gotten a value other than zero. we should probably investigate what's going on. We could have a bug in our code, or the variable could have changed its definition." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Distribution checks\n", + "\n", + "### First, check proportion of missing values\n", + "\n", + "We know that BsmtQual can take missing data. We can, and should test, whether the proportion of missing data that we are getting live, is the same that we considered in our training data set. We can do so, using the Chi-square test, as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 False\n", + "1 False\n", + "2 False\n", + "3 False\n", + "4 False\n", + " ... \n", + "1455 False\n", + "1456 False\n", + "1457 False\n", + "1458 False\n", + "1459 False\n", + "Name: BsmtQual, Length: 1460, dtype: bool" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check number of na in the variable in the train set\n", + "\n", + "train_data['BsmtQual'].isnull()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 False\n", + "1 False\n", + "2 False\n", + "3 False\n", + "4 False\n", + " ... \n", + "1228 False\n", + "1229 False\n", + "1230 False\n", + "1231 False\n", + "1232 False\n", + "Name: BsmtQual, Length: 1233, dtype: bool" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check number of na in the live data\n", + "\n", + "live_data['BsmtQual'].isnull()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that we get zero, that already tells us that something may not be quite right.\n", + "\n", + "We can go ahead and test this with a statistical test. The fisher exact test as implemented in Scipy. In order to run this test, we need to create a 2x2 table, where live and train data are the columns and 0, or 1 indicating missing values are the rows, and the numbers represent the number of observations within each cell.\n", + "\n", + "Let's go ahead and do that:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# first, make 2 binary variables where we indicate if the variable took\n", + "# a missing value or not:\n", + "\n", + "train_data['BsmtQual_na'] = np.where(train_data['BsmtQual'].isnull(), 1, 0)\n", + "live_data['BsmtQual_na'] = np.where(live_data['BsmtQual'].isnull(), 1, 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
trainlive
BsmtQual_na
014231233.0
137NaN
\n", + "
" + ], + "text/plain": [ + " train live\n", + "BsmtQual_na \n", + "0 1423 1233.0\n", + "1 37 NaN" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# now let's generate the 2x2 table:\n", + "\n", + "ct = pd.concat([\n", + " train_data.groupby('BsmtQual_na')['BsmtQual_na'].count(),\n", + " live_data.groupby('BsmtQual_na')['BsmtQual_na'].count()\n", + "], axis=1)\n", + "\n", + "ct.columns = ['train', 'live']\n", + "\n", + "ct" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As expected, the table contains missing data in the second row of the live data, because as we saw, there were no missing values. We can't pass a table with np.nan to the test, so we need to fill it out." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
trainlive
BsmtQual_na
014231233.0
1370.0
\n", + "
" + ], + "text/plain": [ + " train live\n", + "BsmtQual_na \n", + "0 1423 1233.0\n", + "1 37 0.0" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# now we need to build a contingency table for this test\n", + "\n", + "ct.fillna(0, inplace=True)\n", + "ct" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1.2719419961163252e-10" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# and now we compare frequencies with chi-square\n", + "\n", + "oddsratio, pvalue = stats.fisher_exact(ct)\n", + "\n", + "pvalue" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Aha, the p_value indicates that the distributions are significantly different. Which is not surprising given that we got 0 missing values in our live data. \n", + "\n", + "Is this a problem?\n", + "\n", + "We should start digging in our production code to see if we introduced either a bug, an exception or some sort of data filtering. If none if this happened, maybe the variable definition changed, for example.\n", + "\n", + "\n", + "### Categorical distribution test\n", + "\n", + "Similarly, we can use the same test to evaluate the proportion of categories we are getting in the categorical variables of our live data. \n", + "\n", + "As this contingency table is not a 2x2 table, we need to use a different implementation of the test, as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TA 649\n", + "Gd 618\n", + "Ex 121\n", + "Missing 37\n", + "Fa 35\n", + "Name: BsmtQual, dtype: int64" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# we first need the expected frequencies in each category\n", + "# that is the number of observations per category in the train data\n", + "\n", + "# we fill missing values with the string \"Missing\" as we did in our\n", + "# preprocessing steps\n", + "\n", + "train_data['BsmtQual'].fillna('Missing').value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TA 555\n", + "Gd 489\n", + "Ex 129\n", + "Fa 60\n", + "Name: BsmtQual, dtype: int64" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# now we need the received frequencies in each category\n", + "# in the live data data\n", + "\n", + "# we also fill in missing values with the string \"Missing\"\n", + "\n", + "live_data['BsmtQual'].fillna('Missing').value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "# we need to create 2 series of the same size\n", + "# with the counts we displayed in the previous cells\n", + "\n", + "ct = train_data['BsmtQual'].fillna('Missing').value_counts()\n", + "cl = live_data['BsmtQual'].fillna('Missing').value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TA 555\n", + "Gd 489\n", + "Ex 129\n", + "Fa 60\n", + "Name: BsmtQual, dtype: int64" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cl" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TA 555.0\n", + "Gd 489.0\n", + "Ex 129.0\n", + "Fa 60.0\n", + "Missing 0.1\n", + "Name: BsmtQual, dtype: float64" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's add the missing category to the live data\n", + "# (I add 0.1 to avoid divide by zero errors in the test below)\n", + "\n", + "cl['Missing'] = 0.1\n", + "cl" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# let's sort the index\n", + "\n", + "ct.sort_index(inplace=True)\n", + "cl.sort_index(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ex 121\n", + "Fa 35\n", + "Gd 618\n", + "Missing 37\n", + "TA 649\n", + "Name: BsmtQual, dtype: int64" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ct" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ex 129.0\n", + "Fa 60.0\n", + "Gd 489.0\n", + "Missing 0.1\n", + "TA 555.0\n", + "Name: BsmtQual, dtype: float64" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cl" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Power_divergenceResult(statistic=13676.964186265019, pvalue=0.0)" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# and now we compare frequencies with chi-square\n", + "\n", + "stats.chisquare(f_obs=ct,\n", + " f_exp=cl)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see from the test, the p-value is 0, so the distributions in the live and train data are significantly different, which is what we expect given that we are not getting missing data in our live variable.\n", + "\n", + "For variables that are discrete in nature, like OverallQual, we could also use the above test, to compare the distributions. Let's do that:" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Power_divergenceResult(statistic=137.5069575787079, pvalue=3.3726853043045783e-25)" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create value counts series\n", + "\n", + "ct = train_data['OverallQual'].value_counts()\n", + "cl = live_data['OverallQual'].value_counts()\n", + "\n", + "cl[1] = 0.1\n", + "\n", + "ct.sort_index(inplace=True)\n", + "cl.sort_index(inplace=True)\n", + "\n", + "# and now we compare frequencies with chi-square\n", + "\n", + "stats.chisquare(f_obs=ct,\n", + " f_exp=cl)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This variable as well shows a different category distribution in the live data, compared to train data." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXCklEQVR4nO3df5BV5X3H8fdHJPxQArhud5DFLk2IAaxZ40ro2EypJoqYEdKhaNpJqNrQNmSSTtq0JK2NmZaZNWliJa0aGkywTYNE40Cjbf2FzaSNkkUJgpiIZiNLEFciG39X9Ns/7gO5kIV7d++9Z5fHz2vmzj3neZ5zz3eX5bNnn3vuOYoIzMwsL8cNdQFmZlZ/Dnczsww53M3MMuRwNzPLkMPdzCxDDnczswwdP9QFAJx88snR1tY21GWYmR1TNm3a9ExENPfXNyzCva2tja6urqEuw8zsmCLpJ0fq87SMmVmGHO5mZhlyuJuZZajqOXdJI4AuYFdEvE/SVGAN0ARsAj4YEf8naRRwE3AWsBe4JCK66165mb3hvfrqq/T09PDyyy8PdSkNNXr0aFpbWxk5cmTV2wzkDdWPA9uBN6f1q4FrImKNpBuAK4Dr0/OzEfFWSZemcZcMYD9mZlXp6elh3LhxtLW1IWmoy2mIiGDv3r309PQwderUqreralpGUitwEfCVtC7gXOCWNGQ1sCAtz0/rpP7zlOt33cyG1Msvv0xTU1O2wQ4giaampgH/dVLtnPs/AH8BvJ7Wm4B9EbE/rfcAk9PyZGAnQOrvS+PNzOou52A/YDBfY8Vwl/Q+4OmI2DSYoo7yukskdUnq6u3tredLm5kVYt++fVx33XUD3m7evHns27evARX9QjVz7ucAF0uaB4ymNOd+LTBB0vHp6LwV2JXG7wKmAD2SjgfGU3pj9RARsRJYCdDR0eE7htiAtS27veKY7s6LCqjEhotqfiYGotLPz4Fw/8hHPnJI+/79+zn++CPH6x133FGX+o6m4pF7RHwqIlojog24FLg3In4f2AAsTMMWA+vS8vq0Tuq/N3y7JzPL0LJly3j88cdpb2/n7LPP5t3vfjcXX3wxM2bMAGDBggWcddZZzJw5k5UrVx7crq2tjWeeeYbu7m6mT5/Ohz/8YWbOnMn555/PSy+9VJfaajnP/S+BT0jaQWlOfVVqXwU0pfZPAMtqK9HMbHjq7OzkLW95C5s3b+bzn/88Dz74INdeey0/+tGPALjxxhvZtGkTXV1drFixgr17f2kSg8cee4ylS5eybds2JkyYwK233lqX2gZ0bZmIuA+4Ly0/AczqZ8zLwO/WoTYzs2PKrFmzDjldccWKFdx2220A7Ny5k8cee4ympkPPL5k6dSrt7e0AnHXWWXR3d9ellmFx4TAzsxyccMIJB5fvu+8+7r77br73ve8xduxY5syZ0+/pjKNGjTq4PGLEiGExLWNm9oY2btw4nnvuuX77+vr6mDhxImPHjuXRRx/l/vvvL7Q2H7mbmQ1SU1MT55xzDqeffjpjxoyhpaXlYN/cuXO54YYbmD59OqeddhqzZ88utDYNhxNZOjo6wtdzt4HyqZC2fft2pk+fPtRlFKK/r1XSpojo6G+8p2XMzDLkcDczy5DD3cwsQw53M7MM+WwZsxpVemPXb+raUPCRu5lZhhzuZmY1OPHEEwH46U9/ysKFCyuMLo6nZcwsH1eNr/Pr9VU99JRTTuGWW26pPLAgPnI3M6uD7u5uTj/9dABmz57Ntm3bDvbNmTOHrq4uXnjhBS6//HJmzZrFmWeeybp16470cjVzuJuZ1dkll1zC2rVrAdi9eze7d++mo6OD5cuXc+6557Jx40Y2bNjAJz/5SV544YWG1OBwNzOrs0WLFh2colm7du3Bufg777yTzs5O2tvbD14l8sknn2xIDZ5zNzOrs8mTJ9PU1MSWLVu4+eabueGGGwCICG699VZOO+20htfgI3czswa45JJL+NznPkdfXx9nnHEGABdccAFf+tKXOHDBxoceeqhh+68Y7pJGS9oo6QeStkn6bGr/mqQfS9qcHu2pXZJWSNohaYukdzasejOzYWrhwoWsWbOGRYsWHWy78sorefXVVznjjDOYOXMmV155ZcP2X820zCvAuRHxvKSRwHcl/Ufq+2REHH7uz4XAtPR4F3B9ejYza6wBnLpYL88//zxQuun11q1bD7a3tLSwf//+Q8aOGTOGL3/5y4XUVfHIPUqeT6sj0+NoF4GfD9yUtrsfmCBpUu2lmplZtap6Q1XSCGAT8FbgnyLiAUl/AiyX9DfAPcCyiHgFmAzsLNu8J7XtrmvlZtWo9KGWITjSMytCVW+oRsRrEdEOtAKzJJ0OfAp4O3A2cBLwlwPZsaQlkrokdfX29g6wbDMzO5oBnS0TEfuADcDciNidpl5eAb4KzErDdgFTyjZrTW2Hv9bKiOiIiI7m5ubBVW9mb3jD4VahjTaYr7Gas2WaJU1Iy2OA9wKPHphHlyRgAXDgnYT1wIfSWTOzgb6I8JSMmdXd6NGj2bt3b9YBHxHs3buX0aNHD2i7aubcJwGr07z7ccDaiPi2pHslNQMCNgN/nMbfAcwDdgAvApcNqCIzsyq1trbS09ND7lO7o0ePprW1dUDbVAz3iNgCnNlP+7lHGB/A0gFVYWY2CCNHjmTq1KlDXcaw5E+ompllyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYaqulmHmdWg0g1DoOabhrQtu73imO7Oi2rahx1bfORuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpahau6hOlrSRkk/kLRN0mdT+1RJD0jaIelmSW9K7aPS+o7U39bYL8HMzA5XzZH7K8C5EfEOoB2Ym258fTVwTUS8FXgWuCKNvwJ4NrVfk8aZmVmBKoZ7lDyfVkemRwDnArek9tXAgrQ8P62T+s+TpLpVbGZmFVU15y5phKTNwNPAXcDjwL6I2J+G9ACT0/JkYCdA6u8DmupZtJmZHV1V4R4Rr0VEO9AKzALeXuuOJS2R1CWpq7e3t9aXMzOzMgM6WyYi9gEbgN8AJkg6cPmCVmBXWt4FTAFI/eOBvf281sqI6IiIjubm5kGWb2Zm/anmbJlmSRPS8hjgvcB2SiG/MA1bDKxLy+vTOqn/3oiIehZtZmZHV82FwyYBqyWNoPTLYG1EfFvSI8AaSX8HPASsSuNXAf8iaQfwM+DSBtRtZmZHUTHcI2ILcGY/7U9Qmn8/vP1l4HfrUp2ZmQ2KP6FqZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpahau6hOkXSBkmPSNom6eOp/SpJuyRtTo95Zdt8StIOST+UdEEjvwAzM/tl1dxDdT/wZxHxoKRxwCZJd6W+ayLi78sHS5pB6b6pM4FTgLslvS0iXqtn4WZmdmQVj9wjYndEPJiWnwO2A5OPssl8YE1EvBIRPwZ20M+9Vs3MrHEGNOcuqY3SzbIfSE0flbRF0o2SJqa2ycDOss16OPovAzMzq7Oqw13SicCtwJ9GxM+B64G3AO3AbuALA9mxpCWSuiR19fb2DmRTMzOroKpwlzSSUrB/PSK+BRAReyLitYh4HfhnfjH1sguYUrZ5a2o7RESsjIiOiOhobm6u5WswM7PDVHO2jIBVwPaI+GJZ+6SyYe8Htqbl9cClkkZJmgpMAzbWr2QzM6ukmrNlzgE+CDwsaXNq+zTwAUntQADdwB8BRMQ2SWuBRyidabPUZ8qYmRWrYrhHxHcB9dN1x1G2WQ4sr6EuMzOrgT+hamaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGqjnP3cxycNX4Cv19xdRhhfCRu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZquYeqlMkbZD0iKRtkj6e2k+SdJekx9LzxNQuSSsk7ZC0RdI7G/1FmJnZoao5ct8P/FlEzABmA0slzQCWAfdExDTgnrQOcCGlm2JPA5YA19e9ajMzO6qK4R4RuyPiwbT8HLAdmAzMB1anYauBBWl5PnBTlNwPTJA0qe6Vm5nZEQ1ozl1SG3Am8ADQEhG7U9dTQEtangzsLNusJ7WZmVlBqg53SScCtwJ/GhE/L++LiABiIDuWtERSl6Su3t7egWxqZmYVVBXukkZSCvavR8S3UvOeA9Mt6fnp1L4LmFK2eWtqO0RErIyIjojoaG5uHmz9ZmbWj2rOlhGwCtgeEV8s61oPLE7Li4F1Ze0fSmfNzAb6yqZvzMysANXciekc4IPAw5I2p7ZPA53AWklXAD8BFqW+O4B5wA7gReCyulZsZmYVVQz3iPguoCN0n9fP+ACW1liXmZnVwJ9QNTPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMlTNVSHNzKrStuz2o/Z3d15UUCXmI3czsww53M3MMuRwNzPLkMPdzCxD1dxD9UZJT0vaWtZ2laRdkjanx7yyvk9J2iHph5IuaFThZmZ2ZNUcuX8NmNtP+zUR0Z4edwBImgFcCsxM21wnaUS9ijUzs+pUDPeI+A7wsypfbz6wJiJeiYgfU7pJ9qwa6jMzs0GoZc79o5K2pGmbialtMrCzbExPajMzswIN9kNM1wN/C0R6/gJw+UBeQNISYAnAqaeeOsgyzOyYctX4Ksb0Nb6ON4BBHblHxJ6IeC0iXgf+mV9MvewCppQNbU1t/b3GyojoiIiO5ubmwZRhZmZHMKhwlzSpbPX9wIEzadYDl0oaJWkqMA3YWFuJZmY2UBWnZSR9A5gDnCypB/gMMEdSO6VpmW7gjwAiYpuktcAjwH5gaUS81pjSzczsSCqGe0R8oJ/mVUcZvxxYXktRZmZWG39C1cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQ4O9E5O9wbUtu/2o/d2dFxVUiZn1x0fuZmYZcribmWXI4W5mlqGK4S7pRklPS9pa1naSpLskPZaeJ6Z2SVohaYekLZLe2cjizcysf9UcuX8NmHtY2zLgnoiYBtyT1gEupHRT7GnAEuD6+pRpZmYDUTHcI+I7wM8Oa54PrE7Lq4EFZe03Rcn9wARJk+pVrJmZVWewc+4tEbE7LT8FtKTlycDOsnE9qc3MzApU8xuqERFADHQ7SUskdUnq6u3trbUMMzMrM9hw33NguiU9P53adwFTysa1prZfEhErI6IjIjqam5sHWYaZmfVnsOG+HliclhcD68raP5TOmpkN9JVN35iZWUEqXn5A0jeAOcDJknqAzwCdwFpJVwA/ARal4XcA84AdwIvAZQ2o2czMKqgY7hHxgSN0ndfP2ACW1lqUZeCq8VWM6Wt8HWZvUP6EqplZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhipeFdLMLDuVrlqawRVLfeRuZpYhh7uZWYYc7mZmGXK4m5llqKY3VCV1A88BrwH7I6JD0knAzUAb0A0siohnayvTzMwGoh5H7r8dEe0R0ZHWlwH3RMQ04J60bmZmBWrEtMx8YHVaXg0saMA+zMzsKGoN9wDulLRJ0pLU1hIRu9PyU0BLfxtKWiKpS1JXb29vjWWYmVm5Wj/E9JsRsUvSrwB3SXq0vDMiQlL0t2FErARWAnR0dPQ7xszMBqemI/eI2JWenwZuA2YBeyRNAkjPT9dapJmZDcygw13SCZLGHVgGzge2AuuBxWnYYmBdrUWamdnA1DIt0wLcJunA6/xbRPynpO8DayVdAfwEWFR7mWZm1WlbdnvFMd2jCyhkiA063CPiCeAd/bTvBc6rpSgzM6uNP6FqZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYZqveSvDYFK187o7ryooErMbLhyuJuZNcBQH4R5WsbMLEMOdzOzDHlaJkdXja/Q31dMHWY2ZBzu9VQpVMHBamaF8LSMmVmGGnbkLmkucC0wAvhKRHQ2al9mZsecBv+l35Ajd0kjgH8CLgRmAB+QNKMR+zIzs1/WqCP3WcCOdJ9VJK0B5gOPDPYFh/qc0apqeAPcdNfMjg2KiPq/qLQQmBsRf5jWPwi8KyI+WjZmCbAkrZ4G/LDG3Z4MPFPja9RqONQAw6OO4VADDI86hkMNMDzqGA41wPCoox41/GpENPfXMWRny0TESmBlvV5PUldEdNTr9Y7VGoZLHcOhhuFSx3CoYbjUMRxqGC51NLqGRp0tswuYUrbemtrMzKwAjQr37wPTJE2V9CbgUmB9g/ZlZmaHaci0TETsl/RR4L8onQp5Y0Rsa8S+ytRtiqcGw6EGGB51DIcaYHjUMRxqgOFRx3CoAYZHHQ2toSFvqJqZ2dDyJ1TNzDLkcDczy5DD3cwsQw73Gkh6u6TzJJ14WPvcguuYJenstDxD0ickzSuyhn5qumko959q+M30vTi/wH2+S9Kb0/IYSZ+V9O+SrpZUxcVE6lbHxyRNqTyyoTW8SdKHJL0nrf+epH+UtFTSyALr+DVJfy7pWklflPTHB/6NcpbdG6qSLouIrxawn48BS4HtQDvw8YhYl/oejIh3NrqGtK/PULqGz/HAXcC7gA3Ae4H/iojlBdRw+GmuAn4buBcgIi5udA2pjo0RMSstf5jSv89twPnAvxdx8TpJ24B3pDPGVgIvArcA56X232l0DamOPuAF4HHgG8A3I6K3iH2X1fB1Sj+XY4F9wInAtyh9LxQRiwuo4WPA+4DvAPOAh1It7wc+EhH3NbqGIRMRWT2AJwvaz8PAiWm5DeiiFPAADxX49T5M6XTTscDPgTen9jHAloJqeBD4V2AO8FvpeXda/q0CvxcPlS1/H2hOyycADxdUw/by78thfZuL/F5Q+sv8fGAV0Av8J7AYGFdQDVvS8/HAHmBEWleBP5sPl+13LHBfWj614P+n44FO4FHgZ8BeSgeGncCERuzzmJyWkbTlCI+HgZaCyjguIp4HiIhuSoF2oaQvUvrhLcr+iHgtIl4EHo+In6eaXgJeL6iGDmAT8FdAX5SOhl6KiP+OiP8uqAaA4yRNlNRE6ciwFyAiXgD2F1TDVkmXpeUfSOoAkPQ24NWCagCIiHg9Iu6MiCuAU4DrgLnAEwXVcFz6EOM4SsF6YFpqFFDYtAy/+DzPKEp/PRARTxZcw1rgWWBORJwUEU2U/rp9NvXV3bF6J6YW4AJK35hyAv63oBr2SGqPiM0AEfG8pPcBNwK/XlANAP8naWwK97MONKb53ULCPSJeB66R9M30vIeh+dkaT+mXjICQNCkidqf3RIr6hfuHwLWS/prSRaG+J2knsDP1FeWQrzciXqX0KfH1ksYWVMMqSkeqIyj94v+mpCeA2cCagmr4CvB9SQ8A7wauBpDUTOkIuihtEXF1eUNEPAVcLenyRuzwmJxzl7QK+GpEfLefvn+LiN8roIZWSkfNT/XTd05E/E+ja0j7GhURr/TTfjIwKSIeLqKOw/Z9EXBORHy66H33J4VZS0T8uMB9vhmYSumXXE9E7Clq32n/b4uIHxW5zyPUcQpARPxU0gTgPZSmTjcWWMNMYDqwNSIeLWq/h9VwJ3A3sPrAz4KkFuAPgPdGxHvqvs9jMdzNzI4lkiYCyyjd1+JXUvMeSn9NdUbE4bMQte/T4W5mNnQadYafw93MbAhJejIiTq336x6rb6iamR0zJG05UhcNOsPP4W5m1niFn+HncDcza7xvU/rQ4+bDOyTd14gdes7dzCxDx+QnVM3M7Ogc7mZmGXK4m5llyOFuZpYhh7uZWYb+H1TCZQWfUcBOAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# to investigate this further we can plot the number of observations\n", + "# per value, in train and live data, as follows:\n", + "\n", + "tmp = pd.concat([ct,cl], axis=1)\n", + "tmp.columns = ['train', 'live']\n", + "tmp.plot.bar()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " We do see that there is a slight difference in the distribution of the 2 sources of data: we expected less observations for the values 2 and 9, and more for the values 1 and 10 in the live data.\n", + " \n", + "As a follow up, we need to dig out the source of these discrepancies. Either we have a bug, or some sort of data filtering in our production code, or the variables may have changed their definitions\n", + "\n", + "For continuous variables, we can use Kolmogorov-Smirnov as follows:\n", + "\n", + "### Numerical variable tests" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmIAAAJOCAYAAAAUOGurAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdebgcVZ3/8feHhB0kQPQSAQkMjIgyIGYABwYjuLCNYUZBFJEwOLiA4hhHgvpTdFxARTYVBwUJsm8KA4yKwHUZTdQAEhaRGIMkkxBCFgibBL6/P87pUPem7719by/V3ffzep5+btWp6qpv963TdeqcU3UUEZiZmZlZ661TdgBmZmZmo5ULYmZmZmYlcUHMzMzMrCQuiJmZmZmVxAUxMzMzs5K4IGZmZmZWEhfEupSkqZJ+WXYcZmWqNR9ImixpQStiMutEklZJ2qHsOLqRC2IlkXSipN9JelbSRTW+Z76kNxXmJ0qKnEEqr98PI4Ypku6S9LikpZJuk7R9XnaqpOf6bfsTw/6gZsMkaX1JF0h6SNIT+Rg9qEHbDklPFo7pFcN4776SfiVppaRlkv5X0t/nZVMlPd8vv3yjETHb6JN/65/Ox9FySTdJ2rbB+zhV0iX90nolPdPvOH49QERsEhHzatz2iM4tko7IeewpSb2N/LztbGzZAYxi/wd8AXgrsGGd2xoXEatrXVnSWGAicDHwL8BtwCbAW4DnC6teGRHvqTM2s+EaCzwMvAH4C3AwcJWkXSNifgO2v1tEzK115ZxfNgJuBD4IXAWsB/wj8Gxh1V9HxL4NiM8M4J8i4qeSNgC+BZwLHNaC/Z4YEd8d6Zsl7cjIzy3LgLOAnYH9RxpDp3GNWEki4rqI+CHwWDFd0nhJN0paka+6fyFpHUnfB14B/PdIaqdyTcAJkh4EHgR2B/4cEbdG8kREXBsRf2nUZzQbiYh4MiJOjYj5EfFCRNwI/Bl4XaUJUdI0SUskLZJ0bOW9kraUdEO+Ev8N8DcjiSHXSJws6W7gSeBvc2yXR8TzEfF0RPwkIu5uwEc2G1BEPANcA+wCIOlgSffl2uKFkj6e0yt54xOFvHFYXv+P+XzyybzugcAngXfW2pKSzyE75umLJH0z19Q9IWmWpEpeG/G5JSJ+GhFXkSoqRg0XxNrPNGAB8FKgh5RZIiKOJtUO/FOuIv7KCLZ9GLAXKUPfAews6UxJb5S0SWPCN2ssST2kgtC9OWkrYDNga+A44JuSNs/Lvgk8A0wA/jW/RupdwCHAOOCPwPOSZkg6qLA/s6aStBHwTmBmTroAeH9EbAq8hlTrVLEVsAEpb3wG+A7wHuB1pBrc/ydp+4j4EfAlUs3UJhGx2whCOxL4HLA5MBf4Yk73uWWYXBBrP8+RTiLbRcRzEfGLGHpA0KW5Bm1F5epoAF+OiGX5an4eMJmUYa/K27ioX6Y5orDdFZJeXs8HMxsuSesClwIzIuIPOfk54PM5f9wMrAJeKWkM8HbgM7lW7R5gRpXN3lE4ps8ZZPfnRMTDOb88DuwLBOnk9miueesprL93v/yyd32f3ka5Hyr1YVwJvBn4ak5/DthF0ksiYnlE3FF4z3PAFyPiOeAKYDxwdq6Vuhe4Dxiq0HVO4Ri+Y5D1fhARv8ndYi4l1YThc8vwuSDWfr5Kurr4iaR5kqbX8J7xETEuv742yHoPF2ciYmZEHBERLyVdLe0HfKqwylWF7Y6LiFFVXWzlkrQO8H3gr8CJhUWP9esT+RSpH8pLebF/WcVDVTa9R+GY/sggIfTPL/dHxNSI2IZUE/FyUn+Wipn98stMzEbusIgYR6rhOhH4maStSBcbBwMPSfpZpTN99lhEVPpiPZ3/PlJY/jQprwzmI4VjeI9B1ltcmK7kQcDnluFyQazN5CuXaRGxA/A24GOSDqgsrnfzg+z3t8B1pBOMWakkidQE0wO8PV/hD+VRYDVQvLvsFXWEMVh++QNwEc4v1mS5T+J1pM7u+0bEbyNiCvAy4IekWqcRbbpRMQ66E59bhuSCWEkkjc13w4wBxkjaIKcdKmnHfCJaScp8L+S3PQI05DkuSrfi/5ukl+X5nUkFP1/FWzs4D3gVqU/k00OtDOmERfrBP1XSRpJ2AY5pRDCSds43CGyT57cl9SFzfrGmUjKF1BfrQUlHSdosX5w8zovnh+F6BJiYa54bpp5zi6Qx+bw4FlgnnxfXbWR87cgFsfJ8mlRNPJ3UmfLpnLYT8FNSv5dfA9+KiNvze74MfLqGvmC1WEHKHHMkrQJ+BPwAGMlNAGYNI2k74P2kPieL9eKzho6q4e0nkppIFpNqrL7XoLCeIN3oMkvSk6STyj2km2vMmuG/82/z46SO8McA9wNHA/MlPQ58AKglX1Rzdf772BB9wYarnnPL0aRz4XmkJs2nSX0yu5qG7gduZmZmZs3gGjEzMzOzkrggZmZmZlYSF8TMzMzMSuKCmJmZmVlJ2nrQ7/Hjx8fEiRNbus8nn3ySjTfeuKX7bKf9t0MMZex/9uzZS/PDBzvOYPmk7P+l4+iuODo1n3RCHqmXP0d7GFEeiYi2fb3uda+LVrv99ttbvs922n87xFDG/oHfRRsc8yN5DZZPyv5fVjiOvjo1jk7NJ52QR+rlz9EeRpJH3DRpZmZmVhIXxMzMzMxK4oKYmZmZWUnaurN+M0ycftNaafNPO6SESMyab87ClUztd8z7eDd7kfOIlc01YmZNJunfJd0r6R5Jl+eBbLeXNEvSXElXSlovr7t+np+bl08sN3ozM2smF8TMmkjS1sBHgEkR8RpgDHAkcDpwZkTsCCwHjstvOQ5YntPPzOuZmVmXckHMrPnGAhtKGgtsBCwC9geuyctnAIfl6Sl5nrz8AElqYaxmZtZCo66PmFkrRcRCSV8D/gI8DfwEmA2siIjVebUFwNZ5emvg4fze1ZJWAlsCS4vblXQ8cDxAT08Pvb29VfffsyFM23V1n7SB1m2mVatWlbJfx9EZcZiNZi6ImTWRpM1JtVzbAyuAq4ED691uRJwPnA8wadKkmDx5ctX1zr30es6Y0zebzz+q+rrN1Nvby0AxOg7HYTaauWnSrLneBPw5Ih6NiOeA64B9gHG5qRJgG2Bhnl4IbAuQl28GPNbakM3MrFVcEDNrrr8Ae0vaKPf1OgC4D7gdeEde5xjg+jx9Q54nL78tD5thZmZdyAUxsyaKiFmkTvd3AHNIee584GTgY5LmkvqAXZDfcgGwZU7/GDC95UGbmVnLuI8YfR/yOm3X1UydfpMf6GcNExGfBT7bL3kesGeVdZ8BDm9FXGZmVj7XiJmZmZmVxAUxMzMzs5K4IGZmZmZWEhfEzMzMzErigpiZmZlZSVwQMzMzMyuJC2JmZmZmJXFBzMzMzKwkLoiZmZmZlcQFMTMzM7OSuCBmZmZmVhIXxMzMzMxK4oKYmZmZWUlcEDMzMzMrydiyA2hXE6fftFba/NMOKSESMzMz61auETMzMzMryYgLYpK2lXS7pPsk3SvppJy+haRbJD2Y/26e0yXpHElzJd0taY9GfQgzMzOzTlRPjdhqYFpE7ALsDZwgaRdgOnBrROwE3JrnAQ4Cdsqv44Hz6ti3WceQNE7SNZL+IOl+Sa/3BYuZmUEdBbGIWBQRd+TpJ4D7ga2BKcCMvNoM4LA8PQW4OJKZwDhJE0YcuVnnOBv4UUTsDOxGyiu+YDErkDRG0p2Sbszz20ualS9KrpS0Xk5fP8/Pzcsnlhm3Wb0a0lk/Z4TXArOAnohYlBctBnry9NbAw4W3LchpiwppSDqedAKip6eH3t7eRoS4xrRdVw+6vGfDgddpdCzVrFq1qiX7aecYyt5/I0naDNgPmAoQEX8F/ippCjA5rzYD6AVOpnDBAszMtWkTCnnKrFudRLpIeUmePx04MyKukPRt4DjShclxwPKI2FHSkXm9d5YRsFkj1F0Qk7QJcC3w0Yh4XNKaZRERkmI424uI84HzASZNmhSTJ0+uN8Q+pla5G7Jo2q6rOWNO9a9l/lGNjaWa3t5eGv2ZOy2GsvffYNsDjwLfk7QbMJt0wmnJBUu1C4syCrntUrh2HO0Zh6RtgEOALwIfUzqR7A+8O68yAziVVBCbkqcBrgG+IUn54sWs49RVEJO0LqkQdmlEXJeTH6lcweemxyU5fSGwbeHt2+Q0s242FtgD+HBEzJJ0Ni82QwLNvWA599Lr17qwaMUFRX/tUrh2HO0ZB3AW8Alg0zy/JbAiIipXEZULEihcrETEakkr8/pLixvstIuVerVLobpe3fI5hmPEBbF8xXIBcH9EfL2w6AbgGOC0/Pf6QvqJkq4A9gJWurnFRoEFwIKImJXnryEVxHzBYgZIOhRYEhGzJU1u1HY77WKlXm1UqK5Lt3yO4ajnrsl9gKOB/SXdlV8Hkwpgb5b0IPCmPA9wMzAPmAt8B/hQHfs26wgRsRh4WNIrc9IBwH28eMECa1+wvDffPbk3vmCx7rcP8DZJ84ErSE2SZ5Nu6KqUkIoXJGsuVvLyzYDHWhmwWSONuEYsIn4JaIDFB1RZP4ATRro/sw72YeDSfNfXPOBY0kXQVZKOAx4Cjsjr3gwcTLpgeSqva9a1IuIU4BSAXCP28Yg4StLVwDtIhbP+FyvHAL/Oy29z/zDrZB7iyKzJIuIuYFKVRb5gMRvYycAVkr4A3EnqCkP++31Jc4FlwJElxWfWEC6ImZlZW4iIXtKjXIiIecCeVdZ5Bji8pYGZNZHHmjQzMzMriQtiZmZmZiVxQczMzMysJC6ImZmZmZXEBTEzMzOzkrggZmZmZlYSF8TMzMzMSuKCmJmZmVlJ/EDXYZg4/aY+8/NPO6SkSMzMzKwbuEbMzMzMrCQuiJmZmZmVxAUxMzMzs5K4IGZmZmZWEhfEzMzMzErigpiZmZlZSVwQM2sBSWMk3Snpxjy/vaRZkuZKulLSejl9/Tw/Ny+fWGbcZmbWXC6ImbXGScD9hfnTgTMjYkdgOXBcTj8OWJ7Tz8zrmZlZl3JBzKzJJG0DHAJ8N88L2B+4Jq8yAzgsT0/J8+TlB+T1zcysC/nJ+mbNdxbwCWDTPL8lsCIiVuf5BcDWeXpr4GGAiFgtaWVef2lxg5KOB44H6Onpobe3t+qOezaEabuu7pM20LrNtGrVqlL26zg6Iw6z0cwFMbMmknQosCQiZkua3KjtRsT5wPkAkyZNismTq2/63Euv54w5fbP5/KMaFkbNent7GShGx+E4zEYzF8TMmmsf4G2SDgY2AF4CnA2MkzQ214ptAyzM6y8EtgUWSBoLbAY81vqwzcysFVwQq0P/QcDBA4FbXxFxCnAKQK4R+3hEHCXpauAdwBXAMcD1+S035Plf5+W3RUS0Om4zM2uNri+IVSssmbWBk4ErJH0BuBO4IKdfAHxf0lxgGXBkSfGZmVkLdH1BzKxdREQv0Jun5wF7VlnnGeDwlgZmZmal8eMrzMzMzErigpiZmZlZSVwQMzMzMytJXQUxSRdKWiLpnkLaFpJukfRg/rt5Tpekc/IYendL2qPe4M3MzMw6Wb01YhcBB/ZLmw7cGhE7AbfmeYCDgJ3y63jgvDr3bWZmZtbR6rprMiJ+Lmliv+QpwOQ8PYN0l9jJOf3i/EykmZLGSZoQEYvqicHMzDqXpG2Bi4EeIIDzI+JsSVsAVwITgfnAERGxPI+9ejZwMPAUMDUi7mh13H6OpDVKMx5f0VMoXC0mZS4ojKGXVcbX61MQq3UMvVr1H2dvKNXG5huOcy+9fq20XbferOb3t8PYb2XHUPb+zaylVgPTIuIOSZsCsyXdAkwlta6cJmk6qXXlZPq2ruxFal3Zq5TIzRqgqc8Ri4iQNKyngtc6hl6tpg7zga7Tdl291th89RrO2H7tMPZb2TGUvX8za5184b4oTz8h6X7SRbpbV2xUaEZB7JFKppA0AViS0ytj6FUUx9czM7NRLnd1eS0wixa1rlRrBam27pyFK/vMT9t17W25JaF+3fI5hqMZBbHKWHmnsfYYeidKuoJUjbzSVzBmZgYgaRPgWuCjEfF46gqWNLN15dxLr1+rFaRaK0YtrSvDaf1otG5pSeiWzzEcdRXEJF1OqjoeL2kB8FlSAewqSccBDwFH5NVvJnWunEvqYHlsPfs2M7PuIGldUiHs0oi4Lie7dcVGhXrvmnzXAIsOqLJuACfUs79O1f/uGt9ZY2aW5LsgLwDuj4ivFxa5dcVGBQ/6bWZmZdoHOBqYI+munPZJ3Lpio4QLYmZmVpqI+CWgARa7dcW6nseaNDMzMyuJa8TMzMwKqj0136xZXCNmZmZmVhIXxMyaSNK2km6XdJ+keyWdlNO3kHSLpAfz381zuiSdI2mupLsl7VHuJzAzs2ZyQcysuSrj6O0C7A2cIGkX0rh5t0bETsCteR76jqN3PGkcPTMz61IuiJk1UUQsiog78vQTQHEcvRl5tRnAYXl6zTh6ETETGJcfZmlmZl3InfXNWqSdx9FrtnYZP85xtGccZqOZC2JmLdDu4+g1W7uMH+c42jMOs9HMTZNmTTbYOHp5ucfRMzMbpVwjZtZEHkfPbPSo9vwxjy1sQ3FBzKy5PI6emZkNyAUxsybyOHpmZjYY9xEzMzMzK4kLYmZmZmYlcUHMzMzMrCTuI2ZmZtYkvpPShtJVBbFqB7yZmZlZu3LTpJmZmVlJuqpGrFO4qtrMzMzABTEzM7OW6n8x7gvx0c0FMTNrS645NrPRwAWxNlE56UzbdTVT87RPOmaDc2HNzDqdO+ubmZmZlcQ1YmZWOj96xsxGKxfEzKxjuMBm3chN7KObC2Jm1lIuTJmZvcgFsTbmW5ytG0ycflOfm1Basb/+nHfMrF25s76Zdb2J029i4vSbmLNwpWvkzKyttLxGTNKBwNnAGOC7EXFaq2Mwa2ednEc6pZBTa5yuSWtfnZxPzIpaWhCTNAb4JvBmYAHwW0k3RMR9rYzDrF11Uh7plEJXPUb6GV2Aa65Oyicj5Sb20aPVNWJ7AnMjYh6ApCuAKcCwM89oOAn0V89nHmkGdj+1lmtYHqnHaMxf1lHaIp+0mn+Pu1OrC2JbAw8X5hcAexVXkHQ8cHyeXSXpgRbFBsBHYDywtJX7bMX+dfqwVh8whmFuZ6TK+B9s1+L9DWTIPALDyidrfZct+h/2UXa+anUcNXzHbfF9MPw4Oiaf1JNHOkW/46xjP0c/nf45hp1H2u6uyYg4Hzi/rP1L+l1ETBqt+2+HGMrefyeoNZ+0y3fpOBxHq3VaHqmXP0fnavVdkwuBbQvz2+Q0M0ucR8yG5nxiXaPVBbHfAjtJ2l7SesCRwA0tjsGsnTmPmA3N+cS6RkubJiNitaQTgR+Tbjm+MCLubWUMNSitWbRN9g/lx1D2/kvThDzSLt+l4+jLcdShwfmkI7+DKvw5OpQiouwYzMzMzEYlP1nfzMzMrCQuiJmZmZmVpOsLYpIulLRE0j2FtC0k3SLpwfx385wuSedImivpbkl7FN5zTF7/QUnHDGP/20q6XdJ9ku6VdFIJMWwg6TeSfp9j+FxO317SrLyvK3OnVyStn+fn5uUTC9s6Jac/IOmttcaQ3ztG0p2Sbixj/6OJpAPzdzRX0vQW7G++pDmS7pL0u5w27GN8hPsuNY8PEcepkhbm7+UuSQcXllU9luv937XDb06naHU+GY5u+z/6938QEdHVL2A/YA/gnkLaV4DpeXo6cHqePhj4H0DA3sCsnL4FMC//3TxPb17j/icAe+TpTYE/Aru0OAYBm+TpdYFZedtXAUfm9G8DH8zTHwK+naePBK7M07sAvwfWB7YH/gSMGcb/4mPAZcCNeb6l+x8tL1Ln5T8BOwDr5e9slybvcz4wvl/asI7xOvZdah4fIo5TgY9XWbfqsdyI/x1t8JvTCa8y8slo/j/i3/8BX11fIxYRPweW9UueAszI0zOAwwrpF0cyExgnaQLwVuCWiFgWEcuBW4ADa9z/ooi4I08/AdxPeip0K2OIiFiVZ9fNrwD2B64ZIIZKbNcAB0hSTr8iIp6NiD8Dc0lDjQxJ0jbAIcB387xauf9RZs3wLxHxV6Ay/EurDfcYH5Gy8/gQcQxkoGO57v9dO/zmdIh2ySdVddP/0b//g+v6gtgAeiJiUZ5eDPTk6WrDZmw9SPqw5CrW15JqpFoaQ64WvgtYQsqIfwJWRMTqKttbs6+8fCWwZZ0xnAV8Anghz2/Z4v2PJmV8TwH8RNJspaFlYPjHeCOVkscHcGJuKrqw0ozUqjjK/M3pAB3z+brg/+jf/0GM1oLYGhERpJNIU0naBLgW+GhEPN7qGCLi+YjYnfQE6j2BnZu5vyJJhwJLImJ2q/ZpLbdvROwBHAScIGm/4sJW5bNqytw3cB7wN8DuwCLgjFbtuOzfHGuMTv8/+vd/aKO1IPZIpSkk/12S0wcaNqOu4TQkrUvKSJdGxHVlxFARESuA24HXk6quKw/1LW5vzb7y8s2Ax+qIYR/gbZLmk6r/9wfObuH+R5uWf08RsTD/XQL8gFTYH+4x3kil5K/+IuKRfBH0AvAdXmxKaWoc7fSb08ba/vN1yf/Rv/9DqbeTWSe8gIn07UD7Vfp2dvxKnj6Evp0df5PTtwD+TOrouHme3qLGfQu4GDirX3orY3gpMC5Pbwj8AjgUuJq+nSU/lKdPoG9nyavy9Kvp21lyHsPsLAlM5sXOmi3f/2h4kUbMmJe/o0on5Fc3cX8bA5sWpn9F6oMyrGO8zhhKy+NDxDGhMP3vpD4uAx7Ljfjf0Qa/OZ3wanU+GUF8Xfd/xL//1b+XsgNowT/+clKTwHOkNuXjSO3NtwIPAj+tHJT5AP4mqf/UHGBSYTv/SuocOBc4dhj735dUdXw3cFd+HdziGP4OuDPHcA/wmZy+A/CbvL2rgfVz+gZ5fm5evkNhW5/KsT0AHDSC/0cxI7Z8/6PllY+xP+bv6lNN3tcO+Qfy98C9lf2N5Bgf4f5LzeNDxPH9vJ+7SWMhFgtmVY/lev93tMFvTqe8WplP/H/07/9ALw9xZGZmZlaS0dpHzMzMzKx0LoiZmZmNQpJWSdqh7DhGOxfErA9JkyUtKDsO616SjsxDlzypNCTQLEkfyg9tbCuS9pR0s6QVkpYpDRV2bNlx2eg13Pwj6SJJX6i2LCI2iYh5w9j3VEkh6Z0jjd/W5oLYCCmNrfd0vqJYLukmSdsO/c5h7eNUSZdUST80nxCelPSYpEskdcWD7ay7SZpGunX9q8BWpIdRfoB0i/t6w9zW2KHXGjlJrwduA34G7EjqJP1B0rPShrstSfLvrdVluPlH0pgGh3AMafSI9w4RZ1PzZrfxD0N9/ikiNiGNCfYIcG6zdyjpHaTxus4CxpNu6f0r8AtJ45q9f7ORkrQZ8HnSberXRMQTkdwZEUdFxLOSDlEaGPhxSQ9LOrXw/on5avw4SX8hFZKQdLWkxZJWSvq5pFcX3rOlpP/O2/utpC9I+mVh+c5KAycvUxpI+IhCyF8FZkTE6RGxNMc6OyKOyO/dXNKNkh7NF2M3Kg3lUtl2r6QvSvpf4Clgh1yjME/SE5L+LOmo5nzb1m1qzD8XSTov1+I+CbxxiG2GpB0l7ZXz0JjCsn+WdHdhfjvgDcDxwFslbVVYNlnSAkknS1oMfC+nH6o02P0KSb+S9HeF90yX9KecF+6T9M8N+qo6jgtiDRARz5DGxNoFQNLB+cB6QtJCSR/P6ZWD9RO5SnmRpMPy+n/MJ4NP5nUPBD4JvDPXuv0+Vz2fAXwhIi6LiKcjYjHwPtIP/Un5vX1q0gonsLF5/lhJ9+f45kl6/3A/c64R/LjS0C0rJV0paYO8bNATlI1aryc9A+j6QdZ5knS1PY70XKQPSjqs3zpvAF5FGkMP0rOTdgJeBtwBXFpY95t5m1uRruaPqSyQtDFpuK/L8nuPBL4laRdJG+V4r2Fg65BOONsBrwCeBr7Rb52jSSeuTYFHgXNIt91vCvwD6ZEEZrWoJf8AvBv4IumY++UQ6wIQEbNI+WT/ftu5rDD/XuB3EXEtadzL/hcRW5GeWbYdcLyk1wIXAu8n1Sb/F3CDpPXz+n8C/pH0wNbPAZeojjFnO5kLYg2Qf7TfCczMSRcA788/tq8hX7lnW5Gek7I18BnS07bfA7yOdFD+P0nbR8SPgC+RRp7fJCJ2A15J+sG/urj/SE/tvhZ4S40hLyE90PUlwLHAmZL2GNaHTo4gPbhze9Kzyqbm9FpOUDb6jAeWxovjy5GvklcoNfPvFxG9ETEnIl6IiLtJz+Z6Q7/tnBoRT0bE0wARcWGuHXgWOBXYTdJm+er+7cBnI+KpiLiPFwcThpQH5kfE9yJidUTcScpHh5MefLkO6blgVUXEYxFxbd72E6STX/9YL4qIe/NnXk0aa+81kjaMNKjzvcP6Bm00GzL/5OTrI+J/cx56Zhjbvxx4V97upqRnll1eWP5eXiyYXcbazZMvkPLaszlvHg/8V0TMijS6xAzgWdLDZomIqyPi/3KcV5Kei9YVg3gPlwti9fmhpBWkQUnfTGrKgPRAx10kvSQilkfEHYX3PAd8MSKeIw33MB44O59I7gXuA3YbYH/j899qJ4dFpCfoDykiboqIP+Vq7Z8BPyEVAofrnJyRlgH/TRpPr9YTlI0+jwHjVeg/EhH/EBHj8rJ1chPJ7bk2dSWp/8v4fttZM/Cv0mD2p+UmjseB+XnReFJ+GEvfgYKL09sBe+UT2Yqcl48iXSwtJ51YBrxCl7SRpP+S9FDe989Jw7YU++Ws2V9EPEm6YPsAsEipX2nLxny1jjdk/snJD1d7cw0uA/4l11j9C3BHRDwEIGkf0gX3FYV1d5W0e+H9j/Yr+G0HTOuXv7YFXp63+d5Cs+UKUqVF/7w+KrggVp/DcibYADgR+FluN3876WriIUk/U+r0W/FYRDyfp5/Ofx8pLH8a2GSA/S3Nf6udHCYUlg9K0kGSZuam0BU51pFkgMWF6afIcdd4grLR59ekK+Ipg6xzGekJ9NtGxGakoU/63w1WfAr1u/P23kRq4piY00VqClxNGpOuonhDzcPAzyJiXOG1SUR8MCKeyvG+fZBYp5FqqfeKiJcAlRqJYrx9npgdET+OiDeT8usfSDXiZrWoJf/ACHY8mzcAACAASURBVAcBzzXGD5FuRunfLHkM6bi+K/cBm1VIH2i/D5MqHYr5a6OIuFypv9l3SOfNLfN59B7WzuujggtiDZCrXa8Dngf2jYjfRsQUUr+THwJXjXTT/eYfIA2dcngxUelurLcDvTnpSWCjwirFTpXrk5pfvgb05AxwM43NALWcoGyUiTTg/OdI/bDeIWlTSevkq+qN82qbAssi4hlJe5JOCIPZlHRyeox0zH+psL/ngeuAU/PFwc70bU65EfhbSUdLWje//l7Sq/LyTwBTJf2HpC0BJO0mqVIrsCnpwmmFpC2Azw4WqKQeSVNy37RngVWkWjezIdWYfwYyRtIGhddAdyhfRuprvB+5C0zu+3sEqalx98Lrw8C7NfAdkt8BPpBruSVpY6WbcTbN8QbpYgmlR8K8pqYvogu5INYA+SCbQupX8qCkoyRtlpsfH2fkP7aPABNzQYuICODjwKclvTtnqK2A75JqtCp3bd4F7CfpFUp32pxS2OZ6pA6fjwKrJR1E7X3LajWsE5SNHhHxFeBjpELOI/n1X8DJpMHCPwR8XtITpD6UQ13EXEy6il9Iataf2W/5iaSassWkcR8vJxWCyM3mbyF10v+/vM7ppPxBRPyK1Hl5f2CepGXA+aQLF0h3Lm9IqomeCfxoiFjXyZ/9/0iPAHgD6XEYZjWpIf8MZDrpN7nyum2A9Sp9Mm+LiEoLy2H5PRdHxOLKi9QRfyypn3C1WH8H/Bupf/By0tiRU/Oy+0g3nv06f4Zdgf8d/NN3L481OUKS5pOe4fI8qWT/EPBl0lXEDcBewBhSLda/R8QvJU0GLomIbfI2xpL6jG0fEfNz2i9JI89fkq/Cryc9ouLPEbFHXmcK8OmcviFpgNfDI+KBQnzfJPV3WUo6uZwPrBsRqyWdQDrJrU/q27UuMDciPt0/xiE+//si4qd5/lRgx4h4j6SXk66sJpFOOmeQmpjWLXY0NWs1SacDW0XEMUOubGbWAi6IdThJbyEVet4UEb4V3qwgN0euR7pY+XtSbdb7IuKHpQZmZpa5abLDRcRPSI+g2LvsWMza0KakfmJPAleSameHeg6TmVnLuEbMqpL0ClKfm2p2iYi/tDIeMzOzbuSCmJmZmVlJ2npgzvHjx8fEiROrLnvyySfZeOOh7thtnXaKx7FUN1gss2fPXhoRNT0Qt910Sj5xLANrp3i6MZ8Mlkc6UTsdL43SLZ9pRHkkItr29brXvS4Gcvvttw+4rAztFI9jqW6wWEhjqJV+zI/k1Sn5xLEMrJ3i6cZ8Mlge6UTtdLw0Srd8ppHkEXfWNzMzMyuJC2JmZmZmJXFBzMzMzKwkLoiZmZmZlaSt75oczJyFK5k6/aY+afNPO6SkaMwGJunfgfeRhsKaQ3oA7wTgCmBLYDZwdET8NQ/KfjHwOtJA1u+MPPyVNY5/P6ydTOx3LIKPx9HENWJmTSRpa+AjwKSIeA1p/NEjSeN/nhkRO5IGxD0uv+U4YHlOPzOvZ9a1JG0r6XZJ90m6V9JJOX0LSbdIejD/3TynS9I5kuZKulvSHuV+ArP6dGyNmFkHGQtsKOk5YCNgEbA/8O68fAZwKnAeMCVPA1wDfEOS8m3RZt1oNTAtIu6QtCkwW9ItwFTg1og4TdJ0YDpwMnAQsFN+7UXKN3uVEnkNqtV2mRW5IGbWRBGxUNLXgL8ATwM/ITVFroiI1Xm1BcDWeXpr4OH83tWSVpKaL5cWtyvpeOB4gJ6eHnp7e6vuf9WqVQMua7V2iqVnQ5i26+o+aWXG1k7fTatjiYhFpIsTIuIJSfeT8sEUYHJebQbQSyqITQEuzhcnMyWNkzQhb8es47ggZtZEuTllCrA9sAK4Gjiw3u1GxPnA+QCTJk2KyZMnV12vt7eXgZa1WjvFcu6l13PGnL4/f/OPmlxOMLTXd1NmLJImAq8FZgE9hcLVYqAnT6+5WMkqFzJ9CmK1Xqw0W/8Cf60Gi7edCu6N0o2fqVYuiJk115uAP0fEowCSrgP2AcZJGptrxbYBFub1FwLbAgskjQU2I3XaN+tqkjYBrgU+GhGPS1qzLCJC0rCa52u9WGm2/jeF1GqwC4N2Krg3Sjd+plq5s75Zc/0F2FvSRkpnlgOA+4DbgXfkdY4Brs/TN+R58vLb3D/Mup2kdUmFsEsj4rqc/IikCXn5BGBJTq9crFQUL2TMOo4LYmZNFBGzSJ3u7yA9umId0lX6ycDHJM0l9QG7IL/lAmDLnP4xUgdls66VL1AuAO6PiK8XFhUvSvpfrLw33z25N7DS/cOsk7lp0qzJIuKzwGf7Jc8D9qyy7jPA4a2Iy6xN7AMcDcyRdFdO+yRwGnCVpOOAh4Aj8rKbgYOBucBTpOfytQXfIWkj4YKYmZmVJiJ+CWiAxQdUWT+AE5oalFkLuWnSzMzMrCQuiJmZmZmVxAUxMzMzs5K4IGZmZmZWEhfEzMzMzErigpiZmZlZSVwQMzMzMyuJC2JmZmZmJXFBzMzMzKwkLoiZmZmZlcQFMTMzM7OSuCBmZmZmVhIXxMzMzMxK4oKYmZmZWUlcEDMzMzMrydiyAzDrdpLGAd8FXgME8K/AA8CVwERgPnBERCyXJOBs4GDgKWBqRNxRQthmNoiJ028qOwTrEq4RM2u+s4EfRcTOwG7A/cB04NaI2Am4Nc8DHATslF/HA+e1Plyz1pF0oaQlku4ppJ0qaaGku/Lr4MKyUyTNlfSApLeWE7VZ47ggZtZEkjYD9gMuAIiIv0bECmAKMCOvNgM4LE9PAS6OZCYwTtKEFodt1koXAQdWST8zInbPr5sBJO0CHAm8Or/nW5LGtCxSsyYYsmlS0oXAocCSiHhNTtuCYTarSDoG+HTe7BciYgZm3W974FHge5J2A2YDJwE9EbEor7MY6MnTWwMPF96/IKctKqQh6XhSjRk9PT309vZW3fmqVasGXNZq7RRLz4YwbdfVfdLKjK2dvptWxxIRP5c0scbVpwBXRMSzwJ8lzQX2BH7dpPDMmq6WPmIXAd8ALi6kVZpVTpM0Pc+fTN9mlb1IzSp75YLbZ4FJpD4ysyXdEBHLG/VBzNrUWGAP4MMRMUvS2bzYDAlARISkGM5GI+J84HyASZMmxeTJk6uu19vby0DLWq2dYjn30us5Y07fn7/5R00uJxja67tpo1hOlPRe4HfAtHy+2BqYWVincqGyllovVkaqf0G+0QaLt50K7o3SjZ+pVkMWxAa4WpkCTM7TM4BeUkFsTbMKMFNSpVllMnBLRCwDkHQLqVr58ro/gVl7WwAsiIhZef4aUkHsEUkTImJRziNL8vKFwLaF92+T08xGk/OA/yRduP8ncAbpJpea1XqxMlJTm9xZf7ALgzYqLDdMN36mWo30rsnhNqsMlL6WWq9i3LQwMMdSXRmxRMRiSQ9LemVEPAAcANyXX8cAp+W/1+e33ECqCbiCVKu8spDXzEaFiHikMi3pO8CNedYXKtZ16n58xUiaVYbYXk1XMW5aGJhjqa7EWD4MXCppPWAecCzpRpmrJB0HPAQckde9mdTHci6pn+WxrQ/XrFyV2uI8+89A5Y7KG4DLJH0deDmpG8xvSgjRrGFGWhAbbrPKQl5syqyk945w32YdJSLuIvWP7O+AKusGcELTgzJrE5IuJ50fxktaQOpPPFnS7qSmyfnA+wEi4l5JV5FqlFcDJ0TE82XEbdYoIy2I3cAwmlUk/Rj4kqTN83pvAU4ZedhmZtYNIuJdVZIvGGT9LwJfbF5EZq1Vy+Mrql2tnMYwmlUiYpmk/wR+m9f7fKXjvpmZmdloVctdk9WuVmCYzSoRcSFw4bCiMzMzM+tifrK+mZmZWUlcEDMzMzMriQtiZmZmZiVxQczMzMysJC6ImZmZmZWk7ifrm5mZWWNNrDKW5fzTDikhEms214iZmZmZlcQFMTMzM7OSuCBmZmZmVhIXxMzMzMxK4oKYmZmZWUlcEDMzMzMriQtiZmZmZiVxQczMzMysJC6ImbWApDGS7pR0Y57fXtIsSXMlXSlpvZy+fp6fm5dPLDNuMzNrLhfEzFrjJOD+wvzpwJkRsSOwHDgupx8HLM/pZ+b1zLqapAslLZF0TyFtC0m3SHow/908p0vSOfli5W5Je5QXuVn9XBAzazJJ2wCHAN/N8wL2B67Jq8wADsvTU/I8efkBeX2zbnYRcGC/tOnArRGxE3Brngc4CNgpv44HzmtRjGZN4bEmzZrvLOATwKZ5fktgRUSszvMLgK3z9NbAwwARsVrSyrz+0uIGJR1POgnR09NDb29v1R2vWrVqwGWt1k6x9GwI03Zd3SetzNja6bspI5aI+HmVZvgpwOQ8PQPoBU7O6RdHRAAzJY2TNCEiFrUmWrPGckHMrIkkHQosiYjZkiY3arsRcT5wPsCkSZNi8uTqm+7t7WWgZa3WTrGce+n1nDGn78/f/KMmlxMM7fXdtFEsPYXC1WKgJ0+vuVjJKhcyfQpitV6sjFT/gnwrVD5DOxXcG6UbP1OtXBAza659gLdJOhjYAHgJcDYwTtLYXCu2DbAwr78Q2BZYIGkssBnwWOvDNmsfERGSYpjvqeliZaSmTr+podurReVioY0Kyw3TjZ+pVu4jZtZEEXFKRGwTEROBI4HbIuIo4HbgHXm1Y4Dr8/QNeZ68/LbcBGM22jwiaQJA/rskp1cuViqKFzJdbeL0m5g4/SbmLFzJxBIKgtYcrhEzK8fJwBWSvgDcCVyQ0y8Avi9pLrCMVHjraJUTxrRdVzN1+k3MP+2QkiOyDlG5KDmNtS9WTpR0BbAXsLIV/cNc8LFmcUHMrEUiopfU4ZiImAfsWWWdZ4DDWxqYWckkXU7qmD9e0gLgs6QC2FWSjgMeAo7Iq98MHAzMBZ4Cjm15wGYN5IKYmZmVKiLeNcCiA6qsG8AJzY3IrHXcR8zMzMysJC6ImZmZmZXEBTEzMzOzkrggZmZmZlaSugpikuZLmiPpLkm/y2keqNXMzMysBo2oEXtjROweEZPyvAdqNTMzM6tBM5omp5AGaCX/PayQfnEkM0lDvExowv7NzMzMOkK9zxEL4Cd5DLD/ymN7tWSg1p4N1x50tcwBQ9tpwFLHUl07xWJmZgb1F8T2jYiFkl4G3CLpD8WFzRyo9dxLr+eMOX3DrwyIWoZ2GrDUsVTXTrGYmZlBnU2TEbEw/10C/IA0ZIsHajUzMzOrwYgLYpI2lrRpZRp4C3APLw7UCmsP1PrefPfk3rRooFYzMzOzdlVP02QP8ANJle1cFhE/kvRbPFCrmZmZ2ZBGXBCLiHnAblXSH8MDtZqZmZkNyU/WNzMzMyuJC2JmZmZmJXFBzMzMzKwkLoiZNZGkbSXdLuk+SfdKOimne0xWsxoMZ0xjs05U7wNdzWxwq4FpEXFHftzLbEm3AFNJY7KeJmk6aUzWk+k7JutepDFZ9yolcrP28caIWFqYr4xp3D//jCoTp9+0Vtr80w4pIRKrh2vEzJooIhZFxB15+gngftLQXh6T1WzkBso/Zh3HNWJmLSJpIvBaYBYtGpO1HcbXrIwJWxkftux4irEUeazapJ1iyYYzpvEateaRWvU/XspU7fitaLP/Xc3a8LhrGRfEzFpA0ibAtcBHI+Lx/CBkoLljsrbD+JpTc/PJtF1Xc8acsaWOCVvhsWoH1k6xZCMa07jWPFKrqVWaActSyUvVtEP+Gok2PO5axgUxsyaTtC6pEHZpRFyXkx+RNCEiFnlM1tGp2L9n2q6rmTr9JvfvqaI4prGkPmMaV8k/Zh3HfcTMmkip6usC4P6I+HphkcdkNRvCCMY0Nus4rhEza659gKOBOZLuymmfBE7DY7KaDWW4YxqbdRwXxMyaKCJ+CWiAxR6T1WwQwx3T2KwTuSBmZmbWJfxssc7jPmJmZmZmJXFBzMzMzKwkbpo0MzPrYv2bK91U2V5cI2ZmZmZWEhfEzMzMzErigpiZmZlZSVwQMzMzMyuJC2JmZmZmJfFdk2ZdbM7ClUz1HVNmZm3LNWJmZmZmJXGNmJmZ2SjiYZDai2vEzMzMzErigpiZmZlZSdw0aWZmVlCt6c6sWVwQMzMzG+Xcb6w8LW+alHSgpAckzZU0vdX7N2t3ziNmQ3M+sW7R0hoxSWOAbwJvBhYAv5V0Q0Tc18o4bHTof4V30YEblxRJ7ZxHzIbmfNIatTTRutasfq1umtwTmBsR8wAkXQFMAZx5zBLnEbOhNSyfuD9YfWr9/lxgG1irC2JbAw8X5hcAexVXkHQ8cHyeXSXpgQG2NR5Y2ue9pzcoypFZK54SOZYq3nj6oLFs18pYBjFkHoHOzCcfybGUnE8r2uZ7gfb/bgo6Jp8MI490nI+00e9qrWo4tjvuMw1g2Hmk7TrrR8T5wPlDrSfpdxExqQUh1aSd4nEs1bVTLPXqxHziWAbWTvG0Uyz1qDWPdKJu+R8VdeNnqlWrO+svBLYtzG+T08wscR4xG5rziXWNVhfEfgvsJGl7SesBRwI3tDgGs3bmPGI2NOcT6xotbZqMiNWSTgR+DIwBLoyIe0e4uXarcm6neBxLde0US1UNziPQXp/ZsQysneJpp1iqakI+6TRt/z8agW78TDVRRJQdg5mZmdmo5LEmzczMzErigpiZmZlZSdqyIDbU0BWS1pd0ZV4+S9LEwrJTcvoDkt7aglg+Juk+SXdLulXSdoVlz0u6K7/q7khaQyxTJT1a2Of7CsuOkfRgfh1Tbyw1xnNmIZY/SlpRWNaw70bShZKWSLpngOWSdE6O825JexSWNfx7aQftNPzLUP+fFseyraTbc569V9JJJcaygaTfSPp9juVzZcVSiGmMpDsl3Vh2LKPJQMelpC0k3ZJ/n26RtHlO75jftP7HVL7BYlaO/cp8s0VLz+ttJyLa6kXqePknYAdgPeD3wC791vkQ8O08fSRwZZ7eJa+/PrB93s6YJsfyRmCjPP3BSix5flWLv5epwDeqvHcLYF7+u3me3rzZ8fRb/8OkDrXN+G72A/YA7hlg+cHA/wAC9gZmNet7aYfXcP83LYhn0P9Pi2OZAOyRpzcF/ljWd5OPx03y9LrALGDvkr+fjwGXATeW/b8aTa+BjkvgK8D0nD4dOD1Pd8xvWv9jCrgKODJPfxv4YJ5uyXm9HV/tWCO2ZuiKiPgrUBm6omgKMCNPXwMcIEk5/YqIeDYi/gzMzdtrWiwRcXtEPJVnZ5KeZ9MMtXwvA3krcEtELIuI5cAtwIEtjuddwOV17rOqiPg5sGyQVaYAF0cyExgnaQLN+V7aQT3HSsPV8P9pmYhYFBF35OkngPtJT2kvI5aIiFV5dt38Ku3uKUnbAIcA3y0rhtFqkOOyeK6bARyWpzviN63/MZXP0/uTztuw9mdqxXm97bRjQaza0BX9fyjXrBMRq4GVwJY1vrfRsRQdR7pKqdhA0u8kzZR02EBvanAsb89V1ddIqjzwsNHfy7C2mZtrtwduKyQ38rsZykCxNuN7aQfd+rkaKjd9vJZUE1VWDGMk3QUsIZ1AS4sFOAv4BPBCiTGMev2Oy56IWJQXLQZ68nSn/Kb1P6a2BFbk8zb0ja9V5/W2044FsY4k6T3AJOCrheTtIg3Z8G7gLEl/0+Qw/huYGBF/R7oSmjHE+q1yJHBNRDxfSGv1d2O2hqRNgGuBj0bE42XFERHPR8TupJr0PSW9pow4JB0KLImI2WXs35LBjstI7XQd87wpH1O1a8eCWC1DV6xZR9JYYDPgsRrf2+hYkPQm4FPA2yLi2Up6RCzMf+cBvaSrnKbFEhGPFfb/XeB1w/kcjY6n4Ej6NUs2+LsZykCxduswKd36uRpC0rqkk92lEXFd2fEARMQK4HbKa0baB3ibpPmkpuz9JV1SUiyj0gDH5SO5yZH8d0lO74TftLWOKeBsUjNq5WHyxfhadV5vP2V3Uuv/Ij3tfx6pKavS0fjV/dY5gb6d+q7K06+mb6e+edTXWb+WWF5L6jy4U7/0zYH18/R44EHq6BRcYywTCtP/DMzM01sAf84xbZ6nt2j2/ymvtzMwn/zw4GZ8N3k7Exm4s/4h9O3Y+ptmfS/t8Kr1f9PimAb8/7Q4DgEXA2e1QSwvBcbl6Q2BXwCHtkFck3Fn/VZ/51WPS1ILS7Gz/lfydEf9phWPKeBq+nbW/1Cebsl5vR1fpQcwwD/tYNJdI38CPpXTPk+qcQLYIP8z5wK/AXYovPdT+X0PAAe1IJafAo8Ad+XXDTn9H4A5+QCaAxzXgli+DNyb93k7sHPhvf+av6+5wLGt+D/l+VOB0/q9r6HfDam2bRHwHKn/wHHAB4AP5OUCvpnjnANMaub30g6vav+bEmNZ6/9TYiz7kpp37i7k2YNLiuXvgDtzLPcAnyn7uMlxTcYFsVZ/51WPS1IfqVtJF6s/JReqOu03jb4FsR1I5+25pPN45aK8Zef1dnt5iCMzMzOzkrRjHzEzMzOzUcEFMTMzM7OSuCBmZqOSpFMrdwZKmigpCndzNXvfkyUtaMW+zCokzc93+Tdj26sk7dCMbXc7F8RKNJJMkU8WO1ZJ317SC5LOa1yEZo2nNCbqHElPSVos6TxJ48qOqz9Jh+axIJ+U9JikSyR11YMkrTNJ2lfSryStlLRM0v9K+vsGbr9yYbIqv+ZriDFrI2KTSI8jsmFyQax7vBdYDrxT0voDrdSqK36zaiRNA04H/oP0nKC9ge2AWyqD/zZoP3Ud55LeQRof7yzSI1ZeDfwV+EU7Fhpt9JD0EuBG4FzS4yq2Bj4HPDvY+0ZoXERsQhqi7jOS1nrOnc8p9XNBrA1J+rc80vwySTdIenlO/3le5ff5KuWdOV2kgtinSY8I+Kd+2wtJJ0h6kHQbNJJ2lnRL3scDko4orH+IpDslPS7pYUmnNv1DW9fLJ5DPAR+OiB9FxHMRMR84gvScsY9LelrSFoX3vFbS0vywSyT9q6T7JS2X9OM8fFZl3WrH+dn5GH5c0mxJ/1hDnALOAL4QEZdFxNMRsRh4H/AUcFJeb03TZp7v07wp6dgc6xOS5kl6f11foFnytwARcXmkkRmejoifRMTdkv5G0m25BneppEsHunCQtI6k6ZL+lNe/qpj3iiLi16RHI70mv7daXlvTWiNpQ0lnSHoo19r9UtKGedneuTZvhaTfS5rc6C+o07gg1mYk7U96HtgRwATgIdJTiYmI/fJqu+Vq4Cvz/L6kpw1fQRrZ/pgqmz4M2AvYRdLGpCGQLgNeRnp43rck7ZLXfZJUsBtHenDgB9X88SCt+/0D6VlBfZ5mH2ng65uBXYFfA28vLH43aXis5yRNAT4J/AvpYai/YO2B5Ncc53n+t8DupJqDy4CrJW0wRJyvBF5BeqZRMc4XSE8+f8tQHzRbAhwKvAQ4FjhT0h41vtdsIH8Enpc0Q9JBkjYvLBPp/PFy4FWkJ9KfOsB2PkzKL2/I6y8nPZusDyX7kGqF7yws6p/Xir5GGtnlH0h57xPAC7lp/ybgCzn948C1kl469MfuXi6ItZ+jgAsj4o5IwxWdArxeaSDYgRwD/E9ELCedbA6U9LJ+63w5IpZFxNOkk8P8iPheRKyOiDtJJ5jDASKiNyLmRMQLEXE36WT3hkZ+SBuVxgNL48UBf4sW5eWXkZpBKjVTR+Y0SA/p/XJE3J+38SVg92KtGH2PcyLikkhDf62OiDNIT+d+ZQ1xVmKqFmdNJ42IuCki/hTJz4CfAEPWyJkNJtIYlJUHwH4HeDS3nPRExNyIuCUino2IR4GvM/Bv9wdID3tekM81pwLv6NfUuBRYRhoyb3pE3FpY1ievVUhah/RQ2ZMiYmGutftV3sd7gJsj4uZ8frkF+B3p4bWjlgti7eflpFowYE1twWMMMNp8ru49HLg0r/9r4C+kmoSi4uj12wF75arhFZJWkAqAW+Vt7iXpdkmPSlpJyrDjMavPUmD8AH1KJuTl15IuPCYA+wEvkGq+IB23ZxeO2WWkGoBi3ige50j6eG4eXJnfsxlDH8tLCzENFOeQcm3FzNz8v4J0snE+srrli5GpEbENqbnw5cBZknokXSFpoaTHgUsY+JjbDvhBIT/dDzwP9BTWGR8Rm0fEqyLinH7vf5jqxpNqvv80wD4P73fu2ZfqeW3UcEGs/fwf6WAFIDcjbsnAg5z+M6np41tKd6AtJp2Y+jdPFodQeBj4WUSMK7w2iYgP5uWXATcA20bEZqTxwFTvB7NR79ekDsX/UkyUtAlwEHBrrtX9CfBO0sXEFfHi8B8PA+/vd9xuGBG/KmwuCtv9R1KTyBHA5hExDljJ0MfyA6ShmA7vF+c6pGbT3pz0JLBRYZWtCuuuTypUfg3oyfu+uYZ9mw1LRPwBuIhUIPsSKQ/sGhEvIdVADXTMPUwaLqiYnzaIiFoH1B5oWJ6lwDPA3wywz+/32+fGEXFajfvsSi6IlW9dSRtUXqRmwGMl7Z5/zL8EzMqdmiGNa1l8VssxwIWk/jW759c+wG6Sdh1gnzcCfyvpaEnr5tffS3pVXr4psCwinpG0J2vXrpkNW0SsJHXWP1fSgfm4m0jq17gA+H5e9TJSH8XKnYsV3wZOkfRqAEmbSepTWOpnU2A18CgwVtJnSBctQ8UZpL4rn5b07pw3tyI1z4wn3a0GaTzA/SS9QtJmpG4EFeuRmkEfBVZLOoja+5aZDUjpRqtpkrbJ89uSmvNnko75VcDK3B/rPwbZ1LeBL1aa9iW9NPfDrEvuS3kh8HVJL5c0RtLr8/nsEuCfJL01p2+g9Ey9berdbydzQax8NwNPF16Tgf9HuppeRLqqOLKw/qnAjFytewxwAHBWRCwuvGYDP6J6p30i4gnSSeFIUg3cYtIjBSqPvfgQ8HlJTwCfIZ0ozeoWEV8hdbj/GvA4MIt0lXxA7kMCqTZ2J2BxRPy+8N4fkI7TK3Kzyz2kmrSB/JiUD/5Iau5/hoGbU/rHeSVwNPDvpCbQRcAk4P+3d+/xdlT1/f9fbwjX24Z7KAAAIABJREFUBAkEPI0ETfyKtNQolxShWE1BaAQr+vuhgnwhQfzl0QotaqwGbSu92F+0IkXlh6IgIEi4CIUCKhRzHn5tIWIUCQkgB4ySNCRyCwSv0c/vj7V2MtnZ55J9m9n7vJ+Px36cmTWzZz5n9l6z16xZa9YbImJtXudO4FrSQM3LSBc4tfc/D/w1Ke88Q7qYuWUs+zYbxfOkRvJLJb1AKoA9ACwgXegcSqr5vY26jjF1LiR9J+/I5/p78nbb4YOkwcjvJeWfTwA7RMTjQK3Tzc9I+fFvGOdlEQ/6bWY2CknHkWrn3hgR95Udj5n1DxfEzMzGQNKfA/tFxOfLjsXM+ocLYmZmZmYlGdf3Zc3MzMzK5IKYmZmZWUkqPVjnPvvsE9OnT2+47IUXXmDixIndDahC+3cM7Y1h2bJlT0ZETw6zMVI+aVWZn6/3XY6R9t+r+WSfffaJfffdt/Rz1XDK/sxH4ti2T1N5JCIq+zrssMNiOEuWLBl2WTeUvX/H0N4YgO9FB7/LwI6kcdpuzfMzSI9uGCI9AmHnnL5Lnh/Ky6ePtu2R8kmryvx8ve/q7b/T+aRTr8MOO6z04zoSx9acKsbWTB7xrUmz7jiHNIRIzSeACyLiFaTnTJ2Z088EnsnpF+T1zMysT7kgZtZh+anRJ5CezF4bzPpo4Ia8yhXAW/P0iXmevPyYvL6ZmfWhSrcRM+sT/0Ya83CPPD8FeDYiNuX51WwZuHo/8tPfI2JTHnR9CnUDTUuaD8wHGBgYYHBwsCOBb9y4sWPb9r6rt+8q7N9svOmrgtj0hbdtk7Zq0QklRGKWSHozsD4ilkma3a7tRsQlwCUAs2bNitmz27bprQwODtKpbXdq3+04D/Ti/90v++9l/g2yZvRVQcysgo4C3iLpeGBX0qDTFwKTJU3ItWLTgDV5/TXA/sBqSROAPYGnuh+2mZl1Q0ttxCS9X9IKSQ9IuiaPpD5D0lJJQ5KulbRzXneXPD+Ul09vxz9gVmURcW5ETIuI6aRB1r8VEacCS4CT8mpzgZvz9C1sGaz9pLy+h78wM+tTTRfEJO0H/DUwKyJeReqefzLuDWY2Fh8GPiBpiNQG7NKcfikwJad/AFhYUnxmZtYFrd6anADsJuk3wO7AWlJvsHfl5VcA5wEXk3qDnZfTbwA+J0m+2rfxIiIGgcE8/RhweIN1fgm8vauBmZlZaZouiEXEGkmfAn4K/AK4A1hGl3qDNerZs2Dmpm3W68feZI6hejGYmZk1o+mCmKS9SLVcM4BngeuBOa0GNNbeYI169sxr1GPl1Mbvb1UVehY5hurEYL2v2ONtwcxNzFt4m3u8mVnHtdJY/43AjyPiZxHxG+BGUg+xybm3FzTuDYZ7g5mZmZm1VhD7KXCEpN3zk7+PAVbi3mBmZmZmY9J0QSwilpIa3X8fWJ63dQnuDWZmZmY2Ji31moyIjwEfq0t2bzAzMzOzMfCg32ZmZmYlcUHMzMzMrCQuiJmZmZmVxAUxMzMzs5K4IGZmZmZWklbHmjQzMxt3pjcYycWsGa4RMzMzMyuJC2JmZmZmJXFBzMzMzKwkLoiZmZmZlcSN9c3M2qxRQ+5Vi04oIRIrm78LNhrXiJmZmZmVxAUxsw6StKuk70r6oaQVkv4hp8+QtFTSkKRrJe2c03fJ80N5+fQy4zczs85yQcyss34FHB0RrwEOBuZIOgL4BHBBRLwCeAY4M69/JvBMTr8gr2dmZn3KBTGzDopkY57dKb8COBq4IadfAbw1T5+Y58nLj5GkLoVr1nWS9pe0RNLKXGt8Tk7fW9Kdkh7Jf/fK6ZL0mVxrfL+kQ8v9D8xa48b6Zh0maUdgGfAK4CLgUeDZiNiUV1kN7Jen9wMeB4iITZI2AFOAJ+u2OR+YDzAwMMDg4GBHYt+4cWPHtt2pfS+YuWmbtLFsp/i+gd3SfLP/e7MxQLnHvKT9bwIWRMT3Je0BLJN0JzAPuCsiFklaCCwEPgy8CTggv14LXJz/mvWklgpikiYDXwJeRbrKfzfwMHAtMB1YBbwjIp7JV/UXAscDPwfmRcT3W9m/WS+IiN8CB+f8chPw+23Y5iXAJQCzZs2K2bNnt7rJhgYHB+nUtju173mNeqmdOvp2iu9bMHMT5y+fMKb3tTMGKPeYl7H/iFgLrM3Tz0t6kHRBciJQC+QKYJBUEDsRuDIiArhH0mRJU/N2zHpOqzViFwLfiIiTcmPj3YGP4KsYs21ExLOSlgBHApMlTci1YtOANXm1NcD+wGpJE4A9gadKCdisy3LnlEOApcBAoXD1BDCQpzfXGme1GuWtCmL1tcbtrulrVOs5VvVxlF0LOhLH1nlNF8Qk7Qm8nlR9TET8Gvi1JF/FmGWS9gV+kwthuwHHkhrgLwFOAhYDc4Gb81tuyfN35+XfynnGrK9JmgR8DXhfRDxXbBoZESFpu/JBfa3xpEmT2lrT16jWc6zqa0fLrgUdiWPrvFZqxGYAPwO+LOk1pDYw59Dmq5jhSruNSsKttMvYXlUoiTuG6sQwgqnAFbmd2A7AdRFxq6SVwGJJ/wz8ALg0r38p8BVJQ8DTwMllBG3WTZJ2IhXCro6IG3PyutrFuqSpwPqcXqs1rinWKJv1nFYKYhOAQ4G/ioilki4k3YbcrB1XMcOVdhuVhFtpl7G9qlASdwzViWE4EXE/6VZLffpjwOEN0n8JvL0LoZlVQm4/fCnwYER8urCoVju8iG1rjc+WtJjUvGWD76xYL2vl8RWrgdURsTTP30AqmK3LVy/4KsbMzEZxFHAacLSk+/LreFIB7FhJjwBvzPMAtwOPAUPAF4H3lhCzWds0XSMWEU9IelzSgRHxMHAMsDK/fBVjZmajiojvAMM9K++YBusHcFZHgzLrolZ7Tf4VcHXuMfkYcAa5HYykM4GfAO/I695OenTFEOnxFWe0uG8zMzOzntZSQSwi7gNmNVjkqxgzMzOzUXiIIzMzM7OSeIgjMzOzLppe18P/8jkTS4rEqsA1YmZmZmYlcUHMzMzMrCQuiJmZmZmVxAUxMzMzs5L0bGP95Ws2tDToqpmZmVnZXCNmZmZmVhIXxMzMzMxK4oKYmZmZWUlcEDMzMzMriQtiZmZmZiVxQczMzMysJC6ImXWQpP0lLZG0UtIKSefk9L0l3Snpkfx3r5wuSZ+RNCTpfkmHlvsfmJlZJ7kgZtZZm4AFEXEQcARwlqSDgIXAXRFxAHBXngd4E3BAfs0HLu5+yGZm1i0uiJl1UESsjYjv5+nngQeB/YATgSvyalcAb83TJwJXRnIPMFnS1C6HbWZmXdLyk/Ul7Qh8D1gTEW+WNANYDEwBlgGnRcSvJe0CXAkcBjwFvDMiVrW6f7NeIWk6cAiwFBiIiLV50RPAQJ7eD3i88LbVOW1tIQ1J80k1ZgwMDDA4ONiRmDdu3NixbXdq3wtmbtombSzbKb5vYLc03+z/3mwMUO4xr8L+zcabdgxxdA7pKv9Fef4TwAURsVjS54EzSbdXzgSeiYhXSDo5r/fONuzfrPIkTQK+BrwvIp6TtHlZRISk2J7tRcQlwCUAs2bNitmzZ7cx2i0GBwfp1LY7te9GQ5+tOnX07RTft2DmJs5fPmFM72tnDFDuMa/C/s3Gm5ZuTUqaBpwAfCnPCzgauCGvUn/LpXYr5gbgGBV/jcz6lKSdSIWwqyPixpy8rnbLMf9dn9PXAPsX3j4tp5mZWR9qtUbs34APAXvk+SnAsxFRq5ev3VaBwi2XiNgkaUNe/8niBsd6y6V262A0/XjLxjFUL4bh5IuNS4EHI+LThUW3AHOBRfnvzYX0syUtBl4LbCjcwjQzsz7TdEFM0puB9RGxTNLsdgU01lsun736Zs5fPnr4zd5aGE0Vqu8dQ3ViGMFRwGnAckn35bSPkApg10k6E/gJ8I687HbgeGAI+DlwRnfDNTOzbmqlRuwo4C2Sjgd2JbURu5DUy2tCrhUr3lap3XJZLWkCsCep0b5Z34qI7wDD3YI/psH6AZzV0aDMzKwymm4jFhHnRsS0iJgOnAx8KyJOBZYAJ+XV6m+5zM3TJ+X1t6uBspmZmVk/6cRzxD4MfEDSEKkN2KU5/VJgSk7/AFseYGlmZmY2LrXj8RVExCAwmKcfAw5vsM4vgbe3Y39mZmZm/aAtBTEzM2vd9IW3sWDmpq2eQ7Zq0QklRmRmneYhjszMzMxK4oKYmZmZWUlcEDMzs1JJukzSekkPFNL2lnSnpEfy371yuiR9RtKQpPslHVpe5Gatc0HMzMzKdjkwpy5tIXBXRBwA3MWWnvZvAg7Ir/mksYx72vI1G5i+8LatXjZ+uCBmZmaliohvA0/XJRfHJ64ft/jKSO4hPUR8anciNWs/95o0M7MqGiiMs/oEMJCnN49bnNXGNN5qTNb6cYtbHZN2+ZoNW80vmNn0prbRaOzkqoyfW+WxfKsc2/ZwQczMzCotIkLSdo3EUj9u8aRJk1oak3ZeB28XLpi5aZuxkzs1TvL2qvJYvlWObXv41qSZmVXRutotx/x3fU6vjVtcUxzT2KznuEbMzMyqqDY+8SK2Hbf4bEmLgdcCGwq3MPtGowb7frhvf3JBzMzMSiXpGmA2sI+k1cDHSAWw6ySdCfwEeEde/XbgeGAI+DlwRtcDNmsjF8TMzKxUEXHKMIuOabBuAGd1NiKz7nEbMTMzM7OSuCBmZmZmVhLfmjSzSlq+ZsM2jwxwY2Uz6zeuETPrII+hZ2ZmI2m6ICZpf0lLJK2UtELSOTndPzJmW1zOOB5Dz8zMRtZKjdgmYEFEHAQcAZwl6SD8I2O2mcfQMzOzkTTdRiw/QG9tnn5e0oOk8b5OJD0PBtKPzCDwYQo/MsA9kiZLmtqPD+IzG0VLY+jBtuPodWq8tTLHcmt2/L369zTzvtq+m/3fW4mh/v/u9vHvl/H7zHpFWxrrS5oOHAIspc0DtQ53Qmh0km6kH3+gHEP1YmhWM2Po5fdtNY5ep8ZbK3Mst89efXNT4+81GhNwe99XG/uv2fH+WomhftzBbo852C/j95n1ipYLYpImAV8D3hcRz0navKwdA7UOd0JodJJupFMnsSqcrBxDdWLYTutqtcEeQ8/MbHxrqdekpJ1IhbCrI+LGnOyBWs1GVhtDD7YdQ+/03LHlCPp0DD0zM9ui6RoxpaqvS4EHI+LThUXjeqBWsyKPoWdm7VI/ELifq9cfWrk1eRRwGrBc0n057SP4R8ZsM4+hZ2ZmI2ml1+R3AA2z2D8yZmZmZqPwk/XNzMzMSuKCmJmZmVlJXBAzMzMzK0lbHuhqZmZm3VXfixLck7IXuUbMzMzMrCSuETMzMytoVNNk1ikuiJmZmfUJ367sPb41aWZmZlYS14iZmfU414KY9S7XiJmZmZmVpO9rxDxIqpmZjWf+Haw214iZmZmZlcQFMTMzM7OS9P2tSTMzMxuZO3yUxwUxM2sbn8zNzLaPC2JmZuNUo4Lz5XMmlhBJefwUfSubC2JmZmbjyFgLn9MX3saCmZuYV1jfNdzt1/WCmKQ5wIXAjsCXImJRt2MwqzLnEbPROZ+Uw4/CaL+uFsQk7QhcBBwLrAbulXRLRKzsVgxuw2JVVoU8YlZ1zifV4d/U1nW7RuxwYCgiHgOQtBg4ESg18zTTRqC+unZ7dPpLWoWMUYWrpirE0IS25pFmvwu19xW/5z1y/Gx8qORviSWdbHfX6DzU6d+8Tv+WKCLausERdyadBMyJiPfk+dOA10bE2YV15gPz8+yBwMPDbG4f4MkOhjuasvfvGNobw8siYt92BNOKseSRnD7WfNKqMj9f77t6+++ZfNIgjzxF+eeq4ZT9mY/EsW2f7c4jlWusHxGXAJeMtp6k70XErC6EVMn9O4ZqxdBtY80nrSrz2Hrf5Sh7/+1Sn0eq/H85tuZUObbt0e0n668B9i/MT8tpZpY4j5iNzvnE+ka3C2L3AgdImiFpZ+Bk4JYux2BWZc4jZqNzPrG+0dVbkxGxSdLZwDdJXY4vi4gVTW6u47dlKr5/cAw1VYihLdqcR9qhzGPrfY/P/Y+qyXxS5f/LsTWnyrGNWVcb65uZmZnZFt2+NWlmZmZmmQtiZmZmZiXpuYKYpDmSHpY0JGlhm7a5StJySfdJ+l5O21vSnZIeyX/3yumS9Jm8//slHVrYzty8/iOS5hbSD8vbH8rvlaTLJK2X9EBhvY7us8E+1kj6WV0M5+X0+/Lr+MKyc/P2Hpb0Z6N9Jrkh7dKcfm1uVIukXfL8KknP5VhWSDqnpOOweR8GkvaXtETSyuLn0uUYdpT0A0m3lrDvyZJukPSQpAclHdnFfb8/H/MHJF0jadcO72/M56FeNdz5qU3bbphXqnQOq89LYzgvD+Xl0wvbaMu5vy6ubfJZlY5bV0VEz7xIjTIfBV4O7Az8EDioDdtdBexTl/ZJYGGeXgh8Ik8fD3wdEHAEsDSn7w08lv/ulaf3ysu+m9dVfu+bgNcDhwIPdGufDfbxeeDyuhjOAz7Y4BgdlI/3LsCM/DnsONJnAlwHnFzY11/m6ffm+an5/7wW2AP4Ud5Pt4/D5n34FeTP5dA8vflz6XIMHwC+Ctxawv9/BfCePL0zMLlL+90P+DGwW56/DpjX4X2O+TzUi6+Rzk9t2n7DvFKlc1h9XhrtvJynTwauzdNtO/fXxbVNPqvScevq97TsALbzS38k8M3C/LnAuW3Y7iq2LYg9DEwtZLaH8/QXgFPq1wNOAb5QSP9CTpsKPFRI37weMJ2tT4Dd2Gf9Ph5lbAWxrY41qbfSkcN9JvnL/yQwof6zq703T0/I6wm4mTR2XBnH4eGyv99VfdU+ly7ubxpwF3A0XS6IAXuSCkMq4TjvBzxO+lGZANwKHNeF/Y7pPNSLr+HOTx3cX6XOYfV5qcnzctvO/YV1G+azqhy3br967dZk7URVszqntSqAOyQtUxoWA2AgItbm6SeAgVFiGCl99Rhj7sY+6/fRaCiGs3P172WFatvtjWEK8GxEbGoQw+b35OUbgIOBQ4ClJR2HAWwb+fZE7XPpln8DPgT8rov7rJkB/Az4cr6d8yVJE7ux44hYA3wK+CmwFtgQEXd0Y991+ilvdOo3Yxt1eaUq57D6vLS95+UpTcQ80j5qhstnVTluXdVrBbFOeV1EHEq6ZXiWpNcXF0YqOkc3A+rGPofZx8XA/yIVjNYC53cyhkzAl4H3RcRzY4ixrcr4fHuBpEnA12jwuXRwn28G1kfEsm7sr4EJpFt1F0fEIcALpNsXHZcvek4k/Ui9BJgo6X93Y9/Dcd4Ym5HySlnnsArkpZGMms/G07m/1wpiHRnWIl+JEhHrgZuAw4F1kqYC5L/rR4lhpPRpY4y5G/us38dWA6ZGxLqI+G1E/A74Yj4WzcTwFDBZ0oS69K22pdQYeRpwZUTcWOJxWI9tJmkn0g/L1YXPpRuOAt4iaRWwGDha0lVd3P9qYHVE1GoAbyD9YHTDG4EfR8TPIuI3wI3AH3dp30X9lDc6PhTSMHmlCuewbfIScCFjOy9PIN0+fKqJmEc699cMl8+qcNy6rtcKYm0f1kLSREl71KaB44AH8nbn5tXmku79k9NPz704jiDdPlhLum9+nKS98pXtcaT74muB5yQdkXttnF7YVr1u7LN+H3fWHY+phdm35WNRe9/JSj1rZgAHkBpDNvxM8pXGEuCkYf6fuTm2O4BVEfHpko/DcJ/JuJOP1aXAg3WfS8dFxLkRMS0ippO+S9+KiK7VCkXEE8Djkg7MSccAK7u0+58CR0jaPX8GxwAPdmnfRf2UNzo6FNIIeaX0c9gweelURjkv5+mT8vpBe8/9tdiGy2elH7dSlN1IbXtfpN4TPyI1Mv9oG7b3clJvjx8CK2rbJN3nvgt4BPhPYO+cLuCivP/lwKzCtt4NDOXXGYX0WaQCzaPA5/I2riHd+vsN6ergzE7vs8H/9UR+FWP4St7H/aQv7NTCNj+at/cwuQfKSJ9JPrbfzbFdD+yS03fN86tJ1cIPAffl1/ElHIfN+/ArAF6XP5f7i59LCXHMppxekwcD38v//7+Te2F1ad//kPPDAzkv7tLh/Y35PNSrr+HOT23adsO8UrVzWDEvjeG8PJSXv7zw/rac++ti2iafVe24devlIY7MzMzMStJrtybNzMzM+oYLYj1C0q6SQtK00dc2s26S9B1J81p4/2pJs4dZ9sbc4NqsUiQNSnrPMMum59+sCY2W2xYuiLVA0sbC63eSflGYP3WU986RNNTCvhdL+lXe1/OS7pXU9h5WjeKUNEXSlZLWKQ1R9LCkD+RltQLjC4Vj8US747JqkHSVpC/Xpb1B0lN1HT/ata9f5+/U05LukPTKdu6jnXIBKiQtKDsWGz+2N08qDTV0maQn8m/Jj9TmoaBsZC6ItSAiJtVepB5Pf15Iu7oLIfxT3veepOEivtaFfcKWDgevJA1L8TbSU5KLDiwci9/rUlzWfecAb5J0LGx+HMkXgQWx5aGJLZO0Y578l/yd34/U7fyL7dpHB8wFnib12DLrljHnyVxbdQEwCfgD0m/JW0gN361LXBDrIEm7SbpI0tp86+FfJe0kaQrpeWUvL9QaTZF0lNJAqc9K+h9JF4ylWjfSM7++CvyepL3zvn8/3y7ZoDSw95U5vVZj9ReSHs01Wn8r6UBJ383rXy1pwnBxAn9EembOhoj4XUSsjIibOnUcrboi4ingr4BLlB7/8jHg0Yi4XNIOkj6Sv2dP5lrc2iC+OygN+PtE/r4PSvqD2nbzVf1Fkr4h6QXgT+r2+wvSeHYHF9MlvUdpEOFnJH1d0ubnIuXv/V/meJ6X9DFJB0i6J+eDa5SeCVXb1l8oDRj8lKR/L9YmaMtgxxskXUi6MCnGsQfwf5HG7ztIUn2c8yT9JB+XhXXLdpf0lfw/rAAO275PxcazUfLkeTnfXSXpOWAe6Xz+1Yh4Jp/PH4qIG2rbk/THSndcNmiEOy9Kg4t/Kn+nHwNO6Pg/2y/K7rbZLy/SeJVvrEv7JPB/gH1Iwyjcy5bHY8wBhurWP5yUKXYkPd1+CPiLvGxXUjfpaXl+MfC3eXoC8D5St/daT9ibgA+SfiB2A46q2871pKugQ0hd178JvIw0zt0jwDtHiPMq0uM+5gKvqFu2VZx+jY8XqTb2FtLDHPfPaQuA/yLVXu0KfAn4Sl62A+lHYI+87HPA9wrbuwp4hjRO3Q6kAYevAs7LyyeRHr2wrPCe/5vUvf7AnCfOA/5PXjYhfy9vzPt8NfBr0nP0ppO6zj8EnJrXP45U43Zwju//Iz1XCeDFwEZSTfBOwN8AmygM0A2cQXocxA6kAYcvKCybmd9/VP6/PpPfPzsv/xQwmGN6Gen5SqvK/oz96q3XMHnyvHy+f2v+bu6W8+WK/J09oG4be+d8eFrOQ6fk+Sl5+SBbBu7+i5yH9s/vW5Lz3ISyj0XVX6UH0C8vGhfE1gBHF+ZPJA9ESoMCToNtLgSuydONCmK/AJ4Ffpmn315473WkH7epddusbeewQtoK4JzC/EXAouHiBCYCf096Zs4m0o/fG+u2vyHH9izwybI/H786+yJdaGys+x49AryhML9//q7u0OD9++TvzcQ8fxVwWd06V+X3P5vXfRR4VWH5ncDcwvwE4FekgmCtIPbawvIfkm7X1OYvBD6Vp68g3QatLXsR8FvSE7rfDXynsGwH0rO45hXSBgvbOg1Yx5ZBkP8RuKqw7qS87dl5/qfFcwmpVm1V2Z+xX731GiZPngd8u2693YCPAMtIhbQh8rPC8nf3u3Xr3137rrN1Qexb5IqDPH8cLoiN6eVbkx0iScDvAT8pJP+EEQaclXRQvp2yLlcb/z3pB2o4H4+IyaSMdCTwOUlH52XvB3YHfqA0gHf908nXFaZ/0WB+0nA7jYgXIuIfI+Jg0sPx/gP4Wr4dU/OHETE5vz40wv9gfSAi1pGGy1pRSH4p8B/51uOzpAcxArw438b4pKTH8ne91ial+H0vDuZbsyh/52eQfjQOKCx7GXBRYX9PkgY7LvY0Huv3/iUU8m6k8QOfIeXflxRji9Q0YPMAw0qDP78eqLUTvSlvd05h28X3byS1JauZWve/F88hZmMyTJ6EunwVEb+IiH+JiMNI5/PrgOtzM5et8kE23O/YVt/rBu+zYbgg1iGRLgmeIP041LyULeNdNXqS7heB7wP/KyJeRLpyVoP1ttlXRNxHepLx8TltTUS8m3RS/2vgMkkvbeZfGWXfG4BFpBqDZrZv/Ws1cGyhQD45InaNNLzJ6aTv6tGkBsKvyO8pft+H/e5FxCrSxcZnJe2Skx8Hzqzb326xZTy77fE/FPJuvsjYi5R/11IY307SDmxd2Ds9/x9fV+oxPATszJZhVerfP4l0K6fmCbYeP8/5ytpppHz1HPAvpLseM6jLB1nxd6xoq+81/t6OmQtinXUN8LHcEP/FpGEiagMYryPVDBRrnvYgjaG1UdIfAv/PWHck6VXAEeSrH0nvlPSSXCB8Nq/22yb+h23izA0+D1XqeLAbqaD3JO5pY1v7PPAvtQsASS+W9Ja8bA/SbcOnSDW3H9/ejUfE10nfu9pzjD4PfLTW6F+pW/5Jw71/FNcAZ0p6dS7o/b+k9margVuBgyWdmBv3vx/Yt/De00m12QcXXu8E/jx3VrgeOFHSkXnb/8zWP47XAR/J8b8UOLvJ/8FsVJL+TtIfSdpZqYflOaTfjIeB24FXSnpX7vDyTuAgUh6odx3w15Km5e+5H4ExRi6IddbfkxrariC1p/ovUgN+SO1TbgF+km+l7E06ob9H0kZSO61rR9n+3yn3ZgRuIzUovjwvOxJYlpddD8yPiEZXMaNpFOcOpALl06Raj6NIYxH+qontW//6NPAN4C5JzwP/TeqMAvBl0tX2/5Dyx383uY9/BT4saeeIuD7v8/p8u/N+4M+a2WhEfINUI30T6Ur/pcCpedk6UsHqX0kFwZduNLn3AAAVlklEQVQCSwEkvY50i+aiiHii9srbWUXqBHM/6cfuOlLNQm3M15qP5X2uIjX0v7KZ/8FsjIKUH58k5cdjgRMiYmOkHphvJnW8eQr4EPDmiHiywXa+SOr09UPSnZ0buxB7X/BYk2ZmZmYlcY2YmZmZWUlcEDMzMzMriQtiZmZmZiVxQczMzMysJKOOY1imffbZJ6ZPn95w2QsvvMDEiRO7G1Ab9Grc0LuxjyXuZcuWPRkR+464UkWNlE86qVe/D+DYm9Wr+aSXfkuqFI9jaWykWJrKI2U/2n+k12GHHRbDWbJkybDLqqxX447o3djHEjeFcQ577TVSPumkXv0+RDj2ZvVqPuml35IqxeNYGhsplmbyiG9NmpmZmZXEBTEzMzOzkrggZmZmZlaSSjfW75bpC2/bJm3VohNKiMSsvfzdNrNeVjuHLZi5iXkLb+vL85drxMzMzMxK4oKYmZmZWUl8a9LMzKxExSYE/XwLzhpzjZiZmZlZSVwjZmY2Drjjhlk1uUbMzMzMrCQuiJmZmZmVxLcmzcxs3Fq+ZgPz6m7b+patdZNrxMzMrDSSdpX0XUk/lLRC0j/k9BmSlkoaknStpJ1z+i55figvn15m/GatckHMzMzK9Cvg6Ih4DXAwMEfSEcAngAsi4hXAM8CZef0zgWdy+gV5PbOe5VuTZmZWmogIYGOe3Sm/AjgaeFdOvwI4D7gYODFPA9wAfE6S8nasjXzbtjtcEDMzs1JJ2hFYBrwCuAh4FHg2IjblVVYD++Xp/YDHASJik6QNwBTgybptzgfmAwwMDDA4ONhw3wO7pYeoFg23bqcU91+Lp9sxNFKlY1Ol47Jx48a2xuGCmJmZlSoifgscLGkycBPw+23Y5iXAJQCzZs2K2bNnN1zvs1ffzPnLt/4pXHVq43U7ZV7dk/XPXz6h6zE0UqVjU6XjMjg4yHDfp2a4jZiZmVVCRDwLLAGOBCZLqpUCpgFr8vQaYH+AvHxP4Kkuh2rWNi6ImZlZaSTtm2vCkLQbcCzwIKlAdlJebS5wc56+Jc+Tl3/L7cOsl7kgZtZBki6TtF7SA4W08yStkXRffh1fWHZu7pb/sKQ/Kydqs66aCiyRdD9wL3BnRNwKfBj4gKQhUhuwS/P6lwJTcvoHgIUlxGzWNk23EZO0K/BtYJe8nRsi4mOSZgCLSRlnGXBaRPxa0i7AlcBhpGrkd0bEqhbjN6u6y4HPkb77RRdExKeKCZIOAk4G/hB4CfCfkl6Z28+Y9aWIuB84pEH6Y8DhDdJ/Cby9C6GZdUUrNWJ+9ovZKCLi28DTY1z9RGBxRPwqIn4MDNHgh8jMzPpH0zVifvaLWUvOlnQ68D1gQUQ8Q+qWf09hnWKX/a2MtWt+fddzaF/383Z34e6m8Rh7J78LZta8lh5fUeazX9p5Iu3mCWo8/gCUrYJxXwz8E+nC5Z+A84F3b88Gxto1v/5hjNC+7uft7sLdTeMx9k5+F8yseS0VxMp89ks7T6TdPEGNxx+AslUt7ohYV5uW9EXg1jy7uVt+Vuyyb2ZmfagtvSb97BezsZM0tTD7NqDWo/IW4OQ8qPEM4ADgu92Oz8zMuqfpgpif/WI2OknXAHcDB0paLelM4JOSlufu+n8KvB8gIlYA1wErgW8AZ7nHpJlZf2vl1uRU4IrcTmwH4LqIuFXSSmCxpH8GfsDWz375Sn72y9OkbvpmfS0iTmmQfGmDtNr6Hwc+3rmIzMysSlrpNelnv5iZmZm1wE/WNzMzMytJS70me9H0Bj0kzcz6jc91Zr3BNWJmZmZmJXFBzMzMzKwkLoiZmZmZlcQFMTMzM7OSjLvG+mPVqKHrqkUnlBCJmZmZ9SvXiJmZmZmVxAUxMzMzs5K4IGZmZmZWEhfEzMzMzErigpiZmZlZSVwQMzMzMyuJC2JmZmZmJXFBzMzMSiNpf0lLJK2UtELSOTl9b0l3Snok/90rp0vSZyQNSbpf0qHl/gdmrXFBzMzMyrQJWBARBwFHAGdJOghYCNwVEQcAd+V5gDcBB+TXfODi7ods1j4uiJmZWWkiYm1EfD9PPw88COwHnAhckVe7Anhrnj4RuDKSe4DJkqZ2OWyztvEQR2ZmVgmSpgOHAEuBgYhYmxc9AQzk6f2AxwtvW53T1hbSkDSfVGPGwMAAg4ODDfc5sBssmLlpq7Th1u2U4v5r8XQ7hkaqdGyqdFw2btzY1jhcEDMzs9JJmgR8DXhfRDwnafOyiAhJsT3bi4hLgEsAZs2aFbNnz2643mevvpnzl2/9U7jq1Mbrdsq8wtjGC2Zu4vzlE7oeQyNVOjZVOi6Dg4MM931qhm9NmnWQpMskrZf0QCHNjZDNCiTtRCqEXR0RN+bkdbVbjvnv+py+Bti/8PZpOc2sJzVdEHNPF7MxuRyYU5fmRshmmVLV16XAgxHx6cKiW4C5eXoucHMh/fT8m3IEsKFwC9Os57RSI+aeLmajiIhvA0/XJbsRstkWRwGnAUdLui+/jgcWAcdKegR4Y54HuB14DBgCvgi8t4SYzdqm6TZi+QpkbZ5+XlKxp8vsvNoVwCDwYQo/MsA9kiZLmuorGRuHWmqEDGNviFzf0Bba19i23Q1Wu6nfYl++ZsM26y2YOfq2qnAMIuI7gIZZfEyD9QM4q6NBmXVRWxrrl9HTpdkTaaMfprFqx0mr334AekGV426mEXJ+35gaIhcbAde0q7FruxusdlO/xd7ocx6LKjR8NhvvWi6IldXTpdkTabMnLGjPSavffgB6QQXjXlerDXYjZDOz8a2lgthIPV38I2M2rFoj5EVs2wj5bEmLgdfiRshmZqWbXleBc/mciW3dfiu9Jt3TxWwUkq4B7gYOlLRa0pm4EbKZmWWt1IjVerosl3RfTvsI6UfluvyD8xPgHXnZ7cDxpB+ZnwNntLBvs54QEacMs8iNkM3MrKVek+7pYmZmZtYCP1nfzMzMrCQuiJmZmZmVxAUxMzMzs5K4IGZmZmZWkrY8WX+8qH+WyKpFJ5QUiZmZmfUD14iZmZmZlcQFMTMzM7OS+NakmVkPWb5mQ0tj5ppZtbhGzMzMzKwkLoiZmZmZlcQFMTMzM7OSuI1YC+ofZwF+pIWZ9Q6fw8zK54KYmVmF1ReWFswsKRAz6wjfmjQzMzMriQtiZmZmZiXxrUkzs4po1GbLzPqba8TMzMzMSuKCmJmZlUrSZZLWS3qgkLa3pDslPZL/7pXTJekzkoYk3S/p0PIiN2udC2JmZla2y4E5dWkLgbsi4gDgrjwP8CbggPyaD1zcpRjNOqKlNmKSLgPeDKyPiFfltL2Ba4HpwCrgHRHxjCQBFwLHAz8H5kXE91vZv1kvk7QKeB74LbApImYNl3/KirEX+dlYvScivi1pel3yicDsPH0FMAh8OKdfGREB3CNpsqSpEbG2O9GatVerjfUvBz4HXFlIq13FLJK0MM9/mK2vYl5Luop5bYv7N+t1fxoRTxbmh8s/1mfcMH9UA4XC1RPAQJ7eD3i8sN7qnLZVQUzSfFKNGQMDAwwODjbeyW6wYOamrdKGW7dTivuvxdPtGBqp0rEp87jUH4ONGze2NY6WCmK+ijFru+Hyj9m4FREhKbbzPZcAlwDMmjUrZs+e3XC9z159M+cv3/qncNWpjdftlHmFQvmCmZs4f/mErsfQSJWOTZnHZV7dRdPlcyYy3PepGZ14fEVXrmKaLZHWl2zbbbSY2l2S7qZejb3CcQdwR/6B+UL+4Rgu/2xlrPmk0fe9Xceiqsd1LP9zFWJv9lzUqJainco+LgXrahfrkqYC63P6GmD/wnrTcppZT+roc8Q6eRUzODjYVIm0vmTbbqOV1puNuwp6NfYKx/26iFgj6cXAnZIeKi4cKf+MNZ80+r6364qyqsd1LP9zFWJv9lxUqxnolCrUxGS3AHOBRfnvzYX0syUtJjVv2eA7K9bLOtFrcl2+esFXMWbDi4g1+e964CbgcIbPP2Z9S9I1wN3AgZJWSzqTVAA7VtIjwBvzPMDtwGPAEPBF4L0lhGzWNp24rPJVjNkoJE0EdoiI5/P0ccA/Mnz+GXfc+3H8iIhThll0TIN1AzirsxGZdU+rj6+4htSweB9Jq4GPkX5ArstXND8B3pFXv5306Ioh0uMrzmhl32Y9bgC4KT3VhQnAVyPiG5LupXH+MTOzPtRqr0lfxZg1ISIeA17TIP0pGuQfMzPrT36yvpmZmVlJXBAzMzMzK0lHH19hZmaJn6RvZo24INZm7ullZmZmY9X3BTFfhZq1R31eunzOxJIiMTPrH31fEDOzctUX4BrVEI/1gmks2zIz6yUuiJmZtZlr4s1srFwQM7O2GUsBxIUUM7MtXBAzs77n9m1mVlUuiJmNc+7pa2ZWHj/Q1czMzKwkrhEzs6YsX7OBeT3a3qtR7M3WArrNm5m1omcLYu08kZrZ1ly4MDPrjp4tiPWS4o/agpmbmF1eKGZ9xQVGM+t1LoiZjTMuvIydj5WZdZob65uZmZmVxDViZma49svMyuEaMTMzM7OSuCBmZmZmVpK+ujXZK7cW/CRzMzMzgxJqxCTNkfSwpCFJC7u9f7Oqcx4xG53zifWLrhbEJO0IXAS8CTgIOEXSQd2MwazKnEfMRud8Yv2k27cmDweGIuIxAEmLgROBlV2Oo3LGclt1rLcvfeuzpzmPmI3O+cT6hiKiezuTTgLmRMR78vxpwGsj4uzCOvOB+Xn2QODhYTa3D/BkB8PtlF6NG3o39rHE/bKI2LcbwYxkLHkkp481n3RSr34fwLE3q2fySQ//llQpHsfS2EixbHceqVxj/Yi4BLhktPUkfS8iZnUhpLbq1bihd2Pv1bhHMtZ80km9fFwde//r1d+SKsXjWBprdyzdbqy/Bti/MD8tp5lZ4jxiNjrnE+sb3S6I3QscIGmGpJ2Bk4FbuhyDWZU5j5iNzvnE+kZXb01GxCZJZwPfBHYELouIFU1urtTbMi3o1bihd2PvmbjbnEc6rWeOawOOvYf1+W9JleJxLI21NZauNtY3MzMzsy08xJGZmZlZSVwQMzMzMytJTxbEqja0haT9JS2RtFLSCknn5PS9Jd0p6ZH8d6+cLkmfyfHfL+nQwrbm5vUfkTS3S/HvKOkHkm7N8zMkLc3xXZsbwyJplzw/lJdPL2zj3Jz+sKQ/60LMkyXdIOkhSQ9KOrJXjneVSbpM0npJDxTSXiPpbknLJf2HpBcVlr06L1uRl++a0w/L80P52KtKsUvaSdIVOf1BSecW3tPV80uvnz+qps3H86WS7sjfkZXFc16345H0p5LuK7x+KemtJR6bT+ZtPNhMHm9zLJ+Q9EB+vXN74mgylt9XOq/8StIH67a1/eePiOipF6lh5qPAy4GdgR8CB5Uc01Tg0Dy9B/Aj0rAbnwQW5vSFwCfy9PHA1wEBRwBLc/rewGP57155eq8uxP8B4KvArXn+OuDkPP154C/z9HuBz+fpk4Fr8/RB+XPYBZiRP58dOxzzFcB78vTOwOReOd5VfgGvBw4FHiik3Qu8IU+/G/inPD0BuB94TZ6fUvvcge/mY6187N9UsdjfBSzO07sDq4DpZZxfev38UbVXu45nXjYIHJunJwG7lxlPYZt7A09vbzxt/K79MfBfOb/sCNwNzC4plhOAO0nno4mkPP+iDsfyYuCPgI8DHyxsp6nzR+mZpokv9ZHANwvz5wLnlh1XXYw3A8eSnuQ8tfBBP5ynvwCcUlj/4bz8FOALhfSt1utQrNOAu4CjgVvzl/xJYEL98Sb1UDoyT0/I66n+Myiu16GY9wR+TO5sUn8cq3y8e+FFKpAUCzMbasea9OymlXn6eOCqBu+fCjxUmN/qOFck9lOA/8jf4yn5xLt3Fc4vvXT+6IVXC8fzIOA7VYmnbhvzgatLPDZHAsuA3UgXMt8D/qCkWP4G+LtC+qXAOzoZS2G989i6INbU+aMXb03uBzxemF+d0yohV10fAiwFBiJibV70BDCQp4f7H8r43/4N+BDwuzw/BXg2IjY1iGFzfHn5hrx+t+OeAfwM+LLSLdUvSZpIbxzvXrSCNI4fwNvZ8iDNVwIh6ZuSvi/pQzl9P9KxrCnzuA4X+w3AC8Ba4KfApyLiaUr+TvTg+aPSWjyerwSelXRjPs/8q9Jg42XFU3QycE1ZsUTE3cASUv5ZSyp8PFhGLKRapzmSdpe0D/CnbP2w307EMpym8mAvFsQqS9Ik4GvA+yLiueKySMXjSj0rRNKbgfURsazsWLbTBNItqIsj4hDSD+pW9+KreLx72LuB90paRqq2/3VOnwC8Djg1/32bpGPKCXFYw8V+OPBb4CWkgv0CSS8vJ8Sk184fVdeG4zkB+BPgg6TbUC8H5pUYT207U4GZpDsPpcQi6RXAH5DuqOwHHC3pT8qIJSLuAG4H/ptUOL2blLe7HkuzerEgVsmhLSTtRPoAr46IG3PyupxpaplnfU4f7n/o9v92FPAWSauAxaTbkxcCkyXVHvZbjGFzfHn5nsBTJcS9GlgdEUvz/A2kglnVj3dPioiHIuK4iDiMdKJ7NC9aDXw7Ip6MiJ+TToaHko7htMImSjuuI8T+LuAbEfGbiFhPau8yi5K+Ez16/qisNh3P1cB9EfFYvgPw76Tvd1nx1LwDuCkiflNiLG8D7omIjRGxkdR268iSYiEiPh4RB0fEsaTmMj/qcCzDaSoP9mJBrHJDW+TeIpcCD0bEpwuLbgHm5um5pPvOtfTTcy+QI4ANufrzm8BxkvbKvTOOo4WrntFExLkRMS0ippOO47ci4lRSlfNJw8Rd+39OyutHTj9ZqVflDOAAUmPtTsX9BPC4pANz0jHASip+vHuVpBfnvzsAf0vqwAHpWM3MtwQmAG8gtcFaCzwn6YicN05ny2fRVSPE/lPShQf5tvYRwEOUcH7p1fNHVbXxeN5LuijdN693NOk8U1Y8NafQ5G3JNsbyU+ANkibkAswbgO26NdmuWJR6/U/J23w18Grgjg7HMpzmzh+tNGgr60VqJPwj0tXtRysQz+tIVZb3A/fl1/Gk9lN3AY8A/wnsndcXcFGOfzkwq7CtdwND+XVGF/+H2WzpNflyUkFqCLge2CWn75rnh/Lylxfe/9H8/zxMd3rIHUxqIHo/6Up1r1463lV9kU7wa4HfkGoEzgTOyfntR8AiCp0kgP9Naof1APDJQvqsnPYo8Lnie6oQO6kH3PU59pXA3xS209XzSz+cP6r0avPxPDZvZzlwObBzyfFMJ9Ww7FDmsSH1DvwCqfC1Evh0ibHsmmNYCdwDHNyFWH6PdI55Dng2T78oL9vu84eHODIzMzMrSS/emjQzMzPrCy6ImZmZmZXEBTEzMzOzkrggZmZmZlYSF8TMzMzMSuKCmJmZmVlJXBAzMzMzK8n/D0ul5NhY0osVAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# let's inspect the distributions of our variables first\n", + "\n", + "train_data.hist(bins=30, figsize=(10,10))\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ks_2sampResult(statistic=0.6634714306347143, pvalue=5.1793802892230377e-256)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let's test a few variables with the KS test:\n", + "\n", + "stats.ks_2samp(train_data['LotArea'], live_data['LotArea'])" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ks_2sampResult(statistic=0.08687964536879646, pvalue=7.606756582567709e-05)" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stats.ks_2samp(train_data['GrLivArea'], live_data['GrLivArea'])" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ks_2sampResult(statistic=0.04766523347665234, pvalue=0.09196049437201348)" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stats.ks_2samp(train_data['BsmtFinSF1'], live_data['BsmtFinSF1'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The p-values may change slightly between yours and this notebook, but mostly the distributions seem statistically different for all the variables.\n", + "\n", + "So let's inspect this in more detail:" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD4CAYAAAAQP7oXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xc1X3v/c9PM7rZlm+ya7BsYyU2BGMIEGFMQhMCBGxoYtLHAdNwwmlI3XMCTXrSPAm0TZqQcBpyXic09ECAFp5S2sS4Di1OH+dACZCTEC4WARzbYCPAwTI3I8nC1m00o9/5Y6+xx2IkjYQ0s8d836+XXt6z9tprr70tz8/rstc2d0dERKQQFaWugIiIlA8FDRERKZiChoiIFExBQ0RECqagISIiBUuWugITadasWb5w4cJSV0NEpKw8+eSTb7r77Hz7juigsXDhQpqbm0tdDRGRsmJmvx1qn7qnRESkYAoaIiJSMAUNEREpWEFjGma2Avg+kAD+3t2/M2h/NfCPwAeANuASd98V9l0DXAFkgC+4+33DlWlmVwF/CrwXmO3ubw4612nAo8Aad98whmsWERlWf38/ra2t9Pb2lroqE6qmpoZ58+ZRWVlZ8DEjBg0zSwA3AR8DWoHNZrbR3bfnZLsC6HD3RWa2BrgeuMTMlgBrgBOAucADZnZsOGaoMh8B/h14eIi6XA/cX/AVioiMUmtrK3V1dSxcuBAzK3V1JoS709bWRmtrK42NjQUfV0j31DKgxd1fdPcUsA5YNSjPKuDOsL0BOMeiO70KWOfufe7+EtASyhuyTHd/KttKyeNPgB8DbxR6gSIio9Xb20t9ff0RGzAAzIz6+vpRt6YKCRoNwO6cz60hLW8ed08DnUD9MMcWUuZhzKwB+CTwgxHyrTWzZjNr3rt373BZRUSGdCQHjKyxXGM5DYT/DfBVdx8YLpO73+buTe7eNHt23mdTRERkjAoJGnuA+Tmf54W0vHnMLAlMIxoQH+rYQsocrAlYZ2a7gNXAzWZ2UQH1j7Xbf/kSl972WKmrISIxsm/fPm6++eZRH3fBBRewb9++CajRIYUEjc3AYjNrNLMqooHtjYPybAQuD9urgQc9ervTRmCNmVWbWSOwGHiiwDIP4+6N7r7Q3RcSjZt83t3/raCrjLFtr3SydU9nqashIjEyVNBIp9PDHrdp0yamT58+UdUCCpg95e7pMA32PqLpsXe4+zYzuxZodveNwO3AXWbWArQTBQFCvvXAdiANXOnuGTg4tfawMkP6F4CvAEcBW8xsk7t/blyvOkZ6Uhl6+jOlroaIxMjVV1/NCy+8wMknn0xlZSU1NTXMmDGD5557jp07d3LRRRexe/duent7+eIXv8jatWuBQ0snHThwgJUrV3LmmWfyq1/9ioaGBu69915qa2vfcd3sSH7da1NTk8d97anL73iCn+/cy/PXraQyUU5DTCJHrmeffZbjjz8egG/+ZBvbX3lrXMtfMncqf/XxE4bcv2vXLn7v936PrVu38vDDD3PhhReydevWg1Nj29vbmTlzJj09PZx22mn8/Oc/p76+/rCgsWjRIpqbmzn55JO5+OKL+cQnPsFll1027LVmmdmT7t6Ur276liqxnlTUylBrQ0SGsmzZssOepbjxxht5//vfz/Lly9m9ezfPP//8245pbGzk5JNPBuADH/gAu3btGpe6HNGr3JaD7v6oj7I3lWFqTeFPZYpIcQzXIiiWyZMnH9x++OGHeeCBB3j00UeZNGkSZ511Vt5nLaqrqw9uJxIJenp6xqUuammUWLdaGiIySF1dHfv378+7r7OzkxkzZjBp0iSee+45HnusuLMv1dIoMXVPichg9fX1fOhDH2Lp0qXU1tYyZ86cg/tWrFjBLbfcwvHHH89xxx3H8uXLi1o3BY0Sy7Y0sn+KiAD88Ic/zJteXV3NT3/607z7suMWs2bNYuvWrQfTv/zlL49bvdQ9VWLZFkavgoaIlAEFjRLKDDipdLQqirqnRKQcKGiUUHfq0NOdChoiUg4UNEqoJ6dLqkfdUyJSBhQ0Sih38LtXLQ0RKQMKGiWUGzTUPSUi5UBBo4R6+nPGNFLDviZERN5lpkyZAsArr7zC6tWrS1ybQxQ0SkgtDREZydy5c9mwYUOpq3GQgkYJaUxDREaya9culi5dCsDy5cvZtm3bwX1nnXUWzc3NdHV18dnPfpZly5ZxyimncO+9905YffREeAnlzpjKnX4rIjHy06vhtd+Mb5lHnQgrvzPqwy655BLWr1/PN7/5TV599VVeffVVmpqa+PM//3POPvts7rjjDvbt28eyZcs499xzD1vocLyopVFC2ZZGbWWCnn6NaYjI8C6++OKDXVXr168/ONZx//33853vfIeTTz754Kq3L7/88oTUQS2NEsq2LmZOrtJzGiJxNYYWwURpaGigvr6eLVu2cPfdd3PLLbcA4O78+Mc/5rjjjpvwOqilUULZQFE/pUpjGiJSkEsuuYTvfve7dHZ2ctJJJwFw/vnn87d/+7dk38T61FNPTdj5FTRKqLs/Q7LCmFpTqdlTIlKQ1atXs27dOi6++OKDaV/72tfo7+/npJNO4oQTTuBrX/vahJ1f3VMl1JPKUFuVoKYyQXtXqtTVEZEYOXDgAAALFy48bJnzOXPmkE4fPnGmtraWW2+9tSj1UkujhLpTaSZVJaitSqh7SkTKQkFBw8xWmNkOM2sxs6vz7K82s7vD/sfNbGHOvmtC+g4zO3+kMs3sqpDmZjYrJ/3TZrbFzH5jZr8ys/eP9aLjojuVYVJVkkmVCXVPiUhZGDFomFkCuAlYCSwBLjWzJYOyXQF0uPsi4Abg+nDsEmANcAKwArjZzBIjlPkIcC7w20HneAn4iLufCHwLuG2U1xo7PakMtZVRS0NBQyResoPKR7KxXGMhLY1lQIu7v+juKWAdsGpQnlXAnWF7A3COmVlIX+fufe7+EtASyhuyTHd/yt135bm4X7l7R/j4GDBvFNcZSz39GSaFMQ1NuRWJj5qaGtra2o7owOHutLW1UVNTM6rjChkIbwB253xuBU4fKo+7p82sE6gP6Y8NOrYhbI9U5nCuAPK+JNfM1gJrARYsWDCKIouvO5WhriZJbWWCvvQAmQEnUWGlrpbIu968efNobW1l7969pa7KhKqpqWHevNH9/7vsZk+Z2UeJgsaZ+fa7+22ErqumpqZY/zehJ5VhztRqaquiBl9vf4bJ1WX3VyJyxKmsrKSxsbHU1YilQrqn9gDzcz7PC2l585hZEpgGtA1zbCFlvo2ZnQT8PbDK3dsKqHusdfenozGNygSglW5FJP4KCRqbgcVm1mhmVUQD2xsH5dkIXB62VwMPetQZuBFYE2ZXNQKLgScKLPMwZrYAuAf4T+6+s7DLi7foOY0kNdmgoXENEYm5EftCwhjFVcB9QAK4w923mdm1QLO7bwRuB+4ysxagnSgIEPKtB7YDaeBKd89ANLV2cJkh/QvAV4CjgC1mtsndPwd8nWic5OZojJ20uzeN140ohWjKbTR7CrQ8uojEX0Ed6O6+Cdg0KO3rOdu9wKeGOPY64LpCygzpNwI35kn/HPC5QupbDtz94OwpdU+JSLnQE+El0ts/gDvU5gYNdU+JSMwpaJRIdln0SZWHuqfU0hCRuFPQKJHsC5gmVSU1piEiZUNBo0SyrYrc7qludU+JSMwpaJTIoZaGBsJFpHwoaJRIdkyjtipBTZUGwkWkPCholEhP7phGpcY0RKQ8KGiUSG73VGWigmSFqXtKRGJPQaNEsi2NbCujtjJBT2qglFUSERmRgkaJHHxOI4xn1OhFTCJSBhQ0SqS7/9CYRvSn3hMuIvGnoFEivakMZlBTGf0V1OrtfSJSBhQ0SqQ7vB88rNgbvfJVLQ0RiTkFjRLpDivcZqmlISLlQEGjRKIXMOUEDQ2Ei0gZUNAoke5UmkmVh15nUqvuKREpAwoaJdI9qKVRo+4pESkDChol0pMaNKZRVaEptyISewoaJdI9OGioe0pEyoCCRon09GeoqcxtaSTp6c/g7iWslYjI8BQ0SqQnPKeRVVuZwB360lp/SkTiq6CgYWYrzGyHmbWY2dV59leb2d1h/+NmtjBn3zUhfYeZnT9SmWZ2VUhzM5uVk25mdmPYt8XMTh3rRcdBX3pQSyM8Ga5xDRGJsxGDhpklgJuAlcAS4FIzWzIo2xVAh7svAm4Arg/HLgHWACcAK4CbzSwxQpmPAOcCvx10jpXA4vCzFvjB6C41XlLpAaqSh25/diaVXvkqInFWSEtjGdDi7i+6ewpYB6walGcVcGfY3gCcY9H6GKuAde7e5+4vAS2hvCHLdPen3H1XnnqsAv7RI48B083s6NFcbJz0pQeozgkaNXrlq4iUgUKCRgOwO+dza0jLm8fd00AnUD/MsYWUOZZ6YGZrzazZzJr37t07QpGlkRlw0gN+eEujUq98FZH4O+IGwt39Nndvcvem2bNnl7o6eaXCYHd18vCH+yAa6xARiatCgsYeYH7O53khLW8eM0sC04C2YY4tpMyx1KMsZINGVZ7uqd5+zZ4SkfgqJGhsBhabWaOZVRENbG8clGcjcHnYXg086NEDBxuBNWF2VSPRIPYTBZY52EbgM2EW1XKg091fLaD+sZNtTeSOaWS3NXtKROIsOVIGd0+b2VXAfUACuMPdt5nZtUCzu28EbgfuMrMWoJ0oCBDyrQe2A2ngSnfPQDS1dnCZIf0LwFeAo4AtZrbJ3T8HbAIuIBpM7wb+cLxuQrH1HeyeentLQ89piEicjRg0ANx9E9GXdm7a13O2e4FPDXHsdcB1hZQZ0m8EbsyT7sCVhdQ37vrydk+ppSEi8XfEDYSXg+EGwjWmISJxpqBRAhrTEJFypaBRAhrTEJFypaBRAvmm3KqlISLlQEGjBPryjGmYGVXJCnr1cJ+IxJiCRgnka2kA1CQr6NNAuIjEmIJGCeQbCIdoXEPLiIhInClolMCQLY3KhKbcikisKWiUQL7ZU9nPGggXkThT0CiBgw/35by5D7LdU2ppiEh8KWiUQHbcoioxuHtKLQ0RiTcFjRJIpQcwg8qEHZZenUwoaIhIrClolEBfeoCqRAXRG3EPiVoa6p4SkfhS0CiBwe8Hz6rWlFsRiTkFjRLoSw9QlUy8Lb0mqSm3IhJvCholkBqypVGhloaIxJqCRgn0pTN5g4ZaGiISdwoaJRB1T+UJGmppiEjMKWiUwJDdU8kE/RknM+AlqJWIyMgUNEog6p7KMxCu94SLSMwpaJRAKj1AdWW+7im9vU9E4q2goGFmK8xsh5m1mNnVefZXm9ndYf/jZrYwZ981IX2HmZ0/Uplm1hjKaAllVoX0BWb2kJk9ZWZbzOyCd3LhpZR9uG8wtTREJO5GDBpmlgBuAlYCS4BLzWzJoGxXAB3uvgi4Abg+HLsEWAOcAKwAbjazxAhlXg/cEMrqCGUD/CWw3t1PCWXePLZLLr2hWhrZLisFDRGJq0JaGsuAFnd/0d1TwDpg1aA8q4A7w/YG4ByL1shYBaxz9z53fwloCeXlLTMcc3Yog1DmRWHbgalhexrwyuguNT5Gbmmoe0pE4qmQoNEA7M753BrS8uZx9zTQCdQPc+xQ6fXAvlDG4HN9A7jMzFqBTcCf5Kusma01s2Yza967d28Bl1d80eyptw+EVx8c01BLQ0TiqZwGwi8F/sHd5wEXAHeZ2dvq7+63uXuTuzfNnj276JUsRF86k/c5jew0XLU0RCSuCgkae4D5OZ/nhbS8ecwsSdR91DbMsUOltwHTQxmDz3UFsB7A3R8FaoBZBdQ/doZ6TiM7e6pXLQ0RialCgsZmYHGY1VRFNAi9cVCejcDlYXs18KC7e0hfE2ZXNQKLgSeGKjMc81Aog1DmvWH7ZeAcADM7nihoxLP/aQRDPhEeuqz6NBAuIjGVHCmDu6fN7CrgPiAB3OHu28zsWqDZ3TcCtxN1F7UA7URBgJBvPbAdSANXunsGIF+Z4ZRfBdaZ2beBp0LZAH8G/J2Z/TeiQfH/HIJMWckMOOkBH/bhPj2nISJxNWLQAHD3TUSDz7lpX8/Z7gU+NcSx1wHXFVJmSH+RaHbV4PTtwIcKqW+cZd8PnndMo1JTbkUk3sppIPyIkA0a+Ve51UC4iMSbgkaRZafT5l/lVlNuRSTeFDSKrG+Yloam3IpI3CloFNnBoFH59oHwZKKCZIVpTENEYktBo8gOdk/lWUYEoi4qtTREJK4UNIrs4EB4ngULQW/vE5F4U9AosoPdU0O0NKr1nnARiTEFjSIbqaVRXVmhZUREJLYUNIos29KoSrx9IByipUS0jIiIxJWCRpEV0tLQMiIiElcKGkU24uypZEJTbkUkthQ0iqyQ2VMaCBeRuFLQKLJDYxrDPaehloaIxJOCRpGlhnkiHKKlRDSmISJxpaBRZNkxjXxrT4FaGiISbwoaRZZKD2AGyQrLu19BQ0TiTEGjyPrC+8HN8gcNdU+JSJwpaBRZX3pgyEFwiMY6+tIDlOGbbEXkXUBBo8j60gNDDoKD3hMuIvGmoFFkfenMsC2NmqTeEy4i8aWgUWSp9MCQD/bBoYf+1NIQkTgqKGiY2Qoz22FmLWZ2dZ791WZ2d9j/uJktzNl3TUjfYWbnj1SmmTWGMlpCmVU5+y42s+1mts3MfjjWiy6lkcY01NIQkTgbMWiYWQK4CVgJLAEuNbMlg7JdAXS4+yLgBuD6cOwSYA1wArACuNnMEiOUeT1wQyirI5SNmS0GrgE+5O4nAH865qsuodSIYxrZoKGWhojETyEtjWVAi7u/6O4pYB2walCeVcCdYXsDcI5Fc0pXAevcvc/dXwJaQnl5ywzHnB3KIJR5Udj+I+Amd+8AcPc3Rn+5pdeXzgz5AiY49NCfWhoiEkeFBI0GYHfO59aQljePu6eBTqB+mGOHSq8H9oUyBp/rWOBYM3vEzB4zsxX5Kmtma82s2cya9+7dW8DlFddIYxrZlobGNEQkjsppIDwJLAbOAi4F/s7Mpg/O5O63uXuTuzfNnj27yFUcWfbhvqFkp9yqpSEicVRI0NgDzM/5PC+k5c1jZklgGtA2zLFDpbcB00MZg8/VCmx09/7Q1bWTKIiUlVR6gKphg4YGwkUkvgoJGpuBxWFWUxXRwPbGQXk2ApeH7dXAgx490rwRWBNmVzUSfck/MVSZ4ZiHQhmEMu8N2/9G1MrAzGYRdVe9OMrrLbmopTH0QPjBMQ11T4lIDCVHyuDuaTO7CrgPSAB3uPs2M7sWaHb3jcDtwF1m1gK0EwUBQr71wHYgDVzp7hmAfGWGU34VWGdm3waeCmUT8p5nZtuBDPD/unvbO78FxZUaacptdkxDLQ0RiaERgwaAu28CNg1K+3rOdi/wqSGOvQ64rpAyQ/qLRLOrBqc78KXwU7b60pmCHu5TS0NE4qicBsKPCCO1NLJdV2ppiEgcKWgUWd+IU261jIiIxJeCRhFlBpz0gFOVGHogvCpRQaLC6OpLD5lHRKRUFDSK6ND7wYe+7WZGXU2S/b0KGiISPwoaRZR9P/hwYxpACBr9xaiSiMioKGgUUSEtDYCpNZW8pZaGiMSQgkYRZQe31dIQkXKloFFEfQdbGkMPhAPU1VRqTENEYklBo4h6UtGYRu0IQWOqgoaIxJSCRhG1dfUBMHNy5bD56mqSvNWj7ikRiR8FjSLq6E4BMHNy9bD5ptYkOZBKMzDgxaiWiEjBFDSKqO1ANmhUDZtvam0l7nAgpS4qEYkXBY0iau9KkawwptYMv05kXdivLioRiRsFjSLq6E4xY3IV0avQh1ZXE415aDBcROJGQaOI2g6kmDlp+K4piGZPgYKGiMSPgkYRtXelRhzPAHVPiUh8KWgUUXt3iplTCg8a+/sUNEQkXhQ0iqi9q7DuKY1piEhcKWgUSTozQGdPv7qnRKSsKWgUyb6eftyhvoDuqZrKBFXJCrU0RCR2FDSKpL0rerBvRgHdUxA9Fa7l0UUkbgoKGma2wsx2mFmLmV2dZ3+1md0d9j9uZgtz9l0T0neY2fkjlWlmjaGMllBm1aBz/T9m5mbWNJYLLpVs0KgvoHsKsosWqntKROJlxKBhZgngJmAlsAS41MyWDMp2BdDh7ouAG4Drw7FLgDXACcAK4GYzS4xQ5vXADaGsjlB2ti51wBeBx8d2uaVzsKVRYNCoU0tDRGKokJbGMqDF3V909xSwDlg1KM8q4M6wvQE4x6LHnlcB69y9z91fAlpCeXnLDMecHcoglHlRznm+RRRUekd5nSXXNsqWRp1aGiISQ4UEjQZgd87n1pCWN4+7p4FOoH6YY4dKrwf2hTIOO5eZnQrMd/f/f7jKmtlaM2s2s+a9e/cWcHnF0THKlsbU2qQGwkUkdspiINzMKoDvAX82Ul53v83dm9y9afbs2RNfuQK1d6Woq0lSOcKrXrPqqis15VZEYqeQb7A9wPycz/NCWt48ZpYEpgFtwxw7VHobMD2UkZteBywFHjazXcByYGM5DYa3d6UK7pqC7HvC1dIQkXgpJGhsBhaHWU1VRAPbGwfl2QhcHrZXAw+6u4f0NWF2VSOwGHhiqDLDMQ+FMghl3uvune4+y90XuvtC4DHgE+7ePMbrLrpC153KmlpbSU9/hv7MwATWSkRkdEYMGmF84SrgPuBZYL27bzOza83sEyHb7UC9mbUAXwKuDsduA9YD24H/DVzp7pmhygxlfRX4UiirPpRd9tpGGTQOrj+l1oaIxMjwbwMK3H0TsGlQ2tdztnuBTw1x7HXAdYWUGdJfJJpdNVx9ziqk3nHS0ZXixIapBec/tP5UYUuPiIgUQ1kMhJc7d6e9K1XwzClQS0NE4klBowgO9KVJZQZGNRCefRHTW3pWQ0RiREGjCDq6oi/+mZOrCz7m0Eq3ammISHwoaBRBW1cfADMnVxZ8zNScMQ0RkbhQ0CiCju7oafDRtDSm1mpMQ0TiR0GjCNoOhKBR4LLoAFOqQ/eUWhoiEiMKGkWQXeG2kPeDZyUTFUyqSqilISKxoqBRBO3dKaqSFUyuSozqOL1TQ0TiRkGjCNoPpJg5qYpo5ffC1dUkNXtKRGJFQaMIRrvuVFZdTZL9fWppiEh8KGgUQXt3ivpRjGdkTautZF+3goaIxIeCRhG0d6WYMYqZU1lzptbw+lt9E1AjEZGxUdAogrF2Tx09rZY3D/TRl85MQK1EREZPQWOCpdID7O9Njy1oTK8B4PVOtTZEJB4UNCbYoafBx9LSiILGK50941onEZGxUtCYYNkH+0azwm3W0dNqAXhVQUNEYkJBY4Jlg8Zo3qWRNTd0T72yr3dc6yQiMlYKGhOs7R20NCZVJZlWW8lrnQoaIhIPChoTrKNr7GMaEI1rqHtKROJCQWOCtXWlMIPpY3hOA2Du9Fp1T4lIbChoTLCOrhTTaytJVIxu3amso9TSEJEYKShomNkKM9thZi1mdnWe/dVmdnfY/7iZLczZd01I32Fm549Uppk1hjJaQplVIf1LZrbdzLaY2c/M7Jh3cuHF0t6VGtMgeNbcaTV0dPfTk9IDfiJSeiMGDTNLADcBK4ElwKVmtmRQtiuADndfBNwAXB+OXQKsAU4AVgA3m1lihDKvB24IZXWEsgGeAprc/SRgA/DdsV1ycbV19Y1pEDxL025FJE6SBeRZBrS4+4sAZrYOWAVsz8mzCvhG2N4A/C+L1gFfBaxz9z7gJTNrCeWRr0wzexY4G/iDkOfOUO4P3P2hnPM9Blw2iussmY6ufhbOmjTm47NPhb/W2ct7Zk8ZWyGpbnjlKXh9K3S3QaYfaqfDtPlw9PthRiNUqKdSREZWSNBoAHbnfG4FTh8qj7unzawTqA/pjw06tiFs5yuzHtjn7uk8+XNdAfw0X2XNbC2wFmDBggXDXVdRtHWlOPWY6WM+fm5oabwylmm3rz4Dj9wIOzZBf/eh9IokDOS8p6OqDuY1wYIzYMHyaLtq8tDlprrhzZ3w5vPQ1wnuMP0YmHsyTPmd0ddTRMpGIUEjVszsMqAJ+Ei+/e5+G3AbQFNTkxexam8zMOB0dI9tscKso8JSIq/uG0X31FuvwM++Bc/8CGqmwkmXwHEro1bF5N8BM0gdgLYX4LUtUStk9xPw8F8DDpaI8jacCpNnR59790FbC+x9Djp+G+UbzCqg8SPw4S/DwjPHfM0iEl+FBI09wPycz/NCWr48rWaWBKYBbSMcmy+9DZhuZsnQ2jjsXGZ2LvAXwEdCl1es7e9Nkxnww5dFH8jAU3fBM+ug/UWomgJzT4FF58DxH4fqusPKqKlMMHNyVWEtjVRX1LL41Y1RS+KDfwK/+2dRV9Rg1XVRy2DuyXDqZ6K0nn3QuhlefhRefgy2rIe+t6J9yVqY2RjV9f2Xwuz3wezjoHYm4NCxC1oegKd/CP9wYRSoLvweVI+xS01EYqmQoLEZWGxmjURf4Gs4NOaQtRG4HHgUWA086O5uZhuBH5rZ94C5wGLgCcDylRmOeSiUsS6UeS+AmZ0C3AqscPc33sE1F01bVxTXDr6AqacDfvQH8PKvYM6JcOz50N0Ov30Etm6Af/8SvO/C6Ev5PWdBIvrrGfEBv4FMFIQe/BbsfxWWXATnfiP6kh+N2umw+GPRT1YmvAQqUTn8sXVHRV1bZ34JfvE/4Zffg7074A/WQ92c0dVDRGJrxKARxiiuAu4DEsAd7r7NzK4Fmt19I3A7cFcY6G4nCgKEfOuJBs3TwJXungHIV2Y45VeBdWb2baIZU7eH9P8BTAH+Jbxr+2V3/8Q7vgMT6NAKt9XQ3wv/tDrqDvrkrdH/xLPvDHePuoe2rIOt90QBZPLvwImfgvddwPypCXble8Av1Q3P/iT6kn5zBzR8AD71D9GX93gZKVgMVjUJzvlaNC6y4bNw58fhivvzt3ZEpOyYe0m7/SdUU1OTNzc3l+z89297jbV3PclPrjqTE3/z1/D4D+Diu2DJMLEu3QfP3x+1HHbeBwP9pCpqeHrgvSxrOh2qp0ImFf0vfvfj0djE7PfBWVfD8aviNQvqpV/AXZ+EhR+CT28YfQASkZIwsyfdvSnfvrIbCC8n2RVu53Q/B0/cCqf90fABAyBZHY1tHP9x6O2EXYQj2KkAAA19SURBVL9kxy/upWr3ZnzrPVh/dzT7qX5R1BI5cTUs+GC8gkVW4+/Cx78P934eHvrvcO5flbpGIvIOKWhMoPbuFMYAsx6+GibNgrP/cnQF1EyD913IK5kP8Md3Pck9n/sgpy6YMTGVnSinfDoaw3nkb6Lxmnl5//MiImUihv89PXK0H0jxiconqXjl13Det8bcr7+0YRoA2/Z0jmf1iuf8/w51c+Ff/0vU/SYiZUtBYwK1H+hjbfLfYeZ7oq6kMZo7rYYZkyrZuuetcaxdEdVMg098H9qeh8dvKXVtROQdUNCYQLM6nuQEfx7OuBIqEmMux8xY2jCNra+UaUsDYNG5cOwK+Pn/gANlMWNaRPJQ0JhA57SvY3/FNDj50++4rKUN09j5+n760mW82u1510G6Bx78dqlrIiJjpKAxUd54jtP7N7P5d1ZDZe07Lm7p3Gn0Z5ydrx0Yh8qVyKxFcNrnoifi9+4sdW1EZAwUNCZI6pc30uNV7H7v4Ifnx+bEMBhe1l1UAL/7ZaicFD29LiJlR0FjIux/jeTWf+FfMh9h1px8i/SO3vyZtdTVJNlarjOosqbMjtbEenYjtD5Z6tqIyCgpaEyEx2/BPM3fZy6gYcY775qCMBg+d1r5Bw2IJgZMmgUP/FW0hIqIlA0FjfHWtx8238FvZ5/Nyz6HeeMUNABOnDeNZ1/bT39mYNzKLInqOvjIV2DXL+CFn5W6NiIyCgoa4+3X/wh9nTw4cw01lRXv6FWvg50wdyqp9AAtb5TxYHjWB/4zTF8AD3wTBso8CIq8iyhojKdMPzz2AzjmQzzR/x7mzZiEZVeyHQfZwfBfv9wxbmWWTLIaPvqX0aq/2+4pdW1EpEAKGuPpmXXQuRs++AVa93XTMH38uqYAGmdN5pj6Sfzvra+Na7klc+KnYM7S6LmNdKrUtRGRAihojJd0Cv7Pd6M32x17Pns6esZ1PAOiwfCVS4/mVy+00dF1BHzJVlTAOV+Hjpfg13eWujYiUgAFjfHy9D/Bvpfho3/BgVSGju5+5s2YNO6nufDEo8kMOP+x/XUAnnipnRv+Yydl+16UxedFS7v//LvR62pFJNYUNMZDbyc8fD3MOw0WncuejujVrOPd0gBY2jCVeTNq2bT1VTq6Unz+n5/k+z97ns27ynScwww+9k3oegN+eUOpayMiI1DQGA8PfCP60lt5PZjR2tENMG7PaOQyMy488WgeaXmTr/54C/u6+6mrTnLHL18a93MVzfxlcNIa+MX3YPfmUtdGRIahoPFO7XoEmu+A0/9r9I5uYM++iWtpAKw88Wj6M87921/n82e9l8vOOIb7t7/G7vYoWKXSA+XXXXXBd2FqA9zzR1HLTURiSUHjnejcAxs+CzMa4aN/fjC5taOH6mQFs6dUT8hp3z9vGgtmTuLYOVO48uxFfOaMYzAz/vHRXWz6zaucdt0D/Le7ny6vwFEzDX7/NuhshR9eAqnuUtdIRPJQ0Birjt/CnR+PBm/X/BCqpxzc1drRTcOM2nF9RiOXmfGjtcv50R8tpzqZ4OhptaxcehT/3yO7+Pw//5rJVQn+7elX+NsHWybk/BPmmDOiwPHyY/CjNdD1ZqlrJCKDFBQ0zGyFme0wsxYzuzrP/mozuzvsf9zMFubsuyak7zCz80cq08waQxktocyqkc5RVJl+eOqf4bazoPtNuOzHMGfJYVlaO3rG/RmNwRqm11Kf05L54w+/l9rKBFd+9L38/Csf5fdPbeB7/7GT23/5Elv3dLKv+9AUXXeneVc7T7zUzsBA1Bp56uUOrv3Jdn7x/N7StlCW/j5c9AN4+VG45UzY9q8wUMbvEBE5wthIXxBmlgB2Ah8DWoHNwKXuvj0nz+eBk9z9v5jZGuCT7n6JmS0BfgQsA+YCDwDHhsPylmlm64F73H2dmd0CPOPuPxjqHMPVvampyZubm0d3R7IGMtDfHXWTHHgd2l+E3Y/D9nvhrT3R+MUnb4veEQF09vSzd38fU2uTrPibX3D+CUfx179/4tjOPUbufrB105fOcNnfP37YrKoTG6axrHEmv3h+Lztfj5YimT+zlobptTz2YvvBfKc3zuSD751FdypNX3oAM6hOJjixYRqnLZzB7LpqzIydr+/nnl/vYUvrPo6dU8eJDdOYN6OWWXXVdPdleKmti11vdvHSm1283N5NR3eKt3rSnDB3Kp8+fQEnNExjy+59vNzezdKGaZyyYDqTqpIMDDj9e54h+W9rSbTtoGfKAnbP/gg2fxkNC49l0qxjGKicRMqq6MskONCfobW9mz37ephdV83SudOYMY7Lt8TFgb40u9u72dfdD0Blwjh6ei1HTa0hUTExrdpS6e3PsHlXO4++0MacqTV85NjZLJw1GYh+t1/c28WLe7uoqaxgdl01s6ZUUz+liurk2N+QWY4yA05fOkPbgRTPvbafPR3dHDunjvfPn87k6uSYyzWzJ929Ke++AoLGGcA33P388PkaAHf/65w894U8j5pZEngNmA1cnZs3my8c9rYyge8Ae4Gj3D2de+6hzuHDXMCYg8bWe2DDH749PVkDx3wITv9jWPSx6OG04N6n9/DFdU8f/PyVFcfx+bMWjf7c46g/M8D2V97i1c5eWt7Yz8937uXJ33Zw/NFTufyMhVQmjQ1PtrLrzW4uW34Ml5w2n5888wo3PdTCG/v7qE5WUJWsAIe+9ACpsFCiGVQmKkilB0hUGMfNqWNXWxfdqfwtgrnTalhQP4n6KdVMqkzwf57fy+tv9b0tX4VBosLoz0R/pRUMcF5FM3+Q+BnLKp6jxvrfdszLA7P5cOr7b0ufVJU4uICuE20M96ueryfRsALyFIfDkPc3WWHR39NwxxfYeMzeq3da1mjaqtl7aHbonvdnBkgPOIkKIxNaw7WVCQbc6c8MMDDECWorE3n/nsbDRDbAHc/5feXgDRycnv268xHqU2Fw1UcX8aXzjhtTfd5p0FgNrHD3z4XP/wk43d2vysmzNeRpDZ9fAE4nChCPufs/hfTbgZ+Gw95WZk7+RSF9PvBTd1861Dnc/bCObzNbC6wNH48Ddgx/eybELEAd8roPuXQvDtG9iMT5Phzj7rPz7Rh7+yWm3P024LZS1sHMmoeK0u8mug+H6F4consRKdf7UMhA+B5gfs7neSEtb57QdTQNaBvm2KHS24DpoYzB5xrqHCIiUiSFBI3NwOIwq6kKWANsHJRnI3B52F4NPBjGGjYCa8LMp0ZgMfDEUGWGYx4KZRDKvHeEc4iISJGM2D0VBqSvAu4DEsAd7r7NzK4Fmt19I3A7cJeZtQDtREGAkG89sB1IA1e6ewYgX5nhlF8F1pnZt4GnQtkMdY6YKmn3WIzoPhyie3GI7kWkLO/DiAPhIiIiWXoiXERECqagISIiBVPQGEcjLbdSrsxsl5n9xsyeNrPmkDbTzP7DzJ4Pf84I6WZmN4Z7sMXMTs0p5/KQ/3kzuzwn/QOh/JZwbGwebzazO8zsjfCcUDZtwq99qHOU0hD34htmtif8bjxtZhfk7DuylhA6VM/5ZvaQmW03s21m9sWQ/u74vXB3/YzDD9GA/gvAe4Aq4BlgSanrNU7XtguYNSjtu8DVYftq4PqwfQHRA5wGLAceD+kzgRfDnzPC9oyw74mQ18KxK0t9zTnX+WHgVGBrMa99qHPE8F58A/hynrxLwr+BaqAx/NtIDPfvBFgPrAnbtwD/NWx/HrglbK8B7i7xfTgaODVs1xEtibTk3fJ7UfJ/lEfKD3AGcF/O52uAa0pdr3G6tl28PWjsAI4O20cDO8L2rUTriB2WD7gUuDUn/daQdjTwXE76Yfni8AMsHPRFOeHXPtQ5Sv2T5158g/xB47Dff6KZkmcM9e8kfDm+CSRD+sF82WPDdjLks1Lfi5xruJdoHb13xe+FuqfGTwOwO+dza0g7Ejhwv5k9adEyLQBz3P3VsP0aMCdsD3UfhktvzZMeZ8W49qHOEUdXhW6XO3K6S0Z7L+qBfe6eHpR+WFlhf2fIX3Khq+wU4HHeJb8XChpSiDPd/VRgJXClmX04d6dH/+15V87dLsa1x/z+/gB4L3Ay8CrwP0tbneIxsynAj4E/dfe3cvcdyb8XChrjp5DlVsqSu+8Jf74B/CvRUvevm9nRAOHPN0L20S4dsydsD06Ps2Jc+1DniBV3f93dM+4+APwd0e8GHOFLCJlZJVHA+Gd3vyckvyt+LxQ0xk8hy62UHTObbGZ12W3gPGArhy/rMni5l8+EGSPLgc7QnL4POM/MZoQujPOI+qxfBd4ys+VhhshncsqKq2Jc+1DniJXsF1jwSaLfDTiClxAKf1e3A8+6+/dydr07fi9KPYh0JP0QzZLYSTQ75C9KXZ9xuqb3EM1weQbYlr0uoj7lnwHPE71ca2ZIN+CmcA9+AzTllPVZoCX8/GFOehPRl80LwP8iXoOcPyLqdukn6lu+ohjXPtQ5Yngv7grXuoXoC+3onPx/Ea5rBzkz4ob6dxJ+154I9+hfgOqQXhM+t4T97ynxfTiTqFtoC/B0+Lng3fJ7oWVERESkYOqeEhGRgiloiIhIwRQ0RESkYAoaIiJSMAUNEREpmIKGiIgUTEFDREQK9n8B8i9dzpf1VgQAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "sns.kdeplot(train_data['LotArea'], ax=ax, label='train')\n", + "sns.kdeplot(live_data['LotArea'], ax=ax, label='live')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see from this plot, the variable distribution is dramatically different!\n", + "\n", + "This should send all sort of alerts, and we should investigate further the reason of this data shift, as the performance of our model could be impaired." + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3hc1Zn48e87o16sLhfJtmRb7jYusjGdYIqBBNMxCQlsKOnZJPtLYrIhm7DJLiS7KSQQSnBCWIJxgASHDgk9YFs27lVYstV776M5vz/ulZHkkTS2Jd2Z0ft5Hj9z59xzz7znYdA79557zhVjDEoppVQPl9MBKKWUCiyaGJRSSvWhiUEppVQfmhiUUkr1oYlBKaVUH2FOBzAcUlNTTVZWltNhKKVUUNm6dWu1MSatf3lIJIasrCzy8vKcDkMppYKKiBzxVa6XkpRSSvWhiUEppVQfmhiUUkr1ERJjDEopdaK6urooLi6mvb3d6VBGXFRUFJmZmYSHh/tVXxODUmpMKi4uJj4+nqysLETE6XBGjDGGmpoaiouLyc7O9usYvZSklBqT2tvbSUlJCemkACAipKSknNCZkSYGpdSYFepJoceJ9lMTg/KfLtGu1JjgV2IQkVUickBE8kVkrY/9kSLylL1/k4hk9dp3p11+QEQu6VW+TkQqRWR3v7aSReQ1ETlkvyadfPfUsNnxFNwzFX57NhTrZEKlTlV9fT0PPPDACR932WWXUV9fPwIRfWzIxCAibuB+4FJgLnCjiMztV+1WoM4YMwP4BXCvfexcYA0wD1gFPGC3B/AHu6y/tcDfjTE5wN/t98pJhe/Bc1+GlOnQVgvP3g5dbU5HpVRQGygxeDyeQY978cUXSUxMHKmwAP/OGJYD+caYw8aYTmA9sLpfndXAY/b208BKsS5qrQbWG2M6jDEFQL7dHsaYt4FaH5/Xu63HgCtPoD9qJLzxXxA/ET73V7jyt1B7GN75udNRKRXU1q5dy0cffcSiRYtYtmwZ55xzDldccQVz51q/u6+88kqWLl3KvHnzePjhh48dl5WVRXV1NYWFhcyZM4fbb7+defPmcfHFF9PWNjw/2Py5XTUDKOr1vhg4faA6xhiPiDQAKXb5B/2OzRji88YbY8rs7XJgvK9KInIHcAfAlClThu6FOjkVe+HIu3DR3RCVANPOg1mXwbbH4LzvglvveFbB70d/28Pe0sZhbXPupHH8x6fmDbj/nnvuYffu3Wzfvp0333yTyy+/nN27dx+7pXTdunUkJyfT1tbGsmXLuOaaa0hJSenTxqFDh3jyySd55JFHuP7663nmmWe46aabTjn2gB58NtYDqX2OeBpjHjbG5BpjctPSjlscUA2XvEfBHQmLen3ZFn0amivg8JuOhaVUqFm+fHmfeQb33Xcfp512GitWrKCoqIhDhw4dd0x2djaLFi0CYOnSpRQWFg5LLP783CsBJvd6n2mX+apTLCJhQAJQ4+ex/VWIyERjTJmITAQq/YhRjQSvF/Y+B7Mvh9hev1RyLoboJNjxJ8i50Ln4lBomg/2yHy2xsbHHtt98801ef/113n//fWJiYjj//PN9zkOIjIw8tu12u4ftUpI/ZwxbgBwRyRaRCKzB5I396mwEbra3rwX+Yf/a3wisse9aygZygM1DfF7vtm4GnvMjRjUSSj+EliqYdWnf8rBImLsaDr4C3V3OxKZUkIuPj6epqcnnvoaGBpKSkoiJiWH//v188MEHPuuNlCETgzHGA3wVeAXYB2wwxuwRkbtF5Aq72qNAiojkA9/CvpPIGLMH2ADsBV4GvmKM6QYQkSeB94FZIlIsIrfabd0DXCQih4AL7ffKCQdfBnHBDB9nBdNXQmez3rqq1ElKSUnhrLPOYv78+Xz729/us2/VqlV4PB7mzJnD2rVrWbFixajGJiYEJi3l5uYafVDPCHjoXAiPgc+/fPy+tjq4NxvO+w584nujH5tSp2jfvn3MmTPH6TBGja/+ishWY0xu/7oBPfisHNRSDWU7fJ8tgDXGMGmxDkArFYI0MSjfCt+xXrPPG7jO9E9Yl5Lah/c2P6WUszQxKN8K34WIOJi0aOA6U88E0w2l20YvLqXUiNPEoHwreAemrAD3IA/2yFhqveoAtFIhRRODOl5TBVQfgKxzBq8XnQQpMzQxKBViNDGo4x1513odKjEAZC6DkjxdklupEKKJQR2v4B2IiIeJpw1dN2OpNQmu/ujIx6VUiImLiwOgtLSUa6+91uFoPqaJQR2v8B2YeoZ/C+Rl2rdAl2wd2ZiUCmGTJk3i6aefdjqMYzQxqL4ay6Am37/LSADpc8EVBhW7h66rlPKpsLCQ+fPnA7BixQr27NlzbN/5559PXl4eLS0tfP7zn2f58uUsXryY554budWCdM1k1VehPb6Q7WdiCIuE1JlQrolBBbGX1kL5ruFtc8ICuPTEV/S54YYb2LBhAz/60Y8oKyujrKyM3Nxcvve973HBBRewbt066uvrWb58ORdeeGGfxfeGi54xqL4K34bIBJiw0P9jxs/XMwalhsn1119/7LLShg0bjo09vPrqq9xzzz0sWrTo2GqrR4+OzNienjGovgrftSauudxD1+0xYT7s2gCttRCTPHKxKTVSTuKX/UjJyMggJSWFnTt38tRTT/Hggw8CYIzhmWeeYdasWSMeg54xqI81lFiP7fT3MlKP8da1UT1rUGp43HDDDfz0pz+loaGBhQuts/dLLrmEX//61/QsfPrhhx+O2OdrYlAfKzyB+Qu9TVhgvVbsGbyeUsov1157LevXr+f6668/VnbXXXfR1dXFwoULmTdvHnfdddeIfb5eSlIfK3gbohI/PgPwV1w6xKRqYlDqBDU3NwOQlZXF7t0fn3GPHz8ej8fTp250dDQPPfTQqMSlZwzK4vXCoVdgxkpwncTXIm0WVB//TFqlVPDRxKAsJVvtx3hednLHp8601lfSpTGUCnqaGJTl4EsgbuuM4WSkzrSe6tZSPbxxKTWCQuEJlv440X5qYlDWr/z9L1i3qUYnnVwbaTOt1+qDwxeXUiMoKiqKmpqakE8OxhhqamqIiory+xgdfFZQ+iFU7Yfld5x8G6n2vdXVByDrrOGJS6kRlJmZSXFxMVVVVU6HMuKioqLIzMz0u74mBgXbn4CwKJh/zcm3MS4DwmOgSs8YVHAIDw8nOzvb6TACkl5KGuu62mDX0zD7kxCdePLtuFyQmmOdMSilgpomhrFu2+PQXg+5/3LqbaXMsGZOK6WCmiaGsczTAe/9EqacAVOHYVwgeZr1wB5P56m3pZRyjCaGseyD30JjCZz7bRA59faSp4HxQkPRqbellHKMJoaxqnI/vPETmPMpmH7B8LSZPM161ctJSgU1TQxjUbcH/vpFiIyHy38xPGcLoIlBqRCht6uORe/90pq7cN0fIC5t+NqNTYOIOE0MSgU5PWMYa8p3w5v3wLyrYd5Vw9u2CCRna2JQKshpYhhLuj3w1y9Z8xUu+5+R+YzkaZoYlApymhjGkrx1UL4TLvsZxKaMzGckZUPdEfB2j0z7SqkRp4lhrGipgTd+DNPOh7lXjtznJE4Bbxc0lY/cZyilRpRfiUFEVonIARHJF5G1PvZHishT9v5NIpLVa9+ddvkBEblkqDZFZKWIbBOR7SLyrojMOLUuKgA+eADaG2HVPcN3F5IviVOsV53LoFTQGjIxiIgbuB+4FJgL3Cgic/tVuxWoM8bMAH4B3GsfOxdYA8wDVgEPiIh7iDZ/C3zGGLMI+BPw/VProqK9ETY/Ys1ZSJ8zsp+VMNl6rdfEoFSw8ueMYTmQb4w5bIzpBNYDq/vVWQ08Zm8/DawUEbHL1xtjOowxBUC+3d5gbRpgnL2dAJSeXNfUMR8+Dh0NcM63TrqJbq+hpcNDQ2vX4OvXJ9qJoeHoSX+WUspZ/sxjyAB6//wrBk4fqI4xxiMiDUCKXf5Bv2Mz7O2B2rwNeFFE2oBGYIWvoETkDuAOgClTpvjRjTHKGGuhvMxlMGnxCR9+qKKJX7x+kJd2lx97aueiyYn8v4tncdaMFKT/ZamIWIhO1jMGpYJYIE5w+yZwmTFmk4h8G/g5VrLowxjzMPAwQG5ubmg/gulUlG6Dqn3wyV+e0GGdHi93P7+HJzYdJSbczS1nZjFhXBQer+GJD45w06ObuHBOOvd/ZgmRYe6+BydO1jEGpYKYP4mhBJjc632mXearTrGIhGFdAqoZ4tjjykUkDTjNGLPJLn8KeNmPGNVAtv/JfgjP1X4fUt/ayRce38qmglpuOTOLr6/MITk24tj+287J5vfvFXLPS/v59p938ssbFuFy9TpzSJgM1YeGsxdKqVHkzxjDFiBHRLJFJAJrMHljvzobgZvt7WuBfxjrQvRGYI1911I2kANsHqTNOiBBROwHCHMRsO/kuzfGeb2w72+QczFEJfh1SHFdK1c/8E8+PFrPr9Ys4odXzOuTFAAiw9x88bzpfGfVLDbuKOWnr/R7OE/iVOuMIcSfpatUqBryjMEeM/gq8ArgBtYZY/aIyN1AnjFmI/Ao8LiI5AO1WH/osettAPYCHuArxphuAF9t2uW3A8+IiBcrUXx+WHs8lpRug+YK6+lsfiiua2XNwx/Q2NbFE7efzrKs5EHrf+m86ZTWt/HgWx+xLCuJlXPGWzsSJ0NXK7TWjtxEOqXUiPFrjMEY8yLwYr+yH/TabgeuG+DYnwA/8adNu/wvwF/8iUsNYf/zIG6YefGQVXsnhf+77XQWZg79mE8R4T8+NY93D1Xzs1cO8IlZ6dYlpYRedyZpYlAq6OjM51B24CXIOhuikwatdjJJoUe428U3L5rJ/vImXtxdZhUm6lwGpYKZJoZQ1VgKVfsh56JBqxXVnnxS6PHJhZOYOT6On792EE+3t9cZgyYGpYKRJoZQdfgt63Xa+QNWKapt5cZH7DGF21acVFIAcLuEb100k8NVLfx1e6l1hhIRp2cMSgUpTQyhquAtiEmB9Hk+d/dPCgsy/btraSCXzJvA7Anx/PH9QmstpgSdy6BUsNLEEIqMgcNvQva54Dr+P3Fx3fAmBbAGoq9anMHO4gaKalutcYZ6XRZDqWCkiSEU1R6GpjLIPu+4XTXNHXzu0c00DGNS6HHZgokAvLCrTM8YlApimhhCUdFm63VK32Wmmjs83PL7LZQ2tPH7W5YNa1IAmJwcw2mTE3lhZ5l1xtBWBx3Nw/oZSqmRp4khFBVvhshxkDrrWFG31/Cl/9vK3rJGHvjMEnKHmLx2sj65YCK7ShqocqdbBXrWoFTQ0cQQioq3QMbSPuML9/39EO8cqubHV87ngtnjR+yjL10wAYC3K6OtAh1nUCroaGIINR3NULEHJi8/VvTOoSru+8chrlmSyZplkwc5+NRlJsWweEoify2wv1qaGJQKOpoYQk3pNjBe6/kLQFVTB99Yv52c9Dj+88p5xz8/YQRcNn8i75a7Ma4wa6KdUiqoaGIINcVbrNeMpQDc89J+Gtu7uP/TS4iJGJ3Hb5w5IwWDi7bIdE0MSgUhTQyhpmgLpORATDJbj9TyzLZibjtnGjnj40cthNkTxhEfFUaFpEJj/0d3KKUCnSaGUGKMdUfS5OV0ew0/eG4PExOi+OonZoxqGG6XsDwrmYLOBE0MSgUhTQyhpK4AWmsgcxnrtxxlT2kj/375HGIjR/8JrsuzkznUPg7TWKoP7FEqyGhiCCVF1viCZ9JSHnjjI5ZOTeJyezbyaFuenUyZSUE87dYDe5RSQUMTQygpyYOIOF6sTKKkvo0vnjd9VO5C8mV+RgK17lTrjV5OUiqoaGIIJWU7MRMW8PA7hUxLi2Xl7HTHQgl3u0iakGW90cSgVFDRxBAqvF6o2E159Ax2lzRy+znTrMdsOmhq9kwAWqt1kptSwUQTQ6ioK4DOZl6sTic1LpKrFmc4HRHzZ86gy7gpL/rI6VCUUidAE0OoKN8JwLOlydy0YgpR4W6HA4LTpiZTSRItesagVFDRxBAqynbilTAOmUxWL3L+bAEgMsxNQ3g6riad/axUMNHEECrKd3HEPZmcSSlkp8Y6Hc0xXbETieuowOhcBqWChiaGENFdtpNtHZlcvtCZeQsDiUiezHhTw9GaFqdDUUr5SRNDKGiuxN1SwV7vVD65YJLT0fSROGEqUdLF/oIjToeilPKTJoZQYA88t6XMY0pKjMPB9JU6aRoAxYX5DkeilPKXJoYQUP/RVgByFp7hcCTHC0+yHgxUW1bgcCRKKX+N/upqatjVHt5Ks0ll5eJZQ1cebeOsS1vttUUYYxxbokMp5T89YwgBMbV7OOyeHnCXkQCIS8crYSR5qjhS0+p0NEopP2hiCHKms4X0rhLakuc4HYpvLjee2PFMlBp2ljQ4HY1Syg+aGIJc8aEduDDET1nodCgDCkvMYJLUsVsTg1JBQRNDkCs6sA2AqbOXOBzJwFwJGUwJq2NXsSYGpYKBX4lBRFaJyAERyReRtT72R4rIU/b+TSKS1WvfnXb5ARG5ZKg2xfITETkoIvtE5Oun1sXQ1lK8Bw9uJk2b53QoAxuXQbqpZndpvc6AVioIDJkYRMQN3A9cCswFbhSRuf2q3QrUGWNmAL8A7rWPnQusAeYBq4AHRMQ9RJu3AJOB2caYOcD6U+phCDPGEFl3iKqIyUhYhNPhDGxcBuGmE3d7PcV1bU5Ho5Qagj9nDMuBfGPMYWNMJ9Yf6tX96qwGHrO3nwZWinVf4mpgvTGmwxhTAOTb7Q3W5peAu40xXgBjTOXJdy+0FdW2Mbn7KJ1JOU6HMrgEa1G/iVLLntJGh4NRSg3Fn8SQART1el9sl/msY4zxAA1AyiDHDtbmdOAGEckTkZdExOdfPRG5w66TV1VV5Uc3Qs+W/BKmSCVxmQF8GQlgnPWfNsNVw95SHWdQKtAF4uBzJNBujMkFHgHW+apkjHnYGJNrjMlNS0sb1QADxdEDO3CLISnrNKdDGZw9yW1BfAt7y/SMQalA509iKMG65t8j0y7zWUdEwoAEoGaQYwdrsxh41t7+CxC492E6rK1sLwCu9NkORzKEuPEgbubENumlJKWCgD+JYQuQIyLZIhKBNZi8sV+djcDN9va1wD+MdfvJRmCNfddSNpADbB6izb8Cn7C3zwMOnlzXQlt7VzcJzR/hxQ0p050OZ3AuN8RPJCu8nrKGdmpbOp2OSCk1iCHXSjLGeETkq8ArgBtYZ4zZIyJ3A3nGmI3Ao8DjIpIP1GL9oceutwHYC3iArxhjugF8tWl/5D3AEyLyTaAZuG34uhs68iubmU4JrXFTiAuLdDqcoY2bRHpXDQB7Sxs5OyfV4YCUUgPxaxE9Y8yLwIv9yn7Qa7sduG6AY38C/MSfNu3yeuByf+Iay/aVNbJEipH0xU6H4p+EDOJLdwCwt6xBE4NSASwQB5+VHw6V1pAl5URn9J9SEqDGZeBuKmPiuEgdZ1AqwGliCFKNJftwi8GVHqCL5/U3bhJ42lg+wbqUpJQKXJoYgpAxBioPWG/SAvAZDL7YcxmWJrXxUVUzbZ3dDgeklBqIJoYgVNXcwYSuIxgEUgJ81nMPOzHMi2vBa2B/uZ41KBWoNDEEof1lTUyVCjpjJ0J4lNPh+MdeFiM7og6A3Xo5SamApYkhCO0vbyRLKnClzHA6FP/Zk9ySPNUkxoSzW5fgVipgaWIIQvvLmsh2VRCeFuAT23pzuSF+AtJYwoKMBHbpQ3uUCliaGIJQUWkZiTRB8jSnQzkx4zKgsYT5GQkcrGiivUsHoJUKRJoYgkxXtxdPTb71JugSwyRoLGVBRgIer+FAeZPTESmlfNDEEGQKqlvI9JZbb4IuMWRAQwkLJo0DYLcuwa1UQNLEEGQOV7UwVSqsN0lZjsZywhIywNNGZnQHCdHh7NZxBqUCkiaGIFNY00KWqwJv3ESIiHE6nBNjP5dBB6CVCmyaGIJMYXUL092VuFKD6FbVHvYkNxqsAegD5U10eHQAWqlAo4khyPScMZCc7XQoJy7BfjZTQxHzM8bR1W04WN7sbExKqeNoYggylVU1JHnrgm/gGaxJbu5IqD/CgowEAL2cpFQA0sQQRNo6u4lqPmK9CcbE4HJB4mSoO8KU5BjGRYVpYlAqAGliCCKFNS1kSZDeqtojcSrUH0FEOG1yIh8erXM6IqVUP5oYgsiRmhayjt2qGoRjDABJU6H+KACLpyRxsKKJ5g6Pw0EppXrTxBBECqpbmSoVeGPTITLO6XBOTuJUaKuD9kaWTk3Ca2BHUb3TUSmletHEEEQKq1uYEVYZXKuq9pc01XqtP8KiyYkAbDuil5OUCiSaGIJIQTDfqtojcYr1WneEhOhwctLj2KbjDEoFFE0MQaSkqp4Ub411OSZYJWZZr/Y4w5IpSXxYVG89rlQpFRA0MQSJ1k4PYS2l1pvEyc4GcypikiEiDuqt226XTE2kvrWLw9UtDgemlOqhiSFIFFa3kiHV1puEIE4MItYZT52VGJZOTQJgq44zKBUwNDEEicKaFjKlynrTc50+WCVNPXbGMC01jnFRYTqfQakAookhSBRUt5Ah1RhxHVulNGglTrHGGIzB5RIWT0li2xG9ZVWpQKGJIUgcrWllengtEj8J3OFOh3NqEqdCZzO01gLW5aSDlU00tHU5HJhSCjQxBI2iulaywmqDe+C5x7G5DIUALM9OxhjYXFDrXExKqWM0MQSJo7WtTDSVwT++AB/fbmsPQC+ekkhkmIv3P6pxMCilVA9NDEHA0+2lsqGFJE91cN+R1KMnudlzGSLD3CydmsT7hzUxKBUINDEEgbKGdlK9tbjoDo1LSVHjIDrp2J1JAGdMS2FfWSN1LZ0OBqaUAk0MQaGotvXjW1VD4YwB+sxlADhjegoAmwr0rEEpp/mVGERklYgcEJF8EVnrY3+kiDxl798kIlm99t1plx8QkUtOoM37RESf+4g18HxsclswL4fRW3I21B4+9nZhZiLR4W4dZ1AqAAyZGETEDdwPXArMBW4Ukbn9qt0K1BljZgC/AO61j50LrAHmAauAB0TEPVSbIpILJJ1i30JGUW0bk109s54znQ1muKTkWJeSPNalo4gwF7lZOs6gVCDw54xhOZBvjDlsjOkE1gOr+9VZDTxmbz8NrBQRscvXG2M6jDEFQL7d3oBt2knjZ8B3Tq1roeNobSszIusgNh3Co5wOZ3ikzADjhbrCY0VnTE/hYEUz1c0dzsWllPIrMWQARb3eF9tlPusYYzxAA5AyyLGDtflVYKMxpmywoETkDhHJE5G8qqoqP7oRvIrqWsl214TGwHOPnmdK1Bw6VnTGNGuc4QM9a1DKUQE1+Cwik4DrgF8PVdcY87AxJtcYk5uWljbywTmoqLaNCVSFxhyGHinTrdea/GNFCzISiI8K452D1Q4FpZQC/xJDCdD7p2qmXeazjoiEAQlAzSDHDlS+GJgB5ItIIRAjIvmMYW2d3dQ0t5HUVRk6dyQBRCdCbFqfxBDmdnH2jFTeOlilz2dQykH+JIYtQI6IZItIBNZg8sZ+dTYCN9vb1wL/MNb/2RuBNfZdS9lADrB5oDaNMS8YYyYYY7KMMVlAqz2gPWYV17WSSiNhpjO0zhjAupxU3TfvnzczjfLGdg5W6A1pSjllyMRgjxl8FXgF2AdsMMbsEZG7ReQKu9qjQIr96/5bwFr72D3ABmAv8DLwFWNM90BtDm/XQsPRUJzD0CNlRp8zBoDzZlmXBd86WOlEREopIMyfSsaYF4EX+5X9oNd2O9bYgK9jfwL8xJ82fdSJ8ye+UFZU23sOQ4idMaTmwIePQ1udNRMamJgQzazx8bx1sIo7zp3ucIBKjU0BNfisjldU18bUMPsunVC6KwkgbY71Wrm/T/F5s9LYUlBHS4fHgaCUUpoYAlxRbSuzIushKhEi450OZ3il9ySGvX2Kz5uZRme3V2dBK+UQTQwBzjpjqA69swWwZnFHxENV3zOG3KwkosPdvHUwtOenKBWoNDEEMGMMRbWtTDBVobNGUm8ikD4bKvf1KY4Mc3Pm9BTeOFCpt60q5QBNDAGsvrWL5o4ukroqQu+OpB7pc467lASwcs54iuva9LZVpRygiSGAFdW1kkgzEd2toXkpCawB6NYaaO572WjlnHQAXt9X4URUSo1pmhgCWFFt28e3qobyGQNAxe4+xePHRXFaZoImBqUcoIkhgBXVtZIZqnMYekw8zXot33ncrgvnjGd7UT2VTe2jHJRSY5smhgBW1LPcNoRuYohJhoQpULr9uF0r54zHGHhjv86CVmo0aWIIYEdrW5kZWQfhscdmBoekiQuhbMdxxXMmxpORGM1rezUxKDWaNDEEsOK6Nqa6q62zBRGnwxk5ExdB7UfQ3tinWES4cE467+ZX0dbZ7VBwSo09mhgClNdrKKlrs+cwhOhlpB7Hxhl2Hbfrwrnjae/y8vYhneym1GjRxBCgKpra6ez2ktRZFvqJYdIi67X0w+N2rZiWQmJMOC/vLh/loJQauzQxBKii2jbG0UKkpyn0E0NcujUAXbz5uF3hbhcXzx3P63sr6PDo5SSlRoMmhgB1NJSX2/ZlyulwdBP4WALj0gUTaerw8O4hfeSnUqNBE0OAKqptZbLLvhtnLCSGyadDcznUHz1u11nTUxkXFcaLu/RyklKjQRNDgCqqa2VOVL31JinL0VhGxeTTrdeiTcftighzcdHcCby2t5xOj3eUA1Nq7NHEEKCKa9uYGVkLEXGhPYehx/h5Vl+PfuBz92ULJtDY7uG9j/RyklIjTRNDgCqqa2WKuyb05zD0cLlhygoofMfn7rNzUomPDOOFnWWjHJhSY48mhgDU4emmvLGd8d6KsTG+0GPa+VB9EBpKjtsVGeZm1fwJvLy7XCe7KTXCNDEEoJK6NoyBxI7ysZcYAAre8rn76iWZNHd4eHWvDkIrNZI0MQSgojprDkPEWJjD0Fv6PIhNg4/e8Ln79OxkMhKjeWbb8WcUSqnho4khAB2taSFT7CUgxlJicLkg+zw4/AZ4j7/7yOUSrlqcwbuHqqho1KW4lRopmhgC0OHqFqaF1VpvxlJiAJi5ClqqoCTP5+6rlmTgNfDcdj1rUGqkaGIIQAXVLSyIa7DeJIYgsykAABVSSURBVE51NpjRlnMRuMJg/ws+d09Pi2PR5ESe2VqC8TFLWil16jQxBKCC6paxNYeht+hEmHoWHHhxwCrXLs3kQEUT247Wj2JgSo0dmhgCTIen214Oo2rszGHob/bl1m2rVQd97r5qcQbxkWH84Z+FoxuXUmOEJoYAU1TbitdAqmeMzWHobc4VgMDup33ujo0M49rcTF7aVaaD0EqNAE0MAeZwVQsA8e1j4DkMAxk3EbLPgV1/9rnaKsDnzsjC4zU8sen4RfeUUqdGE0OAKahuIYFm3J2NYzcxACy4DmoPQ+k2n7uzU2P5xKw0/rTpqC6sp9Qw08QQYA5XtbAo1l4oLnm6s8E4ac4VEBYF2x4fsMrNZ2ZR3dzB33aUjmJgSoU+TQwBpqC6haVx9t02ydOcDcZJ0Ykw72rrclJHk88q5+akMXtCPPe/kY+nW88alBoufiUGEVklIgdEJF9E1vrYHykiT9n7N4lIVq99d9rlB0TkkqHaFJEn7PLdIrJORMJPrYvB5XB1C3MiqgAZG89hGEzuv0BnM+zyPQjtcgnfuHAmh6tbeG67njUoNVyGTAwi4gbuBy4F5gI3isjcftVuBeqMMTOAXwD32sfOBdYA84BVwAMi4h6izSeA2cACIBq47ZR6GEQa27uobu5gqqscEiZDeJTTITkrc5m1ftLW3w9Y5ZJ545k3aRz3/eMQXXrWoNSw8OeMYTmQb4w5bIzpBNYDq/vVWQ08Zm8/DawUEbHL1xtjOowxBUC+3d6AbRpjXjQ2YDOQeWpdDB4F9h1J47tKIDnb4WgCgIh11lC2A0p8D0KLCN+8cCZHalr5iy6up9Sw8CcxZABFvd4X22U+6xhjPEADkDLIsUO2aV9C+izwsq+gROQOEckTkbyqqio/uhH4CqqtxBDXchRSxvDAc28Lr4fwGMhbN2CVlXPSOS0zgV+8fpDWTs8oBqdUaArkwecHgLeNMT4f6WWMedgYk2uMyU1LSxvl0EbG4eoWkqQZd0f92L4jqbeoBOvW1V1/hmbfPwBEhO9/ci5lDe3c9/f8UQ5QqdDjT2IoASb3ep9pl/msIyJhQAJQM8ixg7YpIv8BpAHf8qcToeJwVTOnJ+gdScc54yvgaYctjwxYZVlWMtcuzeR37xwmv9L3XUxKKf/4kxi2ADkiki0iEViDyRv71dkI3GxvXwv8wx4j2Aisse9aygZysMYNBmxTRG4DLgFuNMaMqdHE/MpmcmPtX8WpM50NJpCkzYKZl8LmR6CzdcBqay+dTUyEm7v+ukdXXlXqFAyZGOwxg68CrwD7gA3GmD0icreIXGFXexRIEZF8rF/5a+1j9wAbgL1YYwVfMcZ0D9Sm3daDwHjgfRHZLiI/GKa+BrT2rm4OVTazMLIM3BF6q2p/Z30d2mph+xMDVkmNi+Tbq2bz/uEa1m8pGrCeUmpwEgq/rHJzc01enu8HuwSLncX1XPGb99ic9RDppga+9J7TIQUWY+B3F0JrNXxtG7jcPqt5vYbPrtvEtiP1PP/1s5meFjfKgSoVPERkqzEmt395IA8+jyl7ShsBSG49bF06UX2JwFn/CnWFsPuZAau5XML/XreIyHAX31i/XddRUuokaGIIEHtKG0iP9BDWWARps50OJzDN/iSMnw9v/Bd0dw1YbUJCFPdes5BdJQ3c+/L+UQxQqdCgiSFA7Clt5BNp9uM8NTH45nLBBd+HuoJBxxoALpk3gVvOzOLRdwv4c56ONyh1IjQxBIBur2F/WRMr4iqtAk0MA5u5yloq462fQtfgD+n5/uVzOHtGKt/7yy62FNaOUoBKBT9NDAGgoLqZtq5u5rmKwB2py2EMRgQuuAsaSyDv0UGrhrld3P/pJUxOiuELj2/lUIXOb1DKH5oYAkDPwHNG+wEYPw/cY2pB2RM37TyY9gl4694BZ0P3SIgJZ90tywhzCTc+/AFlW56Dpz9v/ds58BPilBrLNDEEgD2ljUSGQUzNbpi0yOlwgsOlP7Umu71215BVs1JjefKWBfy393+Z+MLn8Hz0Fhz9AJ69DZ5cA56OUQhYqeChiSEA7Clt4LzUZqSjCSZqYvBL2kw4+xuw40nY97fB67Y3Mv2lm7iQTdwnn2FF+6/Ju+ptuOS/4eDL8Ozt4NXbWpXqoYnBYcYY9pQ2cn68vVSUnjH479zvWIl049es50P70t4I/3cNlG5DrvsDq7/6M+JjY/j0o1vYGHMlXPSfsPc52PTg6MauVADTxOCwkvo26lu7OM1daC2FkTbH6ZCCR1gEXGsvx/341dDQb23Hpgp4/Coo3QbX/QHmrmZqSizPfulMTstM4OtPfshP6lbizVkFr/8QKveNdg+UCkiaGBz2/kc1AGR17Lcmb4VFOBxRkEmZDp95Glqq4KFzIO/3UL7ben3oXKjYA9f/EeZ86tghSbERPHHbCj53xlQeebeQLzTcgjc81jrz8HY72BmlAoMmBoe9m19NRqwhpvJDmHqm0+EEp8xcuP0NSMiE578BD55lvY6bCLe9BrMvP+6QiDAXd6+ezy9vWMQ7ZfDDrpugeAtsGfwWWKXGgjCnAxjLjDG8l1/NzZMqkaJOyDrH6ZCCV9pMuOMtKNturaeUMAUylljzHgZx5eIMZk2I50uPR/J285uc8ep/ED77MivJKDVG6RmDg/aXN1Hd3MkFUQdBXDBlhdMhBTcRmLQY5l0FmUuHTAo95kwcx3NfO4fnJ3+HLk83B9fdQXe33qWkxi5NDA56L78agBmt22HCAohOdDiisSshOpz/vvWTvJ35BWY2vMfvHvpf2rt0vEGNTZoYHPTOoWrmpwqR5Vv1MlIAcLuEVbf+kKpx87im4j6+9Yc3NDmoMUkTg0M6PN1sLqjl5tSD0N1pLSmtnOdyk/bph0h2tfKJI7/i9j/maXJQY44mBodsO1JPW1c353g+gNh0mLzc6ZBUjwkLcJ39r1wX9jbhh1/j9j/m0dapyUGNHZoYHPLWwSqiXV2Mr3wbZl824KMqlUPO+y6MX8ADsb/jQP4hbvvjFk0OaszQxOCArm4vz2wr5puT9iOdLTD3SqdDUv2FRcK164gyHfwt43He/6iKWx/T5KDGBk0MDvj7vgqqmjq43rwCydMh+zynQ1K+pM2ES+9lfPUHbFy0lfcP1+iYgxoTNDE44E+bizg3vozEmm2Q+3nrkZUqMC3+LMy9kvkHfs3vVgrvfVStyUGFPP2LNMqKalt551AVd417HiLiYdGnnQ5JDUYEPvUriJ/Eyj138vMrpvFufjU3r9tMU3uX09EpNSI0MYyyJzcf5TTJJ6fmDTjzaxCT7HRIaijRiXDNI1B/lKtKf84vb1jE1iN13PS7TdS1dDodnVLDThPDKGps7+LpzYX8PP5PEJsGZ3zZ6ZCUv6asgPPvhF0bWC3v8OBNS9lX3sQ1v/0nh6uanY5OqWGliWEU3fvSfq7u/CvTOvbDpfdCZLzTIakTcc6/wdSz4IV/48L0Rp647XTq27q48v73ji1volQo0MQwSrYU1rJv8+t8O3yD9WyAeVc7HZI6US43XP2IdSvrkzeybIKb575yFhMSovjso5v4n1cO0KWL76kQoIlhFHR4uvnV06/xcOSvkIQpcMVv/F75UwWYhAy4/nGoK4BnbmNyYiTPfvksrlmSyW/eyOfqB/7JjqJ6p6NU6pRoYhhhnR4vP/y/1/ivxn8nIcKL68Y/6SqqwS7rLLj0p3DoVXj1+8RFuPnZdafx4E1LKK1vY/X973HHH/PYUVSPMcbpaJU6YfqgnhHU3tXN2j/+na8d+QYTwlsJv/lvMH6u02Gp4bDsVqg+CB88AN1dsOq/WTV/ImfNSGXdu4X87p3DvLq3gsykaC6eO4E5E+OZlhZHSmwEUeFuosPdRIa7iAxzIXr2qAKMhMIvmtzcXJOXl+d0GH3sKKrnoWde4rt1PyAjrImwz/0Fpp7hdFhqOBkDr34f3v8NZC6zbijIWApAQ2sXr+wp58XdZfwzv4bOAcYeRCAqzE1UuIukmAgykqKZnBzDjLQ4ZqTHkTM+jgnjojR5qBEhIluNMbnHlWtiGD7GGHaVNPDHfxZidjzJ3eF/ICwylsjPbrCeS6xC0+5n4IV/g7Y66wlyWWdDygxIngYJk/HETaSosZuC6mYa2rpo6/TS3tVNW1c3HfZrW1c3tS2dFNe1cbS2lfrWjyfPxUWGMT09jpx0K1lMGBdFSlwEKbGRpMZHkBwTQZhbrwqrE3dKiUFEVgG/AtzA74wx9/TbHwn8EVgK1AA3GGMK7X13ArcC3cDXjTGvDNamiGQD64EUYCvwWWPMoLOInEwMDa1dbDtax6aCWl7fW05ydR7fDH+WM1x78GSuIOy6R/X5wWNBeyNs/QPsfQ7Kd0F3R6+dAnHpkJJj/UDIzLXOMOIn+GzKGENNSyf5lc0cqmwmv6KJ/KpmDlU0U9nUcVx9EesJdMkxESTFRpAUE0FafASZSTFkJEaTmRRNRlI04+OjcLn0zEN97KQTg4i4gYPARUAxsAW40Rizt1edLwMLjTFfFJE1wFXGmBtEZC7wJLAcmAS8Dsy0D/PZpohsAJ41xqwXkQeBHcaY3w4W43AkBmMM3V6Dx/vxa4enm8a2Lupbu2hos/5VNnVQWttIeXUd5ZVVhDeXkClVnOYuZFXEDiZ1l+CNTsF1wfdg6b/octpjkbcbGkug9jA0FENDCTQchcp9ULYTvPbZQPwkyFgCExZCcjYkTrVmwkfEWXNcwmOOW0erqb2LqqYOqps7qWnuoLq5g6rmTupbO6lt6aSutZPali4qG9up6TcrO8LtOnaparL9mhYXSVJsOIkxVkJJiA4n3C24XYJLhDCXtd1zKavn/5OubkNntxdPt5eubkNXt5e2rm5aOjy0dHTT0umxtjutstZe2z2vHq/1t0ewkpv1KgjgcgkRbhcRYS7C3WK/unqVWa/R4W5iItzERIQRE+kmNiLMfv9xWVSY2+4Pp3RJzhiDMeA1BoP9aqwrigaD10CXx0uHx0uHp5vOXtsdPdtdXjq7vXR0ddPZ7SXMZfUtwu3u09fIXmV9ynuVuYchyQ+UGPwZfF4O5BtjDtsNrQdWA3t71VkN/NDefhr4jVj/BVYD640xHUCBiOTb7eGrTRHZB1wA9Cwg9Jjd7qCJ4WR94fE83jxQdSwR+ONP4T/mi+5eXY+0XkxYFDL5dJj/XVwLroOImBGIWAUFlxsSp1j/+utqt84oirdA6TYo2Qb7nx+kMYFbXrDuhALio8KJjwpnWtrQYbR1dlNS30ZxXSvFdW0U1bVSXGtdqtpZXN/nctVQRMAtQrf9x/BEiXDsj3ZspPUa7nZhAOw/tD1/YAE8drLp6jZ0erx0dXvp9Fh/VDu7vScVQ+9YXPJxonDZ73v/gcdHAgg0bpcQ7hae/9o5zEiPG9a2/UkMGUBRr/fFwOkD1THGeESkAetSUAbwQb9jM+xtX22mAPXGGI+P+n2IyB3AHfbbZhE54EdfBpIK+DV19awB9zQCf7P/3XIKoZwyv/sS4EKlH3CqffnR2cMXyanR/yYBKOfHp9SXqb4Kg/Z2VWPMw8DDw9GWiOT5Op0KRqHSl1DpB4ROX0KlH6B9GYo/tzKUAJN7vc+0y3zWEZEwIAFrEHqgYwcqrwES7TYG+iyllFIjyJ/EsAXIEZFsEYkA1gAb+9XZCNxsb18L/MNYo9obgTUiEmnfbZQDbB6oTfuYN+w2sNt87uS7p5RS6kQNeSnJHjP4KvAK1q2l64wxe0TkbiDPGLMReBR43B5crsX6Q49dbwPWQLUH+IoxphvAV5v2R34XWC8iPwY+tNseacNySSpAhEpfQqUfEDp9CZV+gPZlUCExwU0ppdTw0emSSiml+tDEoJRSqo8xnxhEZJWIHBCRfBFZ63Q8/YnIOhGpFJHdvcqSReQ1ETlkvybZ5SIi99l92SkiS3odc7Nd/5CI3Ozrs0a4H5NF5A0R2Ssie0TkX4O4L1EisllEdth9+ZFdni0im+yYn7JvrMC++eIpu3yTiGT1autOu/yAiFwy2n2xY3CLyIci8nyQ96NQRHaJyHYRybPLgu77ZceQKCJPi8h+EdknImeMal+sad5j8x/WwPdHwDQgAtgBzHU6rn4xngssAXb3KvspsNbeXgvca29fBryEtbrACmCTXZ4MHLZfk+ztpFHux0Rgib0dj7Ukytwg7YsAcfZ2OLDJjnEDsMYufxD4kr39ZeBBe3sN8JS9Pdf+zkUC2fZ30e3Ad+xbwJ+A5+33wdqPQiC1X1nQfb/sOB4DbrO3I4DE0ezLqHY20P4BZwCv9Hp/J3Cn03H5iDOLvonhADDR3p4IHLC3H8Jac6pPPeBG4KFe5X3qOdSn57DWygrqvgAxwDasmfvVQFj/7xbW3Xdn2Nthdj3p/33rXW8U488E/o61FM3zdlxB1w/7cws5PjEE3fcLax5YAfbNQU70ZaxfSvK13IfPJTgCzHhjTJm9XQ6Mt7cH6k9A9dO+BLEY65d2UPbFvvyyHagEXsP6lTzQci59lowBei8Z43Rffgl8B+h5YMRgy9IEcj8ADPCqiGwVa8kcCM7vVzZQBfzevsT3OxGJZRT7MtYTQ9Az1k+BoLnnWETigGeAbxhjGnvvC6a+GGO6jTGLsH5xLwdmOxzSCRORTwKVxpitTscyTM42xiwBLgW+IiLn9t4ZRN+vMKzLx781xiwGWrAuHR0z0n0Z64nBn+U+AlGFiEwEsF8r7fITXYJkVIlIOFZSeMIY86xdHJR96WGMqcearX8GAy/ncqJLxoyWs4ArRKQQ6xkoF2A9IyXY+gGAMabEfq0E/oKVsIPx+1UMFBtjNtnvn8ZKFKPWl7GeGPxZ7iMQ9V6CpPeyIRuBz9l3KawAGuxTz1eAi0Ukyb6T4WK7bNSIiGDNYt9njPl5r13B2Jc0EUm0t6Oxxkr2MfByLie6ZMyoMMbcaYzJNMZkYX33/2GM+QxB1g8AEYkVkfiebazvxW6C8PtljCkHikRkll20Emv1iNHry2gPEAXaP6wR/YNY14j/3el4fMT3JFAGdGH9krgV67ru34FDWA8/SrbrCnC/3ZddQG6vdj4P5Nv//sWBfpyNdeq7E9hu/7ssSPuyEGu5lp1Yf3x+YJdPw/qDmA/8GYi0y6Ps9/n2/mm92vp3u48HgEsd/J6dz8d3JQVdP+yYd9j/9vT8vxyM3y87hkVAnv0d+yvWXUWj1hddEkMppVQfY/1SklJKqX40MSillOpDE4NSSqk+NDEopZTqQxODUkqpPjQxKKWU6kMTg1JKqT7+PxkBGEVCUmiTAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "sns.kdeplot(train_data['GrLivArea'], ax=ax, label='train')\n", + "sns.kdeplot(live_data['GrLivArea'], ax=ax, label='live')" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZhU1Zn48e9bS1fvK83WDXQja4MI0iDGaFyigiZiEhcckzjRxEmiv6wzE4gTHZ2YaDITJ4vGmEhiJonAaFTi4BrXzCiCisoqLSI0izRN711VXcv5/XFvY3Vb3V1VvdwqeD/P0w+3zj331Htjpd4699x7jhhjUEoppbq5nA5AKaVUetHEoJRSqgdNDEoppXrQxKCUUqoHTQxKKaV68DgdwFAYNWqUqaqqcjoMpZTKKK+++uphY0x57/JjIjFUVVWxceNGp8NQSqmMIiLvxSvXS0lKKaV60MSglFKqB00MSimlejgmxhiUUipZoVCI+vp6AoGA06EMu+zsbCorK/F6vQnV18SglDou1dfXU1BQQFVVFSLidDjDxhhDY2Mj9fX1VFdXJ3SMXkpSSh2XAoEAZWVlx3RSABARysrKkuoZaWJQSh23jvWk0C3Z89TEoJRSqgdNDEPgh49t40u/1wfslFLJaW5u5q677kr6uAsuuIDm5uZhiMiiiWEIrN91hOffbiAUiTodilIqg/SVGMLhcL/HrVu3juLi4uEKSxPDUNjX7KcrHOWdhnanQ1FKZZDly5fzzjvvMHfuXBYsWMDpp5/ORRddRE1NDQAXX3wx8+fPZ9asWdxzzz1Hj6uqquLw4cPs3r2bmTNn8qUvfYlZs2Zx3nnn4ff7Bx2X3q46SIFQhIa2IABb97cyY2yhwxEppZJ181+2sHV/65C2WTO+kJs+OavfOrfddhubN29m06ZNPPfcc1x44YVs3rz56G2lK1eupLS0FL/fz4IFC/jMZz5DWVlZjzZ27tzJ/fffz69//Wsuu+wyHnzwQT772c8OKnbtMQzSvuYPsvOWIf5gKaWOLwsXLuzxrMHPfvYzTjrpJBYtWsTevXvZuXPnh46prq5m7ty5AMyfP5/du3cPOg7tMQxSfZOVGLLcriH/xaGUGhkD/bIfKXl5eUe3n3vuOZ5++mleeuklcnNzOfPMM+M+i+Dz+Y5uu93uIbmUpD2GQapv6gTgtCllbD3QijHG4YiUUpmioKCAtra2uPtaWlooKSkhNzeX7du38/LLL49YXNpjGKT6Jj8el3DWjNE8u6OBfc1+KktynQ5LKZUBysrKOO2005g9ezY5OTmMGTPm6L7Fixdz9913M3PmTKZPn86iRYtGLC5NDINU3+RnfHEOsyuKAGsAWhODUipRf/rTn+KW+3w+Hnvssbj7uscRRo0axebNm4+W/+M//uOQxKSXkgapvqmTypIcZowtQAS2HtBxBqVUZtPEMEj1TX6mFEbIff1e5pZF9M4kpVTG08QwCN3PMJwXeBwe+2f+0PlVjtTXOR2WUkoNiiaGQdhvP8Mw2b8FfIXkRduY2fEyzZ1dDkemlFKp08QwCNYzDIby5k0w/QJCWcXMlnd5p6HD6dCUUiplmhgGob7Jz0Q5hDfQCBNPwT/qRE50vUuLX3sMSqnMpYlhEOqbOlnoth9Rn3AK0bEnMU3qaW3TyfSUUgPLz88HYP/+/VxyySUOR/MBTQyDUN/kZ372fvBkQ/kMvBPm4ZUIrkNbnQ5NKZVBxo8fzwMPPOB0GEcllBhEZLGI7BCROhFZHme/T0RW2/vXi0hVzL4VdvkOETk/pnyliBwSkc292vqxiGwXkTdF5CERGb5JxwepvqmTSd5mKBgHLjfZE+cDkHNEE4NSKnG7d+9m9uzZACxatIgtW7Yc3XfmmWeyceNGOjo6uPrqq1m4cCHz5s3jkUceGbZ4BnzyWUTcwJ3AuUA9sEFE1hpjYr/9rgGajDFTRGQZcDtwuYjUAMuAWcB44GkRmWaMiQC/A34B/L7XWz4FrDDGhEXkdmAF8J3BnORwqW/yM9Z7BAorAHCXTCSMC197vcORKaWS8thyOPjW0LY59kRYclvSh11++eWsWbOGm2++mQMHDnDgwAFqa2v57ne/y9lnn83KlStpbm5m4cKFfPzjH+8x8d5QSaTHsBCoM8bsMsZ0AauApb3qLAXus7cfAM4Ra/XppcAqY0zQGPMuUGe3hzHmBeBI7zczxjxpjOlevuhloDLJcxoRwXCEQ21ByiKHoXCcVehy0yBl5PgPOBucUipjXXbZZUcvK61Zs+bo2MOTTz7Jbbfdxty5c4/OtLpnz55hiSGRuZIqgL0xr+uBU/qqY//SbwHK7PKXex1bkUR8VwOr4+0QkWuBawEmTpyYRJNDo6UzBBjyQ4etS0m2I+7RFAQPjng8SqlBSOGX/XCpqKigrKyMN998k9WrV3P33XcDYIzhwQcfZPr06cMeQ9oOPovIDUAY+GO8/caYe4wxtcaY2vLy8pENDmgLhimhDXe06+ilJIDmrLGUhA6NeDxKqWPH5Zdfzo9+9CNaWlqYM2cOAOeffz4///nPj07t//rrrw/b+yeSGPYBE2JeV9plceuIiAcoAhoTPPZDROTvgU8AV5o0XeCgIxhmnNhXwgo/6DG0Z4+jLHoYohGHIlNKZbpLLrmEVatWcdlllx0t+973vkcoFGLOnDnMmjWL733ve8P2/olcStoATBWRaqwv9WXA3/Wqsxa4CngJuAR4xhhjRGQt8CcR+QnW4PNU4JX+3kxEFgP/DHzMGNOZzMmMpPZAmDHSZL0oGH+0PJg3Ds+RCLQdhKJkrpoppY437e3WM09VVVU9ps8eM2YM4XC4R92cnBx+9atfjUhcA/YY7IHg64EngG3AGmPMFhG5RUQusqvdC5SJSB3wLWC5fewWYA2wFXgcuM6+IwkRuR8rkUwXkXoRucZu6xdAAfCUiGwSkbuH6FyHVHuPHsMHiSGcbyWDaPPeeIcppVTaS2ihHmPMOmBdr7IbY7YDwKV9HHsrcGuc8iv6qD8lkZic1h4MM0aOYMSF5H+w6pIpsm6iChx+j9xJI7fiklJKDZW0HXxOd+3BMGNpIppbDu4P8qunxLpDKtg4PLeRKaWGTpoOYQ65ZM9TE0OK2oNhyqQFyR/dozy/sJhWk0ukWR9yUyqdZWdn09jYeMwnB2MMjY2NZGdnJ3yMrvmcovZAmFJpR3LH9SgvzvXSYIooaNdbVpVKZ5WVldTX19PQ0OB0KMMuOzubysrEnxXWxJCijmCYElcHklPao7woJ4vDFFHUeex/2JTKZF6vl+rqaqfDSEt6KSlFbcEwxbRDbu/E4OWwKcTrP+xQZEopNTiaGFLU4e+ikHb4UI/By2FThC/Y6FBkSik1OJoYUhQNtuAmCjklPcqzPC5aXSVkh1shrCu5KaUyjyaGFLn8zdZGr0tJAJ1ZZdZGh44zKKUyjyaGFLmD9nQYOR9ODEGfnRja3x/BiJRSamhoYkhRVpfdY+h1KQkgnDvK2tAeg1IqA2liSJGvq8XaiHMpyeTZ04DrswxKqQykiSEF0aghJ9JqvYhzKeno3EkdmhiUUplHE0MKOrrClIg1XS45xR/an5dfQLvJxmiPQSmVgTQxpKAjGKGYNro8BeByf2h/sf0sQ6RVE4NSKvNoYkhBezBEibQT8n144Bns+ZIoItKmdyUppTKPJoYUtAWs6TAivg9fRgJ7viRThNExBqVUBtLEkIKOYIQi6SCaHT8xFOdal5LcnTpfklIq82hiSEF7MEQ+fiS7KO7+7sTgDTZBJDTC0Sml1OBoYkhBezBCgfhx5xTE3V9sT70NQIf2GpRSmUUTQwraA1aPwZ3Tf48B0GcZlFIZRxfqSUFnMEiuBAnnFsbdn+110+Kyxx/adVoMpVRm0R5DCoId1lPPnj56DABdOd3zJWmPQSmVWRJKDCKyWER2iEidiCyPs98nIqvt/etFpCpm3wq7fIeInB9TvlJEDonI5l5tlYrIUyKy0/43/sMCDgr77XmSfPHHGAAiOTpfklIqMw2YGETEDdwJLAFqgCtEpKZXtWuAJmPMFOAO4Hb72BpgGTALWAzcZbcH8Du7rLflwF+NMVOBv9qv00rU32Zt9JMYfHmFBMSnM6wqpTJOIj2GhUCdMWaXMaYLWAUs7VVnKXCfvf0AcI6IiF2+yhgTNMa8C9TZ7WGMeQE4Euf9Ytu6D7g4ifMZEdHAwImhOMdLkxRrj0EplXESSQwVwN6Y1/V2Wdw6xpgw0AKUJXhsb2OMMQfs7YPAmHiVRORaEdkoIhsbGkb2V7kE7ZlVffEHnyHmziQdY1BKZZi0Hnw2xhjA9LHvHmNMrTGmtry8fETjki57ZtX+egy5WbwfLdS7kpRSGSeRxLAPmBDzutIui1tHRDxAEdCY4LG9vS8i4+y2xgFp95PbFRr4UlJRjpdDkUKdL0kplXESSQwbgKkiUi0iWViDyWt71VkLXGVvXwI8Y//aXwsss+9aqgamAq8M8H6xbV0FPJJAjCPKE7J7DFn5fdbpnmGVzkaIRkYoMqWUGrwBE4M9ZnA98ASwDVhjjNkiIreIyEV2tXuBMhGpA76FfSeRMWYLsAbYCjwOXGeMiQCIyP3AS8B0EakXkWvstm4DzhWRncDH7ddpxRvusDYG6DEcNkWIiVrJQSmlMkRCTz4bY9YB63qV3RizHQAu7ePYW4Fb45Rf0Uf9RuCcROJyQiRqyI520OXJISvOIj3dinOyaDT24HT7IcgfPUIRKqXU4KT14HM6ag+GySNAyJPXbz2dL0kplak0MSSpPRimQPxEvH2PL4B9KUlnWFVKZSBNDEnqCIbJx080q+/xBejVY9CH3JRSGUQTQ5LaAmHyxY8ZIDHk+zx0uPIIi1cvJSmlMoomhiR19xj6uyMJQEQozsmi3VOiD7kppTKKJoYktQetHoMru+/pMLoV5XqtdRm0x6CUyiCaGJLUHghTQCeunIETg06kp5TKRJoYktS9rKe3j9XbYhXnZtFgCvWuJKVURtHEkKSuQDtuMXgS7DEcjBRal5Ki0RGITimlBk8TQ5LC9iI9iSSGolwv9eFCiIbBH2/pCaWUSj+aGJIUPbqsZyI9hiz2dNnPMrQd6L+yUkqlCU0MSTLBgafc7lac6+WQKbZetL0/jFEppdTQ0cSQrCQTw/uUWC+0x6CUyhCaGJIkXYknhqIcLw3dPYb2g8MYlVJKDR1NDElyJbCsZ7fi3CyCZBHKKoI2TQxKqcygiSFJnqPLeiZ2uyqA31euiUEplTE0MSTJk8Dqbd2Kc63E0OYt08SglMoYmhiSlBXusGZM9fgGrFuQ7UUEWtyjoF3vSlJKZQZNDEnKinbQ5cpNqK7bJRRmezniKrF6DMYMc3RKKTV4mhiSYIzBF+mka4BlPWNZzzKUQDQEnY3DGJ1SSg0NTQxJ6IpEycdP2NP/sp6xinO81JtS60XL3mGKTCmlho4mhiR0BiNWYhhgvedYRblZ7A6VWS+aNTEopdJfQolBRBaLyA4RqROR5XH2+0Rktb1/vYhUxexbYZfvEJHzB2pTRM4RkddEZJOI/E1EpgzuFIdOR1eYfOkkmpVEYsjx8k7Ifvq5pX6YIlNKqaEzYGIQETdwJ7AEqAGuEJGaXtWuAZqMMVOAO4Db7WNrgGXALGAxcJeIuAdo85fAlcaYucCfgH8Z3CkOHX+X1WMYaL3nWMU5XvZ0+sCbp5eSlFIZIZEew0KgzhizyxjTBawClvaqsxS4z95+ADhHRMQuX2WMCRpj3gXq7Pb6a9MA3U+PFQH7Uzu1odfRFSFfBl7vOVZJrpeWQBhTVAnNe4YxOqWUGhqeBOpUALE/deuBU/qqY4wJi0gLUGaXv9zr2Ap7u682vwisExE/0AosSiDGEdEZDFOAnyMJrPfcbXRhNlEDXfkV+LTHoJTKAOk4+PxN4AJjTCXwW+An8SqJyLUislFENjY0NIxIYP5AgGwJ4UoiMYwtzAagLXucDj4rpTJCIolhHzAh5nWlXRa3joh4sC4BNfZzbNxyESkHTjLGrLfLVwMfiReUMeYeY0ytMaa2vLw8gdMYvGCntUhPIqu3dRtjJ4YjnjHWKm5dHcMSm1JKDZVEEsMGYKqIVItIFtZg8tpeddYCV9nblwDPGGOMXb7MvmupGpgKvNJPm01AkYhMs9s6F9iW+ukNrXBnKwCe3MTHGMYUWVNnvC928tI7k5RSaW7AMQZ7zOB64AnADaw0xmwRkVuAjcaYtcC9wH+JSB1wBOuLHrveGmArEAauM8ZEAOK1aZd/CXhQRKJYieLqIT3jQQj7rcTgzS1O+JhReT48LmF3tJzTAY7sgvLpwxOgUkoNgUQGnzHGrAPW9Sq7MWY7AFzax7G3Arcm0qZd/hDwUCJxjbSonRh8eUUJH+NyCaMLfOwI25efDu+E6UuGIzyllBoSCSUGZYkG7UtJOYknBoAxRdns7vBA3mg4vGM4QlNKqSGTjnclpS1JYr3nWGMLsznYGoBR06weg1JKpTFNDMlIMTGMKczm/ZYAlE+Dhh06/bZSKq1pYkiCO5R6YmgLhgkWT4FA86Cm3zaaVJRSw0zHGJLgDrUTRXB5E1+PAWCsfctqY84kxoPVa8gblVQbbYEQNz2yhUffOsC8CcWcPWM0Vy6aRL5P/xMqpYaW9hiS4Al1EpAccCX3P1v3Q277vZOsgkNbkzr+tT1NLPnpizy8aR8XnjiOFn+IHz62nRV/fiupdpRSKhGaGJLgDbcTcCXXW4APpsXYGy6G/LFQvyHhY3c1tPP5e18B4L+/fCp3XD6Xx79xBl87ewp/eWM/G3YfSToepZTqjyaGJPgiHQTdia33HKu7x3CwtQsqaxNODJ1dYb7yh9fwuoXV/3Aq8yeVHt335TNPYFxRNv+6dguRqI47KKWGjiaGJPiiHYTcyfcY8nweCnwe3m8NQOUC6+nnjv4HoI0x3PDQZt4+1MZ/LptHRXFOj/25WR6WL5nBlv2tPPCqTs6nlBo6mhiSkBPtJORJPjGA9ZDbwRY7McCAvYYnthzkodf38fVzpvKxafEnCbzopPHMn1TCz/5ap3crKaWGjCaGBBljyDGdRLzJ3ara7ehDbuPngcsD7/2tz7qBUIR/e3QbM8YWcP1Zfa9sKiJcsXAi+5r9bN7XmlJcSinVmyaGBAXDUfLFTySJ9Z5jjSnMti4lZeXC5DNh69o+H3S754Vd7Gv2c9MnZ+Fx9/+f6OwZo3EJPLn1YEpxKaVUb5oYEtR5dL3n1BLD2CIfh9qCRKMGai6G5vfgwKYP1dvX7Oeu5+q48MRxnHpC2YDtluZlsaCqlKe2vp9SXEop1ZsmhgR1BkPkE8D4El+kJ9bYwmwiUcPhjiDMuNC6nLT5zx+q9x9P7sAYWHHBjITbPm/WWLYfbOO9Rl0ESCk1eJoYEuTvbMMlBleS02F0G23fsvp+SxByS63ksPG3Pe5O2tPYySOb9vPZRZOoLEn8ttjzasYAaK9BKTUkNDEkKNDeDIAksd5zrLFHn2UIWAVn3QChDnj+9qN1fvl8HW6XcO0Zk5Nqe0JpLjPGFvCkJgal1BDQxJCgcIe13rM7ifWeY00qy0UEth2w7x4qnw6118Arv4JN97O/2c8Dr9Zzee2Eow/EJeO8mjFs3H2ExvZgSvEppVQ3TQwJ6uq0EoM3N7lFeroV52Yxp7KY53Yc+qDw/B9A9Rnw8FeoW/UdPCbMP3wsud5Ct7NnjiFq4P/eSX3mVqWUAk0MCYv4uxNDaj0GgDOnlfP63maaOrqsAk8WXLEa/6zLOePgfTxf8C9UNr+aUtuzxxeSm+Vmo86dpJQaJE0MCYoErLUYfHnFKbdx5vRyjIEXdjZ8UJiVyx153+Dq0D9R4jNw3yfgf74NIX9SbXvcLk6eWMKG3U0px6eUUqCJIWFRvzU24MtP7VISwJzKYkpyvTy/44PEcKSjiz+8/B6FJ16I9/r1sOg62PAb+P1S6Ezu139tVQnbDrbSGgilHKNSSmliSJS9rGfOIHoMbpdwxrRynn+7wXrQDVj5t3fxhyJcd9YU66noxT+AS++D/a/DysXQnPgEeQuqSjEGXntPew1KqdRpYkiQdFmJwZPiXUndzpxeTmNHF5v3t9DYHuS+/9vNktljmTom5vmIWRfD5x6CtoPw2yXQ9F5Cbc+dUIzbJWzUy0lKqUFIKDGIyGIR2SEidSKyPM5+n4istvevF5GqmH0r7PIdInL+QG2K5VYReVtEtonI1wZ3ikNDutoJ4LUGjAfhjKnliMBNa7dw5o+f+6C30FvVR+Hv/2L1VH73CWjeM2DbeT4Ps8cX6uI9SqlBGTAxiIgbuBNYAtQAV4hITa9q1wBNxpgpwB3A7faxNcAyYBawGLhLRNwDtPn3wARghjFmJrBqUGc4RNyhdjpJfpGe3sryfZw8sYRNe5s5fdooHrn+NGaN72PcYtxJ8PmHIdgCv7swoctKtVWlbNrbTDAcGXSsSqnjUyI9hoVAnTFmlzGmC+uLemmvOkuB++ztB4BzRETs8lXGmKAx5l2gzm6vvza/AtxijIkCGGNibvx3jifcjt81+MQA8MsrT+a5fzyTu66c33dS6DZ+HnzuYfDbyeFwXb/VF1SVEAxHdRpupVTKEkkMFUDsT9V6uyxuHWNMGGgByvo5tr82TwAuF5GNIvKYiEyNF5SIXGvX2djQ0BCvypDyhjrwp7DeczyjC7OZVJZEWxUnw+cfgq52+M3Z8M6zfVatrbKW/9TnGZRSqUrHwWcfEDDG1AK/BlbGq2SMuccYU2uMqS0vj7/C2ZAGFWknOEQ9hpRUzIcvPQOFFfCHz8CzP4RQ4EPVRuX7qB6Vp88zKKVSlkhi2Id1zb9bpV0Wt46IeIAioLGfY/trsx7ono/6IWBOAjEOO98glvUcMiVVcM2TMPsz8Pxt8J8nwvM//tD60fMmFvP6niZd7lMplZJEEsMGYKqIVItIFtZg8tpeddYCV9nblwDPGOtbaS2wzL5rqRqYCrwyQJsPA2fZ2x8D3k7t1IZWdrSDkNvhxADgK4DP/BquetQanH72+/CTmfDQl6Hemk7j5IklNHZ0sedIp8PBKqUykWegCsaYsIhcDzwBuIGVxpgtInILsNEYsxa4F/gvEakDjmB90WPXWwNsBcLAdcaYCEC8Nu23vA34o4h8E2gHvjh0p5u6XOMnmuLqbcOi+nTr79A260npN1bBG/fDrE+xYP7NALy2pym5sQyllALkWLjcUFtbazZu3Dis79F1Uxmvjr+CU//hF8P6PikLtsFLd8GL/44prGBJ4zdYcHIt/3bxbKcjU0qlKRF51R7P7SEdB5/TTlfAT5aErcs46cpXAGd+B/7+f5BAC7/1/oiduxN7YloppWJpYkhAR5u9eluK6z2PqAkL4YpVlEcb+NKRf6czqBPqKaWSo4khAZ12Ykh19bYRN/EU3p37bc5xvca+F37vdDRKqQyjiSEBgXbrmQBPiqu3OWHU2V9nU/QExm34YdJrOyiljm+aGBIQ7Bj86m0jraQgh9/lXU1+V4N115JSSiVIE0MCuuzEMJjV25zgrj6dl5iD+d+fxn1KWiml4tHEkICwvd5zdl7m9BjAWtHt512fQDoaYPODToejlMoQmhgSEPFbg885hWUOR5KcRZPL+L/oLJrzp8DLv4Rj4JkVpdTw08SQAGOv95xfVOpwJMmpKstldEE2j+ddBO+/BftfczokpVQG0MSQiGArXcaDL9vB2VVTICKcMrmMXzXOxXhy4PU/OB2SUioDaGJIgDvYQrvkYq09lFkWTS7l3XYP7SdcCG89oLeuKqUGpIkhAe5QGx2SmZPRnVJtjYtsKDoPgq1Q97TDESml0p0mhgR4Q+34XWk0s2oSTijPY1S+j0dbJkNuGWx5yOmQlFJpThNDAnzhNgLpsBZDCqxxhlJe2t2KmXkR7HgcunSdBqVU3zQxJMAX7aDLk5k9BoBF1aUcaAlwaMISCHVA3VNOh6SUSmOaGBKQG+0g5M2sh9tiLZpsjTM8G5gKuaNgy8MOR6SUSmeaGBKQazqIeNN4LYYBTBmdz/iibP769hGouQje1stJSqm+aWIYSCRMHgFMOi/SMwAR4eM1Y3hxZwPBaUsh1Ak7n3Q6LKVUmtLEMICuTmueJDJhkZ5+nFszhkAoyotd06y7k7b9xemQlFJpShPDADpbjwAg2ZmdGE6pLiPf5+HpHYdh+gVWjyEcdDospVQa0sQwAH+blRjcuSUORzI4WR4XH5teztPbDhGd8UnrYbd3X3A6LKVUGtLEMICjq7flZc7qbX05d+YYDrcHeTPrJMgqgG1rnQ5JKZWGEkoMIrJYRHaISJ2ILI+z3yciq+3960WkKmbfCrt8h4icn0SbPxOR9tROa+h0tVtTbmflZtYiPfGcNX00bpfw1NvNMO082L4OohGnw1JKpZkBE4OIuIE7gSVADXCFiNT0qnYN0GSMmQLcAdxuH1sDLANmAYuBu0TEPVCbIlILpMW1m3Cn1WPw5adFOINSlOtlYVUpj711EDPjE9B5GPa87HRYSqk0k0iPYSFQZ4zZZYzpAlYBS3vVWQrcZ28/AJwj1lSkS4FVxpigMeZdoM5ur8827aTxY+CfB3dqQyNi35WUU5hZazH05dMnV7DrcAev+WrB7YPtjzodklIqzSSSGCqAvTGv6+2yuHWMMWGgBSjr59j+2rweWGuMOdBfUCJyrYhsFJGNDQ0NCZxGaqIBKzHkFmR+jwHgwjnjyPd5uH9TE5xwtnXbqq7sppSKkVaDzyIyHrgU+PlAdY0x9xhjao0xteXl5cMXU6CZdpNNQV5mLdLTl9wsD588aRz/8+YB/FOWQMteOLDJ6bCUUmkkkcSwD5gQ87rSLotbR0Q8QBHQ2M+xfZXPA6YAdSKyG8gVkboEz2VYuAPNtJCPz5NWOXRQLqudgD8UYV1wLogbtunlJKXUBxL5ttsATBWRahHJwhpM7n2f41rgKnv7EuAZY4yxy5fZdy1VA1OBV/pq0xjzP8aYscaYKmNMFdBpD2g7xtvVQpvkZ+TqbX2ZO6GY6WMK+P2b7TDpI/oUtFKqhwETgz1mcD3wBLANWGOM2SIit4jIRXa1eyOTvUEAABLaSURBVIEy+9f9t4Dl9rFbgDXAVuBx4DpjTKSvNof21IZGVqiFDlfmzpMUj4hw2YIJvLG3mf3jz4XDO6DhbafDUkqlCU8ilYwx64B1vcpujNkOYI0NxDv2VuDWRNqMU8fxRRB84VY63RMGrphhLplfyR1Pvc3dB2dwC8D2v0D5t50OSymVBo6dC+fDJDfSRsCT2fMkxVOU4+XKUybyh21hgmPmwVZ9ClopZdHE0B9jyIu2ZfQiPf25+qPVeFwunnV/xLoz6fBOp0NSSqUBTQz96erAS5iQL/Onw4hnTGE2nz65gu/vmY0RF7xxv9MhKaXSgCaG/vit6TCix2hiALj2jMnsixSxq/AUeGOVzp2klNLE0K/uxJBz7CaGyeX5fHLOeO5sWgit+3QqbqWUJob+dHVYazG4co6N6TD68q1zp/F4+GT87ny9nKSU0sTQn2BrIwCu3GNjAr2+VI3K4+IFJ/Bw1ylEtz4CgVanQ1JKOUgTQz+62qzE4M0vcziS4fe1s6fyoDkLVzhgjTUopY5bmhj6EWq3E0PBsZ8YxhZlM/+0j7MpegKB/70LolGnQ1JKOUQTQz/CHUcIGg8F+cfWlBh9uf6sKTzg/STZre8SefsJp8NRSjlEE0M/wh1HaCGfUQXZTocyIgqyvSy44AvUm1E0Pf4DXadBqeOUJoZ+SOdhGk0Bo/KznA5lxFx08iQeK1rGqOY3adv6lNPhKKUcoImhHx5/I0cooijH63QoI0ZEOP2yb7DflNH0lxt0rEGp45Amhn5kdR2h3VNyTK3FkIgZleW8Of1rTAy8zaZ19zgdjlJqhGli6EdeqIlA1rH9cFtfzrnset72TKNy4w9oaDjodDhKqRGkiaEvoQC5ppMu37F/q2o8Xo+H7E//nGLTxo7fXU80qgPRSh0vNDH0pfMwACa33OFAnDOxZhFbT7iGj3Y8xQv//VOnw1FKjRBNDH0w7Q0ASMEohyNx1olX/pDt2XM5ZeutbH/jJafDUUqNAE0Mfehssq6rZxWOcTgSZ4nby/hr/kSH5JH38BdoPfK+0yEppYaZJoY+dDQdACC7+PhODACF5RUcvuDXjI4epuGeT2G6Op0OSSk1jDQx9CHQbP0yzi8d53Ak6WHGwnN5btatVPu3svfez+qCPkodwzQx9CHceoiA8VJafHzerhrPuZdcy59KvszE9//KkQe/qVNmKHWMSigxiMhiEdkhInUisjzOfp+IrLb3rxeRqph9K+zyHSJy/kBtisgf7fLNIrJSRBx57Nh0NHCYouNmnqREuFzC4i/ezO/lIkq33EfkxTucDkkpNQwGTAwi4gbuBJYANcAVIlLTq9o1QJMxZgpwB3C7fWwNsAyYBSwG7hIR9wBt/hGYAZwI5ABfHNQZpsjVeZhGU0hp3vEzT1IiRuX7KP/UbayNnIr7mZvhzTVOh6SUGmKJ9BgWAnXGmF3GmC5gFbC0V52lwH329gPAOWLNI7EUWGWMCRpj3gXq7Pb6bNMYs87YgFeAysGdYmp8gQZaXCW4XcfXdBiJWDKngien3sRL0VmYh78K7zzrdEhKqSGUSGKoAPbGvK63y+LWMcaEgRagrJ9jB2zTvoT0OeDxeEGJyLUislFENjY0NCRwGskp7DpES9boIW/3WHHjxfP4tuuf2OuqwDxwNbTudzokpdQQSefB57uAF4wxL8bbaYy5xxhTa4ypLS8f4qeTQ37yo610Zuutqn0ZXZjN9Uvmc1XH/yPSFYA/X6t3Kil1jEgkMewDJsS8rrTL4tYREQ9QBDT2c2y/bYrITUA58K1ETmLI2b9+u/LGO/L2meLyBRPIHjud2+Ua2P0i/O0nToeklBoCiSSGDcBUEakWkSysweS1veqsBa6yty8BnrHHCNYCy+y7lqqBqVjjBn22KSJfBM4HrjDGOLMYQKuVo0yBJob+uF3CTZ+s4dfti9hRfj48+0M4+JbTYSmlBmnAxGCPGVwPPAFsA9YYY7aIyC0icpFd7V6gTETqsH7lL7eP3QKsAbZijRVcZ4yJ9NWm3dbdwBjgJRHZJCI3DtG5JqzrSD0A7uLeQymqt0WTy7jgxHF8/uClRLKL4S/f0MV9lMpwnkQqGWPWAet6ld0Ysx0ALu3j2FuBWxNp0y5PKKbh1Hl4D1lAdpkjN0RlnBVLZnLO1kOsKf0yV+y7FV5dCQscuctYKTUE0nnw2TGhpnqOmHxKi4qdDiUjTCjN5aqPTOK7u2roqDgNnr4F2nSyPaUylSaGeFr3cdCUUZavD7cl6rqzplDg83KL+SKEOuGvNzsdklIqRZoY4nC3H+CAKWVUvs/pUDJGcW4W1589hdW7fOyb8QXY9EfY96rTYSmlUqCJIY6czv0cMKXaY0jS50+toqI4h28eOBeTNxoe+45OtKdUBtLE0FvnEXLCLez3VODzuJ2OJqNke918+7xpvHIgxKZpX4P6DfDWfzsdllIqSZoYemusA6Ajv9rhQDLTxXMrmDmukK9tm0l03Dx46kYItjsdllIqCZoYerMTg2/sNIcDyUwul/DdC2awtznIoxVfh7YD8L//6XRYSqkkaGLoxX9gOyHjZnSlJoZUnT61nNOnjuJ7r+YRnHkJ/O/PoGm302EppRKkiaGXzgM72GNGM71CV24bjBsunEl7MMxP5e/A5YYnv+d0SEqpBGli6EWOvMMuM44ZYwudDiWjzRhbyOcWTeKXrwd4f85XYdtaePcFp8NSSiVAE0OsaJSCjvc46KmgvECfYRisb547jbK8LL6+5zRM0QR4bDmEu5wOSyk1AE0MsRp34jVddBZPdzqSY0JRjpflS2by8l4/L075Jzi0BV74sdNhKaUGoIkhRmTvRgBMxXyHIzl2fHpeBR85oYyvbBhD+8zL4cX/gPqNToellOqHJoYYbbvW02ZyKK+a7XQoxwyXS/j3S0/C5RK+cvhSTOE4eOgfoKvD6dCUUn3QxBCr/lXejE5mxvgipyM5powvzuH7F8/mxb1dPDTxX6DxHV0KVKk0pomhWyhAQct23uQEpozOdzqaY87SuRVcPHc8395YyNY5K2D7o9ZT0UqptKOJoVv9K7hNmIMFs3WOpGFy22fmMG9CMRe/diLvz/g8vPQL+Js+Fa1UutHE0G37OoJk0VFxutORHLOyvW5+c9UCKotzWbL9ApqqL4Snb4LHV+hyoEqlEU0MAMbQtfVRXozMombSOKejOaaV5mVx39ULycv18ZGdV/LuCZ+Dl++CVX+nq74plSY0MQAcfJOstr08Lwv5zHxd53m4TSjN5eGvnkZNRQlnbVnMU5O+iXnnGbhzIbz+Bx2UVsphmhiAzhd/gd9kUTBvKUU5XqfDOS6U5fv44xdPYdmCiXxpxwL+zv3vNOVVwyPXwc/mwfpfpd6DiIR1gSClBkHMMfB/oNraWrNxY4oPTTW9R/Sn8/hd+DzO/dZKJpTmDm1wakDrdzXyLw9vpu5QK58v3sx12Y8xuvkNa+eY2TB2DhSOg+xiiIYh0gWBFvA3g78JAs09t8MBQMDjs/7yRkPJJCibChXzobIWSnW9DaVE5FVjTG3vco8TwaSNaITwI/+PsHHxztQvcLUmBUecMrmMdV8/nXVvHWDl30pYWD+H2e73uLJ0J2cE36J857N4/YcQE3OJKSsfckqsZJFTDKOmfPDaVwCRkJUgwgFoOwjN78F7/wfrf2kdX1INU8+DqedC1UfBm+PMySuVhhLqMYjIYuCngBv4jTHmtl77fcDvgflAI3C5MWa3vW8FcA0QAb5mjHmivzZFpBpYBZQBrwKfM8b0O/Nayj2GZ74PL/yYfwpdy+e+cgNzKouTb0MNKWMMb+1r4YktB3lyy/vsPGSt/uZzG2aWuakqL6ZqTBFTxhZTVZbH6EIfZXk+3C4ZuPFIGBq2w56XYOdT1myvYT94smHSaTD5TJj8MRhzIrgSvMoaaAGXB7LyUj5npZzSV49hwMQgIm7gbeBcoB7YAFxhjNkaU+erwBxjzJdFZBnwKWPM5SJSA9wPLATGA08D3SvgxG1TRNYAfzbGrBKRu4E3jDG/7C/GlBPD208SqnuWpyZ8jQtO1LuR0lFDW5BX32ti095m6g61sfNQO3uOdH5oCMHrFtwuweNy4XbJ0UQR+/n2ul0U5ngpyvFSmO1hlC/KSdEtnNj5ClWtGyhqfweAiK+IUPmJmDGzcY+ajKdkIq68UdaXvwgEWq2V6d59Ad5cDWfdAKd+dcT+N1E9DfQd1t/u/o7sr92Bfk5HjSEUMYTCUUKRKF2RKF3hqFVmvw7FvI4ag8/jJsvjwudx4fO68Hnc1rbHhc/rJsvtImoMwVCUQ20Bth5oZduBNq48ZWLKl8AHkxhOBf7VGHO+/XoFgDHmhzF1nrDrvCQiHuAgUA4sj63bXc8+7ENtArcBDcBYY0y493v3ZVBjDCrjBEIRdjV0sOdIBw1tQQ63dxGKRIlEDeGosf+NIljJQezORFc4SmsgRIv/g7/mjhBtwTAAo2niI64tLHRto8b1HjNkL9kS6jOOoPGyzpzK78yF7JSqfmPu/8upny+gAb6B+t2d4nsO9L7D9WV6DAx3jjivW7jnc7WcNWN0SscPZoyhAtgb87oeOKWvOvYXegvWpaAK4OVex1bY2/HaLAOajTHhOPV7n9C1wLX2y3YR2ZHAuQynUcBhh2MYjOM+/vewuq7JedT+G5Tj/n97h2V0/Gf/YFDxT4pXmLGDz8aYe4B7nI6jm4hsjJd5M4XG75xMjh00fqcNR/yJjLDtAybEvK60y+LWsS8lFWENQvd1bF/ljUCx3UZf76WUUmoYJZIYNgBTRaRaRLKAZcDaXnXWAlfZ25cAzxjrYuNaYJmI+Oy7jaYCr/TVpn3Ms3Yb2G0+kvrpKaWUStaAl5LsMYPrgSewbi1daYzZIiK3ABuNMWuBe4H/EpE64AjWFz12vTXAViAMXGeMdTN6vDbtt/wOsEpEvg+8bredCdLmslaKNH7nZHLsoPE7bcjjPyaefFZKKTV0dK4kpZRSPWhiUEop1YMmhkESkcUiskNE6kRkudPxdBORlSJySEQ2x5SVishTIrLT/rfELhcR+Zl9Dm+KyMkxx1xl198pIlfFe69hin+CiDwrIltFZIuIfD2TzkFEskXkFRF5w47/Zru8WkTW23Gutm++wL5BY7Vdvl5EqmLaWmGX7xCRfh/2HOJzcIvI6yLyaKbFbr/3bhF5S0Q2ichGuywjPj/2+xaLyAMisl1EtonIqSMWvzFG/1L8wxo4fweYDGQBbwA1Tsdlx3YGcDKwOabsR8Bye3s5cLu9fQHwGCDAImC9XV4K7LL/LbG3S0Yo/nHAyfZ2AdYUKjWZcg52HPn2thdYb8e1Blhml98NfMXe/ipwt729DFhtb9fYnysfUG1/3twj9N/gW8CfgEft1xkTu/3+u4FRvcoy4vNjv/d9wBft7SygeKTiH5H/QMfqH3Aq8ETM6xXACqfjiomnip6JYQcwzt4eB+ywt3+FNVdVj3rAFcCvYsp71Bvhc3kEa26tjDsHIBd4Devp/sOAp/fnB+sOvVPtbY9dT3p/pmLrDXPMlcBfgbOxHu2WTIk95v128+HEkBGfH6xnwd7FvkFopOPXS0mDE2+6kLhTeKSJMcaYA/b2QWCMvd3XeaTF+dmXJuZh/erOmHOwL8VsAg4BT2H9Yu5rypce08oAsdPKOBH/fwL/DHQvxt3fdDXpFns3AzwpIq+KNYUOZM7npxpr3rjf2pfzfiMieYxQ/JoYjlPG+vmQ9vcqi0g+8CDwDWNMa+y+dD8HY0zEGDMX69f3QmCGwyElREQ+ARwyxrzqdCyD9FFjzMnAEuA6ETkjdmeaf348WJeCf2mMmQd0YE9K2m0449fEMDiJTBeSTt4XkXEA9r+H7PJkpy4ZESLixUoKfzTG/NkuzqhzADDGNGM90X8qfU/5kuy0MsPpNOAiEdmNtTbK2Vhrp2RC7EcZY/bZ/x4CHsJKzpny+akH6o0x6+3XD2AlihGJXxPD4CQyXUg6iZ26JHa6kbXA5+07GxYBLXZ39QngPBEpse9+OM8uG3YiIlhPvW8zxvwk085BRMpFpNjezsEaH9lG31O+JDutzLAxxqwwxlQaY6qwPtPPGGOuzITYu4lInogUdG9j/XffTIZ8fowxB4G9IjLdLjoHawaJkYl/pAaCjtU/rLsB3sa6fnyD0/HExHU/cAAIYf36uAbruu9fgZ1YiyaV2nUFuNM+h7eA2ph2rgbq7L8vjGD8H8XqJr8JbLL/LsiUcwDmYE3p8ibWF9KNdvlkrC/HOuC/AZ9dnm2/rrP3T45p6wb7vHYAS0b4c3QmH9yVlDGx27G+Yf9t6f7/ZqZ8fuz3nQtstD9DD2PdVTQi8euUGEoppXrQS0lKKaV60MSglFKqB00MSimletDEoJRSqgdNDEoppXrQxKCUUqoHTQxKKaV6+P9e1PB6S2tLUQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "sns.kdeplot(train_data['BsmtFinSF1'], ax=ax, label='train')\n", + "sns.kdeplot(live_data['BsmtFinSF1'], ax=ax, label='live')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Finally - compare predictions\n", + "\n", + "Finally, we compare the predictions of our model.\n", + "\n", + "Given that we observed differences in the amount of missing data in BsmtQual and differences in the distribution of the other variables, there is sufficient reason to believe that the distribution of the predictions will not hold.\n", + "\n", + "Let's test that anyhow.\n", + "\n", + "In this particular scenario, we do not have the real value of the Sale Price during shadow mode, because, the houses do take some time to sell.\n", + "\n", + "So in order to evaluate the performance of the model, we can compare the distributions of the predictions between the live and train data, using again, the KS test." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ks_2sampResult(statistic=0.15722705507227056, pvalue=4.440892098500626e-15)" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stats.ks_2samp(train_data['SalePrice'], live_data['SalePrice'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As expected, the distributions are significantly different. And we can visualize that below:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAD4CAYAAADPccAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO29e3zU1Zn4/35mcr9fSAIkQJB7uIgSEYtaC1bwUnErKrbd2tbVdqu77brdirvfuq1bd6vdrb1atdX+bLcWqLaVrVatRW21igZBuUOAEBICud/vmfP743MCk2EmGWIyk8w879drXp/PnM85z3k+n5nkmXOe5zxHjDEoiqIoykjhCrcCiqIoSmShhkVRFEUZUdSwKIqiKCOKGhZFURRlRFHDoiiKoowoMeFWIBxMmDDBFBYWhlsNRVGUccW2bdtqjTE5Q9WLSsNSWFhISUlJuNVQFEUZV4jI0WDq6VSYoiiKMqKoYVEURVFGFDUsiqIoyogSlT4WRVGU4dDT00NFRQWdnZ3hVmVUSUhIoKCggNjY2GG1V8OiKIoSJBUVFaSmplJYWIiIhFudUcEYQ11dHRUVFUyfPn1YMnQqTFEUJUg6OzvJzs6OWKMCICJkZ2d/oFGZGhZFUZSzIJKNSj8f9B7VsCiKoigjihoWJTCePtjxFHx3ITz3lXBroyhRT2NjIw8//PBZt7vqqqtobGwcBY38o4ZFCcxLX4Pf/T20N8C7T0Jbbbg1UpSoJpBh6e3tHbTd888/T0ZGxmipdQZqWBT/GAN7noXZq+HvXoa+btj+i3BrpShRzfr16zl06BCLFy/mggsu4JJLLuHaa6+lqKgIgOuuu44lS5Ywf/58HnvssVPtCgsLqa2tpaysjHnz5nHbbbcxf/58rrjiCjo6OkZcTw03VvzTcASaK+DiL0PuXCi8BEqegA/9I7jc4dZOUcLON/5vN3uON4+ozKLJafz7x+YHvP6tb32LXbt2sWPHDl599VWuvvpqdu3adSos+IknniArK4uOjg4uuOACrr/+erKzswfIOHjwIL/61a/4yU9+wo033sgzzzzDpz71qRG9Dx2xKP458hfnOP1S53jB30FjOZS+HD6dFEUZwNKlSwesNfn+97/Pueeey7Jlyzh27BgHDx48o8306dNZvHgxAEuWLKGsrGzE9dIRi+Kfsr9ASh5MmO28n3s1xKfBgRdh9qrw6qYoY4DBRhahIjk5+dT5q6++yssvv8ybb75JUlISl112md+1KPHx8afO3W73qEyF6YhFORNjnBFL4cXQH8/ujoX886Hi7fDqpihRTGpqKi0tLX6vNTU1kZmZSVJSEvv27eOtt94KsXan0RGLcia1B6H1hONX8abgAvjL/0B3G8Ql+2+rKMqokZ2dzfLly1mwYAGJiYnk5eWdurZ69WoeeeQR5s2bx5w5c1i2bFnY9FTDopzJ0dedY79/pZ+CC8B44Ph2ZzSjKErIeeqpp/yWx8fH84c//MHvtX4/yoQJE9i1a9ep8q98ZXTWp+lUmHIm1fsgLhWyzhlYnl/sHCveCb1OiqKMG4IyLCKyWkT2i0ipiKz3cz1eRDba61tFpNDr2j22fL+IrBpKpohMtzJKrcy4wfoQkVgReVJEdorIXhG5Z7gPQ7HUH4aswtP+lX6SsyFrBhxTw6IoSmCGNCwi4gZ+BFwJFAE3i0iRT7VbgQZjzEzgIeAB27YIWAfMB1YDD4uIewiZDwAPWVkNVnbAPoAbgHhjzEJgCfB5b8OmDIOGI2eOVvopuMAZsRgTWp0URRk3BDNiWQqUGmMOG2O6gQ3AGp86a4An7fnTwEpx0mOuATYYY7qMMUeAUivPr0zbZoWVgZV53RB9GCBZRGKARKAbGNlVS9GEpw8ajkJmgH0YCoqhrdpZ06IoiuKHYAxLPnDM632FLfNbxxjTCzQB2YO0DVSeDTRaGb59BerjaaANqALKgf82xtT73oSI3C4iJSJSUlNTE8RtRylNFeDpGWTEYv0sldtCp5OiKOOKSHDeLwX6gMnAdOCfReSM/4rGmMeMMcXGmOKcnJxQ6zh+qD/sHAMZlpx5IC6o3hM6nRRFGVcEY1gqgSle7wtsmd86dkoqHagbpG2g8jogw8rw7StQH58AXjDG9BhjqoE3gOIg7kvxR8MR55gVYCosNgGyZ8JJNSyKEg5SUlIAOH78OGvXrg2zNv4JxrC8A8yy0VpxOM74zT51NgO32PO1wBZjjLHl62xE13RgFvB2IJm2zStWBlbms0P0UY7jl0FEkoFlwL5gH4DiQ/1hcMdD6uTAdXKLoHp36HRSFOUMJk+ezNNPPz10xTAwpGGx/ow7gReBvcAmY8xuEblPRK611R4HskWkFLgLWG/b7gY2AXuAF4A7jDF9gWRaWXcDd1lZ2VZ2wD5wostSRGQ3jsH6mTHm/eE9DoX6I85oxTXIVyNvATSUQZf/1BKKoow+ZWVlLFiwAIBly5axe/fpH3uXXXYZJSUltLW18bnPfY6lS5dy3nnn8eyzzwYSN6IEtfLeGPM88LxP2b1e5504Yb/+2t4P3B+MTFt+GMdv4lvutw9jTGugvpVhUH8kcERYP3k2Mrx6H0y5YPR1UpSxyB/Ww4mdIytz4kK48ltn3eymm25i06ZNfOMb36CqqoqqqiqKi4v513/9V1asWMETTzxBY2MjS5cu5fLLLx+QvHI0iATnvTJSGDP4GpZ+cvsNi06HKcpY4MYbbzw1LbZp06ZTvpeXXnqJb33rWyxevPhUtuPy8tFfKqC5wpTTtJ6EnvbAjvt+MqZBXAqcVMOiRDHDGFmMFvn5+WRnZ/P++++zceNGHnnkEQCMMTzzzDPMmTMnpProiEU5zalQ4yEMi8sFufM0MkxRxhA33XQTDz74IE1NTSxatAiAVatW8YMf/ABjM2Vs3749JLqoYVFO02Qju9OnDF4PTkeGaWoXRRkTrF27lg0bNnDjjTeeKvva175GT08PixYtYv78+Xzta18LiS46FaacpvWEc0ydNHTdvAXw7pPQUgVpg4QmK4oyorS2tgJQWFg4IAV+Xl4evb29A+omJiby6KOPhlQ/0BGL4k3LCYhNhvjUoev2R4bpdJiiKD6oYVFO01IFqXlnpsv3h0aGKYoSADUsymlaTgQ3DQaQlOWsztcRixJlmCjwK37Qe1TDogDQ3t1LVcURtlS6WP3dP/PtF4PIipNXpCHHSlSRkJBAXV1dRBsXYwx1dXUkJCQMW4Y67xUA/rj7BB/traM77SJi3MLDrx7ixuIpTMseZIVubhEc+TP09YA7NnTKKkqYKCgooKKigkjfeiMhIYGCgoJht1fDogDw4vZS1kgXVyxbzHkLL+DiB7bwszfK+Pq18wM3ylsAfd1Qdwhy54ZOWUUJE7GxsUyfPsQ6L0WnwhSob+umtPQgAK60yeSlJXDNosn8uuQYTR09gRueigzbFbiOoihRhxoWhed2VpFNg/MmJQ+AWy+eTlt3HxvfGSSv0ITZIG7d9EtRlAGoYVHYvKOSc9PanTc2KmxBfjoXTs/i/3ujjN4+j/+GMfGOcdHIMEVRvFDDEuVUNLTzTlkDl0yyK3ZT805d++zy6Rxv6uT10trAAjQyTFEUH4IyLCKyWkT2i0ipiKz3cz1eRDba61tFpNDr2j22fL+IrBpKpt1Vcqst32h3mAzYh4h8UkR2eL08IrJ4uA8k2nhhl5PGZWFaB8SlDlh1f9mcHBJiXbyyrzqwgNwiaCqHzubRVlVRlHHCkIZFRNw4uzReCRQBN4tIkU+1W4EGY8xM4CHgAdu2CGfb4fnAauBhEXEPIfMB4CErq8HKDtiHMeaXxpjFxpjFwN8CR4wxO87+UUQnOyubmJyeQFpvHaROHHAtIdbN8hkT2LK/OnDcfp6NGqveO8qaKooyXghmxLIUKDXGHDbGdAMbgDU+ddYAT9rzp4GVIiK2fIMxpssYcwQotfL8yrRtVlgZWJnXDdGHNzdbWUqQ7K1qZt6kNLvqfuIZ1z8yN5dj9R2UVrf6F9BvWDQyTFEUSzCGJR845vW+wpb5rWP3s2/C2a8+UNtA5dlAo5Xh21egPry5CfiVv5sQkdtFpERESiJ9cVOwdPX2caimjbmTUm2esDMNy4q5uQBsCTQdlj4F4tM0MkxRlFNEjPNeRC4E2o0xfn86G2MeM8YUG2OKc3JyQqzd2OTgyVb6PIZ5E1MDjlgmZyQyd2IqfwpkWER00y9FUQYQjGGpBLx3fiqwZX7riEgMkA7UDdI2UHkdkGFl+PYVqI9+1hFgtKL4Z9+JFgCKMg30dkLKmYYFYOW8XLYdbaCpPcBiybz5TmRYBOdPUhQleIIxLO8As2y0VhzOP/DNPnU2A7fY87XAFuN4ezcD62xE13RgFvB2IJm2zStWBlbms0P0gYi4gBtR/8pZsbeqmfgYF9MS2pyClDy/9VbMzaPPY3jtYIApxNwi6GqCZt/fG4qiRCNDGhbrz7gTeBHYC2wyxuwWkftE5Fpb7XEgW0RKgbuA9bbtbmATsAd4AbjDGNMXSKaVdTdwl5WVbWUH7MNyKXDMGHN4OA8hWtl3opk5E1Nxd9iBX/IEv/UWT8kgKzkucNjxKQe+TocpihJkEkpjzPPA8z5l93qddwI3BGh7P3B/MDJt+WGcqDHf8sH6eBVYNtg9KAMxxrC3qoXL5+VCW5lTGMCwuF3CZbNzeGV/NX0eg9vlE4yX65UzbPYVo6e0oijjgohx3itnR01LF/Vt3U6ocZtdWZ/k37CAE3bc0N7DjmMNZ15MzIC0Ao0MUxQFUMMStey1jvu5E9Og3U6FJflGb5/m0tk5uF3Cn/YGmA6buBCq3htpNRVFGYeoYYlS9lY5KVjmTUqFthpISIeYuID10xNjKZ6WGXg9S/4SqD0AHY2joa6iKOMINSxRyr6qZialJ5CRFOdMhSUPvbZn5bxc9p1oobKx48yLBUuc4/F3R1hTRVHGG2pYopR9J1oc/wo4I5ZB/Cv99K/C9xsdNvl851ixbaRUVBRlnKKGJQrxeAyHa9uYkWP3s2+vCxgR5s2MnBSmZiX5nw5LzHD2ZqksGWFtFUUZb6hhiUJOtnTS3etharY1LG21QRkWEWHF3FzeKK2lvbv3zAr5S6Bym67AV5QoRw1LFFJe5+wWOTUrCTweZ8QSxFQYwBXz8+jq9fDqfj+r8POXONNqjYNsZ6woSsSjhiUKOVrvGJZpWUnQ2QimLyjnPcDSwiyykuP4g90gbAAFxc5Rp8MUJapRwxKFlNe143YJ+ZmJzggDgpoKA4hxu7iiKI8te0/S2dM38GLeAohJgGPvjLDGiqKMJ9SwRCHl9e1Mzkgg1u3yWnUfeHGkL1cunERbdx+vH6wdeMEdC/nFcPSNEdRWUZTxhhqWKORofbvjXwFot8YhyKkwgIvOySYtIYbnd1WdebHwYjixUxdKKkoUo4YlCimva2NqVn9E2NlNhQHExbi4vCiPl/ecpLvXM/Bi4XLAQPlbI6OsoijjDjUsUUZzZw8N7T1My7Yjlrah84T548oFk2ju7OXNw3UDLxRcAO44OPr6CGirKMp4RA1LlDEg1BhsnrAMxz9yFlwyawLJcW7+sNNnOiw20Qk7LlM/i6JEK2pYoozyeh/D0h7c4khfEmLdrJiXx0t7TtLb5zMdNm25k+m4q+WDqqsoyjgkKMMiIqtFZL+IlIrIej/X40Vko72+VUQKva7dY8v3i8iqoWTa7Yq32vKNduviofpYJCJvishuEdkpIgnDeRjRQL9hOT0VFlwCSn9cuWAi9W3dvF1WP/BC4XJnbUz51g+iqqIo45QhDYuIuIEfAVcCRcDNIlLkU+1WoMEYMxN4CHjAti3C2c9+PrAaeFhE3EPIfAB4yMpqsLIH6yMG+F/gC8aY+cBlQM9ZPoeo4WhdO1nJcaQm2Kmvttqz9q/0c9mcHBJiXbzgu1hyyoXgilE/i6JEKcGMWJYCpcaYw8aYbmADsManzhrgSXv+NLBSRMSWbzDGdBljjgClVp5fmbbNCisDK/O6Ifq4AnjfGPMegDGmzhjjs3JP6ae8vo0p/dNgMOypMICkuBgum53LC7tO4PF45QeLS3ayHaufRVGikmAMSz5wzOt9hS3zW8cY0ws0AdmDtA1Ung00Whm+fQXqYzZgRORFEXlXRL7q7yZE5HYRKRGRkpoaP3muooTy+nYnlQuczhM2zKkwgCsXTqS6pYt3y322LC5c7uzN0t32AbRVFGU8EgnO+xjgYuCT9vg3IrLSt5Ix5jFjTLExpjgnZ/j/SMczPX0ejjd2nvavdDSA8QSdgNIfK+bmEud2nZk7bNrF4OmFY29/AI0VRRmPBGNYKoEpXu8LbJnfOtbnkQ7UDdI2UHkdkGFl+PYVqI8K4M/GmFpjTDvwPHB+EPcVdVQ2dNDnMaenwk6tuh++YUlNiOXiWRN4ac8JjHe6/KkXgrihTP0sihJtBGNY3gFm2WitOBxn/GafOpuBW+z5WmCLcf7LbAbW2Yiu6cAs4O1AMm2bV6wMrMxnh+jjRWChiCRZg/NhYE/wjyB6KPfOagzDWnXvj4/MyeFYfQdldo0MAPGpMOlczRumKFHIkIbF+jPuxPkHvhfYZIzZLSL3ici1ttrjQLaIlAJ3Aett293AJpx/9C8Adxhj+gLJtLLuBu6ysrKt7MH6aAC+g2OsdgDvGmOeG+4DiWQqGpy96guyvEKN4QNNhQFcOtuZWnxtv8/OkoXLnY2/ejo+kHxFUcYXMUNXAWPM8zhTTN5l93qddwI3BGh7P3B/MDJt+WGcqDHf8sH6+F+ckGNlECob24lxCXmp8U7BqRHLB/M5TctOpjA7iT8frOUzy6d7XbgY/voDqHgHpl/6gfpQFGX8EAnOeyVIKho6mJSRQIzbfuzt/XnCsj6w7A/PzuHNQ3UD92iZugzEpWHHihJlqGGJIioaOsjPSDxd0FY7rDxh/vjwnBw6evooKfMKO07MgIkL1c+iKFGGGpYoorKhg4JMr8WRbTUfeBqsn2XnZBPndvHaAR8/y7SLnamw3q4R6UdRlLGPGpYoobvXw8mWzoEjlva6DxwR1k9SXAxLp2fx2gGfxaeFy6G303HiK4oSFahhiRKqmjowBgoyvafCakbMsIDjZzlwspWqJq8osKkXAaJ+FkWJItSwRAmnQo0HTIXVfuBQY28umuEks3z7iFe246QsyJuvCSkVJYpQwxIlVDQ4ixdPjVg8fdBRP6IjlrkTU0mKc/PuUZ+8YdOWOyn0+zTptKJEA2pYooTKhg5cAhPT7VY1/XnCRsh5DxDjdrF4SgYlvoZl6oXQ2wEnd/tvqChKRKGGJUqoaOhgYloCsf1rWE6tuh/eXiyBKJ6Wyd6qZtq6ek8X5hc7x8qSEe1LUZSxiRqWKKGi0SfUeAQSUPrj/GmZeAy8d6zxdGHGVGdkVKGRYYoSDahhiRKcNSw+EWEwolNhAOdNzUSEgdNhIs6oRUcsihIVqGGJAnr6PFQ1dZCf6bPqHkY0KgwgPTGW2bmpbPP1sxQsgdoD0NHov6GiKBGDGpYo4ERTJ54z1rCMjo8FnOmwd8sbBm5XnL/EOR5/d8T7UxRlbKGGJQroX8OSn+HjY0nMBHdQCa7PiiXTMmnp7OVgdevpwsl27zX1syhKxKOGJQqobOxfHOkzYhnhabB+iqdlAgycDkvMgAmz1c+iKFFAUIZFRFaLyH4RKRWR9X6ux4vIRnt9q4gUel27x5bvF5FVQ8m0u0puteUb7Q6TAfsQkUIR6RCRHfb1yHAfRqRS0dCOCEzKSDhd2FY74o77fqZlJ5GdHMe75T5+lvwlcHz7qPSpKMrYYUjDIiJu4EfAlUARcLOIFPlUuxVoMMbMBB4CHrBti3C2HZ4PrAYeFhH3EDIfAB6yshqs7IB9WA4ZYxbb1xfO6glEARUNHeSmxhMf4z5d2F4LySPvXwEQERYWpLOzomnghYmLoPUktFb7b6goSkQQzIhlKVBqjDlsjOkGNgBrfOqsAZ60508DK0VEbPkGY0yXMeYIUGrl+ZVp26ywMrAyrxuiD2UIzkiXDyOaMt8fi/LTOVjdQke318ZfExc6xxPvj1q/iqKEn2AMSz5wzOt9hS3zW8fuZ9+Es199oLaByrOBRivDt69AfQBMF5HtIvKaiFzi7yZE5HYRKRGRkpqaGn9VIpaKxvaB6fI9fdBeP2o+FoAF+el4DOyp8hq1nDIsO0etX0VRwk8kOO+rgKnGmPOAu4CnRCTNt5Ix5jFjTLExpjgnZ/R+qY81+jyGqsbOgY77jgbAjPiqe28WFWQADJwOS8xwVuFX6YhFUSKZYAxLJTDF632BLfNbR0RigHSgbpC2gcrrgAwrw7cvv33YabY6AGPMNuAQMDuI+4oKTjZ30usxZ+4cCaNqWPLS4pmQEs/7lX78LDpiUZSIJhjD8g4wy0ZrxeE44zf71NkM3GLP1wJbjDHGlq+zEV3TgVnA24Fk2javWBlYmc8O1oeI5NhgAETkHNvH4eAfQWRzag1LCFbdeyMiLCpIZ9cZhmUh1JVCd9uo9a0oSngZ0rBYf8adwIvAXmCTMWa3iNwnItfaao8D2SJSijMdtd623Q1sAvYALwB3GGP6Asm0su4G7rKysq3sgH0AlwLvi8gOHKf+F4wxXjtNRTeVjT77sEBIRiwAC/PTKa1uHZjpeOIiwGgKfUWJYIJadm2MeR543qfsXq/zTuCGAG3vB+4PRqYtP4wTNeZb7rcPY8wzwDND3kSUUlHfv+reZ697GNWoMHAMi+PAb+aCwiyn0DsybMoZH7OiKBFAJDjvlUGobOxgQko8CbFea1j6p8ISs0a174UF6YCPAz+9ABIy1M+iKBGMGpYIp8I3XT44U2GJWaOSJ8ybvLQEclPj2entZxGBSYs0MkxRIhg1LBFORUP7QMc92FX3o+tf6WdRQfpAwwKOn6V6D/T1+m+kKMq4Rg1LBOPxGI77rmGBUU1A6cvC/AwO1bTSOsCBvxB6O53oMEVRIg41LBFMTWsX3X0eCjL8GJYQjVgWFqRhDOw53ny6UFO7KEpEo4Ylgulfw3JGnrAQToUtyHcc+O9XeO0cOWE2uOPVsChKhKKGJYKpaPCzhqU/T9gohxr3k5uawMS0hIF+Fncs5M7TyDBFiVDUsEQwflfdt9cDJmQ+FnDCjs904C90IsOM8d9IUZRxixqWCKaysYOs5DiS4rzCik+tuh+dvVj8sSg/ncM1bbR09pwunHQudNRD8/GQ6aEoSmhQwxLBVDR0DFxxD45/BUI2FQawwC6U3O3Xga/TYYoSaahhiWAqG9r9hxpDaKfC8v2swM+b7xzVga8oEYcalgjFGBNg1X3/iCV0hmVCSjz5GYkD/SzxqZB1jo5YFCUCUcMSodS0dtHV6wkwFSajnifMlwX5aWc68PMWqGFRlAhEDUuEUl7nhBpPy04eeKGtBhIzRz1PmC+LCjI4UttGs7cDf+IiaDgCXS0h1UVRlNFFDUuEctQalqnZPosjW6shJS/k+vQvlByw8dfEBc5R92ZRlIgiKMMiIqtFZL+IlIrIej/X40Vko72+VUQKva7dY8v3i8iqoWTaXSW32vKNdofJQfuw16eKSKuIfOVsH0IkcrS+HRHO9LG0noSU3JDrs8galu3lXivwNTJMUSKSIQ2L3fb3R8CVQBFws4gU+VS7FWgwxswEHgIesG2LcLYdng+sBh4WEfcQMh8AHrKyGqzsgH148R3gD8HeeKRTXtfG5PRE4mPcAy+EacSSmRzHjJxkth1tOF2Ylq97syhKBBLMiGUpUGqMOWyM6QY2AGt86qwBnrTnTwMrRURs+QZjTJcx5ghQauX5lWnbrLAysDKvG6IPROQ64AigcyqWo/XtTM3ymQYzxhqW0I9YAIqnZbHtaAMej11tL+KMWk7uCos+iqKMDsEYlnzgmNf7Clvmt47dz74JZ7/6QG0DlWcDjVaGb19++xCRFOBu4BuD3YSI3C4iJSJSUlNTM8Qtj3/K69qZ5utf6W6F3o6wGZYlhZk0dfRwqKb1dOHERXByj5PDTFGUiCASnPdfx5k6ax2skjHmMWNMsTGmOCcndKvOw0FrVy91bd3+HfcAyeEasWQCUOI9HTZxgWPs6g6FRSdFUUaeYAxLJTDF632BLfNbR0RigHSgbpC2gcrrgAwrw7evQH1cCDwoImXAl4F/FZE7g7iviOVoXRsA07J8Qo37DUuYRizTJySTnRxHSZm3YbEO/Kr3wqKToigjTzCG5R1glo3WisNxxm/2qbMZuMWerwW2GGOMLV9nI7qmA7OAtwPJtG1esTKwMp8drA9jzCXGmEJjTCHwXeA/jTE/PItnEHGcXsPiO2I56RzDZFhEhCXTMik5Wn+6MGeuszdL1Y6w6KQoysgzpGGx/ow7gReBvcAmY8xuEblPRK611R7H8XeUAncB623b3cAmYA/wAnCHMaYvkEwr627gLisr28oO2IdyJkfrB1nDAmGJCuunuDCTo3Xt1LR0OQXuWGfUclwNi6JECkEtvzbGPA8871N2r9d5J3BDgLb3A/cHI9OWH8aJGvMtD9iHV52vD3Y9Wjha105mUixpCbEDL7RVg7ggKXQp831ZMs1JJbPtaD2rF0xyCicvhvc2gscDrkhw+ylKdKN/xRFIeX0bU31TuYAzFZY0AVzuM6+FiAX5acTFuAb6WSYthu4WqFcHvqJEAmpYIpCjde1M813DAtBaE9ZpMID4GDeLCzLYesTLzzL5POeo02GKEhGoYYkwuns9HG/sONNxDzadS/hDrS+ZNYGdlU1Ut3Q6BTlzISYBjm8Pr2KKoowIalgijMrGDjyGM1fdQ9jSufiyYp4TlfbqfrtQ1R3jpNDXyDBFiQjUsEQYp9aw+PpYjHGc9yHckjgQRZPSmJiWwCv7qk8XTj7PWcvi8YRPMUVRRgQ1LBFGeX2ANSydjdDXPSZGLCLCR+bm8JeDtXT3WkMy+Twn5Uzt/vAqpyjKB0YNS4RxuKaNpDg3uanxAy+02mmnMC2O9OUjc3Jp7erlnTLrxJ9yoXM8tjV8SimKMiKoYYkwDtW0MiMnBZv4+TRhXnXvy/KZE4iLcbGlfzose4YTCl2uhkVRxjtqWNe0mGsAACAASURBVCKMwzVtnJMTYA0LhC0BpS/J8TEsOyf7tGERcUYtOmJRlHGPGpYIoqO7j8rGDmbkpJx5seWEc0ydGFqlBmHFnByO1LZRWm33vJ+y1Fkk2Rr52xooSiSjhiWCOFzr7Bzg37BUQUwiJGaGWKvAXLVoEm6X8OttFU7B1GXOUUctijKuUcMSQRyucUKNZ+T6mQprroS0Sc6U0xghNzWBlXNzeWZbBT19Hie1iztODYuijHPUsEQQh2paEYFCf3nCmqucPebHGOuWTqG2tZs/7T0JsQlO2LEaFkUZ16hhiSAO1bRRkJlIQqyfJJMtxyF1UuiVGoJLZ+UwMS2BDe/YnaqnLoPKd6Fr0A1BFUUZw6hhiSAOVbf69694PHbEMjn0Sg1BjNvFDcUFvHaghuONHTBjBXh64Ogb4VZNUZRhooYlQvB4DEdq2/wblvY655/1GDQsADcWT8EYnFHLlGVOkMGhLeFWS1GUYRKUYRGR1SKyX0RKReSMnRvt1sMb7fWtIlLode0eW75fRFYNJdNuV7zVlm+0WxcH7ENElorIDvt6T0T+ZrgPYzxT1dxJR0+f/zUszZXOcQxOhQFMyUri8nm5/PzNMlo9MVC4HEr/FG61FEUZJkMaFhFxAz8CrgSKgJtFpMin2q1AgzFmJvAQ8IBtW4Szn/18YDXwsIi4h5D5APCQldVgZQfsA9gFFBtjFts+HhWRoHbGjCQOVQ8Ragxj0nnfzx0fmUljew+/fOuoMx1WdxAay8OtlqIowyCYEctSoNQYc9gY0w1sANb41FkDPGnPnwZWipNTZA2wwRjTZYw5ApRaeX5l2jYrrAyszOsG68MY026M6bXlCYAJ9uYjiUM1gxiW5uPOMW1sjlgAzpuaycUzJ/CTvxyha9plTuGhV8Kqk6IowyMYw5IPHPN6X2HL/Nax/+SbgOxB2gYqzwYavQyFd1+B+kBELhSR3cBO4Ate7U8hIreLSImIlNTURN7K7sM1baQlxDAhJe7Mi83HQdxjIrPxYNy5Yia1rV1sOJIEqZOh9OVwq6QoyjCICOe9MWarMWY+cAFwj4gk+KnzmDGm2BhTnJMT/j1JRppDNa2c4y/5JDhTYSl5Yd3rPhgunJ7FBYWZPPLnw/TNWuUYlu62cKulKMpZEoxhqQSmeL0vsGV+61j/RjpQN0jbQOV1QIaXj8S7r0B9nMIYsxdoBRYEcV8RRWmgUGOwq+7HZkSYNyLCP66cRVVTJ39yL4eedjj4UrjVUhTlLAnGsLwDzLLRWnE4zvjNPnU2A7fY87XAFmOMseXrbETXdGAW8HYgmbbNK1YGVuazg/VhZcQAiMg0YC5QFvQTiAAa2rqpbuli7sRU/xWaq8a0f8Wbi2dOYMm0TL7xXjomOQd2/zbcKimKcpYMaVisv+JO4EVgL7DJGLNbRO4TkWtttceBbBEpBe4C1tu2u4FNwB7gBeAOY0xfIJlW1t3AXVZWtpUdsA/gYuA9EdkB/Bb4ojGmdniPY3yy74STHXh2IMPSMjbTufhDRPiny2dT2dzDgawVcOAlXYWvKOOMoMJyjTHPA8/7lN3rdd4J3BCg7f3A/cHItOWHcaLGfMv99mGM+QXwiyFvIoLZf6IZwP+IpasFuprH7BoWfyyfmU3xtEy+W7WAH/duhAMvwMK1QzdUFGVMEBHO+2hn/8kWMpJiz9yOGJxpMBgXPpZ+RIR/+uhsXmydTlt8Lry3IdwqKYpyFqhhiQD2nWhhTl6q/4iw/kWG6VPOvDaG+dCMbIoLJ/Crng9jSl+GhqPhVklRlCBRwzLOMcZw4ERLYMd9Y5lzzCwMlUojgojw5ctn8Xj7pRgE3n1y6EaKoowJ1LCMcyoaOmjr7gvsuG8oA3f8mF8c6Y+LZmQzZfosXpclmHd/AX094VZJUZQgUMMyztlvI8ICjlgajkLGVHCNv4+6f9TyRNdlSFs17PWNclcUZSwy/v7bKAPYf9KGGucFmgo7CpnTQqjRyPKhGRPonPoRjjIZz+vfAxOVqeAUZVyhhmWcs+9EC/kZiaQmxPqv0FA27vwrvvzD5XN4uOdqXCfegyOvhVsdRVGGQA3LOGf/iebA02AdjdDZBBnjd8QCToRY2eSrqSUDz1++G251FEUZAjUs45juXg+Ha9oCO+4bbYjuOJ4KA8fX8oXL5/PTntW4jrwCldvCrZKiKIOghmUcc7i2lV6PGdxxD+N+Kgzgstk5bM+7niZS8bzyn+FWR1GUQVDDMo7Zc7w/lUua/woNZc5xnE+FgTNquXXlIn7cczWu0pfh2NvhVklRlACoYRnH7D7eTEKsixn+9rkHZyosIR0SM0Kr2Cjx0aI83pxwPQ2SjtlyRvo5RVHGCGpYxjG7KpuYOzGNGHeAj7HhaESMVvoREW5fuZAfdl+DHHkVyt4It0qKovhBDcs4xeMx7DnezIL8ANNgEBGhxr5cuWAib2ZdR51kYbZ8U9e1KMoYRA3LOKW8vp2Wrl4WTE73X8HjcRJQjvOIMF9cLuH2FfP5XvfHkPK/6roWRRmDBGVYRGS1iOwXkVIRWe/neryIbLTXt4pIode1e2z5fhFZNZRMuyPkVlu+0e4wGbAPEfmoiGwTkZ32uGK4D2M8sds67hfkBzAsTcegrwuyZ4ZQq9BwzaJJbM28hmrJxvOn/9BRi6KMMYY0LCLiBn4EXAkUATeLSJFPtVuBBmPMTOAh4AHbtghn2+H5wGrgYRFxDyHzAeAhK6vByg7YB1ALfMwYsxBn6+Ko2PRr1/EmYt3CrLwA+9zX7HeOE+aETqkQEeN28ZWrzuXb3dfjqiyBPb8Lt0qKongRzIhlKVBqjDlsjOkGNgBrfOqsAfrzmj8NrBRnc5A1wAZjTJcx5ghQauX5lWnbrLAysDKvG6wPY8x2Y8xxW74bSBQRPzteRRa7KpuYlZtKfIzbf4Vaa1hyIs+wAFw+L5eKqWs4yFT6/vh16O0Kt0qKoliCMSz5wDGv9xW2zG8du599E85+9YHaBirPBhqtDN++AvXhzfXAu8aYM/7LiMjtIlIiIiU1NTVD3PLYxhjD7qEc9zX7IWkCJGWFTrEQIiL82zUL+Wb3zbgby+Dtn4RbJUVRLBHjvBeR+TjTY5/3d90Y85gxptgYU5yTkxNa5UaYqqZO6tu6A/tXwDEsOXNDp1QYWJCfTs55V/Oa51w8r/wXtJwIt0qKohCcYakEvPe1LbBlfuuISAyQDtQN0jZQeR2QYWX49hWoD0SkAPgt8GljzKEg7mlc0++4nx8oIswYZyosZ3YItQoP66+cy3+7bqWvpwvz4v8LtzqKohCcYXkHmGWjteJwnPG+Oy5txnGcA6wFthhjjC1fZyO6pgOzgLcDybRtXrEysDKfHawPEckAngPWG2OiYsXcrsomXALzJgXIEdZa7WQ1jkDHvS8TUuL51NUf4eHejyG7fg2HNfxYUcLNkIbF+jPuBF4E9gKbjDG7ReQ+EbnWVnscyBaRUuAuYL1tuxvYBOwBXgDuMMb0BZJpZd0N3GVlZVvZAfuwcmYC94rIDvvKHebzGBfsrGzinJwUkuJi/FeIcMe9LzcWT+Htgls4ykT6fncndLWGWyVFiWrEROEagOLiYlNSUhJuNYaFMYYl33yZlXNz+fYN5/qv9PZP4PmvwF17IW1yaBUME6XVrfy/7/+Up2K+jqv4s3DNQ+FWSVEiDhHZZowpHqpexDjvo4Wjde3Ut3Vz3tTMwJVq9kNcKqROCp1iYWZmbgoXfvgqftp7FZQ8AQdeDLdKihK1qGEZZ2w/1gDAeVMHyVjc77gXCZFWY4MvfmQGz2R8hoNSiPnt56Hx2JBtFEUZedSwjDO2lzeSHOdmdl4Ax70xcGIX5M4LrWJjgPgYN9/4+BJu6/wHuru64Nefgd7ucKulKFGHGpZxxvbyRs6dkoHbFWA00nQMOuph8nmhVWyMsOycbJYuuYC7um6HyhL4473hVklRog41LOOIju4+9lY1Dz4Ndny7c4xSwwLwr1fN462Ei/m/xGth649hz7NDN1IUZcRQwzKO2HW8iV6P4bwpgzjuj28HVwzkzg+dYmOMjKQ4vnZNEXc1rKUmfSH87g6o3htutRQlalDDMo7YXu447hcPOmLZAblFEJsQIq3GJmsWT2bZrImsa/wifTGJ8NRN0FYXbrUUJSpQwzKO2F7eyNSsJCakBEjebIwzYpm8OLSKjUFEhG9et4CKvky+nfnvTh6xDZ+Ano5wq6YoEY8alnHE9vLGwf0rjUehszGq/SveTMtO5kuXz+KRQ5m8t/RBOLYVfv1Z6OsdurGiKMNGDcs44Vh9OyeaOzl/sIWR/Y77STpi6ee2S85h7sRUvvDuFDpXPQgH/gDPfA56OsOtmqJELGpYxglvlNYC8KEZvlvQeHF8B7hiIS96Hfe+xLpd/OfHF3KiuZMHapfDqv90osR+uRba68OtnqJEJGpYxglvHKojNzWembkBtiIGZ6pn0iKIifgNNM+K86dm8qkLp/HkX8t4f8on4eM/gfK34OFlsO85xzelKMqIoYZlHGCM4c1DtXxoRjYSKE1LdztUlEDhxaFVbpzwL6vnMCElnvXP7KR3/lq4bQsk5zgO/Z+uhF3PqGNfUUYINSzjgP0nW6ht7eZDMycErlTxNnh6oPCS0Ck2jkhLiOUb185nT1UzP3ujzBnZ3fYKXP0d6GiApz8H354Fv/k8HHgJ+nrCrbKijFvUsIwD3ih11l8sH8ywHPkLiBumLguRVuOP1Qsmcvm8PL790n52HGuEmDi44Fa4swT+9ncw/zrHuf/UDfD982Dro+rkV5RhEJRhEZHVIrJfREpFZL2f6/EistFe3yoihV7X7rHl+0Vk1VAy7a6SW235RrvDZMA+RCRbRF4RkVYR+eFwH8RY5q+ltRRmJ5GfkRi4UtnrTphxfIDklAoiwoNrF5GXFs9tPy+hqslOfbncMOMjsOaH8JWDcNP/QnoB/OGr8MhyOPrX8CquKOOMIQ2LiLiBHwFXAkXAzSJS5FPtVqDBGDMTeAh4wLYtwtl2eD6wGnhYRNxDyHwAeMjKarCyA/YBdAJfA75ylvc+Lujt87D1SP3g02DdbVC5Tf0rQZCVHMfjt1xAR3cft/28hNYunzUtMfEw72PwuRfgU7+Bvm742ZXw2oPq5FeUIAlmxLIUKDXGHDbGdAMbgDU+ddYAT9rzp4GV4niZ1wAbjDFdxpgjQKmV51embbPCysDKvG6wPowxbcaY13EMTMTxXkUTrV29LJ8xiGE5pv6Vs2F2Xirfv3kxe44387EfvM7Oiib/FWeuhC++BYtuglfud9Lw69SYogxJMIYlH/DeManClvmtY/ezb8LZrz5Q20Dl2UCjleHbV6A+gkJEbheREhEpqampCbZZ2HntQA0icNFg61cObXHWr0y9MHSKjXNWzM3jqduW0dnTx8d//AYPvLCPwzWtZ1aMS4a/eRQ+eh/s+Z2z/qWrJfQKK8o4Imqc98aYx4wxxcaY4pycnHCrExTGGJ7fWcWF07PISo4LXHH/8840mPpXzopl52Tz/D9ewhXzJ/LIa4dY8T+vce0PX+c7fzxASVk9fR479SUCy7/krH85+ld48lpdXKkogxCMYakEpni9L7BlfuuISAyQDtQN0jZQeR2QYWX49hWoj4hl/8kWSqtbuXrR5MCVag5AXSnMvTp0ikUQmclx/OgT5/Pm+pX821XzcInwwy0HWfvIm6z4n1f5xZtldHT3OZUX3Qjrfgkndzt+l+bjYdVdUcYqwRiWd4BZNlorDscZv9mnzmbgFnu+FthijDG2fJ2N6JoOzALeDiTTtnnFysDKfHaIPiKW596vwiWwev7EwJX2P+cc51wZGqUilInpCdx26Tn87o7lvPu1j/K9dYudfV2e3c2K/3mVkjI7QplzJXzqGWiqhCdWQd2h8CquKGOQIQ2L9WfcCbwI7AU2GWN2i8h9InKtrfY4kC0ipcBdwHrbdjewCdgDvADcYYzpCyTTyrobuMvKyrayA/YBICJlwHeAz4hIhZ+otXGHMYbn3q9i2TnZ5KQOkqJl3/Mw6VwnPFYZETKS4lizOJ/fffFD/Oq2ZcTFuLjpsbd49LVDGGNg+iVwy2boaoUnVjs52hRFOYVE+I9+vxQXF5uSkpJwqzEou483cfX3X+c//2Yhn7hwqv9KrdXw37PhsnvgsrtDq2AU0dzZw/pn3uf5nSf45IVT+Y81C3C5BGr2wy8+Dm01cOW3YMlnHX+MokQoIrLNGFM8VL2ocd6PN557vwq3S1g1Py9wpV2/AQzMuyZkekUjaQmx/OgT5/P3l83gl1vL+bff7cTjMZAzBz7/mhM48ft/gic/Bid2hVtdRQk7MUNXUUJNb5+HZ3cc50MzsskOtFskwI7/dabBNE3+qCMifHXVHNwi/PCVUjwe+K+PL8SVPAE++TSUPO6sdXlkOUz/MJx7M5zzYUgbJPBCUSIUNSxjkOd2VlHZ2MG/f2wQV1HV+3BiJ1z57dApFuWICP98xWxcAt/fUorHGL51/SLcLhcsvQ0WXA/v/BS2/wJ+9wWnUfpUJ+HlxEXOcepFkDjILqCKEgGoYRljGGN45LXDzMxN4fJ5g0yD7fgluONg4drAdZQRR0S464o5uFzCd18+iMfAg2sX4XYJJGXBh78Kl3wFTrzv5G+r3Oac73sOMOCKgWkfguJbndQxLne4b0lRRhw1LGOM1w7UsLeqmQfXLnIcxP7o6YT3N8Kcq5x/ZkrI+fLlsxGEh14+gDGGb99wrmNcAFwumLzYefXT1QpV70HpH2H3b+HXt0DmdFj9XxoqrkQc6rwfYzz62mEmpiVw3WLfrDlevPeUs4dI8WdDp5hyBl+6fBZfuWI2v9leyV2bdtDb5wlcOT4FCpfD5V+Hf3gXbvyFk/DyV+tgwyedCD9FiRDUsIwh3jpcx5uH67j14unExQT4aPp64Y3vweTzHSexElbuXDGLf1k1h2d3HOeTP91KeV370I1cbii6Fj7/F7j8G3Dwj842yXueHbqtoowD1LCMETq6+7j7mfeZkpXIJ5cFWLcCTiLEhjK45J91zcQY4Y6PzOTbaxex+3gzq7/3Zx597RC1rV1DN4yJg4u/DJ//M6RPgU2fht/cDh2No6+0oowiukByjHDf/+3hiTeO8KvblgXOZNzXC49eAsYDf/+mM5evjBkqGzu45zc7+fOBGtwu4aJzspmWnURmUhwugZauXlo7e2nr7qW1q4/UhBjyUhOYnpPMJdPTmbbnx8ifvw0pec6mYzNXhvuWFGUAwS6QVOf9GOCdsnp+9tcjfPqiaYOnx3/np1C9B278uRqVMUh+RiI//9xS9p1oZvOO4/xpbzV7qpppbO/GAClxMaQkxJASH0NSnJuK+nZeaa6m3Sa5nJq1jC8sfIIbK75JzP9+3AnOuPRfIP/88N6YopwlOmIJM4drWrnx0TdJjHPzwpcuJTk+gK1vOQE/KIYpS50kiDoNNm7w2PT7gaL8ymrb+MvBGl7ac5LXS2tJoJv/ynuVq9t+Q2xPM+TMdYxMwQWQOw9SJ0FsQihvQVGA4EcsaljCSGVjBzf8+K909XrY9IWLmJGT4r+iMbDxU3DwJWdHw+wZoVVUCRnH6tv51dvlbCqpoLO1gc+mbGVd0jYmt+xAjFfUWWKWs6o/MdNZcJmQDgkZzqv/fdpkyJ7pTK3pDxFlBNCpsDFOaXUrf/fkO7R09bLh9mWBjQrA6w/Bvt/DR/9DjUqEMyUria+unsuXL5/Ny3tP8tTWqfyg9CMku7q5flIdK3JbmJfcygRTh7vlhBN2XlsKnY3Q2QQ9fqLSUibC1GWOz2bWKkgdZOGtoowAOmIJA8/vrOJffv0eCbFuHvt0MUumZQauvPf3zmhlwcfh+sf1l2cUUlbbxq+3HeNPe6vZd8LZFjnO7WJWXgpFk9KYPzmNORPTmJmbwoREkM5mx9A0HYPag1BRAkffgGa7Z97k82H2api9ysk1p98pJUh0KmwQwmVYalq6ePCFffx6WwWLp2Tw40+dz6T0xMAN3v0F/N+XnD/+zzwHcUmhU1YZk1Q2dlBSVs+eqmb2HHdedW3dp65nJMUyKzeFmbkpzMxNPXU+KS0eqd4DB15wXhUlgHH8NbNXOf6b7FnO1FnyIAEkSlSjhmUQQm1Ymjt7eGprOT/aUkpnbx+3XnwO//TRWcTHBMgT1dEAf/x3ePdJmLHCWaUdP8hUmRK1GGOobuniwMkWDp5spbSmldKTrRysbqGhvedUvZT4GGbkpjAzJ4VZeSnMTOpgZvOb5FW9QkL5a0h362mhCRmQWXj6lT3TSaCZM89Ze6NELSNqWERkNfA9wA381BjzLZ/r8cDPgSU4+9DfZIwps9fuAW4F+oB/NMa8OJhMu4XxBpzdI7cBf2uM6R5OH4EIhWHp7fPwbnkjz71/nKe3VdDW3cdlc3L42jVFgf0pzcdh25NOWHFHA1z0RVhxr/4xK8OirrWLg9WtHKxupfRkC6U1rRw82Up1y8DFmzHSx6LkJs5NqmVubDXTpYq8vhNkdVeR3FGJy2MNlCvWiUqbdK7zmnyes2VD7CCjbj/09nlo6eylvaePGJcQ53YRF+O8YlyC6NTcmGXEDIuIuIEDwEeBCpz96m82xuzxqvNFYJEx5gsisg74G2PMTXaL4F8BS4HJwMvAbNvMr0wR2QT8xhizQUQeAd4zxvz4bPswxvQFuqcPaliMMXT3eejq9dDV46G9u5eali5ONju/HPdWNfNOWT0N7T3EuV1cs2gin/vQVBZMTIK+bsdotNdBez00lkPNPjj6JpzcCQjMvBxW3uv8SlSUEaapo4fKhg5ONHdwoqmLE82dnGjq4ERzFyebOqlq6qC5sxcAFx6mykkWSBnnxZVzrvsosz2HSTPNAHhwU5s4nZqkc2iMm0RTXB6NcRNpII263gRquuOo6Y6hqdPQ1NlHY5eH9u7+6DaDGw+x9BJLH7H0kiA9ZMf1kRXbQ0ZsDxkxPaS5e0iL6SFFukmgk1jTg8cVS68rjl6Jp0vi6JBEOiSBTkmknQTaSaKdBDqIo9cI8bGxpCfFkZYQS1pijD3Gkp4YS0q8XVsU7z61xijW7cLtEtwigZPBniV9HkNnTx8dPX10dPfR2dNHa1cvLZ29NHf20NLZS0tnD80dztG7PMYtJMbGkBzvJinOTVKccw/piTGkJzn3kZ4YZ4+xp4y02yWnjiNhsEcyKmwpUGqMOWwFbwDW4Oxj388a4Ov2/Gngh+LcxRpggzGmCzhi96tfauudIVNE9gIrgE/YOk9auT8eRh9vBnFvZ8V7xxq54dE36e4NnGzQJTB9QjJPJv+A+byFy/Qhe/tg7yCC41KcX38r74Wi6zTySxlV+v/5FE1OC1invbuXk81dVDV1cLK5kxNNXZQ3dbCttYum9m7i2o4zsf0Ahd0HmNN2hJnt25nNn4iVgL/nHFxgEgQjLsR4EAL8sO21rxGkDxceI/Th4se913Jf3/VBtz1tZEAQDAZjcLQ3YOx99JcZY+zx7PV0CaQmxJKaEOMc42Po7PFQ19pOR08f7d19tHf10tY9xLP2dw8u4ZqFk/jOTYuHbvABCMaw5APHvN5XABcGqmOM6RWRJpyprHzgLZ+2/Wl7/cnMBhqNMb1+6g+nj1OIyO3A7fZtq4jsD3zLZ80EoLb/zRHg3LNq3gwcB54DvjIqOo0hxqJeY1EnGJt6jUWd4Kz1+pl9jSpj8lk9BBMeWjdsvaYFUylq1rEYYx4DHhsN2SJSEszwMJSMRZ1gbOo1FnWCsanXWNQJxqZeY1EnCI1ewSScqgSmeL0vsGV+64hIDJCO42AP1DZQeR2QYWX49nW2fSiKoihhIBjD8g4wS0Smi0gcsA7Y7FNnM3CLPV8LbDFOVMBmYJ2IxNtor1nA24Fk2javWBlYmc8Osw9FURQlDAw5FWb9GXcCL+KEBj9hjNktIvcBJcaYzcDjwC+s47wex1Bg623CcfT3Anf0R2v5k2m7vBvYICLfBLZb2QynjxAyKlNsH5CxqBOMTb3Gok4wNvUaizrB2NRrLOoEIdArKhdIKoqiKKOHbuqhKIqijChqWBRFUZSRxRijr2G+gNXAfqAUWD+Ccp8AqoFdXmVZwB+Bg/aYacsF+L7V4X3gfK82t9j6B4FbvMqXADttm+9zekrUbx/22hScwIo9wG7gS+HWC0jACdR4z+r0DVs+Hdhq5WwE4mx5vH1faq8XevV9jy3fD6wa6jMO1IfXdTeOj/D3Y0inMvt8d+D4R8P6+dlrGTgLnvfhLCO+aAzoNMc+o/5XM/DlMaDXP+F8z3fhZBtJCPSZE8Lvld//YaH4BxyJL5x/HIeAc4A4nH9uRSMk+1LgfAYalgf7P2xgPfCAPb8K+IP9ci8Dtnp9QQ/bY6Y97/9DeNvWFdv2ysH6sO8n9f/BAKk4KXmKwqmXrZdiz2Ptl38ZsAlYZ8sfAf7enn8ReMSerwM22vMi+/nF2z+iQ/bzDfgZB+rD63ndBTzFacMyFnQqAyb4lIX7e/Uk8Hf2PA7H0IRVJz9/5ydwFgaG87uej7P2OtHrs/5MoM+cEH6v/D63UP9DjpQXzi+rF73e3wPcM4LyCxloWPYDk+z5JGC/PX8UJ8/agHrAzcCjXuWP2rJJwD6v8lP1AvURQL9ncXK9jQm9gCTgXZwMDrVAjO/nhBOFeJE9j7H1xPez668X6DO2bfz2Yd8XAH/CSU/0+8Hqh0onW1bGmYYlbJ8fzlq0I9hf62NBJz/fqyuAN8KtF6czj2TZ78nvgVWBPnNC+L3y91Ify/Dxl+rmjFQyI0ieMabKnp8A+rcBDKTHYOUVfsoH62MAIlIInIczQgirXiLiFpEdOFOHf8T51RVUWiDAOy3Q2eg6WOohgO8CXwX6k8oFnapoFHUCJ43VSyKyzaY4gvB+ftOBGuBnIrJdRH4qIslhxWjK0wAAArFJREFU1smXdTjTToO1GXW9jDGVwH8D5UAVzvdkG2Pje3UGaljGIcb56WDC0YeIpADPAF82xqa4DaNexpg+Y8xinFHCUmDuaPY/FCJyDVBtjNkWTj0CcLEx5nzgSuAOEbnU+2IYPr8YnCnfHxtjzgPacKZ/wqnTKezi7WuBXwfbZrT0EpFMnIS703GyuCfj+ETGJGpYhk+oU8mcFJFJAPZYPYQeg5UX+CkfrA9sWSyOUfmlMeY3Y0UvAGNMI05wwUWMXFqg4aQeWg5cKyJlOPsKrcDZdyicOvU/o0p7rAZ+i2OIw/n5VQAVxpit9v3TOIZmTHyncAzwu8aYk0O0CYVelwNHjDE1xpge4Dc437Wwf6/8oYZl+AST6mYk8U5pcwsDU918WhyWAU12KP0icIWIZNpfO1fgzI1WAc0issxuO/Bp/KfN8e4DW/dxYK8x5jtjQS8RyRGRDHueiOPz2cvIpQU669RDxph7jDEFxphCW3+LMeaT4dTJPp9kEUntP7fPfVc4Pz9jzAngmIjMsddW4kQdhvW77sXNnJ4GG6xNKPQqB5aJSJJt0/+swvq9CshQThh9DepgvwonOuoQ8G8jKPdXOPOoPTi/6m7Fmev8E04Y4stAlq0rwI+sDjuBYi85n8MJESwFPutVXozzT+UQ8ENOhzr67cNeuxhnWP4+p8MwrwqnXsAinJDe9227e235OfaPpRRnGiPelifY96X2+jleff+b7Xc/NkJnsM84UB8+n+NlnI4KC6tO9tp7nA7N/rchnm2ovleLgRL7Gf4OJ3oqrDrZ68k4v9bTvcrC/ay+gROWvQv4BU5k15j4rvu+NKWLoiiKMqLoVJiiKIoyoqhhURRFUUYUNSyKoijKiKKGRVEURRlR1LAoiqIoI4oaFkVRFGVEUcOiKIqijCj/P+Th5VpVjizUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "sns.kdeplot(train_data['SalePrice'], ax=ax, label='train')\n", + "sns.kdeplot(live_data['SalePrice'], ax=ax, label='live')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As in the live environment, we are getting houses with bigger LotArea, the predictions naturally provide bigger Sale Prices. \n", + "\n", + "Does this mean that the model will not perform well? Unclear at this stage. More expensive prices are expected for bigger houses, so in principle that should not worry us too much.\n", + "\n", + "But we should definitely investigate the reasons behind the distribution changes in the input variables.\n", + "\n", + "I hope we could give you a flavour of what we should be looking at in shadow mode, and how we should be thinking or reacting when things do not go as planned." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + }, + "toc": { + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "toc_cell": false, + "toc_position": {}, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/exercise_notebooks/shadow_mode_exercise/requirements.txt b/exercise_notebooks/shadow_mode_exercise/requirements.txt new file mode 100644 index 0000000..0e55c35 --- /dev/null +++ b/exercise_notebooks/shadow_mode_exercise/requirements.txt @@ -0,0 +1,15 @@ +# ML requirements +numpy>=1.18.1,<1.19.0 +scikit-learn>=0.22.1,<0.23.0 +pandas>=0.25.3,<0.26.0 +feature_engine>=0.3.1,<0.4.0 +joblib>=0.14.1,<0.15.0 +matplotlib>=3.1.3,<3.2.0 +seaborn>=0.10.0,<0.11.0 +jupyter>=1.0.0,<1.1.0 + +# Persistence +sqlalchemy>=1.3.11,<1.4.0 # ORM +psycopg2>=2.8.4,<2.9.0 # DB Driver +alembic>=1.3.1,<1.4.0 # DB Migrations +sqlalchemy_utils>=0.36.0,<0.37.0 # DB Utils \ No newline at end of file diff --git a/packages/gradient_boosting_model/mypy.ini b/packages/gradient_boosting_model/mypy.ini new file mode 100644 index 0000000..97e52a5 --- /dev/null +++ b/packages/gradient_boosting_model/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +warn_unused_ignores = True +follow_imports = skip +show_error_context = True +warn_incomplete_stub = True +ignore_missing_imports = True +check_untyped_defs = True +cache_dir = /dev/null +warn_redundant_casts = True +warn_unused_configs = True +strict_optional = True diff --git a/packages/gradient_boosting_model/test_requirements.txt b/packages/gradient_boosting_model/test_requirements.txt new file mode 100644 index 0000000..c44378e --- /dev/null +++ b/packages/gradient_boosting_model/test_requirements.txt @@ -0,0 +1,16 @@ +-r requirements.txt + +# testing requirements +pytest>=5.3.2,<6.0.0 + +# old model for testing purposes +# source code: https://github.com/trainindata/deploying-machine-learning-models/tree/master/packages/regression_model +tid-regression-model>=2.0.20,<2.1.0 + +# repo maintenance tooling +black>=19.10b0,<20.0 +flake8>=3.7.9,<4.0 +mypy>=0.740 + +# kaggle cli +kaggle>=1.5.6,<1.6.0 diff --git a/packages/gradient_boosting_model/tests/conftest.py b/packages/gradient_boosting_model/tests/conftest.py new file mode 100644 index 0000000..c896d72 --- /dev/null +++ b/packages/gradient_boosting_model/tests/conftest.py @@ -0,0 +1,34 @@ +import pytest +from sklearn.model_selection import train_test_split + +from gradient_boosting_model.config.core import config +from gradient_boosting_model.processing.data_management import load_dataset + + +@pytest.fixture(scope="session") +def pipeline_inputs(): + # For larger datasets, here we would use a testing sub-sample. + data = load_dataset(file_name=config.app_config.training_data_file) + + # Divide train and test + X_train, X_test, y_train, y_test = train_test_split( + data[config.model_config.features], # predictors + data[config.model_config.target], + test_size=config.model_config.test_size, + # we are setting the random seed here + # for reproducibility + random_state=config.model_config.random_state, + ) + + return X_train, X_test, y_train, y_test + + +@pytest.fixture() +def raw_training_data(): + # For larger datasets, here we would use a testing sub-sample. + return load_dataset(file_name=config.app_config.training_data_file) + + +@pytest.fixture() +def sample_input_data(): + return load_dataset(file_name=config.app_config.test_data_file) diff --git a/packages/gradient_boosting_model/tests/test_config.py b/packages/gradient_boosting_model/tests/test_config.py new file mode 100644 index 0000000..5a82241 --- /dev/null +++ b/packages/gradient_boosting_model/tests/test_config.py @@ -0,0 +1,124 @@ +from pathlib import Path + +from gradient_boosting_model.config.core import ( + create_and_validate_config, + fetch_config_from_yaml, +) + +import pytest +from pydantic import ValidationError + + +TEST_CONFIG_TEXT = """ +package_name: gradient_boosting_model +training_data_file: houseprice.csv +test_data_file: test.csv +drop_features: YrSold +pipeline_name: gb_regression +pipeline_save_file: gb_regression_output_v +target: SalePrice +variables_to_rename: + foo: bar +test_size: 0.1 +features: + - LotArea +numerical_vars: + - LotArea +categorical_vars: + - BsmtQual +temporal_vars: YearRemodAdd +numerical_vars_with_na: + - LotFrontage +numerical_na_not_allowed: + - LotArea +random_state: 0 +n_estimators: 50 +rare_label_tol: 0.01 +rare_label_n_categories: 5 +loss: ls +allowed_loss_functions: + - ls + - huber +""" + +INVALID_TEST_CONFIG_TEXT = """ +package_name: gradient_boosting_model +training_data_file: houseprice.csv +test_data_file: test.csv +drop_features: YrSold +pipeline_name: gb_regression +pipeline_save_file: gb_regression_output_v +target: SalePrice +features: + - LotArea +numerical_vars: + - LotArea +categorical_vars: + - BsmtQual +temporal_vars: YearRemodAdd +numerical_vars_with_na: + - LotFrontage +numerical_na_not_allowed: + - LotArea +random_state: 0 +n_estimators: 50 +rare_label_tol: 0.01 +rare_label_n_categories: 5 +loss: ls +allowed_loss_functions: + - huber +""" + + +def test_fetch_config_structure(tmpdir): + # Given + # We make use of the pytest built-in tmpdir fixture + configs_dir = Path(tmpdir) + config_1 = configs_dir / "sample_config.yml" + config_1.write_text(TEST_CONFIG_TEXT) + parsed_config = fetch_config_from_yaml(cfg_path=config_1) + + # When + config = create_and_validate_config(parsed_config=parsed_config) + + # Then + assert config.model_config + assert config.app_config + + +def test_config_validation_raises_error_for_invalid_config(tmpdir): + # Given + # We make use of the pytest built-in tmpdir fixture + configs_dir = Path(tmpdir) + config_1 = configs_dir / "sample_config.yml" + + # invalid config attempts to set a prohibited loss + # function which we validate against an allowed set of + # loss function parameters. + config_1.write_text(INVALID_TEST_CONFIG_TEXT) + parsed_config = fetch_config_from_yaml(cfg_path=config_1) + + # When + with pytest.raises(ValidationError) as excinfo: + create_and_validate_config(parsed_config=parsed_config) + + # Then + assert "not in the allowed set" in str(excinfo.value) + + +def test_missing_config_field_raises_validation_error(tmpdir): + # Given + # We make use of the pytest built-in tmpdir fixture + configs_dir = Path(tmpdir) + config_1 = configs_dir / "sample_config.yml" + TEST_CONFIG_TEXT = """package_name: gradient_boosting_model""" + config_1.write_text(TEST_CONFIG_TEXT) + parsed_config = fetch_config_from_yaml(cfg_path=config_1) + + # When + with pytest.raises(ValidationError) as excinfo: + create_and_validate_config(parsed_config=parsed_config) + + # Then + assert "field required" in str(excinfo.value) + assert "pipeline_name" in str(excinfo.value) diff --git a/packages/gradient_boosting_model/tests/test_pipeline.py b/packages/gradient_boosting_model/tests/test_pipeline.py new file mode 100644 index 0000000..3820995 --- /dev/null +++ b/packages/gradient_boosting_model/tests/test_pipeline.py @@ -0,0 +1,54 @@ +from gradient_boosting_model import pipeline +from gradient_boosting_model.config.core import config +from gradient_boosting_model.processing.validation import validate_inputs + + +def test_pipeline_drops_unnecessary_features(pipeline_inputs): + # Given + X_train, X_test, y_train, y_test = pipeline_inputs + assert config.model_config.drop_features in X_train.columns + + # When + # We use the scikit-learn Pipeline private method `_fit` which is called + # by the `fit` method, since this allows us to access the transformed + # dataframe. For other models we could use the `transform` method, but + # the GradientBoostingRegressor does not have a `transform` method. + X_transformed, _ = pipeline.price_pipe._fit(X_train, y_train) + + # Then + assert config.model_config.drop_features in X_train.columns + assert config.model_config.drop_features not in X_transformed.columns + + +def test_pipeline_transforms_temporal_features(pipeline_inputs): + # Given + X_train, X_test, y_train, y_test = pipeline_inputs + + # When + # We use the scikit-learn Pipeline private method `_fit` which is called + # by the `fit` method, since this allows us to access the transformed + # dataframe. For other models we could use the `transform` method, but + # the GradientBoostingRegressor does not have a `transform` method. + X_transformed, _ = pipeline.price_pipe._fit(X_train, y_train) + + # Then + assert ( + X_transformed.iloc[0]["YearRemodAdd"] + == X_train.iloc[0]["YrSold"] - X_train.iloc[0]["YearRemodAdd"] + ) + + +def test_pipeline_predict_takes_validated_input(pipeline_inputs, sample_input_data): + # Given + X_train, X_test, y_train, y_test = pipeline_inputs + pipeline.price_pipe.fit(X_train, y_train) + + # When + validated_inputs, errors = validate_inputs(input_data=sample_input_data) + predictions = pipeline.price_pipe.predict( + validated_inputs[config.model_config.features] + ) + + # Then + assert predictions is not None + assert errors is None diff --git a/packages/gradient_boosting_model/tests/test_predict.py b/packages/gradient_boosting_model/tests/test_predict.py new file mode 100644 index 0000000..5be393e --- /dev/null +++ b/packages/gradient_boosting_model/tests/test_predict.py @@ -0,0 +1,62 @@ +from gradient_boosting_model.predict import make_prediction +from gradient_boosting_model.config.core import config + +from sklearn.metrics import mean_squared_error + +from regression_model.predict import make_prediction as alt_make_prediction + + +def test_prediction_quality_against_benchmark(raw_training_data, sample_input_data): + # Given + input_df = raw_training_data.drop(config.model_config.target, axis=1) + output_df = raw_training_data[config.model_config.target] + + # Generate rough benchmarks (you would tweak depending on your model) + benchmark_flexibility = 50000 + # setting ndigits to -4 will round the value to the nearest 10,000 i.e. 210,000 + benchmark_lower_boundary = ( + round(output_df.iloc[0], ndigits=-4) - benchmark_flexibility + ) # 210,000 - 50000 = 160000 + benchmark_upper_boundary = ( + round(output_df.iloc[0], ndigits=-4) + benchmark_flexibility + ) # 210000 + 50000 = 260000 + + # When + subject = make_prediction(input_data=input_df[0:1]) + + # Then + assert subject is not None + prediction = subject.get("predictions")[0] + assert isinstance(prediction, float) + assert prediction > benchmark_lower_boundary + assert prediction < benchmark_upper_boundary + + +def test_prediction_quality_against_another_model(raw_training_data, sample_input_data): + # Given + input_df = raw_training_data.drop(config.model_config.target, axis=1) + output_df = raw_training_data[config.model_config.target] + current_predictions = make_prediction(input_data=input_df) + + # the older model has these variable names reversed + input_df.rename( + columns={ + "FirstFlrSF": "1stFlrSF", + "SecondFlrSF": "2ndFlrSF", + "ThreeSsnPortch": "3SsnPorch", + }, + inplace=True, + ) + alternative_predictions = alt_make_prediction(input_data=input_df) + + # When + current_mse = mean_squared_error( + y_true=output_df.values, y_pred=current_predictions["predictions"] + ) + + alternative_mse = mean_squared_error( + y_true=output_df.values, y_pred=alternative_predictions["predictions"] + ) + + # Then + assert current_mse < alternative_mse diff --git a/packages/gradient_boosting_model/tests/test_preprocessors.py b/packages/gradient_boosting_model/tests/test_preprocessors.py new file mode 100644 index 0000000..11a4900 --- /dev/null +++ b/packages/gradient_boosting_model/tests/test_preprocessors.py @@ -0,0 +1,37 @@ +from gradient_boosting_model.config.core import config +from gradient_boosting_model.processing import preprocessors as pp + + +def test_drop_unnecessary_features_transformer(pipeline_inputs): + # Given + X_train, X_test, y_train, y_test = pipeline_inputs + assert config.model_config.drop_features in X_train.columns + + transformer = pp.DropUnecessaryFeatures( + variables_to_drop=config.model_config.drop_features, + ) + + # When + X_transformed = transformer.transform(X_train) + + # Then + assert config.model_config.drop_features not in X_transformed.columns + + +def test_temporal_variable_estimator(pipeline_inputs): + # Given + X_train, X_test, y_train, y_test = pipeline_inputs + + transformer = pp.TemporalVariableEstimator( + variables=config.model_config.temporal_vars, + reference_variable=config.model_config.drop_features, + ) + + # When + X_transformed = transformer.transform(X_train) + + # Then + assert ( + X_transformed.iloc[0]["YearRemodAdd"] + == X_train.iloc[0]["YrSold"] - X_train.iloc[0]["YearRemodAdd"] + ) diff --git a/packages/gradient_boosting_model/tests/test_validation.py b/packages/gradient_boosting_model/tests/test_validation.py new file mode 100644 index 0000000..b636674 --- /dev/null +++ b/packages/gradient_boosting_model/tests/test_validation.py @@ -0,0 +1,30 @@ +from gradient_boosting_model.processing.validation import validate_inputs + + +def test_validate_inputs(sample_input_data): + # When + validated_inputs, errors = validate_inputs(input_data=sample_input_data) + + # Then + assert not errors + + # we expect that 2 rows are removed due to missing vars + # 1459 is the total number of rows in the test data set (test.csv) + # and 1457 number returned after 2 rows are filtered out. + assert len(sample_input_data) == 1459 + assert len(validated_inputs) == 1457 + + +def test_validate_inputs_identifies_errors(sample_input_data): + # Given + test_inputs = sample_input_data.copy() + + # introduce errors + test_inputs.at[1, "BldgType"] = 50 # we expect a string + + # When + validated_inputs, errors = validate_inputs(input_data=test_inputs) + + # Then + assert errors + assert errors[1] == {"BldgType": ["Not a valid string."]} diff --git a/packages/gradient_boosting_model/tox.ini b/packages/gradient_boosting_model/tox.ini new file mode 100644 index 0000000..7c9059d --- /dev/null +++ b/packages/gradient_boosting_model/tox.ini @@ -0,0 +1,65 @@ +[tox] +envlist = unit_tests,typechecks,stylechecks +skipsdist = True + + +[testenv] +install_command = pip install {opts} {packages} +deps = + -rtest_requirements.txt + +passenv = + KAGGLE_USERNAME + KAGGLE_KEY + +setenv = + PYTHONPATH=. + +commands= + kaggle competitions download -c house-prices-advanced-regression-techniques -p gradient_boosting_model/datasets/ + unzip -o gradient_boosting_model/datasets/house-prices-advanced-regression-techniques.zip -d gradient_boosting_model/datasets + mv gradient_boosting_model/datasets/train.csv gradient_boosting_model/datasets/houseprice.csv + python gradient_boosting_model/train_pipeline.py + pytest \ + -s \ + -vv \ + {posargs:tests/} + + +[testenv:unit_tests] +envdir = {toxworkdir}/unit_tests +deps = + {[testenv]deps} + +setenv = + PYTHONPATH=. + +commands = + python gradient_boosting_model/train_pipeline.py + pytest \ + -s \ + -vv \ + {posargs:tests/} + + +[testenv:typechecks] +envdir = {toxworkdir}/unit_tests + +deps = + {[testenv:unit_tests]deps} + +commands = {posargs:mypy gradient_boosting_model} + + +[testenv:stylechecks] +envdir = {toxworkdir}/unit_tests + +deps = + {[testenv:unit_tests]deps} + +commands = {posargs:flake8 gradient_boosting_model tests} + + +[flake8] +exclude = .git,env +max-line-length = 90 \ No newline at end of file diff --git a/packages/ml_api/.dockerignore b/packages/ml_api/.dockerignore new file mode 100644 index 0000000..26ab026 --- /dev/null +++ b/packages/ml_api/.dockerignore @@ -0,0 +1,20 @@ +exercise_notebooks/* +*env* +*venv* +.circleci* +packages/gradient_boosting_model +*.env +*.log +.git +.gitignore +.dockerignore +*.mypy_cache +*.pytest_cache +*.tox + +# alembic +!alembic/env.py + +# Byte-compiled / optimized / DLL files +*__pycache__* +*.py[cod] \ No newline at end of file diff --git a/packages/ml_api/Makefile b/packages/ml_api/Makefile new file mode 100644 index 0000000..a544913 --- /dev/null +++ b/packages/ml_api/Makefile @@ -0,0 +1,40 @@ +# For details on Makefiles, see the section notes. +NAME=ml_api +VERSION=$(shell git rev-parse HEAD) +REPO=UPDATEME +PASSWORD=UPDATEME + +# Specify phony list to ensure make recipes do not conflict with real file names +.PHONY: run-service-development tag-push-master tag-push-local db-migrations + + +tag-push-local: + @echo "+ $@" + docker login --username $(REPO) --password $(PASSWORD) + env TARGET=$(VERSION) docker-compose -f docker/docker-compose-ci-candidate.yml build + docker push $(REPO)/$(NAME):$(VERSION) + +tag-push-master: + @echo "+ $@" + docker login --username $(REPO) --password $(PASSWORD) + env TARGET=master docker-compose -f docker/docker-compose-ci-master.yml build + docker push $(REPO)/$(NAME):master + +# start up Flask API service +run-service-development: + @echo "+ $@" + python run.py + +run-service-wsgi: + @echo "+ $@" + gunicorn --bind 0.0.0.0:5000 \ + --workers=1 \ + --log-config gunicorn_logging.conf \ + --log-level=DEBUG \ + --access-logfile=- \ + --error-logfile=- \ + run:application + +db-migrations: + @echo "+ $@" + PYTHONPATH=. alembic -c alembic.ini upgrade head diff --git a/packages/ml_api/__init__.py b/packages/ml_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/alembic.ini b/packages/ml_api/alembic.ini new file mode 100644 index 0000000..604e701 --- /dev/null +++ b/packages/ml_api/alembic.ini @@ -0,0 +1,49 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +timezone = UTC + +sqlalchemy.url = VALUE_IS_SET_AT_RUNTIME + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/packages/ml_api/alembic/env.py b/packages/ml_api/alembic/env.py new file mode 100644 index 0000000..377cb2e --- /dev/null +++ b/packages/ml_api/alembic/env.py @@ -0,0 +1,63 @@ +import os + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Import the models so the changes in them are automatically reflected in the +# generated migrations. +from api.persistence import models # noqa +from api.config import DevelopmentConfig as user_config +from api.persistence.core import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +database_url = os.environ.get("ALEMBIC_DB_URI", user_config.SQLALCHEMY_DATABASE_URI) +config.set_main_option("sqlalchemy.url", database_url) + +# add your model's MetaData object here +# for 'autogenerate' support +target_metadata = Base.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + This configures the context with just a URL + and not a user_ratings, though a user_ratings is acceptable + here as well. By skipping the user_ratings creation + we don't even need a DBAPI to be available. + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + In this scenario we need to create a user_ratings + and associate a connection with the context. + """ + alembic_config = config.get_section(config.config_ini_section) + connectable = engine_from_config( + alembic_config, prefix="sqlalchemy.", poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/packages/ml_api/alembic/script.py.mako b/packages/ml_api/alembic/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/packages/ml_api/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/packages/ml_api/alembic/versions/cf4abb13368d_create_prediction_tables.py b/packages/ml_api/alembic/versions/cf4abb13368d_create_prediction_tables.py new file mode 100644 index 0000000..a26fb19 --- /dev/null +++ b/packages/ml_api/alembic/versions/cf4abb13368d_create_prediction_tables.py @@ -0,0 +1,78 @@ +"""create prediction tables + +Revision ID: cf4abb13368d +Revises: +Create Date: 2019-12-15 14:54:07.857500+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "cf4abb13368d" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "gradient_boosting_model_predictions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column( + "datetime_captured", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("model_version", sa.String(length=36), nullable=False), + sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_gradient_boosting_model_predictions_datetime_captured"), + "gradient_boosting_model_predictions", + ["datetime_captured"], + unique=False, + ) + op.create_table( + "regression_model_predictions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column( + "datetime_captured", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("model_version", sa.String(length=36), nullable=False), + sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_regression_model_predictions_datetime_captured"), + "regression_model_predictions", + ["datetime_captured"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_regression_model_predictions_datetime_captured"), + table_name="regression_model_predictions", + ) + op.drop_table("regression_model_predictions") + op.drop_index( + op.f("ix_gradient_boosting_model_predictions_datetime_captured"), + table_name="gradient_boosting_model_predictions", + ) + op.drop_table("gradient_boosting_model_predictions") + # ### end Alembic commands ### diff --git a/packages/ml_api/api/__init__.py b/packages/ml_api/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/api/app.py b/packages/ml_api/api/app.py new file mode 100644 index 0000000..f8f74e0 --- /dev/null +++ b/packages/ml_api/api/app.py @@ -0,0 +1,33 @@ +import logging + +import connexion +from sqlalchemy.orm import scoped_session + +from api.config import Config +from api.monitoring.middleware import setup_metrics +from api.persistence.core import init_database + +_logger = logging.getLogger('mlapi') + + +def create_app( + *, config_object: Config, db_session: scoped_session = None +) -> connexion.App: + """Create app instance.""" + + connexion_app = connexion.App( + __name__, debug=config_object.DEBUG, specification_dir="spec/" + ) + flask_app = connexion_app.app + flask_app.config.from_object(config_object) + + # Setup database + init_database(flask_app, config=config_object, db_session=db_session) + + # Setup prometheus monitoring + setup_metrics(flask_app) + + connexion_app.add_api("api.yaml") + _logger.info("Application instance created") + + return connexion_app diff --git a/packages/ml_api/api/config.py b/packages/ml_api/api/config.py new file mode 100644 index 0000000..1f528c4 --- /dev/null +++ b/packages/ml_api/api/config.py @@ -0,0 +1,101 @@ +import logging +import os +import pathlib +import sys +from logging.config import fileConfig + +import api + +# logging format +FORMATTER = logging.Formatter( + "%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s" +) + +# Project Directories +ROOT = pathlib.Path(api.__file__).resolve().parent.parent + +APP_NAME = 'ml_api' + + +class Config: + DEBUG = False + TESTING = False + ENV = os.getenv("FLASK_ENV", "production") + SERVER_PORT = int(os.getenv("SERVER_PORT", 5000)) + SERVER_HOST = os.getenv("SERVER_HOST", "0.0.0.0") + LOGGING_LEVEL = os.getenv("LOGGING_LEVEL", logging.INFO) + SHADOW_MODE_ACTIVE = os.getenv('SHADOW_MODE_ACTIVE', True) + SQLALCHEMY_DATABASE_URI = ( + f"postgresql+psycopg2://{os.getenv('DB_USER')}:" + f"{os.getenv('DB_PASSWORD')}@{os.getenv('DB_HOST')}/{os.getenv('DB_NAME')}" + ) + # DB config matches docker container + DB_USER = os.getenv("DB_USER", "user") + DB_PASSWORD = os.getenv("DB_PASSWORD", "password") + DB_PORT = os.getenv("DB_PORT", 6609) + DB_HOST = os.getenv("DB_HOST", "0.0.0.0") + DB_NAME = os.getenv("DB_NAME", "ml_api_dev") + + +class DevelopmentConfig(Config): + DEBUG = True + ENV = "development" # do not use in production! + LOGGING_LEVEL = logging.DEBUG + + +class TestingConfig(Config): + DEBUG = True + TESTING = True + LOGGING_LEVEL = logging.DEBUG + + # DB config matches test docker container + DB_USER = os.getenv("DB_USER", "test_user") + DB_PASSWORD = os.getenv("DB_PASSWORD", "password") + DB_PORT = os.getenv("DB_PORT", 6608) + DB_HOST = os.getenv("DB_HOST", "0.0.0.0") + DB_NAME = "ml_api_test" + SQLALCHEMY_DATABASE_URI = ( + f"postgresql+psycopg2://{DB_USER}:" + f"{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + ) + + +class ProductionConfig(Config): + DB_USER = os.getenv("DB_USER", "user") + DB_PASSWORD = os.getenv("DB_PASSWORD", "password") + DB_PORT = os.getenv("DB_PORT", 6609) + DB_HOST = os.getenv("DB_HOST", "database") + DB_NAME = os.getenv("DB_NAME", "ml_api") + SQLALCHEMY_DATABASE_URI = ( + f"postgresql+psycopg2://{DB_USER}:" + f"{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + ) + + +def get_console_handler(): + """Setup console logging handler.""" + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler + + +def setup_app_logging(config: Config) -> None: + """Prepare custom logging for our application.""" + _disable_irrelevant_loggers() + fileConfig(ROOT / 'gunicorn_logging.conf') + logger = logging.getLogger('mlapi') + logger.setLevel(config.LOGGING_LEVEL) + + +def _disable_irrelevant_loggers() -> None: + """Disable loggers created by packages which create a lot of noise.""" + for logger_name in ( + "connexion.apis.flask_api", + "connexion.apis.abstract", + "connexion.decorators", + "connexion.operation", + "connexion.operations", + "connexion.app", + "openapi_spec_validator", + ): + logging.getLogger(logger_name).level = logging.WARNING diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py new file mode 100644 index 0000000..47d9f07 --- /dev/null +++ b/packages/ml_api/api/controller.py @@ -0,0 +1,131 @@ +import json +import logging +import threading + +from flask import request, jsonify, Response, current_app +from prometheus_client import Histogram, Gauge, Info +from regression_model import __version__ as live_version + +from api.config import APP_NAME +from api.persistence.data_access import PredictionPersistence, ModelType +from gradient_boosting_model import __version__ as shadow_version +from gradient_boosting_model.predict import make_prediction + +_logger = logging.getLogger('mlapi') + + +PREDICTION_TRACKER = Histogram( + name='house_price_prediction_dollars', + documentation='ML Model Prediction on House Price', + labelnames=['app_name', 'model_name', 'model_version'] +) + +PREDICTION_GAUGE = Gauge( + name='house_price_gauge_dollars', + documentation='ML Model Prediction on House Price for min max calcs', + labelnames=['app_name', 'model_name', 'model_version'] +) + +PREDICTION_GAUGE.labels( + app_name=APP_NAME, + model_name=ModelType.LASSO.name, + model_version=live_version) + +MODEL_VERSIONS = Info( + 'model_version_details', + 'Capture model version information', +) + +MODEL_VERSIONS.info({ + 'live_model': ModelType.LASSO.name, + 'live_version': live_version, + 'shadow_model': ModelType.GRADIENT_BOOSTING.name, + 'shadow_version': shadow_version}) + + +def health(): + if request.method == "GET": + status = {"status": "ok"} + _logger.debug(status) + return jsonify(status) + + +def predict(): + if request.method == "POST": + # Step 1: Extract POST data from request body as JSON + json_data = request.get_json() + for entry in json_data: + _logger.info(entry) + + # Step 2a: Get and save live model predictions + persistence = PredictionPersistence(db_session=current_app.db_session) + result = persistence.make_save_predictions( + db_model=ModelType.LASSO, input_data=json_data + ) + + # Step 2b: Get and save shadow predictions asynchronously + if current_app.config.get("SHADOW_MODE_ACTIVE"): + _logger.debug( + f"Calling shadow model asynchronously: " + f"{ModelType.GRADIENT_BOOSTING.value}" + ) + thread = threading.Thread( + target=persistence.make_save_predictions, + kwargs={ + "db_model": ModelType.GRADIENT_BOOSTING, + "input_data": json_data, + }, + ) + thread.start() + + # Step 3: Handle errors + if result.errors: + _logger.warning(f"errors during prediction: {result.errors}") + return Response(json.dumps(result.errors), status=400) + + # Step 4: Monitoring + for _prediction in result.predictions: + PREDICTION_TRACKER.labels( + app_name=APP_NAME, + model_name=ModelType.LASSO.name, + model_version=live_version).observe(_prediction) + PREDICTION_GAUGE.labels( + app_name=APP_NAME, + model_name=ModelType.LASSO.name, + model_version=live_version).set(_prediction) + _logger.info( + f'Prediction results for model: {ModelType.LASSO.name} ' + f'version: {result.model_version} ' + f'Output values: {result.predictions}') + + # Step 5: Prepare prediction response + return jsonify( + { + "predictions": result.predictions, + "version": result.model_version, + "errors": result.errors, + } + ) + + +def predict_previous(): + if request.method == "POST": + # Step 1: Extract POST data from request body as JSON + json_data = request.get_json() + + # Step 2: Access the model prediction function (also validates data) + result = make_prediction(input_data=json_data) + + # Step 3: Handle errors + errors = result.get("errors") + if errors: + return Response(json.dumps(errors), status=400) + + # Step 4: Split out results + predictions = result.get("predictions").tolist() + version = result.get("version") + + # Step 5: Prepare prediction response + return jsonify( + {"predictions": predictions, "version": version, "errors": errors} + ) diff --git a/packages/ml_api/api/monitoring/__init__.py b/packages/ml_api/api/monitoring/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/api/monitoring/middleware.py b/packages/ml_api/api/monitoring/middleware.py new file mode 100644 index 0000000..31712ef --- /dev/null +++ b/packages/ml_api/api/monitoring/middleware.py @@ -0,0 +1,60 @@ +from flask import request, Flask +from flask.wrappers import Response +from prometheus_client import Counter, Histogram +import time + +from api.config import APP_NAME + + +# Counter and Histogram are examples of default metrics +# available from the prometheus Python client. +REQUEST_COUNT = Counter( + name='http_request_count', + documentation='App Request Count', + labelnames=['app_name', 'method', 'endpoint', 'http_status'] +) +REQUEST_LATENCY = Histogram( + name='http_request_latency_seconds', + documentation='Request latency', + labelnames=['app_name', 'endpoint'] +) + + +def start_timer() -> None: + """Get start time of a request.""" + request._prometheus_metrics_request_start_time = time.time() + + +def stop_timer(response: Response) -> Response: + """Get stop time of a request..""" + request_latency = time.time() - request._prometheus_metrics_request_start_time + REQUEST_LATENCY.labels( + app_name=APP_NAME, + endpoint=request.path).observe(request_latency) + return response + + +def record_request_data(response: Response) -> Response: + """Capture request data. + + Uses the flask request object to extract information such as + the HTTP request method, endpoint and HTTP status. + """ + REQUEST_COUNT.labels( + app_name=APP_NAME, + method=request.method, + endpoint=request.path, + http_status=response.status_code).inc() + return response + + +def setup_metrics(app: Flask) -> None: + """Setup Prometheus metrics. + + This function uses the flask before_request + and after_request hooks to capture metrics + with each HTTP request to the application. + """ + app.before_request(start_timer) + app.after_request(record_request_data) + app.after_request(stop_timer) diff --git a/packages/ml_api/api/persistence/__init__.py b/packages/ml_api/api/persistence/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/api/persistence/core.py b/packages/ml_api/api/persistence/core.py new file mode 100644 index 0000000..aab5555 --- /dev/null +++ b/packages/ml_api/api/persistence/core.py @@ -0,0 +1,67 @@ +import logging +import os + +import alembic.config +from flask import Flask +from sqlalchemy import create_engine +from sqlalchemy.engine import Engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy_utils import database_exists, create_database + +from api.config import Config, ROOT + +_logger = logging.getLogger('mlapi') + +# Base class for SQLAlchemy models +Base = declarative_base() + + +def create_db_engine_from_config(*, config: Config) -> Engine: + """The Engine is the starting point for any SQLAlchemy application. + + It’s “home base” for the actual database and its DBAPI, delivered to the SQLAlchemy + application through a connection pool and a Dialect, which describes how to talk to + a specific kind of database / DBAPI combination. + """ + + db_url = config.SQLALCHEMY_DATABASE_URI + if not database_exists(db_url): + create_database(db_url) + engine = create_engine(db_url) + + _logger.info(f"creating DB conn with URI: {db_url}") + return engine + + +def create_db_session(*, engine: Engine) -> scoped_session: + """Broadly speaking, the Session establishes all conversations with the database. + + It represents a “holding zone” for all the objects which you’ve loaded or + associated with it during its lifespan. + """ + return scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) + + +def init_database(app: Flask, config: Config, db_session=None) -> None: + """Connect to the database and attach DB session to the app.""" + + if not db_session: + engine = create_db_engine_from_config(config=config) + db_session = create_db_session(engine=engine) + + app.db_session = db_session + + @app.teardown_appcontext + def shutdown_session(exception=None): + db_session.remove() + + +def run_migrations(): + """Run the DB migrations prior to the tests.""" + + # alembic looks for the migrations in the current + # directory so we change to the correct directory. + os.chdir(str(ROOT)) + alembicArgs = ["--raiseerr", "upgrade", "head"] + alembic.config.main(argv=alembicArgs) diff --git a/packages/ml_api/api/persistence/data_access.py b/packages/ml_api/api/persistence/data_access.py new file mode 100644 index 0000000..f46d24f --- /dev/null +++ b/packages/ml_api/api/persistence/data_access.py @@ -0,0 +1,113 @@ +import enum +import json +import logging +import typing as t + +import numpy as np +import pandas as pd +from regression_model.predict import make_prediction as make_live_prediction +from sqlalchemy.orm.session import Session + +from api.persistence.models import ( + LassoModelPredictions, + GradientBoostingModelPredictions, +) +from gradient_boosting_model.predict import make_prediction as make_shadow_prediction + +_logger = logging.getLogger('mlapi') + + +SECONDARY_VARIABLES_TO_RENAME = { + "FirstFlrSF": "1stFlrSF", + "SecondFlrSF": "2ndFlrSF", + "ThreeSsnPortch": "3SsnPorch", +} + + +class ModelType(enum.Enum): + LASSO = "lasso" + GRADIENT_BOOSTING = "gradient_boosting" + + +class PredictionResult(t.NamedTuple): + errors: t.Any + predictions: np.array + model_version: str + + +MODEL_PREDICTION_MAP = { + ModelType.GRADIENT_BOOSTING: make_shadow_prediction, + ModelType.LASSO: make_live_prediction, +} + + +class PredictionPersistence: + def __init__(self, *, db_session: Session, user_id: str = None) -> None: + self.db_session = db_session + if not user_id: + # in reality, here we would use something like a UUID for anonymous users + # and if we had user logins, we would record the user ID. + self.user_id = "007" + + def make_save_predictions( + self, *, db_model: ModelType, input_data: t.List + ) -> PredictionResult: + """Get the prediction from a given model and persist it.""" + # Access the model prediction function via mapping + if db_model == ModelType.LASSO: + # we have to rename a few of the columns for backwards + # compatibility with the regression model package. + live_frame = pd.DataFrame(input_data) + input_data = live_frame.rename( + columns=SECONDARY_VARIABLES_TO_RENAME + ).to_dict(orient="records") + + result = MODEL_PREDICTION_MAP[db_model](input_data=input_data) + errors = None + try: + errors = result["errors"] + except KeyError: + # regression model `make_prediction` does not include errors + pass + + prediction_result = PredictionResult( + errors=errors, + predictions=result.get("predictions").tolist() if not errors else None, + model_version=result.get("version"), + ) + + if prediction_result.errors: + return prediction_result + + self.save_predictions( + inputs=input_data, prediction_result=prediction_result, db_model=db_model + ) + + return prediction_result + + def save_predictions( + self, + *, + inputs: t.List, + prediction_result: PredictionResult, + db_model: ModelType, + ) -> None: + """Persist model predictions to storage.""" + if db_model == db_model.LASSO: + prediction_data = LassoModelPredictions( + user_id=self.user_id, + model_version=prediction_result.model_version, + inputs=json.dumps(inputs), + outputs=json.dumps(prediction_result.predictions), + ) + else: + prediction_data = GradientBoostingModelPredictions( + user_id=self.user_id, + model_version=prediction_result.model_version, + inputs=json.dumps(inputs), + outputs=json.dumps(prediction_result.predictions), + ) + + self.db_session.add(prediction_data) + self.db_session.commit() + _logger.debug(f"saved data for model: {db_model}") diff --git a/packages/ml_api/api/persistence/models.py b/packages/ml_api/api/persistence/models.py new file mode 100644 index 0000000..65da0b8 --- /dev/null +++ b/packages/ml_api/api/persistence/models.py @@ -0,0 +1,29 @@ +from sqlalchemy import Column, String, DateTime, Integer +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import func + +from api.persistence.core import Base + + +class LassoModelPredictions(Base): + __tablename__ = "regression_model_predictions" + id = Column(Integer, primary_key=True) + user_id = Column(String(36), nullable=False) + datetime_captured = Column( + DateTime(timezone=True), server_default=func.now(), index=True + ) + model_version = Column(String(36), nullable=False) + inputs = Column(JSONB) + outputs = Column(JSONB) + + +class GradientBoostingModelPredictions(Base): + __tablename__ = "gradient_boosting_model_predictions" + id = Column(Integer, primary_key=True) + user_id = Column(String(36), nullable=False) + datetime_captured = Column( + DateTime(timezone=True), server_default=func.now(), index=True + ) + model_version = Column(String(36), nullable=False) + inputs = Column(JSONB) + outputs = Column(JSONB) diff --git a/packages/ml_api/api/spec/__init__.py b/packages/ml_api/api/spec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/api/spec/api.yaml b/packages/ml_api/api/spec/api.yaml new file mode 100644 index 0000000..84c3075 --- /dev/null +++ b/packages/ml_api/api/spec/api.yaml @@ -0,0 +1,147 @@ +openapi: 3.0.0 + +info: + title: Spec for House Price Prediction API + version: '1' + +servers: +- url: http://{base}:5000/ + description: API for performing house price predictions. + variables: + base: + default: 0.0.0.0 + +paths: + /: + get: + operationId: api.controller.health + responses: + '200': + description: API Health Status + + /v1/predictions/regression: + post: + operationId: api.controller.predict + requestBody: + description: House details used to make price prediction + required: true + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/HouseDetails' + responses: + '200': + description: House Price Predictions + '400': + description: Bad request, house data validation failed + '5XX': + description: Unexpected error + + /v1/predictions/gradient: + post: + operationId: api.controller.predict_previous + requestBody: + description: House details used to make price prediction + required: true + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/HouseDetails' + responses: + '200': + description: House Price Predictions + '400': + description: Bad request, house data validation failed + '5XX': + description: Unexpected error + +components: + schemas: + HouseDetails: + type: object + description: "List of the houses to get predictions for." + example: + Id: 1461 + MSSubClass: 20 + MSZoning: RH + LotFrontage: 80.0 + LotArea: 11622 + Street: Pave + Alley: null + LotShape: Reg + LandContour: Lvl + Utilities: AllPub + LotConfig: Inside + LandSlope: Gtl + Neighborhood: NAmes + Condition1: Feedr + Condition2: Norm + BldgType: 1Fam + HouseStyle: 1Story + OverallQual: 5 + OverallCond: 6 + YearBuilt: 1961 + YearRemodAdd: 1961 + RoofStyle: Gable + RoofMatl: CompShg + Exterior1st: VinylSd + Exterior2nd: VinylSd + MasVnrType: None + MasVnrArea: 0.0 + ExterQual: TA + ExterCond: TA + Foundation: CBlock + BsmtQual: TA + BsmtCond: TA + BsmtExposure: null + BsmtFinType1: Rec + BsmtFinSF1: 468.0 + BsmtFinType2: LwQ + BsmtFinSF2: 144.0 + BsmtUnfSF: 270.0 + TotalBsmtSF: 882.0 + Heating: GasA + HeatingQC: TA + CentralAir: Y + Electrical: SBrkr + 1stFlrSF: 896 + 2ndFlrSF: 0 + LowQualFinSF: 0 + GrLivArea: 896 + BsmtFullBath: 0.0 + BsmtHalfBath: 0.0 + FullBath: 1 + HalfBath: 0 + BedroomAbvGr: 2 + KitchenAbvGr: 1 + KitchenQual: TA + TotRmsAbvGrd: 5 + Functional: Typ + Fireplaces: 0 + FireplaceQu: null + GarageType: Attchd + GarageYrBlt: 1961.0 + GarageFinish: Unf + GarageCars: 1.0 + GarageArea: 730.0 + GarageQual: TA + GarageCond: TA + PavedDrive: Y + WoodDeckSF: 140 + OpenPorchSF: 0 + EnclosedPorch: 0 + 3SsnPorch: 0 + ScreenPorch: 120 + PoolArea: 0 + PoolQC: null + Fence: MnPrv + MiscFeature: null + MiscVal: 0 + MoSold: 6 + YrSold: 2010 + SaleType: WD + SaleCondition: Normal diff --git a/packages/ml_api/differential_tests/__init__.py b/packages/ml_api/differential_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/differential_tests/__main__.py b/packages/ml_api/differential_tests/__main__.py new file mode 100644 index 0000000..cf86222 --- /dev/null +++ b/packages/ml_api/differential_tests/__main__.py @@ -0,0 +1,100 @@ +import json +from argparse import ArgumentParser, Namespace +from pathlib import Path +from typing import Mapping + +from differential_tests.compare import compare_predictions +from api.config import ROOT + +from termcolor import cprint +from yarl import URL +import requests + +Marginals = Mapping[str, Mapping[str, float]] + + +def parse_args() -> Namespace: + parser = ArgumentParser() + + subparsers = parser.add_subparsers(dest="command") + + compute_parser = subparsers.add_parser( + "compute", help="Compute the predictions for a test set" + ) + compute_parser.add_argument( + "--base-url", + default=URL("http://0.0.0.0:5000"), + type=URL, + help="Base URL of the service to test", + ) + compute_parser.add_argument( + "tests_dir", type=Path, help="Directory containing the test set to use" + ) + compute_parser.add_argument( + "results_dir", type=Path, help="Directory to save the prediction results to" + ) + + compare_parser = subparsers.add_parser( + "compare", help="Compare the actual results with the expected results" + ) + compare_parser.add_argument( + "--absolute-tolerance", + dest="abs_tol", + metavar="X", + type=float, + help="math.isclose(a, b, abs_tol=X)", + default=1e-5, + ) + compare_parser.add_argument( + "--relative-tolerance", + dest="rel_tol", + metavar="X", + type=float, + default=1e-5, + help="math.isclose(a, b, rel_tol=X)", + ) + compare_parser.add_argument( + "expected_results_dir", + type=Path, + help="Directory containing the expected results", + ) + compare_parser.add_argument( + "actual_results_dir", type=Path, help="Directory containing the actual results" + ) + + return parser.parse_args() + + +def main(args: Namespace) -> None: + if args.command == "compute": + compute_predictions(args) + elif args.command == "compare": + compare_predictions(args) + + +def compute_predictions(args: Namespace) -> None: + print("computing") + + diff_test_dir = ROOT / "differential_tests" + results_dir = args.results_dir + results_dir.mkdir(parents=True, exist_ok=True) + prepared_test_dir = diff_test_dir / Path(args.tests_dir) + + for test_filename in sorted(prepared_test_dir.glob("*.json")): + results_filename = results_dir / test_filename.name + print(f"Computing {results_filename} from {test_filename} ... ", end="") + + with test_filename.open() as f: + test = json.load(f) + + results = requests.post(f"{args.base_url}/v1/predictions/primary", json=test) + + with results_filename.open("w") as f: + json.dump(results.json(), f, indent=2, sort_keys=True) + + cprint("OK", "green") + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/packages/ml_api/differential_tests/compare.py b/packages/ml_api/differential_tests/compare.py new file mode 100644 index 0000000..012dd16 --- /dev/null +++ b/packages/ml_api/differential_tests/compare.py @@ -0,0 +1,95 @@ +import json +import math +import sys +import typing as t +from argparse import Namespace + +from termcolor import cprint + +from api.config import ROOT + + +def compare_differences( + *, + expected_predictions: t.List, + actual_predictions: t.List, + rel_tol: t.Optional[float] = None, + abs_tol: t.Optional[float] = None, +) -> None: + """ + :param rel_tol: is the relative tolerance – it is the maximum allowed difference + between a and b, relative to the larger absolute value of a or b. + For example, to set a tolerance of 5%, pass rel_tol=0.05. The default + tolerance is 1e-09, which assures that the two values are the same within + about 9 decimal digits. rel_tol must be greater than zero. + + :param abs_tol: abs_tol is the minimum absolute tolerance – useful for comparisons + near zero. abs_tol must be at least zero. + """ + only_in_expected = len(expected_predictions) - len(actual_predictions) + + if only_in_expected: + raise ValueError(f"Missing {only_in_expected} predictions") + + only_in_actual = len(actual_predictions) - len(expected_predictions) + + if only_in_actual: + raise ValueError(f"Found {only_in_actual} unexpected predictions") + + thresholds = {} + + if abs_tol is not None: + thresholds["abs_tol"] = abs_tol + + if rel_tol is not None: + thresholds["rel_tol"] = rel_tol + + for index, (actual_prediction, expected_prediction) in enumerate( + zip(actual_predictions, expected_predictions) + ): + if not math.isclose(expected_prediction, actual_prediction, **thresholds): + raise ValueError( + f"Price prediction {index} has changed by more " + f"than the thresholds: {thresholds}: " + f"{expected_prediction} (expected) vs " + f"{actual_prediction} (actual)" + ) + + +def compare_predictions(args: Namespace) -> None: + expected_results_dir = ROOT / args.expected_results_dir + actual_results_dir = ROOT / args.actual_results_dir + + expected_results_filenames = list(expected_results_dir.glob("*.json")) + + if not expected_results_filenames: + print("No results found!") + sys.exit(1) + + for expected_results_filename in sorted(expected_results_filenames): + name = expected_results_filename.name + actual_results_filename = actual_results_dir / name + + print( + f"Comparing {expected_results_filename} with {actual_results_filename} ... ", + end="", + ) + + with expected_results_filename.open() as f: + expected_results = json.load(f) + + with actual_results_filename.open() as f: + actual_results = json.load(f) + + try: + compare_differences( + expected_predictions=expected_results["predictions"], + actual_predictions=actual_results["predictions"], + rel_tol=args.rel_tol, + abs_tol=args.abs_tol, + ) + except ValueError as exc: + cprint("ERROR", "red") + cprint(f" • {exc}", "red") + else: + cprint("OK", "green") diff --git a/packages/ml_api/differential_tests/sample_payloads/sample_input1.json b/packages/ml_api/differential_tests/sample_payloads/sample_input1.json new file mode 100644 index 0000000..61f96e6 --- /dev/null +++ b/packages/ml_api/differential_tests/sample_payloads/sample_input1.json @@ -0,0 +1,488 @@ +[{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 11622, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1961, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 896, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +}, { + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 11689, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1969, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 752, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +}, +{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 22689, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1969, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 752, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +},{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 11689, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1969, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 988, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +},{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 11689, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1969, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 752, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2008, + "SaleType": "WD", + "SaleCondition": "Normal" +},{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 25000, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1969, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 752, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +}] \ No newline at end of file diff --git a/packages/ml_api/docker/Dockerfile b/packages/ml_api/docker/Dockerfile new file mode 100644 index 0000000..9c948fc --- /dev/null +++ b/packages/ml_api/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.7.5-slim-buster + +RUN mkdir -p /opt/app +COPY requirements /opt/app/requirements +RUN pip install --upgrade pip + +# ensure we can run the make commands +RUN apt-get update -y && \ + apt-get install -y make && \ + apt-get install -y libffi-dev gcc && \ + # for swagger + apt-get install -y curl && \ + # for postgres driver + apt-get install -y libpq-dev + +RUN pip install -r /opt/app/requirements/requirements.txt +ENV PYTHONPATH "${PYTHONPATH}:/opt/app/" + +ADD . /opt/app +WORKDIR /opt/app diff --git a/packages/ml_api/docker/Dockerfile.test b/packages/ml_api/docker/Dockerfile.test new file mode 100644 index 0000000..46c29ac --- /dev/null +++ b/packages/ml_api/docker/Dockerfile.test @@ -0,0 +1,18 @@ +FROM python:3.7.5-slim-buster + +RUN mkdir -p /opt/app +COPY requirements /opt/app/requirements +RUN pip install --upgrade pip + +# ensure we can run the make commands +RUN apt-get update -y && \ + apt-get install -y make && \ + apt-get install -y libffi-dev gcc && \ + # for swagger + apt-get install -y curl + +ENV PYTHONPATH "${PYTHONPATH}:/opt/app" +RUN pip install -r /opt/app/requirements/test_requirements.txt + +ADD . /opt/app +WORKDIR /opt/app diff --git a/packages/ml_api/docker/config/grafana/basic_cadvisor_dashboard_ml_api.json b/packages/ml_api/docker/config/grafana/basic_cadvisor_dashboard_ml_api.json new file mode 100644 index 0000000..58b24a1 --- /dev/null +++ b/packages/ml_api/docker/config/grafana/basic_cadvisor_dashboard_ml_api.json @@ -0,0 +1,605 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Docker monitoring with Prometheus and cAdvisor with node selection", + "editable": true, + "gnetId": 8321, + "graphTooltip": 1, + "id": 1, + "iteration": 1578230538273, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 0 + }, + "height": "20", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_last_seen", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Running containers", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "mbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 0 + }, + "height": "20", + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})/1024/1024", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 0 + }, + "height": "20", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}[5m]) * 100)", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total CPU Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(container_cpu_user_seconds_total{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}[5m]) * 100", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_usage_bytes{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 21, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(container_cpu_user_seconds_total, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "/([^:]+):.*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "8080", + "value": "8080" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Port", + "multi": false, + "name": "port", + "options": [], + "query": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", + "refresh": 1, + "regex": "/[^:]+:(.*)/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Docker monitoring with node selection", + "uid": "pHUTSjLZk", + "version": 2 +} \ No newline at end of file diff --git a/packages/ml_api/docker/config/grafana/grafana_flask_basic_dashboard_ml_api.json b/packages/ml_api/docker/config/grafana/grafana_flask_basic_dashboard_ml_api.json new file mode 100644 index 0000000..39224a7 --- /dev/null +++ b/packages/ml_api/docker/config/grafana/grafana_flask_basic_dashboard_ml_api.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(http_request_count_total{job=\"ml_api\"}[5m])", + "legendFormat": "{{app_name}} {{method}} {{endpoint}} {{http_status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(http_request_latency_seconds_sum{job=\"ml_api\"}[5m])) / sum (rate(http_request_latency_seconds_count{job=\"ml_api\"}[5m]))", + "legendFormat": "Average (seconds)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Really Simple Flask Dashboard", + "uid": "q8vgEpLZl", + "version": 3 +} \ No newline at end of file diff --git a/packages/ml_api/docker/config/grafana/ml_api_dashboard.json b/packages/ml_api/docker/config/grafana/ml_api_dashboard.json new file mode 100644 index 0000000..b55b2ce --- /dev/null +++ b/packages/ml_api/docker/config/grafana/ml_api_dashboard.json @@ -0,0 +1,625 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.6.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Machine learning-specific metrics", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 7, + "links": [], + "options": {}, + "pageSize": 1, + "pluginVersion": "6.5.2", + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by(live_model, live_version, shadow_model, shadow_version, version)(model_version_details_info\n* on (instance, job) group_left(version)\npython_info)", + "format": "table", + "legendFormat": "{{model_version}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Model Versions", + "transform": "table", + "transparent": true, + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(house_price_prediction_dollars_sum{job=\"ml_api\"}[1m])) / sum (rate(house_price_prediction_dollars_count{job=\"ml_api\"}[1m]))", + "legendFormat": "Average Prediction Amount ($)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average House Price Prediction Amount (USD)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(house_price_prediction_dollars_count{job=\"ml_api\"}[1m])", + "legendFormat": "Average Prediction Rate", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average House Price Prediction Rate (/second)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg_over_time(house_price_gauge_dollars[1m])", + "legendFormat": "AVG", + "refId": "A" + }, + { + "expr": "stddev_over_time(house_price_gauge_dollars[1m])", + "legendFormat": "STD", + "refId": "B" + }, + { + "expr": "stddev_over_time(house_price_gauge_dollars[1m]) / (sqrt(count_over_time(house_price_prediction_dollars_count[1m])))", + "legendFormat": "SEM", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Standard Error of the Mean (SEM)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(avg_over_time(house_price_gauge_dollars[1m]) - avg_over_time(house_price_gauge_dollars[1w])) / (stddev_over_time(house_price_gauge_dollars[1w]))", + "legendFormat": "Z-Score", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Standard Score (Z-Score)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "options": { + "fieldOptions": { + "calcs": [ + "logmin" + ], + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "house_price_gauge_dollars", + "legendFormat": "Average Prediction Amount ($)", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Prediction", + "type": "gauge" + }, + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8, + "options": { + "fieldOptions": { + "calcs": [ + "max" + ], + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "expr": "house_price_gauge_dollars", + "legendFormat": "Average Prediction Amount ($)", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Prediction", + "type": "gauge" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ML API Dashboard", + "uid": "q8vgEpLZk", + "version": 2 +} \ No newline at end of file diff --git a/packages/ml_api/docker/config/prometheus/prometheus.yml b/packages/ml_api/docker/config/prometheus/prometheus.yml new file mode 100644 index 0000000..1e9fa32 --- /dev/null +++ b/packages/ml_api/docker/config/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +# my global config +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'my-project' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['prometheus:9090'] + - job_name: 'ml_api' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ['ml_api:5000'] + + - job_name: 'cadvisor' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['cadvisor:8080'] diff --git a/packages/ml_api/docker/docker-compose-ci-candidate.yml b/packages/ml_api/docker/docker-compose-ci-candidate.yml new file mode 100644 index 0000000..47bbc16 --- /dev/null +++ b/packages/ml_api/docker/docker-compose-ci-candidate.yml @@ -0,0 +1,20 @@ +version: '3' +services: + + ml_api: + image: christophergs/ml_api:${TARGET} + environment: + SERVER_PORT: ${SERVER_PORT:-5001} + build: + context: ../ + dockerfile: docker/Dockerfile.test + ports: + - "5001:5001" + tty: true + command: bash -c "make run-service-development" + + differential-tests: + image: christophergs/ml_api:${TARGET} + command: ["true"] + depends_on: + - ml_api \ No newline at end of file diff --git a/packages/ml_api/docker/docker-compose-ci-master.yml b/packages/ml_api/docker/docker-compose-ci-master.yml new file mode 100644 index 0000000..c0844e3 --- /dev/null +++ b/packages/ml_api/docker/docker-compose-ci-master.yml @@ -0,0 +1,20 @@ +version: '3' +services: + + ml_api: + image: christophergs/ml_api:${TARGET} + environment: + SERVER_PORT: ${SERVER_PORT:-5000} + build: + context: ../ + dockerfile: docker/Dockerfile.test + ports: + - "5000:5000" + tty: true + command: bash -c "make run-service-development" + + differential-tests: + image: christophergs/ml_api:${TARGET} + command: ["true"] + depends_on: + - ml_api \ No newline at end of file diff --git a/packages/ml_api/docker/docker-compose-elk.yml b/packages/ml_api/docker/docker-compose-elk.yml new file mode 100644 index 0000000..3ed1806 --- /dev/null +++ b/packages/ml_api/docker/docker-compose-elk.yml @@ -0,0 +1,99 @@ +version: '3.2' +services: + ml_api: + build: + context: ../ + dockerfile: docker/Dockerfile + environment: + DB_HOST: database + DB_PORT: 5432 + DB_USER: user + DB_PASSWORD: ${DB_PASSWORD:-password} + DB_NAME: ml_api_dev + networks: + - elk + depends_on: + - database + - logstash + ports: + - "5000:5000" # expose webserver to localhost host:container + command: bash -c "make db-migrations && make run-service-wsgi" + + database: + image: postgres:latest + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_dev + ports: + # expose postgres container on different host port to default (host:container) + - "6609:5432" + volumes: + - my_dbdata:/var/lib/postgresql/data + networks: + - elk + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + volumes: + - type: bind + source: ./elasticsearch/config/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme + # Use single node discovery in order to disable production mode and avoid bootstrap checks + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html + discovery.type: single-node + networks: + - elk + + logstash: + image: docker.elastic.co/logstash/logstash:${ELK_VERSION} + volumes: + - type: bind + source: ./logstash/config/logstash.yml + target: /usr/share/logstash/config/logstash.yml + read_only: true + - type: bind + source: ./logstash/pipeline + target: /usr/share/logstash/pipeline + read_only: true + ports: + - "5001:5001" + - "9600:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + image: docker.elastic.co/kibana/kibana:${ELK_VERSION} + volumes: + - type: bind + source: ./kibana/config/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + my_dbdata: + elasticsearch: \ No newline at end of file diff --git a/packages/ml_api/docker/docker-compose.test.yml b/packages/ml_api/docker/docker-compose.test.yml new file mode 100644 index 0000000..44109b6 --- /dev/null +++ b/packages/ml_api/docker/docker-compose.test.yml @@ -0,0 +1,33 @@ +version: '3' +services: + ml_api_test: + image: christophergs/ml_api:master + build: + context: ../ + dockerfile: docker/Dockerfile.test + environment: + DB_HOST: test_database + DB_PORT: 5432 + DB_USER: test_user + DB_PASSWORD: ${DB_PASSWORD:-password} + DB_NAME: ml_api_test + depends_on: + - test_database + ports: + - "5000:5000" # expose webserver to localhost host:container + command: bash -c "make db-migrations && make run-service-development" + + test_database: + image: postgres:latest + environment: + POSTGRES_USER: test_user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_test + ports: + # expose postgres container on different host port to default (host:container) + - "6608:5432" + volumes: + - my_dbdata_test:/var/lib/postgresql/test_data + +volumes: + my_dbdata_test: diff --git a/packages/ml_api/docker/docker-compose.yml b/packages/ml_api/docker/docker-compose.yml new file mode 100644 index 0000000..92cc691 --- /dev/null +++ b/packages/ml_api/docker/docker-compose.yml @@ -0,0 +1,72 @@ +version: '3' +services: + ml_api: + build: + context: ../ + dockerfile: docker/Dockerfile + environment: + DB_HOST: database + DB_PORT: 5432 + DB_USER: user + DB_PASSWORD: ${DB_PASSWORD:-password} + DB_NAME: ml_api_dev + depends_on: + - database + - cadvisor + ports: + - "5000:5000" # expose webserver to localhost host:container + command: bash -c "make db-migrations && make run-service-wsgi" + + database: + image: postgres:latest + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: ml_api_dev + ports: + # expose postgres container on different host port to default (host:container) + - "6609:5432" + volumes: + - my_dbdata:/var/lib/postgresql/data + + prometheus: + image: prom/prometheus + container_name: prometheus + volumes: + - ./config/prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + expose: + - 9090 + ports: + - 9090:9090 + depends_on: + - cadvisor + + grafana: + image: grafana/grafana + depends_on: + - prometheus + ports: + - 3000:3000 + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=foobar + - GF_USERS_ALLOW_SIGN_UP=false + + cadvisor: + image: google/cadvisor + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - 8080:8080 + +volumes: + my_dbdata: {} + prometheus_data: {} + grafana_data: {} diff --git a/packages/ml_api/docker/elasticsearch/config/elasticsearch.yml b/packages/ml_api/docker/elasticsearch/config/elasticsearch.yml new file mode 100644 index 0000000..b831729 --- /dev/null +++ b/packages/ml_api/docker/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,10 @@ +## Default Elasticsearch configuration from Elasticsearch base image. +## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +## X-Pack settings +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +xpack.license.self_generated.type: basic +xpack.security.enabled: true +xpack.monitoring.collection.enabled: true diff --git a/packages/ml_api/docker/kibana/config/kibana.yml b/packages/ml_api/docker/kibana/config/kibana.yml new file mode 100644 index 0000000..6f0a087 --- /dev/null +++ b/packages/ml_api/docker/kibana/config/kibana.yml @@ -0,0 +1,12 @@ +--- +## Default Kibana configuration from Kibana base image. +## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js + +server.name: kibana +server.host: "0" +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +xpack.monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +elasticsearch.username: elastic +elasticsearch.password: changeme diff --git a/packages/ml_api/docker/kibana/config/kibana_example_inputs_dashboard.ndjson b/packages/ml_api/docker/kibana/config/kibana_example_inputs_dashboard.ndjson new file mode 100644 index 0000000..266941d --- /dev/null +++ b/packages/ml_api/docker/kibana/config/kibana_example_inputs_dashboard.ndjson @@ -0,0 +1,5 @@ +{"attributes":{"fields":"[{\"name\":\"1stFlrSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"2ndFlrSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"3SsnPorch\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"@version\",\"subType\":\"multi\"},{\"name\":\"Alley\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Alley.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Alley\",\"subType\":\"multi\"},{\"name\":\"BedroomAbvGr\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"BldgType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BldgType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BldgType\",\"subType\":\"multi\"},{\"name\":\"BsmtCond\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BsmtCond.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BsmtCond\",\"subType\":\"multi\"},{\"name\":\"BsmtExposure\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BsmtExposure.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BsmtExposure\",\"subType\":\"multi\"},{\"name\":\"BsmtFinSF1\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"BsmtFinSF2\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"BsmtFinType1\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BsmtFinType1.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BsmtFinType1\",\"subType\":\"multi\"},{\"name\":\"BsmtFinType2\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BsmtFinType2.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BsmtFinType2\",\"subType\":\"multi\"},{\"name\":\"BsmtFullBath\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"BsmtHalfBath\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"BsmtQual\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"BsmtQual.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"BsmtQual\",\"subType\":\"multi\"},{\"name\":\"BsmtUnfSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"CentralAir\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"CentralAir.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"CentralAir\",\"subType\":\"multi\"},{\"name\":\"Condition1\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Condition1.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Condition1\",\"subType\":\"multi\"},{\"name\":\"Condition2\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Condition2.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Condition2\",\"subType\":\"multi\"},{\"name\":\"Electrical\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Electrical.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Electrical\",\"subType\":\"multi\"},{\"name\":\"EnclosedPorch\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ExterCond\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ExterCond.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ExterCond\",\"subType\":\"multi\"},{\"name\":\"ExterQual\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ExterQual.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ExterQual\",\"subType\":\"multi\"},{\"name\":\"Exterior1st\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Exterior1st.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Exterior1st\",\"subType\":\"multi\"},{\"name\":\"Exterior2nd\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Exterior2nd.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Exterior2nd\",\"subType\":\"multi\"},{\"name\":\"Fence\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Fence.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Fence\",\"subType\":\"multi\"},{\"name\":\"FireplaceQu\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"FireplaceQu.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"FireplaceQu\",\"subType\":\"multi\"},{\"name\":\"Fireplaces\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"FirstFlrSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Foundation\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Foundation.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Foundation\",\"subType\":\"multi\"},{\"name\":\"FullBath\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Functional\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Functional.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Functional\",\"subType\":\"multi\"},{\"name\":\"GarageArea\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"GarageCars\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"GarageCond\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"GarageCond.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"GarageCond\",\"subType\":\"multi\"},{\"name\":\"GarageFinish\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"GarageFinish.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"GarageFinish\",\"subType\":\"multi\"},{\"name\":\"GarageQual\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"GarageQual.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"GarageQual\",\"subType\":\"multi\"},{\"name\":\"GarageType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"GarageType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"GarageType\",\"subType\":\"multi\"},{\"name\":\"GarageYrBlt\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"GrLivArea\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"HalfBath\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Heating\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Heating.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Heating\",\"subType\":\"multi\"},{\"name\":\"HeatingQC\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"HeatingQC.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"HeatingQC\",\"subType\":\"multi\"},{\"name\":\"HouseStyle\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"HouseStyle.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"HouseStyle\",\"subType\":\"multi\"},{\"name\":\"Id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"KitchenAbvGr\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"KitchenQual\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"KitchenQual.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"KitchenQual\",\"subType\":\"multi\"},{\"name\":\"LandContour\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"LandContour.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"LandContour\",\"subType\":\"multi\"},{\"name\":\"LandSlope\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"LandSlope.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"LandSlope\",\"subType\":\"multi\"},{\"name\":\"LotArea\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"LotConfig\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"LotConfig.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"LotConfig\",\"subType\":\"multi\"},{\"name\":\"LotFrontage\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"LotShape\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"LotShape.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"LotShape\",\"subType\":\"multi\"},{\"name\":\"LowQualFinSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"MSSubClass\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"MSZoning\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"MSZoning.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"MSZoning\",\"subType\":\"multi\"},{\"name\":\"MasVnrArea\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"MasVnrType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"MasVnrType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"MasVnrType\",\"subType\":\"multi\"},{\"name\":\"MiscFeature\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"MiscFeature.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"MiscFeature\",\"subType\":\"multi\"},{\"name\":\"MiscVal\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"MoSold\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Neighborhood\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Neighborhood.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Neighborhood\",\"subType\":\"multi\"},{\"name\":\"OpenPorchSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"OverallCond\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"OverallQual\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"PavedDrive\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"PavedDrive.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"PavedDrive\",\"subType\":\"multi\"},{\"name\":\"PoolArea\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RoofMatl\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"RoofMatl.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"RoofMatl\",\"subType\":\"multi\"},{\"name\":\"RoofStyle\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"RoofStyle.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"RoofStyle\",\"subType\":\"multi\"},{\"name\":\"SaleCondition\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"SaleCondition.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"SaleCondition\",\"subType\":\"multi\"},{\"name\":\"SaleType\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"SaleType.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"SaleType\",\"subType\":\"multi\"},{\"name\":\"ScreenPorch\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"SecondFlrSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Street\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Street.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Street\",\"subType\":\"multi\"},{\"name\":\"ThreeSsnPortch\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TotRmsAbvGrd\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TotalBsmtSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Utilities\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Utilities.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Utilities\",\"subType\":\"multi\"},{\"name\":\"WoodDeckSF\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"YearBuilt\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"YearRemodAdd\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"YrSold\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tags.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tags\",\"subType\":\"multi\"},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]","timeFieldName":"@timestamp","title":"input*"},"id":"61d12f10-4b74-11ea-a505-e57bbdfb6038","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-02-15T08:38:19.009Z","version":"WzExLDJd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"BsmtQual Pie Chart","uiStateJSON":"{\"vis\":{\"legendOpen\":true}}","version":1,"visState":"{\"title\":\"BsmtQual Pie Chart\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"BsmtQual.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"1d3afa50-4b76-11ea-a505-e57bbdfb6038","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"61d12f10-4b74-11ea-a505-e57bbdfb6038","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-02-15T08:38:19.009Z","version":"WzEyLDJd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"LotArea Line Graph","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"LotArea Line Graph\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"range\",\"schema\":\"segment\",\"params\":{\"field\":\"LotArea\",\"ranges\":[{\"from\":0,\"to\":10000},{\"from\":10000,\"to\":20000},{\"from\":20000,\"to\":30000},{\"from\":40000,\"to\":50000},{\"from\":50000}]}}]}"},"id":"49eceef0-4b76-11ea-a505-e57bbdfb6038","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"61d12f10-4b74-11ea-a505-e57bbdfb6038","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-02-15T08:38:19.009Z","version":"WzEzLDJd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"76c9e383-bdbb-4658-8795-118493af4ec3\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"76c9e383-bdbb-4658-8795-118493af4ec3\",\"version\":\"7.5.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"17e4efa7-9495-4df4-b7cf-1c3900042d8d\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"17e4efa7-9495-4df4-b7cf-1c3900042d8d\",\"version\":\"7.5.1\",\"panelRefName\":\"panel_1\"}]","timeRestore":false,"title":"Example Inputs Dashboard","version":1},"id":"2daf01d0-4fcf-11ea-bad8-6dbf60384395","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"1d3afa50-4b76-11ea-a505-e57bbdfb6038","name":"panel_0","type":"visualization"},{"id":"49eceef0-4b76-11ea-a505-e57bbdfb6038","name":"panel_1","type":"visualization"}],"type":"dashboard","updated_at":"2020-02-15T08:43:03.149Z","version":"WzIyLDJd"} +{"exportedCount":4,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/packages/ml_api/docker/logstash/config/logstash.yml b/packages/ml_api/docker/logstash/config/logstash.yml new file mode 100644 index 0000000..9f69eac --- /dev/null +++ b/packages/ml_api/docker/logstash/config/logstash.yml @@ -0,0 +1,11 @@ +--- +## Default Logstash configuration from Logstash base image. +## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml +# +http.host: "0.0.0.0" +xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] + +## X-Pack security credentials +xpack.monitoring.enabled: true +xpack.monitoring.elasticsearch.username: elastic +xpack.monitoring.elasticsearch.password: changeme \ No newline at end of file diff --git a/packages/ml_api/docker/logstash/pipeline/logstash.conf b/packages/ml_api/docker/logstash/pipeline/logstash.conf new file mode 100644 index 0000000..8cb6770 --- /dev/null +++ b/packages/ml_api/docker/logstash/pipeline/logstash.conf @@ -0,0 +1,26 @@ +input { + tcp { + port => 5001 + tags => ["webapp_logs"] + type => "webapp_logs" + codec => json + } +} + +output { + if [LotArea] { + elasticsearch { + hosts => "elasticsearch:9200" + user => "elastic" + password => "changeme" + index => "input_logs-%{+YYYY.MM.dd}" + } + } else { + elasticsearch { + hosts => "elasticsearch:9200" + user => "elastic" + password => "changeme" + index => "webapp_logs-%{+YYYY.MM.dd}" + } + } +} \ No newline at end of file diff --git a/packages/ml_api/docker/workaround_32_os/Dockerfile.workaround b/packages/ml_api/docker/workaround_32_os/Dockerfile.workaround new file mode 100644 index 0000000..ca8c7e9 --- /dev/null +++ b/packages/ml_api/docker/workaround_32_os/Dockerfile.workaround @@ -0,0 +1,20 @@ +FROM python:3.7.5-slim-buster + +RUN mkdir -p /opt/app +COPY requirements /opt/app/requirements +RUN pip install --upgrade pip +RUN pip install tox + +# ensure we can run the make commands +RUN apt-get update -y && \ + apt-get install -y make && \ + apt-get install -y libffi-dev gcc && \ + # for swagger + apt-get install -y curl + +RUN pip install -r /opt/app/requirements/test_requirements.txt +COPY tests /opt/app/tests +COPY tox.ini /opt/app/tox.ini +COPY api /opt/app/api +COPY run.py /opt/app/run.py +WORKDIR /opt/app diff --git a/packages/ml_api/docker/workaround_32_os/docker-compose-workaround.yml b/packages/ml_api/docker/workaround_32_os/docker-compose-workaround.yml new file mode 100644 index 0000000..171d4e2 --- /dev/null +++ b/packages/ml_api/docker/workaround_32_os/docker-compose-workaround.yml @@ -0,0 +1,13 @@ +# This is only to be used as a workaround for students who +# are unable to install the gradient_boosting_model package +# because they are on a 32 bit operating system + +version: '3' +services: + ml_api: + build: + context: ../../ + dockerfile: docker/workaround_32_os/Dockerfile.workaround + ports: + - "5000:5000" + command: bash -c "tox -e integration_tests" diff --git a/packages/ml_api/gunicorn_logging.conf b/packages/ml_api/gunicorn_logging.conf new file mode 100644 index 0000000..f4a2bdf --- /dev/null +++ b/packages/ml_api/gunicorn_logging.conf @@ -0,0 +1,49 @@ +[loggers] +keys=root, mlapi, logstash.error, logstash.access + +[handlers] +keys=console, logstash + +[formatters] +keys=generic, json + +[logger_root] +level=INFO +handlers=console +propagate=1 + +[logger_mlapi] +level=INFO +handlers=console,logstash +propagate=0 +qualname=mlapi + +[logger_logstash.error] +level=INFO +handlers=logstash +propagate=1 +qualname=gunicorn.error + +[logger_logstash.access] +level=INFO +handlers=logstash +propagate=0 +qualname=gunicorn.access + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[handler_logstash] +class=logstash.TCPLogstashHandler +formatter=json +args=('logstash', 5001) + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s +datefmt=%Y-%m-%d %H:%M:%S +class=logging.Formatter + +[formatter_json] +class=pythonjsonlogger.jsonlogger.JsonFormatter diff --git a/packages/ml_api/mypy.ini b/packages/ml_api/mypy.ini new file mode 100644 index 0000000..97e52a5 --- /dev/null +++ b/packages/ml_api/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +warn_unused_ignores = True +follow_imports = skip +show_error_context = True +warn_incomplete_stub = True +ignore_missing_imports = True +check_untyped_defs = True +cache_dir = /dev/null +warn_redundant_casts = True +warn_unused_configs = True +strict_optional = True diff --git a/packages/ml_api/requirements/requirements.txt b/packages/ml_api/requirements/requirements.txt new file mode 100644 index 0000000..2cab5cd --- /dev/null +++ b/packages/ml_api/requirements/requirements.txt @@ -0,0 +1,30 @@ +# ML Model +tid-gradient-boosting-model>=0.1.18,<0.2.0 + +# Old model +tid-regression-model>=2.0.20,<2.1.0 + +# Web microframework for the API +flask>=1.1.1,<1.2.0 +connexion[swagger-ui]>=2.5.1,<2.6.0 + +# repo maintenance tooling +black>=19.10b0,<20.0 +flake8>=3.7.9,<4.0 +mypy>=0.740 + +# Persistence +sqlalchemy>=1.3.11,<1.4.0 # ORM +psycopg2>=2.8.4,<2.9.0 # DB Driver +alembic>=1.3.1,<1.4.0 # DB Migrations +sqlalchemy_utils>=0.36.0,<0.37.0 # DB Utils + +# Metrics +prometheus_client>=0.7.1,<0.8.0 + +# Logging +python3-logstash>=0.4.80,<0.5.0 +python-json-logger>=0.1.11,<0.2.0 + +# Deployment +gunicorn>=20.0.4,<20.1.0 diff --git a/packages/ml_api/requirements/test_requirements.txt b/packages/ml_api/requirements/test_requirements.txt new file mode 100644 index 0000000..c909f64 --- /dev/null +++ b/packages/ml_api/requirements/test_requirements.txt @@ -0,0 +1,14 @@ +-r requirements.txt + +# testing requirements +pytest>=5.3.2,<6.0.0 +requests>=2.22.0,<2.23.0 + +# repo maintenance tooling +black>=19.10b0,<20.0 +flake8>=3.7.9,<4.0 +mypy>=0.740 + +# diff test tooling +termcolor==1.1.0 +yarl==1.3.0 \ No newline at end of file diff --git a/packages/ml_api/run.py b/packages/ml_api/run.py new file mode 100644 index 0000000..5bb5e0a --- /dev/null +++ b/packages/ml_api/run.py @@ -0,0 +1,19 @@ +import prometheus_client +from werkzeug.middleware.dispatcher import DispatcherMiddleware + +from api.app import create_app +from api.config import DevelopmentConfig, setup_app_logging + +_config = DevelopmentConfig() + +# setup logging as early as possible +setup_app_logging(config=_config) +main_app = create_app(config_object=_config).app +application = DispatcherMiddleware( + app=main_app.wsgi_app, + mounts={'/metrics': prometheus_client.make_wsgi_app()} + ) + + +if __name__ == "__main__": + main_app.run(port=_config.SERVER_PORT, host=_config.SERVER_HOST) diff --git a/packages/ml_api/scripts/differential_tests.sh b/packages/ml_api/scripts/differential_tests.sh new file mode 100755 index 0000000..98c8a91 --- /dev/null +++ b/packages/ml_api/scripts/differential_tests.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -euox pipefail + +MODEL_VERSION="master" +MODEL_VARIANT="candidate" +NUMBER_OF_TESTS="50" + +CANDIDATE_MODEL_SHA="$(git rev-parse HEAD)" + +# required once only (or whenever you make local changes): +# comment these two lines out otherwise as they can take some time. +make tag-push-local + +# should only be run once a model version has been finalized +# best practice is to run as part of a CI pipeline on merge to master branch. +make tag-push-master + +## Pull latest published image +env TARGET=master docker-compose --file docker/docker-compose.yml pull + +# start latest (master) image and local image +env TARGET=master SERVER_PORT=5000 docker-compose --project-name master --file docker/docker-compose-ci-master.yml up --no-recreate -d ml_api +env TARGET=$CANDIDATE_MODEL_SHA SERVER_PORT=5001 docker-compose --project-name head --file docker/docker-compose-ci-candidate.yml up --no-recreate -d ml_api + +## Start the test runner containers +env TARGET=master docker-compose --project-name master --file docker/docker-compose-ci-master.yml run -d --name differential-tests-expected differential-tests sleep infinity +env TARGET=$CANDIDATE_MODEL_SHA docker-compose --project-name head --file docker/docker-compose-ci-candidate.yml run -d --name differential-tests-actual differential-tests sleep infinity + +docker ps --all + +echo "===== Running $CANDIDATE_MODEL_SHA ... =====" + +## Compute the actual predictions (i.e. candidate model) +docker exec --user root differential-tests-actual \ + python3 differential_tests compute sample_payloads differential_tests/actual_results --base-url http://head_ml_api_1:5001 + +## Copy the actual predictions +docker cp differential-tests-actual:/opt/app/differential_tests/actual_results/. differential_tests/actual_results + +echo "===== Running master ... =====" +## Compute the expected marginals (i.e. existing model) +docker exec --user root differential-tests-expected \ + python3 differential_tests compute sample_payloads differential_tests/expected_results --base-url http://master_ml_api_1:5000 + +## Copy the expected marginals +docker cp differential-tests-expected:/opt/app/differential_tests/expected_results/. differential_tests/expected_results + +# then copy all results into the differential-tests-actual container for comparison +docker cp differential_tests/expected_results/. differential-tests-actual:/opt/app/differential_tests/expected_results + +echo "===== Comparing $CANDIDATE_MODEL_SHA vs. master ... =====" +## Compare the expected and actual marginals +docker exec differential-tests-actual \ + python3 -m differential_tests compare differential_tests/expected_results differential_tests/actual_results + +# clear any docker containers (will stop the script if no containers found) +docker rm $(docker ps -a -q) -f diff --git a/packages/ml_api/scripts/populate_database.py b/packages/ml_api/scripts/populate_database.py new file mode 100644 index 0000000..1a40323 --- /dev/null +++ b/packages/ml_api/scripts/populate_database.py @@ -0,0 +1,121 @@ +import argparse +import os +import time +import typing as t +from random import randint, choice + +import pandas as pd +import requests +from gradient_boosting_model.config.core import config +from gradient_boosting_model.processing.data_management import load_dataset + +LOCAL_URL = f'http://{os.getenv("DB_HOST", "localhost")}:5000' + +HEADERS = {"Accept": "application/json", "Content-Type": "application/json"} + +LOT_AREA_MAP = {"min": 1470, "max": 56600} + +FIRST_FLR_SF_MAP = {"min": 407, "max": 5095} + +SECOND_FLR_SF_MAP = {"min": 0, "max": 1862} + +BSMT_QUAL_VALUES = ('Gd', 'TA', 'Ex', 'Fa') + + +def _generate_random_int(value: int, value_ranges: t.Mapping) -> int: + """Generate random integer within a min and max range.""" + random_value = randint(value_ranges["min"], value_ranges["max"]) + return int(random_value) + + +def _select_random_category(value: str, value_options: t.Sequence) -> str: + """Select random category given a sequence of categories.""" + random_category = choice(value_options) + return random_category + + +def _prepare_inputs(dataframe: pd.DataFrame) -> pd.DataFrame: + """Prepare input data by removing key rows with NA values.""" + clean_inputs_df = dataframe.dropna( + subset=config.model_config.features + ["KitchenQual", "LotFrontage"] + ).copy() + + clean_inputs_df.loc[:, "FirstFlrSF"] = clean_inputs_df["FirstFlrSF"].apply( + _generate_random_int, value_ranges=FIRST_FLR_SF_MAP + ) + clean_inputs_df.loc[:, "SecondFlrSF"] = clean_inputs_df["SecondFlrSF"].apply( + _generate_random_int, value_ranges=SECOND_FLR_SF_MAP + ) + clean_inputs_df.loc[:, "LotArea"] = clean_inputs_df["LotArea"].apply( + _generate_random_int, value_ranges=LOT_AREA_MAP + ) + + clean_inputs_df.loc[:, "BsmtQual"] = clean_inputs_df["BsmtQual"].apply( + _select_random_category, value_options=BSMT_QUAL_VALUES + ) + + return clean_inputs_df + + +def populate_database(n_predictions: int = 500, anomaly: bool = False) -> None: + """ + Manipulate the test data to generate random + predictions and save them to the database. + Before running this script, ensure that the + API and Database docker containers are running. + """ + + print(f"Preparing to generate: {n_predictions} predictions.") + + # Load the gradient boosting test dataset which + # is included in the model package + test_inputs_df = load_dataset(file_name="test.csv") + clean_inputs_df = _prepare_inputs(dataframe=test_inputs_df) + if len(clean_inputs_df) < n_predictions: + print( + f"If you want {n_predictions} predictions, you need to" + "extend the script to handle more predictions." + ) + + if anomaly: + # set extremely low values to generate an outlier + n_predictions = 1 + clean_inputs_df.loc[:, "FirstFlrSF"] = 1 + clean_inputs_df.loc[:, "LotArea"] = 1 + clean_inputs_df.loc[:, "OverallQual"] = 1 + clean_inputs_df.loc[:, "GrLivArea"] = 1 + + clean_inputs_df = clean_inputs_df.where(pd.notnull(clean_inputs_df), None) + for index, data in clean_inputs_df.iterrows(): + if index > n_predictions: + if anomaly: + print('Created 1 anomaly') + break + + response = requests.post( + f"{LOCAL_URL}/v1/predictions/regression", + headers=HEADERS, + json=[data.to_dict()], + ) + response.raise_for_status() + + if index % 50 == 0: + print(f"{index} predictions complete") + + # prevent overloading the server + time.sleep(0.5) + + print("Prediction generation complete.") + + +if __name__ == "__main__": + anomaly = False + parser = argparse.ArgumentParser( + description='Send random requests to House Price API.') + parser.add_argument('--anomaly', help="generate unusual inputs") + args = parser.parse_args() + if args.anomaly: + print("Generating unusual inputs") + anomaly = True + + populate_database(n_predictions=500, anomaly=anomaly) diff --git a/packages/ml_api/tests/__init__.py b/packages/ml_api/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/ml_api/tests/conftest.py b/packages/ml_api/tests/conftest.py new file mode 100644 index 0000000..8939e04 --- /dev/null +++ b/packages/ml_api/tests/conftest.py @@ -0,0 +1,54 @@ +import os + +from unittest import mock +import pytest +from gradient_boosting_model.processing.data_management import load_dataset +from sqlalchemy_utils import create_database, database_exists + +from api.app import create_app +from api.config import TestingConfig +from api.persistence import core + + +@pytest.fixture(scope='session') +def _db(): + db_url = TestingConfig.SQLALCHEMY_DATABASE_URI + if not database_exists(db_url): + create_database(db_url) + # alembic can be configured through the configuration file. For testing + # purposes 'env.py' also checks the 'ALEMBIC_DB_URI' variable first. + engine = core.create_db_engine_from_config(config=TestingConfig()) + evars = {"ALEMBIC_DB_URI": db_url} + with mock.patch.dict(os.environ, evars): + core.run_migrations() + + yield engine + + +@pytest.fixture(scope='session') +def _db_session(_db): + """ Create DB session for testing. + """ + session = core.create_db_session(engine=_db) + yield session + + +@pytest.fixture(scope='session') +def app(_db_session): + app = create_app(config_object=TestingConfig(), db_session=_db_session).app + with app.app_context(): + yield app + + +@pytest.fixture +def client(app): + with app.test_client() as client: + yield client # Has to be yielded to access session cookies + + +@pytest.fixture +def test_inputs_df(): + # Load the gradient boosting test dataset which + # is included in the model package + test_inputs_df = load_dataset(file_name="test.csv") + return test_inputs_df.copy(deep=True) diff --git a/packages/ml_api/tests/test_api.py b/packages/ml_api/tests/test_api.py new file mode 100644 index 0000000..3d9fb5b --- /dev/null +++ b/packages/ml_api/tests/test_api.py @@ -0,0 +1,135 @@ +import json +import time + +import numpy as np +import pytest + +from api.persistence.data_access import SECONDARY_VARIABLES_TO_RENAME +from api.persistence.models import ( + GradientBoostingModelPredictions, + LassoModelPredictions, +) +from gradient_boosting_model.processing.data_management import load_dataset + + +@pytest.mark.integration +def test_health_endpoint(client): + # When + response = client.get("/") + + # Then + assert response.status_code == 200 + assert json.loads(response.data) == {"status": "ok"} + + +@pytest.mark.integration +@pytest.mark.parametrize( + "api_endpoint, expected_no_predictions", + ( + ( + "v1/predictions/regression", + # test csv contains 1459 rows + # we expect 2 rows to be filtered + 1451, + ), + ( + "v1/predictions/gradient", + # we expect 8 rows to be filtered + 1457, + ), + ), +) +def test_prediction_endpoint( + api_endpoint, expected_no_predictions, client, test_inputs_df +): + # Given + # Load the test dataset which is included in the model package + test_inputs_df = load_dataset(file_name="test.csv") # dataframe + if api_endpoint == "v1/predictions/regression": + # adjust column names to those expected by the secondary model + test_inputs_df.rename(columns=SECONDARY_VARIABLES_TO_RENAME, inplace=True) + + # When + response = client.post(api_endpoint, json=test_inputs_df.to_dict(orient="records")) + + # Then + assert response.status_code == 200 + data = json.loads(response.data) + assert data["errors"] is None + assert len(data["predictions"]) == expected_no_predictions + + +# parameterizationa allows us to try many combinations of data +# within the same test, see the pytest docs for details: +# https://docs.pytest.org/en/latest/parametrize.html +@pytest.mark.parametrize( + "field, field_value, index, expected_error", + ( + ( + "BldgType", + 1, # expected str + 33, + {"33": {"BldgType": ["Not a valid string."]}}, + ), + ( + "GarageArea", # model feature + "abc", # expected float + 45, + {"45": {"GarageArea": ["Not a valid number."]}}, + ), + ( + "CentralAir", + np.nan, # nan not allowed + 34, + {"34": {"CentralAir": ["Field may not be null."]}}, + ), + ("LotArea", "", 2, {"2": {"LotArea": ["Not a valid integer."]}}), + ), +) +@pytest.mark.integration +def test_prediction_validation( + field, field_value, index, expected_error, client, test_inputs_df +): + # Given + # Check gradient_boosting_model.processing.validation import HouseDataInputSchema + # and you will see the expected values for the inputs to the house price prediction + # model. In this test, inputs are changed to incorrect values to check the validation. + test_inputs_df.loc[index, field] = field_value + + # When + response = client.post( + "/v1/predictions/gradient", json=test_inputs_df.to_dict(orient="records") + ) + + # Then + assert response.status_code == 400 + data = json.loads(response.data) + assert data == expected_error + + +@pytest.mark.integration +def test_prediction_data_saved(client, app, test_inputs_df): + # Given + initial_gradient_count = app.db_session.query( + GradientBoostingModelPredictions + ).count() + initial_lasso_count = app.db_session.query(LassoModelPredictions).count() + + # When + response = client.post( + "/v1/predictions/regression", json=test_inputs_df.to_dict(orient="records") + ) + + # Then + assert response.status_code == 200 + assert ( + app.db_session.query(LassoModelPredictions).count() == initial_lasso_count + 1 + ) + + # The gradient prediction save occurs on a separate async thread which can take + # time to complete. We pause the test briefly to allow the save operation to finish. + time.sleep(2) + assert ( + app.db_session.query(GradientBoostingModelPredictions).count() + == initial_gradient_count + 1 + ) diff --git a/packages/ml_api/tests/test_back_to_back_models.py b/packages/ml_api/tests/test_back_to_back_models.py new file mode 100644 index 0000000..af98797 --- /dev/null +++ b/packages/ml_api/tests/test_back_to_back_models.py @@ -0,0 +1,37 @@ +import json + +import pytest +from gradient_boosting_model.processing.data_management import load_dataset + +from api.persistence.data_access import SECONDARY_VARIABLES_TO_RENAME +from differential_tests.compare import compare_differences + + +@pytest.mark.differential +def test_model_prediction_differentials(client): + test_inputs_df = load_dataset(file_name="test.csv") + old_model_inputs_df = test_inputs_df.rename( + columns=SECONDARY_VARIABLES_TO_RENAME + ) + + new_model_response = client.post( + "v1/predictions/gradient", json=test_inputs_df.to_dict(orient="records") + ) + new_model_predictions = json.loads(new_model_response.data)["predictions"] + + old_model_response = client.post( + "v1/predictions/regression", + json=old_model_inputs_df.to_dict(orient="records"), + ) + old_model_predictions = json.loads(old_model_response.data)["predictions"] + + # We just pass in the first 10 rows as the two models' validation differs + # which means they filter out a slightly different number of rows + # which would cause the differential tests to fail. + compare_differences( + expected_predictions=new_model_predictions[:10], + actual_predictions=old_model_predictions[:10], + # you would adjust the rel_tol level parameter on your model. + # right now this is extremely permissive of variation. + rel_tol=0.2, + ) diff --git a/packages/ml_api/tests/test_persistence.py b/packages/ml_api/tests/test_persistence.py new file mode 100644 index 0000000..8172157 --- /dev/null +++ b/packages/ml_api/tests/test_persistence.py @@ -0,0 +1,36 @@ +from unittest import mock +import pytest + +from api.persistence.data_access import PredictionPersistence, ModelType + +from api.persistence.models import ( + GradientBoostingModelPredictions, + LassoModelPredictions, +) + + +# parameterizationa allows us to try many combinations of data +# within the same test, see the pytest docs for details: +# https://docs.pytest.org/en/latest/parametrize.html +@pytest.mark.parametrize( + "model_type, model,", + ( + (ModelType.GRADIENT_BOOSTING, GradientBoostingModelPredictions), + (ModelType.LASSO, LassoModelPredictions), + ), +) +def test_data_access(model_type, model, test_inputs_df): + # Given + # We mock the database session + mock_session = mock.MagicMock() + _persistence = PredictionPersistence(db_session=mock_session) + + # When + _persistence.make_save_predictions( + db_model=model_type, input_data=test_inputs_df.to_dict(orient="records") + ) + + # Then + assert mock_session.commit.call_count == 1 + assert mock_session.add.call_count == 1 + assert isinstance(mock_session.add.call_args[0][0], model) diff --git a/packages/ml_api/tox.ini b/packages/ml_api/tox.ini new file mode 100644 index 0000000..031f45f --- /dev/null +++ b/packages/ml_api/tox.ini @@ -0,0 +1,141 @@ +[tox] +envlist = integration_tests,unit_tests,differential_tests,typechecks,stylechecks +skipsdist = True + + +[testenv] +install_command = pip install {opts} {packages} + +deps = + -rrequirements/test_requirements.txt + +setenv = + PYTHONPATH=. + +passenv = +# A list of wildcard environment variable names which shall be copied from +# the tox invocation environment to the test environment when executing test commands + DB_* + SHADOW_MODE_ACTIVE + +commands= + py.test + + +[testenv:integration_tests] +envdir = {toxworkdir}/integration_tests +deps = + {[testenv]deps} + +passenv = + {[testenv]passenv} + +setenv = + PYTHONPATH=. + DB_USER={env:DB_USER:test_user} + DB_PASSWORD={env:DB_PASSWORD:password} + DB_HOST={env:DB_HOST:localhost} + DB_PORT={env:DB_PORT:6608} + DB_NAME={env:DB_NAME:ml_api_test} + SHADOW_MODE_ACTIVE={env:SHADOW_MODE_ACTIVE:true} + +commands = + pytest \ + -s \ + -vv \ + -m integration \ + {posargs:tests/} + + +[testenv:unit_tests] +envdir = {toxworkdir}/integration_tests +deps = + {[testenv]deps} + +passenv = + {[testenv]passenv} + +setenv = + PYTHONPATH=. + +commands = + pytest \ + -s \ + -vv \ + -m "not integration and not differential" \ + {posargs:tests/} + + +[testenv:differential_tests] +envdir = {toxworkdir}/integration_tests +deps = + {[testenv]deps} + +passenv = + {[testenv]passenv} + +setenv = + PYTHONPATH=. + DB_USER={env:DB_USER:test_user} + DB_PASSWORD={env:DB_PASSWORD:password} + DB_HOST={env:DB_HOST:localhost} + DB_PORT={env:DB_PORT:6608} + DB_NAME={env:DB_NAME:ml_api_test} + SHADOW_MODE_ACTIVE={env:SHADOW_MODE_ACTIVE:true} + +commands = + pytest \ + -s \ + -vv \ + -m differential \ + {posargs:tests/} + + +[testenv:generate_predictions] +envdir = {toxworkdir}/generate_predictions +deps = + {[testenv]deps} + +passenv = + {[testenv]passenv} + +setenv = + PYTHONPATH=. + DB_HOST=localhost + +commands = python scripts/populate_database.py {posargs} + + +[testenv:typechecks] +envdir = {toxworkdir}/integration_tests + +deps = + {[testenv:integration_tests]deps} + +commands = {posargs:mypy api} + + +[testenv:stylechecks] +envdir = {toxworkdir}/integration_tests + +deps = + {[testenv:integration_tests]deps} + +commands = {posargs:flake8 api tests} + + +[flake8] +exclude = .git,env +max-line-length = 90 + + +[pytest] +markers = + integration: mark a test as an integration test. + differential: mark a test as a differential test. + +filterwarnings = + ignore::DeprecationWarning + ignore::RuntimeWarning + ignore::UserWarning + ignore::FutureWarning diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..e69de29