diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e9a8189c6..597650918 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,7 +6,7 @@ - [ ] I have performed a self-review of the changes -*List here tasks to do in order to complete this PR.* +*List here tasks to complete in order to mark this PR as ready for review.* # To Consider diff --git a/.github/process_github_events.py b/.github/process_github_events.py new file mode 100644 index 000000000..92edc2081 --- /dev/null +++ b/.github/process_github_events.py @@ -0,0 +1,81 @@ +# flake8: noqa +import os +import json +import requests + +GITHUB_ARGS = { + "GITHUB_TOKEN": None, + "GITHUB_REPOSITORY": None, + "GITHUB_EVENT_PATH": None, + "GITHUB_EVENT_NAME": None, +} + +for arg in GITHUB_ARGS: + GITHUB_ARGS[arg] = os.getenv(arg) + + if GITHUB_ARGS[arg] is None: + raise RuntimeError(f"`{arg}` is not set") + + +def post_comment_on_pr(comment: str, pr_number: int): + """ + Leave a comment as `github-actions` bot on a PR. + """ + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f'Bearer {GITHUB_ARGS["GITHUB_TOKEN"]}', + "X-GitHub-Api-Version": "2022-11-28", + } + + escaped_comment = comment.replace("\n", "\\n") + + data = f'{{"body": "{escaped_comment}","event": "COMMENT","comments": []}}' + + response = requests.post( + f'https://api.github.com/repos/{GITHUB_ARGS["GITHUB_REPOSITORY"]}/pulls/{pr_number}/reviews', + headers=headers, + data=data, + ) + + if not response.status_code == 200: + raise RuntimeError(response.__dict__) + + +RELEASE_CHECKLIST = """It appears this PR is a release PR (change its base from `master` if that is not the case). + +Here's a release checklist: + +- [ ] Update package version +- [ ] Change PR merge option +- [ ] Test modules without automated testing: + - [ ] Requiring telegram `api_id` and `api_hash` + - [ ] Requiring `HF_API_KEY` +- [ ] Search for objects to be deprecated +""" + + +def post_release_checklist(pr_payload: dict): + pr_number = pr_payload["number"] + pr_base = pr_payload["base"] + + if pr_base["ref"] == "master": + print("post_release_checklist") + post_comment_on_pr(RELEASE_CHECKLIST, pr_number) + + +def on_opened_pull_request(event_info: dict): + print("on_opened_pull_request") + + post_release_checklist(event_info["pull_request"]) + + +def main(): + with open(GITHUB_ARGS["GITHUB_EVENT_PATH"], "r", encoding="utf-8") as fd: + event_info = json.load(fd) + print(f"event info: {event_info}") + if GITHUB_ARGS["GITHUB_EVENT_NAME"] == "pull_request_target" and event_info["action"] == "opened": + on_opened_pull_request(event_info) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/event_handler.yml b/.github/workflows/event_handler.yml new file mode 100644 index 000000000..4c0832171 --- /dev/null +++ b/.github/workflows/event_handler.yml @@ -0,0 +1,28 @@ +on: + pull_request_target: + types: [ opened ] + +jobs: + event_handler: + strategy: + fail-fast: false + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: install dependencies + run: python -m pip install requests + shell: bash + + - name: handle event + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python .github/process_github_events.py + shell: bash \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4055262c..4b556751b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ To execute all tests, including integration with DBs and APIs tests, run ```bash make test_all ``` -for successful execution of this command `Docker` and `docker-compose` are required. +for successful execution of this command `Docker` and `docker compose` are required. To make sure that the code satisfies only the style requirements, run ```bash @@ -127,7 +127,7 @@ DFF uses docker images for two purposes: The first group can be launched via ```bash -docker-compose --profile context_storage up +docker compose --profile context_storage up ``` This will download and run all the databases (`mysql`, `postgres`, `redis`, `mongo`, `ydb`). @@ -135,14 +135,14 @@ This will download and run all the databases (`mysql`, `postgres`, `redis`, `mon The second group can be launched via ```bash -docker-compose --profile stats up +docker compose --profile stats up ``` This will download and launch Superset Dashboard, Clickhouse, OpenTelemetry Collector. To launch both groups run ```bash -docker-compose --profile context_storage --profile stats up +docker compose --profile context_storage --profile stats up ``` or ```bash diff --git a/README.md b/README.md index 1c5bc1986..ff4e42887 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ pip install dff[benchmark] # dependencies for benchmarking For example, if you are going to use one of the database backends, you can specify the corresponding requirements yourself. Multiple dependencies can be installed at once, e.g. ```bash -pip install dff[postgresql, mysql] +pip install dff[postgresql,mysql] ``` ## Basic example diff --git a/docker-compose.yml b/compose.yml similarity index 76% rename from docker-compose.yml rename to compose.yml index bb5cb93dd..a753b946b 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -1,5 +1,6 @@ version: "3.9" services: + mysql: env_file: [.env_file] image: mysql:latest @@ -10,6 +11,13 @@ services: - 3307:3306 volumes: - mysql-data:/var/lib/mysql + healthcheck: + test: mysql -u $${MYSQL_USERNAME} -p$${MYSQL_PASSWORD} -e "select 1;" + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s + psql: env_file: [.env_file] image: postgres:latest @@ -20,6 +28,13 @@ services: - 5432:5432 volumes: - postgres-data:/var/lib/postgresql/data + healthcheck: + test: psql pg_isready -U $${POSTGRES_USERNAME} -d $${POSTGRES_DB} + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s + redis: env_file: [.env_file] image: redis:latest @@ -31,6 +46,13 @@ services: - 6379:6379 volumes: - redis-data:/data + healthcheck: + test: redis-cli --raw incr ping + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s + mongo: env_file: [.env_file] image: mongo:latest @@ -41,6 +63,13 @@ services: - 27017:27017 volumes: - mongo-data:/data/db + healthcheck: + test: mongosh --norc --quiet --eval 'db.runCommand("ping").ok' localhost:27017/test + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s + ydb: env_file: [.env_file] image: cr.yandex/yc/yandex-docker-local-ydb:latest @@ -55,6 +84,13 @@ services: volumes: - ydb-data:/ydb_data - ydb-certs:/ydb_certs + healthcheck: + test: sh ./health_check + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s + dashboard: env_file: [.env_file] build: @@ -70,6 +106,7 @@ services: - stats ports: - "8088:8088" + dashboard-metadata: env_file: [.env_file] image: postgres:latest @@ -83,11 +120,13 @@ services: command: -p 5433 healthcheck: test: pg_isready -p 5433 --username=$${POSTGRES_USERNAME} - interval: 4s - timeout: 3s - retries: 3 + interval: 5s + timeout: 10s + retries: 5 + start_period: 30s volumes: - dashboard-data:/var/lib/postgresql/data + clickhouse: env_file: [.env_file] image: clickhouse/clickhouse-server:latest @@ -101,10 +140,12 @@ services: volumes: - ch-data:/var/lib/clickhouse/ healthcheck: - test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1 + test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping interval: 5s - timeout: 4s + timeout: 10s retries: 5 + start_period: 30s + otelcol: image: otel/opentelemetry-collector-contrib:latest profiles: @@ -121,6 +162,7 @@ services: ports: - "4317:4317" # OTLP over gRPC receiver - "4318:4318" # OTLP over HTTP receiver + volumes: ch-data: dashboard-data: diff --git a/dff/__init__.py b/dff/__init__.py index 64096bbbb..2d606ee54 100644 --- a/dff/__init__.py +++ b/dff/__init__.py @@ -3,7 +3,7 @@ __author__ = "Denis Kuznetsov" __email__ = "kuznetsov.den.p@gmail.com" -__version__ = "0.6.3" +__version__ = "0.6.4" import nest_asyncio diff --git a/dff/messengers/telegram/interface.py b/dff/messengers/telegram/interface.py index 96b86102f..5d8f1a902 100644 --- a/dff/messengers/telegram/interface.py +++ b/dff/messengers/telegram/interface.py @@ -7,12 +7,11 @@ import asyncio from typing import Any, Optional, List, Tuple, Callable -from telebot import types, logger +from telebot import types, apihelper -from dff.script import Context -from dff.messengers.common import PollingMessengerInterface, PipelineRunnerFunction, CallbackMessengerInterface +from dff.messengers.common import MessengerInterface, PipelineRunnerFunction, CallbackMessengerInterface from .messenger import TelegramMessenger -from .message import TelegramMessage, Message +from .message import TelegramMessage try: from flask import Flask, request, abort @@ -24,6 +23,9 @@ request, abort = None, None +apihelper.ENABLE_MIDDLEWARE = True + + def extract_telegram_request_and_id( update: types.Update, messenger: Optional[TelegramMessenger] = None ) -> Tuple[TelegramMessage, int]: # pragma: no cover @@ -77,7 +79,7 @@ def extract_telegram_request_and_id( return message, ctx_id -class PollingTelegramInterface(PollingMessengerInterface): # pragma: no cover +class PollingTelegramInterface(MessengerInterface): # pragma: no cover """ Telegram interface that retrieves updates by polling. Multi-threaded polling is currently not supported. @@ -111,60 +113,27 @@ def __init__( long_polling_timeout: int = 20, messenger: Optional[TelegramMessenger] = None, ): - self.messenger = messenger if messenger is not None else TelegramMessenger(token) - self.interval = interval + self.messenger = ( + messenger if messenger is not None else TelegramMessenger(token, suppress_middleware_excepions=True) + ) self.allowed_updates = allowed_updates + self.interval = interval self.timeout = timeout self.long_polling_timeout = long_polling_timeout - self._last_processed_update = -1 - self._stop_polling = asyncio.Event() - - def _request(self) -> List[Tuple[Message, int]]: - updates = self.messenger.get_updates( - offset=(self.messenger.last_update_id + 1), - allowed_updates=self.allowed_updates, - timeout=self.timeout, - long_polling_timeout=self.long_polling_timeout, - ) - update_list = [extract_telegram_request_and_id(update, self.messenger) for update in updates] - return update_list - - def _respond(self, response: List[Context]): - for resp in response: - self.messenger.send_response(resp.id, resp.last_response) - update_id = getattr(resp.last_request, "update_id", None) - if update_id is not None: - if update_id > self._last_processed_update: - self._last_processed_update = update_id - - def _on_exception(self, e: Exception): - logger.error(e) - self._stop_polling.set() - - def forget_processed_updates(self): - """ - Forget updates already processed by the pipeline. - """ - self.messenger.get_updates( - offset=self._last_processed_update + 1, - allowed_updates=self.allowed_updates, - timeout=1, - long_polling_timeout=1, - ) async def connect(self, callback: PipelineRunnerFunction, loop: Optional[Callable] = None, *args, **kwargs): - self._stop_polling.clear() - - try: - await super().connect( - callback, loop=loop or (lambda: not self._stop_polling.is_set()), timeout=self.interval - ) - finally: - self.forget_processed_updates() - - def stop(self): - """Stop polling.""" - self._stop_polling.set() + def dff_middleware(bot_instance, update): + message, ctx_id = extract_telegram_request_and_id(update, self.messenger) + + ctx = asyncio.run(callback(message, ctx_id)) + + bot_instance.send_response(ctx_id, ctx.last_response) + + self.messenger.middleware_handler()(dff_middleware) + + self.messenger.infinity_polling( + timeout=self.timeout, long_polling_timeout=self.long_polling_timeout, interval=self.interval + ) class CallbackTelegramInterface(CallbackMessengerInterface): # pragma: no cover diff --git a/dff/messengers/telegram/messenger.py b/dff/messengers/telegram/messenger.py index b1152c578..07919c3d0 100644 --- a/dff/messengers/telegram/messenger.py +++ b/dff/messengers/telegram/messenger.py @@ -85,7 +85,7 @@ def send_response(self, chat_id: Union[str, int], response: Union[str, dict, Mes with open(attachment.source, "rb") as file: method(chat_id, file, **params) else: - method(chat_id, attachment.source or attachment.id, **params) + method(chat_id, str(attachment.source or attachment.id), **params) else: def cast(file): @@ -99,7 +99,7 @@ def cast(file): cast_to_media_type = types.InputMediaVideo else: raise TypeError(type(file)) - return cast_to_media_type(media=file.source or file.id, caption=file.title) + return cast_to_media_type(media=str(file.source or file.id), caption=file.title) files = map(cast, ready_response.attachments.files) with batch_open_io(files) as media: diff --git a/dff/pipeline/pipeline/pipeline.py b/dff/pipeline/pipeline/pipeline.py index c65ac9e7a..49864f7d1 100644 --- a/dff/pipeline/pipeline/pipeline.py +++ b/dff/pipeline/pipeline/pipeline.py @@ -322,7 +322,9 @@ async def _run_pipeline(self, request: Message, ctx_id: Optional[Hashable] = Non :param ctx_id: Current dialog id; if `None`, new dialog will be created. :return: Dialog `Context`. """ - if isinstance(self.context_storage, DBContextStorage): + if ctx_id is None: + ctx = Context() + elif isinstance(self.context_storage, DBContextStorage): ctx = await self.context_storage.get_async(ctx_id, Context(id=ctx_id)) else: ctx = self.context_storage.get(ctx_id, Context(id=ctx_id)) diff --git a/docs/source/conf.py b/docs/source/conf.py index e445f0eaa..489fd814f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ author = "DeepPavlov" # The full version, including alpha/beta/rc tags -release = "0.6.3" +release = "0.6.4" # -- General configuration --------------------------------------------------- diff --git a/docs/source/user_guides/context_guide.rst b/docs/source/user_guides/context_guide.rst index 0583aecc1..4f3f2dbc7 100644 --- a/docs/source/user_guides/context_guide.rst +++ b/docs/source/user_guides/context_guide.rst @@ -214,7 +214,7 @@ and the connection parameters, for example, *mongodb://admin:pass@localhost:2701 The GitHub-based distribution of DFF includes Docker images for each of the supported database types. Therefore, the easiest way to deploy your service together with a database is to clone the GitHub distribution and to take advantage of the packaged -`docker-compose file `_. +`docker compose file `_. .. code-block:: shell :linenos: @@ -222,9 +222,9 @@ distribution and to take advantage of the packaged git clone https://github.com/deeppavlov/dialog_flow_framework.git cd dialog_flow_framework # assuming we need to deploy mongodb - docker-compose up mongo + docker compose up mongo -The images can be configured using the docker-compose file or the +The images can be configured using the docker compose file or the `environment file `_, also available in the distribution. Consult these files for more options. diff --git a/docs/source/user_guides/superset_guide.rst b/docs/source/user_guides/superset_guide.rst index d3b4a29a1..a819fd544 100644 --- a/docs/source/user_guides/superset_guide.rst +++ b/docs/source/user_guides/superset_guide.rst @@ -21,7 +21,7 @@ Collection procedure .. code-block:: shell :linenos: - # clone the original repository to access the docker-compose file + # clone the original repository to access the docker compose file git clone https://github.com/deeppavlov/dialog_flow_framework.git # install with the stats extra cd dialog_flow_framework @@ -32,11 +32,11 @@ Collection procedure .. code-block:: shell :linenos: - # clone the original repository to access the docker-compose file + # clone the original repository to access the docker compose file git clone https://github.com/deeppavlov/dialog_flow_framework.git # launch the required services cd dialog_flow_framework - docker-compose --profile stats up + docker compose --profile stats up **Collecting data** diff --git a/makefile b/makefile index 97c51b1c6..b7d8fea18 100644 --- a/makefile +++ b/makefile @@ -5,7 +5,7 @@ SHELL = /bin/bash PYTHON = python3 VENV_PATH = venv VERSIONING_FILES = setup.py makefile docs/source/conf.py dff/__init__.py -CURRENT_VERSION = 0.6.3 +CURRENT_VERSION = 0.6.4 TEST_COVERAGE_THRESHOLD=95 TEST_ALLOW_SKIP=all # for more info, see tests/conftest.py @@ -50,19 +50,14 @@ lint: venv .PHONY: lint docker_up: - docker-compose --profile context_storage --profile stats up -d --build + docker compose --profile context_storage --profile stats up -d --build --wait .PHONY: docker_up -wait_db: docker_up - while ! docker-compose exec psql pg_isready; do sleep 1; done > /dev/null - while ! docker-compose exec mysql bash -c 'mysql -u $$MYSQL_USERNAME -p$$MYSQL_PASSWORD -e "select 1;"'; do sleep 1; done &> /dev/null -.PHONY: wait_db - test: venv source <(cat .env_file | sed 's/=/=/' | sed 's/^/export /') && pytest -m "not no_coverage" --cov-fail-under=$(TEST_COVERAGE_THRESHOLD) --cov-report html --cov-report term --cov=dff --allow-skip=$(TEST_ALLOW_SKIP) tests/ .PHONY: test -test_all: venv wait_db test lint +test_all: venv docker_up test lint .PHONY: test_all build_drawio: diff --git a/setup.py b/setup.py index c83f481b7..f38e18e46 100644 --- a/setup.py +++ b/setup.py @@ -216,7 +216,7 @@ def merge_req_lists(*req_lists: List[str]) -> List[str]: setup( name="dff", - version="0.6.3", + version="0.6.4", description=description, long_description=long_description, long_description_content_type="text/markdown", diff --git a/tutorials/messengers/telegram/5_conditions_with_media.py b/tutorials/messengers/telegram/5_conditions_with_media.py index 25b555138..7f80ae3c6 100644 --- a/tutorials/messengers/telegram/5_conditions_with_media.py +++ b/tutorials/messengers/telegram/5_conditions_with_media.py @@ -34,11 +34,7 @@ # %% -picture_url = ( - "https://gist.githubusercontent.com/scotthaleen/" - "32f76a413e0dfd4b4d79c2a534d49c0b/raw" - "/6c1036b1eca90b341caf06d4056d36f64fc11e88/tiny.jpg" -) +picture_url = "https://avatars.githubusercontent.com/u/29918795?s=200&v=4" # %% [markdown] diff --git a/tutorials/messengers/telegram/6_conditions_extras.py b/tutorials/messengers/telegram/6_conditions_extras.py index eee19866f..db161df03 100644 --- a/tutorials/messengers/telegram/6_conditions_extras.py +++ b/tutorials/messengers/telegram/6_conditions_extras.py @@ -83,7 +83,6 @@ }, "greeting_flow": { "start_node": { - RESPONSE: TelegramMessage(text="Bot running"), TRANSITIONS: { "node1": telegram_condition(commands=["start", "restart"]) }, diff --git a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py index a2ae9a647..ccd1e13a7 100644 --- a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py +++ b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py @@ -23,7 +23,7 @@ ```python import sys from locust import main - + sys.argv = ["locust", "-f", {file_name}] main.main() ```