diff --git a/.flake8 b/.flake8 index b81b197..9096043 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,4 @@ inline-quotes = " multiline-quotes = """ docstring-quotes = """ avoid-escape = False -ignore = Q000,WPS306,I001,I005,WPS229,D400,WPS317,S101,WPS507 +ignore = Q000,WPS306,I001,I005,WPS229,D400,WPS317,S101,WPS507,DAR101,DAR201,WPS112,F401,WPS300,WPS412,DAR301,D401,D205,WPS615,I004,WPS110,WPS420,C812 diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 9bc4f54..340c366 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -13,24 +13,26 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11.4"] + python-version: ["3.9", "3.10", "3.11.5"] steps: - name: Checkout uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + sudo apt-get install libjack-jackd2-dev portaudio19-dev python -m pip install --upgrade pip pip install -r requirements.txt pip install pylint reorder-python-imports pip install mypy reorder-python-imports pip install wemake-python-styleguide reorder-python-imports pip install black reorder-python-imports + - name: Analysing the code with pylint id: pylint continue-on-error: true @@ -81,7 +83,7 @@ jobs: fi - name: Check runner state run: | - if [[ "${{ steps.pylint.outcome }}" == "failure" || "${{ steps.black.outcome }}" == "failure" ]] || "${{ steps.mypy.outcome }}" == "failure" ]]; then + if [[ "${{ steps.pylint.outcome }}" == "failure" || "${{ steps.black.outcome }}" == "failure" || "${{ steps.mypy.outcome }}" == "failure" ]]; then echo "Linters failed, refer to related sections for info" exit 1 fi diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ca4d8e1 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "leonardo_api"] + path = leonardo_api + url = git@github.com:wwakabobik/leonardo_api.git +[submodule "openai_api"] + path = openai_api + url = git@github.com:wwakabobik/openai_api.git diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b28de..e125406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1] - 2023-08-25 ### Added -- ChatGPT engine -- DALL-E engine + - audio_recorder - logger - transcriptors +- translators - tts +- OpenAI API +- Leonardo API diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..18c9147 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..350de5b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,54 @@ +# Contributing to Transcriptase +We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's: + +- Reporting a bug +- Discussing the current state of the code +- Submitting a fix +- Proposing new features +- Becoming a maintainer + +## We Develop with Github +We use github to host code, to track issues and feature requests, as well as accept pull requests. + +## We Use [Github Flow](https://guides.github.com/introduction/flow/index.html), So All Code Changes Happen Through Pull Requests +Pull requests are the best way to propose changes to the codebase (we use [Github Flow](https://guides.github.com/introduction/flow/index.html)). We actively welcome your pull requests: + +1. Fork the repo and create your branch from `master`. +2. If you've added code that should be tested, add tests. +3. Always update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints: pylint, mypy and black code formatter. +6. Issue that pull request! + +## Any contributions you make will be under the MIT Software License +In short, when you submit code changes, your submissions are understood to be under the same [MIT License](http://choosealicense.com/licenses/mit/) that covers the project. Feel free to contact the maintainers if that's a concern. + +## Report bugs using Github's [issues](https://github.com/briandk/transcriptase-atom/issues) +We use GitHub issues to track public bugs. Report a bug by [opening a new issue](); it's that easy! + +## Write bug reports with detail, background, and sample code +[This is an example](http://stackoverflow.com/q/12488905/180626) of a bug report I wrote, and I think it's not a bad model. Here's [another example from Craig Hockenberry](http://www.openradar.me/11905408), an app developer whom I greatly respect. + +**Great Bug Reports** tend to have: + +- A quick summary and/or background +- Steps to reproduce + - Be specific! + - Give sample code if you can. [My stackoverflow question](http://stackoverflow.com/q/12488905/180626) includes sample code that *anyone* with a base R setup can run to reproduce what I was seeing +- What you expected would happen +- What actually happens +- Notes (possibly including why you think this might be happening, or stuff you tried that didn't work) + +People *love* thorough bug reports. I'm not even kidding. + +## Use a Consistent Coding Style +I'm again borrowing these from [Facebook's Guidelines](https://github.com/facebook/draft-js/blob/a9316a723f9e918afde44dea68b5f9f39b7d9b00/CONTRIBUTING.md) + +* 2 spaces for indentation rather than tabs +* You can try running `npm run lint` for style unification + +## License +By contributing, you agree that your contributions will be licensed under its MIT License. + +## References +This document was adapted from the open-source contribution guidelines for [Facebook's Draft](https://github.com/facebook/draft-js/blob/a9316a723f9e918afde44dea68b5f9f39b7d9b00/CONTRIBUTING.md) diff --git a/README.md b/README.md index 8e2bdc7..2973a52 100644 --- a/README.md +++ b/README.md @@ -1 +1,2 @@ -# ai_engines \ No newline at end of file +# ai_engines +Playground and utils libraries for AI stuff diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..87cd27b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 0.0.1 | :white_check_mark: | + + +## Reporting a Vulnerability + +In case of issues, please report to: https://github.com/wwakabobik/leonardo_api/issues + +Some ignored/declined issues will be described bellow, please check them prior to create new issues. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..5659fec --- /dev/null +++ b/TODO.md @@ -0,0 +1,3 @@ +- Add more params and fixes +- Add image handling methods (generalistic) +- Rework as pypi package diff --git a/__init__.py b/__init__.py index 90b7e87..451bb7f 100644 --- a/__init__.py +++ b/__init__.py @@ -1,12 +1,24 @@ +# -*- coding: utf-8 -*- +""" +Filename: __main__.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 12.09.2023 + +Description: +This file is init point for project-wide structure. +""" + # Engines -from .testrail_api_reporter.engines.at_coverage_reporter import ATCoverageReporter -from .testrail_api_reporter.engines.results_reporter import TestRailResultsReporter -from .testrail_api_reporter.engines.plotly_reporter import PlotlyReporter -from .testrail_api_reporter.engines.case_backup import TCBackup -# Publishers -from .testrail_api_reporter.publishers.confluence_sender import ConfluenceSender -from .testrail_api_reporter.publishers.email_sender import EmailSender -from .testrail_api_reporter.publishers.slack_sender import SlackSender -from .testrail_api_reporter.publishers.gdrive_uploader import GoogleDriveUploader +from .openai_api.src.openai_api.chatgpt import ChatGPT # pylint: disable=unused-import +from .openai_api.src.openai_api.dalle import DALLE # pylint: disable=unused-import + # Utils -from .testrail_api_reporter.utils.reporter_utils import upload_image, zip_file, delete_file +from .utils.tts import CustomTTS # pylint: disable=unused-import +from .utils.transcriptors import CustomTranscriptor # pylint: disable=unused-import +from .utils.translators import CustomTranslator # pylint: disable=unused-import +from .utils.audio_recorder import AudioRecorder, record_and_convert_audio # pylint: disable=unused-import +from .utils.logger_config import setup_logger # pylint: disable=unused-import +from .utils.other import is_heroku_environment # pylint: disable=unused-import diff --git a/__main__.py b/__main__.py index b38aefd..b0d22f8 100644 --- a/__main__.py +++ b/__main__.py @@ -1,12 +1,24 @@ +# -*- coding: utf-8 -*- +""" +Filename: __main__.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 12.09.2023 + +Description: +This file is entry point for project-wide structure. +""" + # Engines -from testrail_api_reporter.engines.at_coverage_reporter import ATCoverageReporter -from testrail_api_reporter.engines.results_reporter import TestRailResultsReporter -from testrail_api_reporter.engines.plotly_reporter import PlotlyReporter -from testrail_api_reporter.engines.case_backup import TCBackup -# Publishers -from testrail_api_reporter.publishers.confluence_sender import ConfluenceSender -from testrail_api_reporter.publishers.email_sender import EmailSender -from testrail_api_reporter.publishers.slack_sender import SlackSender -from testrail_api_reporter.publishers.gdrive_uploader import GoogleDriveUploader +from openai_api.src.openai_api.chatgpt import ChatGPT # pylint: disable=unused-import +from openai_api.src.openai_api.dalle import DALLE # pylint: disable=unused-import + # Utils -from testrail_api_reporter.utils.reporter_utils import upload_image, zip_file, delete_file +from utils.tts import CustomTTS # pylint: disable=unused-import +from utils.transcriptors import CustomTranscriptor # pylint: disable=unused-import +from utils.translators import CustomTranslator # pylint: disable=unused-import +from utils.audio_recorder import AudioRecorder, record_and_convert_audio # pylint: disable=unused-import +from utils.logger_config import setup_logger # pylint: disable=unused-import +from utils.other import is_heroku_environment # pylint: disable=unused-import diff --git a/leonardo_api b/leonardo_api new file mode 160000 index 0000000..a1baea0 --- /dev/null +++ b/leonardo_api @@ -0,0 +1 @@ +Subproject commit a1baea0fb4e8350783bfd9360025cd719775d9b7 diff --git a/openai_api b/openai_api new file mode 160000 index 0000000..51ee463 --- /dev/null +++ b/openai_api @@ -0,0 +1 @@ +Subproject commit 51ee4630af01dfb6c62971f28c6d02beda718282 diff --git a/openai_engine/TODO b/openai_engine/TODO deleted file mode 100644 index 5c579c2..0000000 --- a/openai_engine/TODO +++ /dev/null @@ -1,3 +0,0 @@ -- ADD STATISTICS tracking -- ADD Logger -- ADD Translator \ No newline at end of file diff --git a/openai_engine/__init__.py b/openai_engine/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/openai_engine/chatgpt.py b/openai_engine/chatgpt.py deleted file mode 100644 index dd464e1..0000000 --- a/openai_engine/chatgpt.py +++ /dev/null @@ -1,787 +0,0 @@ -""" This module contains implementation for ChatGPT """ -# -*- coding: utf-8 -*- -import logging - -import openai - -from models import COMPLETIONS, TRANSCRIPTIONS, TRANSLATIONS - - -class GPTStatistics: - """ - The GPTStatistics class is for creating an instance of the GPTStatistics model. - - Parameters: - prompt_tokens (int): The number of tokens in the prompt. Default is 0. - completion_tokens (int): The number of tokens in the completion. Default is 0. - total_tokens (int): The total number of tokens. Default is 0. - """ - - def __init__( - self, - prompt_tokens: int = 0, - completion_tokens: int = 0, - total_tokens: int = 0, - ): - """ - Constructs all the necessary attributes for the GPTStatistics object. - - :param prompt_tokens: The number of tokens in the prompt. Default is 0. - :param completion_tokens: The number of tokens in the completion. Default is 0. - :param total_tokens: The total number of tokens. Default is 0. - """ - self.___prompt_tokens = prompt_tokens - self.___completion_tokens = completion_tokens - self.___total_tokens = total_tokens - - @property - def prompt_tokens(self): - """ - Getter for prompt_tokens. - - :return: The prompt tokens. - """ - return self.___prompt_tokens - - @prompt_tokens.setter - def prompt_tokens(self, value): - """ - Setter for prompt_tokens. - - :param value: The prompt tokens. - """ - self.___prompt_tokens = value - - @property - def completion_tokens(self): - """ - Getter for completion_tokens. - - :return: The completion tokens. - """ - return self.___completion_tokens - - @completion_tokens.setter - def completion_tokens(self, value): - """ - Setter for completion_tokens. - - :param value: The completion tokens. - """ - self.___completion_tokens = value - - @property - def total_tokens(self): - """ - Getter for completion_tokens. - - :return: The completion tokens. - """ - return self.___total_tokens - - @total_tokens.setter - def total_tokens(self, value): - """ - Setter for total_tokens. - - :param value: The total tokens. - """ - self.___total_tokens = value - - def add_prompt_tokens(self, value): - """ - Adder for prompt_tokens. - - :param value: The prompt tokens. - """ - self.prompt_tokens += value - - def add_completion_tokens(self, value): - """ - Adder for completion_tokens. - - :param value: The completion tokens. - """ - self.completion_tokens += value - - def add_total_tokens(self, value): - """ - Adder for total_tokens. - - :param value: The total tokens. - """ - self.total_tokens += value - - def set_tokens(self, prompt_tokens, completion_tokens, total_tokens): - """ - Sets all tokens statistics in bulk - - :param prompt_tokens: The prompt tokens. - :param completion_tokens: The prompt tokens. - :param total_tokens: The prompt tokens. - """ - self.prompt_tokens = prompt_tokens - self.completion_tokens = completion_tokens - self.total_tokens = total_tokens - - def add_tokens(self, prompt_tokens, completion_tokens, total_tokens): - """ - Adds all tokens statistics in bulk - - :param prompt_tokens: The prompt tokens. - :param completion_tokens: The prompt tokens. - :param total_tokens: The prompt tokens. - """ - self.prompt_tokens += prompt_tokens - self.completion_tokens += completion_tokens - self.total_tokens += total_tokens - - def get_tokens(self): - """ - Returns a dictionary of the class attributes and their values. - - :return: dict with tokens statistics - """ - return { - "prompt_tokens": self.___prompt_tokens, - "completion_tokens": self.___completion_tokens, - "total_tokens": self.___total_tokens, - } - - -class ChatGPT: - """ - The ChatGPT class is for creating an instance of the ChatGPT model. - - Parameters: - auth_token (str): Authentication bearer token. Required. - organization (str): Organization uses auth toke. Required. - model (str): The name of the model, Default is 'gpt-4'. - choices (int, optional): The number of response options. Default is 1. - temperature (float, optional): The temperature of the model's output. Default is 1. - top_p (float, optional): The top-p value for nucleus sampling. Default is 1. - stream (bool, optional): If True, the model will return intermediate results. Default is False. - stop (str, optional): The stop sequence at which the model should stop generating further tokens. Default is None. - max_tokens (int, optional): The maximum number of tokens in the output. Default is 1024. - presence_penalty (float, optional): The penalty for new token presence. Default is 0. - frequency_penalty (float, optional): The penalty for token frequency. Default is 0. - logit_bias (map, optional): The bias for the logits before sampling. Default is None. - user (str, optional): The user ID. Default is ''. - functions (list, optional): The list of functions. Default is None. - function_call (str, optional): The function call. Default is None. - history_length (int, optional): Length of history. Default is 5. - chats (dict, optional): Chats dictionary, contains all chat. Default is None. - current_chat (str, optional): Default chat will be used. Default is None. - prompt_method (bool, optional): prompt method. Use messages if False, otherwise - prompt. Default if False. - logger (logging.Logger, optional): default logger. Default is None. - statistic (GPTStatistics, optional): statistics logger. If none, will be initialized with zeros. - system_settings (str, optional): general instructions for chat. Default is None. - echo # TODO - best_of # TODO - suffix # TODO - """ - - def __init__( - self, - auth_token, - organization, - model: str = COMPLETIONS[0], - choices: int = 1, - temperature: float = 1, - top_p: float = 1, - stream: bool = False, - stop: str = None, - max_tokens: int = 1024, - presence_penalty: float = 0, - frequency_penalty: float = 0, - logit_bias: map = None, - user: str = "", - functions: list = None, - function_call: str = None, - history_length: int = 5, - chats: dict = None, - current_chat: str = None, - prompt_method: bool = False, - logger: logging.Logger = None, - statistics: GPTStatistics = GPTStatistics(), - system_settings: str = None, - ): - """ - General init - - :param auth_token (str): Authentication bearer token. Required. - :param organization (str): Organization uses auth toke. Required. - :param model: The name of the model. - :param choices: The number of response options. Default is 1. - :param temperature: The temperature of the model's output. Default is 1. - :param top_p: The top-p value for nucleus sampling. Default is 1. - :param stream: If True, the model will return intermediate results. Default is False. - :param stop: The stop sequence at which the model should stop generating further tokens. Default is None. - :param max_tokens: The maximum number of tokens in the output. Default is 1024. - :param presence_penalty: The penalty for new token presence. Default is 0. - :param frequency_penalty: The penalty for token frequency. Default is 0. - :param logit_bias: The bias for the logits before sampling. Default is None. - :param user: The user ID. Default is ''. - :param functions: The list of functions. Default is None. - :param function_call: The function call. Default is None. - :param history_length: Length of history. Default is 5. - :param chats: Chats dictionary, contains all chat. Default is None. - :param current_chat: Default chat will be used. Default is None. - :param prompt_method: prompt method. Use messages if False, otherwise - prompt. Default if False. - :param logger: default logger. Default is None. - :param statistics: statistics logger. If none, will be initialized with zeros. - :param system_settings: general system instructions for bot. Default is ''. - """ - self.___model = model - self.___choices = choices - self.___temperature = temperature - self.___top_p = top_p - self.___stream = stream - self.___stop = stop - self.___max_tokens = max_tokens - self.___presence_penalty = presence_penalty - self.___frequency_penalty = frequency_penalty - self.___logit_bias = logit_bias - self.___user = user - self.___functions = functions - self.___function_call = function_call - self.___history_length = history_length - self.___chats = chats if chats else {} - self.___current_chat = current_chat - self.___prompt_method = prompt_method - self.___set_auth(auth_token, organization) - self.___logger = logger - self.___statistics = statistics - self.___system_settings = system_settings if system_settings else "" - - @staticmethod - def ___set_auth(token, organization): - """ - Method to set auth bearer. - - :param token: authentication bearer token. - :param organization: organization, which drives the chat. - """ - openai.api_key = token - openai.organization = organization - - @property - def model(self): - """ - Getter for model. - - :return: The name of the model. - """ - return self.___model - - @model.setter - def model(self, value): - """ - Setter for model. - - :param value: The new name of the model. - """ - self.___model = value - - @property - def choices(self): - """ - Getter for choices. - - :return: The number of response options. - """ - return self.___choices - - @choices.setter - def choices(self, value): - """ - Setter for choices. - - :param value: The new number of response options. - """ - self.___choices = value - - @property - def temperature(self): - """ - Getter for temperature. - - :return: The temperature of the model's output. - """ - return self.___temperature - - @temperature.setter - def temperature(self, value): - """ - Setter for temperature. - - :param value: The new temperature of the model's output. - """ - self.___temperature = value - - @property - def top_p(self): - """ - Getter for top_p. - - :return: The top-p value for nucleus sampling. - """ - return self.___top_p - - @top_p.setter - def top_p(self, value): - """ - Setter for top_p. - - :param value: The new top-p value for nucleus sampling. - """ - self.___top_p = value - - @property - def stream(self): - """ - Getter for stream. - - :return: If True, the model will return intermediate results. - """ - return self.___stream - - @stream.setter - def stream(self, value): - """ - Setter for stream. - - :param value: The new value for stream. - """ - self.___stream = value - - @property - def stop(self): - """ - Getter for stop. - - :return: The stop sequence at which the model should stop generating further tokens. - """ - return self.___stop - - @stop.setter - def stop(self, value): - """ - Setter for stop. - - :param value: The new stop sequence. - """ - self.___stop = value - - @property - def max_tokens(self): - """ - Getter for max_tokens. - - :return: The maximum number of tokens in the output. - """ - return self.___max_tokens - - @max_tokens.setter - def max_tokens(self, value): - """ - Setter for max_tokens. - - :param value: The new maximum number of tokens in the output. - """ - self.___max_tokens = value - - @property - def presence_penalty(self): - """ - Getter for presence_penalty. - - :return: The penalty for new token presence. - """ - return self.___presence_penalty - - @presence_penalty.setter - def presence_penalty(self, value): - """ - Setter for presence_penalty. - - :param value: The new penalty for new token presence. - """ - self.___presence_penalty = value - - @property - def frequency_penalty(self): - """ - Getter for frequency_penalty. - - :return: The penalty for token frequency. - """ - return self.___frequency_penalty - - @frequency_penalty.setter - def frequency_penalty(self, value): - """ - Setter for frequency_penalty. - - :param value: The new penalty for token frequency. - """ - self.___frequency_penalty = value - - @property - def logit_bias(self): - """ - Getter for logit_bias. - - :return: The bias for the logits before sampling. - """ - return self.___logit_bias - - @logit_bias.setter - def logit_bias(self, value): - """ - Setter for logit_bias. - - :param value: The new bias for the logits before sampling. - """ - self.___logit_bias = value - - @property - def user(self): - """ - Getter for user. - - :return: The user ID. - """ - return self.___user - - @user.setter - def user(self, value): - """ - Setter for user. - - :param value: The new user ID. - """ - self.___user = value - - @property - def functions(self): - """ - Getter for functions. - - :return: The list of functions. - """ - return self.___functions - - @functions.setter - def functions(self, value): - """ - Setter for functions. - - :param value: The new list of functions. - """ - self.___functions = value - - @property - def function_call(self): - """ - Getter for function_call. - - :return: The function call. - """ - return self.___function_call - - @function_call.setter - def function_call(self, value): - """ - Setter for function_call. - - :param value: The new function call. - """ - self.___function_call = value - - @property - def history_length(self): - """ - Getter for history_length. - - :return: The history length. - """ - return self.___history_length - - @history_length.setter - def history_length(self, value): - """ - Setter for history_length. - - :param value: The new history length. - """ - self.___history_length = value - - @property - def chats(self): - """ - Getter for chats. - - :return: The chats. - """ - return self.___chats - - @chats.setter - def chats(self, value): - """ - Setter for chats. - - :param value: The new chats. - """ - self.___chats = value - - @property - def current_chat(self): - """ - Getter for current_chat. - - :return: The current chat. - """ - return self.___current_chat - - @current_chat.setter - def current_chat(self, value): - """ - Setter for current_chat. - - :param value: The current chat. - """ - self.___current_chat = value - - @property - def prompt_method(self): - """ - Getter for prompt_method. - - :return: The prompt method. - """ - return self.___prompt_method - - @prompt_method.setter - def prompt_method(self, value): - """ - Setter for prompt_method. - - :param value: The prompt method. - """ - self.___prompt_method = value - - @property - def system_settings(self): - """ - Getter for system_settings. - - :return: The system settings method. - """ - return self.___system_settings - - @system_settings.setter - def system_settings(self, value): - """ - Setter for system_settings. - - :param value: The system settings. - """ - self.___system_settings = value - - async def process_chat(self, prompt, default_choice=None): - """ - Creates a new chat completion for the provided messages and parameters. - - :param prompt: The prompt to pass to the model. - :param default_choice: Default number of choice to monitor for stream end. By default, is None. - - :return: Returns answers by chunk if 'stream' is false True, otherwise return complete answer. - """ - # Prepare parameters - params = { - "model": self.model, - "max_tokens": self.max_tokens, - "n": self.choices, - "temperature": self.temperature, - "top_p": self.top_p, - "presence_penalty": self.presence_penalty, - "frequency_penalty": self.frequency_penalty, - "user": self.user, - "functions": self.functions, - "function_call": self.function_call, - "stream": self.stream, - } - - # Remove None values - params = {k: v for k, v in params.items() if v is not None} - - # Add 'prompt' or 'messages' parameter - if self.prompt_method: - params["prompt"] = prompt - else: - params["messages"] = prompt - - # Get response - if self.stream: - try: - async for chunk in await openai.ChatCompletion.acreate(**params): - if default_choice is not None: - if chunk["choices"][default_choice]["finish_reason"] is not None: - break - yield chunk - except GeneratorExit: - pass - else: - response = await openai.ChatCompletion.acreate(**params) - yield response - - async def __handle_chat_name(self, chat_name, prompt): - """ - Handles the chat name. If chat_name is None, sets it to the first 40 characters of the prompt. - If chat_name is not present in self.chats, adds it. - - :param chat_name: Name of the chat. - :param prompt: Message from the user. - :return: Processed chat name. - """ - if chat_name is None: - chat_name = prompt[:40] - self.current_chat = chat_name - if chat_name not in self.chats: - self.chats[chat_name] = [] - return chat_name - - async def chat(self, prompt, chat_name=None, default_choice=0): - """ - Wrapper for the process_chat function. Adds new messages to the chat and calls process_chat. - - :param prompt: Message from the user. - :param chat_name: Name of the chat. If None, uses self.current_chat. - :param default_choice: Index of the model's response choice. - """ - # Set chat_name - chat_name = chat_name if chat_name is not None else self.current_chat - chat_name = await self.__handle_chat_name(chat_name, prompt) - - # Add new message to chat - self.chats[chat_name].append({"role": "user", "content": prompt}) - - # Call process_chat - full_prompt = "" - if self.prompt_method: - try: - async for response in self.process_chat(prompt=prompt, default_choice=default_choice): - if isinstance(response, dict): - finish_reason = response["choices"][default_choice].get("finish_reason", "") - yield response - if self.stream: - if "content" in response["choices"][default_choice]["delta"].keys(): - full_prompt += response["choices"][default_choice]["delta"]["content"] - else: - full_prompt += response["choices"][default_choice]["message"]["content"] - if finish_reason in ["stop", "length", "function_call"]: - yield "" - else: - break - except GeneratorExit: - pass - else: - # Get last 'history_length' messages - messages = self.chats[chat_name][-self.history_length :] - messages.insert(0, {"role": "system", "content": self.system_settings}) - - try: - async for response in self.process_chat(prompt=messages, default_choice=default_choice): - if isinstance(response, dict): - finish_reason = response["choices"][default_choice].get("finish_reason", "") - yield response - if self.stream: - if "content" in response["choices"][default_choice]["delta"].keys(): - full_prompt += response["choices"][default_choice]["delta"]["content"] - else: - full_prompt += response["choices"][default_choice]["message"]["content"] - if finish_reason in ["stop", "length", "function_call"]: - yield "" - else: - break - except GeneratorExit: - pass - - # Add last response to chat - self.chats[chat_name].append({"role": "assistant", "content": full_prompt}) - - async def str_chat(self, prompt, chat_name=None, default_choice=0): - """ - Wrapper for the chat function. Returns only the content of the message. - - :param prompt: Message from the user. - :param chat_name: Name of the chat. If None, uses self.current_chat. - :param default_choice: Index of the model's response choice. - - :return: Content of the message. - """ - try: - async for response in self.chat(prompt, chat_name, default_choice): - if isinstance(response, dict): - if self.stream: - if "content" in response["choices"][default_choice]["delta"].keys(): - yield response["choices"][default_choice]["delta"]["content"] - else: - yield "" - else: - yield response["choices"][default_choice]["message"]["content"] - else: - break - except GeneratorExit: - pass - - async def transcript(self, file, prompt=None, language="en", response_format="text"): - """ - Wrapper for the chat function. Returns only the content of the message. - - :param file: Path with filename to transcript. - :param prompt: Previous prompt. Default is None. - :param language: Language on which audio is. Default is 'en'. - :param response_format: default response format, by default is 'text'. - Possible values are: json, text, srt, verbose_json, or vtt. - - - :return: transcription (text, json, srt, verbose_json or vtt) - """ - kwargs = {} - if prompt is not None: - kwargs["prompt"] = prompt - transcription = await openai.Audio.atranscribe( - model=TRANSCRIPTIONS[0], - file=file, - language=language, - response_format=response_format, - temperature=self.temperature, - **kwargs - ) - return transcription - - async def translate(self, file, prompt=None, response_format="text"): - """ - Wrapper for the chat function. Returns only the content of the message. - - :param file: Path with filename to transcript. - :param prompt: Previous prompt. Default is None. - :param response_format: default response format, by default is 'text'. - Possible values are: json, text, srt, verbose_json, or vtt. - - - :return: transcription (text, json, srt, verbose_json or vtt) - """ - kwargs = {} - if prompt is not None: - kwargs["prompt"] = prompt - translation = await openai.Audio.atranslate( - model=TRANSLATIONS[0], - file=file, - response_format=response_format, - temperature=self.temperature, - **kwargs - ) - return translation diff --git a/openai_engine/dalle.py b/openai_engine/dalle.py deleted file mode 100644 index 2629258..0000000 --- a/openai_engine/dalle.py +++ /dev/null @@ -1,230 +0,0 @@ -""" This module contains implementation for DALL-E """ -# -*- coding: utf-8 -*- -import tempfile -import os -import uuid - -import aiohttp -from io import BytesIO -import openai -from PIL import Image - - -class DALLE: - def __init__(self, auth_token, organization, default_count=1, default_size="512x512", default_file_format='PNG', - user=None): - self.___default_count = default_count - self.___default_size = default_size - self.___default_file_format = default_file_format - self.___user = user - self.___set_auth(auth_token, organization) - - @staticmethod - def ___set_auth(token, organization): - """ - Method to set auth bearer. - - :param token: authentication bearer token. - :param organization: organization, which drives the chat. - """ - openai.api_key = token - openai.organization = organization - - @property - def default_count(self): - """ - Getter for default_count. - - :return: Returns default_count value. - """ - return self.___default_count - - @default_count.setter - def default_count(self, value): - """ - Setter for default_count. - - :param value: The new value of default_count. - """ - self.___default_count = value - - @property - def default_size(self): - """ - Getter for default_size. - - :return: Returns default_size value. - """ - return self.___default_size - - @default_size.setter - def default_size(self, value): - """ - Setter for default_size. - - :param value: The new value of default_size. - """ - self.___default_size = value - - @property - def default_file_format(self): - """ - Getter for default_file_format. - - :return: Returns default_file_format value. - """ - return self.___default_file_format - - @default_file_format.setter - def default_file_format(self, value): - """ - Setter for default_size. - - :param value: The new value of default_file_format. - """ - self.___default_file_format = value - - @property - def user(self): - """ - Getter for user. - - :return: The user. - """ - return self.___user - - @user.setter - def user(self, value): - """ - Setter for user. - - :param value: The user. - """ - self.___user = value - - async def create_image(self, prompt): - """ - Creates an image using DALL-E Image API. - - :param prompt: The prompt to be used for image creation. - - :return: A PIL.Image object created from the image data received from the API. - """ - response = await openai.Image.acreate(prompt=prompt, n=self.default_count, size=self.default_size, - user=self.user) - image_url = response["data"][0]["url"] - async with aiohttp.ClientSession() as session: - async with session.get(image_url) as resp: - image_data = await resp.read() - return Image.open(BytesIO(image_data)) - - @staticmethod - def show_image(image): - """ - Shows image interactively. - - :param image: image object. - """ - image.show() - - def save_image(self, image, filename=None, file_format=None): - """ Saves an image to a file. - - :param image: A PIL.Image object to be saved. - :param filename: The name of the file where the image will be saved. - If None, a random filename in the system's temporary directory will be used. - :param file_format: The format of the file. This is optional and defaults to 'PNG'. - - :return: The full path of the file where the image was saved, or None if the image could not be saved. - """ - if file_format is None: - file_format = self.default_file_format - if filename is None: - filename = os.path.join(tempfile.gettempdir(), f'{uuid.uuid4()}.{file_format.lower()}') - try: - image.save(filename, format=format) - except Exception as error: - print(f"Can't save image: {error}") - return None - return filename - - async def create_variation_from_file(self, file): - """ - Creates an image variation from file using DALL-E Image API. - - :param file: file of the image (bytes). - - :return: A PIL.Image object created from the image data received from the API. - """ - response = await openai.Image.acreate_variation(file=file, n=self.default_count, size=self.default_size, - user=self.user) - image_url = response["data"][0]["url"] - async with aiohttp.ClientSession() as session: - async with session.get(image_url) as resp: - image_data = await resp.read() - return Image.open(BytesIO(image_data)) - - async def create_variation_from_url(self, url): - """ - Creates an image variation from URL using DALL-E Image API. - - :param url: URL of the image. - - :return: A PIL.Image object created from the image data received from the API. - """ - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - image_data = await resp.read() - - response = await openai.Image.acreate_variation(BytesIO(image_data), n=self.default_count, - size=self.default_size, - user=self.user) - image_url = response["data"][0]["url"] - async with aiohttp.ClientSession() as session: - async with session.get(image_url) as resp: - variation_image_data = await resp.read() - return Image.open(BytesIO(variation_image_data)) - - async def edit_image_from_file(self, file, prompt, mask=None): - """ - Edits an image using OpenAI's Image API. - - :param file: A file-like object opened in binary mode containing the image to be edited. - :param prompt: The prompt to be used for image editing. - :param mask: An optional file-like object opened in binary mode containing the mask image. - If provided, the mask will be applied to the image. - :return: A PIL.Image object created from the image data received from the API. - """ - response = await openai.Image.acreate_edit(file=file, prompt=prompt, mask=mask, - n=self.default_count, size=self.default_size, - user=self.user) - image_url = response["data"][0]["url"] - async with aiohttp.ClientSession() as session: - async with session.get(image_url) as resp: - image_data = await resp.read() - return Image.open(BytesIO(image_data)) - - async def edit_image_from_url(self, url, prompt, mask_url=None): - """ - Edits an image using OpenAI's Image API. - - :param url: A url of image to be edited. - :param prompt: The prompt to be used for image editing. - :param mask_url: Url containing mask image. If provided, the mask will be applied to the image. - :return: A PIL.Image object created from the image data received from the API. - """ - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - image_data = await resp.read() - - async with aiohttp.ClientSession() as session: - async with session.get(mask_url) as resp: - mask_data = await resp.read() - response = await openai.Image.acreate_edit(file=BytesIO(image_data), prompt=prompt, mask=BytesIO(mask_data), - n=self.default_count, size=self.default_size, - user=self.user) - image_url = response["data"][0]["url"] - async with aiohttp.ClientSession() as session: - async with session.get(image_url) as resp: - image_data = await resp.read() - return Image.open(BytesIO(image_data)) diff --git a/openai_engine/models.py b/openai_engine/models.py deleted file mode 100644 index 5f30d3a..0000000 --- a/openai_engine/models.py +++ /dev/null @@ -1,17 +0,0 @@ -""" This file contains OpenAI constants """ - -COMPLETIONS = [ - "gpt-4", - "gpt-4-0613", - "gpt-4-32k", - "gpt-4-32k-0613", - "gpt-3.5-turbo", - "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-16k-0613", -] -TRANSCRIPTIONS = ["whisper-1"] -TRANSLATIONS = ["whisper-1"] -FINE_TUNES = ["davinci", "curie", "babbage", "ada"] -EMBEDDINGS = ["text-embedding-ada-002", "text-similarity-*-001", "text-search-*-*-001", "code-search-*-*-001"] -MODERATIONS = ["text-moderation-stable", "text-moderation-latest"] diff --git a/pyproject.toml b/pyproject.toml index 8610684..78b1e78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,21 @@ extend-ignore = """ D400, WPS317, S101, - WPS507 + WPS507, + DAR101, + DAR201, + WPS112, + F401, + WPS300, + WPS412, + DAR301, + D401, + D205, + WPS615, + I004, + WPS110, + WPS420, + C812 """ [tool.pylint] diff --git a/requirements.txt b/requirements.txt index 6cc84c8..8149784 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -aiohttp==3.8.5 -pyobjc -# OpenAI -openai==0.28.0 # TTS gtts==2.3.2 pyttsx4==3.0.15 @@ -19,3 +15,5 @@ soundfile==0.12.1 numpy==1.25.2 # Image pillow==10.0.0 +# Articles +readability==0.3.1 diff --git a/setup.cfg b/setup.cfg index d2dfd68..fb69a11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [flake8] max-line-length = 120 -extend-ignore = Q000,WPS306,I001,I005,WPS229,D400,WPS317,S101,WPS507 +extend-ignore = Q000,WPS306,I001,I005,WPS229,D400,WPS317,S101,WPS507,DAR201,DAR101,WPS112,F401,WPS300,WPS412,DAR301,D401,D205,WPS615,I004,WPS110,WPS420,C812 diff --git a/utils/article_extractor.py b/utils/article_extractor.py new file mode 100644 index 0000000..98dc334 --- /dev/null +++ b/utils/article_extractor.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" +Filename: article_extractor.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 12.09.2023 + +Description: +This file contains implementation for Article Extractor from internet page +""" + +import requests +from readability import Document + +# FIXME: This is a temporary solution. We need to find a better way to extract + + +def get_content(url): + """ + This function extracts content from internet page. + Args: + url: URL of internet page. + + Returns: + Content of internet page. + + """ + session = requests.Session() + response = session.get(url) + doc = Document(response.text) + content = doc.summary() + return content diff --git a/utils/audio_recorder.py b/utils/audio_recorder.py index 47d050b..ee50285 100644 --- a/utils/audio_recorder.py +++ b/utils/audio_recorder.py @@ -1,36 +1,79 @@ -""" This module contains implementation for Audio Recorder """ +# -*- coding: utf-8 -*- +""" +Filename: audio_recorder.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 26.08.2023 + +Description: +This file contains implementation for Audio Recorder +""" + +import math import os import struct import tempfile import time +import uuid import wave -# -*- coding: utf-8 -*- -import math import pyaudio import sounddevice as sd import soundfile as sf from pydub import AudioSegment -def record_and_convert_audio(duration=10, fs=44100): +def record_and_convert_audio(duration: int = 5, frequency_sample: int = 16000): + """ + Records audio for a specified duration and converts it to MP3 format. + + This function records audio for a given duration (in seconds) with a specified frequency sample. + The audio is then saved as a temporary .wav file, converted to .mp3 format, and the .wav file is deleted. + The function returns the path to the .mp3 file. + + :param duration: The duration of the audio recording in seconds. Default is 5 seconds. + :param frequency_sample: The frequency sample rate of the audio recording. Default is 16000 Hz. + + :return: The path to the saved .mp3 file. + """ print(f"Listening beginning for {duration}s...") - myrecording = sd.rec(int(duration * fs), samplerate=fs, channels=1) + recording = sd.rec(int(duration * frequency_sample), samplerate=frequency_sample, channels=1) sd.wait() # Wait until recording is finished print("Recording complete!") temp_dir = tempfile.gettempdir() - wav_file_path = os.path.join(temp_dir, "my_audio.wav") - sf.write(wav_file_path, myrecording, fs) - print(f"Temp audiofile saved: {wav_file_path}") - audio = AudioSegment.from_wav(wav_file_path) - os.remove(os.path.join(temp_dir, "my_audio.wav")) - mp3_file_path = os.path.join(temp_dir, "my_audio.mp3") - audio.export(mp3_file_path, format="mp3") - print("Audio converted to MP3 and stored into {mp3_file_path}") - return mp3_file_path - - -class Recorder: + wave_file = f"{temp_dir}/{str(uuid.uuid4())}.wav" + sf.write(wave_file, recording, frequency_sample) + print(f"Temp audiofile saved: {wave_file}") + audio = AudioSegment.from_wav(wave_file) + os.remove(wave_file) + mp3_file = f"{temp_dir}/{str(uuid.uuid4())}.mp3" + audio.export(mp3_file, format="mp3") + print(f"Audio converted to MP3 and stored into {mp3_file}") + return mp3_file + + +# pylint: disable=too-many-instance-attributes +class AudioRecorder: + """ + The AudioRecorder class is for managing an instance of the audio recording and conversion process. + + Parameters: + pyaudio_obj (PyAudio): Instance of PyAudio. Default is pyaudio.PyAudio(). + threshold (int): The RMS threshold for starting the recording. Default is 15. + channels (int): The number of channels in the audio stream. Default is 1. + chunk (int): The number of frames per buffer. Default is 1024. + f_format (int): The format of the audio stream. Default is pyaudio.paInt16. + rate (int): The sample rate of the audio stream. Default is 16000 Hz. + sample_width (int): The sample width (in bytes) of the audio stream. Default is 2. + timeout_length (int): The length of the timeout for the recording (in seconds). Default is 2 seconds. + temp_dir (str): The directory for storing the temporary .wav and .mp3 files. Default is the system's temporary dir. + normalize (float): The normalization factor for the audio samples. Default is 1.0 / 32768.0. + pa_input (bool): Specifies whether the stream is an input stream. Default is True. + pa_output (bool): Specifies whether the stream is an output stream. Default is True. + """ + def __init__( self, pyaudio_obj=pyaudio.PyAudio(), @@ -39,20 +82,39 @@ def __init__( chunk=1024, f_format=pyaudio.paInt16, rate=16000, - swidth=2, + sample_width=2, timeout_length=2, temp_dir=tempfile.gettempdir(), normalize=(1.0 / 32768.0), pa_input=True, pa_output=True, ): + """ + General init. + + This method initializes an instance of the AudioRecorder class with the specified parameters. + The default values are used for any parameters that are not provided. + + :param pyaudio_obj: Instance of PyAudio. Default is pyaudio.PyAudio(). + :param threshold: The RMS threshold for starting the recording. Default is 15. + :param channels: The number of channels in the audio stream. Default is 1. + :param chunk: The number of frames per buffer. Default is 1024. + :param f_format: The format of the audio stream. Default is pyaudio.paInt16. + :param rate: The sample rate of the audio stream. Default is 16000 Hz. + :param sample_width: The sample width (in bytes) of the audio stream. Default is 2. + :param timeout_length: The length of the timeout for the recording (in seconds). Default is 2 seconds. + :param temp_dir: The directory for storing the temporary .wav and .mp3 files. Default is temp dir. + :param normalize: The normalization factor for the audio samples. Default is 1.0 / 32768.0. + :param pa_input: Specifies whether the stream is an input stream. Default is True. + :param pa_output: Specifies whether the stream is an output stream. Default is True. + """ self.___pyaudio = pyaudio_obj self.___threshold = threshold self.___channels = channels self.___chunk = chunk self.___format = f_format self.___rate = rate - self.___swidth = swidth + self.___sample_width = sample_width self.___timeout_length = timeout_length self.___temp_dir = temp_dir self.___normalize = normalize @@ -68,6 +130,22 @@ def __init__( ) def init_stream(self, f_format, channels, rate, pa_input, pa_output, frames_per_buffer): + """ + Initializes an audio stream with the specified parameters. + + This function uses PyAudio to open an audio stream with the given format, channels, rate, input, output, + and frames per buffer. + + :param f_format: The format of the audio stream. + :param channels: The number of channels in the audio stream. + :param rate: The sample rate of the audio stream. + :param pa_input: Specifies whether the stream is an input stream. A true value indicates an input stream. + :param pa_output: Specifies whether the stream is an output stream. A true value indicates an output stream. + :param frames_per_buffer: The number of frames per buffer. + :type frames_per_buffer: int + + :return: The initialized audio stream. + """ return self.___pyaudio.open( format=f_format, channels=channels, @@ -78,13 +156,22 @@ def init_stream(self, f_format, channels, rate, pa_input, pa_output, frames_per_ ) def record(self): + """ + Starts recording audio when noise is detected. + + This function starts recording audio when noise above a certain threshold is detected. + The recording continues for a specified timeout length. + The recorded audio is then saved as a .wav file, converted to .mp3 format, and the .wav file is deleted. + The function returns the path to the .mp3 file. + + :return: The path to the saved .mp3 file. + """ print("Noise detected, recording beginning") rec = [] current = time.time() end = time.time() + self.___timeout_length while current <= end: - data = self.stream.read(self.___chunk) if self.rms(data) >= self.___threshold: end = time.time() + self.___timeout_length @@ -95,25 +182,52 @@ def record(self): return self.convert_to_mp3(filename) def write(self, recording): - n_files = len(os.listdir(self.___temp_dir)) - filename = os.path.join(self.___temp_dir, f"{n_files}.wav") - - wf = wave.open(filename, "wb") - wf.setnchannels(self.___channels) - wf.setsampwidth(self.p.get_sample_size(self.___format)) - wf.setframerate(self.___rate) - wf.writeframes(recording) - wf.close() + """ + Saves the recorded audio to a .wav file. + + This function saves the recorded audio to a .wav file with a unique filename. + The .wav file is saved in the specified temporary directory. + + :param recording: The recorded audio data. + + :return: The path to the saved .wav file. + """ + filename = os.path.join(self.___temp_dir, f"{str(uuid.uuid4())}.wav") + + wave_form = wave.open(filename, "wb") + wave_form.setnchannels(self.___channels) + wave_form.setsampwidth(self.___pyaudio.get_sample_size(self.___format)) + wave_form.setframerate(self.___rate) + wave_form.writeframes(recording) + wave_form.close() return filename def convert_to_mp3(self, filename): + """ + Converts a .wav file to .mp3 format. + + This function converts a .wav file to .mp3 format. The .wav file is deleted after the conversion. + The .mp3 file is saved with a unique filename in the specified temporary directory. + + :param filename: The path to the .wav file to be converted. + + :return: The path to the saved .mp3 file. + """ audio = AudioSegment.from_wav(filename) - mp3_file_path = os.path.join(self.___temp_dir, "my_audio.mp3") + mp3_file_path = os.path.join(self.___temp_dir, f"{str(uuid.uuid4())}.mp3") audio.export(mp3_file_path, format="mp3") os.remove(filename) return mp3_file_path def listen(self): + """ + Starts listening for audio. + + This function continuously listens for audio and starts recording when the + RMS value of the audio exceeds a certain threshold. + + :return: The path to the saved .mp3 file if recording was triggered. + """ print("Listening beginning...") while True: mic_input = self.stream.read(self.___chunk) @@ -122,14 +236,24 @@ def listen(self): return self.record() def rms(self, frame): - count = len(frame) / self.___swidth + """ + Calculates the Root Mean Square (RMS) value of the audio frame. + + This function calculates the RMS value of the audio frame, which is a measure of the power in the audio signal. + + :param frame: The audio frame for which to calculate the RMS value. + + :return: The RMS value of the audio frame. + """ + count = len(frame) / self.___sample_width + # pylint: disable=C0209 f_format = "%dh" % count shorts = struct.unpack(f_format, frame) sum_squares = 0.0 for sample in shorts: - n = sample * self.___normalize - sum_squares += n * n + normal_sample = sample * self.___normalize + sum_squares += normal_sample * normal_sample rms = math.pow(sum_squares / count, 0.5) return rms * 1000 diff --git a/utils/logger_config.py b/utils/logger_config.py index aba0161..8f82425 100644 --- a/utils/logger_config.py +++ b/utils/logger_config.py @@ -1,4 +1,15 @@ -""" This file contains an configuration for loggers. """ +# -*- coding: utf-8 -*- +""" +Filename: logger_config.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 25.08.2023 + +Description: +This file contains configuration for loggers. +""" import logging import sys @@ -6,7 +17,17 @@ from utils.other import is_heroku_environment -def setup_logger(name, log_file, level=logging.DEBUG): +def setup_logger(name: str, log_file: str, level=logging.DEBUG): + """ + Method to setup logger + + :param name: (string) Name of the logger. + :param log_file: path to log_file + :param level: logging level. Default is logging.DEBUG + + :returns: logger object + + """ formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s") file_handler = logging.FileHandler(log_file) diff --git a/utils/other.py b/utils/other.py index a616737..7111eba 100644 --- a/utils/other.py +++ b/utils/other.py @@ -1,4 +1,15 @@ -""" This file contains several other stuff. """ +# -*- coding: utf-8 -*- +""" +Filename: other.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 25.08.2023 + +Description: +This file contains several other stuff. +""" import os diff --git a/utils/transcriptors.py b/utils/transcriptors.py index 4a4e078..9661a88 100644 --- a/utils/transcriptors.py +++ b/utils/transcriptors.py @@ -1,5 +1,15 @@ -""" This module contains implementation for Custom Transcriptor """ # -*- coding: utf-8 -*- +""" +Filename: transcriptors.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 25.08.2023 + +Description: +This module contains implementation for Custom Transcriptor +""" import speech_recognition as sr @@ -44,7 +54,8 @@ def transcript(self): :return: transcripted text (string). """ print("Listening beginning...") - audio = self.___recognizer.listen(self.___source, timeout=5) + with self.___source as source: + audio = self.___recognizer.listen(source, timeout=5) user_input = None try: diff --git a/utils/translators.py b/utils/translators.py index 0682d9a..c4d6ec9 100644 --- a/utils/translators.py +++ b/utils/translators.py @@ -1,5 +1,15 @@ -""" This module contains implementation for Custom Translator """ # -*- coding: utf-8 -*- +""" +Filename: translators.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 25.08.2023 + +Description: +This module contains implementation for Custom Translator +""" from deep_translator import GoogleTranslator @@ -8,6 +18,7 @@ class CustomTranslator(GoogleTranslator): """ This class implements wrapper for GoogleTranslator """ + def __init__(self, source, target, **kwargs): """ General init. diff --git a/utils/tts.py b/utils/tts.py index 364e583..d681cb9 100644 --- a/utils/tts.py +++ b/utils/tts.py @@ -1,60 +1,118 @@ # -*- coding: utf-8 -*- +""" +Filename: tts.py +Author: Iliya Vereshchagin +Copyright (c) 2023. All rights reserved. + +Created: 25.08.2023 +Last Modified: 29.08.2023 + +Description: +This module contains implementation for Text-to-Speach tools +""" + import os +import threading import tempfile from time import sleep +from uuid import uuid4 from gtts import gTTS -from mpyg321.mpyg321 import MPyg321Player -from mutagen.mp3 import MP3 -from pydub import AudioSegment from pyttsx4 import init as pyttsx_init +from pydub import AudioSegment, playback class CustomTTS: + """ + The GPTStatistics class is for managing an instance of the Custom Text-to-Speach models. + + Parameters: + method (str): Default method to use TTS. Default is 'google'. + lang: language in ISO 639-1 format. Default is 'en'. + speedup (float): Speedup ratio. Default is 1.3. + frame (float): audio sample frame in seconds. Default is 0.1. + voice (str): default TTS voice to use. Default is 'com.apple.voice.enhanced.ru-RU.Katya' + """ + def __init__( self, method="google", lang="en", speedup=1.3, frame=0.1, voice="com.apple.voice.enhanced.ru-RU.Katya" ): + """ + General init. + + :param method: Default method to use TTS. Default is 'google'. + :param lang: language in ISO 639-1 format. Default is 'en'. + :param speedup: Speedup ratio. Default is 1.3. + :param frame: audio sample frame in seconds. Default is 0.1. + :param voice: default TTS voice to use. Default is 'com.apple.voice.enhanced.ru-RU.Katya' + """ self.___method = method - self.___player = MPyg321Player() self.___pytts = pyttsx_init() self.___lang = lang self.___voice = voice self.___speedup = speedup self.___frame = frame + self.semaphore = threading.Semaphore(1) + + def play_audio(self, audio): + """ Service method to play audio in monopoly mode using pydub + + :param audio: AudioSegment needs to be played. + """ + playback.play(audio) + self.semaphore.release() - def __process_via_gtts(self, answer): + async def __process_via_gtts(self, text): + """ + Converts text to speach using gtts text-to-speach method + + :param text: Text needs to be converted to speach. + """ temp_dir = tempfile.gettempdir() - # gtts - tts = gTTS(answer, lang=self.___lang) - tts.save(f"{temp_dir}/raw.mp3") - audio = AudioSegment.from_file(f"{temp_dir}/raw.mp3", format="mp3") - new = audio.speedup(1.3) # speed up by 2x - os.remove(f"{temp_dir}/raw.mp3") - new.export(f"{temp_dir}/response.mp3", format="mp3") - # player - self.___player.play_song(f"{temp_dir}/response.mp3") - audio = MP3(f"{temp_dir}/response.mp3") - sleep(audio.info.length) - self.___player.stop() - os.remove(f"{temp_dir}/response.mp3") - - def __process_via_pytts(self, answer): + tts = gTTS(text, lang=self.___lang) + raw_file = f"{temp_dir}/{str(uuid4())}.mp3" + tts.save(raw_file) + audio = AudioSegment.from_file(raw_file, format="mp3").speedup(self.___speedup) + os.remove(raw_file) + self.semaphore.acquire() + player_thread = threading.Thread(target=self.play_audio, args=(audio,)) + player_thread.start() + + def __process_via_pytts(self, text): + """ + Converts text to speach using python-tts text-to-speach method + + :param text: Text needs to be converted to speach. + """ engine = self.___pytts engine.setProperty("voice", self.___voice) - engine.say(answer) + engine.say(text) engine.startLoop(False) while engine.isBusy(): engine.iterate() - sleep(0.1) + sleep(self.___frame) engine.endLoop() - - async def process(self, answer): - if "google" in self.___method: - self.__process_via_gtts(answer) - else: - self.__process_via_pytts(answer) + self.semaphore.release() def get_pytts_voices_list(self): + """ + Returns list of possible voices + + :return: (list) of possible voices + """ return self.___pytts.getProperty("voices") + + async def process(self, text): + """ + Converts text to speach using pre-defined model + + :param text: Text needs to be converted to speach. + """ + if "google" in self.___method: + await self.__process_via_gtts(text) + else: + self.semaphore.acquire() + player_thread = threading.Thread(target=self.__process_via_pytts, args=(text,)) + player_thread.start()