diff --git a/.alfred.toml b/.alfred.toml new file mode 100644 index 000000000..1f609067e --- /dev/null +++ b/.alfred.toml @@ -0,0 +1,13 @@ +[alfred] +# name = "" # name of a subproject, use the name of the directory if not set +# description = "" # inline documentation for a sub project +# subprojects = [] + +# [alfred.project] +# command = [ "alfred/*.py" ] +# python_path_project_root = true +# python_path_extends = [] +# venv = "src/.." + +# more info about project manifest +# into https://alfred-cli.readthedocs.io/en/latest/project.html#setting-up-a-project-with-alfred-toml diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml new file mode 100644 index 000000000..01cfede5c --- /dev/null +++ b/.github/workflows/ci-macos.yml @@ -0,0 +1,37 @@ +name: ci-macos + +on: [ push, pull_request ] + +jobs: + build: + runs-on: macos-latest + strategy: + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: "18.x" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + sudo pip install poetry + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml new file mode 100644 index 000000000..c82b8318b --- /dev/null +++ b/.github/workflows/ci-windows.yml @@ -0,0 +1,37 @@ +name: ci-windows + +on: [ push, pull_request ] + +jobs: + build: + runs-on: windows-latest + strategy: + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: "18.x" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + pip install poetry + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..b148906e0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +name: ci + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + sudo pip install poetry + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file diff --git a/README.md b/README.md index ce08b4292..06c9edb6c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ ## What is Streamsync? +[![PyPi](https://img.shields.io/pypi/v/streamsync.svg?label=Version)](https://pypi.org/project/streamsync/) +[![CI](https://github.com/streamsync-cloud/streamsync/actions/workflows/ci.yml/badge.svg)](https://github.com/streamsync-cloud/streamsync/actions/workflows/ci.yml) +[![Discord](https://img.shields.io/badge/discord-streamsync-sn677E3Pd3?logo=discord&logoColor=white)](https://discord.gg/sn677E3Pd3) +[![License](https://img.shields.io/pypi/l/streamsync)](LICENSE) + Streamsync is an open-source framework for creating data apps. Build user interfaces using a visual editor; write the backend code in Python. ![Streamsync Builder screenshot](https://raw.githubusercontent.com/streamsync-cloud/streamsync/master/docs/docs/public/sc1.png "Streamsync Builder screenshot") diff --git a/alfred/build.py b/alfred/build.py new file mode 100644 index 000000000..3879c2e21 --- /dev/null +++ b/alfred/build.py @@ -0,0 +1,33 @@ +import os.path +import shutil + +import alfred + + +@alfred.command("build", help="build apps package for pypi") +@alfred.option("--ignore-ci", is_flag=True, help="ignore continuous integration pipeline") +def build(ignore_ci: bool = False): + if not ignore_ci: + alfred.invoke_command("ci") + else: + alfred.invoke_command("npm.build") + + alfred.invoke_command("build.app_provisionning") + alfred.invoke_command("build.poetry") + +@alfred.command("build.app_provisionning", help="update app templates using ./apps", hidden=True) +def build_app_provisionning(): + if os.path.isdir('src/streamsync/app_templates'): + shutil.rmtree('src/streamsync/app_templates') + + shutil.copytree( 'apps/default', 'src/streamsync/app_templates/default') + shutil.copytree( 'apps/hello', 'src/streamsync/app_templates/hello') + +@alfred.command("build.poetry", help="build python packages with poetry", hidden=True) +def build_poetry(): + removed_directories = ['dist', 'build'] + for directory in removed_directories: + if os.path.isdir(directory): + shutil.rmtree(directory) + + alfred.run("poetry build") \ No newline at end of file diff --git a/alfred/ci.py b/alfred/ci.py new file mode 100644 index 000000000..b91ddcb14 --- /dev/null +++ b/alfred/ci.py @@ -0,0 +1,21 @@ +import os + +import alfred + + +@alfred.command("ci", help="continuous integration pipeline") +def ci(): + alfred.invoke_command("ci.mypy") + alfred.invoke_command("ci.pytest") + alfred.invoke_command("npm.build") + + +@alfred.command("ci.mypy", help="typing checking with mypy on ./src/streamsync") +def ci_mypy(): + alfred.run("mypy ./src/streamsync --exclude app_templates/*") + + +@alfred.command("ci.pytest", help="run pytest on ./tests") +def ci_test(): + os.chdir("tests") + alfred.run("pytest") \ No newline at end of file diff --git a/alfred/npm.py b/alfred/npm.py new file mode 100644 index 000000000..c1d980e03 --- /dev/null +++ b/alfred/npm.py @@ -0,0 +1,12 @@ +import alfred +import os + +@alfred.command("npm.build", help="build ui code") +def npm_build(): + os.chdir("ui") + alfred.run("npm run build") + +@alfred.command("npm.build_custom_components", help="build custom components") +def ui_build_custom(): + os.chdir("ui") + alfred.run("npm run custom.build") \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 90f37cec3..11be96bed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,22 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "alfred-cli" +version = "2.2.7" +description = "Alfred is an extensible automation tool. It allows you to build your continuous integration scripts in python, and much more. You can replace any scripts using the best of both worlds, shell and python." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "alfred_cli-2.2.7-py3-none-any.whl", hash = "sha256:35059a1525c792f77d1911d8d67df698f9d93bc2c06f28dc3deba125ce886228"}, + {file = "alfred_cli-2.2.7.tar.gz", hash = "sha256:674aafa6fe5445e0e60ebad94b7c5a84be1e4fdd64fb2c51223c357734981050"}, +] + +[package.dependencies] +click = ">=8.1.0,<9.0.0" +prompt-toolkit = ">=3.0.41,<4.0.0" +shellingham = ">=1.3.0,<2.0.0" +toml = ">=0.10,<0.11" + [[package]] name = "altair" version = "5.2.0" @@ -575,6 +592,20 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "prompt-toolkit" +version = "3.0.43" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "pyarrow" version = "15.0.0" @@ -921,6 +952,17 @@ files = [ {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, ] +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + [[package]] name = "six" version = "1.16.0" @@ -975,6 +1017,17 @@ files = [ [package.extras] doc = ["reno", "sphinx", "tornado (>=4.5)"] +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -1088,6 +1141,17 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + [[package]] name = "websockets" version = "12.0" @@ -1169,7 +1233,10 @@ files = [ {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] +[extras] +ds = ["pandas", "plotly", "pyarrow"] + [metadata] lock-version = "2.0" python-versions = ">=3.9.2, <4.0" -content-hash = "a1b429c27b38c251b207151039018f3eab470309ee2f52efff2f6e079b587c69" +content-hash = "eda7baf2623c0f476a9d2a5ae111af040aedab615fac92e438ba18b3312ad4d9" diff --git a/pyproject.toml b/pyproject.toml index ad2f579c4..f705be7fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,10 @@ repository = "https://www.github.com/streamsync-cloud/streamsync" documentation = "https://www.streamsync.cloud/getting-started.html" keywords = ["data apps", "gui", "ui"] classifiers = [ + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Development Status :: 4 - Beta" ] packages = [ @@ -45,6 +49,7 @@ pytest-asyncio = ">= 0.23.4, < 1" pytest = ">= 7.0.0, < 8" altair = ">= 5.2.0, < 6" httpx = ">=0.26.0, < 1" +alfred-cli = "^2.2.7" [tool.poetry.extras] ds = ["pandas", "pyarrow", "plotly"] diff --git a/src/streamsync/app_runner.py b/src/streamsync/app_runner.py index ebe95704f..e817093e3 100644 --- a/src/streamsync/app_runner.py +++ b/src/streamsync/app_runner.py @@ -13,7 +13,7 @@ import logging.handlers from types import ModuleType import json -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, cast from watchdog.observers.polling import PollingObserver @@ -748,7 +748,10 @@ def _start_app_process(self) -> None: raise ValueError( "Cannot start app process. Components haven't been set.") self.is_app_process_server_ready.clear() - self.client_conn, self.server_conn = multiprocessing.Pipe(duplex=True) + client_conn, server_conn = multiprocessing.Pipe(duplex=True) + self.client_conn = cast(multiprocessing.connection.Connection, client_conn) # for mypy type checking on windows + self.server_conn = cast(multiprocessing.connection.Connection, server_conn) # for mypy type checking on windows + self.app_process = AppProcess( client_conn=self.client_conn, server_conn=self.server_conn,