From e9dca4bf09b628ea5c1ff2c1434415d1e5fffce7 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 18:18:33 +0100 Subject: [PATCH 1/8] feat: github action enforce PR continuous integration rules * configure github action --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..f48ae7c80 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: ci + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + sudo pip install poetry + poetry install \ No newline at end of file From b45935df9de353bfffeb3a62ae1467eba74a6ccd Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 18:20:24 +0100 Subject: [PATCH 2/8] feat: github action enforce PR continuous integration rules * configure runner to execute on ubuntu, macos and windows --- .github/workflows/ci-macos.yml | 23 +++++++++++++++++++++++ .github/workflows/ci-windows.yml | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/workflows/ci-macos.yml create mode 100644 .github/workflows/ci-windows.yml diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml new file mode 100644 index 000000000..03fcb13c0 --- /dev/null +++ b/.github/workflows/ci-macos.yml @@ -0,0 +1,23 @@ +name: ci-macos + +on: [ push, pull_request ] + +jobs: + build: + runs-on: macos-latest + strategy: + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + sudo pip install poetry + poetry install \ No newline at end of file diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml new file mode 100644 index 000000000..4e5befab6 --- /dev/null +++ b/.github/workflows/ci-windows.yml @@ -0,0 +1,23 @@ +name: ci-windows + +on: [ push, pull_request ] + +jobs: + build: + runs-on: windows-latest + strategy: + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: update package manager & install python3 environment + run: | + pip install poetry + poetry install \ No newline at end of file From 1c7e542df3862c061c780c96d84211a15ae24512 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 19:01:12 +0100 Subject: [PATCH 3/8] feat: github action enforce PR continuous integration rules * configure continuous integration pipeline that run with alfred.ci * configure an equivalent of ./build.sh script that run with alfred.build --- .alfred.toml | 13 ++++++++++ alfred/build.py | 33 +++++++++++++++++++++++ alfred/ci.py | 21 +++++++++++++++ alfred/npm.py | 12 +++++++++ poetry.lock | 69 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 .alfred.toml create mode 100644 alfred/build.py create mode 100644 alfred/ci.py create mode 100644 alfred/npm.py diff --git a/.alfred.toml b/.alfred.toml new file mode 100644 index 000000000..1f609067e --- /dev/null +++ b/.alfred.toml @@ -0,0 +1,13 @@ +[alfred] +# name = "" # name of a subproject, use the name of the directory if not set +# description = "" # inline documentation for a sub project +# subprojects = [] + +# [alfred.project] +# command = [ "alfred/*.py" ] +# python_path_project_root = true +# python_path_extends = [] +# venv = "src/.." + +# more info about project manifest +# into https://alfred-cli.readthedocs.io/en/latest/project.html#setting-up-a-project-with-alfred-toml diff --git a/alfred/build.py b/alfred/build.py new file mode 100644 index 000000000..3879c2e21 --- /dev/null +++ b/alfred/build.py @@ -0,0 +1,33 @@ +import os.path +import shutil + +import alfred + + +@alfred.command("build", help="build apps package for pypi") +@alfred.option("--ignore-ci", is_flag=True, help="ignore continuous integration pipeline") +def build(ignore_ci: bool = False): + if not ignore_ci: + alfred.invoke_command("ci") + else: + alfred.invoke_command("npm.build") + + alfred.invoke_command("build.app_provisionning") + alfred.invoke_command("build.poetry") + +@alfred.command("build.app_provisionning", help="update app templates using ./apps", hidden=True) +def build_app_provisionning(): + if os.path.isdir('src/streamsync/app_templates'): + shutil.rmtree('src/streamsync/app_templates') + + shutil.copytree( 'apps/default', 'src/streamsync/app_templates/default') + shutil.copytree( 'apps/hello', 'src/streamsync/app_templates/hello') + +@alfred.command("build.poetry", help="build python packages with poetry", hidden=True) +def build_poetry(): + removed_directories = ['dist', 'build'] + for directory in removed_directories: + if os.path.isdir(directory): + shutil.rmtree(directory) + + alfred.run("poetry build") \ No newline at end of file diff --git a/alfred/ci.py b/alfred/ci.py new file mode 100644 index 000000000..b91ddcb14 --- /dev/null +++ b/alfred/ci.py @@ -0,0 +1,21 @@ +import os + +import alfred + + +@alfred.command("ci", help="continuous integration pipeline") +def ci(): + alfred.invoke_command("ci.mypy") + alfred.invoke_command("ci.pytest") + alfred.invoke_command("npm.build") + + +@alfred.command("ci.mypy", help="typing checking with mypy on ./src/streamsync") +def ci_mypy(): + alfred.run("mypy ./src/streamsync --exclude app_templates/*") + + +@alfred.command("ci.pytest", help="run pytest on ./tests") +def ci_test(): + os.chdir("tests") + alfred.run("pytest") \ No newline at end of file diff --git a/alfred/npm.py b/alfred/npm.py new file mode 100644 index 000000000..c1d980e03 --- /dev/null +++ b/alfred/npm.py @@ -0,0 +1,12 @@ +import alfred +import os + +@alfred.command("npm.build", help="build ui code") +def npm_build(): + os.chdir("ui") + alfred.run("npm run build") + +@alfred.command("npm.build_custom_components", help="build custom components") +def ui_build_custom(): + os.chdir("ui") + alfred.run("npm run custom.build") \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 90f37cec3..fe3f03d92 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,22 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "alfred-cli" +version = "2.2.6" +description = "Alfred is an extensible automation tool. It allows you to build your continuous integration scripts in python, and much more. You can replace any scripts using the best of both worlds, shell and python." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "alfred_cli-2.2.6-py3-none-any.whl", hash = "sha256:afbfaff7ea2772c5716b4fcab921e518998bd78e919776eeb91379ffec9a5bb0"}, + {file = "alfred_cli-2.2.6.tar.gz", hash = "sha256:b02453ebecce04b32bc52f1c2ea8db83919cf0fda95dcb476a4b7e2c7a6c450f"}, +] + +[package.dependencies] +click = ">=8.1.0,<9.0.0" +prompt-toolkit = ">=3.0.41,<4.0.0" +shellingham = ">=1.3.0,<2.0.0" +toml = ">=0.10,<0.11" + [[package]] name = "altair" version = "5.2.0" @@ -575,6 +592,20 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "prompt-toolkit" +version = "3.0.43" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "pyarrow" version = "15.0.0" @@ -921,6 +952,17 @@ files = [ {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, ] +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + [[package]] name = "six" version = "1.16.0" @@ -975,6 +1017,17 @@ files = [ [package.extras] doc = ["reno", "sphinx", "tornado (>=4.5)"] +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -1088,6 +1141,17 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + [[package]] name = "websockets" version = "12.0" @@ -1169,7 +1233,10 @@ files = [ {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] +[extras] +ds = ["pandas", "plotly", "pyarrow"] + [metadata] lock-version = "2.0" python-versions = ">=3.9.2, <4.0" -content-hash = "a1b429c27b38c251b207151039018f3eab470309ee2f52efff2f6e079b587c69" +content-hash = "82329fe83a8558191025e96c7ca62911bec6da67fcfdbec518c98c8e44948cfa" diff --git a/pyproject.toml b/pyproject.toml index ad2f579c4..dbcf1ed62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ pytest-asyncio = ">= 0.23.4, < 1" pytest = ">= 7.0.0, < 8" altair = ">= 5.2.0, < 6" httpx = ">=0.26.0, < 1" +alfred-cli = "^2.2.6" [tool.poetry.extras] ds = ["pandas", "pyarrow", "plotly"] From 7a72e9913ad8658d528a7618ded45d8c6c21d8b6 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 19:02:21 +0100 Subject: [PATCH 4/8] feat: github action enforce PR continuous integration rules * configure github action to run continuous integration pipeline --- .github/workflows/ci-macos.yml | 16 +++++++++++++++- .github/workflows/ci-windows.yml | 16 +++++++++++++++- .github/workflows/ci.yml | 11 ++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 03fcb13c0..23d16d7fe 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -12,6 +12,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: "18.x" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -20,4 +25,13 @@ jobs: - name: update package manager & install python3 environment run: | sudo pip install poetry - poetry install \ No newline at end of file + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 4e5befab6..e1fb21fa2 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -12,6 +12,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: "18.x" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -20,4 +25,13 @@ jobs: - name: update package manager & install python3 environment run: | pip install poetry - poetry install \ No newline at end of file + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f48ae7c80..f229bf28f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,4 +20,13 @@ jobs: - name: update package manager & install python3 environment run: | sudo pip install poetry - poetry install \ No newline at end of file + poetry install --with build + + - name: install npm environment + run: | + cd ui + npm install + + - name: run continuous integration pipeline + run: | + poetry run alfred ci \ No newline at end of file From cf6c18f3210d4d59b07a08d2d4145da2ce6ff7a6 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 19:31:27 +0100 Subject: [PATCH 5/8] feat: github action enforce PR continuous integration rules * fix typing error checking on windows --- src/streamsync/app_runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/streamsync/app_runner.py b/src/streamsync/app_runner.py index ebe95704f..e817093e3 100644 --- a/src/streamsync/app_runner.py +++ b/src/streamsync/app_runner.py @@ -13,7 +13,7 @@ import logging.handlers from types import ModuleType import json -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, cast from watchdog.observers.polling import PollingObserver @@ -748,7 +748,10 @@ def _start_app_process(self) -> None: raise ValueError( "Cannot start app process. Components haven't been set.") self.is_app_process_server_ready.clear() - self.client_conn, self.server_conn = multiprocessing.Pipe(duplex=True) + client_conn, server_conn = multiprocessing.Pipe(duplex=True) + self.client_conn = cast(multiprocessing.connection.Connection, client_conn) # for mypy type checking on windows + self.server_conn = cast(multiprocessing.connection.Connection, server_conn) # for mypy type checking on windows + self.app_process = AppProcess( client_conn=self.client_conn, server_conn=self.server_conn, From ec4c4ed5a6f855438f7a03e65178589e663aee81 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 19:47:35 +0100 Subject: [PATCH 6/8] feat: github action enforce PR continuous integration rules * configure badges on readme --- README.md | 5 +++++ pyproject.toml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index ce08b4292..06c9edb6c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ ## What is Streamsync? +[![PyPi](https://img.shields.io/pypi/v/streamsync.svg?label=Version)](https://pypi.org/project/streamsync/) +[![CI](https://github.com/streamsync-cloud/streamsync/actions/workflows/ci.yml/badge.svg)](https://github.com/streamsync-cloud/streamsync/actions/workflows/ci.yml) +[![Discord](https://img.shields.io/badge/discord-streamsync-sn677E3Pd3?logo=discord&logoColor=white)](https://discord.gg/sn677E3Pd3) +[![License](https://img.shields.io/pypi/l/streamsync)](LICENSE) + Streamsync is an open-source framework for creating data apps. Build user interfaces using a visual editor; write the backend code in Python. ![Streamsync Builder screenshot](https://raw.githubusercontent.com/streamsync-cloud/streamsync/master/docs/docs/public/sc1.png "Streamsync Builder screenshot") diff --git a/pyproject.toml b/pyproject.toml index dbcf1ed62..d2c1abcaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,9 @@ repository = "https://www.github.com/streamsync-cloud/streamsync" documentation = "https://www.streamsync.cloud/getting-started.html" keywords = ["data apps", "gui", "ui"] classifiers = [ + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Development Status :: 4 - Beta" ] packages = [ From a55ec1a94f66368605f259a12f14a3dc971bf9c1 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Sat, 10 Feb 2024 20:04:40 +0100 Subject: [PATCH 7/8] feat: github action enforce PR continuous integration rules * fix charmap error in continuous integration on windows --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index fe3f03d92..11be96bed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "alfred-cli" -version = "2.2.6" +version = "2.2.7" description = "Alfred is an extensible automation tool. It allows you to build your continuous integration scripts in python, and much more. You can replace any scripts using the best of both worlds, shell and python." optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "alfred_cli-2.2.6-py3-none-any.whl", hash = "sha256:afbfaff7ea2772c5716b4fcab921e518998bd78e919776eeb91379ffec9a5bb0"}, - {file = "alfred_cli-2.2.6.tar.gz", hash = "sha256:b02453ebecce04b32bc52f1c2ea8db83919cf0fda95dcb476a4b7e2c7a6c450f"}, + {file = "alfred_cli-2.2.7-py3-none-any.whl", hash = "sha256:35059a1525c792f77d1911d8d67df698f9d93bc2c06f28dc3deba125ce886228"}, + {file = "alfred_cli-2.2.7.tar.gz", hash = "sha256:674aafa6fe5445e0e60ebad94b7c5a84be1e4fdd64fb2c51223c357734981050"}, ] [package.dependencies] @@ -1239,4 +1239,4 @@ ds = ["pandas", "plotly", "pyarrow"] [metadata] lock-version = "2.0" python-versions = ">=3.9.2, <4.0" -content-hash = "82329fe83a8558191025e96c7ca62911bec6da67fcfdbec518c98c8e44948cfa" +content-hash = "eda7baf2623c0f476a9d2a5ae111af040aedab615fac92e438ba18b3312ad4d9" diff --git a/pyproject.toml b/pyproject.toml index d2c1abcaf..d391d18e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ pytest-asyncio = ">= 0.23.4, < 1" pytest = ">= 7.0.0, < 8" altair = ">= 5.2.0, < 6" httpx = ">=0.26.0, < 1" -alfred-cli = "^2.2.6" +alfred-cli = "^2.2.7" [tool.poetry.extras] ds = ["pandas", "pyarrow", "plotly"] From eca6ca3f0f93fc32b0a6f61165dc9a262c496269 Mon Sep 17 00:00:00 2001 From: Fabien Arcellier Date: Mon, 12 Feb 2024 14:14:42 +0100 Subject: [PATCH 8/8] feat: github action enforce PR continuous integration rules * enable version 3.9 --- .github/workflows/ci-macos.yml | 2 +- .github/workflows/ci-windows.yml | 2 +- .github/workflows/ci.yml | 2 +- pyproject.toml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 23d16d7fe..01cfede5c 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -7,7 +7,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [ "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index e1fb21fa2..c82b8318b 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -7,7 +7,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: [ "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f229bf28f..b148906e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index d391d18e7..f705be7fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ repository = "https://www.github.com/streamsync-cloud/streamsync" documentation = "https://www.streamsync.cloud/getting-started.html" keywords = ["data apps", "gui", "ui"] classifiers = [ + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12",