From 64bb45a5770f2443d39a72f53ead9ec04deb972f Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 6 Sep 2024 18:18:13 +0100 Subject: [PATCH] feat/automations (#4) --- .github/workflows/conventional-label.yaml | 10 ++ .github/workflows/dev2master.yml | 20 +++ .github/workflows/publish_alpha.yml | 118 ++++++++++++ .github/workflows/publish_stable.yml | 73 ++++++++ .github/workflows/pypi_upload.yml | 2 +- CHANGELOG.md | 46 +++++ readme.md => README.md | 0 examples/iptv.py | 210 ++++++++++++++++++++++ examples/livestreams.py | 9 + scripts/remove_alpha.py | 16 ++ scripts/update_version.py | 66 +++++++ setup.py | 33 +++- tutubo/pytube/__main__.py | 4 +- tutubo/version.py | 6 + 14 files changed, 608 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/conventional-label.yaml create mode 100644 .github/workflows/dev2master.yml create mode 100644 .github/workflows/publish_alpha.yml create mode 100644 .github/workflows/publish_stable.yml create mode 100644 CHANGELOG.md rename readme.md => README.md (100%) create mode 100644 examples/iptv.py create mode 100644 examples/livestreams.py create mode 100644 scripts/remove_alpha.py create mode 100644 scripts/update_version.py create mode 100644 tutubo/version.py diff --git a/.github/workflows/conventional-label.yaml b/.github/workflows/conventional-label.yaml new file mode 100644 index 0000000..0a449cb --- /dev/null +++ b/.github/workflows/conventional-label.yaml @@ -0,0 +1,10 @@ +# auto add labels to PRs +on: + pull_request_target: + types: [ opened, edited ] +name: conventional-release-labels +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: bcoe/conventional-release-labels@v1 \ No newline at end of file diff --git a/.github/workflows/dev2master.yml b/.github/workflows/dev2master.yml new file mode 100644 index 0000000..cc76fee --- /dev/null +++ b/.github/workflows/dev2master.yml @@ -0,0 +1,20 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Push dev -> master +on: + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + ref: dev + - name: Push dev -> master + uses: ad-m/github-push-action@master + + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: master \ No newline at end of file diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml new file mode 100644 index 0000000..07a79de --- /dev/null +++ b/.github/workflows/publish_alpha.yml @@ -0,0 +1,118 @@ +# This workflow will generate an ALPHA release distribution and upload it to PyPI +name: Publish Alpha Build +on: + pull_request: + types: [closed] + branches: + - dev + paths-ignore: + - 'version.py' + - 'test/**' + - 'examples/**' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'README.md' + - 'scripts/**' + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # Avoid errors when pushing refs to the destination repository + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Build Tools + run: | + python -m pip install build wheel + + - name: Debug GitHub Labels + run: | + echo "Labels in Pull Request:" + echo "${{ toJson(github.event.pull_request.labels) }}" + + # Convert the labels array into text using jq + LABELS=$(echo '${{ toJson(github.event.pull_request.labels) }}' | jq -r '.[].name') + + # Handle the case where there are no labels + if [ -z "$LABELS" ]; then + echo "No labels found on the pull request." + else + echo "Labels: $LABELS" + fi + + - name: Determine version bump + id: version_bump + run: | + # Convert the labels array into text using jq + LABELS=$(echo '${{ toJson(github.event.pull_request.labels) }}' | jq -r '.[].name') + + # Handle the case where there are no labels + if [ -z "$LABELS" ]; then + echo "No labels found on the pull request." + LABELS="" + fi + + echo "Labels: $LABELS" + + MAJOR=0 + MINOR=0 + BUILD=0 + + # Loop over the labels and determine the version bump + for label in $LABELS; do + echo "Processing label: $label" + if [ "$label" == "breaking" ]; then + MAJOR=1 + elif [ "$label" == "feature" ]; then + MINOR=1 + elif [ "$label" == "fix" ]; then + BUILD=1 + fi + done + + # Set the output based on the labels found + if [ $MAJOR -eq 1 ]; then + echo "::set-output name=part::major" + elif [ $MINOR -eq 1 ]; then + echo "::set-output name=part::minor" + elif [ $BUILD -eq 1 ]; then + echo "::set-output name=part::build" + else + echo "::set-output name=part::alpha" + fi + + - name: Update version in version.py + run: | + python scripts/update_version.py ${{ steps.version_bump.outputs.part }} --version-file $GITHUB_WORKSPACE/tutubo/version.py + + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + maxIssues: 50 + id: changelog + + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version + branch: dev + + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml new file mode 100644 index 0000000..5ce8df0 --- /dev/null +++ b/.github/workflows/publish_stable.yml @@ -0,0 +1,73 @@ +# This workflow will generate a STABLE distribution and upload it to PyPI + +name: Publish Stable Release +on: + push: + branches: + - master + paths-ignore: + - 'version.py' + - 'test/**' + - 'examples/**' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'README.md' + - 'scripts/**' + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Remove Alpha tag + run: | + python scripts/remove_alpha.py + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + maxIssues: 50 + id: changelog + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version + branch: dev + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: true + commitish: dev + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 747142c..d1f69fb 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -38,7 +38,7 @@ jobs: commitish: dev - name: Build Distribution Packages run: | - python setup.py bdist_wheel + python setup.py sdist bdist_wheel - name: Publish to Test PyPI uses: pypa/gh-action-pypi-publish@master with: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..cf04259 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +## [Unreleased](https://github.com/OpenJarbas/tutubo/tree/HEAD) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/V0.0.2a5...HEAD) + +**Breaking changes:** + +- test "breaking" label [\#15](https://github.com/OpenJarbas/tutubo/pull/15) ([JarbasAl](https://github.com/JarbasAl)) + +**Fixed bugs:** + +- fix/videos [\#3](https://github.com/OpenJarbas/tutubo/pull/3) ([JarbasAl](https://github.com/JarbasAl)) +- fix/channel\_playlist\_parsing [\#2](https://github.com/OpenJarbas/tutubo/pull/2) ([JarbasAl](https://github.com/JarbasAl)) + +**Merged pull requests:** + +- fix:test [\#14](https://github.com/OpenJarbas/tutubo/pull/14) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.0.2a5](https://github.com/OpenJarbas/tutubo/tree/V0.0.2a5) (2024-09-06) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/V0.0.2a4...V0.0.2a5) + +## [V0.0.2a4](https://github.com/OpenJarbas/tutubo/tree/V0.0.2a4) (2024-09-06) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/V0.0.2a3...V0.0.2a4) + +## [V0.0.2a3](https://github.com/OpenJarbas/tutubo/tree/V0.0.2a3) (2024-09-06) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/V0.0.2...V0.0.2a3) + +## [V0.0.2](https://github.com/OpenJarbas/tutubo/tree/V0.0.2) (2024-06-22) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/V0.0.2a1...V0.0.2) + +**Fixed bugs:** + +- Zero-indexing where a list may be empty [\#1](https://github.com/OpenJarbas/tutubo/issues/1) + +## [V0.0.2a1](https://github.com/OpenJarbas/tutubo/tree/V0.0.2a1) (2024-05-24) + +[Full Changelog](https://github.com/OpenJarbas/tutubo/compare/cc472cda4ac3f28838dbb3f4d7197569dc8ddf2a...V0.0.2a1) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/readme.md b/README.md similarity index 100% rename from readme.md rename to README.md diff --git a/examples/iptv.py b/examples/iptv.py new file mode 100644 index 0000000..b2549e7 --- /dev/null +++ b/examples/iptv.py @@ -0,0 +1,210 @@ +from tutubo.models import Channel + +kids = [ + "https://www.youtube.com/@hasbroOfficial", + "https://www.youtube.com/@BoomerangUK", + "https://www.youtube.com/@MarvelHQ", + "https://www.youtube.com/@DinoKids", + "https://www.youtube.com/@TheSmurfsEnglish", + "https://www.youtube.com/@ZipZipOfficial", + "https://www.youtube.com/@cartoonito", + "https://www.youtube.com/@HorridHenry", + "https://www.youtube.com/@teletubbies", + "https://www.youtube.com/@PowerRangersOfficial", +] + +movies = [ + "https://www.youtube.com/@ClassicMrBean", + "https://www.youtube.com/@ShoutStudios", + "https://www.youtube.com/@MST3K", + "https://www.youtube.com/@adultswim", +] +news = [ + "https://www.youtube.com/@tpaonline3152", # Angola + "https://www.youtube.com/@euronewspt", + "https://www.youtube.com/@airevolutionx" +] +misc = [ + "https://www.youtube.com/@SpacedustDOC", + "https://www.youtube.com/@DryBarComedy", + "https://www.youtube.com/@EnglishClass101", + # "https://www.youtube.com/@pixeldry", + "https://www.youtube.com/@BitcoinLIVEyt" +] +space = [ + "https://www.youtube.com/@Whataboutit", + "https://www.youtube.com/@WorldCamLIVE", + "https://www.youtube.com/@WNSPACELIVE", +] + +reading = [ + "https://www.youtube.com/@AgroSquerril", + "https://www.youtube.com/@Frequency2156", + "https://www.youtube.com/@CreepsMcPasta" +] + +music = [ + "https://www.youtube.com/@NuclearBlastRecords", + "https://www.youtube.com/@centurymedia", + "https://www.youtube.com/@Herknungr", + "https://www.youtube.com/@solitudeprod", + "https://www.youtube.com/@dadabots_", + "https://www.youtube.com/@LofiGirl", +] +pets = [ + "https://www.youtube.com/@TVfordogs-2106", + "https://www.youtube.com/@sweetpetmusic", + "https://www.youtube.com/@BirderKing" +] +wildlife_cams = [ + "https://www.youtube.com/@nature-live", + "https://www.youtube.com/@NamibiaCam", + "https://www.youtube.com/c/Africamvideos", + "https://www.youtube.com/c/ExploreAfrica" + # @ExploreAfrica ?? goes to wrong channel, https://www.youtube.com/@exploreafrica6302 +] +beach_cams = [ + "https://www.youtube.com/@PlayoceanLive", + "https://www.youtube.com/@MadeiraWebLive" +] + +def get_readings(): + m3u_content = '#EXTM3U\n' + + for url in reading: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_Readings.m3u8", "w") as f: + f.write(m3u_content) + + +def get_channels(): + m3u_content = '#EXTM3U\n' + + for url in news + movies + misc + space: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV.m3u8", "w") as f: + f.write(m3u_content) + + +def get_kids(): + m3u_content = '#EXTM3U\n' + + for url in kids: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_Kids.m3u8", "w") as f: + f.write(m3u_content) + + +def get_music(): + m3u_content = '#EXTM3U\n' + + for url in music: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_Music.m3u8", "w") as f: + f.write(m3u_content) + + +def get_pets(): + m3u_content = '#EXTM3U\n' + + for url in pets: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_Pets.m3u8", "w") as f: + f.write(m3u_content) + + +def get_wildlife_cams(): + m3u_content = '#EXTM3U\n' + + for url in wildlife_cams: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_wildLife.m3u8", "w") as f: + f.write(m3u_content) + + +def get_beachcams(): + m3u_content = '#EXTM3U\n' + + for url in beach_cams: + print(666, url) + try: + c = Channel(url) + for v in c.live: + if not v.is_live: + break # always come after current lives + print(v.title, v.keywords) + m3u_content += f'\n#EXTINF:-1 group-title="TV" tvg-logo="{v.thumbnail_url}",{v.title}\n{v.watch_url}\n' + except Exception as e: + print("error - rate limited?", e) + + with open("youtubeTV_beachCams.m3u8", "w") as f: + f.write(m3u_content) + + +get_readings() +get_channels() +get_kids() +get_music() +get_wildlife_cams() +get_pets() +get_beachcams() diff --git a/examples/livestreams.py b/examples/livestreams.py new file mode 100644 index 0000000..78b4e86 --- /dev/null +++ b/examples/livestreams.py @@ -0,0 +1,9 @@ +from tutubo.models import Channel + +url = "https://www.youtube.com/@ShoutStudios" +c = Channel(url) +for v in c.live: + if not v.is_live: + continue + print(v.title, v.watch_url, v.keywords) + diff --git a/scripts/remove_alpha.py b/scripts/remove_alpha.py new file mode 100644 index 0000000..6cb70bb --- /dev/null +++ b/scripts/remove_alpha.py @@ -0,0 +1,16 @@ +""" +on merge to master -> declare stable (remove alpha) +""" +import fileinput +from os.path import join, dirname + +# TODO - read from env var to allow script to be reused +VERSION_FILE = f"{dirname(dirname(__file__))}/tutubo/version.py" + +alpha_var_name = "VERSION_ALPHA" + +for line in fileinput.input(VERSION_FILE, inplace=True): + if line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/update_version.py b/scripts/update_version.py new file mode 100644 index 0000000..188eaa1 --- /dev/null +++ b/scripts/update_version.py @@ -0,0 +1,66 @@ +""" +on merge to dev: +- depending on labels (conventional commits) script is called with "major", "minor", "build", "alpha" +- on merge to dev, update version.py string to enforce semver +""" + +import sys +import argparse +from os.path import abspath + +def read_version(version_file): + VERSION_MAJOR = 0 + VERSION_MINOR = 0 + VERSION_BUILD = 0 + VERSION_ALPHA = 0 + + with open(version_file, 'r') as file: + content = file.read() + for l in content.split("\n"): + l = l.strip() + if l.startswith("VERSION_MAJOR"): + VERSION_MAJOR = int(l.split("=")[-1]) + elif l.startswith("VERSION_MINOR"): + VERSION_MINOR = int(l.split("=")[-1]) + elif l.startswith("VERSION_BUILD"): + VERSION_BUILD = int(l.split("=")[-1]) + elif l.startswith("VERSION_ALPHA"): + VERSION_ALPHA = int(l.split("=")[-1]) + return VERSION_MAJOR, VERSION_MINOR, VERSION_BUILD, VERSION_ALPHA + + +def update_version(part, version_file): + VERSION_MAJOR, VERSION_MINOR, VERSION_BUILD, VERSION_ALPHA = read_version(version_file) + + if part == 'major': + VERSION_MAJOR += 1 + VERSION_MINOR = 0 + VERSION_BUILD = 0 + VERSION_ALPHA = 1 + elif part == 'minor': + VERSION_MINOR += 1 + VERSION_BUILD = 0 + VERSION_ALPHA = 1 + elif part == 'build': + VERSION_BUILD += 1 + VERSION_ALPHA = 1 + elif part == 'alpha': + VERSION_ALPHA += 1 + + with open(version_file, 'w') as file: + file.write(f"""# START_VERSION_BLOCK +VERSION_MAJOR = {VERSION_MAJOR} +VERSION_MINOR = {VERSION_MINOR} +VERSION_BUILD = {VERSION_BUILD} +VERSION_ALPHA = {VERSION_ALPHA} +# END_VERSION_BLOCK""") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Update the version based on the specified part (major, minor, build, alpha)') + parser.add_argument('part', help='Part of the version to update (major, minor, build, alpha)') + parser.add_argument('--version-file', help='Path to the version.py file', required=True) + + args = parser.parse_args() + + update_version(args.part, abspath(args.version_file)) diff --git a/setup.py b/setup.py index 428edfe..dc75394 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,36 @@ from setuptools import setup +import os + +PKG_NAME = 'tutubo' + +def get_version(): + """ Find the version of this skill""" + version_file = os.path.join(os.path.dirname(__file__), PKG_NAME, 'version.py') + major, minor, build, alpha = (None, None, None, None) + with open(version_file) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if int(alpha): + version += f"a{alpha}" + return version + setup( - name='tutubo', - version='0.0.2', - packages=['tutubo'], + name=PKG_NAME, + version=get_version(), + packages=[PKG_NAME], url='https://github.com/OpenJarbas/tutubo', license='Apache', author='jarbasAI', diff --git a/tutubo/pytube/__main__.py b/tutubo/pytube/__main__.py index 5d6eee8..13b30fe 100644 --- a/tutubo/pytube/__main__.py +++ b/tutubo/pytube/__main__.py @@ -340,6 +340,8 @@ def title(self) -> str: try: self._title = self.vid_info['videoDetails']['title'] except KeyError: + # 'playabilityStatus': {'status': 'LOGIN_REQUIRED', 'reason': 'Sign in to confirm you’re not a bot'} + # Check_availability will raise the correct exception in most cases # if it doesn't, ask for a report. self.check_availability() @@ -378,7 +380,7 @@ def length(self) -> int: :rtype: int """ - return int(self.vid_info.get('videoDetails', {}).get('lengthSeconds')) + return int(self.vid_info.get('videoDetails', {}).get('lengthSeconds') or 0) @property def is_live(self) -> bool: diff --git a/tutubo/version.py b/tutubo/version.py new file mode 100644 index 0000000..40ea5af --- /dev/null +++ b/tutubo/version.py @@ -0,0 +1,6 @@ +# START_VERSION_BLOCK +VERSION_MAJOR = 1 +VERSION_MINOR = 0 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 +# END_VERSION_BLOCK \ No newline at end of file