From 057a5b877ef0cbbba98934928e8d30eebb6775d2 Mon Sep 17 00:00:00 2001 From: Steve-Tech Date: Tue, 15 Feb 2022 13:59:32 +1000 Subject: [PATCH 01/21] Added Tidal plugin --- requirements.txt | 3 +- ultrasonics/official_plugins/up_tidal.py | 723 +++++++++++++++++++++++ 2 files changed, 725 insertions(+), 1 deletion(-) create mode 100644 ultrasonics/official_plugins/up_tidal.py diff --git a/requirements.txt b/requirements.txt index 9c713ea..7549916 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ fuzzywuzzy==0.18.0 python-Levenshtein==0.12.2 spotipy==2.19.0 tqdm==4.62.3 -PlexAPI==4.7.2 \ No newline at end of file +PlexAPI==4.7.2 +tidalapi==0.7.0 \ No newline at end of file diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py new file mode 100644 index 0000000..abea109 --- /dev/null +++ b/ultrasonics/official_plugins/up_tidal.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python3 + +""" +up_tidal + +input and output plugin for Tidal, based off of up_spotify so it will do most of the things it does + +Steve-Tech & XDGFX, 2022 +""" + +import os +import re +import sqlite3 +import time + +import tidalapi +from tqdm import tqdm + +from app import _ultrasonics +from ultrasonics import logs +from ultrasonics.tools import fuzzymatch, name_filter + +log = logs.create_log(__name__) + +handshake = { + "name": "tidal", + "description": "sync your playlists to and from tidal", + "type": ["inputs", "outputs"], + "mode": ["playlists"], + "version": "0.1", # Optionally, "0.0.0" + "settings": [ + { + "type": "string", + "value": "Tidal Login, run tidal_login.py for these values", + }, + { + "type": "text", + "label": "Session ID", + "name": "session_id", + }, + { + "type": "text", + "label": "Token Type", + "name": "token_type", + }, + { + "type": "text", + "label": "Access Token", + "name": "access_token", + }, + { + "type": "text", + "label": "Refresh Token", + "name": "refresh_token", + }, + { + "type": "string", + "value": "Songs will always attempt to be matched using fixed values like ISRC, however if you're trying to sync music without these tags, fuzzy matching will be used instead.", + }, + { + "type": "string", + "value": "This means that the titles 'You & Me - Flume Remix', and 'You & Me (Flume Remix)' will probably qualify as the same song [with a fuzzy score of 96.85 if all other fields are identical]. However, 'You, Me, & the Log Flume Ride' probably won't đŸŽĸ [the score was 88.45 assuming *all* other fields are identical]. The fuzzyness of this matching is determined with the below setting. A value of 100 means all song fields must be identical to pass as duplicates. A value of 0 means any song will quality as a match, even if they are completely different. đŸ‘Ŋ", + }, + { + "type": "text", + "label": "Default Global Fuzzy Ratio", + "name": "fuzzy_ratio", + "value": "Recommended: 90", + }, + ], +} + + +def run(settings_dict, **kwargs): + """ + Runs the up_tidal plugin. + + Important note: songs will only be appended to playlists if they are new! + No songs will be removed from existing playlists, nothing will be over-written. + This behaviour is different from some other plugins. + """ + + database = kwargs["database"] + global_settings = kwargs["global_settings"] + component = kwargs["component"] + applet_id = kwargs["applet_id"] + songs_dict = kwargs["songs_dict"] + + class Tidal: + """ + Class for interactions with Tidal through the tidalapi library. + """ + + def __init__(self): + self.session = tidalapi.Session() + + def login(self): + success = self.session.load_oauth_session(database["session_id"], database["token_type"], database["access_token"], database["refresh_token"]) + if success: + log.debug("Successfully loaded session") + else: + log.error("Error loading session") + return success + + def request(self, tidal_func, *args, **kwargs): + """ + Used to call a tidalapi function, with automatic catching and renewing on access token errors. + """ + errors = 0 + + while errors <= 1: + try: + return tidal_func(*args, **kwargs) + + except Exception as e: + # Renew token + log.error(e) + self.session.token_refresh(database["refresh_token"]) + errors += 1 + continue + + log.error("An error occurred while trying to contact the Tidal api.") + raise Exception(e) + + def search(self, track): + """ + Used to search the Tidal API for a song, supplied in standard songs_dict format. + Will attempt to match using fixed values (Tidal ID) before moving onto fuzzy values. + Tidal doesn't seem to support searching ISRC, and just searching is pretty sketchy compared to spotify. + + @returns: + Tidal ID, confidence score + """ + + cutoff_regex = [ + "[([](feat|ft|featuring|original|prod).+?[)\]]", + "[ (\- )\-]+(feat|ft|featuring|original|prod).+?(?=[(\n])", + ] + + # 1. Tidal ID + try: + tidal_id = track["id"]["tidal"] + # tidal_uri = f"tidal:track:{tidal_id}" + confidence = 100 + + return tidal_id, confidence + + except KeyError: + # Tidal ID was not supplied + pass + + # 2. Other fields + # Multiple searches are made as Tidal is more likely to return false negative (missing songs) + # than false positive, when specifying many query parameters. + + queries = [] + + try: + title = ( + re.sub(cutoff_regex[0], "", track["title"], flags=re.IGNORECASE) + + "\n" + ) + + title = re.sub( + cutoff_regex[1], " ", title, flags=re.IGNORECASE + ).strip() + except KeyError: + pass + + try: + album = ( + re.sub(cutoff_regex[0], "", track["album"], flags=re.IGNORECASE) + + "\n" + ) + + album = re.sub( + cutoff_regex[1], " ", album, flags=re.IGNORECASE + ).strip() + except KeyError: + pass + + try: + queries.append(f"{title} {album}") + except NameError: + pass + + try: + for artist in track["artists"]: + queries.append(f"{title} {artist}") + except NameError: + pass + + try: + queries.append(title) + except NameError: + pass + + results_list = [] + + # Execute all queries + for query in queries: + results = self.request(self.session.search, query) + + # Convert to ultrasonics format and append to results_list + for item in results['tracks']: + item = s.tidal_to_songs_dict(item) + if item not in results_list: + results_list.append(item) + + if not results_list: + # No items were found + return "", 0 + + # Check results with fuzzy matching + confidence = 0 + + for item in results_list: + score = fuzzymatch.similarity(track, item) + if score > confidence: + matched_track = item + confidence = score + if confidence > 100: + break + + # tidal_uri = f"tidal:track:{matched_track['id']['tidal']}" + + return matched_track['id']['tidal'], confidence + + def current_user_playlists(self): + """ + Wrapper for Tidal `current_user_playlists`. + """ + + playlists = self.request(self.session.user.playlists) + + log.info(f"Found {len(playlists)} playlist(s) on Tidal.") + + return [{'id':{'tidal':playlist.id},'name':playlist.name, 'description':playlist.description} for playlist in playlists] + + def current_user_saved_tracks(self): + """ + Wrapper for TidalAPI `current_user_saved_tracks`. + """ + tracks = self.request(self.session.user.favorites.tracks) + + tidal_ids = [item.id for item in tracks] + + return tidal_ids, tracks + + def playlist_tracks(self, playlist_id): + """ + Wrapper for TidalAPI `playlist_tracks`. + """ + + tracks = self.request(self.session.playlist, playlist_id).tracks() + + track_list = [] + + # Convert from Tidal API format to ultrasonics format + log.info("Converting tracks to ultrasonics format.") + for track in tqdm( + tracks, + desc=f"Converting tracks in {playlist_id}", + ): + try: + track_list.append(s.tidal_to_songs_dict(track)) + except TypeError: + log.error( + f"Could not convert track {track.id} to ultrasonics format." + ) + continue + + return track_list + + def user_playlist_remove_all_occurrences_of_tracks(self, playlist_id, tracks): + """ + Wrapper for the tidalapi function of the same name. + Removes all `tracks` from the specified playlist. + """ + playlist = tidalapi.playlist.UserPlaylist(self.session, playlist_id) + for track in tracks: + playlist.remove_by_id(track['id']) + + def tidal_to_songs_dict(self, track): + """ + Convert dictionary received from Tidal API to ultrasonics songs_dict format. + Assumes title, artist(s), and id field are always present. + """ + + item = { + "title": track.name, + "artists": [artist.name for artist in track.artists], + "album": track.album.name, + "date": track.album.release_date, + "isrc": track.isrc, + "id": {"tidal": track.id} + } + + # Remove any empty fields + item = {k: v for k, v in item.items() if v} + + return item + + def user_id(self): + """ + Get and return the current user's user ID. + """ + user_info = self.request(self.session.user.id) + self.user_id = user_info + + class Database: + """ + Class for interactions with the up_tidal database. + Currently used for storing info about saved songs. + """ + + def __init__(self): + # Create database if required + self.saved_songs_db = os.path.join( + _ultrasonics["config_dir"], "up_tidal", "saved_songs.db" + ) + + # Create the containing folder if it doesn't already exist + try: + os.mkdir(os.path.dirname(self.saved_songs_db)) + except FileExistsError: + # Folder already exists + pass + + + with sqlite3.connect(self.saved_songs_db) as conn: + cursor = conn.cursor() + + # Create saved songs table if needed + query = "CREATE TABLE IF NOT EXISTS saved_songs (applet_id TEXT, tidal_id TEXT)" + cursor.execute(query) + + # Create lastrun table if needed + query = "CREATE TABLE IF NOT EXISTS lastrun (applet_id TEXT PRIMARY KEY, time INTEGER)" + cursor.execute(query) + + conn.commit() + + def lastrun_get(self): + """ + Gets the last run time for saved songs mode. + """ + with sqlite3.connect(self.saved_songs_db) as conn: + cursor = conn.cursor() + + query = "SELECT time FROM lastrun WHERE applet_id = ?" + cursor.execute(query, (applet_id,)) + + rows = cursor.fetchone() + + return None if not rows else rows[0] + + def lastrun_set(self): + """ + Updates the last run time for saved songs mode. + """ + with sqlite3.connect(self.saved_songs_db) as conn: + cursor = conn.cursor() + + query = "REPLACE INTO lastrun (time, applet_id) VALUES (?, ?)" + cursor.execute(query, (int(time.time()), applet_id)) + + def saved_songs_contains(self, tidal_id): + """ + Checks if the input `tidal_id` is present in the saved songs database. + """ + with sqlite3.connect(self.saved_songs_db) as conn: + cursor = conn.cursor() + + query = "SELECT EXISTS(SELECT * FROM saved_songs WHERE applet_id = ? AND tidal_id = ?)" + cursor.execute(query, (applet_id, tidal_id)) + + rows = cursor.fetchone() + + return rows[0] + + def saved_songs_add(self, tidal_ids): + """ + Adds all songs in a list of `tidal_ids` to the saved songs database for this applet. + """ + with sqlite3.connect(self.saved_songs_db) as conn: + cursor = conn.cursor() + + # Add saved songs to database + query = "INSERT INTO saved_songs (applet_id, tidal_id) VALUES (?, ?)" + values = [(applet_id, tidal_id) for tidal_id in tidal_ids] + cursor.executemany(query, values) + + conn.commit() + + s = Tidal() + db = Database() + + s.api_url = global_settings["api_url"] + + # auth = json.loads(database["auth"]) + s.refresh_token = database["refresh_token"] + + s.login() + + if component == "inputs": + if settings_dict["mode"] == "playlists": + # Playlists mode + + # 1. Get a list of users playlists + playlists = s.current_user_playlists() + + songs_dict = [] + + for playlist in playlists: + item = {"name": playlist["name"], "id": playlist["id"]} + + songs_dict.append(item) + + # 2. Filter playlist titles + songs_dict = name_filter.filter(songs_dict, settings_dict["filter"]) + + # 3. Fetch songs from each playlist, build songs_dict + log.info("Building songs_dict for playlists...") + for i, playlist in tqdm(enumerate(songs_dict)): + tracks = s.playlist_tracks(playlist["id"]["tidal"]) + + songs_dict[i]["songs"] = tracks + + return songs_dict + + elif settings_dict["mode"] == "saved": + # Saved songs mode + if db.lastrun_get(): + # Update songs + songs = [] + + tidal_ids, tracks = s.current_user_saved_tracks() + + for tidal_id, track in zip(tidal_ids, tracks): + if db.saved_songs_contains(tidal_id): + break + else: + songs.append(s.tidal_to_songs_dict(track["track"])) + + if not songs: + log.info("No new saved songs were found. Exiting this applet.") + raise Exception("No new saved songs found on this applet run.") + + songs_dict = [ + { + "name": settings_dict["playlist_title"] + or "Tidal Saved Songs", + "id": {}, + "songs": songs, + } + ] + + return songs_dict + + else: + log.info( + "This is the first time this applet plugin has run in saved songs mode." + ) + log.info( + "This first run will be used to get the current state of saved songs, but will not pass any songs in songs_dict." + ) + + # 1. Get some saved songs + tidal_ids, _ = s.current_user_saved_tracks() + + # 2. Update database with saved songs + db.saved_songs_add(tidal_ids) + + # 3. Update lastrun + db.lastrun_set() + + raise Exception( + "Initial run of this plugin will not return a songs_dict. Database is now updated. Next run will continue as normal." + ) + + else: + "Outputs mode" + + # Set the user_id variable + # s.user_id() + + # Get a list of current user playlists + current_playlists = s.current_user_playlists() + + for playlist in songs_dict: + # Check the playlist already exists in Tidal + playlist_id = "" + try: + if playlist["id"]["tidal"] in [ + item["id"] for item in current_playlists + ]: + playlist_id = playlist["id"]["tidal"] + except KeyError: + if playlist["name"] in [item["name"] for item in current_playlists]: + playlist_id = [ + item["id"]["tidal"] + for item in current_playlists + if item["name"] == playlist["name"] + ][0] + if playlist_id: + log.info( + f"Playlist {playlist['name']} already exists, updating that one." + ) + else: + log.info( + f"Playlist {playlist['name']} does not exist, creating it now..." + ) + # Playlist must be created + description = "Created automatically by ultrasonics with 💖" + + response = s.request( + s.session.user.create_playlist, + playlist["name"], + description, + ) + + playlist_id = response.id + + existing_tracks = [] + + # Get all tracks already in the playlist + if "existing_tracks" not in vars(): + existing_tracks = s.playlist_tracks(playlist_id) + + # Add songs which don't already exist in the playlist + ids = [] + duplicate_ids = [] + + log.info("Searching for matching songs in Tidal.") + for song in tqdm( + playlist["songs"], + desc=f"Searching Tidal for songs", + ): + # First check for fuzzy duplicate without Tidal api search + duplicate = False + for item in existing_tracks: + score = fuzzymatch.similarity(song, item) + + if score > float(database.get("fuzzy_ratio") or 90): + # Duplicate was found + duplicate_ids.append(item['id']['tidal']) + duplicate = True + break + + if duplicate: + continue + + id, confidence = s.search(song) + + if id in existing_tracks: + duplicate_ids.append(id) + + if confidence > float(database.get("fuzzy_ratio") or 90): + ids.append(id) + else: + log.debug( + f"Could not find song {song['title']} in Tidal; will not add to playlist." + ) + + if settings_dict["existing_playlists"] == "Update": + # Remove any songs which aren't in `uris` from the playlist + remove_ids = [ + id for id in existing_tracks if id not in ids + duplicate_ids + ] + + s.user_playlist_remove_all_occurrences_of_tracks( + playlist_id, remove_ids + ) + + # Add tracks to playlist + if ids: + api_playlist = s.request( + tidalapi.playlist.UserPlaylist, s.session, playlist_id + ) + + s.request( + api_playlist.add, ids + ) + + + +def test(database, **kwargs): + """ + An optional test function. Used to validate persistent settings supplied in database. + Any errors raised will be caught and displayed to the user for debugging. + If this function is present, test failure will prevent the plugin being added. + """ + + # global_settings = kwargs["global_settings"] + + session = tidalapi.Session() + assert session.load_oauth_session(database["session_id"], database["token_type"], database["access_token"], database["refresh_token"]), "Error logging in" + + +def builder(**kwargs): + component = kwargs["component"] + + if component == "inputs": + settings_dict = [ + """ +
+ +
+ + + + + +
+
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
+ +
+ + + """ + ] + + return settings_dict + + else: + settings_dict = [ + { + "type": "string", + "value": "Do you want to update any existing playlists with the same name (replace any songs already in the playlist), or append to them?", + }, + { + "type": "radio", + "label": "Existing Playlists", + "name": "existing_playlists", + "id": "existing_playlists", + "options": ["Append", "Update"], + "required": True, + }, + ] + + return settings_dict From de113d4a2f32738dd8fece8e0dfb5ee05af010e3 Mon Sep 17 00:00:00 2001 From: Steve-Tech Date: Tue, 15 Feb 2022 13:46:59 +1000 Subject: [PATCH 02/21] Add some temporary documentation for the tidal plugin --- TIDAL_README.md | 12 ++++++++++++ tidal_login.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 TIDAL_README.md create mode 100644 tidal_login.py diff --git a/TIDAL_README.md b/TIDAL_README.md new file mode 100644 index 0000000..1064997 --- /dev/null +++ b/TIDAL_README.md @@ -0,0 +1,12 @@ +# Tidal Plugin +## Setup +The Tidal plugin requires [tidalapi](https://github.com/tamland/python-tidal/tree/0.7.x) version 0.7.x, which isn't on PyPI on the time of writing. +To install run: +```bash +git clone -b 0.7.x https://github.com/tamland/python-tidal.git +cd python-tidal/ +python3 setup.py install +``` +## Logging in +I couldn't work out a good way of logging in from ultrasonic's web UI, so for now run `python3 tidal_login.py` and go to the link. +When logged in, the session ids and keys will show in your terminal, copy them onto ultrasonic's tidal persistant settings page. \ No newline at end of file diff --git a/tidal_login.py b/tidal_login.py new file mode 100644 index 0000000..c7291b9 --- /dev/null +++ b/tidal_login.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import tidalapi + +session = tidalapi.Session() +session.login_oauth_simple() + +print('-'*16) +print("Session ID:", session.session_id) +print('-'*16) +print("Token Type:", session.token_type) +print('-'*16) +print("Access Token:", session.access_token) +print('-'*16) +print("Refresh Token:", session.refresh_token) +print('-'*16) \ No newline at end of file From 901d808235d1851eaf08a71820e2ce197c2317fd Mon Sep 17 00:00:00 2001 From: Steve-Tech Date: Wed, 16 Feb 2022 10:41:08 +1000 Subject: [PATCH 03/21] Fix systemd service wanting an an absolute path Fixes `/etc/systemd/system/ultrasonics.service:11: Neither a valid executable name nor an absolute path: .venv/bin/activate` error on Debian 11 (inside LXC) --- ultrasonics.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ultrasonics.service b/ultrasonics.service index 5d60975..4ef1749 100644 --- a/ultrasonics.service +++ b/ultrasonics.service @@ -8,7 +8,7 @@ ConditionPathIsDirectory=/opt/ultrasonics [Service] Restart=always Type=simple -ExecStart=.venv/bin/activate && python3 app.py +ExecStart=/opt/ultrasonics/.venv/bin/python3 app.py WorkingDirectory=/opt/ultrasonics [Install] From 053dc2ff0ffd2703b30ac28675d0a354382f48eb Mon Sep 17 00:00:00 2001 From: Steve-Tech Date: Wed, 16 Feb 2022 11:36:48 +1000 Subject: [PATCH 04/21] Fixed it deleting all songs in a playlist "Update" is still pretty dodgy, try to stick to "Append" if it's important --- ultrasonics/official_plugins/up_tidal.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index abea109..1395f4d 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -278,8 +278,11 @@ def user_playlist_remove_all_occurrences_of_tracks(self, playlist_id, tracks): Removes all `tracks` from the specified playlist. """ playlist = tidalapi.playlist.UserPlaylist(self.session, playlist_id) - for track in tracks: - playlist.remove_by_id(track['id']) + for track in tqdm( + tracks, + desc=f"Deleting songs from Tidal", + ): + playlist.remove_by_id(track) def tidal_to_songs_dict(self, track): """ @@ -489,6 +492,8 @@ def saved_songs_add(self, tidal_ids): current_playlists = s.current_user_playlists() for playlist in songs_dict: + existing_tracks = None + # Check the playlist already exists in Tidal playlist_id = "" try: @@ -525,7 +530,7 @@ def saved_songs_add(self, tidal_ids): existing_tracks = [] # Get all tracks already in the playlist - if "existing_tracks" not in vars(): + if existing_tracks is None: existing_tracks = s.playlist_tracks(playlist_id) # Add songs which don't already exist in the playlist @@ -566,7 +571,7 @@ def saved_songs_add(self, tidal_ids): if settings_dict["existing_playlists"] == "Update": # Remove any songs which aren't in `uris` from the playlist remove_ids = [ - id for id in existing_tracks if id not in ids + duplicate_ids + id['id']['tidal'] for id in existing_tracks if id['id']['tidal'] not in ids + duplicate_ids ] s.user_playlist_remove_all_occurrences_of_tracks( From f8979ebb510ef66e92ad100a00031addb4a53024 Mon Sep 17 00:00:00 2001 From: Stephen H Date: Thu, 17 Feb 2022 16:35:17 +1000 Subject: [PATCH 05/21] Implemented an update checker Adds a version string to the footer on the homepage, checks hourly and turns it red when there is a different release version on GitHub. Somewhat solves issue #14 --- app.py | 15 ++++++++---- ultrasonics/templates/index.html | 4 +++- ultrasonics/updater.py | 40 ++++++++++++++++++++++++++++++++ ultrasonics/webapp.py | 8 ++++++- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 ultrasonics/updater.py diff --git a/app.py b/app.py index 08bb933..46dac3c 100644 --- a/app.py +++ b/app.py @@ -9,14 +9,19 @@ import os -from ultrasonics import database, plugins, scheduler, webapp +from ultrasonics import updater, database, plugins, scheduler, webapp _ultrasonics = { "version": "1.0.0-rc.1", "config_dir": os.path.join(os.path.dirname(__file__), "config") } -database.Core().connect() -plugins.plugin_gather() -scheduler.scheduler_start() -webapp.server_start() +def start(): + updater.start(_ultrasonics["version"]) + database.Core().connect() + plugins.plugin_gather() + scheduler.scheduler_start() + webapp.server_start() + +if __name__ == "__main__": + start() diff --git a/ultrasonics/templates/index.html b/ultrasonics/templates/index.html index 6e0d318..fd5e054 100644 --- a/ultrasonics/templates/index.html +++ b/ultrasonics/templates/index.html @@ -90,7 +90,9 @@

- ultrasonics by XDGFX. + ultrasonics + {{ version }} + by XDGFX. do more with your music.

diff --git a/ultrasonics/updater.py b/ultrasonics/updater.py new file mode 100644 index 0000000..5bb1a9e --- /dev/null +++ b/ultrasonics/updater.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +""" +updater +Checks for updates, and will possibly update everything in the future. + +Steve-Tech, 2022 +""" + +import requests +from requests.exceptions import RequestException +from threading import Timer +from ultrasonics import logs + +log = logs.create_log(__name__) + +version = None +new_version = None + +def get_version(): + request = requests.get("https://api.github.com/repos/XDGFX/ultrasonics/releases") + return request.json()[0] + +def update_version(): + global new_version + log.debug("Checking for updates") + new_version = get_version()['name'] + log.debug("Newest version: %s", new_version) + if new_version != version: + log.info("Update Available: %s", new_version) + +def start(current): + global version, new_version + version = current + new_version = current + try: + update_version() + thread = Timer(3600, update_version) + except RequestException: + log.warning("Could not check for updates.") diff --git a/ultrasonics/webapp.py b/ultrasonics/webapp.py index f0a7968..a2a4180 100644 --- a/ultrasonics/webapp.py +++ b/ultrasonics/webapp.py @@ -13,7 +13,7 @@ from flask import Flask, redirect, render_template, request from flask_socketio import SocketIO, emit, send -from ultrasonics import database, logs, plugins +from ultrasonics import database, logs, plugins, updater from ultrasonics.tools import random_words log = logs.create_log(__name__) @@ -34,6 +34,12 @@ def send(event, data): sio.emit(str(event), {'data': data}) +# --- WEBSERVER PROCESSORS --- +@app.context_processor +def inject_vars(): + return dict(version=updater.version, new_version=updater.new_version) + + # --- WEBSERVER ROUTES --- # Homepage @app.route('/') From 750c2bb0a928eb680913ef238e79422a1a2fb3c9 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Tue, 2 Aug 2022 14:48:40 +1000 Subject: [PATCH 06/21] Add tidalapi 0.7.0 rcs to requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7549916..6b322a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ python-Levenshtein==0.12.2 spotipy==2.19.0 tqdm==4.62.3 PlexAPI==4.7.2 -tidalapi==0.7.0 \ No newline at end of file +tidalapi~=0.7.0rc0 From bb8b2cbe7d49ea5bbab6097f4a0bc01e3ab84522 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Sat, 10 Sep 2022 15:47:07 +1000 Subject: [PATCH 07/21] Update to tidalapi 0.7.0 --- requirements.txt | 2 +- tidal_login.py | 4 ++-- ultrasonics/official_plugins/up_tidal.py | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6b322a1..3b9bb1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ python-Levenshtein==0.12.2 spotipy==2.19.0 tqdm==4.62.3 PlexAPI==4.7.2 -tidalapi~=0.7.0rc0 +tidalapi~=0.7.0 diff --git a/tidal_login.py b/tidal_login.py index c7291b9..90d7470 100644 --- a/tidal_login.py +++ b/tidal_login.py @@ -5,12 +5,12 @@ session = tidalapi.Session() session.login_oauth_simple() -print('-'*16) -print("Session ID:", session.session_id) print('-'*16) print("Token Type:", session.token_type) print('-'*16) print("Access Token:", session.access_token) print('-'*16) print("Refresh Token:", session.refresh_token) +print('-'*16) +print("Expiry Time:", session.expiry_time.isoformat()) print('-'*16) \ No newline at end of file diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 1395f4d..0097c92 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -15,6 +15,7 @@ import tidalapi from tqdm import tqdm +from datetime import datetime from app import _ultrasonics from ultrasonics import logs @@ -33,11 +34,6 @@ "type": "string", "value": "Tidal Login, run tidal_login.py for these values", }, - { - "type": "text", - "label": "Session ID", - "name": "session_id", - }, { "type": "text", "label": "Token Type", @@ -53,6 +49,11 @@ "label": "Refresh Token", "name": "refresh_token", }, + { + "type": "text", + "label": "Expiry Time", + "name": "expiry_time", + }, { "type": "string", "value": "Songs will always attempt to be matched using fixed values like ISRC, however if you're trying to sync music without these tags, fuzzy matching will be used instead.", @@ -95,7 +96,7 @@ def __init__(self): self.session = tidalapi.Session() def login(self): - success = self.session.load_oauth_session(database["session_id"], database["token_type"], database["access_token"], database["refresh_token"]) + success = self.session.load_oauth_session(database["token_type"], database["access_token"], database["refresh_token"], datetime.fromisoformat(database["expiry_time"])) if success: log.debug("Successfully loaded session") else: @@ -404,7 +405,8 @@ def saved_songs_add(self, tidal_ids): # auth = json.loads(database["auth"]) s.refresh_token = database["refresh_token"] - s.login() + if not (s.login() or s.login()): + return None if component == "inputs": if settings_dict["mode"] == "playlists": @@ -600,7 +602,7 @@ def test(database, **kwargs): # global_settings = kwargs["global_settings"] session = tidalapi.Session() - assert session.load_oauth_session(database["session_id"], database["token_type"], database["access_token"], database["refresh_token"]), "Error logging in" + assert session.load_oauth_session(database["token_type"], database["access_token"], database["refresh_token"], datetime.fromisoformat(database["expiry_time"])), "Error logging in" def builder(**kwargs): From 1e3434d271e0cb56946663e14220d5b46f2733d1 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Wed, 14 Sep 2022 21:25:55 +1000 Subject: [PATCH 08/21] Fix UnboundLocalError in error handler --- ultrasonics/official_plugins/up_spotify mixer.py | 6 +++--- ultrasonics/official_plugins/up_spotify.py | 5 +++-- ultrasonics/official_plugins/up_tidal.py | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ultrasonics/official_plugins/up_spotify mixer.py b/ultrasonics/official_plugins/up_spotify mixer.py index cb03907..fe8892b 100644 --- a/ultrasonics/official_plugins/up_spotify mixer.py +++ b/ultrasonics/official_plugins/up_spotify mixer.py @@ -183,6 +183,7 @@ def request(self, sp_func, *args, **kwargs): """ errors = 0 + # Try once again when error while errors <= 1: try: return sp_func(*args, **kwargs) @@ -194,9 +195,8 @@ def request(self, sp_func, *args, **kwargs): errors += 1 continue - log.error( - "An error occurred while trying to contact the Spotify api.") - raise Exception(e) + # raise exception if no return + raise Exception("An error occurred while trying to contact the Spotify api.") def search(self, track): """ diff --git a/ultrasonics/official_plugins/up_spotify.py b/ultrasonics/official_plugins/up_spotify.py index f763dc8..be104db 100644 --- a/ultrasonics/official_plugins/up_spotify.py +++ b/ultrasonics/official_plugins/up_spotify.py @@ -184,6 +184,7 @@ def request(self, sp_func, *args, **kwargs): """ errors = 0 + # Try once again when error while errors <= 1: try: return sp_func(*args, **kwargs) @@ -195,8 +196,8 @@ def request(self, sp_func, *args, **kwargs): errors += 1 continue - log.error("An error occurred while trying to contact the Spotify api.") - raise Exception(e) + # raise exception if no return + raise Exception("An error occurred while trying to contact the Spotify api.") def search(self, track): """ diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 0097c92..02d2ab0 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -109,6 +109,7 @@ def request(self, tidal_func, *args, **kwargs): """ errors = 0 + # Try once again when error while errors <= 1: try: return tidal_func(*args, **kwargs) @@ -119,9 +120,9 @@ def request(self, tidal_func, *args, **kwargs): self.session.token_refresh(database["refresh_token"]) errors += 1 continue - - log.error("An error occurred while trying to contact the Tidal api.") - raise Exception(e) + + # raise exception if no return + raise Exception("An error occurred while trying to contact the Tidal api.") def search(self, track): """ From ba3aba40c70a1b97c335ada2973cee4ffe4fd64b Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Sat, 12 Nov 2022 21:09:13 +1000 Subject: [PATCH 09/21] Simplify `spotify_to_songs_dict` and update requirements.txt --- requirements.txt | 20 +++++++-------- ultrasonics/official_plugins/up_spotify.py | 30 +++++++--------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b9bb1d..16c6ceb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ # Main Program -Flask==2.0.1 -Flask-SocketIO==5.1.1 -eventlet==0.32.0 +Flask~=2.2.2 +Flask-SocketIO~=5.3.1 +eventlet~=0.33.1 # Included Plugins -mutagen==1.45.1 -requests==2.26.0 -fuzzywuzzy==0.18.0 -python-Levenshtein==0.12.2 -spotipy==2.19.0 -tqdm==4.62.3 -PlexAPI==4.7.2 +mutagen~=1.46.0 +requests~=2.28.1 +fuzzywuzzy~=0.18.0 +python-Levenshtein~=0.20.8 +spotipy~=2.21.0 +tqdm~=4.64.1 +PlexAPI~=4.13.1 tidalapi~=0.7.0 diff --git a/ultrasonics/official_plugins/up_spotify.py b/ultrasonics/official_plugins/up_spotify.py index be104db..6edf1e0 100644 --- a/ultrasonics/official_plugins/up_spotify.py +++ b/ultrasonics/official_plugins/up_spotify.py @@ -415,37 +415,25 @@ def user_playlist_remove_all_occurrences_of_tracks(self, playlist_id, tracks): def spotify_to_songs_dict(self, track): """ Convert dictionary received from Spotify API to ultrasonics songs_dict format. - Assumes title, artist(s), and id field are always present. """ - artists = [artist["name"] for artist in track["artists"]] - - try: - album = track["album"]["name"] - except KeyError: - album = None - - try: - date = track["album"]["release_date"] - except KeyError: - date = None - - try: - isrc = track["external_ids"]["isrc"] - except KeyError: - isrc = None + artists = [artist.get("name") for artist in track.get("artists", [])] + album = track.get("album", {}).get("name") + date = track.get("album", {}).get("release_date") + isrc = track.get("external_ids", {}).get("isrc") item = { - "title": track["name"], + "title": track.get("name"), "artists": artists, "album": album, "date": date, "isrc": isrc, } - if track["id"]: - item["id"] = {"spotify": str(track["id"])} + if track.get("id"): + item["id"] = {"spotify": str(track.get("id"))} else: - log.debug(f"Invalid spotify id for song: {track['name']}") + item["id"] = {} + log.debug(f"Invalid spotify id for song: {track.get('name')}") # Remove any empty fields item = {k: v for k, v in item.items() if v} From b697353d70a8a87ae77b89f716a787ac85dea744 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Sun, 5 Feb 2023 21:48:13 +1000 Subject: [PATCH 10/21] Improve Tidal search Also reduce reliance on try excepts --- .gitignore | 3 + ultrasonics/official_plugins/up_log tracks.py | 6 +- ultrasonics/official_plugins/up_tidal.py | 61 ++++++++----------- ultrasonics/tools/fuzzymatch.py | 32 +++------- 4 files changed, 40 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index de14b5d..221b7e1 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,9 @@ dmypy.json # VSCode .vscode +# Pycharm +.idea + # Ultrasonics specifics .vscode/* !.vscode/launch.json diff --git a/ultrasonics/official_plugins/up_log tracks.py b/ultrasonics/official_plugins/up_log tracks.py index 0fc6172..4fa3711 100755 --- a/ultrasonics/official_plugins/up_log tracks.py +++ b/ultrasonics/official_plugins/up_log tracks.py @@ -20,8 +20,10 @@ def run(settings_dict, **kwargs): songs_dict = kwargs["songs_dict"] log.info(f"Below is the songs_dict passed to this plugin:") - log.info("\n\n" + json.dumps(songs_dict, indent=4) + "\n\n") - + try: + log.info("\n\n" + json.dumps(songs_dict, indent=4) + "\n\n") + except TypeError: + log.info("\n\n" + str(songs_dict) + "\n\n") def builder(**kwargs): return [] diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 02d2ab0..ae77233 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -140,24 +140,20 @@ def search(self, track): ] # 1. Tidal ID - try: + if "id" in track and "tidal" in track["id"]: tidal_id = track["id"]["tidal"] # tidal_uri = f"tidal:track:{tidal_id}" confidence = 100 return tidal_id, confidence - except KeyError: - # Tidal ID was not supplied - pass - # 2. Other fields # Multiple searches are made as Tidal is more likely to return false negative (missing songs) # than false positive, when specifying many query parameters. queries = [] - try: + if "title" in track: title = ( re.sub(cutoff_regex[0], "", track["title"], flags=re.IGNORECASE) + "\n" @@ -166,36 +162,27 @@ def search(self, track): title = re.sub( cutoff_regex[1], " ", title, flags=re.IGNORECASE ).strip() - except KeyError: - pass - - try: - album = ( - re.sub(cutoff_regex[0], "", track["album"], flags=re.IGNORECASE) - + "\n" - ) - - album = re.sub( - cutoff_regex[1], " ", album, flags=re.IGNORECASE - ).strip() - except KeyError: - pass - - try: - queries.append(f"{title} {album}") - except NameError: - pass - try: - for artist in track["artists"]: - queries.append(f"{title} {artist}") - except NameError: - pass - - try: - queries.append(title) - except NameError: - pass + # Tidal doesn't search albums, it seems + # if "album" in track: + # album = ( + # re.sub(cutoff_regex[0], "", track["album"], flags=re.IGNORECASE) + # + "\n" + # ) + # + # album = re.sub( + # cutoff_regex[1], " ", album, flags=re.IGNORECASE + # ).strip() + # + # if "title" in locals() and "album" in locals(): + # queries.append(f"{title} {album}") + + if "title" in locals(): + if "artists" in track: + for artist in track["artists"]: + queries.append(f"{title} {artist}") + else: + queries.append(title) results_list = [] @@ -221,7 +208,7 @@ def search(self, track): if score > confidence: matched_track = item confidence = score - if confidence > 100: + if confidence >= 100: break # tidal_uri = f"tidal:track:{matched_track['id']['tidal']}" @@ -568,7 +555,7 @@ def saved_songs_add(self, tidal_ids): ids.append(id) else: log.debug( - f"Could not find song {song['title']} in Tidal; will not add to playlist." + f"Could not find song '{song['title']}' in Tidal; will not add to playlist." ) if settings_dict["existing_playlists"] == "Update": diff --git a/ultrasonics/tools/fuzzymatch.py b/ultrasonics/tools/fuzzymatch.py index 83b491d..438ae31 100644 --- a/ultrasonics/tools/fuzzymatch.py +++ b/ultrasonics/tools/fuzzymatch.py @@ -68,7 +68,7 @@ def duplicate(song, song_list, threshold): # Name and album scores for key in ["title", "album"]: - try: + if key in song and key in item: a = re.sub(cutoff_regex[0], "", song[key], flags=re.IGNORECASE) + "\n" a = re.sub(cutoff_regex[1], " ", a, @@ -81,21 +81,14 @@ def duplicate(song, song_list, threshold): results[key] = fuzz.ratio(a, b) - except KeyError: - pass - # Date score - try: + if "date" in song: results["date"] = fuzz.token_set_ratio(song["date"], item["date"]) - except KeyError: - pass # Artist score can be a partial match; allowing missing artists - try: + if "artists" in item: results["artist"] = fuzz.partial_token_sort_ratio( - artist_string, " ".join(item["artists"]).lower()) - except KeyError: - pass + artist_string, " ".join(sorted(item["artists"])).lower()) weight = { "title": 8, @@ -133,33 +126,26 @@ def similarity(a, b): @return: a number between 0 and 100 representing the similarity rating, where 100 is the same song. """ # Check exact location match - try: - if a["location"] == b["location"]: - return 100 - except KeyError: - pass + if "location" in a and "location" in b and a["location"] == b["location"]: + return 100 # Check exact ID match if "id" in a and "id" in b: for key in a["id"]: - try: + if key in b["id"]: if a["id"][key].strip() == b["id"][key].strip(): return 100 - except KeyError: - # Missing key in one of the songs - pass # Check fuzzy matches results = {} # ISRC score - try: + if "isrc" in a and "isrc" in b: results["isrc"] = int(a["isrc"].strip().lower() == b["isrc"].strip().lower()) * 100 isrc_match = results["isrc"] == 100 - except KeyError: + else: isrc_match = False - pass # Name and album scores for key in ["title", "album"]: From 03b6d427e42748186adc3cd833f7bd77b5acd9e3 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 21 Apr 2023 12:37:33 +1000 Subject: [PATCH 11/21] Add GHCR Workflow --- .github/workflows/build_container.yaml | 47 ++++++++++++++++++++++++++ requirements.txt | 16 ++++----- 2 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/build_container.yaml diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml new file mode 100644 index 0000000..ba30df9 --- /dev/null +++ b/.github/workflows/build_container.yaml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Create and publish a Docker image + +on: + push: + branches: + - 'master' + +jobs: + push_to_registries: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@v3 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 16c6ceb..066c886 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ # Main Program -Flask~=2.2.2 -Flask-SocketIO~=5.3.1 -eventlet~=0.33.1 +Flask~=2.2.3 +Flask-SocketIO~=5.3.3 +eventlet~=0.33.3 # Included Plugins mutagen~=1.46.0 -requests~=2.28.1 +requests~=2.28.2 fuzzywuzzy~=0.18.0 -python-Levenshtein~=0.20.8 -spotipy~=2.21.0 -tqdm~=4.64.1 -PlexAPI~=4.13.1 +python-Levenshtein~=0.21.0 +spotipy~=2.23.0 +tqdm~=4.65.0 +PlexAPI~=4.13.4 tidalapi~=0.7.0 From 60512718b2f2c67feae222c489a6001c0bec5a55 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 21 Apr 2023 12:48:21 +1000 Subject: [PATCH 12/21] Add cmake and ninja to dockerfile Because I forgot to try it locally --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7a1cd18..c20c04e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR / VOLUME [ "/config", "/plugins" ] -RUN apk --no-cache add gcc g++ musl-dev +RUN apk --no-cache add gcc g++ musl-dev cmake ninja RUN pip install -r requirements.txt EXPOSE 5000 From db4310e004ce1e1714e4a09eee3e2a82e676e0d2 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 08:35:33 +1000 Subject: [PATCH 13/21] Update everything --- .github/workflows/build_container.yaml | 4 ++++ Dockerfile | 2 +- requirements.txt | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml index ba30df9..a8acd08 100644 --- a/.github/workflows/build_container.yaml +++ b/.github/workflows/build_container.yaml @@ -37,6 +37,10 @@ jobs: with: images: | ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha - name: Build and push Docker images uses: docker/build-push-action@v3 diff --git a/Dockerfile b/Dockerfile index c20c04e..b821765 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.5-alpine +FROM python:3.11-alpine COPY . / WORKDIR / diff --git a/requirements.txt b/requirements.txt index 066c886..1ad6244 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ # Main Program -Flask~=2.2.3 -Flask-SocketIO~=5.3.3 +Flask~=2.3.2 +Flask-SocketIO~=5.3.4 eventlet~=0.33.3 # Included Plugins mutagen~=1.46.0 -requests~=2.28.2 +requests~=2.31.0 fuzzywuzzy~=0.18.0 python-Levenshtein~=0.21.0 spotipy~=2.23.0 tqdm~=4.65.0 -PlexAPI~=4.13.4 -tidalapi~=0.7.0 +PlexAPI~=4.14.0 +tidalapi~=0.7.1 From 8b7fe63c575758bef4bfadc8ca7a85cf13294b0d Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 17:58:25 +1000 Subject: [PATCH 14/21] Log whole song dict if title missing --- ultrasonics/official_plugins/up_tidal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index ae77233..432da0b 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -555,7 +555,8 @@ def saved_songs_add(self, tidal_ids): ids.append(id) else: log.debug( - f"Could not find song '{song['title']}' in Tidal; will not add to playlist." + f"Could not find song '{song['title'] if 'title' in song else song}' in Tidal; " + "will not add to playlist." ) if settings_dict["existing_playlists"] == "Update": From 9ef5cba15b0119a869d2d1d9f0a0280299c4eb1b Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 19:54:18 +1000 Subject: [PATCH 15/21] Implement searching songs using album, create not found file The not found file uses the `TIDAL_NOT_FOUND_FILE` environment variable, whatever that is set to is the file that will be written to. --- ultrasonics/official_plugins/up_tidal.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 432da0b..2bb8cb5 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -174,25 +174,31 @@ def search(self, track): # cutoff_regex[1], " ", album, flags=re.IGNORECASE # ).strip() # - # if "title" in locals() and "album" in locals(): + # if "title" in track and "album" in track: # queries.append(f"{title} {album}") - if "title" in locals(): + if "title" in track: if "artists" in track: for artist in track["artists"]: queries.append(f"{title} {artist}") - else: queries.append(title) results_list = [] + # Search albums + if "album" in track: + for result in self.request(self.session.search, track["album"], models=[tidalapi.album.Album])['albums']: + item = s.tidal_to_songs_dict(result) + if item not in results_list: + results_list.append(item) + # Execute all queries for query in queries: - results = self.request(self.session.search, query) + results = self.request(self.session.search, query, models=[tidalapi.media.Track]) # Convert to ultrasonics format and append to results_list - for item in results['tracks']: - item = s.tidal_to_songs_dict(item) + for result in results['tracks']: + item = s.tidal_to_songs_dict(result) if item not in results_list: results_list.append(item) @@ -546,6 +552,7 @@ def saved_songs_add(self, tidal_ids): if duplicate: continue + # Search for song on Tidal id, confidence = s.search(song) if id in existing_tracks: @@ -558,6 +565,9 @@ def saved_songs_add(self, tidal_ids): f"Could not find song '{song['title'] if 'title' in song else song}' in Tidal; " "will not add to playlist." ) + if filename := os.environ.get("TIDAL_NOT_FOUND_FILE"): + with open(filename, "a") as f: + f.write(f"{song}\n") if settings_dict["existing_playlists"] == "Update": # Remove any songs which aren't in `uris` from the playlist From 7f9d1edc28cea7a3a650711ac3c76c5c002d6993 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 19:59:15 +1000 Subject: [PATCH 16/21] Add playlist name to not found file --- ultrasonics/official_plugins/up_tidal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 2bb8cb5..87f3be5 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -567,7 +567,7 @@ def saved_songs_add(self, tidal_ids): ) if filename := os.environ.get("TIDAL_NOT_FOUND_FILE"): with open(filename, "a") as f: - f.write(f"{song}\n") + f.write(f"{playlist['name']}: {song}\n") if settings_dict["existing_playlists"] == "Update": # Remove any songs which aren't in `uris` from the playlist From 4be7466cab5cf2f85da6c93ae5eaebc4c4c7b648 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 20:05:41 +1000 Subject: [PATCH 17/21] Update CodeQL Analysis --- .github/workflows/codeql-analysis.yml | 45 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4742dc2..a503807 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,17 +13,18 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ "master" ] schedule: - - cron: '35 5 * * 3' + - cron: '22 1 * * 3' jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: actions: read contents: read @@ -33,38 +34,44 @@ jobs: fail-fast: false matrix: language: [ 'javascript', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - #- run: | - # make bootstrap - # make release + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From fb877a89c74a3a33647ce24dad3b14174e5ad601 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 20:40:31 +1000 Subject: [PATCH 18/21] Fix album search --- ultrasonics/official_plugins/up_tidal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 87f3be5..4575173 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -188,9 +188,10 @@ def search(self, track): # Search albums if "album" in track: for result in self.request(self.session.search, track["album"], models=[tidalapi.album.Album])['albums']: - item = s.tidal_to_songs_dict(result) - if item not in results_list: - results_list.append(item) + for item in result.items(): + item = s.tidal_to_songs_dict(item) + if item not in results_list: + results_list.append(item) # Execute all queries for query in queries: From 04f58da9601bd941b90eeaa6351ac1e458176e4e Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 9 Jun 2023 21:11:02 +1000 Subject: [PATCH 19/21] Ignore videos when searching in albums --- ultrasonics/official_plugins/up_tidal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 4575173..9d8c8f2 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -189,9 +189,10 @@ def search(self, track): if "album" in track: for result in self.request(self.session.search, track["album"], models=[tidalapi.album.Album])['albums']: for item in result.items(): - item = s.tidal_to_songs_dict(item) - if item not in results_list: - results_list.append(item) + if isinstance(item, tidalapi.media.Track): + item = s.tidal_to_songs_dict(item) + if item not in results_list: + results_list.append(item) # Execute all queries for query in queries: From f72762389aae28d051203292c0f2ac3af66be2f4 Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Fri, 14 Jul 2023 13:15:22 +1000 Subject: [PATCH 20/21] Delete `TIDAL_NOT_FOUND_FILE` on new run. --- ultrasonics/official_plugins/up_tidal.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ultrasonics/official_plugins/up_tidal.py b/ultrasonics/official_plugins/up_tidal.py index 9d8c8f2..293ebf0 100644 --- a/ultrasonics/official_plugins/up_tidal.py +++ b/ultrasonics/official_plugins/up_tidal.py @@ -483,6 +483,11 @@ def saved_songs_add(self, tidal_ids): else: "Outputs mode" + # Remove the not found file if it exists + if (not_found_file := os.environ.get("TIDAL_NOT_FOUND_FILE"))\ + and os.path.exists(not_found_file): + os.remove(not_found_file) + # Set the user_id variable # s.user_id() @@ -567,8 +572,10 @@ def saved_songs_add(self, tidal_ids): f"Could not find song '{song['title'] if 'title' in song else song}' in Tidal; " "will not add to playlist." ) - if filename := os.environ.get("TIDAL_NOT_FOUND_FILE"): - with open(filename, "a") as f: + + # Write to not found file + if not_found_file: + with open(not_found_file, "a") as f: f.write(f"{playlist['name']}: {song}\n") if settings_dict["existing_playlists"] == "Update": From 7fa570627a98bda49f30ed3ce8ed551c00c1a19c Mon Sep 17 00:00:00 2001 From: Stephen Horvath Date: Thu, 20 Jul 2023 08:56:40 +1000 Subject: [PATCH 21/21] Update tidalapi --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1ad6244..4622ef7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ python-Levenshtein~=0.21.0 spotipy~=2.23.0 tqdm~=4.65.0 PlexAPI~=4.14.0 -tidalapi~=0.7.1 +tidalapi~=0.7.2