From 4220a16c6d67bf6bdd8eb6de68bbc74736337898 Mon Sep 17 00:00:00 2001 From: Sathyajith Bhat Date: Sun, 10 May 2020 10:08:28 +0300 Subject: [PATCH] Add cache for saving URLs previously saved (#80) * add test, handle Youtube quota expiry better (#68) * add sigint handler to handle Ctrl+C better * major version bump due to Python version change (now needs 3.6+) * move sentry around so that github action doesn;t complain when trying to fetch version for publishing * move sentry around * oops, moved signal to wrong place * Add HTML Scraping functionality * Update requirements.txt to fit HTML Scrape * change the log structure * some additional error handling when scraping fails * tests for getting link w/ scraping * update requirements for packaging * add deepsource * don't show notice about dev key * fix typo in youtube dev key env variable * remove redundant assignment to dev key * try to go via api only if dev key has been set * add local cache spotifydl now saves links to previously scraped URLs to reduce calls to YouTube * scraper is now the default option * fix bug where non-scrape path was applied when it should have been scraped * update reqs to add peewee * save cache to homedir, remove negative tests * merge conflicts * remove unused imports * remove scrape argument * remove unused args * clean up imports * pep8 line breaks * docstrings * remove validate.py, add later * make deepsource happy * skipchecks Co-authored-by: Luca Koroll --- .deepsource.toml | 1 + .gitignore | 6 +- README.md | 190 ++++++++++++++-------------- requirements.txt | 5 +- spotify_dl/cache.py | 27 ++++ spotify_dl/models.py | 13 ++ spotify_dl/scaffold.py | 9 +- spotify_dl/spotify_dl.py | 258 +++++++++++++++++++------------------- spotify_dl/youtube.py | 150 +++++++++++----------- tests/test_cache_get.py | 20 +++ tests/test_youtube_url.py | 12 +- 11 files changed, 378 insertions(+), 313 deletions(-) create mode 100644 spotify_dl/cache.py create mode 100644 spotify_dl/models.py create mode 100644 tests/test_cache_get.py diff --git a/.deepsource.toml b/.deepsource.toml index d2a7f6a8..a20085c3 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -7,4 +7,5 @@ name = "python" enabled = true [analyzers.meta] + max_line_length = 200 runtime_version = "3.x.x" diff --git a/.gitignore b/.gitignore index 1ba771fa..651eb184 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -__pycache__/ +**/__pycache__/** .idea/ spotify-dl/.cache- spotify-dl/.cache-sathyabhat @@ -10,3 +10,7 @@ dist/ downloaded_songs.txt songs.txt unins +*.pyc +*.mp3 +.coverage +.vscode/** \ No newline at end of file diff --git a/README.md b/README.md index 28578a2b..f35975e2 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,95 @@ -# spotify_dl -Downloads songs from any Spotify playlist or from your "My Music" collection. - -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) -[![PyPI download month](https://img.shields.io/pypi/dm/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) -[![PyPI license](https://img.shields.io/pypi/l/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) -[![PyPI pyversions](https://img.shields.io/pypi/pyversions/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) -[![GitHub release](https://img.shields.io/github/release/SathyaBhat/spotify-dl.svg)](https://GitHub.com/SathyaBhat/spotify-dl/releases/) -[![GitHub stars](https://img.shields.io/github/stars/SathyaBhat/spotify-dl.svg?style=social&label=Star&maxAge=2592000)](https://GitHub.com/SathyaBhat/spotify-dl/stargazers/) -[![GitHub contributors](https://img.shields.io/github/contributors/SathyaBhat/spotify-dl.svg)](https://GitHub.com/SathyaBhat/spotify-dl/graphs/contributors/) - -[![Awesome Badges](https://img.shields.io/badge/badges-awesome-green.svg)](https://github.com/Naereen/badges) - - -# Tell me more! -I wanted an easy way to grab the songs present in my library so I can download it & use it offline(Spotify still hasn't launched here. Y U NO COME?). [spotify_to_mp3](https://github.com/frosas/spotify-to-mp3) worked well but it relied on grooveshark, which unfortunately is no more. - -So I wrote this script which mimics that library, but instead of downloading from grooveshark, it provides you with a file of youtube URLs which you can then plug into [youtube-dl](https://rg3.github.io/youtube-dl/) - -### How do I get this thing running? - -Pre-requisite: You need Python 3.6+ - -1. Install using pip - `sudo pip3 install spotify_dl` - (use `pip` if your distro natively provides Python 3) - -2. Create your Spotify app & fetch the client id and client secret from [Spotify Developer Console](https://developer.spotify.com/my-applications/#!/applications). These keys then need to be assigned as `SPOTIPY_CLIENT_ID`, `SPOTIPY_CLIENT_SECRET` and `SPOTIPY_REDIRECT_URI` environment variables. - - You can set environment variables in Linux like so: - - export SPOTIPY_CLIENT_ID='your-spotify-client-id' - export SPOTIPY_CLIENT_SECRET='your-spotify-client-secret' - export SPOTIPY_REDIRECT_URI='your-app-redirect-url' - - Windows users, check for [this question](http://superuser.com/a/284351/4377) for details on how you can set environment variables. - - Note the redirect URL can be a valid URL, just ensure it matches with what you have entered in the developer console & in the environment variable above. - -3. Create your YouTube API key & fetch the keys from [Google Developer Console](https://console.developers.google.com/apis/api/youtube/overview). Set the key as `YOUTUBE_DEV_KEY` environment variable as mentioned above. -4. Run the script using `spotify_dl`. spotify_dl accepts different parameters, for more details run `spotify_dl -h`. - - For most users `spotify_dl -l spotify_playlist_link -o download_directory` should do where - - - `spotify_playlist_link` is a link to Spotify's playlist. You can get it from the 3-dot menu. - - ![image](images/spotify-playlist.png) - - If the Spotify playlist link is skipped then it will download songs from your "My Music" collection - - `download_directory` is the location where the songs must be downloaded to. If you give a `.` then it will download to the current directory. - - Alternatively, `spotify_dl -p playlist_id -u user_name -o download_directory` will also work - - - `playlist_id` is the id of the playlist where songs need to be downloaded. If this is skipped then it will download songs ftom your "My Music" collection - - `user_name` is the user name who created the playlist. - - `download_directory` is the location where the songs must be downloaded to. -5. A first time run will require authentication; you will need to click on the URL prompted to authenticate. Once logged in, paste the URL back in. -6. To retrieve download songs as MP3, you will need to install ffmpeg. If you prefer to skip MP3 conversion, pass `-m` or `--skip_mp3` as a parameter when running the script - - Linux users can get them by installing libav-tools by using apt-get (`sudo apt-get install -y libav-tools`) or a package manager which comes with your distro - - Windows users can download FFMPEG pre-built binaries from [here](http://ffmpeg.zeranoe.com/builds/). Extract the file using [7-zip](http://7-zip.org/) to a foldrer and [add the folder to your PATH environment variable](http://www.wikihow.com/Install-FFmpeg-on-Windows) - -### How do I set defaults? - -You can set defaults per user by creating a file at `~/.spotify_dl_settings`. Create a key with value for every argument you want a default for. Example: -``` json -{ - "output" : "/home/foo/spotify-dl-output" - , "verbose" : "true" - , "skip_mp3" : "t" -} -``` - -### Running tests - -At the moment, there are barely any tests but PRs always welcome to improve this. Tests are setup and run with pytest, run - - make tests - -to run the tests with [Make](https://www.gnu.org/software/make/) - -### Credits - - - [rhnvrm](https://github.com/rhnvrm) for [adding in youtube-dl](https://github.com/SathyaBhat/spotify-dl/pull/1) - - [mr-karan](https://github.com/mr-karan) for [adding save to directory](https://github.com/SathyaBhat/spotify-dl/pull/6) - - [shantanugoel](https://github.com/shantanugoel) for adding in [User playlist](https://github.com/SathyaBhat/spotify-dl/pull/7), [skip MP3 conversion](https://github.com/SathyaBhat/spotify-dl/pull/34) and [Ability to use custom format string support](https://github.com/SathyaBhat/spotify-dl/pull/34) - - [sildur](https://github.com/sildur) for adding any [user playlist support and other fixes](https://github.com/SathyaBhat/spotify-dl/pulls?q=is%3Apr+author%3Asildur+is%3Aclosed) - - [avinassh](https://github.com/avinassh) for being a [Rockstar](https://github.com/avinassh/rockstar) and not teleporting over to my house to kill me when I innundated him with questions - - [doulwyi](https://github.com/doulwyi) for adding id3 tagging and ability to parse Spotify URI - - [Gowtham](https://github.com/HackToHell) for [create playlist in download](https://github.com/SathyaBhat/spotify-dl/pull/23) directory - - [alvierahman90](https://github.com/alvierahman90) for [config file support](https://github.com/SathyaBhat/spotify-dl/pull/42) and [Spotify playlist URL support](https://github.com/SathyaBhat/spotify-dl/pull/41) - - [Bibhas](https://github.com/iambibhas) for fixing [can only concatenate list (not "str") to list error](https://github.com/SathyaBhat/spotify-dl/issues/44) - - [Nikhil Nagaraju](https://github.com/nikhilnagaraju) for fixing support for playlist url with or without userid #58 - -## Issues, Feedback, Contact details -Feel free to raise any bugs/issues under Github issues. Pull requests are also more than welcome. You can reach me on twitter at [@sathyabhat](https://twitter.com/sathyabhat) or drop an email [sathya@sathyasays.com](mailto:sathya@sathyasays.com) +# spotify_dl +Downloads songs from any Spotify playlist or from your "My Music" collection. + +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) +[![PyPI download month](https://img.shields.io/pypi/dm/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) +[![PyPI license](https://img.shields.io/pypi/l/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) +[![PyPI pyversions](https://img.shields.io/pypi/pyversions/spotify_dl.svg)](https://pypi.python.org/pypi/spotify_dl/) +[![GitHub release](https://img.shields.io/github/release/SathyaBhat/spotify-dl.svg)](https://GitHub.com/SathyaBhat/spotify-dl/releases/) +[![GitHub stars](https://img.shields.io/github/stars/SathyaBhat/spotify-dl.svg?style=social&label=Star&maxAge=2592000)](https://GitHub.com/SathyaBhat/spotify-dl/stargazers/) +[![GitHub contributors](https://img.shields.io/github/contributors/SathyaBhat/spotify-dl.svg)](https://GitHub.com/SathyaBhat/spotify-dl/graphs/contributors/) + +[![Awesome Badges](https://img.shields.io/badge/badges-awesome-green.svg)](https://github.com/Naereen/badges) + + +# Tell me more! +I wanted an easy way to grab the songs present in my library so I can download it & use it offline(Spotify still hasn't launched here. Y U NO COME?). [spotify_to_mp3](https://github.com/frosas/spotify-to-mp3) worked well but it relied on grooveshark, which unfortunately is no more. + +So I wrote this script which mimics that library, but instead of downloading from grooveshark, it provides you with a file of youtube URLs which you can then plug into [youtube-dl](https://rg3.github.io/youtube-dl/) + +### How do I get this thing running? + +Pre-requisite: You need Python 3.6+ + +1. Install using pip + `sudo pip3 install spotify_dl` + (use `pip` if your distro natively provides Python 3) + +2. Create your Spotify app & fetch the client id and client secret from [Spotify Developer Console](https://developer.spotify.com/my-applications/#!/applications). These keys then need to be assigned as `SPOTIPY_CLIENT_ID`, `SPOTIPY_CLIENT_SECRET` and `SPOTIPY_REDIRECT_URI` environment variables. + + You can set environment variables in Linux like so: + + export SPOTIPY_CLIENT_ID='your-spotify-client-id' + export SPOTIPY_CLIENT_SECRET='your-spotify-client-secret' + export SPOTIPY_REDIRECT_URI='your-app-redirect-url' + + Windows users, check for [this question](http://superuser.com/a/284351/4377) for details on how you can set environment variables. + + Note the redirect URL can be a valid URL, just ensure it matches with what you have entered in the developer console & in the environment variable above. + +3. Create your YouTube API key & fetch the keys from [Google Developer Console](https://console.developers.google.com/apis/api/youtube/overview). Set the key as `YOUTUBE_DEV_KEY` environment variable as mentioned above. +4. Run the script using `spotify_dl`. spotify_dl accepts different parameters, for more details run `spotify_dl -h`. + + For most users `spotify_dl -l spotify_playlist_link -o download_directory` should do where + + - `spotify_playlist_link` is a link to Spotify's playlist. You can get it from the 3-dot menu. + + ![image](images/spotify-playlist.png) + + If the Spotify playlist link is skipped then it will download songs from your "My Music" collection + - `download_directory` is the location where the songs must be downloaded to. If you give a `.` then it will download to the current directory. + + Alternatively, `spotify_dl -p playlist_id -u user_name -o download_directory` will also work + + - `playlist_id` is the id of the playlist where songs need to be downloaded. If this is skipped then it will download songs ftom your "My Music" collection + - `user_name` is the user name who created the playlist. + - `download_directory` is the location where the songs must be downloaded to. +5. A first time run will require authentication; you will need to click on the URL prompted to authenticate. Once logged in, paste the URL back in. +6. To retrieve download songs as MP3, you will need to install ffmpeg. If you prefer to skip MP3 conversion, pass `-m` or `--skip_mp3` as a parameter when running the script + - Linux users can get them by installing libav-tools by using apt-get (`sudo apt-get install -y libav-tools`) or a package manager which comes with your distro + - Windows users can download FFMPEG pre-built binaries from [here](http://ffmpeg.zeranoe.com/builds/). Extract the file using [7-zip](http://7-zip.org/) to a foldrer and [add the folder to your PATH environment variable](http://www.wikihow.com/Install-FFmpeg-on-Windows) + +### How do I set defaults? + +You can set defaults per user by creating a file at `~/.spotify_dl_settings`. Create a key with value for every argument you want a default for. Example: +``` json +{ + "output" : "/home/foo/spotify-dl-output" + , "verbose" : "true" + , "skip_mp3" : "t" +} +``` + +### Running tests + +At the moment, there are barely any tests but PRs always welcome to improve this. Tests are setup and run with pytest, run + + make tests + +to run the tests with [Make](https://www.gnu.org/software/make/) + +### Credits + + - [rhnvrm](https://github.com/rhnvrm) for [adding in youtube-dl](https://github.com/SathyaBhat/spotify-dl/pull/1) + - [mr-karan](https://github.com/mr-karan) for [adding save to directory](https://github.com/SathyaBhat/spotify-dl/pull/6) + - [shantanugoel](https://github.com/shantanugoel) for adding in [User playlist](https://github.com/SathyaBhat/spotify-dl/pull/7), [skip MP3 conversion](https://github.com/SathyaBhat/spotify-dl/pull/34) and [Ability to use custom format string support](https://github.com/SathyaBhat/spotify-dl/pull/34) + - [sildur](https://github.com/sildur) for adding any [user playlist support and other fixes](https://github.com/SathyaBhat/spotify-dl/pulls?q=is%3Apr+author%3Asildur+is%3Aclosed) + - [avinassh](https://github.com/avinassh) for being a [Rockstar](https://github.com/avinassh/rockstar) and not teleporting over to my house to kill me when I innundated him with questions + - [doulwyi](https://github.com/doulwyi) for adding id3 tagging and ability to parse Spotify URI + - [Gowtham](https://github.com/HackToHell) for [create playlist in download](https://github.com/SathyaBhat/spotify-dl/pull/23) directory + - [alvierahman90](https://github.com/alvierahman90) for [config file support](https://github.com/SathyaBhat/spotify-dl/pull/42) and [Spotify playlist URL support](https://github.com/SathyaBhat/spotify-dl/pull/41) + - [Bibhas](https://github.com/iambibhas) for fixing [can only concatenate list (not "str") to list error](https://github.com/SathyaBhat/spotify-dl/issues/44) + - [Nikhil Nagaraju](https://github.com/nikhilnagaraju) for fixing support for playlist url with or without userid #58 + +## Issues, Feedback, Contact details +Feel free to raise any bugs/issues under Github issues. Pull requests are also more than welcome. You can reach me on twitter at [@sathyabhat](https://twitter.com/sathyabhat) or drop an email [sathya@sathyasays.com](mailto:sathya@sathyasays.com) diff --git a/requirements.txt b/requirements.txt index f75c77cd..b9611895 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ spotipy==2.3.8 google-api-python-client==1.6.2 youtube-dl>=2015.12.23 -sentry-sdk==0.14.1 +sentry-sdk==0.14.3 colorama==0.4.3 click==7.0 lxml>=4.2 -wheel==0.34.2 \ No newline at end of file +wheel==0.34.2 +peewee==3.13.3 diff --git a/spotify_dl/cache.py b/spotify_dl/cache.py new file mode 100644 index 00000000..65d897e3 --- /dev/null +++ b/spotify_dl/cache.py @@ -0,0 +1,27 @@ +from spotify_dl.models import Song +from peewee import DoesNotExist + + +def check_if_in_cache(search_term): + """ + Checks if the specified search term is in the local database cache + and returns the video id if it exists. + :param search_term: String to be searched for in the cache + :return A tuple with Boolean and video id if it exists + """ + try: + song = Song.get(search_term=search_term) + return True, song.video_id + except DoesNotExist: + return False, None + + +def save_to_cache(search_term, video_id): + """ + Saves the search term and video id to the database cache so it can be looked up later + :param search_term: Search term to be saved to in the cache + :param video_id: Video id to be saved to in the cache + :return Video id saved in the cache + """ + song_info, _ = Song.get_or_create(search_term=search_term, video_id=video_id) + return song_info.video_id diff --git a/spotify_dl/models.py b/spotify_dl/models.py new file mode 100644 index 00000000..9c4222e9 --- /dev/null +++ b/spotify_dl/models.py @@ -0,0 +1,13 @@ +from peewee import SqliteDatabase +from peewee import Model, TextField +from os import path + +db = SqliteDatabase(path.expanduser('~/.songs.db')) + + +class Song(Model): + search_term = TextField() + video_id = TextField() + + class Meta: + database = db diff --git a/spotify_dl/scaffold.py b/spotify_dl/scaffold.py index 041db009..217fa099 100644 --- a/spotify_dl/scaffold.py +++ b/spotify_dl/scaffold.py @@ -11,7 +11,7 @@ sentry_sdk.init("https://7d74a39472c9449dac51eb24bb33bdc3@sentry.io/2383261") -def check_for_tokens(args=None): +def check_for_tokens(): log.debug('Checking for tokens') CLIENT_ID = getenv('SPOTIPY_CLIENT_ID') CLIENT_SECRET = getenv('SPOTIPY_CLIENT_SECRET') @@ -37,7 +37,7 @@ def check_for_tokens(args=None): log.debug("YouTube dev key: {}".format(YOUTUBE_DEV_KEY)) if YOUTUBE_DEV_KEY is None: print(''' - You need to setup Youtube Data API token. You can do this by + Youtube Data API token has not been setup. You can do this by setting environment variables like so: export YOUTUBE_DEV_KEY='your-youtube-dev-key' @@ -45,9 +45,6 @@ def check_for_tokens(args=None): Generate the key from https://console.developers.google.com/apis/api/youtube/overview - Or use the HTML Scraper by specifying -s True + Using HTML Scraper as a fallback. ''') - if args.scrape: - return True - return False return True diff --git a/spotify_dl/spotify_dl.py b/spotify_dl/spotify_dl.py index a06c6be3..bf0c921f 100644 --- a/spotify_dl/spotify_dl.py +++ b/spotify_dl/spotify_dl.py @@ -1,128 +1,130 @@ -#!/usr/bin/env python -import os -import re -from logging import DEBUG -import argparse -import json -import spotipy - -from spotify_dl.scaffold import * -from spotify_dl.spotify import authenticate -from spotify_dl.spotify import fetch_tracks -from spotify_dl.spotify import save_songs_to_file -from spotify_dl.spotify import download_songs -from spotify_dl.spotify import playlist_name -from spotify_dl.youtube import fetch_youtube_url -from spotify_dl.spotify import extract_user_and_playlist_from_uri -from spotify_dl.spotify import get_playlist_name_from_id -from spotify_dl.constants import VERSION -from spotify_dl.youtube import get_youtube_dev_key - -def spotify_dl(): - parser = argparse.ArgumentParser(prog='spotify_dl') - parser.add_argument('-d', '--download', action='store_true', - help='Download using youtube-dl', default=True) - parser.add_argument('-p', '--playlist', action='store', - help='Download from playlist id instead of' - ' saved tracks') - parser.add_argument('-V', '--verbose', action='store_true', - help='Show more information on what''s happening.') - parser.add_argument('-v', '--version', action='store_true', - help='Shows current version of the program') - parser.add_argument('-o', '--output', type=str, action='store', - help='Specify download directory.') - parser.add_argument('-u', '--user_id', action='store', - help='Specify the playlist owner\'s userid when it' - ' is different than your spotify userid') - parser.add_argument('-i', '--uri', type=str, action='store', - nargs='*', help='Given a URI, download it.') - parser.add_argument('-f', '--format_str', type=str, action='store', - help='Specify youtube-dl format string.', - default='bestaudio/best') - parser.add_argument('-m', '--skip_mp3', action='store_true', - help='Don\'t convert downloaded songs to mp3') - parser.add_argument('-l', '--url', action="store", - help="Spotify Playlist link URL") - parser.add_argument('-s', '--scrape', action="store", - help="Use HTML Scraper for YouTube Search") - - args = parser.parse_args() - - playlist_url_pattern = re.compile(r'^https://open.spotify.com/(.+)$') - - if args.version: - print("spotify_dl v{}".format(VERSION)) - exit(0) - - if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')): - with open(os.path.expanduser('~/.spotify_dl_settings')) as file: - config = json.loads(file.read()) - - for key, value in config.items(): - if value and (value.lower() == 'true' or value.lower() == 't'): - setattr(args, key, True) - else: - setattr(args, key, value) - - if args.verbose: - log.setLevel(DEBUG) - - log.info('Starting spotify_dl') - log.debug('Setting debug mode on spotify_dl') - - if not check_for_tokens(args): - exit(1) - - token = authenticate() - sp = spotipy.Spotify(auth=token) - log.debug('Arguments: {}'.format(args)) - - if args.url: - url_match = playlist_url_pattern.match(args.url) - if url_match and len(url_match.groups()) > 0: - uri = "spotify:" + url_match.groups()[0].replace('/', ':') - args.uri = [uri] - else: - raise Exception('Invalid playlist URL ') - if args.uri: - current_user_id, playlist_id = extract_user_and_playlist_from_uri(args.uri[0], sp) - else: - if args.user_id is None: - current_user_id = sp.current_user()['id'] - else: - current_user_id = args.user_id - - if args.output: - if args.uri: - uri = args.uri[0] - playlist = playlist_name(uri, sp) - else: - playlist = get_playlist_name_from_id(args.playlist, current_user_id, sp) - - log.info("Saving songs to: {}".format(playlist)) - download_directory = args.output + '/' + playlist - if len(download_directory) >= 0 and download_directory[-1] != '/': - download_directory += '/' - - if not os.path.exists(download_directory): - os.makedirs(download_directory) - else: - download_directory = '' - - if args.uri: - songs = fetch_tracks(sp, playlist_id, current_user_id) - else: - songs = fetch_tracks(sp, args.playlist, current_user_id) - url = [] - for song, artist in songs.items(): - link = fetch_youtube_url(song + ' - ' + artist, get_youtube_dev_key(), scrape=args.scrape) - if link: - url.append((link, song, artist)) - - save_songs_to_file(url, download_directory) - if args.download is True: - download_songs(url, download_directory, args.format_str, args.skip_mp3) - - -if __name__ == '__main__': - spotify_dl() +#!/usr/bin/env python +import os +import re +from logging import DEBUG +import argparse +import json +import spotipy + +from spotify_dl.scaffold import log, check_for_tokens +from spotify_dl.spotify import authenticate, fetch_tracks, save_songs_to_file +from spotify_dl.spotify import download_songs, playlist_name +from spotify_dl.youtube import fetch_youtube_url +from spotify_dl.spotify import extract_user_and_playlist_from_uri +from spotify_dl.spotify import get_playlist_name_from_id +from spotify_dl.constants import VERSION +from spotify_dl.youtube import get_youtube_dev_key +from spotify_dl.models import db, Song + + +def spotify_dl(): + """Main entry point of the script.""" + parser = argparse.ArgumentParser(prog='spotify_dl') + parser.add_argument('-d', '--download', action='store_true', + help='Download using youtube-dl', default=True) + parser.add_argument('-p', '--playlist', action='store', + help='Download from playlist id instead of' + ' saved tracks') + parser.add_argument('-V', '--verbose', action='store_true', + help='Show more information on what''s happening.') + parser.add_argument('-v', '--version', action='store_true', + help='Shows current version of the program') + parser.add_argument('-o', '--output', type=str, action='store', + help='Specify download directory.') + parser.add_argument('-u', '--user_id', action='store', + help='Specify the playlist owner\'s userid when it' + ' is different than your spotify userid') + parser.add_argument('-i', '--uri', type=str, action='store', + nargs='*', help='Given a URI, download it.') + parser.add_argument('-f', '--format_str', type=str, action='store', + help='Specify youtube-dl format string.', + default='bestaudio/best') + parser.add_argument('-m', '--skip_mp3', action='store_true', + help='Don\'t convert downloaded songs to mp3') + parser.add_argument('-l', '--url', action="store", + help="Spotify Playlist link URL") + parser.add_argument('-s', '--scrape', action="store", + help="Use HTML Scraper for YouTube Search", default=True) + + args = parser.parse_args() + + playlist_url_pattern = re.compile(r'^https://open.spotify.com/(.+)$') + + if args.version: + print("spotify_dl v{}".format(VERSION)) + exit(0) + + db.connect() + db.create_tables([Song]) + if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')): + with open(os.path.expanduser('~/.spotify_dl_settings')) as file: + config = json.loads(file.read()) + + for key, value in config.items(): + if value and (value.lower() == 'true' or value.lower() == 't'): + setattr(args, key, True) + else: + setattr(args, key, value) + + if args.verbose: + log.setLevel(DEBUG) + + log.info('Starting spotify_dl') + log.debug('Setting debug mode on spotify_dl') + + if not check_for_tokens(): + exit(1) + + token = authenticate() + sp = spotipy.Spotify(auth=token) + log.debug('Arguments: {}'.format(args)) + + if args.url: + url_match = playlist_url_pattern.match(args.url) + if url_match and len(url_match.groups()) > 0: + uri = "spotify:" + url_match.groups()[0].replace('/', ':') + args.uri = [uri] + else: + raise Exception('Invalid playlist URL ') + if args.uri: + current_user_id, playlist_id = extract_user_and_playlist_from_uri(args.uri[0], sp) + else: + if args.user_id is None: + current_user_id = sp.current_user()['id'] + else: + current_user_id = args.user_id + + if args.output: + if args.uri: + uri = args.uri[0] + playlist = playlist_name(uri, sp) + else: + playlist = get_playlist_name_from_id(args.playlist, current_user_id, sp) + + log.info("Saving songs to: {}".format(playlist)) + download_directory = args.output + '/' + playlist + if len(download_directory) >= 0 and download_directory[-1] != '/': + download_directory += '/' + + if not os.path.exists(download_directory): + os.makedirs(download_directory) + else: + download_directory = '' + + if args.uri: + songs = fetch_tracks(sp, playlist_id, current_user_id) + else: + songs = fetch_tracks(sp, args.playlist, current_user_id) + url = [] + for song, artist in songs.items(): + link = fetch_youtube_url(song + ' - ' + artist, get_youtube_dev_key()) + if link: + url.append((link, song, artist)) + + save_songs_to_file(url, download_directory) + if args.download is True: + download_songs(url, download_directory, args.format_str, args.skip_mp3) + + +if __name__ == '__main__': + spotify_dl() diff --git a/spotify_dl/youtube.py b/spotify_dl/youtube.py index 104f7ba8..5506b958 100644 --- a/spotify_dl/youtube.py +++ b/spotify_dl/youtube.py @@ -1,71 +1,81 @@ -from os import getenv - -from googleapiclient.discovery import build -from googleapiclient.http import HttpError -from sentry_sdk import capture_exception - - -from spotify_dl.constants import YOUTUBE_API_SERVICE_NAME -from spotify_dl.constants import YOUTUBE_API_VERSION -from spotify_dl.constants import VIDEO -from spotify_dl.constants import YOUTUBE_VIDEO_URL -from spotify_dl.scaffold import log -from json import loads -import requests -from lxml import html -import re - - -from click import secho - -def fetch_youtube_url(search_term, dev_key=None, scrape=False): - """For each song name/artist name combo, fetch the YouTube URL - and return the list of URLs""" - log.info(f"Searching for {search_term}") - if scrape or not dev_key: - YOUTUBE_SEARCH_BASE = "https://www.youtube.com/results?search_query=" - try: - response = requests.get(YOUTUBE_SEARCH_BASE + search_term).content - html_response = html.fromstring(response) - video = html_response.xpath("//a[contains(@class, 'yt-uix-tile-link')]/@href") - video_id = re.search("((\?v=)[a-zA-Z0-9_-]{4,15})", video[0]).group(0)[3:] - log.debug(f"Found video id {video_id} for search term {search_term}") - return YOUTUBE_VIDEO_URL + video_id - except AttributeError as e: - log.warn(f"Could not find scrape details for {search_term}") - capture_exception(e) - return None - except IndexError as e: - log.warn(f"Could not perform scrape search for {search_term}, diff HTML") - capture_exception(e) - return None - else: - YOUTUBE_DEV_KEY = dev_key - youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, - developerKey=YOUTUBE_DEV_KEY, - cache_discovery=False) - try: - search_response = youtube.search().list(q=search_term, - part='id, snippet').execute() - for v in search_response['items']: - if v['id']['kind'] == VIDEO: - log.debug("Adding Video id {}".format(v['id']['videoId'])) - return YOUTUBE_VIDEO_URL + v['id']['videoId'] - except HttpError as err: - err_details = loads(err.content.decode('utf-8')).get('error').get('errors') - secho("Couldn't complete search due to following errors: ", fg='red') - for e in err_details: - error_reason = e.get('reason') - error_domain = e.get('domain') - error_message = e.get('message') - - if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded': - secho(f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.", fg='red') - secho(f"\tThe quota will be reset at midnight Pacific Time (PT)." ,fg='red') - secho(f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.", fg='red') - else: - secho(f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}") - return None - -def get_youtube_dev_key(): +from os import getenv + +from googleapiclient.discovery import build +from googleapiclient.http import HttpError +from sentry_sdk import capture_exception + + +from spotify_dl.constants import YOUTUBE_API_SERVICE_NAME +from spotify_dl.constants import YOUTUBE_API_VERSION +from spotify_dl.constants import VIDEO +from spotify_dl.constants import YOUTUBE_VIDEO_URL +from spotify_dl.scaffold import log +from spotify_dl.cache import check_if_in_cache, save_to_cache +from json import loads +import requests +from lxml import html # skipcq: BAN-B410 +import re + + +from click import secho + +# skipcq: PYL-R1710 +def fetch_youtube_url(search_term, dev_key=None): + """ + For each song name/artist name combo, fetch the YouTube URL + and return the list of URLs + :param search_term: Search term to be looked up on YouTube + :param dev_key: Youtube API key + """ + log.info(f"Searching for {search_term}") + in_cache, video_id = check_if_in_cache(search_term) + if in_cache: + log.info(f"Found id {video_id} for {search_term} in cache") + return YOUTUBE_VIDEO_URL + video_id + if not dev_key: + YOUTUBE_SEARCH_BASE = "https://www.youtube.com/results?search_query=" + try: + response = requests.get(YOUTUBE_SEARCH_BASE + search_term).content + html_response = html.fromstring(response) + video = html_response.xpath("//a[contains(@class, 'yt-uix-tile-link')]/@href") + video_id = re.search("((\?v=)[a-zA-Z0-9_-]{4,15})", video[0]).group(0)[3:] + log.debug(f"Found video id {video_id} for search term {search_term}") + _ = save_to_cache(search_term=search_term, video_id=video_id) + return YOUTUBE_VIDEO_URL + video_id + except AttributeError as e: + log.warning(f"Could not find scrape details for {search_term}") + capture_exception(e) + return None + except IndexError as e: + log.warning(f"Could not perform scrape search for {search_term}, got a different HTML") + capture_exception(e) + return None + else: + youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, + developerKey=dev_key, + cache_discovery=False) + try: + search_response = youtube.search().list(q=search_term, + part='id, snippet').execute() + for v in search_response['items']: + if v['id']['kind'] == VIDEO: + log.debug("Adding Video id {}".format(v['id']['videoId'])) + return YOUTUBE_VIDEO_URL + v['id']['videoId'] + except HttpError as err: + err_details = loads(err.content.decode('utf-8')).get('error').get('errors') + secho("Couldn't complete search due to following errors: ", fg='red') + for e in err_details: + error_reason = e.get('reason') + error_domain = e.get('domain') + error_message = e.get('message') + + if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded': + secho(f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.", fg='red') + secho(f"\tThe quota will be reset at midnight Pacific Time (PT)." ,fg='red') + secho(f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.", fg='red') + else: + secho(f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}") + return None + +def get_youtube_dev_key(): return getenv('YOUTUBE_DEV_KEY') \ No newline at end of file diff --git a/tests/test_cache_get.py b/tests/test_cache_get.py new file mode 100644 index 00000000..e447615b --- /dev/null +++ b/tests/test_cache_get.py @@ -0,0 +1,20 @@ +from spotify_dl.cache import check_if_in_cache, save_to_cache +from spotify_dl.models import db, Song + +search_term_wrong = 'bleh' +search_term = "Red Hot Chili Peppers - Dani California [Official Music Video]" + +video_id = "Sb5aq5HcS1A" +db.connect() +db.create_tables([Song]) + +def test_check_for_cache_miss(): + exists, song_info = check_if_in_cache(search_term=search_term_wrong) + assert exists is False + assert song_info is None + +def test_check_for_cache_hit(): + _ = save_to_cache(search_term=search_term, video_id='Sb5aq5HcS1A') + exists, cache_video_id = check_if_in_cache(search_term=search_term) + assert exists is True + assert cache_video_id == video_id \ No newline at end of file diff --git a/tests/test_youtube_url.py b/tests/test_youtube_url.py index 9fae9e35..79ddd9ef 100644 --- a/tests/test_youtube_url.py +++ b/tests/test_youtube_url.py @@ -2,20 +2,10 @@ def test_fetch_youtube_url(capsys): - song_link = fetch_youtube_url("Red Hot Chili Peppers - Dani California [Official Music Video]", - "12354") - assert song_link is None - captured = capsys.readouterr() - assert "keyInvalid" in captured.out song_link = fetch_youtube_url("Red Hot Chili Peppers - Dani California [Official Music Video]", get_youtube_dev_key()) assert song_link == 'https://www.youtube.com/watch?v=Sb5aq5HcS1A' def test_fetch_youtube_url_wth_scrape(capsys): - song_link = fetch_youtube_url("Red Hot Chili Peppers - Dani California [Official Music Video]", - "12354") - assert song_link is None - captured = capsys.readouterr() - assert "keyInvalid" in captured.out - song_link = fetch_youtube_url("Red Hot Chili Peppers - Dani California [Official Music Video]", dev_key=None, scrape=True) + song_link = fetch_youtube_url("Red Hot Chili Peppers - Dani California [Official Music Video]", dev_key=None) assert song_link == 'https://www.youtube.com/watch?v=Sb5aq5HcS1A'