diff --git a/.gitattributes b/.gitattributes index 0ae5850..512c335 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,3 +9,13 @@ # the files will be saved with the line endings CRLF, which are suitable # for Windows batchfiles. *.bat -text + +# Other files +*.py text eol=lf + +##### + +## Linguist config (https://github.com/github/linguist/#vendored-code) +repository/_extractors/*.py linguist-vendored +## https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-Unspecified +repository/_extractors/__init__.py !linguist-vendored diff --git a/build.bat b/build.bat index c99813c..729a788 100644 --- a/build.bat +++ b/build.bat @@ -1,5 +1,5 @@ @rem - Encoding:utf-8; Mode:Batch; Language:en; LineEndings:CRLF - -:: Video Downloaders (You-Get, Youtube-dl, Annie) One-Click Deployment Batch (Windows) +:: You-Get Unofficial Build Executable for Windows :: Author: Lussac (https://blog.lussac.net) :: Last updated: 2020-02-19 :: >>> Get updated from: https://github.com/LussacZheng/you-get.exe <<< @@ -8,7 +8,22 @@ if exist build\you-get\ rd /S /Q build\you-get if exist dist\you-get.exe del /S /Q dist\you-get.exe >NUL 2>NUL -pushd repository\you-get + +if NOT exist repository\you-get\you-get ( + echo. + echo * Please run "devscripts\init.bat" first or clone the repository of "you-get". + pause > NUL + exit +) + +cd repository +:: First, move out the original `__init__.py` from "you_get.extractors", +:: in order that we can recover everything after build. +:: Then copy all the extractors from `repository\_extractors\`, with a new `__init__.py` +:: which has imported these extractors, into the module "you_get.extractors" +move you-get\src\you_get\extractors\__init__.py .\ > NUL +xcopy _extractors\*.py you-get\src\you_get\extractors\ >NUL +pushd you-get echo. echo ============================================================ @@ -20,6 +35,7 @@ pyinstaller -F --path=src ^ --distpath ..\..\dist ^ --workpath ..\..\build ^ --specpath ..\..\build ^ + --icon ..\dist\you-get.ico ^ --hidden-import=you_get.extractors ^ --hidden-import=you_get.cli_wrapper ^ --hidden-import=you_get.processor ^ @@ -31,7 +47,13 @@ echo ============================================================ echo. popd -cd dist +:: Recover everything in you-get.git after build +for /f "delims=" %%i in ('dir /b /a:a _extractors') do ( + del /Q you-get\src\you_get\extractors\%%i >NUL 2>NUL +) +move __init__.py you-get\src\you_get\extractors\ > NUL + +cd ..\dist if exist file_version_info.txt ( pyi-set_version file_version_info.txt you-get.exe echo. diff --git a/dist/you-get.ico b/dist/you-get.ico new file mode 100644 index 0000000..0895881 Binary files /dev/null and b/dist/you-get.ico differ diff --git a/dist/you-get_extracted_from_0.4.985.ico b/dist/you-get_extracted_from_0.4.985.ico new file mode 100644 index 0000000..d9ed09c Binary files /dev/null and b/dist/you-get_extracted_from_0.4.985.ico differ diff --git a/doc/PyInstaller-Options.md b/doc/PyInstaller-Options.md index 39b7ec7..1462853 100644 --- a/doc/PyInstaller-Options.md +++ b/doc/PyInstaller-Options.md @@ -2,7 +2,7 @@ > [PyInstaller](https://github.com/pyinstaller/pyinstaller) bundles a Python application and all its dependencies into a single package. The user can run the packaged app without installing a Python interpreter or any modules. -下文我将简单叙述一下如何使用 PyInstaller 将 you-get 打包为可独立运行的 exe 程序 (Windows)。 +本文将叙述如何使用 PyInstaller 将 you-get 打包为可独立运行的 exe 程序 (Windows)。 --- @@ -150,13 +150,7 @@ ModuleNotFoundError: No module named 'you_get.extractors' > pyinstaller -F --path=src you-get --hidden-import=you_get.extractors --hidden-import=you_get.cli_wrapper ``` -## Final command - -综上所述，**最后打包的命令为**： - -```shell -> pyinstaller -F --path=src you-get --hidden-import=you_get.extractors --hidden-import=you_get.cli_wrapper --hidden-import=you_get.processor --hidden-import=you_get.util -``` +--- ## Issues @@ -232,4 +226,41 @@ from .ixigua import * 这些 extractors 均以 `_` 开头，而（参照 `Makefile`）直接运行 `python setup.py sdist`生成的 `dist/you-get-0.4.1403.tar.gz` 则不包含这些 extractors 。 暂不知其原因。 -根据上文分析，可以想到解决办法为在 `src/you_get/extractors/__init__.py` 中逐一导入这些模块，再重新打包。 +根据上文分析，可以想到解决办法为：将 `"you-get-0.4.1403.tar.gz" you-get-0.4.1403/src/you_get/extractors/` 中以 `_` 开头的 `py` 文件复制到 clone 目录对应位置，并编辑 `src/you_get/extractors/__init__.py` ，在其中逐一导入这些模块，最后再重新打包。 + +```python +# 为 src/you_get/extractors/__init__.py 文件追加以下语句 +from .baomihua import * +from .giphy import * +from .huomaotv import * +from .iwara import * +from .ixigua import * +from .missevan import * +from .qie_video import * +from .qq_egame import * +from .toutiao import * +from .vidto import * +from .ximalaya import * +from .yizhibo import * + +from ._blip import * +from ._catfun import * +from ._coursera import * +from ._dongting import * +from ._jpopsuki import * +from ._qianmo import * +from ._songtaste import * +from ._thvideo import * +from ._vid48 import * +from ._videobam import * +``` + +--- + +## Final command + +综上所述，**最后打包的命令为**： + +```shell +> pyinstaller -F --path=src you-get --hidden-import=you_get.extractors --hidden-import=you_get.cli_wrapper --hidden-import=you_get.processor --hidden-import=you_get.util +``` \ No newline at end of file diff --git a/repository/.gitkeep b/repository/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/repository/_extractors/__init__.py b/repository/_extractors/__init__.py new file mode 100644 index 0000000..a477fba --- /dev/null +++ b/repository/_extractors/__init__.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +from .acfun import * +from .alive import * +from .archive import * +from .baidu import * +from .bandcamp import * +from .bigthink import * +from .bilibili import * +from .bokecc import * +from .cbs import * +from .ckplayer import * +from .cntv import * +from .coub import * +from .dailymotion import * +from .douban import * +from .douyin import * +from .douyutv import * +from .ehow import * +from .facebook import * +from .fc2video import * +from .flickr import * +from .freesound import * +from .funshion import * +from .google import * +from .heavymusic import * +from .icourses import * +from .ifeng import * +from .imgur import * +from .infoq import * +from .instagram import * +from .interest import * +from .iqilu import * +from .iqiyi import * +from .joy import * +from .khan import * +from .ku6 import * +from .kakao import * +from .kuaishou import * +from .kugou import * +from .kuwo import * +from .le import * +from .lizhi import * +from .longzhu import * +from .magisto import * +from .metacafe import * +from .mgtv import * +from .miaopai import * +from .miomio import * +from .mixcloud import * +from .mtv81 import * +from .musicplayon import * +from .nanagogo import * +from .naver import * +from .netease import * +from .nicovideo import * +from .pinterest import * +from .pixnet import * +from .pptv import * +from .qie import * +from .qingting import * +from .qq import * +from .showroom import * +from .sina import * +from .sohu import * +from .soundcloud import * +from .suntv import * +from .ted import * +from .theplatform import * +from .tiktok import * +from .tucao import * +from .tudou import * +from .tumblr import * +from .twitter import * +from .ucas import * +from .veoh import * +from .videomega import * +from .vimeo import * +from .vine import * +from .vk import * +from .w56 import * +from .wanmen import * +from .xiami import * +from .yinyuetai import * +from .yixia import * +from .youku import * +from .youtube import * +from .zhanqi import * +from .zhibo import * +from .zhihu import * + +# Import additional extractors for PyInstaller +# https://github.com/LussacZheng/you-get.exe/blob/master/doc/PyInstaller-Options.md +from .baomihua import * +from .giphy import * +from .huomaotv import * +from .iwara import * +from .ixigua import * +from .missevan import * +from .qie_video import * +from .qq_egame import * +from .toutiao import * +from .vidto import * +from .ximalaya import * +from .yizhibo import * + +# Copy these files from "you-get-0.4.1403.tar.gz" +from ._blip import * +from ._catfun import * +from ._coursera import * +from ._dongting import * +from ._jpopsuki import * +from ._qianmo import * +from ._songtaste import * +from ._thvideo import * +from ._vid48 import * +from ._videobam import * diff --git a/repository/_extractors/_blip.py b/repository/_extractors/_blip.py new file mode 100644 index 0000000..8308bc4 --- /dev/null +++ b/repository/_extractors/_blip.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +__all__ = ['blip_download'] + +from ..common import * + +import json + +def blip_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): + p_url = url + "?skin=json&version=2&no_wrap=1" + html = get_html(p_url) + metadata = json.loads(html) + + title = metadata['Post']['title'] + real_url = metadata['Post']['media']['url'] + type, ext, size = url_info(real_url) + + print_info(site_info, title, type, size) + if not info_only: + download_urls([real_url], title, ext, size, output_dir, merge = merge) + +site_info = "Blip.tv" +download = blip_download +download_playlist = playlist_not_supported('blip') diff --git a/repository/_extractors/_catfun.py b/repository/_extractors/_catfun.py new file mode 100644 index 0000000..85789e7 --- /dev/null +++ b/repository/_extractors/_catfun.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +__all__ = ['catfun_download'] +from .tudou import tudou_download_by_id +from .sina import sina_download_by_vid + +from ..common import * +from xml.dom.minidom import * + +def parse_item(item): + if item["type"] == "youku": + page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_youku_video_info&youku_id=" + item["vid"]) + dom = parseString(page) + ext = dom.getElementsByTagName("format")[0].firstChild.nodeValue; + size = 0 + urls = [] + for i in dom.getElementsByTagName("durl"): + urls.append(i.getElementsByTagName("url")[0].firstChild.nodeValue) + size += int(i.getElementsByTagName("size")[0].firstChild.nodeValue); + return urls, ext, size + + elif item["type"] == "qq": + page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_qq_video_info&qq_id=" + item["vid"]) + dom = parseString(page) + size = 0 + urls = [] + for i in dom.getElementsByTagName("durl"): + url = i.getElementsByTagName("url")[0].firstChild.nodeValue + urls.append(url) + vtype, ext, _size = url_info(url) + size += _size + return urls, ext, size + + elif item["type"] == "sina": + page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_sina_video_info&sina_id=" + item["vid"]) + try: + dom = parseString(page) + except: + #refresh page encountered + page = get_content(match1(page, r'url=(.+?)"')) + dom = parseString(page) + size = 0 + urls = [] + for i in dom.getElementsByTagName("durl"): + url = i.getElementsByTagName("url")[0].firstChild.nodeValue + urls.append(url) + vtype, ext, _size = url_info(url) + if not ext: + ext = match1(url,r'\.(\w+?)\?') + size += _size + #sina's result does not contains content-type + return urls, ext, size + +def catfun_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): + # html = get_content(url) + title = match1(get_content(url), r'

(.+?)

') + vid = match1(url, r"v\d+/cat(\d+)") + j = json.loads(get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_video&modelid=11&id={}".format(vid))) + for item in j: + if item["name"] != "\u672a\u547d\u540d1": + t = title + "-" + item["name"] + else: + t = title + if item["type"] == "tudou": + tudou_download_by_id(item["vid"], title, output_dir, merge, info_only) + + else: + urls, ext, size = parse_item(item) + + print_info(site_info, title, ext, size) + if not info_only: + download_urls(urls, t, ext, size, output_dir, merge=merge) + +site_info = "CatFun.tv" +download = catfun_download +download_playlist = playlist_not_supported('catfun') diff --git a/repository/_extractors/_coursera.py b/repository/_extractors/_coursera.py new file mode 100644 index 0000000..3454974 --- /dev/null +++ b/repository/_extractors/_coursera.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +__all__ = ['coursera_download'] + +from ..common import * + +def coursera_login(user, password, csrf_token): + url = 'https://www.coursera.org/maestro/api/user/login' + my_headers = { + 'Cookie': ('csrftoken=%s' % csrf_token), + 'Referer': 'https://www.coursera.org', + 'X-CSRFToken': csrf_token, + } + + values = { + 'email_address': user, + 'password': password, + } + form_data = parse.urlencode(values).encode('utf-8') + + response = request.urlopen(request.Request(url, headers = my_headers, data = form_data)) + + return response.headers + +def coursera_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): + course_code = r1(r'coursera.org/([^/]+)', url) + url = "http://class.coursera.org/%s/lecture/index" % course_code + + request.install_opener(request.build_opener(request.HTTPCookieProcessor())) + + import http.client + conn = http.client.HTTPConnection('class.coursera.org') + conn.request('GET', "/%s/lecture/index" % course_code) + response = conn.getresponse() + + csrf_token = r1(r'csrf_token=([^;]+);', response.headers['Set-Cookie']) + + import netrc, getpass + info = netrc.netrc().authenticators('coursera.org') + if info is None: + user = input("User: ") + password = getpass.getpass("Password: ") + else: + user, password = info[0], info[2] + print("Logging in...") + + coursera_login(user, password, csrf_token) + + request.urlopen("https://class.coursera.org/%s/auth/auth_redirector?type=login&subtype=normal" % course_code) # necessary! + + html = get_html(url) + + course_name = "%s (%s)" % (r1(r'course_strings_name = "([^"]+)"', html), course_code) + output_dir = os.path.join(output_dir, course_name) + + materials = re.findall(r'