diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80f622f --- /dev/null +++ b/.gitignore @@ -0,0 +1,205 @@ + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* +### Windows template +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk +### VirtualEnv template +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +pip-selfcheck.json diff --git a/extensions.py b/extensions.py new file mode 100644 index 0000000..2384bbe --- /dev/null +++ b/extensions.py @@ -0,0 +1,131 @@ +# coding: UTF-8 + +extension_id = { + "C++14": 88, + "C": 0, + "C++": 1, + "C++11": 49, + "C++17": 84, + "Java": 3, + "C11": 75, + "Python 2": 6, + "Python 3": 28, + "PyPy": 32, + "PyPy3": 73, + "Ruby 2.5": 68, + "Kotlin": 69, + "Swift": 74, + "C# 6.0": 62, + "Text": 58, + "node.js": 17, + "Go": 12, + "F#": 37, + "PHP": 7, + "Pascal": 2, + "Lua": 16, + "Perl": 8, + "R": 72, + "C (Clang)": 59, + "C++ (Clang)": 60, + "C++11 (Clang)": 66, + "C++14 (Clang)": 67, + "C11 (Clang)": 77, + "C++17 (Clang)": 85, + "Golfscript": 79, + "Fortran": 13, + "Scheme": 14, + "Ada": 19, + "awk": 21, + "OCaml": 22, + "Brainfuck": 23, + "Whitespace": 24, + "Tcl": 26, + "Assembly (32bit)": 27, + "D": 29, + "Clojure": 33, + "Rhino": 34, + "Cobol": 35, + "SpiderMonkey": 38, + "Pike": 41, + "sed": 43, + "Rust": 44, + "Boo": 46, + "Intercal": 47, + "bc": 48, + "Nemerle": 53, + "Cobra": 54, + "Nimrod": 55, + "VB.NET 4.0": 63, + "Algol 68": 70, + "Befunge": 71, + "FreeBASIC": 78, + "Gosu": 80, + "Haxe": 81, + "LOLCODE": 82, + "아희": 83 +} + +extension_name = { + "C++14": 'cpp14.cpp', + "C": 'c', + "C++": 'cpp', + "C++11": 'cpp11.cpp', + "C++17": 'cpp17.cpp', + "Java": 'java', + "C11": 'c11.c', + "Python 2": 'py2.py', + "Python 3": 'py3.py', + "PyPy": 'pypy.py', + "PyPy3": 'pypy3.py', + "Ruby 2.5": 'rb', + "Kotlin": 'kt', + "Swift": 'swift', + "C# 6.0": 'cs', + "Text": 'txt', + "node.js": 'js', + "Go": 'go', + "F#": 'fs', + "PHP": 'php', + "Pascal": 'pas', + "Lua": 'lua', + "Perl": 'pl', + "R": 'R', + "C (Clang)": 'clang.c', + "C++ (Clang)": 'clang.cpp', + "C++11 (Clang)": 'clang_pp11.cpp', + "C++14 (Clang)": 'clang_pp14.cpp', + "C11 (Clang)": 'clang11.c', + "C++17 (Clang)": 'clang_pp17.cpp', + "Golfscript": 'gs', + "Fortran": 'f95', + "Scheme": 'scm', + "Ada": 'ada', + "awk": 'awk', + "OCaml": 'ml', + "Brainfuck": 'bf', + "Whitespace": 'ws', + "Tcl": 'tcl', + "Assembly (32bit)": 'asm', + "D": 'd', + "Clojure": 'clj', + "Rhino": 'rhino.js', + "Cobol": 'cob', + "SpiderMonkey": 'spider.js', + "Pike": 'pike', + "sed": 'sed', + "Rust": 'rs', + "Boo": 'boo', + "Intercal": 'i', + "bc": 'bc', + "Nemerle": 'n', + "Cobra": 'cobra', + "Nimrod": 'nim', + "VB.NET 4.0": 'vb', + "Algol 68": 'a68', + "Befunge": 'befunge.bf', + "FreeBASIC": 'bas', + "Gosu": 'gsp', + "Haxe": 'haxe.py', + "LOLCODE": 'lol', + "아희": 'aheui', +} diff --git a/main.py b/main.py index 40921d5..e1401ce 100755 --- a/main.py +++ b/main.py @@ -1,13 +1,13 @@ #!/usr/bin/python3 -# -*- coding: utf-8 -*- +# coding: utf-8 -import multiprocessing import os import threading from getpass import getpass -from NetworkTool import down_file, login, get_soup -from SubmitRecord import SubmitRecord +import extensions +from network_tools import down_file, get_soup, login +from submit_record import SubmitRecord __author__ = 'isac322' @@ -25,7 +25,7 @@ def get_solved_problems(sp): def make_code_file(problem_num, language): - extension = get_extension(language) + extension = extensions.extension_name[language] file_name = problem_num + '.' + extension directory = os.path.join(working_dir, problem_num) @@ -74,32 +74,21 @@ def analyze_problem(problem_num): problem_name = rows[0].find_all('td')[2].a['title'] screenLock.acquire() - print("{:5s} {}".format(problem_num, problem_name)) + print('{:5s} {}'.format(problem_num, problem_name)) for l, e in table.items(): - print("\t{:10s} {}".format(l, e)) + print('\t{:10s} {}'.format(l, e)) screenLock.release() return table -networkSemaphore = threading.BoundedSemaphore(multiprocessing.cpu_count() * 3) - - def analyze_and_make(problem_num): submitted_codes = analyze_problem(problem_num) - directory = os.path.join(working_dir, problem_num) - - if not os.path.exists(directory): - os.makedirs(directory) - for language, source_code in submitted_codes.items(): - file = make_code_file(problem_num, language) - downloaded = down_file(source_code.judge_id, full_cookie) - file.write(str(downloaded.decode('utf-8'))) - file.close() - - networkSemaphore.release() + with make_code_file(problem_num, language) as file: + downloaded = down_file(source_code.judge_id, full_cookie) + file.write(downloaded.decode()) ignore_list = frozenset(('10947', '9999', '13757')) @@ -107,101 +96,13 @@ def analyze_and_make(problem_num): def get_submitted_files(problems): for problem_num in problems: - networkSemaphore.acquire() - thread_file_maker = threading.Thread(target=analyze_and_make, args=(problem_num,), daemon=False) - thread_file_maker.start() + directory = os.path.join(working_dir, problem_num) + if not os.path.exists(directory): + os.makedirs(directory) -def get_extension(language): - # todo add extensions - if language in ['C++', 'C++ (Clang)']: - return 'cpp' - elif language in ['C++11']: - return 'cpp11.cpp' - elif language in ['C++14']: - return 'cpp14.cpp' - elif language in ['C', 'C (Clang)']: - return 'c' - elif language in ['Python']: - return 'py' - elif language in ['Python3']: - return 'py3.py' - elif language in ['PyPy']: - return 'pypy.py' - elif language in ['PyPy3']: - return 'pypy3.py' - elif language in ['Java']: - return 'java' - elif language in ['Text']: - return 'txt' - elif language in ['PHP']: - return 'php' - elif language in ['Ruby 1.8', 'Ruby 1.9']: - return 'rb' - elif language in ['C# 2.0', 'C# 4.0']: - return 'cs' - elif language in ['Pascal']: - return 'pas' - elif language in ['D']: - return 'd' - elif language in ['Go']: - return 'go' - elif language in ['awk']: - return 'awk' - elif language in ['VB.NET 2.0']: - return 'vb' - elif language in ['Ada']: - return 'ada' - elif language in ['Perl', 'Perl6', 'Prolog']: - return 'pl' - elif language in ['node.js', 'SpiderMonkey']: - return 'js' - elif language in ['Lua']: - return 'lua' - elif language in ['Objective-C']: - return 'm' - elif language in ['Objective-C++']: - return 'mm' - elif language in ['Fortran']: - return 'f95' - elif language in ['Scheme']: - return 'scm' - elif language in ['OCaml']: - return 'ml' - elif language in ['Brainfuck']: - return 'bf' - elif language in ['Whitespace']: - return 'ws' - elif language in ['Groovy']: - return 'groovy' - elif language in ['Tcl']: - return 'tcl' - elif language in ['Assembly']: - return 'asm' - elif language in ['Clojure']: - return 'clj' - elif language in ['Rhino']: - return 'Rhino.js' - elif language in ['Pike']: - return 'pike' - elif language in ['sed']: - return 'sed' - elif language in ['Boo']: - return 'boo' - elif language in ['Intercal']: - return 'i' - elif language in ['bc']: - return 'bc' - elif language in ['Nemerle']: - return 'n' - elif language in ['Cobra']: - return 'cobra' - elif language in ['Nimrod']: - return 'nim' - elif language in ['Io']: - return 'io' - elif language in ['아희']: - return 'aheui' + thread_file_maker = threading.Thread(target=analyze_and_make, args=(problem_num,), daemon=False) + thread_file_maker.start() if __name__ == '__main__': diff --git a/NetworkTool.py b/network_tools.py similarity index 85% rename from NetworkTool.py rename to network_tools.py index 6e323d0..f7bd519 100644 --- a/NetworkTool.py +++ b/network_tools.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- +# coding: utf-8 import http.client import re -import sys import urllib.error from urllib.parse import urlencode from urllib.request import Request, urlopen +import sys from bs4 import BeautifulSoup as Bs __author__ = 'isac322' @@ -19,15 +19,17 @@ def down_file(judge_id, cookie): 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'https://www.acmicpc.net,.', 'Upgrade-Insecure-Requests': '1', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/45.0.2454.101 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'https://www.acmicpc.net/source/{}'.format(judge_id), 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4', - 'Cookie': cookie + ' _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1; _ga=GA1.2.1918118912.1444272664' + 'Cookie': cookie + ' _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1; ' + '_ga=GA1.2.1918118912.1444272664' } - conn = http.client.HTTPSConnection("www.acmicpc.net") + conn = http.client.HTTPSConnection('www.acmicpc.net') conn.request('POST', '/source/download/{}'.format(judge_id), None, header) response = conn.getresponse() @@ -48,14 +50,15 @@ def login(user_name, pw): 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'https://www.acmicpc.net', 'Upgrade-Insecure-Requests': '1', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/45.0.2454.101 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'https://www.acmicpc.net/login/?next=/', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4', } - conn = http.client.HTTPSConnection("www.acmicpc.net") + conn = http.client.HTTPSConnection('www.acmicpc.net') conn.request('POST', '/signin', data, header) response = conn.getresponse() @@ -84,7 +87,8 @@ def get_soup(url, query=None): def get_response(url, query=None, header=None): if header is None: - user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36' + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' \ + 'Chrome/45.0.2454.101 Safari/537.36' header = { 'Connection': 'keep-alive', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cd91942 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +beautifulsoup4==4.6.0 diff --git a/SubmitRecord.py b/submit_record.py similarity index 98% rename from SubmitRecord.py rename to submit_record.py index 444d9a5..ac45c34 100644 --- a/SubmitRecord.py +++ b/submit_record.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# coding: utf-8 import sys