diff --git a/crunch/__version__.py b/crunch/__version__.py index dbc612b..c44f6dc 100644 --- a/crunch/__version__.py +++ b/crunch/__version__.py @@ -1,6 +1,6 @@ __title__ = 'crunch-cli' __description__ = 'crunch-cli - CLI of the CrunchDAO Platform' -__version__ = '2.2.0' +__version__ = '2.3.0' __author__ = 'Enzo CACERES' __author_email__ = 'enzo.caceres@crunchdao.com' __url__ = 'https://github.com/crunchdao/crunch-cli' diff --git a/crunch/command/__init__.py b/crunch/command/__init__.py index 49a076d..d7ee621 100644 --- a/crunch/command/__init__.py +++ b/crunch/command/__init__.py @@ -3,3 +3,4 @@ from .push import push from .setup import setup from .test import test +from .update_token import update_token diff --git a/crunch/command/download.py b/crunch/command/download.py index 34695ba..fe90f22 100644 --- a/crunch/command/download.py +++ b/crunch/command/download.py @@ -105,7 +105,10 @@ def _download(data_file: DataFile, force: bool): if data_file is None: return - print(f"download {data_file.path} from {cut_url(data_file.url)}") + file_length_str = f" ({data_file.size} bytes)" if data_file.has_size else "" + print( + f"download {data_file.path} from {cut_url(data_file.url)}" + file_length_str + ) if not data_file.has_size: print(f"skip: not given by server") @@ -122,21 +125,12 @@ def _download(data_file: DataFile, force: bool): print(f"signature missing: cannot download file without being authenticated") raise click.Abort() - with requests.get(data_file.url, stream=True) as response: - response.raise_for_status() - - file_length = response.headers.get("Content-Length", None) - file_length = int(file_length) if not None else None - - with open(data_file.path, 'wb') as fd, tqdm.tqdm(total=file_length, unit='iB', unit_scale=True, leave=False) as progress: - for chunk in response.iter_content(chunk_size=8192): - progress.update(len(chunk)) - fd.write(chunk) + utils.download(data_file.url, data_file.path, log=False) def download( session: utils.CustomSession, - round_number = "@current", + round_number="@current", force=False, ): project_info = utils.read_project_info() diff --git a/crunch/command/setup.py b/crunch/command/setup.py index ac26bfb..d24133b 100644 --- a/crunch/command/setup.py +++ b/crunch/command/setup.py @@ -53,17 +53,13 @@ def _setup_demo(directory: str, filter: list = None): fd.write(content) -def _setup_submission(directory: str, code_tar: io.BytesIO): - tar = tarfile.open(fileobj=code_tar) - for member in tar.getmembers(): - path = os.path.join(directory, member.name) - print(f"extract {path}") - +def _setup_submission(directory: str, urls: dict): + for relative_path, url in urls.items(): + path = os.path.join(directory, relative_path) + os.makedirs(os.path.dirname(path), exist_ok=True) - - fileobj = tar.extractfile(member) - with open(path, "wb") as fd: - fd.write(fileobj.read()) + + utils.download(url, path) def setup( @@ -99,32 +95,30 @@ def setup( dot_crunchdao_path = _dot_crunchdao(directory) os.makedirs(dot_crunchdao_path, exist_ok=True) + plain = push_token['plain'] + user_id = push_token["project"]["userId"] + project_info = utils.ProjectInfo( competition_name, - push_token["project"]["userId"] + user_id ) utils.write_project_info(project_info, directory) - - token_file_path = os.path.join(dot_crunchdao_path, constants.TOKEN_FILE) - with open(token_file_path, "w") as fd: - fd.write(push_token['plain']) + utils.write_token(plain, directory) try: - code_tar = io.BytesIO( - session.get( - f"/v2/competitions/{competition_name}/projects/{project_info.user_id}/clone", - params={ - "pushToken": push_token['plain'], - "submissionNumber": submission_number, - "includeModel": not no_model, - } - ).content - ) + urls = session.get( + f"/v3/competitions/{competition_name}/projects/{user_id}/clone", + params={ + "pushToken": push_token['plain'], + "submissionNumber": submission_number, + "includeModel": not no_model, + } + ).json() except api.NeverSubmittedException: _setup_demo(directory) else: - _setup_submission(directory, code_tar) + _setup_submission(directory, urls) path = os.path.join(directory, model_directory) os.makedirs(path, exist_ok=True) diff --git a/crunch/command/update_token.py b/crunch/command/update_token.py new file mode 100644 index 0000000..eb6c194 --- /dev/null +++ b/crunch/command/update_token.py @@ -0,0 +1,37 @@ +import click + +from .. import api, utils + + +def update_token( + session: utils.CustomSession, + clone_token: str, +): + project_info = utils.read_project_info() + + try: + push_token = session.post( + f"/v2/project-tokens/upgrade", + json={ + "cloneToken": clone_token + } + ).json() + except api.InvalidProjectTokenException: + print("your token seems to have expired or is invalid") + print("---") + print("please follow this link to copy and paste your new token:") + print(session.format_web_url( + f'/competitions/{project_info.competition_name}/submit' + )) + print("") + + raise click.Abort() + + plain = push_token['plain'] + user_id = push_token['project']["userId"] + + project_info.user_id = user_id + utils.write_project_info(project_info) + utils.write_token(plain) + + print("token updated") diff --git a/crunch/main.py b/crunch/main.py index 89fda1b..fbe4bcb 100644 --- a/crunch/main.py +++ b/crunch/main.py @@ -3,13 +3,17 @@ import click -from . import command, constants, utils, api, library, tester +from . import command, constants, utils, api, library, tester, __version__ session = None debug = False @click.group() +@click.version_option( + __version__.__version__, + package_name="__version__.__title__" +) @click.option("--debug", "enable_debug", envvar=constants.DEBUG_ENV_VAR, is_flag=True, help="Enable debug output.") @click.option("--api-base-url", envvar=constants.API_BASE_URL_ENV_VAR, default=constants.API_BASE_URL_DEFAULT, help="Set the API base url.") @click.option("--web-base-url", envvar=constants.WEB_BASE_URL_ENV_VAR, default=constants.WEB_BASE_URL_DEFAULT, help="Set the Web base url.") @@ -196,5 +200,18 @@ def convert( ) +@cli.command(help="Update a project token.") +@click.argument("clone-token", required=True) +def update_token( + clone_token: str, +): + utils.change_root() + + command.update_token( + session, + clone_token=clone_token + ) + + if __name__ == '__main__': cli() diff --git a/crunch/utils.py b/crunch/utils.py index f291968..c62f31a 100644 --- a/crunch/utils.py +++ b/crunch/utils.py @@ -15,6 +15,7 @@ import pandas import psutil import requests +import tqdm from . import api, constants @@ -99,6 +100,17 @@ def _read_crunchdao_file(name: str, raise_if_missing=True): return fd.read() +def write_token(plain_push_token: str, directory="."): + dot_crunchdao_path = os.path.join( + directory, + constants.DOT_CRUNCHDAO_DIRECTORY + ) + + token_file_path = os.path.join(dot_crunchdao_path, constants.TOKEN_FILE) + with open(token_file_path, "w") as fd: + fd.write(plain_push_token) + + @dataclasses.dataclass() class ProjectInfo: competition_name: str @@ -107,7 +119,9 @@ class ProjectInfo: def write_project_info(info: ProjectInfo, directory=".") -> ProjectInfo: dot_crunchdao_path = os.path.join( - directory, constants.DOT_CRUNCHDAO_DIRECTORY) + directory, + constants.DOT_CRUNCHDAO_DIRECTORY + ) old_path = os.path.join(dot_crunchdao_path, constants.OLD_PROJECT_FILE) if os.path.exists(old_path): @@ -236,3 +250,42 @@ def debug(message: str): arguments[name] = value return function(**arguments) + + +def cut_url(url: str): + try: + return url[:url.index("?")] + except ValueError: + return url + + +def download(url: str, path: str, log=True): + logged = False + + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + + file_length = response.headers.get("Content-Length", None) + file_length = int(file_length) if not None else None + + if log: + if file_length is not None: + file_length_str = f"{file_length} bytes" + else: + file_length_str = "unknown length" + + print( + f"download {path} from {cut_url(url)} ({file_length_str})" + ) + logged = True + + with open(path, 'wb') as fd, tqdm.tqdm(total=file_length, unit='iB', unit_scale=True, leave=False) as progress: + for chunk in response.iter_content(chunk_size=8192): + progress.update(len(chunk)) + fd.write(chunk) + except: + if log and not logged: + print(f"downloading {path} from {cut_url(url)}") + + raise