Merge pull request #30 from crunchdao/dev

2.3.0

crunch/__version__.py

@@ -1,6 +1,6 @@
 __title__ = 'crunch-cli'
 __description__ = 'crunch-cli - CLI of the CrunchDAO Platform'
-__version__ = '2.2.0'
+__version__ = '2.3.0'
 __author__ = 'Enzo CACERES'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/crunchdao/crunch-cli'

crunch/command/__init__.py

@@ -3,3 +3,4 @@
 from .push import push
 from .setup import setup
 from .test import test
+from .update_token import update_token

crunch/command/download.py

@@ -105,7 +105,10 @@ def _download(data_file: DataFile, force: bool):
     if data_file is None:
         return
 
-    print(f"download {data_file.path} from {cut_url(data_file.url)}")
+    file_length_str = f" ({data_file.size} bytes)" if data_file.has_size else ""
+    print(
+        f"download {data_file.path} from {cut_url(data_file.url)}" + file_length_str
+    )
 
     if not data_file.has_size:
         print(f"skip: not given by server")
@@ -122,21 +125,12 @@ def _download(data_file: DataFile, force: bool):
         print(f"signature missing: cannot download file without being authenticated")
         raise click.Abort()
 
-    with requests.get(data_file.url, stream=True) as response:
-        response.raise_for_status()
-
-        file_length = response.headers.get("Content-Length", None)
-        file_length = int(file_length) if not None else None
-
-        with open(data_file.path, 'wb') as fd, tqdm.tqdm(total=file_length, unit='iB', unit_scale=True, leave=False) as progress:
-            for chunk in response.iter_content(chunk_size=8192):
-                progress.update(len(chunk))
-                fd.write(chunk)
+    utils.download(data_file.url, data_file.path, log=False)
 
 
 def download(
     session: utils.CustomSession,
-    round_number = "@current",
+    round_number="@current",
     force=False,
 ):
     project_info = utils.read_project_info()

crunch/command/setup.py

@@ -53,17 +53,13 @@ def _setup_demo(directory: str, filter: list = None):
             fd.write(content)
 
 
-def _setup_submission(directory: str, code_tar: io.BytesIO):
-    tar = tarfile.open(fileobj=code_tar)
-    for member in tar.getmembers():
-        path = os.path.join(directory, member.name)
-        print(f"extract {path}")
-
+def _setup_submission(directory: str, urls: dict):
+    for relative_path, url in urls.items():
+        path = os.path.join(directory, relative_path)
+
         os.makedirs(os.path.dirname(path), exist_ok=True)
-
-        fileobj = tar.extractfile(member)
-        with open(path, "wb") as fd:
-            fd.write(fileobj.read())
+
+        utils.download(url, path)
 
 
 def setup(
@@ -99,32 +95,30 @@ def setup(
     dot_crunchdao_path = _dot_crunchdao(directory)
     os.makedirs(dot_crunchdao_path, exist_ok=True)
 
+    plain = push_token['plain']
+    user_id = push_token["project"]["userId"]
+
     project_info = utils.ProjectInfo(
         competition_name,
-        push_token["project"]["userId"]
+        user_id
     )
 
     utils.write_project_info(project_info, directory)
-
-    token_file_path = os.path.join(dot_crunchdao_path, constants.TOKEN_FILE)
-    with open(token_file_path, "w") as fd:
-        fd.write(push_token['plain'])
+    utils.write_token(plain, directory)
 
     try:
-        code_tar = io.BytesIO(
-            session.get(
-                f"/v2/competitions/{competition_name}/projects/{project_info.user_id}/clone",
-                params={
-                    "pushToken": push_token['plain'],
-                    "submissionNumber": submission_number,
-                    "includeModel": not no_model,
-                }
-            ).content
-        )
+        urls = session.get(
+            f"/v3/competitions/{competition_name}/projects/{user_id}/clone",
+            params={
+                "pushToken": push_token['plain'],
+                "submissionNumber": submission_number,
+                "includeModel": not no_model,
+            }
+        ).json()
     except api.NeverSubmittedException:
         _setup_demo(directory)
     else:
-        _setup_submission(directory, code_tar)
+        _setup_submission(directory, urls)
 
     path = os.path.join(directory, model_directory)
     os.makedirs(path, exist_ok=True)

crunch/command/update_token.py

@@ -0,0 +1,37 @@
+import click
+
+from .. import api, utils
+
+
+def update_token(
+    session: utils.CustomSession,
+    clone_token: str,
+):
+    project_info = utils.read_project_info()
+
+    try:
+        push_token = session.post(
+            f"/v2/project-tokens/upgrade",
+            json={
+                "cloneToken": clone_token
+            }
+        ).json()
+    except api.InvalidProjectTokenException:
+        print("your token seems to have expired or is invalid")
+        print("---")
+        print("please follow this link to copy and paste your new token:")
+        print(session.format_web_url(
+            f'/competitions/{project_info.competition_name}/submit'
+        ))
+        print("")
+
+        raise click.Abort()
+
+    plain = push_token['plain']
+    user_id = push_token['project']["userId"]
+
+    project_info.user_id = user_id
+    utils.write_project_info(project_info)
+    utils.write_token(plain)
+
+    print("token updated")

crunch/main.py

@@ -3,13 +3,17 @@
 
 import click
 
-from . import command, constants, utils, api, library, tester
+from . import command, constants, utils, api, library, tester, __version__
 
 session = None
 debug = False
 
 
 @click.group()
+@click.version_option(
+    __version__.__version__,
+    package_name="__version__.__title__"
+)
 @click.option("--debug", "enable_debug", envvar=constants.DEBUG_ENV_VAR, is_flag=True, help="Enable debug output.")
 @click.option("--api-base-url", envvar=constants.API_BASE_URL_ENV_VAR, default=constants.API_BASE_URL_DEFAULT, help="Set the API base url.")
 @click.option("--web-base-url", envvar=constants.WEB_BASE_URL_ENV_VAR, default=constants.WEB_BASE_URL_DEFAULT, help="Set the Web base url.")
@@ -196,5 +200,18 @@ def convert(
     )
 
 
+@cli.command(help="Update a project token.")
+@click.argument("clone-token", required=True)
+def update_token(
+    clone_token: str,
+):
+    utils.change_root()
+
+    command.update_token(
+        session,
+        clone_token=clone_token
+    )
+
+
 if __name__ == '__main__':
     cli()

crunch/utils.py

@@ -15,6 +15,7 @@
 import pandas
 import psutil
 import requests
+import tqdm
 
 from . import api, constants
 
@@ -99,6 +100,17 @@ def _read_crunchdao_file(name: str, raise_if_missing=True):
         return fd.read()
 
 
+def write_token(plain_push_token: str, directory="."):
+    dot_crunchdao_path = os.path.join(
+        directory,
+        constants.DOT_CRUNCHDAO_DIRECTORY
+    )
+
+    token_file_path = os.path.join(dot_crunchdao_path, constants.TOKEN_FILE)
+    with open(token_file_path, "w") as fd:
+        fd.write(plain_push_token)
+
+
 @dataclasses.dataclass()
 class ProjectInfo:
     competition_name: str
@@ -107,7 +119,9 @@ class ProjectInfo:
 
 def write_project_info(info: ProjectInfo, directory=".") -> ProjectInfo:
     dot_crunchdao_path = os.path.join(
-        directory, constants.DOT_CRUNCHDAO_DIRECTORY)
+        directory,
+        constants.DOT_CRUNCHDAO_DIRECTORY
+    )
 
     old_path = os.path.join(dot_crunchdao_path, constants.OLD_PROJECT_FILE)
     if os.path.exists(old_path):
@@ -236,3 +250,42 @@ def debug(message: str):
         arguments[name] = value
 
     return function(**arguments)
+
+
+def cut_url(url: str):
+    try:
+        return url[:url.index("?")]
+    except ValueError:
+        return url
+
+
+def download(url: str, path: str, log=True):
+    logged = False
+
+    try:
+        with requests.get(url, stream=True) as response:
+            response.raise_for_status()
+
+            file_length = response.headers.get("Content-Length", None)
+            file_length = int(file_length) if not None else None
+
+            if log:
+                if file_length is not None:
+                    file_length_str = f"{file_length} bytes"
+                else:
+                    file_length_str = "unknown length"
+
+                print(
+                    f"download {path} from {cut_url(url)} ({file_length_str})"
+                )
+                logged = True
+
+            with open(path, 'wb') as fd, tqdm.tqdm(total=file_length, unit='iB', unit_scale=True, leave=False) as progress:
+                for chunk in response.iter_content(chunk_size=8192):
+                    progress.update(len(chunk))
+                    fd.write(chunk)
+    except:
+        if log and not logged:
+            print(f"downloading {path} from {cut_url(url)}")
+
+        raise