From 2929669bf901c267af780d258bd0f4023710d28f Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 24 Jun 2023 16:39:12 +0200 Subject: [PATCH 01/15] Implement more methods and fix security hotspot --- src/pcloud/api.py | 35 ++++++++++++++++++++++++++++++++--- src/pcloud/tests/server.py | 21 ++++++++++++++------- src/pcloud/tests/test_api.py | 6 ++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 573ddc9..58283f6 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -357,9 +357,6 @@ def deletetoken(self, **kwargs): return self._do_request("deletetoken", **kwargs) # Streaming - def getfilelink(self, **kwargs): - raise OnlyPcloudError(ONLY_PCLOUD_MSG) - def getvideolink(self, **kwargs): raise OnlyPcloudError(ONLY_PCLOUD_MSG) @@ -462,6 +459,38 @@ def listshares(self, **kwargs): return self._do_request("listshares", **kwargs) # Public links + def getfilepublink(self, **kwargs): + raise OnlyPcloudError(ONLY_PCLOUD_MSG) + + def getpublinkdownload(self, **kwargs): + raise OnlyPcloudError(ONLY_PCLOUD_MSG) + + @RequiredParameterCheck(("path", "folderid")) + def gettreepublink(self, **kwargs): + raise NotImplementedError + + @RequiredParameterCheck(("code",)) + def showpublink(self, **kwargs): + return self._do_request("showpublink", authenticate=False, **kwargs) + + @RequiredParameterCheck(("code",)) + def copypubfile(self, **kwargs): + return self._do_request("copypubfile", **kwargs) + + def listpublinks(self, **kwargs): + return self._do_request("listpublinks", **kwargs) + + def listplshort(self, **kwargs): + return self._do_request("listplshort", **kwargs) + + @RequiredParameterCheck(("linkid",)) + def deletepublink(self, **kwargs): + return self._do_request("deletepublink", **kwargs) + + @RequiredParameterCheck(("linkid",)) + def changepublink(self, **kwargs): + return self._do_request("changepublink", **kwargs) + @RequiredParameterCheck(("path", "folderid")) def getfolderpublink(self, **kwargs): expire = kwargs.get("expire") diff --git a/src/pcloud/tests/server.py b/src/pcloud/tests/server.py index f79e03d..0a468e7 100644 --- a/src/pcloud/tests/server.py +++ b/src/pcloud/tests/server.py @@ -2,21 +2,28 @@ from http.server import BaseHTTPRequestHandler from multipart import MultipartParser from multipart import parse_options_header -from os.path import dirname -from os.path import join +from os import path import socketserver class MockHandler(BaseHTTPRequestHandler): # Handler for GET requests def do_GET(self): - self.send_response(200) + # Send the json message + method = self.path[1:].split("?") + basepath = path.join(path.dirname(__file__), "data") + safemethod = path.realpath(method[0] + ".json") + prefix = path.commonpath((basepath, safemethod)) + if prefix == basepath: + code = 200 + with open(path.join(basepath, safemethod)) as f: + data = f.read() + else: + code = 404 + data = '{"Error": "Path not found or not accessible!"}' + self.send_response(code) self.send_header("Content-type", "applicaton/json") self.end_headers() - # Send the json message - path = self.path[1:].split("?") - with open(join(dirname(__file__), "data", path[0] + ".json")) as f: - data = f.read() self.wfile.write(bytes(data, "utf-8")) # Handler for POST requests diff --git a/src/pcloud/tests/test_api.py b/src/pcloud/tests/test_api.py index 4fd5db7..83e2915 100644 --- a/src/pcloud/tests/test_api.py +++ b/src/pcloud/tests/test_api.py @@ -91,6 +91,12 @@ def test_getfilelink(self): with pytest.raises(api.OnlyPcloudError): papi.getfilelink(file="/test.txt") + def test_server_security(self): + api = DummyPyCloud("", "") + resp = api.session.get(api.endpoint + "../../bogus.sh", params={}) + assert resp.content == b'{"Error": "Path not found or not accessible!"}' + assert resp.status_code == 404 + @pytest.mark.usefixtures("start_mock_server") class TestPcloudFs(object): From ceb9ff5ba1f4191a7533d2ed6e1e8a9faf579d4c Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 24 Jun 2023 18:06:25 +0200 Subject: [PATCH 02/15] readd --- src/pcloud/api.py | 3 +++ src/pcloud/tests/server.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 58283f6..487ce95 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -357,6 +357,9 @@ def deletetoken(self, **kwargs): return self._do_request("deletetoken", **kwargs) # Streaming + def getfilelink(self, **kwargs): + raise OnlyPcloudError(ONLY_PCLOUD_MSG) + def getvideolink(self, **kwargs): raise OnlyPcloudError(ONLY_PCLOUD_MSG) diff --git a/src/pcloud/tests/server.py b/src/pcloud/tests/server.py index 0a468e7..846f64d 100644 --- a/src/pcloud/tests/server.py +++ b/src/pcloud/tests/server.py @@ -10,14 +10,19 @@ class MockHandler(BaseHTTPRequestHandler): # Handler for GET requests def do_GET(self): # Send the json message - method = self.path[1:].split("?") + methodparts = self.path[1:].split("?") basepath = path.join(path.dirname(__file__), "data") - safemethod = path.realpath(method[0] + ".json") + method = path.join(basepath, methodparts[0] + ".json") + safemethod = path.realpath(method) prefix = path.commonpath((basepath, safemethod)) if prefix == basepath: - code = 200 - with open(path.join(basepath, safemethod)) as f: - data = f.read() + try: + code = 200 + with open(safemethod) as f: + data = f.read() + except FileNotFoundError: + code = 404 + data = '{"Error": "Path not found or not accessible!"}' else: code = 404 data = '{"Error": "Path not found or not accessible!"}' From 0c9f4e37f8b23e234b804d6dabcb8c9f2a54ae30 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 22 Dec 2023 09:21:56 +0100 Subject: [PATCH 03/15] Added more test methods and update test deps --- .coveragerc | 4 +--- src/pcloud/tests/test_api.py | 20 ++++++++++++++++++++ test_requirements.txt | 9 +++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.coveragerc b/.coveragerc index 63f46cb..d6d9e85 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,6 @@ [run] relative_files = True - -[report] include = src/* omit = - */tests/* + src/pcloud/tests/* \ No newline at end of file diff --git a/src/pcloud/tests/test_api.py b/src/pcloud/tests/test_api.py index 83e2915..54053e8 100644 --- a/src/pcloud/tests/test_api.py +++ b/src/pcloud/tests/test_api.py @@ -91,6 +91,26 @@ def test_getfilelink(self): with pytest.raises(api.OnlyPcloudError): papi.getfilelink(file="/test.txt") + def test_getvideolink(self): + papi = DummyPyCloud("foo", "bar") + with pytest.raises(api.OnlyPcloudError): + papi.getvideolink(file="/test.txt") + + def test_getvideolinks(self): + papi = DummyPyCloud("foo", "bar") + with pytest.raises(api.OnlyPcloudError): + papi.getvideolinks(file="/test.txt") + + def test_getfilepublink(self): + papi = DummyPyCloud("foo", "bar") + with pytest.raises(api.OnlyPcloudError): + papi.getfilepublink(file="/test.txt") + + def test_getpublinkdownload(self): + papi = DummyPyCloud("foo", "bar") + with pytest.raises(api.OnlyPcloudError): + papi.getpublinkdownload(file="/test.txt") + def test_server_security(self): api = DummyPyCloud("", "") resp = api.session.get(api.endpoint + "../../bogus.sh", params={}) diff --git a/test_requirements.txt b/test_requirements.txt index 418707b..449b929 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,9 +1,10 @@ -pytest==7.4.0 +pytest==7.4.3 pytest-sugar==0.9.7 -pytest-timeout==2.1.0 +pytest-timeout==2.2.0 pytest-cov==4.1.0 -wheel==0.40.0 +wheel==0.42.0 +# flake8 version 6 requires Python 3.8+ flake8==5.0.4 fs==2.4.16 -playwright==1.35.0 +playwright==1.40.0 multipart==0.2.4 From 503e49e1bcc83fcfb0600702a4b508791f76c0da Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 22 Dec 2023 09:32:22 +0100 Subject: [PATCH 04/15] Choose compatible plyawright version --- test_requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 449b929..afaeb4e 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,5 +6,6 @@ wheel==0.42.0 # flake8 version 6 requires Python 3.8+ flake8==5.0.4 fs==2.4.16 -playwright==1.40.0 +# playwright > 1.35.0 requires Python 3.8+ +playwright==1.35.0 multipart==0.2.4 From 74bd97c6b1fef06ca13d70645d86610dee4fdab1 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 22 Dec 2023 11:20:59 +0100 Subject: [PATCH 05/15] foo --- .github/workflows/pcloud-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pcloud-test.yml b/.github/workflows/pcloud-test.yml index bc1c7fb..813e3bd 100644 --- a/.github/workflows/pcloud-test.yml +++ b/.github/workflows/pcloud-test.yml @@ -11,11 +11,10 @@ on: jobs: build: - runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] max-parallel: 1 steps: - uses: actions/checkout@v3 @@ -58,5 +57,6 @@ jobs: -Dsonar.organization=tomgross-github -Dsonar.python.version=3 -Dsonar.python.coverage.reportPaths=coverage.xml + -Dsonar.python.coverage.exclusions=**/tests/**/* From a2eb6f14bb443187c462cd090970e173290fbb17 Mon Sep 17 00:00:00 2001 From: masrlinu Date: Tue, 1 Aug 2023 15:27:36 +0200 Subject: [PATCH 06/15] fix-oauth2-bug --- src/pcloud/oauth2.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pcloud/oauth2.py b/src/pcloud/oauth2.py index 26a0336..52767db 100644 --- a/src/pcloud/oauth2.py +++ b/src/pcloud/oauth2.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import _thread +import time from http.server import BaseHTTPRequestHandler from http.server import HTTPServer @@ -29,6 +30,12 @@ def do_GET(self): self.server.pc_hostname = query.get("hostname", "api.pcloud.com")[0] self.wfile.write(b"

You may now close this window.

") +class HTTPServerWithAttributes(HTTPServer): + def __init__(self, *args, **kwargs): + self.access_token = None + self.pc_hostname = None + super().__init__(*args, **kwargs) + class TokenHandler(object): """ @@ -49,17 +56,15 @@ def close_browser(self): """Hook which is called after request is handled.""" def get_access_token(self): - http_server = HTTPServer(("localhost", PORT), HTTPServerHandler) + http_server = HTTPServerWithAttributes(("localhost", PORT), HTTPServerHandler) - # Solution taken from https://stackoverflow.com/a/12651298 - # There might be better ways than accessing the internal - # _thread library for starting the http-server non-blocking - # but I did not found any ;-) def start_server(): - http_server.handle_request() + http_server.serve_forever() _thread.start_new_thread(start_server, ()) self.open_browser() + while not (http_server.access_token and http_server.pc_hostname): + time.sleep(1) self.close_browser() - http_server.server_close() + http_server.shutdown() return http_server.access_token, http_server.pc_hostname From 75c7272e5d913929d9d72f319cb7fff210d02501 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 22 Dec 2023 13:22:45 +0100 Subject: [PATCH 07/15] Fix Playwright locators in test --- .github/workflows/pcloud-test.yml | 2 +- setup.py | 2 +- src/pcloud/oauth2.py | 4 ++-- src/pcloud/tests/test_api.py | 17 ++++++++++++----- src/pcloud/tests/test_oauth2.py | 4 ++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pcloud-test.yml b/.github/workflows/pcloud-test.yml index 813e3bd..4647d43 100644 --- a/.github/workflows/pcloud-test.yml +++ b/.github/workflows/pcloud-test.yml @@ -57,6 +57,6 @@ jobs: -Dsonar.organization=tomgross-github -Dsonar.python.version=3 -Dsonar.python.coverage.reportPaths=coverage.xml - -Dsonar.python.coverage.exclusions=**/tests/**/* + -Dsonar.python.coverage.exclusions=**/tests/* diff --git a/setup.py b/setup.py index 65dfee7..aec8d38 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ install_requires=[ "requests", "requests-toolbelt", - "setuptools" + "setuptools>=68.0.0" ], extras_require={"pyfs": ["fs"]}, entry_points={ diff --git a/src/pcloud/oauth2.py b/src/pcloud/oauth2.py index 52767db..ec29ab1 100644 --- a/src/pcloud/oauth2.py +++ b/src/pcloud/oauth2.py @@ -59,12 +59,12 @@ def get_access_token(self): http_server = HTTPServerWithAttributes(("localhost", PORT), HTTPServerHandler) def start_server(): - http_server.serve_forever() + http_server.handle_request() _thread.start_new_thread(start_server, ()) self.open_browser() while not (http_server.access_token and http_server.pc_hostname): time.sleep(1) self.close_browser() - http_server.shutdown() + http_server.server_close() return http_server.access_token, http_server.pc_hostname diff --git a/src/pcloud/tests/test_api.py b/src/pcloud/tests/test_api.py index 54053e8..e28aa97 100644 --- a/src/pcloud/tests/test_api.py +++ b/src/pcloud/tests/test_api.py @@ -53,6 +53,9 @@ def test_getfolderpublink(): @pytest.mark.usefixtures("start_mock_server") class TestPcloudApi(object): + + noop_dummy_file = "/test.txt" + def test_getdigest(self): api = DummyPyCloud("foo", "bar") assert api.getdigest() == b"YGtAxbUpI85Zvs7lC7Z62rBwv907TBXhV2L867Hkh" @@ -89,27 +92,27 @@ def test_extractarchive(self): def test_getfilelink(self): papi = DummyPyCloud("foo", "bar") with pytest.raises(api.OnlyPcloudError): - papi.getfilelink(file="/test.txt") + papi.getfilelink(file=self.noop_dummy_file) def test_getvideolink(self): papi = DummyPyCloud("foo", "bar") with pytest.raises(api.OnlyPcloudError): - papi.getvideolink(file="/test.txt") + papi.getvideolink(file=self.noop_dummy_file) def test_getvideolinks(self): papi = DummyPyCloud("foo", "bar") with pytest.raises(api.OnlyPcloudError): - papi.getvideolinks(file="/test.txt") + papi.getvideolinks(file=self.noop_dummy_file) def test_getfilepublink(self): papi = DummyPyCloud("foo", "bar") with pytest.raises(api.OnlyPcloudError): - papi.getfilepublink(file="/test.txt") + papi.getfilepublink(file=self.noop_dummy_file) def test_getpublinkdownload(self): papi = DummyPyCloud("foo", "bar") with pytest.raises(api.OnlyPcloudError): - papi.getpublinkdownload(file="/test.txt") + papi.getpublinkdownload(file=self.noop_dummy_file) def test_server_security(self): api = DummyPyCloud("", "") @@ -127,3 +130,7 @@ def test_write(self, capsys): fs_f.write(data) captured = capsys.readouterr() assert captured.out == "File: b'hello pcloud fs unittest', Size: 24" + + def test_repr(self): + with DummyPCloudFS(username="foo", password="bar") as fs: + assert repr(fs) == "" diff --git a/src/pcloud/tests/test_oauth2.py b/src/pcloud/tests/test_oauth2.py index 5aad186..b587e26 100644 --- a/src/pcloud/tests/test_oauth2.py +++ b/src/pcloud/tests/test_oauth2.py @@ -25,9 +25,9 @@ def open_browser(self): log.info(self.auth_url) page.goto(self.auth_url) page.get_by_placeholder("Email").fill(os.environ.get("PCLOUD_USERNAME")) - page.get_by_text("Continue").click() + page.get_by_text("Continue", exact=True).click() page.get_by_placeholder("Password").fill(os.environ.get("PCLOUD_PASSWORD")) - page.get_by_text("Log in").click() + page.get_by_text("Login").click() expect(page.get_by_text("You may now close this window.")).to_be_visible() From 204cf46271ef1eaa17e59f758e78d79e123b4a0d Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Tue, 26 Dec 2023 18:10:35 +0100 Subject: [PATCH 08/15] PyFS --- README.rst | 1 - setup.py | 5 +-- src/pcloud/api.py | 45 +++++++++++---------- src/pcloud/oauth2.py | 1 + src/pcloud/pcloudfs.py | 58 +++++++++++++++++++++------- src/pcloud/tests/test_api.py | 27 +++++++------ src/pcloud/tests/test_integration.py | 12 ++++++ src/pcloud/tests/test_oauth2.py | 6 ++- src/pcloud/tests/test_pyfs.py | 39 +++++++++++++++++++ 9 files changed, 138 insertions(+), 56 deletions(-) create mode 100644 src/pcloud/tests/test_pyfs.py diff --git a/README.rst b/README.rst index aa5c628..ccabb71 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,6 @@ Features ======== - Can be used as a library -- Comes with a command line script - Provides a PyFileSystem implementation Examples diff --git a/setup.py b/setup.py index aec8d38..564b09b 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,8 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", "License :: OSI Approved :: MIT License", "Topic :: Software Development :: Libraries :: Python Modules", @@ -52,9 +54,6 @@ ], extras_require={"pyfs": ["fs"]}, entry_points={ - "console_scripts": [ - "pcloud-cli = pcloud.api:main", - ], "fs.opener": ["pcloud = pcloud.pcloudfs:PCloudOpener"], }, ) diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 487ce95..0582cf4 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -7,7 +7,6 @@ from urllib.parse import urlparse from urllib.parse import urlunsplit -import argparse import datetime import logging import os.path @@ -43,10 +42,11 @@ class AuthenticationError(Exception): class OnlyPcloudError(NotImplementedError): """Feature restricted to pCloud""" - -# Helpers +class InvalidFileModeError(Exception): + """File mode not supported""" +# Helpers def to_api_datetime(dt): """Converter to a datetime structure the pCloud API understands @@ -56,20 +56,6 @@ def to_api_datetime(dt): return dt.isoformat() return dt - -def main(): - parser = argparse.ArgumentParser(description="pCloud command line client") - parser.add_argument( - "username", help="The username for login into your pCloud account" - ) - parser.add_argument( - "password", help="The password for login into your pCloud account" - ) - args = parser.parse_args() - pyc = PyCloud(args.username, args.password) - print(pyc) - - class PyCloud(object): endpoints = { "api": "https://api.pcloud.com/", @@ -251,7 +237,6 @@ def _upload(self, method, files, **kwargs): kwargs["auth"] = self.auth_token elif self.access_token: # OAuth2 authentication kwargs["access_token"] = self.access_token - kwargs.pop("fd", None) fields = list(kwargs.items()) fields.extend(files) m = MultipartEncoder(fields=fields) @@ -380,7 +365,7 @@ def gettextfile(self, **kwargs): def file_open(self, **kwargs): return self._do_request("file_open", **kwargs) - @RequiredParameterCheck(("fd",)) + @RequiredParameterCheck(("fd", "count")) def file_read(self, **kwargs): return self._do_request("file_read", json=False, **kwargs) @@ -403,7 +388,9 @@ def file_truncate(self, **kwargs): @RequiredParameterCheck(("fd", "data")) def file_write(self, **kwargs): files = [("file", ("upload-file.io", BytesIO(kwargs.pop("data"))))] + kwargs['fd'] = str(kwargs["fd"]) return self._upload("file_write", files, **kwargs) + #return self._do_request("file_write", **kwargs) @RequiredParameterCheck(("fd",)) def file_pwrite(self, **kwargs): @@ -540,7 +527,19 @@ def trash_restorepath(self, **kwargs): @RequiredParameterCheck(("fileid", "folderid")) def trash_restore(self, **kwargs): raise NotImplementedError - - -if __name__ == "__main__": - main() + + # convenience methods + @RequiredParameterCheck(("path",)) + def file_exists(self, **kwargs): + path = kwargs["path"] + resp = self.file_open(path=path, flags=O_APPEND) + result = resp.get("result") + print(resp) + if result == 0: + self.file_close(fd=resp["fd"]) + return True + elif result == 2009: + return False + else: + raise OSError(f"pCloud error occured ({result}) - {resp['error']}: {path}") +# EOF \ No newline at end of file diff --git a/src/pcloud/oauth2.py b/src/pcloud/oauth2.py index ec29ab1..db2dd70 100644 --- a/src/pcloud/oauth2.py +++ b/src/pcloud/oauth2.py @@ -30,6 +30,7 @@ def do_GET(self): self.server.pc_hostname = query.get("hostname", "api.pcloud.com")[0] self.wfile.write(b"

You may now close this window.

") + class HTTPServerWithAttributes(HTTPServer): def __init__(self, *args, **kwargs): self.access_token = None diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 552cd80..4d622ca 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -4,25 +4,48 @@ from fs.opener import Opener from fs import errors from fs.enums import ResourceType +from fs.path import abspath from io import BytesIO -from pcloud.api import PyCloud -from pcloud.api import O_CREAT +from pcloud import api +FSMODEMMAP = { + 'w': api.O_WRITE, + 'x': api.O_EXCL, + 'a': api.O_APPEND, + 'r': api.O_APPEND # pCloud does not have a read mode +} class PCloudFile(BytesIO): """A file representation for pCloud files""" - def __init__(self, pcloud, path, mode): + def __init__(self, pcloud, path, mode, encoding='utf-8'): self.pcloud = pcloud self.path = path self.mode = mode - # TODO: dependency mode and flags? - flags = O_CREAT + self.encoding = encoding + for pyflag, pcloudflag in FSMODEMMAP.items(): + if pyflag in mode: + flags = pcloudflag + break + else: + raise api.InvalidFileModeError + + if "t" in mode: + self.binary = False + else: + self.binary = True + + # Python and PyFS will create a file, which doesn't exist + # in append-mode `a` but pCloud does not. + if flags == api.O_APPEND and not self.pcloud.file_exists(path=self.path): + resp = self.pcloud.file_open(path=self.path, flags=api.O_CREAT) + self.pcloud.file_close(fd=resp["fd"]) resp = self.pcloud.file_open(path=self.path, flags=flags) - if resp.get("result") == 0: + result = resp.get("result") + if result == 0: self.fd = resp["fd"] else: - raise OSError(f"pCloud error occured ({resp['result']}) - {resp['error']}") + raise OSError(f"pCloud error occured ({result}) - {resp['error']}: {path}") def close(self): self.pcloud.file_close(fd=self.fd) @@ -40,25 +63,26 @@ def seek(self, offset, whence=None): def read(self, size=-1): if size == -1: - size = self.pcloud.file_size(fd=self.fd) + size = self.pcloud.file_size(fd=self.fd)["size"] return self.pcloud.file_read(fd=self.fd, count=size) def truncate(self, size=None): self.pcloud.file_truncate(fd=self.fd) def write(self, b): + if isinstance(b, str): + b = bytes(b, self.encoding) self.pcloud.file_write(fd=self.fd, data=b) - class PCloudFS(FS): """A Python virtual filesystem representation for pCloud""" # make alternative implementations possible (i.e. for testing) - factory = PyCloud + factory = api.PyCloud - def __init__(self, username, password): + def __init__(self, username, password, endpoint="api"): super().__init__() - self.pcloud = self.factory(username, password) + self.pcloud = self.factory(username, password, endpoint) self._meta = { "case_insensitive": False, "invalid_path_chars": ":", # not sure what else @@ -128,7 +152,6 @@ def setinfo(self, path, info): # pylint: disable=too-many-branches def listdir(self, path): _path = self.validatepath(path) - _type = self.gettype(_path) if _type is not ResourceType.directory: raise errors.DirectoryExpected(path) @@ -137,6 +160,11 @@ def listdir(self, path): def makedir(self, path, permissions=None, recreate=False): self.check() + if path == '/' or path == '.': + return self.opendir(path) + + + path = abspath(path) result = self.pcloud.createfolder(path=path) if result["result"] == 2004: if recreate: @@ -151,8 +179,10 @@ def makedir(self, path, permissions=None, recreate=False): ) else: # everything is OK return self.opendir(path) - + def openbin(self, path, mode="r", buffering=-1, **options): + if path == "/": + raise errors.FileExpected(path) return PCloudFile(self.pcloud, path, mode) def remove(self, path): diff --git a/src/pcloud/tests/test_api.py b/src/pcloud/tests/test_api.py index e28aa97..4824009 100644 --- a/src/pcloud/tests/test_api.py +++ b/src/pcloud/tests/test_api.py @@ -53,7 +53,6 @@ def test_getfolderpublink(): @pytest.mark.usefixtures("start_mock_server") class TestPcloudApi(object): - noop_dummy_file = "/test.txt" def test_getdigest(self): @@ -121,16 +120,16 @@ def test_server_security(self): assert resp.status_code == 404 -@pytest.mark.usefixtures("start_mock_server") -class TestPcloudFs(object): - def test_write(self, capsys): - with DummyPCloudFS(username="foo", password="bar") as fs: - data = b"hello pcloud fs unittest" - fs_f = fs.openbin("hello.bin") - fs_f.write(data) - captured = capsys.readouterr() - assert captured.out == "File: b'hello pcloud fs unittest', Size: 24" - - def test_repr(self): - with DummyPCloudFS(username="foo", password="bar") as fs: - assert repr(fs) == "" +# @pytest.mark.usefixtures("start_mock_server") +# class TestPcloudFs(object): +# def test_write(self, capsys): +# with DummyPCloudFS(username="foo", password="bar") as fs: +# data = b"hello pcloud fs unittest" +# fs_f = fs.openbin("hello.bin") +# fs_f.write(data) +# captured = capsys.readouterr() +# assert captured.out == "File: b'hello pcloud fs unittest', Size: 24" + +# def test_repr(self): +# with DummyPCloudFS(username="foo", password="bar") as fs: +# assert repr(fs) == "" diff --git a/src/pcloud/tests/test_integration.py b/src/pcloud/tests/test_integration.py index f41872d..759844d 100644 --- a/src/pcloud/tests/test_integration.py +++ b/src/pcloud/tests/test_integration.py @@ -3,10 +3,12 @@ import time import zipfile +from fs import opener from io import BytesIO from pathlib import Path from pcloud.api import PyCloud from pcloud.api import O_CREAT +from urllib.parse import quote @pytest.fixture @@ -81,3 +83,13 @@ def test_listtokens(pycloud): result = pycloud.listtokens() assert result["result"] == 0 assert len(result["tokens"]) > 1 + + +# def testpyfsopener(pycloud): +# username = quote(os.environ.get("PCLOUD_USERNAME")) +# password = quote(os.environ.get("PCLOUD_PASSWORD")) +# pcloud_url = f'pcloud://{username}:{password}/' +# pcloud_url = 'pcloud://itconsense+pytest%40gmail.com:eXOtICf4TH3r/' +# # import pdb; pdb.set_trace() +# with opener.open_fs(pcloud_url) as pcloud_fs: +# assert pcloud_fs.listdir('/') == {} \ No newline at end of file diff --git a/src/pcloud/tests/test_oauth2.py b/src/pcloud/tests/test_oauth2.py index b587e26..3a485e5 100644 --- a/src/pcloud/tests/test_oauth2.py +++ b/src/pcloud/tests/test_oauth2.py @@ -21,13 +21,17 @@ class PlaywrightTokenHandler(TokenHandler): def open_browser(self): with sync_playwright() as p: self.browser = p.firefox.launch() + self.browser.new_context( + locale="de-DE", + timezone_id="Europe/Berlin", + ) page = self.browser.new_page() log.info(self.auth_url) page.goto(self.auth_url) page.get_by_placeholder("Email").fill(os.environ.get("PCLOUD_USERNAME")) page.get_by_text("Continue", exact=True).click() page.get_by_placeholder("Password").fill(os.environ.get("PCLOUD_PASSWORD")) - page.get_by_text("Login").click() + page.get_by_text("Log in", exact=True).click() expect(page.get_by_text("You may now close this window.")).to_be_visible() diff --git a/src/pcloud/tests/test_pyfs.py b/src/pcloud/tests/test_pyfs.py new file mode 100644 index 0000000..1586134 --- /dev/null +++ b/src/pcloud/tests/test_pyfs.py @@ -0,0 +1,39 @@ +import os +import unittest + +from fs.path import abspath +from fs.test import FSTestCases +from pcloud.pcloudfs import PCloudFS + + +class TestMyFS(FSTestCases, unittest.TestCase): + + testdir = "_pyfs_tests" + + @classmethod + def setUpClass(cls): + username = os.environ.get("PCLOUD_USERNAME") + password = os.environ.get("PCLOUD_PASSWORD") + cls.pcloudfs = PCloudFS(username, password, endpoint="eapi") + + + """ + @classmethod + def tearDownClass(cls): + cls.pcloudfs.removetree(cls.testdir) + """ + + def make_fs(self): + # Return an instance of your FS object here + return self.basedir + + def setUp(self): + testdir = abspath(self.testdir) + if self.pcloudfs.exists(testdir): + self.pcloudfs.removetree(testdir) + self.basedir = self.pcloudfs.makedir(testdir) + super().setUp() + + # override to not destroy filesystem + def tearDown(self): + self.pcloudfs.removetree(self.testdir) \ No newline at end of file From e9937a241d712e9b01aac356921afeb99ce7d2b5 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 5 Jan 2024 08:43:14 +0100 Subject: [PATCH 09/15] foo --- src/pcloud/api.py | 1 - src/pcloud/pcloudfs.py | 299 ++++++++++++++++++++++++++++------ src/pcloud/tests/test_pyfs.py | 46 ++++-- test_requirements.txt | 1 + 4 files changed, 282 insertions(+), 65 deletions(-) diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 0582cf4..8330114 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -534,7 +534,6 @@ def file_exists(self, **kwargs): path = kwargs["path"] resp = self.file_open(path=path, flags=O_APPEND) result = resp.get("result") - print(resp) if result == 0: self.file_close(fd=resp["fd"]) return True diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 4d622ca..a35a949 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -4,9 +4,19 @@ from fs.opener import Opener from fs import errors from fs.enums import ResourceType -from fs.path import abspath -from io import BytesIO +from fs.path import abspath, dirname +from fs.mode import Mode +from fs.subfs import SubFS +import io +import array +import threading from pcloud import api +from fs.enums import ResourceType, Seek +from contextlib import closing + +from datetime import datetime + +DT_FORMAT_STRING = "%a, %d %b %Y %H:%M:%S %z" FSMODEMMAP = { 'w': api.O_WRITE, @@ -15,35 +25,37 @@ 'r': api.O_APPEND # pCloud does not have a read mode } -class PCloudFile(BytesIO): + +class PCloudFile(io.RawIOBase): """A file representation for pCloud files""" def __init__(self, pcloud, path, mode, encoding='utf-8'): self.pcloud = pcloud self.path = path - self.mode = mode + self.mode = Mode(mode) self.encoding = encoding + self._lock = threading.Lock() + self._lines = None + self._index = 0 + self.pos = 0 for pyflag, pcloudflag in FSMODEMMAP.items(): if pyflag in mode: flags = pcloudflag break else: raise api.InvalidFileModeError - - if "t" in mode: - self.binary = False - else: - self.binary = True - + # Python and PyFS will create a file, which doesn't exist - # in append-mode `a` but pCloud does not. - if flags == api.O_APPEND and not self.pcloud.file_exists(path=self.path): + # but pCloud does not. + if not self.pcloud.file_exists(path=self.path): resp = self.pcloud.file_open(path=self.path, flags=api.O_CREAT) self.pcloud.file_close(fd=resp["fd"]) resp = self.pcloud.file_open(path=self.path, flags=flags) result = resp.get("result") if result == 0: self.fd = resp["fd"] + elif result == 2009: + raise errors.ResourceNotFound(path) else: raise OSError(f"pCloud error occured ({result}) - {resp['error']}: {path}") @@ -51,50 +63,184 @@ def close(self): self.pcloud.file_close(fd=self.fd) self.fd = None + def tell(self): + return self.pos + + def seekable(self): + return True + + def readable(self): + return self.mode.reading + + def writable(self): + return self.mode.writing + @property def closed(self): return self.fd is None - def fileno(self): - return self.fd - - def seek(self, offset, whence=None): - self.pcloud.file_seek(fd=self.fd, offset=offset) + def seek(self, offset, whence=Seek.set): + _whence = int(whence) + if _whence not in (Seek.set, Seek.current, Seek.end): + raise ValueError("invalid value for whence") + if _whence == Seek.set: + new_pos = offset + elif _whence == Seek.current: + new_pos = self.pos + offset + elif _whence == Seek.end: + resp = self.pcloud.file_size(fd=self.fd) + file_size = resp["size"] + new_pos = file_size + offset + self.pos = max(0, new_pos) + resp = self.pcloud.file_seek(fd=self.fd, offset=self.pos) + return resp["offset"] + # return self.tell() def read(self, size=-1): + # print(f"pos: {self.pos} fd: {self.fd}") + if not self.mode.reading: + raise IOError("File not open for reading") if size == -1: size = self.pcloud.file_size(fd=self.fd)["size"] - return self.pcloud.file_read(fd=self.fd, count=size) + self.pos += size + resp = self.pcloud.file_read(fd=self.fd, count=size) + return resp + + def _close_and_reopen(self): + self.pcloud.file_close(fd=self.fd) + resp = self.pcloud.file_open(path=self.path, flags=api.O_APPEND) + result = resp.get("result") + if result == 0: + self.fd = resp["fd"] - def truncate(self, size=None): - self.pcloud.file_truncate(fd=self.fd) + def truncate(self, size=None): + with self._lock: + if size is None: + size = self.tell() + self.pcloud.file_truncate(fd=self.fd, length=size) + # file gets truncated on close + self._close_and_reopen() + return size def write(self, b): - if isinstance(b, str): - b = bytes(b, self.encoding) - self.pcloud.file_write(fd=self.fd, data=b) + with self._lock: + if not self.mode.writing: + raise IOError("File not open for writing") + if isinstance(b, str): + b = bytes(b, self.encoding) + #if b==b'O': + # import pdb; pdb.set_trace() + resp = self.seek(self.pos-1) + result = self.pcloud.file_write(fd=self.fd, data=b) + sent_size = result["bytes"] + self.pos += sent_size + self._close_and_reopen() + return sent_size + + def writelines(self,lines): + self.write(b''.join(lines)) + + def readline(self): + result = b'' + char = '' + while char != b'\n': + char = self.read(size=1) + print(char) + result += char + if not char: + break + return result + + def line_iterator(self, size=None): + self.pcloud.file_seek(fd=self.fd, offset=0, whence=0) + line = [] + byte = b"1" + if size is None or size < 0: + while byte: + byte = self.read(1) + line.append(byte) + if byte in b"\n": + yield b"".join(line) + del line[:] + else: + while byte and size: + byte = self.read(1) + size -= len(byte) + line.append(byte) + if byte in b"\n" or not size: + yield b"".join(line) + del line[:] + + def readlines(self, hint=-1): + lines = [] + size = 0 + for line in self.line_iterator(): # type: ignore + lines.append(line) + size += len(line) + if hint != -1 and size > hint: + break + return lines + + def readinto(self, buffer): + data = self.read(len(buffer)) + bytes_read = len(data) + if isinstance(buffer, array.array): + buffer[:bytes_read] = array.array(buffer.typecode, data) + else: + buffer[:bytes_read] = data # type: ignore + return bytes_read + + def __repr__(self): + return f"" + + def __iter__(self): + return iter(self.readlines()) + + def __next__(self): + if self._lines is None: + self._lines = self.readlines() + if self._index >= len(self._lines): + raise StopIteration + result = self._lines[self._index] + self._index += 1 + return result + +class PCloudSubFS(SubFS): + + def __init__(self, parent_fs, path): + super().__init__(parent_fs, path) + self._wrap_fs._wrap_sub_dir = self._sub_dir + class PCloudFS(FS): """A Python virtual filesystem representation for pCloud""" # make alternative implementations possible (i.e. for testing) factory = api.PyCloud + subfs_class = PCloudSubFS + + _meta = { + "invalid_path_chars": "\0:", + "case_insensitive": False, + "max_path_length": None, # don't know what the limit is + "max_sys_path_length": None, # there's no syspath + "supports_rename": False, # since we don't have a syspath... + "network": True, + "read_only": False, +# "thread_safe": True, +# "unicode_paths": True, + "virtual": False, + } def __init__(self, username, password, endpoint="api"): super().__init__() self.pcloud = self.factory(username, password, endpoint) - self._meta = { - "case_insensitive": False, - "invalid_path_chars": ":", # not sure what else - "max_path_length": None, # don't know what the limit is - "max_sys_path_length": None, # there's no syspath - "network": True, - "read_only": False, - "supports_rename": False, # since we don't have a syspath... - } def __repr__(self): return "" + + def _to_datetime(self, dt_str, dt_format=DT_FORMAT_STRING): + return datetime.strptime(dt_str, dt_format).timestamp() def _info_from_metadata(self, metadata, namespaces): info = { @@ -107,9 +253,9 @@ def _info_from_metadata(self, metadata, namespaces): info["details"] = { "type": 1 if metadata.get("isfolder") else 2, "accessed": None, - "modified": metadata.get("modified"), - "created": metadata.get("created"), - "metadata_changed": metadata.get("modified"), + "modified": self._to_datetime(metadata.get("modified")), + "created": self._to_datetime(metadata.get("created")), + "metadata_changed": self._to_datetime(metadata.get("modified")), "size": metadata.get("size", 0), } if "link" in namespaces: @@ -148,7 +294,17 @@ def getinfo(self, path, namespaces=None): def setinfo(self, path, info): # pylint: disable=too-many-branches # pCloud doesn't support changing any of the metadata values - pass + if not self.exists(path): + raise errors.ResourceNotFound(path) + + def create(self, path, wipe=False): + with self._lock: + if self.exists(path) and not wipe: + return False + with closing(self.open(path, "wb")) as f: + if wipe: + f.truncate(size=0) + return True def listdir(self, path): _path = self.validatepath(path) @@ -157,42 +313,85 @@ def listdir(self, path): raise errors.DirectoryExpected(path) result = self.pcloud.listfolder(path=_path) return [item["name"] for item in result["metadata"]["contents"]] - + def makedir(self, path, permissions=None, recreate=False): self.check() - if path == '/' or path == '.': - return self.opendir(path) - - + # import pdb; pdb.set_trace() + subpath = getattr(self, '_wrap_sub_dir', '') + if path == '/' or path == '.' or path == subpath: + if recreate: + return self.opendir(path) + else: + raise errors.DirectoryExists(path) path = abspath(path) - result = self.pcloud.createfolder(path=path) - if result["result"] == 2004: + if self.exists(path): + raise errors.DirectoryExists(path) + resp = self.pcloud.createfolder(path=path) + result = resp["result"] + if result == 2004: if recreate: # If the directory already exists and recreate = True # we don't want to raise an error pass else: raise errors.DirectoryExists(path) - elif result["result"] != 0: + elif result == 2002: + raise errors.ResourceNotFound(path) + elif result != 0: raise errors.OperationFailed( - path=path, msg=f"Create of directory failed with {result['error']}" + path=path, msg=f"Create of directory failed with ({result}) {resp['error']}" ) else: # everything is OK return self.opendir(path) def openbin(self, path, mode="r", buffering=-1, **options): - if path == "/": + _mode = Mode(mode) + _mode.validate_bin() + _path = self.validatepath(path) + if _path == "/": raise errors.FileExpected(path) - return PCloudFile(self.pcloud, path, mode) + with self._lock: + try: + info = self.getinfo(_path) + except errors.ResourceNotFound: + if _mode.reading: + raise errors.ResourceNotFound(path) + if _mode.writing and not self.isdir(dirname(_path)): + raise errors.ResourceNotFound(path) + else: + if info.is_dir: + raise errors.FileExpected(path) + if _mode.exclusive: + raise errors.FileExists(path) + pcloud_file = PCloudFile(self.pcloud, _path, _mode.to_platform_bin()) + return pcloud_file def remove(self, path): - self.pcloud.deletefile(path=path) + _path = self.validatepath(path) + if not self.exists(_path): + raise errors.ResourceNotFound(path=_path) + if self.getinfo(_path).is_dir == True: + raise errors.FileExpected(_path) + self.pcloud.deletefile(path=_path) def removedir(self, path): - self.pcloud.deletefolder(path=path) + _path = self.validatepath(path) + if not self.exists(_path): + raise errors.ResourceNotFound(path=_path) + info = self.getinfo(_path) + if info.is_dir == False: + raise errors.DirectoryExpected(_path) + if not self.isempty(_path): + raise errors.DirectoryNotEmpty(_path) + self.pcloud.deletefolder(path=_path) def removetree(self, dir_path): - self.pcloud.deletefolderrecursive(path=dir_path) + _path = self.validatepath(dir_path) + if not self.exists(_path): + raise errors.ResourceNotFound(path=_path) + if self.getinfo(_path).is_dir == False: + raise errors.DirectoryExpected(_path) + self.pcloud.deletefolderrecursive(path=_path) class PCloudOpener(Opener): diff --git a/src/pcloud/tests/test_pyfs.py b/src/pcloud/tests/test_pyfs.py index 1586134..dd747bc 100644 --- a/src/pcloud/tests/test_pyfs.py +++ b/src/pcloud/tests/test_pyfs.py @@ -1,12 +1,14 @@ import os +import tenacity import unittest +from fs.errors import ResourceNotFound from fs.path import abspath from fs.test import FSTestCases from pcloud.pcloudfs import PCloudFS -class TestMyFS(FSTestCases, unittest.TestCase): +class TestpCloudFS(FSTestCases, unittest.TestCase): testdir = "_pyfs_tests" @@ -16,24 +18,40 @@ def setUpClass(cls): password = os.environ.get("PCLOUD_PASSWORD") cls.pcloudfs = PCloudFS(username, password, endpoint="eapi") - - """ - @classmethod - def tearDownClass(cls): - cls.pcloudfs.removetree(cls.testdir) - """ - def make_fs(self): # Return an instance of your FS object here - return self.basedir + return self.basefs - def setUp(self): + @tenacity.retry( + stop=tenacity.stop_after_attempt(2), + retry=tenacity.retry_if_exception_type(ResourceNotFound), + wait=tenacity.wait_incrementing(1, 2, 3), + reraise=True + ) + def _prepare_basedir(self): testdir = abspath(self.testdir) - if self.pcloudfs.exists(testdir): - self.pcloudfs.removetree(testdir) - self.basedir = self.pcloudfs.makedir(testdir) + try: + if self.pcloudfs.exists(testdir): + self.pcloudfs.removetree(testdir) + except ResourceNotFound: # pragma: no coverage + pass + # use api method directly, since `makedir` checks the + # basepath and prevents creating here + resp = self.pcloudfs.pcloud.createfolder(path=testdir) + result = resp["result"] + if result == 0: + return self.pcloudfs.opendir(testdir) + else: + raise ResourceNotFound(testdir) + # return self.pcloudfs.makedir(testdir, recreate=True) + + def setUp(self): + self.basefs = self._prepare_basedir() super().setUp() # override to not destroy filesystem def tearDown(self): - self.pcloudfs.removetree(self.testdir) \ No newline at end of file + try: + self.pcloudfs.removetree(self.testdir) + except ResourceNotFound: # pragma: no coverage + pass \ No newline at end of file diff --git a/test_requirements.txt b/test_requirements.txt index afaeb4e..a4debab 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -9,3 +9,4 @@ fs==2.4.16 # playwright > 1.35.0 requires Python 3.8+ playwright==1.35.0 multipart==0.2.4 +tenacity==8.2.3 \ No newline at end of file From 0f986d141795eee2f2e60724edba7885e37778ea Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 5 Jan 2024 08:45:42 +0100 Subject: [PATCH 10/15] foo --- src/pcloud/pcloudfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index a35a949..0f87024 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -325,6 +325,8 @@ def makedir(self, path, permissions=None, recreate=False): raise errors.DirectoryExists(path) path = abspath(path) if self.exists(path): + import pdb; pdb.set_trace() + self.exists(path) raise errors.DirectoryExists(path) resp = self.pcloud.createfolder(path=path) result = resp["result"] From 538a7843754d54306687fb13a493d94f037475c6 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Fri, 5 Jan 2024 14:40:55 +0100 Subject: [PATCH 11/15] 20 failures --- src/pcloud/pcloudfs.py | 4 ++-- src/pcloud/tests/test_pyfs.py | 38 ++++++++++------------------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 0f87024..448f71f 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -325,8 +325,8 @@ def makedir(self, path, permissions=None, recreate=False): raise errors.DirectoryExists(path) path = abspath(path) if self.exists(path): - import pdb; pdb.set_trace() - self.exists(path) + # import pdb; pdb.set_trace() + # self.exists(path) raise errors.DirectoryExists(path) resp = self.pcloud.createfolder(path=path) result = resp["result"] diff --git a/src/pcloud/tests/test_pyfs.py b/src/pcloud/tests/test_pyfs.py index dd747bc..59ff5f4 100644 --- a/src/pcloud/tests/test_pyfs.py +++ b/src/pcloud/tests/test_pyfs.py @@ -1,6 +1,6 @@ import os -import tenacity import unittest +import uuid from fs.errors import ResourceNotFound from fs.path import abspath @@ -10,8 +10,6 @@ class TestpCloudFS(FSTestCases, unittest.TestCase): - testdir = "_pyfs_tests" - @classmethod def setUpClass(cls): username = os.environ.get("PCLOUD_USERNAME") @@ -20,33 +18,19 @@ def setUpClass(cls): def make_fs(self): # Return an instance of your FS object here - return self.basefs + # For some unknown (concurrency?) reason we can't use + # opendir not directly as it fails with a RessourceNotFound exception + # we create a subfs object directly. + return self.pcloudfs.subfs_class(self.pcloudfs, self.testdir) - @tenacity.retry( - stop=tenacity.stop_after_attempt(2), - retry=tenacity.retry_if_exception_type(ResourceNotFound), - wait=tenacity.wait_incrementing(1, 2, 3), - reraise=True - ) - def _prepare_basedir(self): - testdir = abspath(self.testdir) - try: - if self.pcloudfs.exists(testdir): - self.pcloudfs.removetree(testdir) - except ResourceNotFound: # pragma: no coverage - pass - # use api method directly, since `makedir` checks the - # basepath and prevents creating here - resp = self.pcloudfs.pcloud.createfolder(path=testdir) - result = resp["result"] - if result == 0: - return self.pcloudfs.opendir(testdir) - else: - raise ResourceNotFound(testdir) - # return self.pcloudfs.makedir(testdir, recreate=True) + def _prepare_testdir(self): + random_uuid = uuid.uuid4() + testdir = f'/_pyfs_tests_{random_uuid}' + self.pcloudfs.pcloud.createfolder(path=testdir) + self.testdir = testdir def setUp(self): - self.basefs = self._prepare_basedir() + self._prepare_testdir() super().setUp() # override to not destroy filesystem From 1cd926ef9a38978114389bf355b6eb7ff5f1fa66 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 6 Jan 2024 19:21:46 +0100 Subject: [PATCH 12/15] 2 failures --- src/pcloud/pcloudfs.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 448f71f..834d925 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -128,9 +128,6 @@ def write(self, b): raise IOError("File not open for writing") if isinstance(b, str): b = bytes(b, self.encoding) - #if b==b'O': - # import pdb; pdb.set_trace() - resp = self.seek(self.pos-1) result = self.pcloud.file_write(fd=self.fd, data=b) sent_size = result["bytes"] self.pos += sent_size @@ -316,18 +313,13 @@ def listdir(self, path): def makedir(self, path, permissions=None, recreate=False): self.check() - # import pdb; pdb.set_trace() subpath = getattr(self, '_wrap_sub_dir', '') - if path == '/' or path == '.' or path == subpath: + path = abspath(path) + if path == '/' or path == subpath or self.exists(path): if recreate: return self.opendir(path) else: raise errors.DirectoryExists(path) - path = abspath(path) - if self.exists(path): - # import pdb; pdb.set_trace() - # self.exists(path) - raise errors.DirectoryExists(path) resp = self.pcloud.createfolder(path=path) result = resp["result"] if result == 2004: From 298e8b0ed5a9c009651507fa1bda24dde8164e1d Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 6 Jan 2024 19:41:30 +0100 Subject: [PATCH 13/15] 1 failure --- src/pcloud/pcloudfs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 834d925..54d4d85 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -206,7 +206,8 @@ class PCloudSubFS(SubFS): def __init__(self, parent_fs, path): super().__init__(parent_fs, path) - self._wrap_fs._wrap_sub_dir = self._sub_dir + if not hasattr(self._wrap_fs, "_wrap_sub_dir"): + self._wrap_fs._wrap_sub_dir = self._sub_dir class PCloudFS(FS): From 8dc881d69d3611d438218d82784524a99e623953 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 6 Jan 2024 19:43:39 +0100 Subject: [PATCH 14/15] black --- src/pcloud/api.py | 12 +++-- src/pcloud/pcloudfs.py | 69 ++++++++++++++-------------- src/pcloud/tests/test_integration.py | 2 +- src/pcloud/tests/test_pyfs.py | 7 ++- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 8330114..41cdbda 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -42,6 +42,7 @@ class AuthenticationError(Exception): class OnlyPcloudError(NotImplementedError): """Feature restricted to pCloud""" + class InvalidFileModeError(Exception): """File mode not supported""" @@ -56,6 +57,7 @@ def to_api_datetime(dt): return dt.isoformat() return dt + class PyCloud(object): endpoints = { "api": "https://api.pcloud.com/", @@ -388,9 +390,9 @@ def file_truncate(self, **kwargs): @RequiredParameterCheck(("fd", "data")) def file_write(self, **kwargs): files = [("file", ("upload-file.io", BytesIO(kwargs.pop("data"))))] - kwargs['fd'] = str(kwargs["fd"]) + kwargs["fd"] = str(kwargs["fd"]) return self._upload("file_write", files, **kwargs) - #return self._do_request("file_write", **kwargs) + # return self._do_request("file_write", **kwargs) @RequiredParameterCheck(("fd",)) def file_pwrite(self, **kwargs): @@ -527,7 +529,7 @@ def trash_restorepath(self, **kwargs): @RequiredParameterCheck(("fileid", "folderid")) def trash_restore(self, **kwargs): raise NotImplementedError - + # convenience methods @RequiredParameterCheck(("path",)) def file_exists(self, **kwargs): @@ -541,4 +543,6 @@ def file_exists(self, **kwargs): return False else: raise OSError(f"pCloud error occured ({result}) - {resp['error']}: {path}") -# EOF \ No newline at end of file + + +# EOF diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 54d4d85..7ba452c 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -19,17 +19,17 @@ DT_FORMAT_STRING = "%a, %d %b %Y %H:%M:%S %z" FSMODEMMAP = { - 'w': api.O_WRITE, - 'x': api.O_EXCL, - 'a': api.O_APPEND, - 'r': api.O_APPEND # pCloud does not have a read mode + "w": api.O_WRITE, + "x": api.O_EXCL, + "a": api.O_APPEND, + "r": api.O_APPEND, # pCloud does not have a read mode } class PCloudFile(io.RawIOBase): """A file representation for pCloud files""" - def __init__(self, pcloud, path, mode, encoding='utf-8'): + def __init__(self, pcloud, path, mode, encoding="utf-8"): self.pcloud = pcloud self.path = path self.mode = Mode(mode) @@ -44,7 +44,7 @@ def __init__(self, pcloud, path, mode, encoding='utf-8'): break else: raise api.InvalidFileModeError - + # Python and PyFS will create a file, which doesn't exist # but pCloud does not. if not self.pcloud.file_exists(path=self.path): @@ -65,16 +65,16 @@ def close(self): def tell(self): return self.pos - + def seekable(self): return True def readable(self): return self.mode.reading - + def writable(self): return self.mode.writing - + @property def closed(self): return self.fd is None @@ -105,7 +105,7 @@ def read(self, size=-1): self.pos += size resp = self.pcloud.file_read(fd=self.fd, count=size) return resp - + def _close_and_reopen(self): self.pcloud.file_close(fd=self.fd) resp = self.pcloud.file_open(path=self.path, flags=api.O_APPEND) @@ -113,7 +113,7 @@ def _close_and_reopen(self): if result == 0: self.fd = resp["fd"] - def truncate(self, size=None): + def truncate(self, size=None): with self._lock: if size is None: size = self.tell() @@ -134,20 +134,20 @@ def write(self, b): self._close_and_reopen() return sent_size - def writelines(self,lines): - self.write(b''.join(lines)) + def writelines(self, lines): + self.write(b"".join(lines)) def readline(self): - result = b'' - char = '' - while char != b'\n': + result = b"" + char = "" + while char != b"\n": char = self.read(size=1) print(char) result += char if not char: break return result - + def line_iterator(self, size=None): self.pcloud.file_seek(fd=self.fd, offset=0, whence=0) line = [] @@ -167,7 +167,7 @@ def line_iterator(self, size=None): if byte in b"\n" or not size: yield b"".join(line) del line[:] - + def readlines(self, hint=-1): lines = [] size = 0 @@ -175,9 +175,9 @@ def readlines(self, hint=-1): lines.append(line) size += len(line) if hint != -1 and size > hint: - break + break return lines - + def readinto(self, buffer): data = self.read(len(buffer)) bytes_read = len(data) @@ -189,10 +189,10 @@ def readinto(self, buffer): def __repr__(self): return f"" - + def __iter__(self): return iter(self.readlines()) - + def __next__(self): if self._lines is None: self._lines = self.readlines() @@ -202,13 +202,13 @@ def __next__(self): self._index += 1 return result -class PCloudSubFS(SubFS): +class PCloudSubFS(SubFS): def __init__(self, parent_fs, path): super().__init__(parent_fs, path) if not hasattr(self._wrap_fs, "_wrap_sub_dir"): self._wrap_fs._wrap_sub_dir = self._sub_dir - + class PCloudFS(FS): """A Python virtual filesystem representation for pCloud""" @@ -225,8 +225,8 @@ class PCloudFS(FS): "supports_rename": False, # since we don't have a syspath... "network": True, "read_only": False, -# "thread_safe": True, -# "unicode_paths": True, + "thread_safe": True, + "unicode_paths": True, "virtual": False, } @@ -236,7 +236,7 @@ def __init__(self, username, password, endpoint="api"): def __repr__(self): return "" - + def _to_datetime(self, dt_str, dt_format=DT_FORMAT_STRING): return datetime.strptime(dt_str, dt_format).timestamp() @@ -294,7 +294,7 @@ def setinfo(self, path, info): # pylint: disable=too-many-branches # pCloud doesn't support changing any of the metadata values if not self.exists(path): raise errors.ResourceNotFound(path) - + def create(self, path, wipe=False): with self._lock: if self.exists(path) and not wipe: @@ -311,16 +311,16 @@ def listdir(self, path): raise errors.DirectoryExpected(path) result = self.pcloud.listfolder(path=_path) return [item["name"] for item in result["metadata"]["contents"]] - + def makedir(self, path, permissions=None, recreate=False): self.check() - subpath = getattr(self, '_wrap_sub_dir', '') + subpath = getattr(self, "_wrap_sub_dir", "") path = abspath(path) - if path == '/' or path == subpath or self.exists(path): + if path == "/" or path == subpath or self.exists(path): if recreate: return self.opendir(path) else: - raise errors.DirectoryExists(path) + raise errors.DirectoryExists(path) resp = self.pcloud.createfolder(path=path) result = resp["result"] if result == 2004: @@ -334,11 +334,12 @@ def makedir(self, path, permissions=None, recreate=False): raise errors.ResourceNotFound(path) elif result != 0: raise errors.OperationFailed( - path=path, msg=f"Create of directory failed with ({result}) {resp['error']}" + path=path, + msg=f"Create of directory failed with ({result}) {resp['error']}", ) else: # everything is OK return self.opendir(path) - + def openbin(self, path, mode="r", buffering=-1, **options): _mode = Mode(mode) _mode.validate_bin() diff --git a/src/pcloud/tests/test_integration.py b/src/pcloud/tests/test_integration.py index 759844d..02d7e78 100644 --- a/src/pcloud/tests/test_integration.py +++ b/src/pcloud/tests/test_integration.py @@ -92,4 +92,4 @@ def test_listtokens(pycloud): # pcloud_url = 'pcloud://itconsense+pytest%40gmail.com:eXOtICf4TH3r/' # # import pdb; pdb.set_trace() # with opener.open_fs(pcloud_url) as pcloud_fs: -# assert pcloud_fs.listdir('/') == {} \ No newline at end of file +# assert pcloud_fs.listdir('/') == {} diff --git a/src/pcloud/tests/test_pyfs.py b/src/pcloud/tests/test_pyfs.py index 59ff5f4..f5c55bd 100644 --- a/src/pcloud/tests/test_pyfs.py +++ b/src/pcloud/tests/test_pyfs.py @@ -9,7 +9,6 @@ class TestpCloudFS(FSTestCases, unittest.TestCase): - @classmethod def setUpClass(cls): username = os.environ.get("PCLOUD_USERNAME") @@ -22,10 +21,10 @@ def make_fs(self): # opendir not directly as it fails with a RessourceNotFound exception # we create a subfs object directly. return self.pcloudfs.subfs_class(self.pcloudfs, self.testdir) - + def _prepare_testdir(self): random_uuid = uuid.uuid4() - testdir = f'/_pyfs_tests_{random_uuid}' + testdir = f"/_pyfs_tests_{random_uuid}" self.pcloudfs.pcloud.createfolder(path=testdir) self.testdir = testdir @@ -38,4 +37,4 @@ def tearDown(self): try: self.pcloudfs.removetree(self.testdir) except ResourceNotFound: # pragma: no coverage - pass \ No newline at end of file + pass From b01fb15d638def3c6290e3bf33c3c64a932cfb80 Mon Sep 17 00:00:00 2001 From: Tom Gross Date: Sat, 20 Jan 2024 08:35:42 +0100 Subject: [PATCH 15/15] foo --- .gitignore | 3 ++- src/pcloud/api.py | 7 ++++++- src/pcloud/pcloudfs.py | 39 ++++++++++++++++++++++++--------------- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 0bee7d0..3ab74da 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ pyvenv.cfg Pipfile* *.whl .vscode -coverage.xml \ No newline at end of file +coverage.xml +.env \ No newline at end of file diff --git a/src/pcloud/api.py b/src/pcloud/api.py index 41cdbda..5b3cde8 100644 --- a/src/pcloud/api.py +++ b/src/pcloud/api.py @@ -242,9 +242,14 @@ def _upload(self, method, files, **kwargs): fields = list(kwargs.items()) fields.extend(files) m = MultipartEncoder(fields=fields) - resp = self.session.post( + # use own request and not session to make sure connection is closed after + # write + resp = requests.post( self.endpoint + method, data=m, headers={"Content-Type": m.content_type} ) + import pdb; pdb.set_trace() + # fix lazy loading of response + # str(resp.content) return resp.json() @RequiredParameterCheck(("files", "data")) diff --git a/src/pcloud/pcloudfs.py b/src/pcloud/pcloudfs.py index 7ba452c..65c77e3 100644 --- a/src/pcloud/pcloudfs.py +++ b/src/pcloud/pcloudfs.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- +import io +import array +import threading +import time + from fs.base import FS from fs.info import Info from fs.opener import Opener @@ -7,30 +12,29 @@ from fs.path import abspath, dirname from fs.mode import Mode from fs.subfs import SubFS -import io -import array -import threading from pcloud import api from fs.enums import ResourceType, Seek from contextlib import closing from datetime import datetime + DT_FORMAT_STRING = "%a, %d %b %Y %H:%M:%S %z" FSMODEMMAP = { "w": api.O_WRITE, "x": api.O_EXCL, "a": api.O_APPEND, - "r": api.O_APPEND, # pCloud does not have a read mode + "r": api.O_WRITE, # pCloud does not have a read mode } class PCloudFile(io.RawIOBase): """A file representation for pCloud files""" - def __init__(self, pcloud, path, mode, encoding="utf-8"): - self.pcloud = pcloud + def __init__(self, fs, path, mode, encoding="utf-8", latency=0): + self.fs = fs + self.pcloud = fs.pcloud self.path = path self.mode = Mode(mode) self.encoding = encoding @@ -38,19 +42,23 @@ def __init__(self, pcloud, path, mode, encoding="utf-8"): self._lines = None self._index = 0 self.pos = 0 + # Wait for some time after write operations to ensure they are fully finished on pCloud side + # I figured no way to determine this (without sending additional requests) + self.latency = latency for pyflag, pcloudflag in FSMODEMMAP.items(): if pyflag in mode: - flags = pcloudflag + self.flags = pcloudflag break else: raise api.InvalidFileModeError # Python and PyFS will create a file, which doesn't exist # but pCloud does not. - if not self.pcloud.file_exists(path=self.path): - resp = self.pcloud.file_open(path=self.path, flags=api.O_CREAT) - self.pcloud.file_close(fd=resp["fd"]) - resp = self.pcloud.file_open(path=self.path, flags=flags) + with self._lock: + if not self.pcloud.file_exists(path=self.path): + resp = self.pcloud.file_open(path=self.path, flags=api.O_CREAT) + self.pcloud.file_close(fd=resp["fd"]) + resp = self.pcloud.file_open(path=self.path, flags=self.flags) result = resp.get("result") if result == 0: self.fd = resp["fd"] @@ -108,7 +116,7 @@ def read(self, size=-1): def _close_and_reopen(self): self.pcloud.file_close(fd=self.fd) - resp = self.pcloud.file_open(path=self.path, flags=api.O_APPEND) + resp = self.pcloud.file_open(path=self.path, flags=self.flags) result = resp.get("result") if result == 0: self.fd = resp["fd"] @@ -120,6 +128,7 @@ def truncate(self, size=None): self.pcloud.file_truncate(fd=self.fd, length=size) # file gets truncated on close self._close_and_reopen() + # time.sleep(self.latency) return size def write(self, b): @@ -131,11 +140,11 @@ def write(self, b): result = self.pcloud.file_write(fd=self.fd, data=b) sent_size = result["bytes"] self.pos += sent_size - self._close_and_reopen() + # time.sleep(self.latency) return sent_size def writelines(self, lines): - self.write(b"".join(lines)) + return self.write(b"".join(lines)) def readline(self): result = b"" @@ -359,7 +368,7 @@ def openbin(self, path, mode="r", buffering=-1, **options): raise errors.FileExpected(path) if _mode.exclusive: raise errors.FileExists(path) - pcloud_file = PCloudFile(self.pcloud, _path, _mode.to_platform_bin()) + pcloud_file = PCloudFile(self, _path, _mode.to_platform_bin()) return pcloud_file def remove(self, path):