From 660a03cdbe3edcb1dda60e66523b23c859a0a7f8 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 30 Dec 2019 13:33:26 +0200 Subject: [PATCH 1/5] change location of solc binaries to ~/.solcx --- solcx/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solcx/install.py b/solcx/install.py index 9e54e78..134f61f 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -48,7 +48,7 @@ def _get_platform(): def get_solc_folder(): - path = Path(__file__).parent.joinpath('bin') + path = Path.home().joinpath('.solcx') path.mkdir(exist_ok=True) return path From 772566285b31c3938bb1dfaba3bcdf713cb17e2d Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 30 Dec 2019 20:59:35 +0200 Subject: [PATCH 2/5] add process lock for solc installations --- solcx/install.py | 43 +++++++++++++++++--------- solcx/utils/lock.py | 74 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 solcx/utils/lock.py diff --git a/solcx/install.py b/solcx/install.py index 134f61f..d2d0b83 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -21,6 +21,9 @@ DownloadError, SolcNotInstalled, ) +from .utils.lock import ( + get_process_lock +) DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/{}/{}" ALL_RELEASES = "https://api.github.com/repos/ethereum/solidity/releases?per_page=100" @@ -197,21 +200,31 @@ def get_installed_solc_versions(): def install_solc(version, allow_osx=False): version = _check_version(version) - platform = _get_platform() - if platform == 'linux': - _install_solc_linux(version) - elif platform == 'darwin': - _install_solc_osx(version, allow_osx) - elif platform == 'win32': - _install_solc_windows(version) - binary_path = get_executable(version) - _check_subprocess_call( - [binary_path, '--version'], - message="Checking installed executable version @ {}".format(binary_path) - ) - if not solc_version: - set_solc_version(version) - LOGGER.info("solc {} successfully installed at: {}".format(version, binary_path)) + lock = get_process_lock(version) + if not lock.acquire(False): + lock.wait() + if not _check_for_installed_version(version): + return + return install_solc(version, allow_osx) + + try: + platform = _get_platform() + if platform == 'linux': + _install_solc_linux(version) + elif platform == 'darwin': + _install_solc_osx(version, allow_osx) + elif platform == 'win32': + _install_solc_windows(version) + binary_path = get_executable(version) + _check_subprocess_call( + [binary_path, '--version'], + message="Checking installed executable version @ {}".format(binary_path) + ) + if not solc_version: + set_solc_version(version) + LOGGER.info("solc {} successfully installed at: {}".format(version, binary_path)) + finally: + lock.release() def _check_version(version): diff --git a/solcx/utils/lock.py b/solcx/utils/lock.py new file mode 100644 index 0000000..0513a5b --- /dev/null +++ b/solcx/utils/lock.py @@ -0,0 +1,74 @@ +import os +from pathlib import Path +import sys +import tempfile +import threading + +if sys.platform == "win32": + import msvcrt + OPEN_MODE = os.O_RDWR | os.O_CREAT | os.O_TRUNC +else: + import fcntl + NON_BLOCKING = fcntl.LOCK_EX | fcntl.LOCK_NB + BLOCKING = fcntl.LOCK_EX + +_locks = {} +_base_lock = threading.Lock() + + +def get_process_lock(lock_id): + with _base_lock: + if lock_id not in _locks: + if sys.platform == "win32": + _locks[lock_id] = WindowsLock(lock_id) + else: + _locks[lock_id] = UnixLock(lock_id) + return _locks[lock_id] + + +class _ProcessLock: + + def __init__(self, lock_id): + self._lock = threading.Lock() + self._lock_path = Path(tempfile.gettempdir()).joinpath(f'.solcx-lock-{lock_id}') + self._lock_file = self._lock_path.open('w') + + def wait(self): + self.acquire(True) + self.release() + + +class UnixLock(_ProcessLock): + + def acquire(self, blocking): + if not self._lock.acquire(blocking): + return False + try: + fcntl.flock(self._lock_file, BLOCKING if blocking else NON_BLOCKING) + except BlockingIOError: + return False + return True + + def release(self): + fcntl.flock(self._lock_file, fcntl.LOCK_UN) + self._lock.release() + + +class WindowsLock(_ProcessLock): + + def acquire(self, blocking): + fd = os.open(self._lock_path, OPEN_MODE) + if not self._lock.acquire(blocking): + return False + while True: + try: + msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) + return True + except OSError: + if not blocking: + return False + + def release(self): + fd = os.open(self._lock_path, OPEN_MODE) + msvcrt.locking(fd, msvcrt.LK_UNLCK) + self._lock.release() From dd97a677b9bf68b131a05dcfa684cc1d798bde8a Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 30 Dec 2019 22:54:09 +0200 Subject: [PATCH 3/5] add test cases for locks --- tests/conftest.py | 12 ++++++++++++ tests/test_locks.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/test_locks.py diff --git a/tests/conftest.py b/tests/conftest.py index 5a583cb..e753f67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import pytest +import shutil import sys import solcx @@ -37,6 +38,17 @@ def all_versions(request): request.applymarker('skip') +# run tests with no installed versions of solc +@pytest.fixture +def nosolc(): + path = solcx.install.get_solc_folder() + temp_path = path.parent.joinpath('.temp') + path.rename(temp_path) + yield + shutil.rmtree(path) + temp_path.rename(path) + + @pytest.fixture() def foo_source(): yield """pragma solidity >=0.4.11; diff --git a/tests/test_locks.py b/tests/test_locks.py new file mode 100644 index 0000000..540d1d8 --- /dev/null +++ b/tests/test_locks.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 + +import threading +import multiprocessing as mp + +import solcx + + +class ThreadWrap: + + def __init__(self, fn, *args, **kwargs): + self.exc = None + self.t = threading.Thread(target=self.wrap, args=(fn,)+args, kwargs=kwargs) + self.t.start() + + def wrap(self, fn, *args, **kwargs): + try: + fn(*args, **kwargs) + except Exception as e: + self.exc = e + + def join(self): + self.t.join() + if self.exc is not None: + raise self.exc + + +def test_threadlock(nosolc): + threads = [ThreadWrap(solcx.install_solc, '0.5.0') for i in range(4)] + for t in threads: + t.join() + + +def test_processlock(): + threads = [mp.Process(target=solcx.install_solc, args=('0.5.0',)) for i in range(4)] + for t in threads: + t.start() + solcx.install_solc('0.5.0') + for t in threads: + t.join() + assert t.exitcode == 0 From 85ac74a3bfcf59d0936af7ef9665adfac3e314e3 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 30 Dec 2019 23:12:23 +0200 Subject: [PATCH 4/5] windows bugfixes --- solcx/install.py | 3 ++- solcx/utils/lock.py | 8 +++++--- tests/test_locks.py | 7 ++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/solcx/install.py b/solcx/install.py index d2d0b83..0c329ce 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -199,7 +199,9 @@ def get_installed_solc_versions(): def install_solc(version, allow_osx=False): + platform = _get_platform() version = _check_version(version) + lock = get_process_lock(version) if not lock.acquire(False): lock.wait() @@ -208,7 +210,6 @@ def install_solc(version, allow_osx=False): return install_solc(version, allow_osx) try: - platform = _get_platform() if platform == 'linux': _install_solc_linux(version) elif platform == 'darwin': diff --git a/solcx/utils/lock.py b/solcx/utils/lock.py index 0513a5b..ad6aa13 100644 --- a/solcx/utils/lock.py +++ b/solcx/utils/lock.py @@ -46,6 +46,7 @@ def acquire(self, blocking): try: fcntl.flock(self._lock_file, BLOCKING if blocking else NON_BLOCKING) except BlockingIOError: + self._lock.release() return False return True @@ -57,18 +58,19 @@ def release(self): class WindowsLock(_ProcessLock): def acquire(self, blocking): - fd = os.open(self._lock_path, OPEN_MODE) if not self._lock.acquire(blocking): return False while True: try: + fd = os.open(self._lock_path, OPEN_MODE) msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) + self._fd = fd return True except OSError: if not blocking: + self._lock.release() return False def release(self): - fd = os.open(self._lock_path, OPEN_MODE) - msvcrt.locking(fd, msvcrt.LK_UNLCK) + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) self._lock.release() diff --git a/tests/test_locks.py b/tests/test_locks.py index 540d1d8..c2b5274 100644 --- a/tests/test_locks.py +++ b/tests/test_locks.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import threading import multiprocessing as mp +import threading import solcx @@ -31,8 +31,9 @@ def test_threadlock(nosolc): t.join() -def test_processlock(): - threads = [mp.Process(target=solcx.install_solc, args=('0.5.0',)) for i in range(4)] +def test_processlock(nosolc): + mp.set_start_method('spawn') + threads = [mp.Process(target=solcx.install_solc, args=('0.5.0',),) for i in range(4)] for t in threads: t.start() solcx.install_solc('0.5.0') From f6db59abd1e066e5f9f39b7aeb667079a974128f Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 31 Dec 2019 01:07:21 +0200 Subject: [PATCH 5/5] update changelog --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 41e627b..540cd1f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +0.7.0 (unreleased) +----- + + - Store solc binaries at $HOME/.solcx + - Add locks for thread and multiprocessing safety + 0.6.1 -----