diff --git a/CHANGELOG b/CHANGELOG index 41e627b..540cd1f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +0.7.0 (unreleased) +----- + + - Store solc binaries at $HOME/.solcx + - Add locks for thread and multiprocessing safety + 0.6.1 ----- diff --git a/solcx/install.py b/solcx/install.py index 9e54e78..0c329ce 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -21,6 +21,9 @@ DownloadError, SolcNotInstalled, ) +from .utils.lock import ( + get_process_lock +) DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/{}/{}" ALL_RELEASES = "https://api.github.com/repos/ethereum/solidity/releases?per_page=100" @@ -48,7 +51,7 @@ def _get_platform(): def get_solc_folder(): - path = Path(__file__).parent.joinpath('bin') + path = Path.home().joinpath('.solcx') path.mkdir(exist_ok=True) return path @@ -196,22 +199,33 @@ def get_installed_solc_versions(): def install_solc(version, allow_osx=False): - version = _check_version(version) platform = _get_platform() - if platform == 'linux': - _install_solc_linux(version) - elif platform == 'darwin': - _install_solc_osx(version, allow_osx) - elif platform == 'win32': - _install_solc_windows(version) - binary_path = get_executable(version) - _check_subprocess_call( - [binary_path, '--version'], - message="Checking installed executable version @ {}".format(binary_path) - ) - if not solc_version: - set_solc_version(version) - LOGGER.info("solc {} successfully installed at: {}".format(version, binary_path)) + version = _check_version(version) + + lock = get_process_lock(version) + if not lock.acquire(False): + lock.wait() + if not _check_for_installed_version(version): + return + return install_solc(version, allow_osx) + + try: + if platform == 'linux': + _install_solc_linux(version) + elif platform == 'darwin': + _install_solc_osx(version, allow_osx) + elif platform == 'win32': + _install_solc_windows(version) + binary_path = get_executable(version) + _check_subprocess_call( + [binary_path, '--version'], + message="Checking installed executable version @ {}".format(binary_path) + ) + if not solc_version: + set_solc_version(version) + LOGGER.info("solc {} successfully installed at: {}".format(version, binary_path)) + finally: + lock.release() def _check_version(version): diff --git a/solcx/utils/lock.py b/solcx/utils/lock.py new file mode 100644 index 0000000..ad6aa13 --- /dev/null +++ b/solcx/utils/lock.py @@ -0,0 +1,76 @@ +import os +from pathlib import Path +import sys +import tempfile +import threading + +if sys.platform == "win32": + import msvcrt + OPEN_MODE = os.O_RDWR | os.O_CREAT | os.O_TRUNC +else: + import fcntl + NON_BLOCKING = fcntl.LOCK_EX | fcntl.LOCK_NB + BLOCKING = fcntl.LOCK_EX + +_locks = {} +_base_lock = threading.Lock() + + +def get_process_lock(lock_id): + with _base_lock: + if lock_id not in _locks: + if sys.platform == "win32": + _locks[lock_id] = WindowsLock(lock_id) + else: + _locks[lock_id] = UnixLock(lock_id) + return _locks[lock_id] + + +class _ProcessLock: + + def __init__(self, lock_id): + self._lock = threading.Lock() + self._lock_path = Path(tempfile.gettempdir()).joinpath(f'.solcx-lock-{lock_id}') + self._lock_file = self._lock_path.open('w') + + def wait(self): + self.acquire(True) + self.release() + + +class UnixLock(_ProcessLock): + + def acquire(self, blocking): + if not self._lock.acquire(blocking): + return False + try: + fcntl.flock(self._lock_file, BLOCKING if blocking else NON_BLOCKING) + except BlockingIOError: + self._lock.release() + return False + return True + + def release(self): + fcntl.flock(self._lock_file, fcntl.LOCK_UN) + self._lock.release() + + +class WindowsLock(_ProcessLock): + + def acquire(self, blocking): + if not self._lock.acquire(blocking): + return False + while True: + try: + fd = os.open(self._lock_path, OPEN_MODE) + msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) + self._fd = fd + return True + except OSError: + if not blocking: + self._lock.release() + return False + + def release(self): + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + self._lock.release() diff --git a/tests/conftest.py b/tests/conftest.py index 5a583cb..e753f67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import pytest +import shutil import sys import solcx @@ -37,6 +38,17 @@ def all_versions(request): request.applymarker('skip') +# run tests with no installed versions of solc +@pytest.fixture +def nosolc(): + path = solcx.install.get_solc_folder() + temp_path = path.parent.joinpath('.temp') + path.rename(temp_path) + yield + shutil.rmtree(path) + temp_path.rename(path) + + @pytest.fixture() def foo_source(): yield """pragma solidity >=0.4.11; diff --git a/tests/test_locks.py b/tests/test_locks.py new file mode 100644 index 0000000..c2b5274 --- /dev/null +++ b/tests/test_locks.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 + +import multiprocessing as mp +import threading + +import solcx + + +class ThreadWrap: + + def __init__(self, fn, *args, **kwargs): + self.exc = None + self.t = threading.Thread(target=self.wrap, args=(fn,)+args, kwargs=kwargs) + self.t.start() + + def wrap(self, fn, *args, **kwargs): + try: + fn(*args, **kwargs) + except Exception as e: + self.exc = e + + def join(self): + self.t.join() + if self.exc is not None: + raise self.exc + + +def test_threadlock(nosolc): + threads = [ThreadWrap(solcx.install_solc, '0.5.0') for i in range(4)] + for t in threads: + t.join() + + +def test_processlock(nosolc): + mp.set_start_method('spawn') + threads = [mp.Process(target=solcx.install_solc, args=('0.5.0',),) for i in range(4)] + for t in threads: + t.start() + solcx.install_solc('0.5.0') + for t in threads: + t.join() + assert t.exitcode == 0