Skip to content

Commit

Permalink
Bundle lz4 (#50)
Browse files Browse the repository at this point in the history
* compile & bundle lz4

* ci

* ci

* ci

* ci

* ci

* cleanup

* black
  • Loading branch information
normanrz authored Dec 2, 2019
1 parent 646b478 commit 3b439a6
Show file tree
Hide file tree
Showing 7 changed files with 53 additions and 14 deletions.
32 changes: 28 additions & 4 deletions .github/workflows/python-module.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,19 @@ jobs:
- name: Build testing docker container
run: |
docker build -f python/Dockerfile -t testing .
- name: Compile lz4
run: |
docker run \
-v$(pwd):/app \
-w/app \
testing \
bash -c "
curl -L https://github.com/lz4/lz4/archive/v1.9.2.tar.gz -o lz4.tar.gz
tar -xzvf lz4.tar.gz
rm -rf lz4
mv lz4-1.9.2/ lz4
cd lz4
make allmost"
- name: Build
run: |
docker run \
Expand Down Expand Up @@ -69,15 +82,25 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
architecture: 'x64' # (x64 or x86)
- name: Install Rust and lz4
- name: Install Rust
run: |
brew install rustup-init lz4
brew install rustup-init
rustup-init -y
- name: Compile lz4
run: |
brew uninstall --ignore-dependencies lz4
curl -L https://github.com/lz4/lz4/archive/v1.9.2.tar.gz -o lz4.tar.gz
tar -xzvf lz4.tar.gz
rm -rf lz4
mv lz4-1.9.2/ lz4
cd lz4
make allmost
- name: Build
run: |
cd python
export PATH=$PATH:$HOME/.cargo/bin
python setup.py install
install_name_tool -change /usr/local/lib/liblz4.1.dylib @loader_path/liblz4.1.dylib wkw/lib/libwkw.dylib
- name: Test
run: |
cd python
Expand Down Expand Up @@ -116,8 +139,9 @@ jobs:
cd lz4
curl -L https://github.com/lz4/lz4/releases/download/v1.8.2/lz4_v1_8_2_win64.zip -o lz4_v1_8_2_win64.zip
unzip lz4_v1_8_2_win64.zip
mv dll/liblz4.so.1.8.2.dll ../c/liblz4.dll
mv dll/liblz4.lib ../c/liblz4.lib
mkdir lib
mv dll/liblz4.so.1.8.2.dll lib/liblz4.dll
mv dll/liblz4.lib lib/liblz4.lib
- name: Build
shell: bash
run: |
Expand Down
3 changes: 1 addition & 2 deletions python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \
PATH=/usr/local/cargo/bin:$PATH \
PYBIN=/opt/python/cp36-cp36m/bin

RUN yum install lz4 lz4-devel -y; \
curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path -y; \
RUN curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path -y; \
chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
rustup --version; \
cargo --version; \
Expand Down
2 changes: 2 additions & 0 deletions python/MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
include wkw/lib/wkw.h
include wkw/lib/libwkw.so
include wkw/lib/liblz4.so.1
include wkw/lib/libwkw.dylib
include wkw/lib/liblz4.1.dylib
include wkw/lib/wkw.dll
include wkw/lib/liblz4.dll
13 changes: 9 additions & 4 deletions python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ def __build_c_library(self):
subprocess.call(["cargo", "clean"], cwd=c_dir)
subprocess.call(["cargo", "build", "--release"], cwd=c_dir)

lz4_dir = os.path.normpath(os.path.join(this_dir, "..", "lz4", "lib"))
lz4_name_platform = {
"Linux": "liblz4.so.1",
"Windows": "liblz4.dll",
"Darwin": "liblz4.1.dylib",
}
lz4_name = lz4_name_platform[platform.system()]

lib_name_platform = {
"Linux": "libwkw.so",
"Windows": "wkw.dll",
Expand All @@ -34,10 +42,7 @@ def __build_c_library(self):
shutil.rmtree(lib_dir)

os.makedirs(lib_dir)
if platform.system() == "Windows":
shutil.copy(
os.path.join(c_dir, "liblz4.dll"), os.path.join(lib_dir, "liblz4.dll")
)
shutil.copy(os.path.join(lz4_dir, lz4_name), os.path.join(lib_dir, lz4_name))
shutil.copy(lib_file, os.path.join(lib_dir, lib_name))
shutil.copy(header_file, os.path.join(lib_dir, "wkw.h"))

Expand Down
13 changes: 9 additions & 4 deletions python/wkw/wkw.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import cffi
import ctypes
import numpy as np
import cffi
import platform
import os
from copy import deepcopy
Expand All @@ -11,6 +11,13 @@ def _init_libwkw():
this_dir = os.path.dirname(__file__)
path_wkw_header = os.path.join(this_dir, "lib", "wkw.h")

lz4_name_platform = {
"Linux": "liblz4.so.1",
"Windows": "liblz4.dll",
"Darwin": "liblz4.1.dylib",
}
path_lz4_lib = os.path.join(this_dir, "lib", lz4_name_platform[platform.system()])

lib_name_platform = {
"Linux": "libwkw.so",
"Windows": "wkw.dll",
Expand All @@ -25,11 +32,9 @@ def _init_libwkw():
wkw_header = filter(lambda l: not l.startswith("#"), wkw_header)
wkw_header = "\n".join(wkw_header)

if platform.system() == "Windows":
os.environ["PATH"] += os.pathsep + os.path.join(this_dir, "lib")

ffi = cffi.FFI()
ffi.cdef(wkw_header)
ffi.dlopen(path_lz4_lib)
libwkw = ffi.dlopen(path_wkw_lib)

return (ffi, libwkw)
Expand Down
1 change: 1 addition & 0 deletions rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ authors = ["Alessandro Motta <[email protected]>"]
description = "webKNOSSOS wrapper is a file format designed for large-scale, three-dimensional voxel datasets. It was optimized for high-speed access to data subvolumes, and supports multi-channel data and dataset compression."
repository = "https://github.com/scalableminds/webknossos-wrap"
license = "AGPL-3.0"
build = "build.rs"

[badges]
circle-ci = { repository = "scalableminds/webknossos", branch = "master" }
Expand Down
3 changes: 3 additions & 0 deletions rust/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
fn main() {
println!("cargo:rustc-link-search={}", "../lz4/lib");
}

0 comments on commit 3b439a6

Please sign in to comment.