forked from h2oai/h2ogpt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
116 lines (99 loc) · 4.65 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
all: clean dist
PACKAGE_VERSION := `cat version.txt | tr -d '\n'`
BUILD_TAG := $(shell git describe --always --dirty)
DOCKER_TEST_IMAGE := harbor.h2o.ai/h2ogpt/test-image:$(BUILD_TAG)
PYTHON_BINARY ?= `which python`
DEFAULT_MARKERS ?= "not need_tokens and not need_gpu"
.PHONY: venv dist test publish docker_build build_info.txt
clean:
rm -rf dist build h2ogpt.egg-info
venv:
$(PYTHON_BINARY) -m virtualenv -p $(PYTHON_BINARY) venv
install:
$(PYTHON_BINARY) -m pip install dist/h2ogpt-$(PACKAGE_VERSION)-py3-none-any.whl
install-%:
$(PYTHON_BINARY) -m pip install dist/h2ogpt-$(PACKAGE_VERSION)-py3-none-any.whl[$*]
dist:
$(PYTHON_BINARY) setup.py bdist_wheel
test:
$(PYTHON_BINARY) -m pip install requirements-parser
$(PYTHON_BINARY) -m pytest tests --disable-warnings --junit-xml=test_report.xml -m "$(DEFAULT_MARKERS)"
test_imports:
$(PYTHON_BINARY) -m pytest tests/test_imports.py --disable-warnings --junit-xml=test_report.xml -m "$(DEFAULT_MARKERS)"
publish:
echo "Publishing not implemented yet."
build_info.txt:
@rm -rf build_info.txt
@echo "commit=\"$(shell git rev-parse HEAD)\"" >> $@
@echo "branch=\"`git rev-parse HEAD | git branch -a --contains | grep -v detached | sed -e 's~remotes/origin/~~g' -e 's~^ *~~' | sort | uniq | tr '*\n' ' '`\"" >> $@
@echo "describe=\"`git describe --always --dirty`\"" >> $@
@echo "build_os=\"`uname -a`\"" >> $@
@echo "build_machine=\"`hostname`\"" >> $@
@echo "build_date=\"$(shell date "+%Y%m%d")\"" >> $@
@echo "build_user=\"`id -u -n`\"" >> $@
@echo "base_version=\"$(PACKAGE_VERSION)\"" >> $@
git_hash.txt:
@echo "$(shell git rev-parse HEAD)" >> $@
# Deprecated for now, no 0.4.1 on pypi, use release binary wheel that has no CUDA errors anymore
docker_build_deps:
@cp docker_build_script_ubuntu.sh docker_build_script_ubuntu.sh.back
@sed -i '/# Install prebuilt dependencies/,$$d' docker_build_script_ubuntu.sh
@docker build -t h2ogpt-deps-builder -f Dockerfile .
@mv docker_build_script_ubuntu.sh.back docker_build_script_ubuntu.sh
@mkdir -p prebuilt_deps
@docker run \
--rm \
-it \
--entrypoint bash \
--runtime nvidia \
-v `pwd`:/dot \
-v /etc/passwd:/etc/passwd:ro \
-v /etc/group:/etc/group:ro \
-u `id -u`:`id -g` \
h2ogpt-deps-builder -c " \
mkdir -p /dot/prebuilt_deps && cd /dot/prebuilt_deps && \
GITHUB_ACTIONS=true python3.10 -m pip install auto-gptq==0.4.2 --no-cache-dir --use-deprecated=legacy-resolver && \
python3.10 -m pip wheel auto-gptq==0.4.2 \
"
@docker run \
--rm \
-it \
--entrypoint bash \
-v `pwd`:/dot \
quay.io/pypa/manylinux2014_x86_64 -c " \
ln -s /usr/local/bin/python3.10 /usr/local/bin/python3 && cd /tmp && \
git clone https://github.com/h2oai/duckdb.git && \
cd duckdb && \
git checkout dcd8c1ffc53dd020623630efb99ba6a3a4cbc5ad && \
BUILD_PYTHON=1 make release && \
cd tools/pythonpkg && \
python3.10 setup.py bdist_wheel && \
cp dist/duckdb-0.*.whl /dot/prebuilt_deps \
"
s3cmd put prebuilt_deps/auto_gptq-0.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl s3://artifacts.h2o.ai/deps/h2ogpt/ && \
s3cmd setacl s3://artifacts.h2o.ai/deps/h2ogpt/auto_gptq-0.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl --acl-public
s3cmd put prebuilt_deps/duckdb-0.8.2.dev4026+gdcd8c1ffc5-cp310-cp310-linux_x86_64.whl s3://artifacts.h2o.ai/deps/h2ogpt/ && \
s3cmd setacl s3://artifacts.h2o.ai/deps/h2ogpt/duckdb-0.8.2.dev4026+gdcd8c1ffc5-cp310-cp310-linux_x86_64.whl --acl-public
docker_build: build_info.txt
ifeq ($(shell curl --connect-timeout 4 --write-out %{http_code} -sS --output /dev/null -X GET http://harbor.h2o.ai/api/v2.0/projects/h2ogpt/repositories/test-image/artifacts/$(BUILD_TAG)/tags),200)
@echo "Image already pushed to Harbor: $(DOCKER_TEST_IMAGE)"
else
DOCKER_BUILDKIT=1 docker build -t $(DOCKER_TEST_IMAGE) -f Dockerfile .
docker push $(DOCKER_TEST_IMAGE)
endif
just_docker_build: build_info.txt
DOCKER_BUILDKIT=1 docker build -t $(DOCKER_TEST_IMAGE) -f Dockerfile .
docker_build_runner: docker_build
-docker pull $(DOCKER_TEST_IMAGE)
docker tag $(DOCKER_TEST_IMAGE) gcr.io/vorvan/h2oai/h2ogpt-runtime:$(BUILD_TAG)
docker tag $(DOCKER_TEST_IMAGE) gcr.io/vorvan/h2oai/h2ogpt-runtime:$(PACKAGE_VERSION)
docker tag $(DOCKER_TEST_IMAGE) gcr.io/vorvan/h2oai/h2ogpt-runtime:latest
docker push gcr.io/vorvan/h2oai/h2ogpt-runtime:$(BUILD_TAG)
docker push gcr.io/vorvan/h2oai/h2ogpt-runtime:$(PACKAGE_VERSION)
docker push gcr.io/vorvan/h2oai/h2ogpt-runtime:latest
ifdef BUILD_ID
docker tag $(DOCKER_TEST_IMAGE) gcr.io/vorvan/h2oai/h2ogpt-runtime:$(PACKAGE_VERSION)-$(BUILD_ID)
docker push gcr.io/vorvan/h2oai/h2ogpt-runtime:$(PACKAGE_VERSION)-$(BUILD_ID)
endif
print-%:
@echo $($*)