Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade ci #49

Merged
merged 7 commits into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-python@v1
- uses: actions/setup-python@v5
with:
python-version: 3.8
- name: Install Flake8
Expand All @@ -36,7 +36,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v1
- uses: actions/setup-python@v5
with:
python-version: 3.8
- name: Install requirements
Expand All @@ -49,27 +49,19 @@ jobs:
run: |
LOCAL_IRODS=0 NETWORK=0 pytest -v tests/biomaj_tests.py

pkg_build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Check that the package build works
run: |
pip install -U pip setuptools build
python -m build --sdist --wheel --outdir dist/ .

pypi:
runs-on: ubuntu-latest
needs: [lint, test, pkg_build]
needs: [lint, test]
name: Deploy release to Pypi
environment:
name: release
url: https://pypi.org/p/biomaj-download
permissions:
id-token: write
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-python@v1
- uses: actions/setup-python@v5
with:
python-version: 3.8
- name: Python install
Expand All @@ -78,6 +70,6 @@ jobs:
run: python -m build --sdist --wheel --outdir dist/ .
- name: Publish distribution 📦 to PyPI
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@master
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.pypi_password }}
61 changes: 60 additions & 1 deletion biomaj_download/download/curl.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import re
import requests
from datetime import datetime
import hashlib
import time
Expand Down Expand Up @@ -298,8 +299,61 @@ def _append_file_to_download(self, rfile):
rfile['url'] = self.url
if 'root' not in rfile or not rfile['root']:
rfile['root'] = self.rootdir

if rfile.get('modified_size'):
# Size parsed is inacurate. Try to get a more accurate size with a HEAD query
try:
self.logger.debug('Trying to get a more accurate size for ' + rfile['name'])
head_size = self._estimate_size(rfile)
if head_size:
rfile['size'] = head_size
except Exception as e:
self.logger.error('Exception while trying to get a more accurate size for ' + rfile['name'] + ' - ' + str(e))

super(CurlDownload, self)._append_file_to_download(rfile)

def _estimate_size(self, rfile):
# Cannot reuse _file_url, since we did not cleanup the name yet
# Mostly pasted for the same stuff in direct download
name = re.sub('//+', '/', rfile['name'])
url = self.url + '/' + rfile['root'] + name
url_elts = url.split('://')
if len(url_elts) == 2:
url_elts[1] = re.sub("/{2,}", "/", url_elts[1])
full_url = '://'.join(url_elts)
else:
full_url = re.sub("/{2,}", "/", url)

return self._head_size_call(full_url)

def _head_size_call(self, full_url):
# Now do a HEAD call on this url

auth = ()
proxies = {}

if self.credentials is not None:
auth = tuple(self.credentials.split(":"))

if self.proxy is not None:
proxy = self.proxy
if not self.proxy.startswith("http"):
proxy = 'http://' + self.proxy
if self.proxy_auth is not None:
# Don't really want to manage properly the various schemes
proxy.replace('http://', 'http://{}@'.format(self.proxy_auth))
proxy.replace('https://', 'https://{}@'.format(self.proxy_auth))
proxies['http'] = proxy
proxies['https'] = proxy

try:
size_response = requests.head(full_url, allow_redirects=True, auth=auth, proxies=proxies)
size = int(size_response.headers.get('content-length', 0))
return size

except Exception:
return 0

def _file_url(self, rfile):
# rfile['root'] is set to self.rootdir if needed but may be different.
# We don't use os.path.join because rfile['name'] may starts with /
Expand Down Expand Up @@ -519,7 +573,12 @@ def _http_parse_result(self, result):
rfile['group'] = ''
rfile['user'] = ''
if self.http_parse.file_size != -1:
rfile['size'] = humanfriendly.parse_size(foundfile[self.http_parse.file_size - 1])
size = humanfriendly.parse_size(foundfile[self.http_parse.file_size - 1])
if not str(size) == foundfile[self.http_parse.file_size - 1]:
# This is an approximation of the real size (conversion to byte)
# We will check later (in match()) if we can get a more accurate size
rfile['modified_size'] = True
rfile['size'] = size
else:
rfile['size'] = 0
if self.http_parse.file_date != -1:
Expand Down