Skip to content

Commit

Permalink
remove SHA check before downloading mmd file
Browse files Browse the repository at this point in the history
- anonymous access to the github API is limited
- we don't want to require authentication for pythesint
- the file is small so the performance difference is negligible
  • Loading branch information
aperrin66 committed Dec 5, 2024
1 parent 58b9299 commit 5372647
Showing 1 changed file with 1 addition and 26 deletions.
27 changes: 1 addition & 26 deletions pythesint/mmd_vocabulary.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,6 @@ def get_element_by_label(self, parent, node_type, label):
return node
return None

@staticmethod
def _get_local_file_github_blob_sha(file_path):
"""Get the local file's blob SHA hash as used by github"""
if os.path.isfile(file_path):
with open(file_path, 'rb') as f_h:
contents = f_h.read()
header_string = f"blob {len(contents)}\0"
blob = bytes(header_string, encoding='utf-8') + contents
return hashlib.sha1(blob).hexdigest()
return None

def _get_remote_file_github_blob_sha(self, version):
"""Get the remote file's blob sha hash from github"""
endpoint = f"https://api.github.com/repos/metno/mmd/git/trees/{version}:thesauri"
result = json.loads(requests.get(endpoint).text)
for file_info in result['tree']:
if file_info['path'] == 'mmd-vocabulary.xml':
return file_info['sha']
return None

def _download_data_file(self, version=None):
"""Download the file containing all the vocabularies
definitions if it is not already present
Expand All @@ -64,12 +44,7 @@ def _download_data_file(self, version=None):
else:
url = self.base_url

# by comparing the hashes, we can check if we need to
# download the file or not
local_sha = self._get_local_file_github_blob_sha(self.base_file)
remote_sha = self._get_remote_file_github_blob_sha(version)

if (not os.path.isfile(self.base_file) or local_sha != remote_sha):
if not os.path.isfile(self.base_file):
with closing(requests.get(url, stream=True)) as response:
with open(self.base_file, 'wb') as f_h:
f_h.write(response.content)
Expand Down

0 comments on commit 5372647

Please sign in to comment.