Hybrid Sim Amend/Revision (#1070)

* changes for amends * version changes * Update pipeline-amend-citation.component.js fix typo * Update pipeline-amend.component.js * Update pipeline-version-changes.component.js * Update pipeline-version.component.js * Update pipeline-version-changes.component.js * add hybrid sim mapping and file tag support for Fedora * use new Fedora operations in the publication pipeline. * fix bug when constructing coordinator paths. --------- Co-authored-by: Jake Rosenberg <[email protected]> Co-authored-by: Jake Rosenberg <[email protected]> Co-authored-by: Jake Rosenberg <[email protected]>
DesignSafe-CI · Sep 18, 2023 · f56ac5b · f56ac5b
1 parent d29e814
commit f56ac5b
Show file tree

Hide file tree

Showing 21 changed files with 1,764 additions and 59 deletions.
diff --git a/designsafe/apps/api/tasks.py b/designsafe/apps/api/tasks.py
@@ -670,6 +670,8 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi
  from designsafe.apps.projects.managers import publication as PublicationManager
  from designsafe.libs.fedora.fedora_operations import amend_project_fedora, ingest_project_experimental
  from designsafe.libs.fedora.sim_operations import ingest_project_sim
+ from designsafe.libs.fedora.fr_operations import ingest_project_fr
+ from designsafe.libs.fedora.hyb_sim_operations import ingest_project_hyb_sim
  try:
  amended_pub = PublicationManager.amend_publication(project_id, amendments, authors, revision)
  PublicationManager.amend_datacite_doi(amended_pub)
@@ -680,6 +682,10 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi
  ingest_project_experimental(project_id, version=revision, amend=True)
  if amended_pub.project.value.projectType == 'simulation':
  ingest_project_sim(project_id, version=revision, amend=True)
+ if amended_pub.project.value.projectType == 'hybrid_simulation':
+ ingest_project_hyb_sim(project_id, version=revision, amend=True)
+ if amended_pub.project.value.projectType == 'field_recon':
+ ingest_project_fr(project_id, version=revision, amend=True)
  except Exception as exc:
  logger.error('Proj Id: %s. %s', project_id, exc, exc_info=True)
  raise self.retry(exc=exc)
@@ -780,41 +786,14 @@ def save_to_fedora(self, project_id, revision=None):
  from designsafe.libs.fedora.sim_operations import ingest_project_sim
  ingest_project_sim(project_id, version=revision)
  return
-
- _root = os.path.join('/corral-repl/tacc/NHERI/published', project_id)
- fedora_base = 'http://fedoraweb01.tacc.utexas.edu:8080/fcrepo/rest/publications_01'
- res = requests.get(fedora_base)
- if res.status_code == 404 or res.status_code == 410:
- requests.put(fedora_base)
-
- fedora_project_base = ''.join([fedora_base, '/', project_id])
- res = requests.get(fedora_project_base)
- if res.status_code == 404 or res.status_code == 410:
- requests.put(fedora_project_base)
-
- headers = {'Content-Type': 'text/plain'}
- #logger.debug('walking: %s', _root)
- for root, dirs, files in os.walk(_root):
- for name in files:
- mime = magic.Magic(mime=True)
- headers['Content-Type'] = mime.from_file(os.path.join(root, name))
- #files
- full_path = os.path.join(root, name)
- _path = full_path.replace(_root, '', 1)
- _path = _path.replace('[', '-')
- _path = _path.replace(']', '-')
- url = ''.join([fedora_project_base, urllib.parse.quote(_path)])
- #logger.debug('uploading: %s', url)
- with open(os.path.join(root, name), 'rb') as _file:
- requests.put(url, data=_file, headers=headers)
-
- for name in dirs:
- #dirs
- full_path = os.path.join(root, name)
- _path = full_path.replace(_root, '', 1)
- url = ''.join([fedora_project_base, _path])
- #logger.debug('creating: %s', _path)
- requests.put(url)
+ if pub.project.value.projectType == 'hybrid_simulation':
+ from designsafe.libs.fedora.hyb_sim_operations import ingest_project_hyb_sim
+ ingest_project_hyb_sim(project_id, version=revision)
+ return
+ if pub.project.value.projectType == 'field_recon':
+ from designsafe.libs.fedora.fr_operations import ingest_project_fr
+ ingest_project_fr(project_id, version=revision)
+ return
 
  except Exception as exc:
  logger.error('Proj Id: %s. %s', project_id, exc)

diff --git a/designsafe/apps/projects/managers/publication.py b/designsafe/apps/projects/managers/publication.py
@@ -252,10 +252,9 @@ def draft_publication(
 def amend_publication(project_id, amendments=None, authors=None, revision=None):
  """Amend a Publication
  
- Update Amendable fields on a publication and the corrosponding DataCite
+ Update Amendable fields on a publication and the corresponding DataCite
  records. These changes do not produce a new version of a publication, but
- they do allow for limited changes to a published project. This is currently
- configured to support "Other" publications only.
+ they do allow for limited changes to a published project. 
  
  :param str project_id: Project uuid to amend
  :param int revision: Revision number to amend

diff --git a/designsafe/libs/fedora/fedora_operations.py b/designsafe/libs/fedora/fedora_operations.py
@@ -114,6 +114,9 @@
  },
  "influenced": {
  "@id": "https://www.w3.org/TR/2013/REC-prov-o-20130430/#influenced"
+ },
+ "Influence": {
+ "@id": "https://www.w3.org/TR/2013/REC-prov-o-20130430/#Influence"
  }
 }
 
@@ -297,19 +300,26 @@ def format_metadata_for_fedora(project_id, version=None):
 
 
 def generate_manifest_other(project_id, version=None):
+ fido_client = Fido()
  doc = IndexedPublication.from_id(project_id, revision=version)
  uuid = doc.project.uuid
+ file_tags = getattr(doc.project.value, 'fileTags', [])
 
  if version:
  project_id = '{}v{}'.format(project_id, str(version))
  manifest = []
  archive_path = os.path.join(PUBLICATIONS_MOUNT_ROOT, project_id)
 
  for path in get_child_paths(archive_path):
+ rel_path = os.path.relpath(path, archive_path)
+
+ tags = filter(lambda t: t['path'].strip('/') == rel_path.strip('/'), file_tags)
  manifest.append({
  'parent_entity': uuid,
  'corral_path': path,
- 'checksum': get_sha1_hash(path)
+ 'checksum': get_sha1_hash(path),
+ 'tags': [t['tagName'] for t in tags],
+ 'ffi': get_fido_output(fido_client, path)
  })
 
  return manifest
@@ -411,7 +421,8 @@ def walk_experimental(project_id, version=None):
  'uuid': doc.project.uuid,
  'container_path': project_id,
  'fedora_mapping': {**project_meta, 'generated': [], 'license': None},
- 'fileObjs': []
+ 'fileObjs': [],
+ 'fileTags': []
  }
 
  experiments_list = doc.experimentsList
@@ -426,7 +437,8 @@ def walk_experimental(project_id, version=None):
  'uuid': expt.uuid,
  'container_path': expt_container_path,
  'fedora_mapping': {**format_experiment(expt), 'license': license, 'wasGeneratedBy': project_id, 'generated': []},
- 'fileObjs': expt.fileObjs
+ 'fileObjs': expt.fileObjs,
+ 'fileTags': getattr(expt.value, 'fileTags', []),
  }
 
  full_author_list += experiment_map['fedora_mapping']['creator']
@@ -443,6 +455,7 @@ def walk_experimental(project_id, version=None):
  report_map = {
  'uuid': report.uuid,
  'fileObjs': report.fileObjs,
+ 'fileTags': getattr(report.value, 'fileTags', []),
  'container_path': report_container_path,
  'fedora_mapping': {**format_report(report), 'wasGeneratedBy': 'Experiment: {}'.format(exp_doi)}
  }
@@ -460,6 +473,7 @@ def walk_experimental(project_id, version=None):
  analysis_map = {
  'uuid': analysis.uuid,
  'fileObjs': analysis.fileObjs,
+ 'fileTags': getattr(analysis.value, 'fileTags', []),
  'container_path': analysis_container_path,
  'fedora_mapping': {**format_analysis(analysis), 'wasGeneratedBy': 'Experiment: {}'.format(exp_doi)}
 
@@ -478,6 +492,7 @@ def walk_experimental(project_id, version=None):
  mc_map = {
  'uuid': mc.uuid,
  'fileObjs': mc.fileObjs,
+ 'fileTags': getattr(mc.value, 'fileTags', []),
  'container_path': configs_container_path,
  'fedora_mapping': {**format_model_config(mc), 'wasGeneratedBy': exp_doi}
  }
@@ -494,6 +509,7 @@ def walk_experimental(project_id, version=None):
  sl_map = {
  'uuid': sl.uuid,
  'fileObjs': sl.fileObjs,
+ 'fileTags': getattr(sl.value, 'fileTags', []),
  'container_path': sl_container_path,
  'fedora_mapping': {**format_sensor_info(sl),
  'wasGeneratedBy': 'Experiment: {}'.format(exp_doi),
@@ -514,6 +530,7 @@ def walk_experimental(project_id, version=None):
  event_map = {
  'uuid': event.uuid,
  'fileObjs': event.fileObjs,
+ 'fileTags': getattr(event.value, 'fileTags', []),
  'container_path': evt_container_path,
  'fedora_mapping': {**format_event(event),
  'wasGeneratedBy': 'Experiment: {}'.format(exp_doi),
@@ -667,27 +684,36 @@ def generate_manifest(walk_result, project_id, version=None):
  archive_path = os.path.join(PUBLICATIONS_MOUNT_ROOT, project_id)
  for entity in walk_result:
  file_objs = entity['fileObjs']
+ file_tags = entity.get('fileTags', [])
  for file in file_objs:
 
+
+
  file_path = os.path.join(archive_path, file['path'].strip('/'))
  rel_path = os.path.join(parse.unquote(entity['container_path']), file['path'].strip('/'))
 
  if file['type'] == 'dir':
  for path in get_child_paths(file_path):
+ tag_path = os.path.relpath(path, archive_path)
+ tags = filter(lambda t: t['path'].strip('/') == tag_path.strip('/'), file_tags)
+
  manifest.append({
  'parent_entity': entity['uuid'],
  'corral_path': path,
  'project_path': path.replace(file_path, rel_path, 1),
  'checksum': get_sha1_hash(path),
+ 'tags': [t['tagName'] for t in tags],
  'ffi': get_fido_output(fido_client, path)
  })
 
  else:
+ tags = filter(lambda t: t['path'] == file['path'], file_tags)
  manifest.append({
  'parent_entity': entity['uuid'],
  'corral_path': file_path,
  'project_path': rel_path,
  'checksum': get_sha1_hash(file_path),
+ 'tags': [t['tagName'] for t in tags],
  'ffi': get_fido_output(fido_client, file_path)
  })
 

diff --git a/designsafe/libs/fedora/fr_operations.py b/designsafe/libs/fedora/fr_operations.py
@@ -45,7 +45,8 @@ def walk_fr(project_id, version=None):
  'uuid': doc.project.uuid,
  'container_path': project_id,
  'fedora_mapping': {**project_meta, 'generated': [], 'license': None},
- 'fileObjs': []
+ 'fileObjs': [],
+ 'fileTags': []
  }
 
  docs_list = getattr(doc, 'reports', [])
@@ -60,7 +61,8 @@ def walk_fr(project_id, version=None):
  'uuid': document.uuid,
  'container_path': doc_container_path,
  'fedora_mapping': {**format_docs(document), 'license': license, 'wasGeneratedBy': project_id, 'generated': [], 'hasVersion': version},
- 'fileObjs': document.fileObjs
+ 'fileObjs': document.fileObjs,
+ 'fileTags': getattr(document.value, 'fileTags', []),
  }
 
  relation_map.append(doc_map)
@@ -77,7 +79,8 @@ def walk_fr(project_id, version=None):
  'uuid': mission.uuid,
  'container_path': mission_container_path,
  'fedora_mapping': {**format_mission(mission), 'license': license, 'wasGeneratedBy': project_id, 'generated': [], 'hasVersion': version},
- 'fileObjs': []
+ 'fileObjs': [],
+ 'fileTags': getattr(mission.value, 'fileTags', []),
  }
 
 
@@ -93,6 +96,7 @@ def walk_fr(project_id, version=None):
  collection_map = {
  'uuid': collection.uuid,
  'fileObjs': collection.fileObjs,
+ 'fileTags': getattr(collection.value, 'fileTags', []),
  'container_path': collection_container_path,
  'fedora_mapping': {**format_geo(collection), 'wasGeneratedBy': 'Mission: {}'.format(mission_doi)}
  }
@@ -110,6 +114,7 @@ def walk_fr(project_id, version=None):
  collection_map = {
  'uuid': geo_collection.uuid,
  'fileObjs': geo_collection.fileObjs,
+ 'fileTags': getattr(geo_collection.value, 'fileTags', []),
  'container_path': geo_container_path,
  'fedora_mapping': {**format_geo(geo_collection), 'wasGeneratedBy': 'Mission: {}'.format(mission_doi)}
  }
@@ -127,6 +132,7 @@ def walk_fr(project_id, version=None):
  collection_map = {
  'uuid': soc_collection.uuid,
  'fileObjs': soc_collection.fileObjs,
+ 'fileTags': getattr(soc_collection.value, 'fileTags', []),
  'container_path': soc_container_path,
  'fedora_mapping': {**format_soc(soc_collection), 'wasGeneratedBy': 'Mission: {}'.format(mission_doi)}
  }
@@ -144,6 +150,7 @@ def walk_fr(project_id, version=None):
  collection_map = {
  'uuid': planning_coll.uuid,
  'fileObjs': planning_coll.fileObjs,
+ 'fileTags': getattr(planning_coll.value, 'fileTags', []),
  'container_path': planning_container_path,
  'fedora_mapping': {**format_planning(planning_coll), 'wasGeneratedBy': 'Mission: {}'.format(mission_doi)}
  }