Merge pull request #2887 from CannonLock/SOFTWARE-5443

Create Institution Endpoint (SOFTWARE-5443)

src/app.py

@@ -18,6 +18,7 @@
 
 from webapp import default_config
 from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, cache_control_private
+from webapp.flask_common import create_accepted_response
 from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
 from webapp.models import GlobalData
@@ -264,6 +265,21 @@ def contacts():
  app.log_exception(sys.exc_info())
  return Response("Error getting users", status=503) # well, it's better than crashing
 
+@app.route('/api/institutions')
+def institutions():
+
+ resource_facilities = set(global_data.get_topology().facilities.keys())
+ project_facilities = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project'])
+
+ facilities = project_facilities.union(resource_facilities)
+
+ facility_data = [["Institution Name", "Has Resource(s)", "Has Project(s)"]]
+ for facility in sorted(facilities):
+ facility_data.append([facility, facility in resource_facilities, facility in project_facilities])
+
+ return create_accepted_response(facility_data, request.headers, default="text/csv")
+
+
 
 @app.route('/miscproject/xml')
 def miscproject_xml():

src/tests/test_api.py

@@ -7,6 +7,8 @@
 # Rewrites the path so the app can be imported like it normally is
 import os
 import sys
+import csv
+import io
 
 topdir = os.path.join(os.path.dirname(__file__), "..")
 sys.path.append(topdir)
@@ -57,8 +59,9 @@
  "/origin/Authfile-public",
  "/origin/scitokens.conf",
  "/cache/scitokens.conf",
+ "/api/institutions",
  "/cache/grid-mapfile",
- "/origin/grid-mapfile",
+ "/origin/grid-mapfile"
 ]
 
 
@@ -239,6 +242,21 @@ def validate_namespace_schema(ns):
  validate_cache_schema(cache)
  assert found_credgen, "At least one namespace with credential_generation"
 
+
+ def test_institution_accept_type(self, client: flask.Flask):
+ """Checks both formats output the same content"""
+
+ json_institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json
+ json_tuples = [tuple(map(str, x)) for x in sorted(json_institutions, key=lambda x: x[0])]
+
+ csv_institutions = csv.reader(io.StringIO(client.get("/api/institutions").data.decode()))
+ csv_tuples = [tuple(x) for x in sorted(csv_institutions, key=lambda x: x[0])]
+
+ assert len(csv_tuples) == len(json_tuples)
+
+ assert tuple(json_tuples) == tuple(csv_tuples)
+
+
  def test_origin_grid_mapfile(self, client: flask.Flask):
  TEST_ORIGIN = "origin-auth2001.chtc.wisc.edu" # This origin serves protected data
  response = client.get("/origin/grid-mapfile")
@@ -835,6 +853,43 @@ def test_facility_defaults(self, client: flask.Flask):
  # Check that the site contains the appropriate keys
  assert set(facilities.popitem()[1]).issuperset(["ID", "Name", "IsCCStar"])
 
+ def test_institution_default(self, client: flask.Flask):
+ institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json
+
+ assert len(institutions) > 0
+
+ # Check facilities exist and have the "have resources" bit flipped
+ assert [i for i in institutions if i[0] == "JINR"][0][1]
+ assert [i for i in institutions if i[0] == "Universidade de São Paulo - Laboratório de Computação Científica Avançada"][0][1]
+
+ # Project Organizations exist and have "has project" bit flipped
+ assert [i for i in institutions if i[0] == "Iolani School"][0][2]
+ assert [i for i in institutions if i[0] == "University of California, San Diego"][0][2]
+
+ # Both
+ assert [i for i in institutions if i[0] == "Harvard University"][0][1] and [i for i in institutions if i[0] == "Harvard University"][0][2]
+
+ # Check Project only doesn't have resource bit
+ assert [i for i in institutions if i[0] == "National Research Council of Canada"][0][1] is False
+
+ # Facility Tests
+ facilities = set(global_data.get_topology().facilities.keys())
+
+ # Check all facilities exist
+ assert set(i[0] for i in institutions).issuperset(facilities)
+
+ # Check all facilities have their facilities bit flipped
+ assert all(x[1] for x in institutions if x[0] in institutions)
+
+ # Project Tests
+ projects = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project'])
+
+ # Check all projects exist
+ assert set(i[0] for i in institutions).issuperset(projects)
+
+ # Check all projects have the project bit flipped
+ assert all(x[2] for x in institutions if x[0] in projects)
+
 
 if __name__ == '__main__':
  pytest.main()

src/webapp/common.py

@@ -13,6 +13,9 @@
 
 import xmltodict
 import yaml
+import csv
+from io import StringIO
+
 try:
  from yaml import CSafeLoader as SafeLoader
 except ImportError:
@@ -53,6 +56,14 @@ def populate_voown_name(self, vo_id_to_name: Dict):
  self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id]
 
 
+def to_csv(data: list) -> str:
+ csv_string = StringIO()
+ writer = csv.writer(csv_string)
+ for row in data:
+ writer.writerow(row)
+ return csv_string.getvalue()
+
+
 def is_null(x, *keys) -> bool:
  for key in keys:
  if not key: continue
@@ -101,7 +112,7 @@ def simplify_attr_list(data: Union[Dict, List], namekey: str, del_name: bool = T
  return new_data
 
 
-def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=True) -> List[OrderedDict]:
+def expand_attr_list_single(data: Dict, namekey: str, valuekey: str, name_first=True) -> List[OrderedDict]:
  """
  Expand
  {"name1": "val1",
@@ -120,7 +131,8 @@ def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=T
  return newdata
 
 
-def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None]=None, ignore_missing=False) -> List[OrderedDict]:
+def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None] = None, ignore_missing=False) -> List[
+ OrderedDict]:
  """
  Expand
  {"name1": {"attr1": "val1", ...},
@@ -263,14 +275,15 @@ def git_clone_or_pull(repo, dir, branch, ssh_key=None) -> bool:
  ok = ok and run_git_cmd(["checkout", branch], dir=dir)
  return ok
 
+
 def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool:
  if os.path.exists(git_dir):
  ok = run_git_cmd(["fetch", "origin"], git_dir=git_dir, ssh_key=ssh_key)
  else:
  ok = run_git_cmd(["clone", "--mirror", repo, git_dir], ssh_key=ssh_key)
  # disable mirror push
  ok = ok and run_git_cmd(["config", "--unset", "remote.origin.mirror"],
-  git_dir=git_dir)
+ git_dir=git_dir)
  return ok
 
 
@@ -315,17 +328,14 @@ def escape(pattern: str) -> str:
 
  unescaped_characters = ['!', '"', '%', "'", ',', '/', ':', ';', '<', '=', '>', '@', "`"]
  for unescaped_character in unescaped_characters:
-
  escaped_string = re.sub(unescaped_character, f"\\{unescaped_character}", escaped_string)
 
  return escaped_string
 
 
 def support_cors(f):
-
  @wraps(f)
  def wrapped():
-
  response = f()
 
  response.headers['Access-Control-Allow-Origin'] = '*'

src/webapp/flask_common.py

@@ -0,0 +1,22 @@
+from flask import Response
+from .common import to_csv, to_json_bytes
+from typing import List
+
+def create_accepted_response(data: List, headers, default=None) -> Response:
+ """Provides CSV or JSON options for list of list(string)"""
+
+ if not default:
+ default = "application/json"
+
+ accepted_response_builders = {
+ "text/csv": lambda: Response(to_csv(data), mimetype="text/csv"),
+ "application/json": lambda: Response(to_json_bytes(data), mimetype="application/json"),
+ }
+
+ requested_types = set(headers.get('Accept', 'default').replace(' ', '').split(","))
+ accepted_and_requested = set(accepted_response_builders.keys()).intersection(requested_types)
+
+ if accepted_and_requested:
+ return accepted_response_builders[accepted_and_requested.pop()]()
+ else:
+ return accepted_response_builders[default]()