From c2146791d6541ce83b663d27b2aff7771a5a93d9 Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Mon, 13 Jan 2025 12:08:45 +0100 Subject: [PATCH 1/2] be more robust against WFH being unavailable --- bin/workflows-fetch.rb | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/workflows-fetch.rb b/bin/workflows-fetch.rb index 2aa95e38aa96b..59554f9a8fc25 100755 --- a/bin/workflows-fetch.rb +++ b/bin/workflows-fetch.rb @@ -11,6 +11,7 @@ def request(url) request['Accept'] = 'application/json' req_options = { use_ssl: uri.scheme == 'https', + max_retries: 5, } Net::HTTP.start(uri.hostname, uri.port, req_options) do |http| http.request(request) @@ -37,19 +38,32 @@ def fetch_workflows(server) end def fetch_workflowhub - projects = JSON.parse(request('https://workflowhub.eu/projects').body) + + begin + projects = JSON.parse(request('https://workflowhub.eu/projects').body) + + response = request('https://workflowhub.eu/workflows?filter[workflow_type]=galaxy') + data = JSON.parse(response.body) + rescue StandardError + puts "ERROR: cannot fetch from WorkflowHub" + return [] + end + project_mapping = projects['data'].to_h { |p| [p['id'], p['attributes']['title']] } - response = request('https://workflowhub.eu/workflows?filter[workflow_type]=galaxy') - data = JSON.parse(response.body) - if !data['links']['next'].nil? + if !data['links']['next'].nil? puts 'ERROR: Cannot yet handle multiple pages' exit 42 end puts "INFO: Fetching #{data['data'].length} workflows from WorkflowHub" data['data'].map.with_index do |w, _i| # {"id"=>"14", "type"=>"workflows", "attributes"=>{"title"=>"Cheminformatics - Docking"}, "links"=>{"self"=>"/workflows/14"}} - wf_info = JSON.parse(request("https://workflowhub.eu#{w['links']['self']}").body) + begin + wf_info = JSON.parse(request("https://workflowhub.eu#{w['links']['self']}").body) + rescue StandardError + puts "ERROR: cannont fetch from WorkflowHub" + return [] + end creator_list = [] creator0 = wf_info['data']['attributes']['creators'][0] From b9dfda34087287fc78f1ffef5910781b79bbf4f8 Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Mon, 13 Jan 2025 12:10:49 +0100 Subject: [PATCH 2/2] fix indent --- bin/workflows-fetch.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/workflows-fetch.rb b/bin/workflows-fetch.rb index 59554f9a8fc25..072dce854884f 100755 --- a/bin/workflows-fetch.rb +++ b/bin/workflows-fetch.rb @@ -51,7 +51,7 @@ def fetch_workflowhub project_mapping = projects['data'].to_h { |p| [p['id'], p['attributes']['title']] } - if !data['links']['next'].nil? + if !data['links']['next'].nil? puts 'ERROR: Cannot yet handle multiple pages' exit 42 end