Skip to content

Commit

Permalink
Add timeout and retry to superagent requests
Browse files Browse the repository at this point in the history
  • Loading branch information
corys committed Oct 12, 2021
1 parent 9ae37a2 commit 56e4c6d
Showing 1 changed file with 14 additions and 8 deletions.
22 changes: 14 additions & 8 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,13 @@ terminalProcedures.currentCycleEffectiveDates = async () => {
const response = await superagent
.get(BASE_URL)
.set('Accept', ACCEPT)

.timeout({ deadline: 30000 })
.retry(3)

const $ = cheerio.load(response.text)
var currentCycle = $('select#cycle > option:contains(Current)').text()
return parseEffectiveDates(currentCycle.replace(/(\n|\t)/gm, ''))
}
}

/**
* Returns the text and values of the targeted <select/> element
Expand All @@ -51,7 +53,9 @@ const fetchCycle = async (cycle = 'Current') => {
const response = await superagent
.get(BASE_URL)
.set('Accept', ACCEPT)

.timeout({ deadline: 30000 })
.retry(3)

const $ = cheerio.load(response.text)
const $cycle = $(`select#cycle > option:contains(${cycle})`)
if (!$cycle) {
Expand All @@ -72,7 +76,7 @@ terminalProcedures.fetchCycle = fetchCycle
terminalProcedures.getCycleEffectiveDates = async (cycle = 'Current') => {
const { text: currentCycle, } = await fetchCycle(cycle)
return parseEffectiveDates(currentCycle.replace(/(\n|\t)/gm, ''))
}
}

terminalProcedures.currentCycleEffectiveDates = async () => {
const { text: currentCycle, } = await fetchCycle()
Expand Down Expand Up @@ -120,7 +124,7 @@ terminalProcedures.fetchNextCycleCode = fetchNextCycleCode
*/
const listOne = async (icao, options) => {
let searchCycle = null

if (options.getNextCycle === true) {
searchCycle = await fetchNextCycleCode()
}
Expand All @@ -146,7 +150,7 @@ const listOne = async (icao, options) => {
// are used to issue separate requests whereas the `urlParams` are
// applied to every request
let filterFlags = []
// Validate the flag option first
// Validate the flag option first
if (typeof options === 'object' && Array.isArray(options.flag) && options.flag.length) {
for (let f = 0, fLen = options.flag.length; f < fLen; f++) {
switch (options.flag[f].toUpperCase()) {
Expand Down Expand Up @@ -206,6 +210,8 @@ const listOne = async (icao, options) => {
const getProcedures = async url => superagent
.get(url)
.set('Accept', ACCEPT)
.timeout({ deadline: 30000 })
.retry(3)
.then(res => parse(res.text))

/**
Expand All @@ -227,7 +233,7 @@ const link = ($row, columnIndex) =>
* Extract the relevant information from the dom node and return
* an object with the data mapped by the appropriate named key
* @param {HTMLNode} $row - The dom node that contains the tabular data
* @param {String} effectiveStartDate - The start date the terminal procedure is effective for
* @param {String} effectiveStartDate - The start date the terminal procedure is effective for
* @param {HTMLNode} effectiveEndDate - The end date the terminal procedure is effective for
*/
const extractRow = ($row, effectiveStartDate, effectiveEndDate) => {
Expand Down Expand Up @@ -311,7 +317,7 @@ const parseEffectiveDates = str => {

/**
* Parse the response HTML into JSON
* @param {string} html
* @param {string} html
* @returns {Object} - The scraped and transformed data and the number of result pages
*/
const parse = html => {
Expand Down

0 comments on commit 56e4c6d

Please sign in to comment.