feat: retry requests on status code 429 and 503

fixes fent/node-ytdl-core#444

lib/index.js

@@ -6,6 +6,7 @@ const PassThrough = require('stream').PassThrough;
 
 const httpLibs = { 'http:': http, 'https:': https };
 const redirectCodes = { 301: true, 302: true, 303: true, 307: true };
+const retryCodes = { 429: true, 503: true };
 const defaults = {
  maxRedirects: 2,
  maxRetries: 2,
@@ -58,7 +59,7 @@ module.exports = (url, options, callback) => {
  }, options.headers);
  }
 
- const doRetry = (err, statusCode) => {
+ const doRetry = (retryOptions = {}) => {
  if (aborted) { return false; }
  // If there is an error when the download has already started,
  // but not finished, try reconnecting.
@@ -67,23 +68,26 @@ module.exports = (url, options, callback) => {
  reconnects++ < options.maxReconnects) {
  myres = null;
  retries = 0;
- let ms = Math.min(options.backoff.inc, options.backoff.max);
+ let inc = options.backoff.inc;
+ let ms = Math.min(inc, options.backoff.max);
  retryTimeout = setTimeout(doDownload, ms);
- stream.emit('reconnect', reconnects, err);
+ stream.emit('reconnect', reconnects, retryOptions.err);
  return true;
  }
- } else if ((!statusCode || err.message === 'ENOTFOUND') &&
+ } else if ((!retryOptions.statusCode ||
+ retryOptions.err && retryOptions.err.message === 'ENOTFOUND') &&
  retries++ < options.maxRetries) {
- let ms = Math.min(retries * options.backoff.inc, options.backoff.max);
+ let ms = retryOptions.retryAfter ||
+ Math.min(retries * options.backoff.inc, options.backoff.max);
  retryTimeout = setTimeout(doDownload, ms);
- stream.emit('retry', retries, err);
+ stream.emit('retry', retries, retryOptions.err);
  return true;
  }
  return false;
  };
 
  const onRequestError = (err, statusCode) => {
- if (!doRetry(err, statusCode)) {
+ if (!doRetry({ err, statusCode })) {
  stream.emit('error', err);
  }
  };
@@ -117,15 +121,21 @@ module.exports = (url, options, callback) => {
  }
 
  myreq = httpLib.get(parsed, (res) => {
- if (redirectCodes[res.statusCode] === true) {
+ if (redirectCodes[res.statusCode]) {
  if (redirects++ >= options.maxRedirects) {
  stream.emit('error', Error('Too many redirects'));
  } else {
  url = res.headers.location;
+ setTimeout(doDownload, parseInt(res.headers['retry-after'] || 0, 10) * 1000);
  stream.emit('redirect', url);
- doDownload();
  }
  return;
+
+ // Check for rate limiting.
+ } else if (retryCodes[res.statusCode]) {
+ doRetry({ retryAfter: parseInt(res.headers['retry-after'], 10) });
+ return;
+
  } else if (res.statusCode < 200 || 400 <= res.statusCode) {
  let err = Error('Status code: ' + res.statusCode);
  if (res.statusCode >= 500) {

test/request-test.js

@@ -173,13 +173,16 @@ describe('Make a request', () => {
  .reply(302, '', { Location: 'http://mysite.com/redirected!' })
  .get('/redirected!')
  .reply(200, 'Helloo!');
- miniget('http://mysite.com/pathy', (err, res, body) => {
- assert.ifError(err);
+ const stream = miniget('http://mysite.com/pathy');
+ stream.on('error', done);
+ stream.on('data', (body) => {
  scope.done();
- assert.equal(res.statusCode, 200);
  assert.equal(body, 'Helloo!');
  done();
  });
+ stream.on('redirect', () => {
+ clock.tick(1);
+ });
  });
 
  describe('too many times', () => {
@@ -191,12 +194,81 @@ describe('Make a request', () => {
  .reply(302, '', { Location: 'http://yoursite.com/redirect-2' })
  .get('/redirect-2')
  .reply(302, '', { Location: 'http://yoursite.com/redirect-3' });
- miniget('http://yoursite.com/first-request', (err) => {
+ const stream = miniget('http://yoursite.com/first-request');
+ stream.on('error', (err) => {
  assert.ok(err);
  scope.done();
  assert.equal(err.message, 'Too many redirects');
  done();
  });
+ stream.on('redirect', () => {
+ clock.tick(1);
+ });
+ });
+ });
+
+ describe('with `retry-after` header', () => {
+ it('Redirects after given time', (done) => {
+ const scope = nock('http://mysite2.com')
+ .get('/pathos/to/resource')
+ .reply(301, '', {
+ Location: 'http://mysite2.com/newpath/to/source',
+ 'Retry-After': '300',
+ })
+ .get('/newpath/to/source')
+ .reply(200, 'hi world!!');
+ const stream = miniget('http://mysite2.com/pathos/to/resource');
+ stream.on('error', done);
+ stream.on('data', (body) => {
+ scope.done();
+ assert.equal(body, 'hi world!!');
+ done();
+ });
+ stream.on('redirect', () => {
+ clock.tick(300 * 1000);
+ });
+ });
+ });
+ });
+
+ describe('that gets api limited', () => {
+ it('Retries the request after some time', (done) => {
+ const scope = nock('https://mysite.io')
+ .get('/api/v1/data')
+ .reply(429, 'slow down')
+ .get('/api/v1/data')
+ .reply(200, 'where are u');
+ const stream = miniget('https://mysite.io/api/v1/data');
+ stream.on('error', done);
+ stream.on('data', (data) => {
+ scope.done();
+ assert.equal(data, 'where are u');
+ done();
+ });
+ stream.on('retry', () => {
+ clock.tick(1000);
+ });
+ });
+ describe('with `retry-after` header', () => {
+ it('Retries after given time', (done) => {
+ const scope = nock('https://mysite.io')
+ .get('/api/v1/dota')
+ .reply(429, 'slow down', { 'Retry-After': '3600' })
+ .get('/api/v1/dota')
+ .reply(200, 'where are u');
+ const stream = miniget('https://mysite.io/api/v1/dota');
+ stream.on('error', done);
+ stream.on('data', (data) => {
+ scope.done();
+ assert.equal(data, 'where are u');
+ done();
+ });
+ stream.on('retry', () => {
+ // Test that ticking by a bit does not retry the request.
+ clock.tick(1000);
+ assert.ok(!scope.isDone());
+ clock.tick(3600 * 1000);
+ });
  });
  });
  });