Skip to content

Commit

Permalink
better handling of 503 errors
Browse files Browse the repository at this point in the history
  • Loading branch information
jlloyd-widen committed Jan 19, 2024
1 parent 37d39d4 commit 62f24b3
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 20 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "tap-salesforce-connect"
version = "0.1.1"
version = "0.1.2"
description = "`tap-salesforce-connect` is a Singer tap for SalesforceConnect, built with the Meltano Singer SDK."
readme = "README.md"
authors = ["Josh Lloyd"]
Expand Down
28 changes: 9 additions & 19 deletions tap_salesforce_connect/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from pathlib import Path
from typing import Any, Callable, Generator, Iterable

import backoff
import requests
from singer_sdk.helpers.jsonpath import extract_jsonpath
from singer_sdk.streams import RESTStream
Expand Down Expand Up @@ -110,24 +109,15 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None:
stringified_row[k] = v
return stringified_row

# there is a one-hour rate limit on the SalesforceConnect API
def backoff_wait_generator(self) -> Generator[float, None, None]:
"""Use wait generator for the backoff decorator on request failure.
See for options:
https://github.com/litl/backoff/blob/master/backoff/_wait_gen.py
def get_wait_time_based_on_response(self, exception):
"""Return the number of seconds to wait before retrying.
And see for examples: `Code Samples <../code_samples.html#custom-backoff>`_
Returns:
The wait generator
Salesforce Connect API has a rate limit scoped to an hour
"""
return backoff.expo(factor=3) # type: ignore # ignore 'Returning Any'
if exception.response.status_code == 503:
return 60 * 60
return exception.response.headers.get("Retry-After", 0)

def backoff_max_tries(self) -> int:
"""Provide the number of attempts before giving up when retrying requests.
Returns:
Number of max retries.
"""
return 20
def backoff_wait_generator(self) -> Generator[float, None, None]:
"""Return a generator of wait times between retries."""
return self.backoff_runtime(value=self.get_wait_time_based_on_response)

0 comments on commit 62f24b3

Please sign in to comment.