Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

increased the timeout ramp #1

Merged
merged 1 commit into from
Jan 17, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
increased the timeout ramp
  • Loading branch information
jlloyd-widen committed Jan 17, 2024
commit 8af2a0bebac06602780507928a6790ad1e10c2be
34 changes: 34 additions & 0 deletions plugins/loaders/target-jsonl--andyh1203.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"plugin_type": "loaders",
"name": "target-jsonl",
"namespace": "target_jsonl",
"variant": "andyh1203",
"label": "JSON Lines (JSONL)",
"docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203",
"repo": "https://github.com/andyh1203/target-jsonl",
"pip_url": "target-jsonl",
"description": "JSONL loader",
"logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png",
"settings": [
{
"name": "destination_path",
"kind": "string",
"value": "output",
"label": "Destination Path",
"description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n"
},
{
"name": "do_timestamp_file",
"kind": "boolean",
"value": false,
"label": "Include Timestamp in File Names",
"description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n"
},
{
"name": "custom_name",
"kind": "string",
"label": "Custom File Name Override",
"description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n"
}
]
}
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "tap-salesforce-connect"
version = "0.1.0"
version = "0.1.1"
description = "`tap-salesforce-connect` is a Singer tap for SalesforceConnect, built with the Meltano Singer SDK."
readme = "README.md"
authors = ["Josh Lloyd"]
25 changes: 24 additions & 1 deletion tap_salesforce_connect/client.py
Original file line number Diff line number Diff line change
@@ -5,8 +5,9 @@
import json
import sys
from pathlib import Path
from typing import Any, Callable, Iterable
from typing import Any, Callable, Generator, Iterable

import backoff
import requests
from singer_sdk.helpers.jsonpath import extract_jsonpath
from singer_sdk.streams import RESTStream
@@ -108,3 +109,25 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None:
else:
stringified_row[k] = v
return stringified_row

# there is a one-hour rate limit on the SalesforceConnect API
def backoff_wait_generator(self) -> Generator[float, None, None]:
"""Use wait generator for the backoff decorator on request failure.

See for options:
https://github.com/litl/backoff/blob/master/backoff/_wait_gen.py

And see for examples: `Code Samples <../code_samples.html#custom-backoff>`_

Returns:
The wait generator
"""
return backoff.expo(factor=3) # type: ignore # ignore 'Returning Any'

def backoff_max_tries(self) -> int:
"""Provide the number of attempts before giving up when retrying requests.

Returns:
Number of max retries.
"""
return 20
2 changes: 1 addition & 1 deletion tap_salesforce_connect/tap.py
Original file line number Diff line number Diff line change
@@ -60,7 +60,7 @@ def discover_streams(self) -> list[streams.SalesforceConnectStream]:
streams.TopicFeedStream(self),
streams.UsersStream(self),
streams.TopicsStream(self),
streams.CommentsStream(self)
streams.CommentsStream(self),
]