add curlify on error (#15)

pyproject.toml

@@ -15,6 +15,7 @@ python = "<3.11,>=3.7.1"
 requests = "^2.25.1"
 singer-sdk = "^0.9.0"
 target-hotglue = "^0.0.2"
+curlify = "^2.2.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.2.1"

target_api/client.py

@@ -7,6 +7,7 @@
 from target_hotglue.client import HotglueBaseSink
 import requests
 from singer_sdk.exceptions import FatalAPIError, RetriableAPIError
+from curlify import to_curl
 
 
 class ApiSink(HotglueBaseSink):
@@ -59,18 +60,24 @@ def unified_schema(self) -> BaseModel:
 
  def response_error_message(self, response: requests.Response) -> str:
  try:
- response_text = f" with response body: {response.text}"
+ response_text = f" with response body: '{response.text}'"
  except:
  response_text = None
  return f"Status code: {response.status_code} with {response.reason} for path: {response.request.url} {response_text}"
+
+ def curlify_on_error(self, response):
+ curl = to_curl(response.request)
+ return curl
 
  def validate_response(self, response: requests.Response) -> None:
  """Validate HTTP response."""
  if response.status_code in [429] or 500 <= response.status_code < 600:
  msg = self.response_error_message(response)
- error = {"status_code": response.status_code, "body": msg}
+ curl = self.curlify_on_error(response)
+ error = {"status_code": response.status_code, "body": msg, "curl": curl}
  raise RetriableAPIError(error)
  elif 400 <= response.status_code < 500:
  msg = self.response_error_message(response)
- error = {"status_code": response.status_code, "body": msg}
+ curl = self.curlify_on_error(response)
+ error = {"status_code": response.status_code, "body": msg, "curl":curl}
  raise FatalAPIError(error)

target_api/sinks.py

@@ -85,3 +85,19 @@ def make_batch_request(self, records: List[dict]):
  self.logger.warning(f"Unable to get response's id: {e}")
 
  return id, response.ok, dict()
+
+ def process_batch(self, context: dict) -> None:
+ if not self.latest_state:
+ self.init_state()
+
+ raw_records = context["records"]
+
+ records = list(map(lambda e: self.process_batch_record(e[1], e[0]), enumerate(raw_records)))
+
+ try:
+ response = self.make_batch_request(records)
+ result = self.handle_batch_response(response)
+ for state in result.get("state_updates", list()):
+ self.update_state(state)
+ except Exception as e:
+ self.update_state({"error": str(e)})