feat!: migrate from httplib2 to requests library

BREAKING CHANGE: See #117.

pyproject.toml

@@ -25,7 +25,9 @@ license = "APACHE-2.0"
 python = "^3.7"
 solnlib = "^4.0.0"
 splunk-sdk = "^1.6.16"
-splunktalib = "^2.0.0"
+splunktalib = "^3.0.0b1"
+requests = "^2.26.0"
+PySocks = "^1.7.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2"

splunktaucclib/alert_actions_base.py

@@ -22,6 +22,7 @@
 from solnlib import log
 
 from splunktaucclib.cim_actions import ModularAction
+from splunktaucclib.rest_handler import util
 from splunktaucclib.splunk_aoblib.rest_helper import TARestHelper
 from splunktaucclib.splunk_aoblib.setup_util import Setup_Util
 
@@ -114,22 +115,8 @@ def get_proxy(self):
  return self.setup_util.get_proxy_settings()
 
  def _get_proxy_uri(self):
- uri = None
  proxy = self.get_proxy()
- if proxy and proxy.get("proxy_url") and proxy.get("proxy_type"):
- uri = proxy["proxy_url"]
- if proxy.get("proxy_port"):
- uri = "{}:{}".format(uri, proxy.get("proxy_port"))
- if proxy.get("proxy_username") and proxy.get("proxy_password"):
- uri = "{}://{}:{}@{}/".format(
- proxy["proxy_type"],
- proxy["proxy_username"],
- proxy["proxy_password"],
- uri,
- )
- else:
- uri = "{}://{}".format(proxy["proxy_type"], uri)
- return uri
+ return util.get_proxy_uri(proxy)
 
  def send_http_request(
  self,
@@ -158,71 +145,10 @@ def send_http_request(
  )
 
  def build_http_connection(self, config, timeout=120, disable_ssl_validation=False):
- from httplib2 import Http, ProxyInfo, socks
-
- """
- :config: dict like, proxy and account information are in the following
- format {
- "username": xx,
- "password": yy,
- "proxy_url": zz,
- "proxy_port": aa,
- "proxy_username": bb,
- "proxy_password": cc,
- "proxy_type": http,http_no_tunnel,sock4,sock5,
- "proxy_rdns": 0 or 1,
- }
- :return: Http2.Http object
- """
- if not config:
- config = {}
-
- proxy_type_to_code = {
- "http": socks.PROXY_TYPE_HTTP,
- "http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
- "socks4": socks.PROXY_TYPE_SOCKS4,
- "socks5": socks.PROXY_TYPE_SOCKS5,
- }
- if config.get("proxy_type") in proxy_type_to_code:
- proxy_type = proxy_type_to_code[config["proxy_type"]]
- else:
- proxy_type = socks.PROXY_TYPE_HTTP
-
- rdns = config.get("proxy_rdns")
-
- proxy_info = None
- if config.get("proxy_url") and config.get("proxy_port"):
- if config.get("proxy_username") and config.get("proxy_password"):
- proxy_info = ProxyInfo(
- proxy_type=proxy_type,
- proxy_host=config["proxy_url"],
- proxy_port=int(config["proxy_port"]),
- proxy_user=config["proxy_username"],
- proxy_pass=config["proxy_password"],
- proxy_rdns=rdns,
- )
- else:
- proxy_info = ProxyInfo(
- proxy_type=proxy_type,
- proxy_host=config["proxy_url"],
- proxy_port=int(config["proxy_port"]),
- proxy_rdns=rdns,
- )
- if proxy_info:
- http = Http(
- proxy_info=proxy_info,
- timeout=timeout,
- disable_ssl_certificate_validation=disable_ssl_validation,
- )
- else:
- http = Http(
- timeout=timeout,
- disable_ssl_certificate_validation=disable_ssl_validation,
- )
-
- if config.get("username") and config.get("password"):
- http.add_credentials(config["username"], config["password"])
- return http
+ raise NotImplementedError(
+ "Replace the usage of this function to send_http_request function of same class "
+ "or use requests.request method"
+ )
 
  def process_event(self, *args, **kwargs):
  raise NotImplemented()

splunktaucclib/config.py

@@ -127,22 +127,20 @@ def load(self):
  retries = 4
  waiting_time = [1, 2, 2]
  for retry in range(retries):
- resp, cont = splunkd_request(
+ resp = splunkd_request(
  splunkd_uri=self.make_uri(ep_id),
  session_key=self.session_key,
  data=data,
  retry=3,
  )
 
- if resp is None or resp.status != 200:
- msg = 'Fail to load endpoint "{ep_id}" - {err}' "".format(
- ep_id=ep_id, err=code_to_msg(resp, cont) if resp else cont
- )
+ if resp is None or resp.status_code != 200:
+ msg = f'Fail to load endpoint "{ep_id}" - {code_to_msg(resp)}'
  log(msg, level=logging.ERROR, need_tb=True)
  raise ConfigException(msg)
 
  try:
- ret[ep_id] = self._parse_content(ep_id, cont)
+ ret[ep_id] = self._parse_content(ep_id, resp.text)
  except ConfigException as exc:
  log(exc, level=logging.WARNING, need_tb=True)
  if retry < retries - 1:
@@ -206,22 +204,15 @@ def update_items(
  continue
  item_uri = self.make_uri(endpoint_id, item_name=item_name)
 
- resp, cont = splunkd_request(
+ resp = splunkd_request(
  splunkd_uri=item_uri,
  session_key=self.session_key,
  data=post_data,
  method="POST",
  retry=3,
  )
- if resp is None or resp.status not in (200, 201):
- msg = (
- 'Fail to update item "{item}" in endpoint "{ep_id}"'
- " - {err}".format(
- ep_id=endpoint_id,
- item=item_name,
- err=code_to_msg(resp, cont) if resp else cont,
- )
- )
+ if resp is None or resp.status_code not in (200, 201):
+ msg = f'Fail to update item "{item_name}" in endpoint "{endpoint_id}" - {code_to_msg(resp)}'
  log(msg, level=logging.ERROR)
  if raise_if_failed:
  raise ConfigException(msg)
@@ -365,8 +356,8 @@ def load_value(self, endpoint_id, item_name, fname, fval):
 
  def try_fix_corrupted_json(self, corrupted_json, value_err):
  """
- When L3 (one of our customers) was testing, they encountered a bug that 'access_token_encrypted' or 'refresh_token' got corrupted when it was saved in the conf file
- (because of the way how we save encrypted data).
+ A bug was encountered that 'access_token_encrypted' or 'refresh_token'
+ got corrupted when it was saved in the conf file
  """
  # value_err.message is in this format:
  # Extra data: line 1 column 2720 - line 1 column 2941 (char 2719 - 2940)

splunktaucclib/rest_handler/base.py

@@ -21,7 +21,6 @@
 import itertools
 import json
 import logging
-import sys
 from inspect import ismethod
 from os import path as op
 
@@ -68,15 +67,15 @@ def user_caps(mgmt_uri, session_key):
  """
  url = mgmt_uri + "/services/authentication/current-context"
 
- resp, cont = splunkd_request(
+ resp = splunkd_request(
  url, session_key, method="GET", data={"output_mode": "json"}, retry=3
  )
  if resp is None:
  RH_Err.ctl(500, logging.ERROR, "Fail to get capabilities of sessioned user")
- elif resp.status not in (200, "200"):
- RH_Err.ctl(resp.status, logging.ERROR, cont)
+ elif resp.status_code != 200:
+ RH_Err.ctl(resp.status_code, logging.ERROR, resp.text)
 
- cont = json.loads(cont)
+ cont = resp.json()
  caps = cont["entry"][0]["content"]["capabilities"]
  return set(caps)
 
@@ -288,7 +287,7 @@ def handleACL(self, confInfo):
  def handleCreate(self, confInfo):
  try:
  self.get(self.callerArgs.id)
- except:
+ except Exception:
  pass
  else:
  RH_Err.ctl(

splunktaucclib/rest_handler/poster.py

@@ -15,17 +15,17 @@
 #
 
 
-import json
 import re
 import urllib.error
 import urllib.parse
 import urllib.request
 
+import requests
 from splunk import admin, rest
 from splunktalib.common.util import is_true
-from splunktalib.rest import build_http_connection, code_to_msg, splunkd_request
+from splunktalib.rest import code_to_msg, splunkd_request
 
-from . import base
+from . import base, util
 from .error_ctl import RestHandlerError as RH_Err
 
 
@@ -75,7 +75,8 @@ def handleEdit(self, confInfo):
  app=app,
  )
  proxy_enabled = proxy_info.get("proxy_enabled", False)
- http = build_http_connection(proxy_info if proxy_enabled else {})
+ proxy_uri = util.get_proxy_uri(proxy_info if proxy_enabled else {})
+ proxies = {"http": proxy_uri, "https": proxy_uri}
  try:
  url = self.callerArgs.data["splunk_poster_url"][0]
  for regex in self.allowedURLs:
@@ -97,15 +98,18 @@ def handleEdit(self, confInfo):
  "Content-Type": "application/x-www-form-urlencoded",
  }
 
- resp, content = http.request(
- url,
+ resp = requests.request(
  method=method,
+ url=url,
  headers=headers,
- body=urllib.parse.urlencode(payload),
+ data=urllib.parse.urlencode(payload),
+ timeout=120,
+ proxies=proxies,
+ verify=True,
  )
- content = json.loads(content)
- if resp.status not in (200, 201, "200", "201"):
- RH_Err.ctl(resp.status, msgx=content)
+ content = resp.json()
+ if resp.status_code not in (200, 201):
+ RH_Err.ctl(resp.status_code, msgx=content)
 
  for key, val in content.items():
  confInfo[self.callerArgs.id][key] = val
@@ -167,17 +171,12 @@ def getProxyInfo(self, splunkdMgmtUri, sessionKey, user, app):
  proxyInfoEndpoint=self.proxyInfoEndpoint,
  )
  data = {"output_mode": "json", "--get-clear-credential--": "1"}
- resp, cont = splunkd_request(url, sessionKey, data=data, retry=3)
- if resp is None or resp.status != 200:
- RH_Err.ctl(
- 1104,
- msgx="failed to load proxy info. {err}".format(
- err=code_to_msg(resp, cont) if resp else cont
- ),
- )
+ resp = splunkd_request(url, sessionKey, data=data, retry=3)
+ if resp is None or resp.status_code != 200:
+ RH_Err.ctl(1104, msgx=f"failed to load proxy info. {code_to_msg(resp)}")
 
  try:
- proxy_info = json.loads(cont)["entry"][0]["content"]
+ proxy_info = resp.json()["entry"][0]["content"]
  except IndexError | KeyError:
  proxy_info = {}