diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml new file mode 100644 index 0000000..e5e5d2d --- /dev/null +++ b/.github/workflows/action.yml @@ -0,0 +1,17 @@ +# This is a basic workflow to generate build +name: "Generate build, run app inspect and update splunklib" + +on: push + +jobs: + pre-release: + name: "Run on push - Add Utilities & App Inspect" + runs-on: "ubuntu-latest" + + steps: + - uses: VatsalJagani/splunk-app-action@v4 + with: + my_github_token: ${{ secrets.MY_GITHUB_TOKEN }} + splunkbase_username: ${{ secrets.SPLUNKBASE_USERNAME }} + splunkbase_password: ${{ secrets.SPLUNKBASE_PASSWORD }} + to_make_permission_changes: true diff --git a/README.md b/README.md index 66b695b..301b341 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,10 @@ As of October 2022, there are still no signs of version control within the Splun - [Search Head Backup](https://splunkbase.splunk.com/app/6438) - backup to an index, works in Splunk Cloud ## Release Notes +### 1.2.11 +Library updates: +- Updated Splunk python SDK to 2.0.1 + ### 1.2.10 Updates: - Disabled urllib3 warnings diff --git a/lib/splunklib/__init__.py b/lib/splunklib/__init__.py index 31787bd..c86dfdb 100644 --- a/lib/splunklib/__init__.py +++ b/lib/splunklib/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,12 +14,10 @@ """Python library for Splunk.""" -from __future__ import absolute_import -from splunklib.six.moves import map import logging DEFAULT_LOG_FORMAT = '%(asctime)s, Level=%(levelname)s, Pid=%(process)s, Logger=%(name)s, File=%(filename)s, ' \ - 'Line=%(lineno)s, %(message)s' + 'Line=%(lineno)s, %(message)s' DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S %Z' @@ -31,5 +29,6 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE format=log_format, datefmt=date_format) -__version_info__ = (1, 7, 3) + +__version_info__ = (2, 0, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/lib/splunklib/binding.py b/lib/splunklib/binding.py index 85cb8d1..958be96 100644 --- a/lib/splunklib/binding.py +++ b/lib/splunklib/binding.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -24,31 +24,24 @@ :mod:`splunklib.client` module. """ -from __future__ import absolute_import - import io +import json import logging import socket import ssl -import sys import time from base64 import b64encode from contextlib import contextmanager from datetime import datetime from functools import wraps from io import BytesIO -from xml.etree.ElementTree import XML - +from urllib import parse +from http import client +from http.cookies import SimpleCookie +from xml.etree.ElementTree import XML, ParseError +from splunklib.data import record from splunklib import __version__ -from splunklib import six -from splunklib.six.moves import urllib - -from .data import record -try: - from xml.etree.ElementTree import ParseError -except ImportError as e: - from xml.parsers.expat import ExpatError as ParseError logger = logging.getLogger(__name__) @@ -57,26 +50,59 @@ "connect", "Context", "handler", - "HTTPError" + "HTTPError", + "UrlEncoded", + "_encode", + "_make_cookie_header", + "_NoAuthenticationToken", + "namespace" ] +SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId', + 'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey', + 'token'] + # If you change these, update the docstring # on _authority as well. DEFAULT_HOST = "localhost" DEFAULT_PORT = "8089" DEFAULT_SCHEME = "https" + def _log_duration(f): @wraps(f) def new_f(*args, **kwargs): start_time = datetime.now() val = f(*args, **kwargs) end_time = datetime.now() - logger.debug("Operation took %s", end_time-start_time) + logger.debug("Operation took %s", end_time - start_time) return val + return new_f +def mask_sensitive_data(data): + ''' + Masked sensitive fields data for logging purpose + ''' + if not isinstance(data, dict): + try: + data = json.loads(data) + except Exception as ex: + return data + + # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type + if not isinstance(data, dict): + return data + mdata = {} + for k, v in data.items(): + if k in SENSITIVE_KEYS: + mdata[k] = "******" + else: + mdata[k] = mask_sensitive_data(v) + return mdata + + def _parse_cookies(cookie_str, dictionary): """Tries to parse any key-value pairs of cookies in a string, then updates the the dictionary with any key-value pairs found. @@ -93,7 +119,7 @@ def _parse_cookies(cookie_str, dictionary): :param dictionary: A dictionary to update with any found key-value pairs. :type dictionary: ``dict`` """ - parsed_cookie = six.moves.http_cookies.SimpleCookie(cookie_str) + parsed_cookie = SimpleCookie(cookie_str) for cookie in parsed_cookie.values(): dictionary[cookie.key] = cookie.coded_value @@ -115,10 +141,11 @@ def _make_cookie_header(cookies): :return: ``str` An HTTP header cookie string. :rtype: ``str`` """ - return "; ".join("%s=%s" % (key, value) for key, value in cookies) + return "; ".join(f"{key}={value}" for key, value in cookies) + # Singleton values to eschew None -class _NoAuthenticationToken(object): +class _NoAuthenticationToken: """The value stored in a :class:`Context` or :class:`splunklib.client.Service` class that is not logged in. @@ -130,7 +157,6 @@ class that is not logged in. Likewise, after a ``Context`` or ``Service`` object has been logged out, the token is set to this value again. """ - pass class UrlEncoded(str): @@ -156,7 +182,7 @@ class UrlEncoded(str): **Example**:: import urllib - UrlEncoded('%s://%s' % (scheme, urllib.quote(host)), skip_encode=True) + UrlEncoded(f'{scheme}://{urllib.quote(host)}', skip_encode=True) If you append ``str`` strings and ``UrlEncoded`` strings, the result is also URL encoded. @@ -166,19 +192,19 @@ class UrlEncoded(str): UrlEncoded('ab c') + 'de f' == UrlEncoded('ab cde f') 'ab c' + UrlEncoded('de f') == UrlEncoded('ab cde f') """ + def __new__(self, val='', skip_encode=False, encode_slash=False): if isinstance(val, UrlEncoded): # Don't urllib.quote something already URL encoded. return val - elif skip_encode: + if skip_encode: return str.__new__(self, val) - elif encode_slash: - return str.__new__(self, urllib.parse.quote_plus(val)) - else: - # When subclassing str, just call str's __new__ method - # with your class and the value you want to have in the - # new string. - return str.__new__(self, urllib.parse.quote(val)) + if encode_slash: + return str.__new__(self, parse.quote_plus(val)) + # When subclassing str, just call str.__new__ method + # with your class and the value you want to have in the + # new string. + return str.__new__(self, parse.quote(val)) def __add__(self, other): """self + other @@ -188,8 +214,8 @@ def __add__(self, other): """ if isinstance(other, UrlEncoded): return UrlEncoded(str.__add__(self, other), skip_encode=True) - else: - return UrlEncoded(str.__add__(self, urllib.parse.quote(other)), skip_encode=True) + + return UrlEncoded(str.__add__(self, parse.quote(other)), skip_encode=True) def __radd__(self, other): """other + self @@ -199,8 +225,8 @@ def __radd__(self, other): """ if isinstance(other, UrlEncoded): return UrlEncoded(str.__radd__(self, other), skip_encode=True) - else: - return UrlEncoded(str.__add__(urllib.parse.quote(other), self), skip_encode=True) + + return UrlEncoded(str.__add__(parse.quote(other), self), skip_encode=True) def __mod__(self, fields): """Interpolation into ``UrlEncoded``s is disabled. @@ -209,15 +235,17 @@ def __mod__(self, fields): ``TypeError``. """ raise TypeError("Cannot interpolate into a UrlEncoded object.") + def __repr__(self): - return "UrlEncoded(%s)" % repr(urllib.parse.unquote(str(self))) + return f"UrlEncoded({repr(parse.unquote(str(self)))})" + @contextmanager def _handle_auth_error(msg): - """Handle reraising HTTP authentication errors as something clearer. + """Handle re-raising HTTP authentication errors as something clearer. If an ``HTTPError`` is raised with status 401 (access denied) in - the body of this context manager, reraise it as an + the body of this context manager, re-raise it as an ``AuthenticationError`` instead, with *msg* as its message. This function adds no round trips to the server. @@ -238,6 +266,7 @@ def _handle_auth_error(msg): else: raise + def _authentication(request_fun): """Decorator to handle autologin and authentication errors. @@ -270,12 +299,12 @@ def _authentication(request_fun): def f(): c.get("/services") return 42 - print _authentication(f) + print(_authentication(f)) """ + @wraps(request_fun) def wrapper(self, *args, **kwargs): - if self.token is _NoAuthenticationToken and \ - not self.has_cookies(): + if self.token is _NoAuthenticationToken and not self.has_cookies(): # Not yet logged in. if self.autologin and self.username and self.password: # This will throw an uncaught @@ -297,8 +326,7 @@ def wrapper(self, *args, **kwargs): # an AuthenticationError and give up. with _handle_auth_error("Autologin failed."): self.login() - with _handle_auth_error( - "Authentication Failed! If session token is used, it seems to have been expired."): + with _handle_auth_error("Authentication Failed! If session token is used, it seems to have been expired."): return request_fun(self, *args, **kwargs) elif he.status == 401 and not self.autologin: raise AuthenticationError( @@ -352,7 +380,8 @@ def _authority(scheme=DEFAULT_SCHEME, host=DEFAULT_HOST, port=DEFAULT_PORT): # IPv6 addresses must be enclosed in [ ] in order to be well # formed. host = '[' + host + ']' - return UrlEncoded("%s://%s:%s" % (scheme, host, port), skip_encode=True) + return UrlEncoded(f"{scheme}://{host}:{port}", skip_encode=True) + # kwargs: sharing, owner, app def namespace(sharing=None, owner=None, app=None, **kwargs): @@ -407,7 +436,7 @@ def namespace(sharing=None, owner=None, app=None, **kwargs): n = binding.namespace(sharing="global", app="search") """ if sharing in ["system"]: - return record({'sharing': sharing, 'owner': "nobody", 'app': "system" }) + return record({'sharing': sharing, 'owner': "nobody", 'app': "system"}) if sharing in ["global", "app"]: return record({'sharing': sharing, 'owner': "nobody", 'app': app}) if sharing in ["user", None]: @@ -415,7 +444,7 @@ def namespace(sharing=None, owner=None, app=None, **kwargs): raise ValueError("Invalid value for argument: 'sharing'") -class Context(object): +class Context: """This class represents a context that encapsulates a splunkd connection. The ``Context`` class encapsulates the details of HTTP requests, @@ -434,7 +463,7 @@ class Context(object): :type port: ``integer`` :param scheme: The scheme for accessing the service (the default is "https"). :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verrification for https connections. + :param verify: Enable (True) or disable (False) SSL verification for https connections. :type verify: ``Boolean`` :param sharing: The sharing mode for the namespace (the default is "user"). :type sharing: "global", "system", "app", or "user" @@ -477,12 +506,14 @@ class Context(object): # Or if you already have a valid cookie c = binding.Context(cookie="splunkd_8089=...") """ + def __init__(self, handler=None, **kwargs): self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), - cert_file=kwargs.get("cert_file"), context=kwargs.get("context"), # Default to False for backward compat + cert_file=kwargs.get("cert_file"), context=kwargs.get("context"), + # Default to False for backward compat retries=kwargs.get("retries", 0), retryDelay=kwargs.get("retryDelay", 10)) self.token = kwargs.get("token", _NoAuthenticationToken) - if self.token is None: # In case someone explicitly passes token=None + if self.token is None: # In case someone explicitly passes token=None self.token = _NoAuthenticationToken self.scheme = kwargs.get("scheme", DEFAULT_SCHEME) self.host = kwargs.get("host", DEFAULT_HOST) @@ -532,9 +563,9 @@ def _auth_headers(self): if self.has_cookies(): return [("Cookie", _make_cookie_header(list(self.get_cookies().items())))] elif self.basic and (self.username and self.password): - token = 'Basic %s' % b64encode(("%s:%s" % (self.username, self.password)).encode('utf-8')).decode('ascii') + token = f'Basic {b64encode(("%s:%s" % (self.username, self.password)).encode("utf-8")).decode("ascii")}' elif self.bearerToken: - token = 'Bearer %s' % self.bearerToken + token = f'Bearer {self.bearerToken}' elif self.token is _NoAuthenticationToken: token = [] else: @@ -542,7 +573,7 @@ def _auth_headers(self): if self.token.startswith('Splunk '): token = self.token else: - token = 'Splunk %s' % self.token + token = f'Splunk {self.token}' if token: header.append(("Authorization", token)) if self.get_cookies(): @@ -631,7 +662,7 @@ def delete(self, path_segment, owner=None, app=None, sharing=None, **query): """ path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - logger.debug("DELETE request to %s (body: %s)", path, repr(query)) + logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query)) response = self.http.delete(path, self._auth_headers, **query) return response @@ -694,7 +725,7 @@ def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, ** path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - logger.debug("GET request to %s (body: %s)", path, repr(query)) + logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query)) all_headers = headers + self.additional_headers + self._auth_headers response = self.http.get(path, all_headers, **query) return response @@ -773,12 +804,7 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, * path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - # To avoid writing sensitive data in debug logs - endpoint_having_sensitive_data = ["/storage/passwords"] - if any(endpoint in path for endpoint in endpoint_having_sensitive_data): - logger.debug("POST request to %s ", path) - else: - logger.debug("POST request to %s (body: %s)", path, repr(query)) + logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query)) all_headers = headers + self.additional_headers + self._auth_headers response = self.http.post(path, all_headers, **query) return response @@ -840,13 +866,12 @@ def request(self, path_segment, method="GET", headers=None, body={}, headers = [] path = self.authority \ - + self._abspath(path_segment, owner=owner, - app=app, sharing=sharing) + + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) all_headers = headers + self.additional_headers + self._auth_headers logger.debug("%s request to %s (headers: %s, body: %s)", - method, path, str(all_headers), repr(body)) - + method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body)) if body: body = _encode(**body) @@ -887,14 +912,14 @@ def login(self): """ if self.has_cookies() and \ - (not self.username and not self.password): + (not self.username and not self.password): # If we were passed session cookie(s), but no username or # password, then login is a nop, since we're automatically # logged in. return if self.token is not _NoAuthenticationToken and \ - (not self.username and not self.password): + (not self.username and not self.password): # If we were passed a session token, but no username or # password, then login is a nop, since we're automatically # logged in. @@ -916,11 +941,11 @@ def login(self): username=self.username, password=self.password, headers=self.additional_headers, - cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header + cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header body = response.body.read() session = XML(body).findtext("./sessionKey") - self.token = "Splunk %s" % session + self.token = f"Splunk {session}" return self except HTTPError as he: if he.status == 401: @@ -935,7 +960,7 @@ def logout(self): return self def _abspath(self, path_segment, - owner=None, app=None, sharing=None): + owner=None, app=None, sharing=None): """Qualifies *path_segment* into an absolute path for a URL. If *path_segment* is already absolute, returns it unchanged. @@ -987,12 +1012,11 @@ def _abspath(self, path_segment, # namespace. If only one of app and owner is specified, use # '-' for the other. if ns.app is None and ns.owner is None: - return UrlEncoded("/services/%s" % path_segment, skip_encode=skip_encode) + return UrlEncoded(f"/services/{path_segment}", skip_encode=skip_encode) oname = "nobody" if ns.owner is None else ns.owner aname = "system" if ns.app is None else ns.app - path = UrlEncoded("/servicesNS/%s/%s/%s" % (oname, aname, path_segment), - skip_encode=skip_encode) + path = UrlEncoded(f"/servicesNS/{oname}/{aname}/{path_segment}", skip_encode=skip_encode) return path @@ -1043,21 +1067,23 @@ def connect(**kwargs): c.login() return c + # Note: the error response schema supports multiple messages but we only # return the first, although we do return the body so that an exception # handler that wants to read multiple messages can do so. class HTTPError(Exception): """This exception is raised for HTTP responses that return an error.""" + def __init__(self, response, _message=None): status = response.status reason = response.reason body = response.body.read() try: detail = XML(body).findtext("./messages/msg") - except ParseError as err: + except ParseError: detail = body - message = "HTTP %d %s%s" % ( - status, reason, "" if detail is None else " -- %s" % detail) + detail_formatted = "" if detail is None else f" -- {detail}" + message = f"HTTP {status} {reason}{detail_formatted}" Exception.__init__(self, _message or message) self.status = status self.reason = reason @@ -1065,6 +1091,7 @@ def __init__(self, response, _message=None): self.body = body self._response = response + class AuthenticationError(HTTPError): """Raised when a login request to Splunk fails. @@ -1072,6 +1099,7 @@ class AuthenticationError(HTTPError): in a call to :meth:`Context.login` or :meth:`splunklib.client.Service.login`, this exception is raised. """ + def __init__(self, message, cause): # Put the body back in the response so that HTTPError's constructor can # read it again. @@ -1079,6 +1107,7 @@ def __init__(self, message, cause): HTTPError.__init__(self, cause._response, message) + # # The HTTP interface used by the Splunk binding layer abstracts the underlying # HTTP library using request & response 'messages' which are implemented as @@ -1106,16 +1135,17 @@ def __init__(self, message, cause): # 'foo=1&foo=2&foo=3'. def _encode(**kwargs): items = [] - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): if isinstance(value, list): items.extend([(key, item) for item in value]) else: items.append((key, value)) - return urllib.parse.urlencode(items) + return parse.urlencode(items) + # Crack the given url into (scheme, host, port, path) def _spliturl(url): - parsed_url = urllib.parse.urlparse(url) + parsed_url = parse.urlparse(url) host = parsed_url.hostname port = parsed_url.port path = '?'.join((parsed_url.path, parsed_url.query)) if parsed_url.query else parsed_url.path @@ -1124,9 +1154,10 @@ def _spliturl(url): if port is None: port = DEFAULT_PORT return parsed_url.scheme, host, port, path + # Given an HTTP request handler, this wrapper objects provides a related # family of convenience methods built using that handler. -class HttpLib(object): +class HttpLib: """A set of convenient methods for making HTTP calls. ``HttpLib`` provides a general :meth:`request` method, and :meth:`delete`, @@ -1168,7 +1199,9 @@ class HttpLib(object): If using the default handler, SSL verification can be disabled by passing verify=False. """ - def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, retryDelay=10): + + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, + retryDelay=10): if custom_handler is None: self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context) else: @@ -1229,7 +1262,7 @@ def get(self, url, headers=None, **kwargs): # the query to be encoded or it will get automatically URL # encoded by being appended to url. url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) - return self.request(url, { 'method': "GET", 'headers': headers }) + return self.request(url, {'method': "GET", 'headers': headers}) def post(self, url, headers=None, **kwargs): """Sends a POST request to a URL. @@ -1325,6 +1358,7 @@ class ResponseReader(io.RawIOBase): types of HTTP libraries used with this SDK. This class also provides a preview of the stream and a few useful predicates. """ + # For testing, you can use a StringIO as the argument to # ``ResponseReader`` instead of an ``httplib.HTTPResponse``. It # will work equally well. @@ -1334,10 +1368,7 @@ def __init__(self, response, connection=None): self._buffer = b'' def __str__(self): - if six.PY2: - return self.read() - else: - return str(self.read(), 'UTF-8') + return str(self.read(), 'UTF-8') @property def empty(self): @@ -1363,7 +1394,7 @@ def close(self): self._connection.close() self._response.close() - def read(self, size = None): + def read(self, size=None): """Reads a given number of characters from the response. :param size: The number of characters to read, or "None" to read the @@ -1416,7 +1447,7 @@ def connect(scheme, host, port): kwargs = {} if timeout is not None: kwargs['timeout'] = timeout if scheme == "http": - return six.moves.http_client.HTTPConnection(host, port, **kwargs) + return client.HTTPConnection(host, port, **kwargs) if scheme == "https": if key_file is not None: kwargs['key_file'] = key_file if cert_file is not None: kwargs['cert_file'] = cert_file @@ -1427,8 +1458,8 @@ def connect(scheme, host, port): # verify is True in elif branch and context is not None kwargs['context'] = context - return six.moves.http_client.HTTPSConnection(host, port, **kwargs) - raise ValueError("unsupported scheme: %s" % scheme) + return client.HTTPSConnection(host, port, **kwargs) + raise ValueError(f"unsupported scheme: {scheme}") def request(url, message, **kwargs): scheme, host, port, path = _spliturl(url) @@ -1439,7 +1470,7 @@ def request(url, message, **kwargs): "User-Agent": "splunk-sdk-python/%s" % __version__, "Accept": "*/*", "Connection": "Close", - } # defaults + } # defaults for key, value in message["headers"]: head[key] = value method = message.get("method", "GET") diff --git a/lib/splunklib/client.py b/lib/splunklib/client.py index 33156bb..4886188 100644 --- a/lib/splunklib/client.py +++ b/lib/splunklib/client.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -54,7 +54,7 @@ are subclasses of :class:`Entity`. An ``Entity`` object has fields for its attributes, and methods that are specific to each kind of entity. For example:: - print my_app['author'] # Or: print my_app.author + print(my_app['author']) # Or: print(my_app.author) my_app.package() # Creates a compressed package of this application """ @@ -66,15 +66,13 @@ import socket from datetime import datetime, timedelta from time import sleep +from urllib import parse -from splunklib import six -from splunklib.six.moves import urllib - -from . import data -from .binding import (AuthenticationError, Context, HTTPError, UrlEncoded, - _encode, _make_cookie_header, _NoAuthenticationToken, - namespace) -from .data import record +from splunklib import data +from splunklib.data import record +from splunklib.binding import (AuthenticationError, Context, HTTPError, UrlEncoded, + _encode, _make_cookie_header, _NoAuthenticationToken, + namespace) logger = logging.getLogger(__name__) @@ -84,7 +82,8 @@ "OperationError", "IncomparableException", "Service", - "namespace" + "namespace", + "AuthenticationError" ] PATH_APPS = "apps/local/" @@ -106,7 +105,7 @@ PATH_MODULAR_INPUTS = "data/modular-inputs" PATH_ROLES = "authorization/roles/" PATH_SAVED_SEARCHES = "saved/searches/" -PATH_STANZA = "configs/conf-%s/%s" # (file, stanza) +PATH_STANZA = "configs/conf-%s/%s" # (file, stanza) PATH_USERS = "authentication/users/" PATH_RECEIVERS_STREAM = "/services/receivers/stream" PATH_RECEIVERS_SIMPLE = "/services/receivers/simple" @@ -116,45 +115,38 @@ XNAME_ENTRY = XNAMEF_ATOM % "entry" XNAME_CONTENT = XNAMEF_ATOM % "content" -MATCH_ENTRY_CONTENT = "%s/%s/*" % (XNAME_ENTRY, XNAME_CONTENT) +MATCH_ENTRY_CONTENT = f"{XNAME_ENTRY}/{XNAME_CONTENT}/*" class IllegalOperationException(Exception): """Thrown when an operation is not possible on the Splunk instance that a :class:`Service` object is connected to.""" - pass class IncomparableException(Exception): """Thrown when trying to compare objects (using ``==``, ``<``, ``>``, and so on) of a type that doesn't support it.""" - pass class AmbiguousReferenceException(ValueError): """Thrown when the name used to fetch an entity matches more than one entity.""" - pass class InvalidNameException(Exception): """Thrown when the specified name contains characters that are not allowed in Splunk entity names.""" - pass class NoSuchCapability(Exception): """Thrown when the capability that has been referred to doesn't exist.""" - pass class OperationError(Exception): - """Raised for a failed operation, such as a time out.""" - pass + """Raised for a failed operation, such as a timeout.""" class NotSupportedError(Exception): """Raised for operations that are not supported on a given object.""" - pass def _trailing(template, *targets): @@ -190,8 +182,9 @@ def _trailing(template, *targets): def _filter_content(content, *args): if len(args) > 0: return record((k, content[k]) for k in args) - return record((k, v) for k, v in six.iteritems(content) - if k not in ['eai:acl', 'eai:attributes', 'type']) + return record((k, v) for k, v in content.items() + if k not in ['eai:acl', 'eai:attributes', 'type']) + # Construct a resource path from the given base path + resource name def _path(base, name): @@ -221,10 +214,9 @@ def _load_atom_entries(response): # its state wrapped in another element, but at the top level. # For example, in XML, it returns ... instead of # .... - else: - entries = r.get('entry', None) - if entries is None: return None - return entries if isinstance(entries, list) else [entries] + entries = r.get('entry', None) + if entries is None: return None + return entries if isinstance(entries, list) else [entries] # Load the sid from the body of the given response @@ -250,7 +242,7 @@ def _parse_atom_entry(entry): metadata = _parse_atom_metadata(content) # Filter some of the noise out of the content record - content = record((k, v) for k, v in six.iteritems(content) + content = record((k, v) for k, v in content.items() if k not in ['eai:acl', 'eai:attributes']) if 'type' in content: @@ -289,6 +281,7 @@ def _parse_atom_metadata(content): return record({'access': access, 'fields': fields}) + # kwargs: scheme, host, port, app, owner, username, password def connect(**kwargs): """This function connects and logs in to a Splunk instance. @@ -417,8 +410,9 @@ class Service(_BaseService): # Or if you already have a valid cookie s = client.Service(cookie="splunkd_8089=...") """ + def __init__(self, **kwargs): - super(Service, self).__init__(**kwargs) + super().__init__(**kwargs) self._splunk_version = None self._kvstore_owner = None self._instance_type = None @@ -538,8 +532,7 @@ def modular_input_kinds(self): """ if self.splunk_version >= (5,): return ReadOnlyCollection(self, PATH_MODULAR_INPUTS, item=ModularInputKind) - else: - raise IllegalOperationException("Modular inputs are not supported before Splunk version 5.") + raise IllegalOperationException("Modular inputs are not supported before Splunk version 5.") @property def storage_passwords(self): @@ -589,7 +582,7 @@ def restart(self, timeout=None): :param timeout: A timeout period, in seconds. :type timeout: ``integer`` """ - msg = { "value": "Restart requested by " + self.username + "via the Splunk SDK for Python"} + msg = {"value": "Restart requested by " + self.username + "via the Splunk SDK for Python"} # This message will be deleted once the server actually restarts. self.messages.create(name="restart_required", **msg) result = self.post("/services/server/control/restart") @@ -693,13 +686,13 @@ def splunk_version(self): :return: A ``tuple`` of ``integers``. """ if self._splunk_version is None: - self._splunk_version = tuple([int(p) for p in self.info['version'].split('.')]) + self._splunk_version = tuple(int(p) for p in self.info['version'].split('.')) return self._splunk_version @property def splunk_instance(self): if self._instance_type is None : - splunk_info = self.info; + splunk_info = self.info if hasattr(splunk_info, 'instance_type') : self._instance_type = splunk_info['instance_type'] else: @@ -751,13 +744,14 @@ def users(self): return Users(self) -class Endpoint(object): +class Endpoint: """This class represents individual Splunk resources in the Splunk REST API. An ``Endpoint`` object represents a URI, such as ``/services/saved/searches``. This class provides the common functionality of :class:`Collection` and :class:`Entity` (essentially HTTP GET and POST methods). """ + def __init__(self, service, path): self.service = service self.path = path @@ -774,11 +768,11 @@ def get_api_version(self, path): # Default to v1 if undefined in the path # For example, "/services/search/jobs" is using API v1 api_version = 1 - + versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) if versionSearch: api_version = int(versionSearch.group(1)) - + return api_version def get(self, path_segment="", owner=None, app=None, sharing=None, **query): @@ -852,7 +846,7 @@ def get(self, path_segment="", owner=None, app=None, sharing=None, **query): # - Fallback from v2+ to v1 if Splunk Version is < 9. # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): # path = path.replace(PATH_JOBS_V2, PATH_JOBS) - + if api_version == 1: if isinstance(path, UrlEncoded): path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) @@ -911,14 +905,14 @@ def post(self, path_segment="", owner=None, app=None, sharing=None, **query): apps.get('nonexistant/path') # raises HTTPError s.logout() apps.get() # raises AuthenticationError - """ + """ if path_segment.startswith('/'): path = path_segment else: if not self.path.endswith('/') and path_segment != "": self.path = self.path + '/' path = self.service._abspath(self.path + path_segment, owner=owner, app=app, sharing=sharing) - + # Get the API version from the path api_version = self.get_api_version(path) @@ -927,7 +921,7 @@ def post(self, path_segment="", owner=None, app=None, sharing=None, **query): # - Fallback from v2+ to v1 if Splunk Version is < 9. # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): # path = path.replace(PATH_JOBS_V2, PATH_JOBS) - + if api_version == 1: if isinstance(path, UrlEncoded): path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) @@ -1003,7 +997,6 @@ def __init__(self, service, path, **kwargs): self._state = None if not kwargs.get('skip_refresh', False): self.refresh(kwargs.get('state', None)) # "Prefresh" - return def __contains__(self, item): try: @@ -1028,24 +1021,21 @@ def __eq__(self, other): but then ``x != saved_searches['asearch']``. whether or not there was a change on the server. Rather than - try to do something fancy, we simple declare that equality is + try to do something fancy, we simply declare that equality is undefined for Entities. Makes no roundtrips to the server. """ - raise IncomparableException( - "Equality is undefined for objects of class %s" % \ - self.__class__.__name__) + raise IncomparableException(f"Equality is undefined for objects of class {self.__class__.__name__}") def __getattr__(self, key): # Called when an attribute was not found by the normal method. In this # case we try to find it in self.content and then self.defaults. if key in self.state.content: return self.state.content[key] - elif key in self.defaults: + if key in self.defaults: return self.defaults[key] - else: - raise AttributeError(key) + raise AttributeError(key) def __getitem__(self, key): # getattr attempts to find a field on the object in the normal way, @@ -1061,9 +1051,8 @@ def _load_atom_entry(self, response): apps = [ele.entry.content.get('eai:appName') for ele in elem] raise AmbiguousReferenceException( - "Fetch from server returned multiple entries for name '%s' in apps %s." % (elem[0].entry.title, apps)) - else: - return elem.entry + f"Fetch from server returned multiple entries for name '{elem[0].entry.title}' in apps {apps}.") + return elem.entry # Load the entity state record from the given response def _load_state(self, response): @@ -1096,17 +1085,15 @@ def _proper_namespace(self, owner=None, app=None, sharing=None): :param sharing: :return: """ - if owner is None and app is None and sharing is None: # No namespace provided + if owner is None and app is None and sharing is None: # No namespace provided if self._state is not None and 'access' in self._state: return (self._state.access.owner, self._state.access.app, self._state.access.sharing) - else: - return (self.service.namespace['owner'], + return (self.service.namespace['owner'], self.service.namespace['app'], self.service.namespace['sharing']) - else: - return (owner,app,sharing) + return owner, app, sharing def delete(self): owner, app, sharing = self._proper_namespace() @@ -1114,11 +1101,11 @@ def delete(self): def get(self, path_segment="", owner=None, app=None, sharing=None, **query): owner, app, sharing = self._proper_namespace(owner, app, sharing) - return super(Entity, self).get(path_segment, owner=owner, app=app, sharing=sharing, **query) + return super().get(path_segment, owner=owner, app=app, sharing=sharing, **query) def post(self, path_segment="", owner=None, app=None, sharing=None, **query): owner, app, sharing = self._proper_namespace(owner, app, sharing) - return super(Entity, self).post(path_segment, owner=owner, app=app, sharing=sharing, **query) + return super().post(path_segment, owner=owner, app=app, sharing=sharing, **query) def refresh(self, state=None): """Refreshes the state of this entity. @@ -1206,8 +1193,8 @@ def read(self, response): # In lower layers of the SDK, we end up trying to URL encode # text to be dispatched via HTTP. However, these links are already # URL encoded when they arrive, and we need to mark them as such. - unquoted_links = dict([(k, UrlEncoded(v, skip_encode=True)) - for k,v in six.iteritems(results['links'])]) + unquoted_links = dict((k, UrlEncoded(v, skip_encode=True)) + for k, v in results['links'].items()) results['links'] = unquoted_links return results @@ -1282,7 +1269,7 @@ def update(self, **kwargs): """ # The peculiarity in question: the REST API creates a new # Entity if we pass name in the dictionary, instead of the - # expected behavior of updating this Entity. Therefore we + # expected behavior of updating this Entity. Therefore, we # check for 'name' in kwargs and throw an error if it is # there. if 'name' in kwargs: @@ -1295,9 +1282,10 @@ class ReadOnlyCollection(Endpoint): """This class represents a read-only collection of entities in the Splunk instance. """ + def __init__(self, service, path, item=Entity): Endpoint.__init__(self, service, path) - self.item = item # Item accessor + self.item = item # Item accessor self.null_count = -1 def __contains__(self, name): @@ -1329,7 +1317,7 @@ def __getitem__(self, key): name. Where there is no conflict, ``__getitem__`` will fetch the - entity given just the name. If there is a conflict and you + entity given just the name. If there is a conflict, and you pass just a name, it will raise a ``ValueError``. In that case, add the namespace as a second argument. @@ -1376,13 +1364,13 @@ def __getitem__(self, key): response = self.get(key) entries = self._load_list(response) if len(entries) > 1: - raise AmbiguousReferenceException("Found multiple entities named '%s'; please specify a namespace." % key) - elif len(entries) == 0: + raise AmbiguousReferenceException( + f"Found multiple entities named '{key}'; please specify a namespace.") + if len(entries) == 0: raise KeyError(key) - else: - return entries[0] + return entries[0] except HTTPError as he: - if he.status == 404: # No entity matching key and namespace. + if he.status == 404: # No entity matching key and namespace. raise KeyError(key) else: raise @@ -1405,7 +1393,7 @@ def __iter__(self, **kwargs): c = client.connect(...) saved_searches = c.saved_searches for entity in saved_searches: - print "Saved search named %s" % entity.name + print(f"Saved search named {entity.name}") """ for item in self.iter(**kwargs): @@ -1446,13 +1434,12 @@ def _entity_path(self, state): # This has been factored out so that it can be easily # overloaded by Configurations, which has to switch its # entities' endpoints from its own properties/ to configs/. - raw_path = urllib.parse.unquote(state.links.alternate) + raw_path = parse.unquote(state.links.alternate) if 'servicesNS/' in raw_path: return _trailing(raw_path, 'servicesNS/', '/', '/') - elif 'services/' in raw_path: + if 'services/' in raw_path: return _trailing(raw_path, 'services/') - else: - return raw_path + return raw_path def _load_list(self, response): """Converts *response* to a list of entities. @@ -1615,8 +1602,6 @@ def list(self, count=None, **kwargs): return list(self.iter(count=count, **kwargs)) - - class Collection(ReadOnlyCollection): """A collection of entities. @@ -1690,8 +1675,8 @@ def create(self, name, **params): applications = s.apps new_app = applications.create("my_fake_app") """ - if not isinstance(name, six.string_types): - raise InvalidNameException("%s is not a valid name for an entity." % name) + if not isinstance(name, str): + raise InvalidNameException(f"{name} is not a valid name for an entity.") if 'namespace' in params: namespace = params.pop('namespace') params['owner'] = namespace.owner @@ -1703,14 +1688,13 @@ def create(self, name, **params): # This endpoint doesn't return the content of the new # item. We have to go fetch it ourselves. return self[name] - else: - entry = atom.entry - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - self._entity_path(state), - state=state) - return entity + entry = atom.entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + self._entity_path(state), + state=state) + return entity def delete(self, name, **params): """Deletes a specified entity from the collection. @@ -1750,7 +1734,7 @@ def delete(self, name, **params): # has already been deleted, and we reraise it as a # KeyError. if he.status == 404: - raise KeyError("No such entity %s" % name) + raise KeyError(f"No such entity {name}") else: raise return self @@ -1801,14 +1785,13 @@ def get(self, name="", owner=None, app=None, sharing=None, **query): """ name = UrlEncoded(name, encode_slash=True) - return super(Collection, self).get(name, owner, app, sharing, **query) - - + return super().get(name, owner, app, sharing, **query) class ConfigurationFile(Collection): """This class contains all of the stanzas from one configuration file. """ + # __init__'s arguments must match those of an Entity, not a # Collection, since it is being created as the elements of a # Configurations, which is a Collection subclass. @@ -1825,6 +1808,7 @@ class Configurations(Collection): stanzas. This collection is unusual in that the values in it are themselves collections of :class:`ConfigurationFile` objects. """ + def __init__(self, service): Collection.__init__(self, service, PATH_PROPERTIES, item=ConfigurationFile) if self.service.namespace.owner == '-' or self.service.namespace.app == '-': @@ -1839,10 +1823,10 @@ def __getitem__(self, key): # This screws up the default implementation of __getitem__ from Collection, which thinks # that multiple entities means a name collision, so we have to override it here. try: - response = self.get(key) + self.get(key) return ConfigurationFile(self.service, PATH_CONF % key, state={'title': key}) except HTTPError as he: - if he.status == 404: # No entity matching key + if he.status == 404: # No entity matching key raise KeyError(key) else: raise @@ -1851,13 +1835,12 @@ def __contains__(self, key): # configs/conf-{name} never returns a 404. We have to post to properties/{name} # in order to find out if a configuration exists. try: - response = self.get(key) + self.get(key) return True except HTTPError as he: - if he.status == 404: # No entity matching key + if he.status == 404: # No entity matching key return False - else: - raise + raise def create(self, name): """ Creates a configuration file named *name*. @@ -1873,15 +1856,14 @@ def create(self, name): # This has to be overridden to handle the plumbing of creating # a ConfigurationFile (which is a Collection) instead of some # Entity. - if not isinstance(name, six.string_types): - raise ValueError("Invalid name: %s" % repr(name)) + if not isinstance(name, str): + raise ValueError(f"Invalid name: {repr(name)}") response = self.post(__conf=name) if response.status == 303: return self[name] - elif response.status == 201: + if response.status == 201: return ConfigurationFile(self.service, PATH_CONF % name, item=Stanza, state={'title': name}) - else: - raise ValueError("Unexpected status code %s returned from creating a stanza" % response.status) + raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") def delete(self, key): """Raises `IllegalOperationException`.""" @@ -1920,10 +1902,11 @@ def __len__(self): class StoragePassword(Entity): """This class contains a storage password. """ + def __init__(self, service, path, **kwargs): state = kwargs.get('state', None) kwargs['skip_refresh'] = kwargs.get('skip_refresh', state is not None) - super(StoragePassword, self).__init__(service, path, **kwargs) + super().__init__(service, path, **kwargs) self._state = state @property @@ -1947,8 +1930,11 @@ class StoragePasswords(Collection): """This class provides access to the storage passwords from this Splunk instance. Retrieve this collection using :meth:`Service.storage_passwords`. """ + def __init__(self, service): - super(StoragePasswords, self).__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) + if service.namespace.owner == '-' or service.namespace.app == '-': + raise ValueError("StoragePasswords cannot have wildcards in namespace.") + super().__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) def create(self, password, username, realm=None): """ Creates a storage password. @@ -1965,11 +1951,8 @@ def create(self, password, username, realm=None): :return: The :class:`StoragePassword` object created. """ - if self.service.namespace.owner == '-' or self.service.namespace.app == '-': - raise ValueError("While creating StoragePasswords, namespace cannot have wildcards.") - - if not isinstance(username, six.string_types): - raise ValueError("Invalid name: %s" % repr(username)) + if not isinstance(username, str): + raise ValueError(f"Invalid name: {repr(username)}") if realm is None: response = self.post(password=password, name=username) @@ -1977,7 +1960,7 @@ def create(self, password, username, realm=None): response = self.post(password=password, realm=realm, name=username) if response.status != 201: - raise ValueError("Unexpected status code %s returned from creating a stanza" % response.status) + raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") entries = _load_atom_entries(response) state = _parse_atom_entry(entries[0]) @@ -1999,9 +1982,6 @@ def delete(self, username, realm=None): :return: The `StoragePassword` collection. :rtype: ``self`` """ - if self.service.namespace.owner == '-' or self.service.namespace.app == '-': - raise ValueError("app context must be specified when removing a password.") - if realm is None: # This case makes the username optional, so # the full name can be passed in as realm. @@ -2020,6 +2000,7 @@ def delete(self, username, realm=None): class AlertGroup(Entity): """This class represents a group of fired alerts for a saved search. Access it using the :meth:`alerts` property.""" + def __init__(self, service, path, **kwargs): Entity.__init__(self, service, path, **kwargs) @@ -2048,6 +2029,7 @@ class Indexes(Collection): """This class contains the collection of indexes in this Splunk instance. Retrieve this collection using :meth:`Service.indexes`. """ + def get_default(self): """ Returns the name of the default index. @@ -2075,6 +2057,7 @@ def delete(self, name): class Index(Entity): """This class represents an index and provides different operations, such as cleaning the index, writing to the index, and so forth.""" + def __init__(self, service, path, **kwargs): Entity.__init__(self, service, path, **kwargs) @@ -2091,26 +2074,26 @@ def attach(self, host=None, source=None, sourcetype=None): :return: A writable socket. """ - args = { 'index': self.name } + args = {'index': self.name} if host is not None: args['host'] = host if source is not None: args['source'] = source if sourcetype is not None: args['sourcetype'] = sourcetype - path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + urllib.parse.urlencode(args), skip_encode=True) + path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + parse.urlencode(args), skip_encode=True) - cookie_or_auth_header = "Authorization: Splunk %s\r\n" % \ - (self.service.token if self.service.token is _NoAuthenticationToken - else self.service.token.replace("Splunk ", "")) + cookie_header = self.service.token if self.service.token is _NoAuthenticationToken else self.service.token.replace("Splunk ", "") + cookie_or_auth_header = f"Authorization: Splunk {cookie_header}\r\n" # If we have cookie(s), use them instead of "Authorization: ..." if self.service.has_cookies(): - cookie_or_auth_header = "Cookie: %s\r\n" % _make_cookie_header(self.service.get_cookies().items()) + cookie_header = _make_cookie_header(self.service.get_cookies().items()) + cookie_or_auth_header = f"Cookie: {cookie_header}\r\n" # Since we need to stream to the index connection, we have to keep # the connection open and use the Splunk extension headers to note # the input mode sock = self.service.connect() - headers = [("POST %s HTTP/1.1\r\n" % str(self.service._abspath(path))).encode('utf-8'), - ("Host: %s:%s\r\n" % (self.service.host, int(self.service.port))).encode('utf-8'), + headers = [f"POST {str(self.service._abspath(path))} HTTP/1.1\r\n".encode('utf-8'), + f"Host: {self.service.host}:{int(self.service.port)}\r\n".encode('utf-8'), b"Accept-Encoding: identity\r\n", cookie_or_auth_header.encode('utf-8'), b"X-Splunk-Input-Mode: Streaming\r\n", @@ -2172,8 +2155,7 @@ def clean(self, timeout=60): ftp = self['frozenTimePeriodInSecs'] was_disabled_initially = self.disabled try: - if (not was_disabled_initially and \ - self.service.splunk_version < (5,)): + if not was_disabled_initially and self.service.splunk_version < (5,): # Need to disable the index first on Splunk 4.x, # but it doesn't work to disable it on 5.0. self.disable() @@ -2183,17 +2165,17 @@ def clean(self, timeout=60): # Wait until event count goes to 0. start = datetime.now() diff = timedelta(seconds=timeout) - while self.content.totalEventCount != '0' and datetime.now() < start+diff: + while self.content.totalEventCount != '0' and datetime.now() < start + diff: sleep(1) self.refresh() if self.content.totalEventCount != '0': - raise OperationError("Cleaning index %s took longer than %s seconds; timing out." % (self.name, timeout)) + raise OperationError( + f"Cleaning index {self.name} took longer than {timeout} seconds; timing out.") finally: # Restore original values self.update(maxTotalDataSizeMB=tds, frozenTimePeriodInSecs=ftp) - if (not was_disabled_initially and \ - self.service.splunk_version < (5,)): + if not was_disabled_initially and self.service.splunk_version < (5,): # Re-enable the index if it was originally enabled and we messed with it. self.enable() @@ -2221,7 +2203,7 @@ def submit(self, event, host=None, source=None, sourcetype=None): :return: The :class:`Index`. """ - args = { 'index': self.name } + args = {'index': self.name} if host is not None: args['host'] = host if source is not None: args['source'] = source if sourcetype is not None: args['sourcetype'] = sourcetype @@ -2255,6 +2237,7 @@ class Input(Entity): typed input classes and is also used when the client does not recognize an input kind. """ + def __init__(self, service, path, kind=None, **kwargs): # kind can be omitted (in which case it is inferred from the path) # Otherwise, valid values are the paths from data/inputs ("udp", @@ -2265,7 +2248,7 @@ def __init__(self, service, path, kind=None, **kwargs): path_segments = path.split('/') i = path_segments.index('inputs') + 1 if path_segments[i] == 'tcp': - self.kind = path_segments[i] + '/' + path_segments[i+1] + self.kind = path_segments[i] + '/' + path_segments[i + 1] else: self.kind = path_segments[i] else: @@ -2291,7 +2274,7 @@ def update(self, **kwargs): # UDP and TCP inputs require special handling due to their restrictToHost # field. For all other inputs kinds, we can dispatch to the superclass method. if self.kind not in ['tcp', 'splunktcp', 'tcp/raw', 'tcp/cooked', 'udp']: - return super(Input, self).update(**kwargs) + return super().update(**kwargs) else: # The behavior of restrictToHost is inconsistent across input kinds and versions of Splunk. # In Splunk 4.x, the name of the entity is only the port, independent of the value of @@ -2309,11 +2292,11 @@ def update(self, **kwargs): if 'restrictToHost' in kwargs: raise IllegalOperationException("Cannot set restrictToHost on an existing input with the SDK.") - elif 'restrictToHost' in self._state.content and self.kind != 'udp': + if 'restrictToHost' in self._state.content and self.kind != 'udp': to_update['restrictToHost'] = self._state.content['restrictToHost'] # Do the actual update operation. - return super(Input, self).update(**to_update) + return super().update(**to_update) # Inputs is a "kinded" collection, which is a heterogenous collection where @@ -2340,13 +2323,12 @@ def __getitem__(self, key): response = self.get(self.kindpath(kind) + "/" + key) entries = self._load_list(response) if len(entries) > 1: - raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key)) - elif len(entries) == 0: + raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") + if len(entries) == 0: raise KeyError((key, kind)) - else: - return entries[0] + return entries[0] except HTTPError as he: - if he.status == 404: # No entity matching kind and key + if he.status == 404: # No entity matching kind and key raise KeyError((key, kind)) else: raise @@ -2360,22 +2342,22 @@ def __getitem__(self, key): response = self.get(kind + "/" + key) entries = self._load_list(response) if len(entries) > 1: - raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key)) - elif len(entries) == 0: + raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") + if len(entries) == 0: pass else: - if candidate is not None: # Already found at least one candidate - raise AmbiguousReferenceException("Found multiple inputs named %s, please specify a kind" % key) + if candidate is not None: # Already found at least one candidate + raise AmbiguousReferenceException( + f"Found multiple inputs named {key}, please specify a kind") candidate = entries[0] except HTTPError as he: if he.status == 404: - pass # Just carry on to the next kind. + pass # Just carry on to the next kind. else: raise if candidate is None: - raise KeyError(key) # Never found a match. - else: - return candidate + raise KeyError(key) # Never found a match. + return candidate def __contains__(self, key): if isinstance(key, tuple) and len(key) == 2: @@ -2395,11 +2377,9 @@ def __contains__(self, key): entries = self._load_list(response) if len(entries) > 0: return True - else: - pass except HTTPError as he: if he.status == 404: - pass # Just carry on to the next kind. + pass # Just carry on to the next kind. else: raise return False @@ -2451,9 +2431,8 @@ def create(self, name, kind, **kwargs): name = UrlEncoded(name, encode_slash=True) path = _path( self.path + kindpath, - '%s:%s' % (kwargs['restrictToHost'], name) \ - if 'restrictToHost' in kwargs else name - ) + f"{kwargs['restrictToHost']}:{name}" if 'restrictToHost' in kwargs else name + ) return Input(self.service, path, kind) def delete(self, name, kind=None): @@ -2523,7 +2502,7 @@ def itemmeta(self, kind): :return: The metadata. :rtype: class:``splunklib.data.Record`` """ - response = self.get("%s/_new" % self._kindmap[kind]) + response = self.get(f"{self._kindmap[kind]}/_new") content = _load_atom(response, MATCH_ENTRY_CONTENT) return _parse_atom_metadata(content) @@ -2538,9 +2517,9 @@ def _get_kind_list(self, subpath=None): this_subpath = subpath + [entry.title] # The "all" endpoint doesn't work yet. # The "tcp/ssl" endpoint is not a real input collection. - if entry.title == 'all' or this_subpath == ['tcp','ssl']: + if entry.title == 'all' or this_subpath == ['tcp', 'ssl']: continue - elif 'create' in [x.rel for x in entry.link]: + if 'create' in [x.rel for x in entry.link]: path = '/'.join(subpath + [entry.title]) kinds.append(path) else: @@ -2589,10 +2568,9 @@ def kindpath(self, kind): """ if kind == 'tcp': return UrlEncoded('tcp/raw', skip_encode=True) - elif kind == 'splunktcp': + if kind == 'splunktcp': return UrlEncoded('tcp/cooked', skip_encode=True) - else: - return UrlEncoded(kind, skip_encode=True) + return UrlEncoded(kind, skip_encode=True) def list(self, *kinds, **kwargs): """Returns a list of inputs that are in the :class:`Inputs` collection. @@ -2660,18 +2638,18 @@ def list(self, *kinds, **kwargs): path = UrlEncoded(path, skip_encode=True) response = self.get(path, **kwargs) except HTTPError as he: - if he.status == 404: # No inputs of this kind + if he.status == 404: # No inputs of this kind return [] entities = [] entries = _load_atom_entries(response) if entries is None: - return [] # No inputs in a collection comes back with no feed or entry in the XML + return [] # No inputs in a collection comes back with no feed or entry in the XML for entry in entries: state = _parse_atom_entry(entry) # Unquote the URL, since all URL encoded in the SDK # should be of type UrlEncoded, and all str should not # be URL encoded. - path = urllib.parse.unquote(state.links.alternate) + path = parse.unquote(state.links.alternate) entity = Input(self.service, path, kind, state=state) entities.append(entity) return entities @@ -2686,18 +2664,18 @@ def list(self, *kinds, **kwargs): response = self.get(self.kindpath(kind), search=search) except HTTPError as e: if e.status == 404: - continue # No inputs of this kind + continue # No inputs of this kind else: raise entries = _load_atom_entries(response) - if entries is None: continue # No inputs to process + if entries is None: continue # No inputs to process for entry in entries: state = _parse_atom_entry(entry) # Unquote the URL, since all URL encoded in the SDK # should be of type UrlEncoded, and all str should not # be URL encoded. - path = urllib.parse.unquote(state.links.alternate) + path = parse.unquote(state.links.alternate) entity = Input(self.service, path, kind, state=state) entities.append(entity) if 'offset' in kwargs: @@ -2765,6 +2743,7 @@ def oneshot(self, path, **kwargs): class Job(Entity): """This class represents a search job.""" + def __init__(self, service, sid, **kwargs): # Default to v2 in Splunk Version 9+ path = "{path}{sid}" @@ -2827,7 +2806,7 @@ def events(self, **kwargs): :return: The ``InputStream`` IO handle to this job's events. """ kwargs['segmentation'] = kwargs.get('segmentation', 'none') - + # Search API v1(GET) and v2(POST) if self.service.disable_v2_api: return self.get("events", **kwargs).body @@ -2898,10 +2877,10 @@ def results(self, **query_params): for result in rr: if isinstance(result, results.Message): # Diagnostic messages may be returned in the results - print '%s: %s' % (result.type, result.message) + print(f'{result.type}: {result.message}') elif isinstance(result, dict): # Normal events are returned as dicts - print result + print(result) assert rr.is_preview == False Results are not available until the job has finished. If called on @@ -2919,7 +2898,7 @@ def results(self, **query_params): :return: The ``InputStream`` IO handle to this job's results. """ query_params['segmentation'] = query_params.get('segmentation', 'none') - + # Search API v1(GET) and v2(POST) if self.service.disable_v2_api: return self.get("results", **query_params).body @@ -2942,14 +2921,14 @@ def preview(self, **query_params): for result in rr: if isinstance(result, results.Message): # Diagnostic messages may be returned in the results - print '%s: %s' % (result.type, result.message) + print(f'{result.type}: {result.message}') elif isinstance(result, dict): # Normal events are returned as dicts - print result + print(result) if rr.is_preview: - print "Preview of a running search job." + print("Preview of a running search job.") else: - print "Job is finished. Results are final." + print("Job is finished. Results are final.") This method makes one roundtrip to the server, plus at most two more if @@ -2964,7 +2943,7 @@ def preview(self, **query_params): :return: The ``InputStream`` IO handle to this job's preview results. """ query_params['segmentation'] = query_params.get('segmentation', 'none') - + # Search API v1(GET) and v2(POST) if self.service.disable_v2_api: return self.get("results_preview", **query_params).body @@ -3056,6 +3035,7 @@ def unpause(self): class Jobs(Collection): """This class represents a collection of search jobs. Retrieve this collection using :meth:`Service.jobs`.""" + def __init__(self, service): # Splunk 9 introduces the v2 endpoint if not service.disable_v2_api: @@ -3114,10 +3094,10 @@ def export(self, query, **params): for result in rr: if isinstance(result, results.Message): # Diagnostic messages may be returned in the results - print '%s: %s' % (result.type, result.message) + print(f'{result.type}: {result.message}') elif isinstance(result, dict): # Normal events are returned as dicts - print result + print(result) assert rr.is_preview == False Running an export search is more efficient as it streams the results @@ -3170,10 +3150,10 @@ def oneshot(self, query, **params): for result in rr: if isinstance(result, results.Message): # Diagnostic messages may be returned in the results - print '%s: %s' % (result.type, result.message) + print(f'{result.type}: {result.message}') elif isinstance(result, dict): # Normal events are returned as dicts - print result + print(result) assert rr.is_preview == False The ``oneshot`` method makes a single roundtrip to the server (as opposed @@ -3214,6 +3194,7 @@ def oneshot(self, query, **params): class Loggers(Collection): """This class represents a collection of service logging categories. Retrieve this collection using :meth:`Service.loggers`.""" + def __init__(self, service): Collection.__init__(self, service, PATH_LOGGER) @@ -3245,19 +3226,18 @@ class ModularInputKind(Entity): """This class contains the different types of modular inputs. Retrieve this collection using :meth:`Service.modular_input_kinds`. """ + def __contains__(self, name): args = self.state.content['endpoints']['args'] if name in args: return True - else: - return Entity.__contains__(self, name) + return Entity.__contains__(self, name) def __getitem__(self, name): args = self.state.content['endpoint']['args'] if name in args: return args['item'] - else: - return Entity.__getitem__(self, name) + return Entity.__getitem__(self, name) @property def arguments(self): @@ -3282,6 +3262,7 @@ def update(self, **kwargs): class SavedSearch(Entity): """This class represents a saved search.""" + def __init__(self, service, path, **kwargs): Entity.__init__(self, service, path, **kwargs) @@ -3423,8 +3404,7 @@ def suppressed(self): r = self._run_action("suppress") if r.suppressed == "1": return int(r.expiration) - else: - return 0 + return 0 def unsuppress(self): """Cancels suppression and makes this search run as scheduled. @@ -3438,6 +3418,7 @@ def unsuppress(self): class SavedSearches(Collection): """This class represents a collection of saved searches. Retrieve this collection using :meth:`Service.saved_searches`.""" + def __init__(self, service): Collection.__init__( self, service, PATH_SAVED_SEARCHES, item=SavedSearch) @@ -3462,6 +3443,7 @@ def create(self, name, search, **kwargs): class Settings(Entity): """This class represents configuration settings for a Splunk service. Retrieve this collection using :meth:`Service.settings`.""" + def __init__(self, service, **kwargs): Entity.__init__(self, service, "/services/server/settings", **kwargs) @@ -3483,6 +3465,7 @@ def update(self, **kwargs): class User(Entity): """This class represents a Splunk user. """ + @property def role_entities(self): """Returns a list of roles assigned to this user. @@ -3490,7 +3473,8 @@ def role_entities(self): :return: The list of roles. :rtype: ``list`` """ - return [self.service.roles[name] for name in self.content.roles] + all_role_names = [r.name for r in self.service.roles.list()] + return [self.service.roles[name] for name in self.content.roles if name in all_role_names] # Splunk automatically lowercases new user names so we need to match that @@ -3499,6 +3483,7 @@ class Users(Collection): """This class represents the collection of Splunk users for this instance of Splunk. Retrieve this collection using :meth:`Service.users`. """ + def __init__(self, service): Collection.__init__(self, service, PATH_USERS, item=User) @@ -3538,8 +3523,8 @@ def create(self, username, password, roles, **params): boris = users.create("boris", "securepassword", roles="user") hilda = users.create("hilda", "anotherpassword", roles=["user","power"]) """ - if not isinstance(username, six.string_types): - raise ValueError("Invalid username: %s" % str(username)) + if not isinstance(username, str): + raise ValueError(f"Invalid username: {str(username)}") username = username.lower() self.post(name=username, password=password, roles=roles, **params) # splunkd doesn't return the user in the POST response body, @@ -3549,7 +3534,7 @@ def create(self, username, password, roles, **params): state = _parse_atom_entry(entry) entity = self.item( self.service, - urllib.parse.unquote(state.links.alternate), + parse.unquote(state.links.alternate), state=state) return entity @@ -3568,6 +3553,7 @@ def delete(self, name): class Role(Entity): """This class represents a user role. """ + def grant(self, *capabilities_to_grant): """Grants additional capabilities to this role. @@ -3618,8 +3604,8 @@ def revoke(self, *capabilities_to_revoke): for c in old_capabilities: if c not in capabilities_to_revoke: new_capabilities.append(c) - if new_capabilities == []: - new_capabilities = '' # Empty lists don't get passed in the body, so we have to force an empty argument. + if not new_capabilities: + new_capabilities = '' # Empty lists don't get passed in the body, so we have to force an empty argument. self.post(capabilities=new_capabilities) return self @@ -3627,8 +3613,9 @@ def revoke(self, *capabilities_to_revoke): class Roles(Collection): """This class represents the collection of roles in the Splunk instance. Retrieve this collection using :meth:`Service.roles`.""" + def __init__(self, service): - return Collection.__init__(self, service, PATH_ROLES, item=Role) + Collection.__init__(self, service, PATH_ROLES, item=Role) def __getitem__(self, key): return Collection.__getitem__(self, key.lower()) @@ -3661,8 +3648,8 @@ def create(self, name, **params): roles = c.roles paltry = roles.create("paltry", imported_roles="user", defaultApp="search") """ - if not isinstance(name, six.string_types): - raise ValueError("Invalid role name: %s" % str(name)) + if not isinstance(name, str): + raise ValueError(f"Invalid role name: {str(name)}") name = name.lower() self.post(name=name, **params) # splunkd doesn't return the user in the POST response body, @@ -3672,7 +3659,7 @@ def create(self, name, **params): state = _parse_atom_entry(entry) entity = self.item( self.service, - urllib.parse.unquote(state.links.alternate), + parse.unquote(state.links.alternate), state=state) return entity @@ -3689,6 +3676,7 @@ def delete(self, name): class Application(Entity): """Represents a locally-installed Splunk app.""" + @property def setupInfo(self): """Returns the setup information for the app. @@ -3705,6 +3693,7 @@ def updateInfo(self): """Returns any update information that is available for the app.""" return self._run_action("update") + class KVStoreCollections(Collection): def __init__(self, service): Collection.__init__(self, service, 'storage/collections/config', item=KVStoreCollection) @@ -3730,14 +3719,15 @@ def create(self, name, accelerated_fields={}, fields={}, **kwargs): :return: Result of POST request """ - for k, v in six.iteritems(accelerated_fields): + for k, v in accelerated_fields.items(): if isinstance(v, dict): v = json.dumps(v) kwargs['accelerated_fields.' + k] = v - for k, v in six.iteritems(fields): + for k, v in fields.items(): kwargs['field.' + k] = v return self.post(name=name, **kwargs) + class KVStoreCollection(Entity): @property def data(self): @@ -3758,9 +3748,7 @@ def update_accelerated_field(self, name, value): :return: Result of POST request """ kwargs = {} - if isinstance(value, dict): - value = json.dumps(value) - kwargs['accelerated_fields.' + name] = value + kwargs['accelerated_fields.' + name] = json.dumps(value) if isinstance(value, dict) else value return self.post(**kwargs) def update_field(self, name, value): @@ -3777,7 +3765,8 @@ def update_field(self, name, value): kwargs['field.' + name] = value return self.post(**kwargs) -class KVStoreCollectionData(object): + +class KVStoreCollectionData: """This class represents the data endpoint for a KVStoreCollection. Retrieve using :meth:`KVStoreCollection.data` @@ -3840,7 +3829,8 @@ def insert(self, data): """ if isinstance(data, dict): data = json.dumps(data) - return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + return json.loads( + self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) def delete(self, query=None): """ @@ -3878,7 +3868,8 @@ def update(self, id, data): """ if isinstance(data, dict): data = json.dumps(data) - return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, + body=data).body.read().decode('utf-8')) def batch_find(self, *dbqueries): """ @@ -3895,7 +3886,8 @@ def batch_find(self, *dbqueries): data = json.dumps(dbqueries) - return json.loads(self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + return json.loads( + self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) def batch_save(self, *documents): """ @@ -3912,4 +3904,5 @@ def batch_save(self, *documents): data = json.dumps(documents) - return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) \ No newline at end of file + return json.loads( + self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) diff --git a/lib/splunklib/data.py b/lib/splunklib/data.py index f9ffb86..34f3ffa 100644 --- a/lib/splunklib/data.py +++ b/lib/splunklib/data.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,16 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -"""The **splunklib.data** module reads the responses from splunkd in Atom Feed +"""The **splunklib.data** module reads the responses from splunkd in Atom Feed format, which is the format used by most of the REST API. """ -from __future__ import absolute_import -import sys from xml.etree.ElementTree import XML -from splunklib import six -__all__ = ["load"] +__all__ = ["load", "record"] # LNAME refers to element names without namespaces; XNAME is the same # name, but with an XML namespace. @@ -36,33 +33,41 @@ XNAME_KEY = XNAMEF_REST % LNAME_KEY XNAME_LIST = XNAMEF_REST % LNAME_LIST + # Some responses don't use namespaces (eg: search/parse) so we look for # both the extended and local versions of the following names. + def isdict(name): - return name == XNAME_DICT or name == LNAME_DICT + return name in (XNAME_DICT, LNAME_DICT) + def isitem(name): - return name == XNAME_ITEM or name == LNAME_ITEM + return name in (XNAME_ITEM, LNAME_ITEM) + def iskey(name): - return name == XNAME_KEY or name == LNAME_KEY + return name in (XNAME_KEY, LNAME_KEY) + def islist(name): - return name == XNAME_LIST or name == LNAME_LIST + return name in (XNAME_LIST, LNAME_LIST) + def hasattrs(element): return len(element.attrib) > 0 + def localname(xname): rcurly = xname.find('}') - return xname if rcurly == -1 else xname[rcurly+1:] + return xname if rcurly == -1 else xname[rcurly + 1:] + def load(text, match=None): - """This function reads a string that contains the XML of an Atom Feed, then - returns the - data in a native Python structure (a ``dict`` or ``list``). If you also - provide a tag name or path to match, only the matching sub-elements are + """This function reads a string that contains the XML of an Atom Feed, then + returns the + data in a native Python structure (a ``dict`` or ``list``). If you also + provide a tag name or path to match, only the matching sub-elements are loaded. :param text: The XML text to load. @@ -78,30 +83,27 @@ def load(text, match=None): 'names': {} } - # Convert to unicode encoding in only python 2 for xml parser - if(sys.version_info < (3, 0, 0) and isinstance(text, unicode)): - text = text.encode('utf-8') - root = XML(text) items = [root] if match is None else root.findall(match) count = len(items) - if count == 0: + if count == 0: return None - elif count == 1: + if count == 1: return load_root(items[0], nametable) - else: - return [load_root(item, nametable) for item in items] + return [load_root(item, nametable) for item in items] + # Load the attributes of the given element. def load_attrs(element): if not hasattrs(element): return None attrs = record() - for key, value in six.iteritems(element.attrib): + for key, value in element.attrib.items(): attrs[key] = value return attrs + # Parse a element and return a Python dict -def load_dict(element, nametable = None): +def load_dict(element, nametable=None): value = record() children = list(element) for child in children: @@ -110,6 +112,7 @@ def load_dict(element, nametable = None): value[name] = load_value(child, nametable) return value + # Loads the given elements attrs & value into single merged dict. def load_elem(element, nametable=None): name = localname(element.tag) @@ -118,12 +121,12 @@ def load_elem(element, nametable=None): if attrs is None: return name, value if value is None: return name, attrs # If value is simple, merge into attrs dict using special key - if isinstance(value, six.string_types): + if isinstance(value, str): attrs["$text"] = value return name, attrs # Both attrs & value are complex, so merge the two dicts, resolving collisions. collision_keys = [] - for key, val in six.iteritems(attrs): + for key, val in attrs.items(): if key in value and key in collision_keys: value[key].append(val) elif key in value and key not in collision_keys: @@ -133,6 +136,7 @@ def load_elem(element, nametable=None): value[key] = val return name, value + # Parse a element and return a Python list def load_list(element, nametable=None): assert islist(element.tag) @@ -143,6 +147,7 @@ def load_list(element, nametable=None): value.append(load_value(child, nametable)) return value + # Load the given root element. def load_root(element, nametable=None): tag = element.tag @@ -151,6 +156,7 @@ def load_root(element, nametable=None): k, v = load_elem(element, nametable) return Record.fromkv(k, v) + # Load the children of the given element. def load_value(element, nametable=None): children = list(element) @@ -159,7 +165,7 @@ def load_value(element, nametable=None): # No children, assume a simple text value if count == 0: text = element.text - if text is None: + if text is None: return None if len(text.strip()) == 0: @@ -179,7 +185,7 @@ def load_value(element, nametable=None): # If we have seen this name before, promote the value to a list if name in value: current = value[name] - if not isinstance(current, list): + if not isinstance(current, list): value[name] = [current] value[name].append(item) else: @@ -187,23 +193,24 @@ def load_value(element, nametable=None): return value + # A generic utility that enables "dot" access to dicts class Record(dict): - """This generic utility class enables dot access to members of a Python + """This generic utility class enables dot access to members of a Python dictionary. - Any key that is also a valid Python identifier can be retrieved as a field. - So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to - ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be - retrieved as a field, because ``-`` and ``.`` are not allowed in + Any key that is also a valid Python identifier can be retrieved as a field. + So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to + ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be + retrieved as a field, because ``-`` and ``.`` are not allowed in identifiers. - Keys of the form ``a.b.c`` are very natural to write in Python as fields. If - a group of keys shares a prefix ending in ``.``, you can retrieve keys as a + Keys of the form ``a.b.c`` are very natural to write in Python as fields. If + a group of keys shares a prefix ending in ``.``, you can retrieve keys as a nested dictionary by calling only the prefix. For example, if ``r`` contains keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record - with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each - one is placed into a nested dictionary, so you can write ``r.bar.qux`` or + with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each + one is placed into a nested dictionary, so you can write ``r.bar.qux`` or ``r['bar.qux']`` interchangeably. """ sep = '.' @@ -215,7 +222,7 @@ def __call__(self, *args): def __getattr__(self, name): try: return self[name] - except KeyError: + except KeyError: raise AttributeError(name) def __delattr__(self, name): @@ -235,7 +242,7 @@ def __getitem__(self, key): return dict.__getitem__(self, key) key += self.sep result = record() - for k,v in six.iteritems(self): + for k, v in self.items(): if not k.startswith(key): continue suffix = k[len(key):] @@ -250,17 +257,16 @@ def __getitem__(self, key): else: result[suffix] = v if len(result) == 0: - raise KeyError("No key or prefix: %s" % key) + raise KeyError(f"No key or prefix: {key}") return result - -def record(value=None): - """This function returns a :class:`Record` instance constructed with an + +def record(value=None): + """This function returns a :class:`Record` instance constructed with an initial value that you provide. - - :param `value`: An initial record value. - :type `value`: ``dict`` + + :param value: An initial record value. + :type value: ``dict`` """ if value is None: value = {} return Record(value) - diff --git a/lib/splunklib/modularinput/argument.py b/lib/splunklib/modularinput/argument.py index 04214d1..ec64387 100644 --- a/lib/splunklib/modularinput/argument.py +++ b/lib/splunklib/modularinput/argument.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,16 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import -try: - import xml.etree.ElementTree as ET -except ImportError: - import xml.etree.cElementTree as ET +import xml.etree.ElementTree as ET + +class Argument: -class Argument(object): """Class representing an argument to a modular input kind. - ``Argument`` is meant to be used with ``Scheme`` to generate an XML + ``Argument`` is meant to be used with ``Scheme`` to generate an XML definition of the modular input kind that Splunk understands. ``name`` is the only required parameter for the constructor. @@ -100,4 +97,4 @@ def add_to_document(self, parent): for name, value in subelements: ET.SubElement(arg, name).text = str(value).lower() - return arg \ No newline at end of file + return arg diff --git a/lib/splunklib/modularinput/event.py b/lib/splunklib/modularinput/event.py index 9cd6cf3..7ee7266 100644 --- a/lib/splunklib/modularinput/event.py +++ b/lib/splunklib/modularinput/event.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,16 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import from io import TextIOBase -from splunklib.six import ensure_text +import xml.etree.ElementTree as ET -try: - import xml.etree.cElementTree as ET -except ImportError as ie: - import xml.etree.ElementTree as ET +from splunklib.utils import ensure_str -class Event(object): + +class Event: """Represents an event or fragment of an event to be written by this modular input to Splunk. To write an input to a stream, call the ``write_to`` function, passing in a stream. @@ -108,7 +105,7 @@ def write_to(self, stream): ET.SubElement(event, "done") if isinstance(stream, TextIOBase): - stream.write(ensure_text(ET.tostring(event))) + stream.write(ensure_str(ET.tostring(event))) else: stream.write(ET.tostring(event)) - stream.flush() \ No newline at end of file + stream.flush() diff --git a/lib/splunklib/modularinput/event_writer.py b/lib/splunklib/modularinput/event_writer.py index 5f8c5aa..c048a5b 100644 --- a/lib/splunklib/modularinput/event_writer.py +++ b/lib/splunklib/modularinput/event_writer.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,18 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import import sys -from splunklib.six import ensure_str +from splunklib.utils import ensure_str from .event import ET -try: - from splunklib.six.moves import cStringIO as StringIO -except ImportError: - from splunklib.six import StringIO -class EventWriter(object): +class EventWriter: """``EventWriter`` writes events and error messages to Splunk from a modular input. Its two important methods are ``writeEvent``, which takes an ``Event`` object, and ``log``, which takes a severity and an error message. @@ -68,7 +63,7 @@ def log(self, severity, message): :param message: ``string``, message to log. """ - self._err.write("%s %s\n" % (severity, message)) + self._err.write(f"{severity} {message}\n") self._err.flush() def write_xml_document(self, document): @@ -77,11 +72,11 @@ def write_xml_document(self, document): :param document: An ``ElementTree`` object. """ - self._out.write(ensure_str(ET.tostring(document))) + self._out.write(ensure_str(ET.tostring(document), errors="replace")) self._out.flush() def close(self): """Write the closing tag to make this XML well formed.""" if self.header_written: - self._out.write("") + self._out.write("") self._out.flush() diff --git a/lib/splunklib/modularinput/input_definition.py b/lib/splunklib/modularinput/input_definition.py index fdc7cbb..190192f 100644 --- a/lib/splunklib/modularinput/input_definition.py +++ b/lib/splunklib/modularinput/input_definition.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,12 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import -try: - import xml.etree.cElementTree as ET -except ImportError as ie: - import xml.etree.ElementTree as ET - +import xml.etree.ElementTree as ET from .utils import parse_xml_data class InputDefinition: @@ -57,4 +52,4 @@ def parse(stream): else: definition.metadata[node.tag] = node.text - return definition \ No newline at end of file + return definition diff --git a/lib/splunklib/modularinput/scheme.py b/lib/splunklib/modularinput/scheme.py index 4104e4a..a3b0868 100644 --- a/lib/splunklib/modularinput/scheme.py +++ b/lib/splunklib/modularinput/scheme.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,13 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import -try: - import xml.etree.cElementTree as ET -except ImportError: - import xml.etree.ElementTree as ET +import xml.etree.ElementTree as ET -class Scheme(object): + +class Scheme: """Class representing the metadata for a modular input kind. A ``Scheme`` specifies a title, description, several options of how Splunk should run modular inputs of this @@ -82,4 +79,4 @@ def to_xml(self): for arg in self.arguments: arg.add_to_document(args) - return root \ No newline at end of file + return root diff --git a/lib/splunklib/modularinput/script.py b/lib/splunklib/modularinput/script.py index 8595dc4..e912d73 100644 --- a/lib/splunklib/modularinput/script.py +++ b/lib/splunklib/modularinput/script.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -12,24 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import from abc import ABCMeta, abstractmethod -from splunklib.six.moves.urllib.parse import urlsplit import sys +import xml.etree.ElementTree as ET +from urllib.parse import urlsplit from ..client import Service from .event_writer import EventWriter from .input_definition import InputDefinition from .validation_definition import ValidationDefinition -from splunklib import six -try: - import xml.etree.cElementTree as ET -except ImportError: - import xml.etree.ElementTree as ET - -class Script(six.with_metaclass(ABCMeta, object)): +class Script(metaclass=ABCMeta): """An abstract base class for implementing modular inputs. Subclasses should override ``get_scheme``, ``stream_events``, @@ -74,7 +68,7 @@ def run_script(self, args, event_writer, input_stream): event_writer.close() return 0 - elif str(args[1]).lower() == "--scheme": + if str(args[1]).lower() == "--scheme": # Splunk has requested XML specifying the scheme for this # modular input Return it and exit. scheme = self.get_scheme() @@ -83,11 +77,10 @@ def run_script(self, args, event_writer, input_stream): EventWriter.FATAL, "Modular input script returned a null scheme.") return 1 - else: - event_writer.write_xml_document(scheme.to_xml()) - return 0 + event_writer.write_xml_document(scheme.to_xml()) + return 0 - elif args[1].lower() == "--validate-arguments": + if args[1].lower() == "--validate-arguments": validation_definition = ValidationDefinition.parse(input_stream) try: self.validate_input(validation_definition) @@ -98,11 +91,10 @@ def run_script(self, args, event_writer, input_stream): event_writer.write_xml_document(root) return 1 - else: - err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( - args) - event_writer._err.write(err_string) - return 1 + err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( + args) + event_writer._err.write(err_string) + return 1 except Exception as e: event_writer.log(EventWriter.ERROR, str(e)) @@ -165,7 +157,6 @@ def validate_input(self, definition): :param definition: The parameters for the proposed input passed by splunkd. """ - pass @abstractmethod def stream_events(self, inputs, ew): diff --git a/lib/splunklib/modularinput/utils.py b/lib/splunklib/modularinput/utils.py index 3d42b63..dad73dd 100644 --- a/lib/splunklib/modularinput/utils.py +++ b/lib/splunklib/modularinput/utils.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,8 +14,7 @@ # File for utility functions -from __future__ import absolute_import -from splunklib.six.moves import zip + def xml_compare(expected, found): """Checks equality of two ``ElementTree`` objects. @@ -39,27 +38,25 @@ def xml_compare(expected, found): return False # compare children - if not all([xml_compare(a, b) for a, b in zip(expected_children, found_children)]): + if not all(xml_compare(a, b) for a, b in zip(expected_children, found_children)): return False # compare elements, if there is no text node, return True if (expected.text is None or expected.text.strip() == "") \ and (found.text is None or found.text.strip() == ""): return True - else: - return expected.tag == found.tag and expected.text == found.text \ + return expected.tag == found.tag and expected.text == found.text \ and expected.attrib == found.attrib def parse_parameters(param_node): if param_node.tag == "param": return param_node.text - elif param_node.tag == "param_list": + if param_node.tag == "param_list": parameters = [] for mvp in param_node: parameters.append(mvp.text) return parameters - else: - raise ValueError("Invalid configuration scheme, %s tag unexpected." % param_node.tag) + raise ValueError(f"Invalid configuration scheme, {param_node.tag} tag unexpected.") def parse_xml_data(parent_node, child_node_tag): data = {} diff --git a/lib/splunklib/modularinput/validation_definition.py b/lib/splunklib/modularinput/validation_definition.py index 3bbe976..b71e1e7 100644 --- a/lib/splunklib/modularinput/validation_definition.py +++ b/lib/splunklib/modularinput/validation_definition.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -13,16 +13,12 @@ # under the License. -from __future__ import absolute_import -try: - import xml.etree.cElementTree as ET -except ImportError as ie: - import xml.etree.ElementTree as ET +import xml.etree.ElementTree as ET from .utils import parse_xml_data -class ValidationDefinition(object): +class ValidationDefinition: """This class represents the XML sent by Splunk for external validation of a new modular input. @@ -83,4 +79,4 @@ def parse(stream): # Store anything else in metadata definition.metadata[node.tag] = node.text - return definition \ No newline at end of file + return definition diff --git a/lib/splunklib/results.py b/lib/splunklib/results.py index 8543ab0..30476c8 100644 --- a/lib/splunklib/results.py +++ b/lib/splunklib/results.py @@ -1,4 +1,4 @@ -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -29,38 +29,27 @@ reader = ResultsReader(result_stream) for item in reader: print(item) - print "Results are a preview: %s" % reader.is_preview + print(f"Results are a preview: {reader.is_preview}") """ -from __future__ import absolute_import - from io import BufferedReader, BytesIO -from splunklib import six - -from splunklib.six import deprecated -try: - import xml.etree.cElementTree as et -except: - import xml.etree.ElementTree as et +import xml.etree.ElementTree as et from collections import OrderedDict from json import loads as json_loads -try: - from splunklib.six.moves import cStringIO as StringIO -except: - from splunklib.six import StringIO - __all__ = [ "ResultsReader", "Message", "JSONResultsReader" ] +import deprecation -class Message(object): + +class Message: """This class represents informational messages that Splunk interleaves in the results stream. ``Message`` takes two arguments: a string giving the message type (e.g., "DEBUG"), and @@ -76,7 +65,7 @@ def __init__(self, type_, message): self.message = message def __repr__(self): - return "%s: %s" % (self.type, self.message) + return f"{self.type}: {self.message}" def __eq__(self, other): return (self.type, self.message) == (other.type, other.message) @@ -85,7 +74,7 @@ def __hash__(self): return hash((self.type, self.message)) -class _ConcatenatedStream(object): +class _ConcatenatedStream: """Lazily concatenate zero or more streams into a stream. As you read from the concatenated stream, you get characters from @@ -117,7 +106,7 @@ def read(self, n=None): return response -class _XMLDTDFilter(object): +class _XMLDTDFilter: """Lazily remove all XML DTDs from a stream. All substrings matching the regular expression ]*> are @@ -144,7 +133,7 @@ def read(self, n=None): c = self.stream.read(1) if c == b"": break - elif c == b"<": + if c == b"<": c += self.stream.read(1) if c == b"`_ 3. `Configure seach assistant with searchbnf.conf `_ - + 4. `Control search distribution with distsearch.conf `_ """ -from __future__ import absolute_import, division, print_function, unicode_literals - from .environment import * from .decorators import * from .validators import * diff --git a/lib/splunklib/searchcommands/decorators.py b/lib/splunklib/searchcommands/decorators.py index d8b3f48..1393d78 100644 --- a/lib/splunklib/searchcommands/decorators.py +++ b/lib/splunklib/searchcommands/decorators.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,19 +14,16 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals -from splunklib import six - -from collections import OrderedDict # must be python 2.7 +from collections import OrderedDict from inspect import getmembers, isclass, isfunction -from splunklib.six.moves import map as imap + from .internals import ConfigurationSettingsType, json_encode_string from .validators import OptionName -class Configuration(object): +class Configuration: """ Defines the configuration settings for a search command. Documents, validates, and ensures that only relevant configuration settings are applied. Adds a :code:`name` class @@ -69,7 +66,7 @@ def __call__(self, o): name = o.__name__ if name.endswith('Command'): name = name[:-len('Command')] - o.name = six.text_type(name.lower()) + o.name = str(name.lower()) # Construct ConfigurationSettings instance for the command class @@ -82,7 +79,7 @@ def __call__(self, o): o.ConfigurationSettings.fix_up(o) Option.fix_up(o) else: - raise TypeError('Incorrect usage: Configuration decorator applied to {0}'.format(type(o), o.__name__)) + raise TypeError(f'Incorrect usage: Configuration decorator applied to {type(o)}') return o @@ -136,7 +133,7 @@ def fix_up(cls, values): for name, setting in definitions: if setting._name is None: - setting._name = name = six.text_type(name) + setting._name = name = str(name) else: name = setting._name @@ -187,14 +184,14 @@ def is_supported_by_protocol(version): continue if setting.fset is None: - raise ValueError('The value of configuration setting {} is fixed'.format(name)) + raise ValueError(f'The value of configuration setting {name} is fixed') setattr(cls, backing_field_name, validate(specification, name, value)) del values[name] if len(values) > 0: - settings = sorted(list(six.iteritems(values))) - settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings) + settings = sorted(list(values.items())) + settings = [f'{n_v[0]}={n_v[1]}' for n_v in settings] raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings)) cls.configuration_setting_definitions = definitions @@ -212,7 +209,7 @@ def _get_specification(self): try: specification = ConfigurationSettingsType.specification_matrix[name] except KeyError: - raise AttributeError('Unknown configuration setting: {}={}'.format(name, repr(self._value))) + raise AttributeError(f'Unknown configuration setting: {name}={repr(self._value)}') return ConfigurationSettingsType.validate_configuration_setting, specification @@ -346,7 +343,7 @@ def _copy_extra_attributes(self, other): # region Types - class Item(object): + class Item: """ Presents an instance/class view over a search command `Option`. This class is used by SearchCommand.process to parse and report on option values. @@ -357,7 +354,7 @@ def __init__(self, command, option): self._option = option self._is_set = False validator = self.validator - self._format = six.text_type if validator is None else validator.format + self._format = str if validator is None else validator.format def __repr__(self): return '(' + repr(self.name) + ', ' + repr(self._format(self.value)) + ')' @@ -405,7 +402,6 @@ def reset(self): self._option.__set__(self._command, self._option.default) self._is_set = False - pass # endregion class View(OrderedDict): @@ -420,27 +416,26 @@ def __init__(self, command): OrderedDict.__init__(self, ((option.name, item_class(command, option)) for (name, option) in definitions)) def __repr__(self): - text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])' + text = 'Option.View([' + ','.join([repr(item) for item in self.values()]) + '])' return text def __str__(self): - text = ' '.join([str(item) for item in six.itervalues(self) if item.is_set]) + text = ' '.join([str(item) for item in self.values() if item.is_set]) return text # region Methods def get_missing(self): - missing = [item.name for item in six.itervalues(self) if item.is_required and not item.is_set] + missing = [item.name for item in self.values() if item.is_required and not item.is_set] return missing if len(missing) > 0 else None def reset(self): - for value in six.itervalues(self): + for value in self.values(): value.reset() - pass # endregion - pass + # endregion diff --git a/lib/splunklib/searchcommands/environment.py b/lib/splunklib/searchcommands/environment.py index e92018f..35f1dea 100644 --- a/lib/splunklib/searchcommands/environment.py +++ b/lib/splunklib/searchcommands/environment.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,16 +14,15 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals + from logging import getLogger, root, StreamHandler from logging.config import fileConfig -from os import chdir, environ, path -from splunklib.six.moves import getcwd - +from os import chdir, environ, path, getcwd import sys + def configure_logging(logger_name, filename=None): """ Configure logging and return the named logger and the location of the logging configuration file loaded. @@ -88,9 +87,9 @@ def configure_logging(logger_name, filename=None): found = True break if not found: - raise ValueError('Logging configuration file "{}" not found in local or default directory'.format(filename)) + raise ValueError(f'Logging configuration file "{filename}" not found in local or default directory') elif not path.exists(filename): - raise ValueError('Logging configuration file "{}" not found'.format(filename)) + raise ValueError(f'Logging configuration file "{filename}" not found') if filename is not None: global _current_logging_configuration_file diff --git a/lib/splunklib/searchcommands/eventing_command.py b/lib/splunklib/searchcommands/eventing_command.py index 27dc13a..d42d056 100644 --- a/lib/splunklib/searchcommands/eventing_command.py +++ b/lib/splunklib/searchcommands/eventing_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,10 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals -from splunklib import six -from splunklib.six.moves import map as imap from .decorators import ConfigurationSetting from .search_command import SearchCommand @@ -140,10 +137,10 @@ def fix_up(cls, command): # N.B.: Does not use Python 2 dict copy semantics def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) - return imap(lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems) + return [(name_value[0], 'events' if name_value[0] == 'type' else name_value[1]) for name_value in iteritems] # N.B.: Does not use Python 3 dict view semantics - if not six.PY2: - items = iteritems + + items = iteritems # endregion diff --git a/lib/splunklib/searchcommands/external_search_command.py b/lib/splunklib/searchcommands/external_search_command.py index c230624..a8929f8 100644 --- a/lib/splunklib/searchcommands/external_search_command.py +++ b/lib/splunklib/searchcommands/external_search_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,34 +14,31 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - from logging import getLogger import os import sys import traceback -from splunklib import six +from . import splunklib_logger as logger + if sys.platform == 'win32': from signal import signal, CTRL_BREAK_EVENT, SIGBREAK, SIGINT, SIGTERM from subprocess import Popen import atexit -from . import splunklib_logger as logger + # P1 [ ] TODO: Add ExternalSearchCommand class documentation -class ExternalSearchCommand(object): - """ - """ +class ExternalSearchCommand: def __init__(self, path, argv=None, environ=None): - if not isinstance(path, (bytes, six.text_type)): - raise ValueError('Expected a string value for path, not {}'.format(repr(path))) + if not isinstance(path, (bytes,str)): + raise ValueError(f'Expected a string value for path, not {repr(path)}') self._logger = getLogger(self.__class__.__name__) - self._path = six.text_type(path) + self._path = str(path) self._argv = None self._environ = None @@ -57,7 +54,7 @@ def argv(self): @argv.setter def argv(self, value): if not (value is None or isinstance(value, (list, tuple))): - raise ValueError('Expected a list, tuple or value of None for argv, not {}'.format(repr(value))) + raise ValueError(f'Expected a list, tuple or value of None for argv, not {repr(value)}') self._argv = value @property @@ -67,7 +64,7 @@ def environ(self): @environ.setter def environ(self, value): if not (value is None or isinstance(value, dict)): - raise ValueError('Expected a dictionary value for environ, not {}'.format(repr(value))) + raise ValueError(f'Expected a dictionary value for environ, not {repr(value)}') self._environ = value @property @@ -90,7 +87,7 @@ def execute(self): self._execute(self._path, self._argv, self._environ) except: error_type, error, tb = sys.exc_info() - message = 'Command execution failed: ' + six.text_type(error) + message = f'Command execution failed: {str(error)}' self._logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb))) sys.exit(1) @@ -120,13 +117,13 @@ def _execute(path, argv=None, environ=None): found = ExternalSearchCommand._search_path(path, search_path) if found is None: - raise ValueError('Cannot find command on path: {}'.format(path)) + raise ValueError(f'Cannot find command on path: {path}') path = found - logger.debug('starting command="%s", arguments=%s', path, argv) + logger.debug(f'starting command="{path}", arguments={argv}') - def terminate(signal_number, frame): - sys.exit('External search command is terminating on receipt of signal={}.'.format(signal_number)) + def terminate(signal_number): + sys.exit(f'External search command is terminating on receipt of signal={signal_number}.') def terminate_child(): if p.pid is not None and p.returncode is None: @@ -206,7 +203,6 @@ def _execute(path, argv, environ): os.execvp(path, argv) else: os.execvpe(path, argv, environ) - return # endregion diff --git a/lib/splunklib/searchcommands/generating_command.py b/lib/splunklib/searchcommands/generating_command.py index 6a75d2c..36b014c 100644 --- a/lib/splunklib/searchcommands/generating_command.py +++ b/lib/splunklib/searchcommands/generating_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,14 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals import sys from .decorators import ConfigurationSetting from .search_command import SearchCommand -from splunklib import six -from splunklib.six.moves import map as imap, filter as ifilter # P1 [O] TODO: Discuss generates_timeorder in the class-level documentation for GeneratingCommand @@ -254,8 +251,7 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_ if not allow_empty_input: raise ValueError("allow_empty_input cannot be False for Generating Commands") - else: - return super(GeneratingCommand, self).process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True) + return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True) # endregion @@ -370,18 +366,14 @@ def iteritems(self): iteritems = SearchCommand.ConfigurationSettings.iteritems(self) version = self.command.protocol_version if version == 2: - iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems) + iteritems = [name_value1 for name_value1 in iteritems if name_value1[0] != 'distributed'] if not self.distributed and self.type == 'streaming': - iteritems = imap( - lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems) + iteritems = [(name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]) for name_value in iteritems] return iteritems # N.B.: Does not use Python 3 dict view semantics - if not six.PY2: - items = iteritems + items = iteritems - pass # endregion - pass # endregion diff --git a/lib/splunklib/searchcommands/internals.py b/lib/splunklib/searchcommands/internals.py index 1ea2833..abceac3 100644 --- a/lib/splunklib/searchcommands/internals.py +++ b/lib/splunklib/searchcommands/internals.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,25 +14,22 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function - -from io import TextIOWrapper -from collections import deque, namedtuple -from splunklib import six -from collections import OrderedDict -from splunklib.six.moves import StringIO -from itertools import chain -from splunklib.six.moves import map as imap -from json import JSONDecoder, JSONEncoder -from json.encoder import encode_basestring_ascii as json_encode_string -from splunklib.six.moves import urllib - import csv import gzip import os import re import sys import warnings +import urllib.parse +from io import TextIOWrapper, StringIO +from collections import deque, namedtuple +from collections import OrderedDict +from itertools import chain +from json import JSONDecoder, JSONEncoder +from json.encoder import encode_basestring_ascii as json_encode_string + + + from . import environment @@ -43,35 +40,19 @@ def set_binary_mode(fh): """ Helper method to set up binary mode for file handles. Emphasis being sys.stdin, sys.stdout, sys.stderr. For python3, we want to return .buffer - For python2+windows we want to set os.O_BINARY """ - typefile = TextIOWrapper if sys.version_info >= (3, 0) else file + typefile = TextIOWrapper # check for file handle if not isinstance(fh, typefile): return fh - # check for python3 and buffer - if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'): + # check for buffer + if hasattr(fh, 'buffer'): return fh.buffer - # check for python3 - elif sys.version_info >= (3, 0): - pass - # check for windows python2. SPL-175233 -- python3 stdout is already binary - elif sys.platform == 'win32': - # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in - # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that - # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption. - from platform import python_implementation - implementation = python_implementation() - if implementation == 'PyPy': - return os.fdopen(fh.fileno(), 'wb', 0) - else: - import msvcrt - msvcrt.setmode(fh.fileno(), os.O_BINARY) return fh -class CommandLineParser(object): +class CommandLineParser: r""" Parses the arguments to a search command. A search command line is described by the following syntax. @@ -144,7 +125,7 @@ def parse(cls, command, argv): command_args = cls._arguments_re.match(argv) if command_args is None: - raise SyntaxError('Syntax error: {}'.format(argv)) + raise SyntaxError(f'Syntax error: {argv}') # Parse options @@ -152,7 +133,7 @@ def parse(cls, command, argv): name, value = option.group('name'), option.group('value') if name not in command.options: raise ValueError( - 'Unrecognized {} command option: {}={}'.format(command.name, name, json_encode_string(value))) + f'Unrecognized {command.name} command option: {name}={json_encode_string(value)}') command.options[name].value = cls.unquote(value) missing = command.options.get_missing() @@ -160,8 +141,8 @@ def parse(cls, command, argv): if missing is not None: if len(missing) > 1: raise ValueError( - 'Values for these {} command options are required: {}'.format(command.name, ', '.join(missing))) - raise ValueError('A value for {} command option {} is required'.format(command.name, missing[0])) + f'Values for these {command.name} command options are required: {", ".join(missing)}') + raise ValueError(f'A value for {command.name} command option {missing[0]} is required') # Parse field names @@ -277,10 +258,10 @@ def validate_configuration_setting(specification, name, value): if isinstance(specification.type, type): type_names = specification.type.__name__ else: - type_names = ', '.join(imap(lambda t: t.__name__, specification.type)) - raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value))) + type_names = ', '.join(map(lambda t: t.__name__, specification.type)) + raise ValueError(f'Expected {type_names} value, not {name}={repr(value)}') if specification.constraint and not specification.constraint(value): - raise ValueError('Illegal value: {}={}'.format(name, repr(value))) + raise ValueError(f'Illegal value: {name}={ repr(value)}') return value specification = namedtuple( @@ -314,7 +295,7 @@ def validate_configuration_setting(specification, name, value): supporting_protocols=[1]), 'maxinputs': specification( type=int, - constraint=lambda value: 0 <= value <= six.MAXSIZE, + constraint=lambda value: 0 <= value <= sys.maxsize, supporting_protocols=[2]), 'overrides_timeorder': specification( type=bool, @@ -341,11 +322,11 @@ def validate_configuration_setting(specification, name, value): constraint=None, supporting_protocols=[1]), 'streaming_preop': specification( - type=(bytes, six.text_type), + type=(bytes, str), constraint=None, supporting_protocols=[1, 2]), 'type': specification( - type=(bytes, six.text_type), + type=(bytes, str), constraint=lambda value: value in ('events', 'reporting', 'streaming'), supporting_protocols=[2])} @@ -368,7 +349,7 @@ class InputHeader(dict): """ def __str__(self): - return '\n'.join([name + ':' + value for name, value in six.iteritems(self)]) + return '\n'.join([name + ':' + value for name, value in self.items()]) def read(self, ifile): """ Reads an input header from an input file. @@ -416,7 +397,7 @@ def _object_hook(dictionary): while len(stack): instance, member_name, dictionary = stack.popleft() - for name, value in six.iteritems(dictionary): + for name, value in dictionary.items(): if isinstance(value, dict): stack.append((dictionary, name, value)) @@ -437,11 +418,14 @@ def default(self, o): _separators = (',', ':') -class ObjectView(object): +class ObjectView: def __init__(self, dictionary): self.__dict__ = dictionary + def update(self, obj): + self.__dict__.update(obj.__dict__) + def __repr__(self): return repr(self.__dict__) @@ -449,7 +433,7 @@ def __str__(self): return str(self.__dict__) -class Recorder(object): +class Recorder: def __init__(self, path, f): self._recording = gzip.open(path + '.gz', 'wb') @@ -487,7 +471,7 @@ def write(self, text): self._recording.flush() -class RecordWriter(object): +class RecordWriter: def __init__(self, ofile, maxresultrows=None): self._maxresultrows = 50000 if maxresultrows is None else maxresultrows @@ -513,7 +497,7 @@ def is_flushed(self): @is_flushed.setter def is_flushed(self, value): - self._flushed = True if value else False + self._flushed = bool(value) @property def ofile(self): @@ -593,7 +577,7 @@ def _write_record(self, record): if fieldnames is None: self._fieldnames = fieldnames = list(record.keys()) self._fieldnames.extend([i for i in self.custom_fields if i not in self._fieldnames]) - value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) + value_list = map(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) self._writerow(list(chain.from_iterable(value_list))) get_value = record.get @@ -632,9 +616,9 @@ def _write_record(self, record): if value_t is bool: value = str(value.real) - elif value_t is six.text_type: + elif value_t is str: value = value - elif isinstance(value, six.integer_types) or value_t is float or value_t is complex: + elif isinstance(value, int) or value_t is float or value_t is complex: value = str(value) elif issubclass(value_t, (dict, list, tuple)): value = str(''.join(RecordWriter._iterencode_json(value, 0))) @@ -658,13 +642,11 @@ def _write_record(self, record): values += (value, None) continue - if value_t is six.text_type: - if six.PY2: - value = value.encode('utf-8') + if value_t is str: values += (value, None) continue - if isinstance(value, six.integer_types) or value_t is float or value_t is complex: + if isinstance(value, int) or value_t is float or value_t is complex: values += (str(value), None) continue @@ -799,16 +781,15 @@ def write_chunk(self, finished=None): if len(inspector) == 0: inspector = None - metadata = [item for item in (('inspector', inspector), ('finished', finished))] + metadata = [('inspector', inspector), ('finished', finished)] self._write_chunk(metadata, self._buffer.getvalue()) self._clear() def write_metadata(self, configuration): self._ensure_validity() - metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),)) + metadata = chain(configuration.items(), (('inspector', self._inspector if self._inspector else None),)) self._write_chunk(metadata, '') - self.write('\n') self._clear() def write_metric(self, name, value): @@ -816,13 +797,13 @@ def write_metric(self, name, value): self._inspector['metric.' + name] = value def _clear(self): - super(RecordWriterV2, self)._clear() + super()._clear() self._fieldnames = None def _write_chunk(self, metadata, body): if metadata: - metadata = str(''.join(self._iterencode_json(dict([(n, v) for n, v in metadata if v is not None]), 0))) + metadata = str(''.join(self._iterencode_json(dict((n, v) for n, v in metadata if v is not None), 0))) if sys.version_info >= (3, 0): metadata = metadata.encode('utf-8') metadata_length = len(metadata) @@ -836,7 +817,7 @@ def _write_chunk(self, metadata, body): if not (metadata_length > 0 or body_length > 0): return - start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length) + start_line = f'chunked 1.0,{metadata_length},{body_length}\n' self.write(start_line) self.write(metadata) self.write(body) diff --git a/lib/splunklib/searchcommands/reporting_command.py b/lib/splunklib/searchcommands/reporting_command.py index 9470861..5df3dc7 100644 --- a/lib/splunklib/searchcommands/reporting_command.py +++ b/lib/splunklib/searchcommands/reporting_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,8 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - from itertools import chain from .internals import ConfigurationSettingsType, json_encode_string @@ -23,7 +21,6 @@ from .streaming_command import StreamingCommand from .search_command import SearchCommand from .validators import Set -from splunklib import six class ReportingCommand(SearchCommand): @@ -94,7 +91,7 @@ def prepare(self): self._configuration.streaming_preop = ' '.join(streaming_preop) return - raise RuntimeError('Unrecognized reporting command phase: {}'.format(json_encode_string(six.text_type(phase)))) + raise RuntimeError(f'Unrecognized reporting command phase: {json_encode_string(str(phase))}') def reduce(self, records): """ Override this method to produce a reporting data structure. @@ -244,7 +241,7 @@ def fix_up(cls, command): """ if not issubclass(command, ReportingCommand): - raise TypeError('{} is not a ReportingCommand'.format( command)) + raise TypeError(f'{command} is not a ReportingCommand') if command.reduce == ReportingCommand.reduce: raise AttributeError('No ReportingCommand.reduce override') @@ -274,8 +271,7 @@ def fix_up(cls, command): ConfigurationSetting.fix_up(f.ConfigurationSettings, settings) del f._settings - pass + # endregion - pass # endregion diff --git a/lib/splunklib/searchcommands/search_command.py b/lib/splunklib/searchcommands/search_command.py index dd11391..7e8f771 100644 --- a/lib/splunklib/searchcommands/search_command.py +++ b/lib/splunklib/searchcommands/search_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright © 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,44 +14,32 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - # Absolute imports -from collections import namedtuple - +import csv import io - -from collections import OrderedDict +import os +import re +import sys +import tempfile +import traceback +from collections import namedtuple, OrderedDict from copy import deepcopy -from splunklib.six.moves import StringIO +from io import StringIO from itertools import chain, islice -from splunklib.six.moves import filter as ifilter, map as imap, zip as izip -from splunklib import six -if six.PY2: - from logging import _levelNames, getLevelName, getLogger -else: - from logging import _nameToLevel as _levelNames, getLevelName, getLogger -try: - from shutil import make_archive -except ImportError: - # Used for recording, skip on python 2.6 - pass +from logging import _nameToLevel as _levelNames, getLevelName, getLogger +from shutil import make_archive from time import time -from splunklib.six.moves.urllib.parse import unquote -from splunklib.six.moves.urllib.parse import urlsplit +from urllib.parse import unquote +from urllib.parse import urlsplit from warnings import warn from xml.etree import ElementTree +from splunklib.utils import ensure_str -import os -import sys -import re -import csv -import tempfile -import traceback # Relative imports - +import splunklib +from . import Boolean, Option, environment from .internals import ( CommandLineParser, CsvDialect, @@ -64,8 +52,6 @@ RecordWriterV1, RecordWriterV2, json_encode_string) - -from . import Boolean, Option, environment from ..client import Service @@ -91,7 +77,7 @@ # P2 [ ] TODO: Consider bumping None formatting up to Option.Item.__str__ -class SearchCommand(object): +class SearchCommand: """ Represents a custom search command. """ @@ -158,16 +144,16 @@ def logging_level(self): def logging_level(self, value): if value is None: value = self._default_logging_level - if isinstance(value, (bytes, six.text_type)): + if isinstance(value, (bytes, str)): try: level = _levelNames[value.upper()] except KeyError: - raise ValueError('Unrecognized logging level: {}'.format(value)) + raise ValueError(f'Unrecognized logging level: {value}') else: try: level = int(value) except ValueError: - raise ValueError('Unrecognized logging level: {}'.format(value)) + raise ValueError(f'Unrecognized logging level: {value}') self._logger.setLevel(level) def add_field(self, current_record, field_name, field_value): @@ -291,7 +277,7 @@ def search_results_info(self): values = next(reader) except IOError as error: if error.errno == 2: - self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path))) + self.logger.error(f'Search results info file {json_encode_string(path)} does not exist.') return raise @@ -306,7 +292,7 @@ def convert_value(value): except ValueError: return value - info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) + info = ObjectView(dict((convert_field(f_v[0]), convert_value(f_v[1])) for f_v in zip(fields, values))) try: count_map = info.countMap @@ -315,7 +301,7 @@ def convert_value(value): else: count_map = count_map.split(';') n = len(count_map) - info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) + info.countMap = dict(list(zip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))) try: msg_type = info.msgType @@ -323,7 +309,7 @@ def convert_value(value): except AttributeError: pass else: - messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) + messages = [t_m for t_m in zip(msg_type.split('\n'), msg_text.split('\n')) if t_m[0] or t_m[1]] info.msg = [Message(message) for message in messages] del info.msgType @@ -417,7 +403,6 @@ def prepare(self): :rtype: NoneType """ - pass def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): """ Process data. @@ -466,7 +451,7 @@ def _map_metadata(self, argv): def _map(metadata_map): metadata = {} - for name, value in six.iteritems(metadata_map): + for name, value in metadata_map.items(): if isinstance(value, dict): value = _map(value) else: @@ -485,7 +470,8 @@ def _map(metadata_map): _metadata_map = { 'action': - (lambda v: 'getinfo' if v == '__GETINFO__' else 'execute' if v == '__EXECUTE__' else None, lambda s: s.argv[1]), + (lambda v: 'getinfo' if v == '__GETINFO__' else 'execute' if v == '__EXECUTE__' else None, + lambda s: s.argv[1]), 'preview': (bool, lambda s: s.input_header.get('preview')), 'searchinfo': { @@ -533,7 +519,7 @@ def _prepare_protocol_v1(self, argv, ifile, ofile): try: tempfile.tempdir = self._metadata.searchinfo.dispatch_dir except AttributeError: - raise RuntimeError('{}.metadata.searchinfo.dispatch_dir is undefined'.format(self.__class__.__name__)) + raise RuntimeError(f'{self.__class__.__name__}.metadata.searchinfo.dispatch_dir is undefined') debug(' tempfile.tempdir=%r', tempfile.tempdir) @@ -603,7 +589,8 @@ def _process_protocol_v1(self, argv, ifile, ofile): ifile = self._prepare_protocol_v1(argv, ifile, ofile) self._record_writer.write_record(dict( - (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in six.iteritems(self._configuration))) + (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in + self._configuration.items())) self.finish() elif argv[1] == '__EXECUTE__': @@ -617,21 +604,21 @@ def _process_protocol_v1(self, argv, ifile, ofile): else: message = ( - 'Command {0} appears to be statically configured for search command protocol version 1 and static ' + f'Command {self.name} appears to be statically configured for search command protocol version 1 and static ' 'configuration is unsupported by splunklib.searchcommands. Please ensure that ' 'default/commands.conf contains this stanza:\n' - '[{0}]\n' - 'filename = {1}\n' + f'[{self.name}]\n' + f'filename = {os.path.basename(argv[0])}\n' 'enableheader = true\n' 'outputheader = true\n' 'requires_srinfo = true\n' 'supports_getinfo = true\n' 'supports_multivalues = true\n' - 'supports_rawargs = true'.format(self.name, os.path.basename(argv[0]))) + 'supports_rawargs = true') raise RuntimeError(message) except (SyntaxError, ValueError) as error: - self.write_error(six.text_type(error)) + self.write_error(str(error)) self.flush() exit(0) @@ -686,7 +673,7 @@ def _process_protocol_v2(self, argv, ifile, ofile): action = getattr(metadata, 'action', None) if action != 'getinfo': - raise RuntimeError('Expected getinfo action, not {}'.format(action)) + raise RuntimeError(f'Expected getinfo action, not {action}') if len(body) > 0: raise RuntimeError('Did not expect data for getinfo action') @@ -706,7 +693,7 @@ def _process_protocol_v2(self, argv, ifile, ofile): try: tempfile.tempdir = self._metadata.searchinfo.dispatch_dir except AttributeError: - raise RuntimeError('%s.metadata.searchinfo.dispatch_dir is undefined'.format(class_name)) + raise RuntimeError(f'{class_name}.metadata.searchinfo.dispatch_dir is undefined') debug(' tempfile.tempdir=%r', tempfile.tempdir) except: @@ -727,7 +714,7 @@ def _process_protocol_v2(self, argv, ifile, ofile): debug('Parsing arguments') - if args and type(args) == list: + if args and isinstance(args, list): for arg in args: result = self._protocol_v2_option_parser(arg) if len(result) == 1: @@ -738,13 +725,13 @@ def _process_protocol_v2(self, argv, ifile, ofile): try: option = self.options[name] except KeyError: - self.write_error('Unrecognized option: {}={}'.format(name, value)) + self.write_error(f'Unrecognized option: {name}={value}') error_count += 1 continue try: option.value = value except ValueError: - self.write_error('Illegal value: {}={}'.format(name, value)) + self.write_error(f'Illegal value: {name}={value}') error_count += 1 continue @@ -752,15 +739,15 @@ def _process_protocol_v2(self, argv, ifile, ofile): if missing is not None: if len(missing) == 1: - self.write_error('A value for "{}" is required'.format(missing[0])) + self.write_error(f'A value for "{missing[0]}" is required') else: - self.write_error('Values for these required options are missing: {}'.format(', '.join(missing))) + self.write_error(f'Values for these required options are missing: {", ".join(missing)}') error_count += 1 if error_count > 0: exit(1) - debug(' command: %s', six.text_type(self)) + debug(' command: %s', str(self)) debug('Preparing for execution') self.prepare() @@ -778,7 +765,7 @@ def _process_protocol_v2(self, argv, ifile, ofile): setattr(info, attr, [arg for arg in getattr(info, attr) if not arg.startswith('record=')]) metadata = MetadataEncoder().encode(self._metadata) - ifile.record('chunked 1.0,', six.text_type(len(metadata)), ',0\n', metadata) + ifile.record('chunked 1.0,', str(len(metadata)), ',0\n', metadata) if self.show_configuration: self.write_info(self.name + ' command configuration: ' + str(self._configuration)) @@ -888,25 +875,25 @@ def _as_binary_stream(ifile): try: return ifile.buffer except AttributeError as error: - raise RuntimeError('Failed to get underlying buffer: {}'.format(error)) + raise RuntimeError(f'Failed to get underlying buffer: {error}') @staticmethod def _read_chunk(istream): # noinspection PyBroadException - assert isinstance(istream.read(0), six.binary_type), 'Stream must be binary' + assert isinstance(istream.read(0), bytes), 'Stream must be binary' try: header = istream.readline() except Exception as error: - raise RuntimeError('Failed to read transport header: {}'.format(error)) + raise RuntimeError(f'Failed to read transport header: {error}') if not header: return None - match = SearchCommand._header.match(six.ensure_str(header)) + match = SearchCommand._header.match(ensure_str(header)) if match is None: - raise RuntimeError('Failed to parse transport header: {}'.format(header)) + raise RuntimeError(f'Failed to parse transport header: {header}') metadata_length, body_length = match.groups() metadata_length = int(metadata_length) @@ -915,14 +902,14 @@ def _read_chunk(istream): try: metadata = istream.read(metadata_length) except Exception as error: - raise RuntimeError('Failed to read metadata of length {}: {}'.format(metadata_length, error)) + raise RuntimeError(f'Failed to read metadata of length {metadata_length}: {error}') decoder = MetadataDecoder() try: - metadata = decoder.decode(six.ensure_str(metadata)) + metadata = decoder.decode(ensure_str(metadata)) except Exception as error: - raise RuntimeError('Failed to parse metadata of length {}: {}'.format(metadata_length, error)) + raise RuntimeError(f'Failed to parse metadata of length {metadata_length}: {error}') # if body_length <= 0: # return metadata, '' @@ -932,9 +919,9 @@ def _read_chunk(istream): if body_length > 0: body = istream.read(body_length) except Exception as error: - raise RuntimeError('Failed to read body of length {}: {}'.format(body_length, error)) + raise RuntimeError(f'Failed to read body of length {body_length}: {error}') - return metadata, six.ensure_str(body) + return metadata, ensure_str(body,errors="replace") _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n') @@ -949,16 +936,16 @@ def _read_csv_records(self, ifile): except StopIteration: return - mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) + mv_fieldnames = dict((name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')) if len(mv_fieldnames) == 0: for values in reader: - yield OrderedDict(izip(fieldnames, values)) + yield OrderedDict(list(zip(fieldnames, values))) return for values in reader: record = OrderedDict() - for fieldname, value in izip(fieldnames, values): + for fieldname, value in zip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[fieldname]] = self._decode_list(value) @@ -978,25 +965,25 @@ def _execute_v2(self, ifile, process): metadata, body = result action = getattr(metadata, 'action', None) if action != 'execute': - raise RuntimeError('Expected execute action, not {}'.format(action)) + raise RuntimeError(f'Expected execute action, not {action}') self._finished = getattr(metadata, 'finished', False) self._record_writer.is_flushed = False - + self._metadata.update(metadata) self._execute_chunk_v2(process, result) self._record_writer.write_chunk(finished=self._finished) def _execute_chunk_v2(self, process, chunk): - metadata, body = chunk + metadata, body = chunk - if len(body) <= 0 and not self._allow_empty_input: - raise ValueError( - "No records found to process. Set allow_empty_input=True in dispatch function to move forward " - "with empty records.") + if len(body) <= 0 and not self._allow_empty_input: + raise ValueError( + "No records found to process. Set allow_empty_input=True in dispatch function to move forward " + "with empty records.") - records = self._read_csv_records(StringIO(body)) - self._record_writer.write_records(process(records)) + records = self._read_csv_records(StringIO(body)) + self._record_writer.write_records(process(records)) def _report_unexpected_error(self): @@ -1008,7 +995,7 @@ def _report_unexpected_error(self): filename = origin.tb_frame.f_code.co_filename lineno = origin.tb_lineno - message = '{0} at "{1}", line {2:d} : {3}'.format(error_type.__name__, filename, lineno, error) + message = f'{error_type.__name__} at "{filename}", line {str(lineno)} : {error}' environment.splunklib_logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb))) self.write_error(message) @@ -1017,10 +1004,11 @@ def _report_unexpected_error(self): # region Types - class ConfigurationSettings(object): + class ConfigurationSettings: """ Represents the configuration settings common to all :class:`SearchCommand` classes. """ + def __init__(self, command): self.command = command @@ -1034,8 +1022,8 @@ def __repr__(self): """ definitions = type(self).configuration_setting_definitions - settings = imap( - lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions) + settings = [repr((setting.name, setting.__get__(self), setting.supporting_protocols)) for setting in + definitions] return '[' + ', '.join(settings) + ']' def __str__(self): @@ -1047,8 +1035,8 @@ def __str__(self): :return: String representation of this instance """ - #text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems())) - text = ', '.join(['{}={}'.format(name, json_encode_string(six.text_type(value))) for (name, value) in six.iteritems(self)]) + # text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems())) + text = ', '.join([f'{name}={json_encode_string(str(value))}' for (name, value) in self.items()]) return text # region Methods @@ -1072,24 +1060,25 @@ def fix_up(cls, command_class): def iteritems(self): definitions = type(self).configuration_setting_definitions version = self.command.protocol_version - return ifilter( - lambda name_value1: name_value1[1] is not None, imap( - lambda setting: (setting.name, setting.__get__(self)), ifilter( - lambda setting: setting.is_supported_by_protocol(version), definitions))) + return [name_value1 for name_value1 in [(setting.name, setting.__get__(self)) for setting in + [setting for setting in definitions if + setting.is_supported_by_protocol(version)]] if + name_value1[1] is not None] # N.B.: Does not use Python 3 dict view semantics - if not six.PY2: - items = iteritems - pass # endregion + items = iteritems - pass # endregion + # endregion + + # endregion SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) -def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_input=True): +def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, + allow_empty_input=True): """ Instantiates and executes a search command class This function implements a `conditional script stanza `_ based on the value of diff --git a/lib/splunklib/searchcommands/streaming_command.py b/lib/splunklib/searchcommands/streaming_command.py index fa075ed..e2a3a40 100644 --- a/lib/splunklib/searchcommands/streaming_command.py +++ b/lib/splunklib/searchcommands/streaming_command.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,10 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - -from splunklib import six -from splunklib.six.moves import map as imap, filter as ifilter from .decorators import ConfigurationSetting from .search_command import SearchCommand @@ -171,7 +167,6 @@ def fix_up(cls, command): """ if command.stream == StreamingCommand.stream: raise AttributeError('No StreamingCommand.stream override') - return # TODO: Stop looking like a dictionary because we don't obey the semantics # N.B.: Does not use Python 2 dict copy semantics @@ -180,16 +175,14 @@ def iteritems(self): version = self.command.protocol_version if version == 1: if self.required_fields is None: - iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems) + iteritems = [name_value for name_value in iteritems if name_value[0] != 'clear_required_fields'] else: - iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems) + iteritems = [name_value2 for name_value2 in iteritems if name_value2[0] != 'distributed'] if not self.distributed: - iteritems = imap( - lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems) + iteritems = [(name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]) for name_value1 in iteritems] return iteritems # N.B.: Does not use Python 3 dict view semantics - if not six.PY2: - items = iteritems + items = iteritems # endregion diff --git a/lib/splunklib/searchcommands/validators.py b/lib/splunklib/searchcommands/validators.py index 22f0e16..ccaebca 100644 --- a/lib/splunklib/searchcommands/validators.py +++ b/lib/splunklib/searchcommands/validators.py @@ -1,6 +1,6 @@ # coding=utf-8 # -# Copyright 2011-2015 Splunk, Inc. +# Copyright © 2011-2024 Splunk, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"): you may # not use this file except in compliance with the License. You may obtain @@ -14,20 +14,17 @@ # License for the specific language governing permissions and limitations # under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - -from json.encoder import encode_basestring_ascii as json_encode_string -from collections import namedtuple -from splunklib.six.moves import StringIO -from io import open import csv import os import re -from splunklib import six -from splunklib.six.moves import getcwd +from io import open, StringIO +from os import getcwd +from json.encoder import encode_basestring_ascii as json_encode_string +from collections import namedtuple -class Validator(object): + +class Validator: """ Base class for validators that check and format search command options. You must inherit from this class and override :code:`Validator.__call__` and @@ -60,14 +57,16 @@ class Boolean(Validator): def __call__(self, value): if not (value is None or isinstance(value, bool)): - value = six.text_type(value).lower() + value = str(value).lower() if value not in Boolean.truth_values: - raise ValueError('Unrecognized truth value: {0}'.format(value)) + raise ValueError(f'Unrecognized truth value: {value}') value = Boolean.truth_values[value] return value def format(self, value): - return None if value is None else 't' if value else 'f' + if value is None: + return None + return 't' if value else 'f' class Code(Validator): @@ -93,11 +92,11 @@ def __call__(self, value): if value is None: return None try: - return Code.object(compile(value, 'string', self._mode), six.text_type(value)) + return Code.object(compile(value, 'string', self._mode), str(value)) except (SyntaxError, TypeError) as error: message = str(error) - six.raise_from(ValueError(message), error) + raise ValueError(message) from error def format(self, value): return None if value is None else value.source @@ -113,9 +112,9 @@ class Fieldname(Validator): def __call__(self, value): if value is not None: - value = six.text_type(value) + value = str(value) if Fieldname.pattern.match(value) is None: - raise ValueError('Illegal characters in fieldname: {}'.format(value)) + raise ValueError(f'Illegal characters in fieldname: {value}') return value def format(self, value): @@ -136,7 +135,7 @@ def __call__(self, value): if value is None: return value - path = six.text_type(value) + path = str(value) if not os.path.isabs(path): path = os.path.join(self.directory, path) @@ -144,8 +143,7 @@ def __call__(self, value): try: value = open(path, self.mode) if self.buffering is None else open(path, self.mode, self.buffering) except IOError as error: - raise ValueError('Cannot open {0} with mode={1} and buffering={2}: {3}'.format( - value, self.mode, self.buffering, error)) + raise ValueError(f'Cannot open {value} with mode={self.mode} and buffering={self.buffering}: {error}') return value @@ -163,42 +161,38 @@ class Integer(Validator): def __init__(self, minimum=None, maximum=None): if minimum is not None and maximum is not None: def check_range(value): - if not (minimum <= value <= maximum): - raise ValueError('Expected integer in the range [{0},{1}], not {2}'.format(minimum, maximum, value)) - return + if not minimum <= value <= maximum: + raise ValueError(f'Expected integer in the range [{minimum},{maximum}], not {value}') + elif minimum is not None: def check_range(value): if value < minimum: - raise ValueError('Expected integer in the range [{0},+∞], not {1}'.format(minimum, value)) - return + raise ValueError(f'Expected integer in the range [{minimum},+∞], not {value}') elif maximum is not None: def check_range(value): if value > maximum: - raise ValueError('Expected integer in the range [-∞,{0}], not {1}'.format(maximum, value)) - return + raise ValueError(f'Expected integer in the range [-∞,{maximum}], not {value}') + else: def check_range(value): return self.check_range = check_range - return + def __call__(self, value): if value is None: return None try: - if six.PY2: - value = long(value) - else: - value = int(value) + value = int(value) except ValueError: - raise ValueError('Expected integer value, not {}'.format(json_encode_string(value))) + raise ValueError(f'Expected integer value, not {json_encode_string(value)}') self.check_range(value) return value def format(self, value): - return None if value is None else six.text_type(int(value)) + return None if value is None else str(int(value)) class Float(Validator): @@ -208,25 +202,21 @@ class Float(Validator): def __init__(self, minimum=None, maximum=None): if minimum is not None and maximum is not None: def check_range(value): - if not (minimum <= value <= maximum): - raise ValueError('Expected float in the range [{0},{1}], not {2}'.format(minimum, maximum, value)) - return + if not minimum <= value <= maximum: + raise ValueError(f'Expected float in the range [{minimum},{maximum}], not {value}') elif minimum is not None: def check_range(value): if value < minimum: - raise ValueError('Expected float in the range [{0},+∞], not {1}'.format(minimum, value)) - return + raise ValueError(f'Expected float in the range [{minimum},+∞], not {value}') elif maximum is not None: def check_range(value): if value > maximum: - raise ValueError('Expected float in the range [-∞,{0}], not {1}'.format(maximum, value)) - return + raise ValueError(f'Expected float in the range [-∞,{maximum}], not {value}') else: def check_range(value): return - self.check_range = check_range - return + def __call__(self, value): if value is None: @@ -234,13 +224,13 @@ def __call__(self, value): try: value = float(value) except ValueError: - raise ValueError('Expected float value, not {}'.format(json_encode_string(value))) + raise ValueError(f'Expected float value, not {json_encode_string(value)}') self.check_range(value) return value def format(self, value): - return None if value is None else six.text_type(float(value)) + return None if value is None else str(float(value)) class Duration(Validator): @@ -265,7 +255,7 @@ def __call__(self, value): if len(p) == 3: result = 3600 * _unsigned(p[0]) + 60 * _60(p[1]) + _60(p[2]) except ValueError: - raise ValueError('Invalid duration value: {0}'.format(value)) + raise ValueError(f'Invalid duration value: {value}') return result @@ -302,7 +292,7 @@ class Dialect(csv.Dialect): def __init__(self, validator=None): if not (validator is None or isinstance(validator, Validator)): - raise ValueError('Expected a Validator instance or None for validator, not {}', repr(validator)) + raise ValueError(f'Expected a Validator instance or None for validator, not {repr(validator)}') self._validator = validator def __call__(self, value): @@ -322,7 +312,7 @@ def __call__(self, value): for index, item in enumerate(value): value[index] = self._validator(item) except ValueError as error: - raise ValueError('Could not convert item {}: {}'.format(index, error)) + raise ValueError(f'Could not convert item {index}: {error}') return value @@ -346,10 +336,10 @@ def __call__(self, value): if value is None: return None - value = six.text_type(value) + value = str(value) if value not in self.membership: - raise ValueError('Unrecognized value: {0}'.format(value)) + raise ValueError(f'Unrecognized value: {value}') return self.membership[value] @@ -362,19 +352,19 @@ class Match(Validator): """ def __init__(self, name, pattern, flags=0): - self.name = six.text_type(name) + self.name = str(name) self.pattern = re.compile(pattern, flags) def __call__(self, value): if value is None: return None - value = six.text_type(value) + value = str(value) if self.pattern.match(value) is None: - raise ValueError('Expected {}, not {}'.format(self.name, json_encode_string(value))) + raise ValueError(f'Expected {self.name}, not {json_encode_string(value)}') return value def format(self, value): - return None if value is None else six.text_type(value) + return None if value is None else str(value) class OptionName(Validator): @@ -385,13 +375,13 @@ class OptionName(Validator): def __call__(self, value): if value is not None: - value = six.text_type(value) + value = str(value) if OptionName.pattern.match(value) is None: - raise ValueError('Illegal characters in option name: {}'.format(value)) + raise ValueError(f'Illegal characters in option name: {value}') return value def format(self, value): - return None if value is None else six.text_type(value) + return None if value is None else str(value) class RegularExpression(Validator): @@ -402,9 +392,9 @@ def __call__(self, value): if value is None: return None try: - value = re.compile(six.text_type(value)) + value = re.compile(str(value)) except re.error as error: - raise ValueError('{}: {}'.format(six.text_type(error).capitalize(), value)) + raise ValueError(f'{str(error).capitalize()}: {value}') return value def format(self, value): @@ -421,9 +411,9 @@ def __init__(self, *args): def __call__(self, value): if value is None: return None - value = six.text_type(value) + value = str(value) if value not in self.membership: - raise ValueError('Unrecognized value: {}'.format(value)) + raise ValueError(f'Unrecognized value: {value}') return value def format(self, value): diff --git a/lib/splunklib/utils.py b/lib/splunklib/utils.py new file mode 100644 index 0000000..db9c312 --- /dev/null +++ b/lib/splunklib/utils.py @@ -0,0 +1,48 @@ +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The **splunklib.utils** File for utility functions. +""" + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + """ + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, str): + return s.encode(encoding, errors) + + if isinstance(s, bytes): + return s + + raise TypeError(f"not expecting type '{type(s)}'") + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """ + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, bytes): + return s.decode(encoding, errors) + + if isinstance(s, str): + return s + + raise TypeError(f"not expecting type '{type(s)}'") + + +def assertRegex(self, *args, **kwargs): + return getattr(self, "assertRegex")(*args, **kwargs)