Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Debug information on nist requests #290

Merged
merged 6 commits into from
May 23, 2024
Merged

Debug information on nist requests #290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9bad5ce
Update api_base_class.py
derIckeBrln May 21, 2024
7403062
Update nvd_nist_api.py
derIckeBrln May 21, 2024
39bdd55
Update api_base_class.py
derIckeBrln May 22, 2024
25851c4
Update api_base_class.py
derIckeBrln May 22, 2024
f9c3cc3
Update nvd_nist_api.py
derIckeBrln May 22, 2024
31ae0d3
Update api_base_class.py
derIckeBrln May 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions CveXplore/core/api_base_class.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,25 @@ def _connect(

request_api_resource.update(self.kwargs)

full_url = self._build_url(resource)
self.logger.debug(f"Making {method} request to URL: {full_url}")
self.logger.debug(f"Request headers: {request_api_resource['headers']}")
self.logger.debug(f"Request data: {data}")

try:
if method == self.methods.POST:
r = session.post(self._build_url(resource), **request_api_resource)
r = session.post(full_url, **request_api_resource)
elif method == self.methods.PUT:
r = session.put(self._build_url(resource), **request_api_resource)
r = session.put(full_url, **request_api_resource)
elif method == self.methods.PATCH:
r = session.patch(self._build_url(resource), **request_api_resource)
r = session.patch(full_url, **request_api_resource)
elif method == self.methods.DELETE:
r = session.delete(self._build_url(resource), **request_api_resource)
r = session.delete(full_url, **request_api_resource)
else:
r = session.get(self._build_url(resource), **request_api_resource)
r = session.get(full_url, **request_api_resource)

self.logger.debug(f"Received response: {r.status_code}")
self.logger.debug(f"Response content: {r.text}")

try:
if isinstance(r, Response):
Expand All @@ -118,9 +126,11 @@ def _connect(

return the_response
except requests.exceptions.ConnectionError as err:
raise requests.exceptions.ConnectionError(err)
self.logger.error(f"Connection error occurred: {err}")
raise
except Exception as err:
raise Exception(err)
self.logger.error(f"An error occurred: {err}")
raise

def get_session(
self,
Expand Down Expand Up @@ -156,8 +166,8 @@ def call(
) -> dict:
"""
Method for requesting free format api resources

"""
self.logger.debug(f"API call with method: {method}, RESOURCE: {resource}")
try:
with self.get_session() as session:
result = self._connect(
Expand All @@ -168,10 +178,13 @@ def call(
timeout=timeout,
return_response_object=return_response_object,
)
self.logger.debug(f"Received API response: {result}")
return result
except requests.ConnectionError:
except requests.ConnectionError as conn_err:
self.logger.error(f"Connection error occurred: {conn_err}")
raise
except Exception:
except Exception as e:
self.logger.error(f"Error during API call: {e}")
raise

@property
Expand Down
61 changes: 48 additions & 13 deletions CveXplore/core/nvd_nist/nvd_nist_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from json import JSONDecodeError
from urllib.parse import urlencode

from requests.exceptions import RequestException, HTTPError

import aiohttp as aiohttp
import requests
from aiohttp import ContentTypeError
Expand Down Expand Up @@ -81,19 +83,22 @@ def _build_url(self, resource: dict = None, data: int = 1) -> str:
resource = urlencode(resource)
if data == self.datasource.CVE:
if self.filter_rejected:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
else:
if data == self.datasource.CVE:
if self.filter_rejected:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"

self.logger.debug(f"Built URL: {url}")
return url

def _connect(
self,
Expand All @@ -113,11 +118,27 @@ def _connect(
"proxies": self.proxies,
}

full_url = self._build_url(resource, data=data)
self.logger.debug(f"Sending {method} request to URL: {full_url}")
self.logger.debug(f"Request headers: {request_api_resource['headers']}")
self.logger.debug(f"Request data: {data}")

try:
self.logger.debug(f"Sending request: resource={resource}, data={data}")
r = session.get(
self._build_url(resource, data=data), **request_api_resource
)
if method == "GET":
r = session.get(full_url, **request_api_resource)
elif method == "POST":
r = session.post(full_url, data=data, **request_api_resource)
elif method == "PUT":
r = session.put(full_url, data=data, **request_api_resource)
elif method == "PATCH":
r = session.patch(full_url, data=data, **request_api_resource)
elif method == "DELETE":
r = session.delete(full_url, **request_api_resource)
else:
raise ValueError(f"Unsupported method: {method}")

self.logger.debug(f"Received response: {r.status_code}")
self.logger.debug(f"Response content: {r.text}")

try:
if isinstance(r, Response):
Expand Down Expand Up @@ -146,8 +167,10 @@ def _connect(
return the_response

except requests.exceptions.ConnectionError as err:
self.logger.error(f"Connection error occurred: {err}")
raise requests.exceptions.ConnectionError(err)
except Exception as err:
self.logger.error(f"An error occurred: {err}")
raise Exception(err)

def __repr__(self):
Expand Down Expand Up @@ -244,17 +267,29 @@ def get_count(
last_mod_end_date=last_mod_end_date,
)

self.logger.debug(f"Getting count for datasource: {datasource}")
self.logger.info(f"Getting count for datasource: {datasource}")

try:
ret_data = self.call(self.methods.GET, resource=resource, data=datasource)

if not isinstance(ret_data, Response):
self.logger.debug(f"API response data: {ret_data}")
return ret_data["totalResults"]
else:
self.logger.error(f"Unexpected response type: {ret_data}")
raise ApiDataRetrievalFailed(resource)
except Exception:
raise ApiMaxRetryError
except HTTPError as http_err:
self.logger.error(f"HTTP error occurred: {http_err}")
raise ApiMaxRetryError from http_err
except RequestException as req_err:
self.logger.error(f"Request exception occurred: {req_err}")
raise ApiMaxRetryError from req_err
except ApiDataRetrievalFailed as data_err:
self.logger.error(f"Data retrieval error: {data_err}")
raise
except Exception as err:
self.logger.error(f"An unexpected error occurred: {err}")
raise ApiMaxRetryError from err

def get_all_data(
self,
Expand Down
Loading