Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add pagination in icurl() to support large amount of objects #174

Merged
merged 2 commits into from
Oct 15, 2024
Merged

Add pagination in icurl() to support large amount of objects #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3abfc44
feat: Add pagination in icurl() to handle 100K+ objects
takishida Oct 4, 2024
3ac9e73
test: Add pytest for icurl
takishida Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 31 additions & 12 deletions aci-preupgrade-validation-script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1044,16 +1044,7 @@ def print_result(title, result, msg='',
prints(output)


def icurl(apitype, query):
if apitype not in ['class', 'mo']:
print('invalid API type - %s' % apitype)
return []
uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query)
cmd = ['icurl', '-gs', uri]
logging.info('cmd = ' + ' '.join(cmd))
response = subprocess.check_output(cmd)
logging.debug('response: ' + str(response))
imdata = json.loads(response)['imdata']
def _icurl_error_handler(imdata):
if imdata and "error" in imdata[0]:
if "not found in class" in imdata[0]['error']['attributes']['text']:
raise OldVerPropNotFound('cversion does not have requested property')
Expand All @@ -1063,8 +1054,36 @@ def icurl(apitype, query):
raise OldVerClassNotFound('cversion does not have requested class')
else:
raise Exception('API call failed! Check debug log')
else:
return imdata


def _icurl(apitype, query, page=0, page_size=100000):
if apitype not in ['class', 'mo']:
print('invalid API type - %s' % apitype)
return []
pre = '&' if '?' in query else '?'
query += '{}page={}&page-size={}'.format(pre, page, page_size)
uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query)
cmd = ['icurl', '-gs', uri]
logging.info('cmd = ' + ' '.join(cmd))
response = subprocess.check_output(cmd)
logging.debug('response: ' + str(response))
data = json.loads(response)
_icurl_error_handler(data['imdata'])
return data


def icurl(apitype, query, page_size=100000):
total_imdata = []
total_cnt = 999999
page = 0
while total_cnt > len(total_imdata):
data = _icurl(apitype, query, page, page_size)
if not data['imdata']:
break
total_imdata += data['imdata']
total_cnt = int(data['totalCount'])
page += 1
return total_imdata


def get_credentials():
Expand Down
100 changes: 79 additions & 21 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,65 @@
@pytest.fixture
def icurl_outputs():
"""Allows each test function to simulate the responses of icurl() with a
test data in the form of { key: value }.
test data in the form of { query: test_data }.

where:
key = icurl query parameter such as `topSystem.json`
value = An expected result of icurl(). Should be the list under
`imdata` in the JSON response from APIC.
query = icurl query parameter such as `topSystem.json`
test_data = An expected result of icurl().
This can take different forms in the test data. See below for details.

If a single check performs multiple icurl quries, provide test data as
shown below:
{
"query1": "result1",
"query2": "result2",
"query1": "test_data1",
"query2": "test_data2",
}

Examples)

Option 1 - test_data is the content of imdata:
{
"object_class1.json?filter1=xxx&filter2=yyy": [],
"object_class2.json": [{"object_class": {"attributes": {}}}],
}

Option 2 - test_data is the whole response of an API query:
{
"object_class1.json?filter1=xxx&filter2=yyy": {
"totalCount": "0",
"imdata": [],
},
"object_class2.json": {
"totalCount": "1",
"imdata": [{"object_class": {"attributes": {}}}],
}
}

Option 3 - test_data is the bundle of API queries with multiple pages:
{
"object_class1.json?filter1=xxx&filter2=yyy": [
{
"totalCount": "0",
"imdata": [],
}
],
"object_class2.json": [
{
"totalCount": "199000",
"imdata": [
{"object_class": {"attributes": {...}}},
...
],
},
{
"totalCount": "199000",
"imdata": [
{"object_class": {"attributes": {...}}},
...
],
},
]
}
"""
return {
"object_class1.json?filter1=xxx&filter2=yyy": [],
Expand All @@ -38,21 +84,33 @@ def icurl_outputs():

@pytest.fixture
def mock_icurl(monkeypatch, icurl_outputs):
def _mock_icurl(apitype, query):
if icurl_outputs.get(query) is None:
def _mock_icurl(apitype, query, page=0, page_size=100000):
output = icurl_outputs.get(query)
if output is None:
log.error("Query `%s` not found in test data", query)

imdata = icurl_outputs.get(query, [])
if imdata and "error" in imdata[0]:
if "not found in class" in imdata[0]['error']['attributes']['text']:
raise script.OldVerPropNotFound('cversion does not have requested property')
elif "unresolved class for" in imdata[0]['error']['attributes']['text']:
raise script.OldVerClassNotFound('cversion does not have requested class')
elif "not found" in imdata[0]['error']['attributes']['text']:
raise script.OldVerClassNotFound('cversion does not have requested class')
data = {"totalCount": "0", "imdata": []}
elif isinstance(output, list):
# icurl_outputs option 1 - output is imdata which is empty
if not output:
data = {"totalCount": "0", "imdata": []}
# icurl_outputs option 1 - output is imdata
elif output[0].get("totalCount") is None:
data = {"totalCount": str(len(output)), "imdata": output}
# icurl_outputs option 3 - output is each page of icurl
elif len(output) > page:
data = output[page]
# icurl_outputs option 3 - output is each page of icurl
# page after the last page which is empty
else:
data = {"totalCount": output[0]["totalCount"], "imdata": []}
# icurl_outputs option 2 - output is full response of icurl without pages
elif isinstance(output, dict):
if page == 0:
data = output
else:
raise Exception('API call failed! Check debug log')
else:
return imdata
data = {"totalCount": output["totalCount"], "imdata": []}

script._icurl_error_handler(data['imdata'])
return data

monkeypatch.setattr(script, "icurl", _mock_icurl)
monkeypatch.setattr(script, "_icurl", _mock_icurl)
141 changes: 141 additions & 0 deletions tests/test_icurl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import pytest
import importlib

script = importlib.import_module("aci-preupgrade-validation-script")

# icurl queries
fabricNodePEps = "fabricNodePEp.json"

data = [
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-101-103/nodepep-101",
"id": "101",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-204-206/nodepep-206",
"id": "206",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-101-103/nodepep-103",
"id": "103",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-204-206/nodepep-204",
"id": "204",
}
}
},
]
long_data = data * 25000 # 100K entries
long_data_all = long_data * 2 + data


@pytest.mark.parametrize(
"apitype, query, icurl_outputs, expected_result",
[
# option 1: test_data is imdata
(
"class",
"fabricNodePEp.json",
{"fabricNodePEp.json": data},
data,
),
# option 2: test_data is the whole response of an API query (totalCount + imdata)
(
"class",
"fabricNodePEp.json",
{"fabricNodePEp.json": {"totalCount": str(len(data)), "imdata": data}},
data,
),
# option 3: test_data is the bundle of API queries with multiple pages
(
"class",
"fabricNodePEp.json",
{
"fabricNodePEp.json": [
{ # page 0
"totalCount": str(len(long_data_all)),
"imdata": long_data,
},
{ # page 1
"totalCount": str(len(long_data_all)),
"imdata": long_data,
},
{ # page 2
"totalCount": str(len(long_data_all)),
"imdata": data,
},
]
},
long_data_all,
),
],
)
def test_icurl(mock_icurl, apitype, query, expected_result):
assert script.icurl(apitype, query) == expected_result


@pytest.mark.parametrize(
"imdata, expected_exception",
[
# /api/class/faultInfo.json?query-target-filter=eq(faultInst.cod,"F2109")
(
[
{
"error": {
"attributes": {
"code": "121",
"text": "Prop 'cod' not found in class 'faultInst' property table",
}
}
}
],
script.OldVerPropNotFound,
),
# /api/class/faultInf.json?query-target-filter=eq(faultInst.code,"F2109")
(
[
{
"error": {
"attributes": {
"code": "400",
"text": "Request failed, unresolved class for faultInf",
}
}
}
],
script.OldVerClassNotFound,
),
# /api/class/faultInfo.json?query-target-filter=eq(faultIns.code,"F2109")
(
[
{
"error": {
"attributes": {
"code": "122",
"text": "class faultIns not found",
}
}
}
],
script.OldVerClassNotFound,
),
],
)
def test_icurl_error_handler(imdata, expected_exception):
with pytest.raises(expected_exception):
script._icurl_error_handler(imdata)