From 0606939179e96bbc4a31e6980ab51771156cf37f Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 15:57:41 -0400 Subject: [PATCH 01/21] prelimenary change for http source follow next page --- lib/oxidized/source/http.rb | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 02956a8c3..424cf58c9 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -18,8 +18,9 @@ def setup def load(node_want = nil) nodes = [] - data = JSON.parse(read_http(node_want)) - data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + #data = JSON.parse(read_http(node_want)) + #data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + data = read_http(node_want) data.each do |node| next if node.empty? @@ -45,6 +46,14 @@ def load(node_want = nil) private + def set_request(l_uri, l_headers, l_node_want) + req_uri = l_uri.request_uri + req_uri = "#{req_uri}/#{l_node_want}" if l_node_want + request = Net::HTTP::Get.new(req_uri, l_headers) + request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? + request + end + def string_navigate(object, wants) wants = wants.split(".").map do |want| head, match, _tail = want.partition(/\[\d+\]/) @@ -71,11 +80,31 @@ def read_http(node_want) headers[header] = value end - req_uri = uri.request_uri - req_uri = "#{req_uri}/#{node_want}" if node_want - request = Net::HTTP::Get.new(req_uri, headers) - request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? - http.request(request).body + request = set_request(uri, headers, node_want) + response = http.request(request) + + node_data = [] + + something = true + if not something + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + return node_data + else + loop do + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + if data['next'].nil? + break + end + if data.key?('next') + new_uri = URI.parse(data['next']) + end + request = set_request(new_uri, headers, node_want) + response = http.request(request) + end + return node_data + end end end end From 1d91df7c902740e29b453d127491152d4cdfb159 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 15:58:37 -0400 Subject: [PATCH 02/21] remove old commented code --- lib/oxidized/source/http.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 424cf58c9..0f1027eb6 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -18,8 +18,6 @@ def setup def load(node_want = nil) nodes = [] - #data = JSON.parse(read_http(node_want)) - #data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? data = read_http(node_want) data.each do |node| next if node.empty? From 0eb1abdd0d3266ef0d434ddee1484f385d3090f5 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 16:37:26 -0400 Subject: [PATCH 03/21] update http.rb to check for config settings --- lib/oxidized/source/http.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 0f1027eb6..4dd7f42db 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -83,20 +83,24 @@ def read_http(node_want) node_data = [] - something = true - if not something + # since new feature; dont break curent configs + if not @cfg.pagination? data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? return node_data else + if not @cfg.pagination_key_name? + raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" + end + next_key = @cfg.pagination_key_name loop do data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - if data['next'].nil? + if data[next_key].nil? break end - if data.key?('next') - new_uri = URI.parse(data['next']) + if data.key?(next_key) + new_uri = URI.parse(data[next_key]) end request = set_request(new_uri, headers, node_want) response = http.request(request) From d113cf14f319c7eeca1855bda08441af2a23d770 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 16:42:44 -0400 Subject: [PATCH 04/21] update docs --- docs/Sources.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/Sources.md b/docs/Sources.md index 06fedd81a..d5bab7781 100644 --- a/docs/Sources.md +++ b/docs/Sources.md @@ -187,3 +187,16 @@ source: scheme: https secure: false ``` + +HTTP source also supports pagination. Two settings must be enabled. (`pagination` as a bool and `pagination_key_name` as a string) +The `pagination_key_name` setting is the key name that an api returns to find the url of the next page. + +**Disclaimer**: currently only tested with netbox as the source + +```yaml +source: + default: http + http: + pagination: true + pagination_key_name: 'next' +``` From a7a8560f350fff1f7daa44b46649f52740a0c44a Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 17:13:31 -0400 Subject: [PATCH 05/21] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3b9a1a0..54eb1e321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Added - model for D-Link cisco like CLI (@mirackle-spb) +- pagination for http source (@davama) ## Changed From fb59b014eb9975df7a3b2644b761f5e6aae6689e Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 29 Jun 2023 18:50:46 -0400 Subject: [PATCH 06/21] remove redundant return --- lib/oxidized/source/http.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 4dd7f42db..7a52ec0d6 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -105,7 +105,7 @@ def read_http(node_want) request = set_request(new_uri, headers, node_want) response = http.request(request) end - return node_data + node_data end end end From 4c54a3f97318ff902411cda2df036c07db3a76dc Mon Sep 17 00:00:00 2001 From: Dave Macias Date: Fri, 30 Jun 2023 00:38:28 +0000 Subject: [PATCH 07/21] attempt to fix indentation --- lib/oxidized/source/http.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 7a52ec0d6..77272fa9d 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -89,19 +89,15 @@ def read_http(node_want) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? return node_data else - if not @cfg.pagination_key_name? + if not @cfg.pagination_key_name? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" - end - next_key = @cfg.pagination_key_name + end + next_key = @cfg.pagination_key_name loop do data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - if data[next_key].nil? - break - end - if data.key?(next_key) - new_uri = URI.parse(data[next_key]) - end + break if data[next_key].nil? + if data.key?(next_key) then new_uri = URI.parse(data[next_key]) end request = set_request(new_uri, headers, node_want) response = http.request(request) end From 97f1efac23b9d40cea83232b2b43bc732b949291 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 08:50:24 -0400 Subject: [PATCH 08/21] attempt 2 at fixing CI checks --- lib/oxidized/source/http.rb | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 77272fa9d..80db77e1e 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -83,24 +83,26 @@ def read_http(node_want) node_data = [] - # since new feature; dont break curent configs - if not @cfg.pagination? - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - return node_data - else - if not @cfg.pagination_key_name? + if @cfg.pagination? + if not @cfg.pagination_key_name? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" - end - next_key = @cfg.pagination_key_name + end + + next_key = @cfg.pagination_key_name loop do data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? - if data.key?(next_key) then new_uri = URI.parse(data[next_key]) end + break if data[next_key].nil? + + if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end request = set_request(new_uri, headers, node_want) response = http.request(request) end + # since new feature; dont break curent configs + else + node_data + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? node_data end end From 52bd054657deff5b2fb0d77e8188b4a4afb59ade Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 08:57:37 -0400 Subject: [PATCH 09/21] fix module... broke by accident --- lib/oxidized/source/http.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 80db77e1e..d3fe66681 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -83,7 +83,12 @@ def read_http(node_want) node_data = [] - if @cfg.pagination? + # since new feature; dont break curent configs + if not @cfg.pagination? + node_data + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + else if not @cfg.pagination_key_name? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" end @@ -98,13 +103,8 @@ def read_http(node_want) request = set_request(new_uri, headers, node_want) response = http.request(request) end - # since new feature; dont break curent configs - else - node_data - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - node_data end + node_data end end end From ed8bbbb250c5893d17259e1a1292fbd68023c1e5 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 09:07:13 -0400 Subject: [PATCH 10/21] removed tabs --- lib/oxidized/source/http.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index d3fe66681..de674e4ae 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -83,26 +83,26 @@ def read_http(node_want) node_data = [] - # since new feature; dont break curent configs - if not @cfg.pagination? - node_data - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - else - if not @cfg.pagination_key_name? + if @cfg.pagination? + if not @cfg.pagination_key_name? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" - end + end - next_key = @cfg.pagination_key_name + next_key = @cfg.pagination_key_name loop do data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? + break if data[next_key].nil? if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end request = set_request(new_uri, headers, node_want) response = http.request(request) end + # since new feature; dont break curent configs + else + node_data + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? end node_data end From 44b92951caa73ea25e5ef3f50985f7bf39e91538 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 09:15:06 -0400 Subject: [PATCH 11/21] updates --- lib/oxidized/source/http.rb | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index de674e4ae..3f5de67c2 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -84,23 +84,22 @@ def read_http(node_want) node_data = [] if @cfg.pagination? - if not @cfg.pagination_key_name? + if @cfg.pagination_key_name? + next_key = @cfg.pagination_key_name + loop do + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + break if data[next_key].nil? + + if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end + request = set_request(new_uri, headers, node_want) + response = http.request(request) + end + else raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" end - - next_key = @cfg.pagination_key_name - loop do - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? - - if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end - request = set_request(new_uri, headers, node_want) - response = http.request(request) - end # since new feature; dont break curent configs else - node_data data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? end From 69e6b179cfd6367b55a98cf1311d45b53c08ebaa Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 09:25:36 -0400 Subject: [PATCH 12/21] updates --- lib/oxidized/source/http.rb | 115 ++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 3f5de67c2..5dc312908 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -107,3 +107,118 @@ def read_http(node_want) end end end +[root@dcom-dave-vm docker-oxidized]# vi http.rb +[root@dcom-dave-vm docker-oxidized]# vi http.rb +[root@dcom-dave-vm docker-oxidized]# cat http.rb +module Oxidized + class HTTP < Source + def initialize + @cfg = Oxidized.config.source.http + super + end + + def setup + return unless @cfg.url.empty? + + raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config' + end + + require "net/http" + require "net/https" + require "uri" + require "json" + + def load(node_want = nil) + nodes = [] + data = read_http(node_want) + data.each do |node| + next if node.empty? + + # map node parameters + keys = {} + @cfg.map.each do |key, want_position| + keys[key.to_sym] = node_var_interpolate string_navigate(node, want_position) + end + keys[:model] = map_model keys[:model] if keys.has_key? :model + keys[:group] = map_group keys[:group] if keys.has_key? :group + + # map node specific vars + vars = {} + @cfg.vars_map.each do |key, want_position| + vars[key.to_sym] = node_var_interpolate string_navigate(node, want_position) + end + keys[:vars] = vars unless vars.empty? + + nodes << keys + end + nodes + end + + private + + def set_request(l_uri, l_headers, l_node_want) + req_uri = l_uri.request_uri + req_uri = "#{req_uri}/#{l_node_want}" if l_node_want + request = Net::HTTP::Get.new(req_uri, l_headers) + request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? + request + end + + def string_navigate(object, wants) + wants = wants.split(".").map do |want| + head, match, _tail = want.partition(/\[\d+\]/) + match.empty? ? head : [head, match[1..-2].to_i] + end + wants.flatten.each do |want| + object = object[want] if object.respond_to? :each + end + object + end + + def check_pagination(response, http, headers, node_want) + node_data = [] + if @cfg.pagination? + if @cfg.pagination_key_name? + next_key = @cfg.pagination_key_name + loop do + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + break if data[next_key].nil? + + if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end + request = set_request(new_uri, headers, node_want) + response = http.request(request) + end + else + raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" + end + # since new feature; dont break curent configs + else + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + end + node_data + end + + def read_http(node_want) + uri = URI.parse(@cfg.url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure + + # Add read_timeout to handle case of big list of nodes (default value is 60 seconds) + http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout" + + # map headers + headers = {} + @cfg.headers.each do |header, value| + headers[header] = value + end + + request = set_request(uri, headers, node_want) + response = http.request(request) + + check_pagination(response, http, headers, node_want) + end + end +end From 45aafbd597616328eaf143dee497344f56ede68d Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 09:27:46 -0400 Subject: [PATCH 13/21] bad copy/paste --- lib/oxidized/source/http.rb | 112 ------------------------------------ 1 file changed, 112 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 5dc312908..99e889d99 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -63,118 +63,6 @@ def string_navigate(object, wants) object end - def read_http(node_want) - uri = URI.parse(@cfg.url) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if uri.scheme == 'https' - http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure - - # Add read_timeout to handle case of big list of nodes (default value is 60 seconds) - http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout" - - # map headers - headers = {} - @cfg.headers.each do |header, value| - headers[header] = value - end - - request = set_request(uri, headers, node_want) - response = http.request(request) - - node_data = [] - - if @cfg.pagination? - if @cfg.pagination_key_name? - next_key = @cfg.pagination_key_name - loop do - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? - - if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end - request = set_request(new_uri, headers, node_want) - response = http.request(request) - end - else - raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" - end - # since new feature; dont break curent configs - else - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - end - node_data - end - end -end -[root@dcom-dave-vm docker-oxidized]# vi http.rb -[root@dcom-dave-vm docker-oxidized]# vi http.rb -[root@dcom-dave-vm docker-oxidized]# cat http.rb -module Oxidized - class HTTP < Source - def initialize - @cfg = Oxidized.config.source.http - super - end - - def setup - return unless @cfg.url.empty? - - raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config' - end - - require "net/http" - require "net/https" - require "uri" - require "json" - - def load(node_want = nil) - nodes = [] - data = read_http(node_want) - data.each do |node| - next if node.empty? - - # map node parameters - keys = {} - @cfg.map.each do |key, want_position| - keys[key.to_sym] = node_var_interpolate string_navigate(node, want_position) - end - keys[:model] = map_model keys[:model] if keys.has_key? :model - keys[:group] = map_group keys[:group] if keys.has_key? :group - - # map node specific vars - vars = {} - @cfg.vars_map.each do |key, want_position| - vars[key.to_sym] = node_var_interpolate string_navigate(node, want_position) - end - keys[:vars] = vars unless vars.empty? - - nodes << keys - end - nodes - end - - private - - def set_request(l_uri, l_headers, l_node_want) - req_uri = l_uri.request_uri - req_uri = "#{req_uri}/#{l_node_want}" if l_node_want - request = Net::HTTP::Get.new(req_uri, l_headers) - request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? - request - end - - def string_navigate(object, wants) - wants = wants.split(".").map do |want| - head, match, _tail = want.partition(/\[\d+\]/) - match.empty? ? head : [head, match[1..-2].to_i] - end - wants.flatten.each do |want| - object = object[want] if object.respond_to? :each - end - object - end - def check_pagination(response, http, headers, node_want) node_data = [] if @cfg.pagination? From ea92baee55a40045e175bf2f84462ba2eb55b6be Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 11:04:57 -0400 Subject: [PATCH 14/21] using control flow --- lib/oxidized/source/http.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 99e889d99..23d06c65a 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -72,8 +72,7 @@ def check_pagination(response, http, headers, node_want) data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? break if data[next_key].nil? - - if data.has_key?(next_key) then new_uri = URI.parse(data[next_key]) end + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) request = set_request(new_uri, headers, node_want) response = http.request(request) end From 1bad0ca9f32a010c015bdaf0760f227df329c484 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 11:07:59 -0400 Subject: [PATCH 15/21] those tabs are annoying... --- lib/oxidized/source/http.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 23d06c65a..5fbb57696 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -72,7 +72,8 @@ def check_pagination(response, http, headers, node_want) data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? break if data[next_key].nil? - new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) + + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) request = set_request(new_uri, headers, node_want) response = http.request(request) end From b2bd02160951eff4074bfede898e4fbf39cf8f8c Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 11:09:47 -0400 Subject: [PATCH 16/21] remove trailing whitespace --- lib/oxidized/source/http.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 5fbb57696..8b6aee12c 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -72,7 +72,7 @@ def check_pagination(response, http, headers, node_want) data = JSON.parse(response.body) node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? break if data[next_key].nil? - + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) request = set_request(new_uri, headers, node_want) response = http.request(request) From fa04822a40a90952945613b58347bada9dff670f Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 11:19:13 -0400 Subject: [PATCH 17/21] using a guard clause --- lib/oxidized/source/http.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 8b6aee12c..96e1b1dbd 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -66,6 +66,7 @@ def string_navigate(object, wants) def check_pagination(response, http, headers, node_want) node_data = [] if @cfg.pagination? + raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? if @cfg.pagination_key_name? next_key = @cfg.pagination_key_name loop do @@ -77,8 +78,6 @@ def check_pagination(response, http, headers, node_want) request = set_request(new_uri, headers, node_want) response = http.request(request) end - else - raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" end # since new feature; dont break curent configs else From 3822bada0ef16d31a931fde0c67609c93788ed6d Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 30 Jun 2023 11:21:11 -0400 Subject: [PATCH 18/21] adding guard line after raise clause --- lib/oxidized/source/http.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 96e1b1dbd..e868ae349 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -67,6 +67,7 @@ def check_pagination(response, http, headers, node_want) node_data = [] if @cfg.pagination? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? + if @cfg.pagination_key_name? next_key = @cfg.pagination_key_name loop do From 4c023570fee8a371c852971a989b31994d2975f1 Mon Sep 17 00:00:00 2001 From: Dave Macias Date: Fri, 30 Jun 2023 14:53:34 -0400 Subject: [PATCH 19/21] removed redundant if condition --- lib/oxidized/source/http.rb | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index e868ae349..32dbeb161 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -68,17 +68,15 @@ def check_pagination(response, http, headers, node_want) if @cfg.pagination? raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? - if @cfg.pagination_key_name? - next_key = @cfg.pagination_key_name - loop do - data = JSON.parse(response.body) - node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? - - new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) - request = set_request(new_uri, headers, node_want) - response = http.request(request) - end + next_key = @cfg.pagination_key_name + loop do + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + break if data[next_key].nil? + + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) + request = set_request(new_uri, headers, node_want) + response = http.request(request) end # since new feature; dont break curent configs else From f695db5dfaad7a80740620fc3bdca687257d1f49 Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 10 Jul 2023 11:31:41 -0400 Subject: [PATCH 20/21] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37dfda89d..02c70db41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Added - model for D-Link cisco like CLI (@mirackle-spb) - model for Ruijie Networks RGOS devices (@spike77453) +- pagination for http source (@davama) + + Added ability to send mail with the Docker container + Documentation to send mail with hooks -- pagination for http source (@davama) ## Changed From fab3a2b75fce9b897c12135f04010496cb858643 Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 10 Jul 2023 11:32:13 -0400 Subject: [PATCH 21/21] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02c70db41..15e7ad607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + Added ability to send mail with the Docker container + Documentation to send mail with hooks - ## Changed ## Fixed