diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e014b5fa..15e7ad607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Added - model for D-Link cisco like CLI (@mirackle-spb) - model for Ruijie Networks RGOS devices (@spike77453) +- pagination for http source (@davama) + Added ability to send mail with the Docker container + Documentation to send mail with hooks diff --git a/docs/Sources.md b/docs/Sources.md index 06fedd81a..d5bab7781 100644 --- a/docs/Sources.md +++ b/docs/Sources.md @@ -187,3 +187,16 @@ source: scheme: https secure: false ``` + +HTTP source also supports pagination. Two settings must be enabled. (`pagination` as a bool and `pagination_key_name` as a string) +The `pagination_key_name` setting is the key name that an api returns to find the url of the next page. + +**Disclaimer**: currently only tested with netbox as the source + +```yaml +source: + default: http + http: + pagination: true + pagination_key_name: 'next' +``` diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 02956a8c3..32dbeb161 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -18,8 +18,7 @@ def setup def load(node_want = nil) nodes = [] - data = JSON.parse(read_http(node_want)) - data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + data = read_http(node_want) data.each do |node| next if node.empty? @@ -45,6 +44,14 @@ def load(node_want = nil) private + def set_request(l_uri, l_headers, l_node_want) + req_uri = l_uri.request_uri + req_uri = "#{req_uri}/#{l_node_want}" if l_node_want + request = Net::HTTP::Get.new(req_uri, l_headers) + request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? + request + end + def string_navigate(object, wants) wants = wants.split(".").map do |want| head, match, _tail = want.partition(/\[\d+\]/) @@ -56,6 +63,29 @@ def string_navigate(object, wants) object end + def check_pagination(response, http, headers, node_want) + node_data = [] + if @cfg.pagination? + raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? + + next_key = @cfg.pagination_key_name + loop do + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + break if data[next_key].nil? + + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) + request = set_request(new_uri, headers, node_want) + response = http.request(request) + end + # since new feature; dont break curent configs + else + data = JSON.parse(response.body) + node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? + end + node_data + end + def read_http(node_want) uri = URI.parse(@cfg.url) http = Net::HTTP.new(uri.host, uri.port) @@ -71,11 +101,10 @@ def read_http(node_want) headers[header] = value end - req_uri = uri.request_uri - req_uri = "#{req_uri}/#{node_want}" if node_want - request = Net::HTTP::Get.new(req_uri, headers) - request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? - http.request(request).body + request = set_request(uri, headers, node_want) + response = http.request(request) + + check_pagination(response, http, headers, node_want) end end end