[Feature]: pagination for source http (#2847)

* prelimenary change for http source follow next page

* remove old commented code

* update http.rb to check for config settings

* update docs

* Update CHANGELOG.md

* remove redundant return

* attempt to fix indentation

* attempt 2 at fixing CI checks

* fix module... broke by accident

* removed tabs

* updates

* updates

* bad copy/paste

* using control flow

* those tabs are annoying...

* remove trailing whitespace

* using a guard clause

* adding guard line after raise clause

* removed redundant if condition

* Update CHANGELOG.md

* Update CHANGELOG.md

---------

Co-authored-by: Dave Macias <[email protected]>

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## Added
 - model for D-Link cisco like CLI (@mirackle-spb)
 - model for Ruijie Networks RGOS devices (@spike77453)
+- pagination for http source (@davama)
 
 + Added ability to send mail with the Docker container
 + Documentation to send mail with hooks

docs/Sources.md

@@ -187,3 +187,16 @@ source:
  scheme: https
  secure: false
 ```
+
+HTTP source also supports pagination. Two settings must be enabled. (`pagination` as a bool and `pagination_key_name` as a string)
+The `pagination_key_name` setting is the key name that an api returns to find the url of the next page.
+
+**Disclaimer**: currently only tested with netbox as the source
+
+```yaml
+source:
+ default: http
+ http:
+ pagination: true
+ pagination_key_name: 'next'
+```

lib/oxidized/source/http.rb

@@ -18,8 +18,7 @@ def setup
 
  def load(node_want = nil)
  nodes = []
- data = JSON.parse(read_http(node_want))
- data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+ data = read_http(node_want)
  data.each do |node|
  next if node.empty?
 
@@ -45,6 +44,14 @@ def load(node_want = nil)
 
  private
 
+ def set_request(l_uri, l_headers, l_node_want)
+ req_uri = l_uri.request_uri
+ req_uri = "#{req_uri}/#{l_node_want}" if l_node_want
+ request = Net::HTTP::Get.new(req_uri, l_headers)
+ request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
+ request
+ end
+
  def string_navigate(object, wants)
  wants = wants.split(".").map do |want|
  head, match, _tail = want.partition(/\[\d+\]/)
@@ -56,6 +63,29 @@ def string_navigate(object, wants)
  object
  end
 
+ def check_pagination(response, http, headers, node_want)
+ node_data = []
+ if @cfg.pagination?
+ raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name?
+
+ next_key = @cfg.pagination_key_name
+ loop do
+ data = JSON.parse(response.body)
+ node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+ break if data[next_key].nil?
+
+ new_uri = URI.parse(data[next_key]) if data.has_key?(next_key)
+ request = set_request(new_uri, headers, node_want)
+ response = http.request(request)
+ end
+ # since new feature; dont break curent configs
+ else
+ data = JSON.parse(response.body)
+ node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+ end
+ node_data
+ end
+
  def read_http(node_want)
  uri = URI.parse(@cfg.url)
  http = Net::HTTP.new(uri.host, uri.port)
@@ -71,11 +101,10 @@ def read_http(node_want)
  headers[header] = value
  end
 
- req_uri = uri.request_uri
- req_uri = "#{req_uri}/#{node_want}" if node_want
- request = Net::HTTP::Get.new(req_uri, headers)
- request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
- http.request(request).body
+ request = set_request(uri, headers, node_want)
+ response = http.request(request)
+
+ check_pagination(response, http, headers, node_want)
  end
  end
 end