From c902515ad11359d648635491854996a721cc7b9f Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Thu, 15 Aug 2024 15:09:06 -0400 Subject: [PATCH 1/9] Neptune path is different It is single tenant so path switching for different db is not supported --- lib/neo4j/http/configuration.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/neo4j/http/configuration.rb b/lib/neo4j/http/configuration.rb index f4f946b..2d7d67f 100644 --- a/lib/neo4j/http/configuration.rb +++ b/lib/neo4j/http/configuration.rb @@ -26,11 +26,8 @@ def initialize(options = ENV) def transaction_path # v3.5 - /db/data/transaction/commit # v4.x - /db/#{database_name}/tx/commit - if database_name - "/db/#{database_name}/tx/commit" - else - "/db/data/transaction/commit" - end + # https://docs.aws.amazon.com/neptune/latest/userguide/access-graph-opencypher-queries.html + "/openCypher" end def auth_token From ee421f99f8290c4dd6210216a99986c8951fb5a0 Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Thu, 15 Aug 2024 15:10:09 -0400 Subject: [PATCH 2/9] Change way we call the query. Neptune takes a single query at a time --- lib/neo4j/http/cypher_client.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/neo4j/http/cypher_client.rb b/lib/neo4j/http/cypher_client.rb index cad3935..c27f108 100644 --- a/lib/neo4j/http/cypher_client.rb +++ b/lib/neo4j/http/cypher_client.rb @@ -24,20 +24,17 @@ def execute_cypher(cypher, parameters = {}) # for improved routing performance on read only queries access_mode = parameters.delete(:access_mode) || @configuration.access_mode + # https://docs.aws.amazon.com/neptune/latest/userguide/opencypher-parameterized-queries.html request_body = { - statements: [ - { - statement: cypher, - parameters: parameters.as_json - } - ] + query: cypher, + parameters: parameters.as_json } @connection = @injected_connection || connection(access_mode) response = @connection.post(transaction_path, request_body) results = check_errors!(cypher, response, parameters) - Neo4j::Http::Results.parse(results&.first || {}) + Neo4j::Http::Results.parse(results || []) end def connection(access_mode) From d5f74d0fa47eeb51e90e79007929c172990cb975 Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Thu, 15 Aug 2024 15:10:55 -0400 Subject: [PATCH 3/9] Change our error detection and handling --- lib/neo4j/http/cypher_client.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/neo4j/http/cypher_client.rb b/lib/neo4j/http/cypher_client.rb index c27f108..d1f095c 100644 --- a/lib/neo4j/http/cypher_client.rb +++ b/lib/neo4j/http/cypher_client.rb @@ -46,12 +46,13 @@ def connection(access_mode) delegate :auth_token, :transaction_path, to: :@configuration def check_errors!(cypher, response, parameters) raise Neo4j::Http::Errors::InvalidConnectionUrl, response.status if response.status == 404 - if response.body["errors"].any? { |error| error["message"][/Routing WRITE queries is not supported/] } - raise Neo4j::Http::Errors::ReadOnlyError - end + # Neptune error format: + # {"requestId":"c1470a57-7bd0-4b88-b86b-71e64e7e6a2d","code":"MalformedQueryException", + # "detailedMessage":"It is not allowed to refer to variables in LIMIT (line 1, column 54 (offset: 53))", + # "message":"It is not allowed to refer to variables in LIMIT (line 1, column 54 (offset: 53))"} body = response.body || {} - errors = body.fetch("errors", []) + errors = body.fetch("message", []) return body.fetch("results", {}) unless errors.present? error = errors.first @@ -59,11 +60,8 @@ def check_errors!(cypher, response, parameters) end def raise_error(error, cypher, parameters = {}) - code = error["code"] - message = error["message"] - klass = find_error_class(code) parameters = JSON.pretty_generate(parameters.as_json) - raise klass, "#{code} - #{message}\n cypher: #{cypher} \n parameters given: \n#{parameters}" + raise Neo4j::Http::Errors::Neo4jCodedError, "#{error}\n cypher: #{cypher} \n parameters given: \n#{parameters}" end def find_error_class(code) From 366beb60b64b04b0a1bd0c8860fca37bdac6cf3e Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Thu, 15 Aug 2024 15:12:36 -0400 Subject: [PATCH 4/9] Ignore SSL errors for now (dev cert is invalid) --- lib/neo4j/http/cypher_client.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/neo4j/http/cypher_client.rb b/lib/neo4j/http/cypher_client.rb index d1f095c..321aee4 100644 --- a/lib/neo4j/http/cypher_client.rb +++ b/lib/neo4j/http/cypher_client.rb @@ -72,8 +72,9 @@ def find_error_class(code) def build_connection(access_mode) # https://neo4j.com/docs/http-api/current/actions/transaction-configuration/ - headers = build_http_headers.merge({"access-mode" => access_mode}) - Faraday.new(url: @configuration.uri, headers: headers, request: build_request_options) do |f| + headers = build_http_headers + # TODO we disable SSL verification for development env + Faraday.new(url: @configuration.uri, headers: headers, request: build_request_options, ssl: { verify: false }) do |f| f.request :json # encode req bodies as JSON f.request :retry # retry transient failures f.response :json # decode response bodies as JSON @@ -90,16 +91,11 @@ def build_request_options end def build_http_headers - { - "User-Agent" => @configuration.user_agent, - "Accept" => "application/json" - }.merge(authentication_headers) + return {} end def authentication_headers - return {} if auth_token.blank? - - {"Authentication" => "Basic #{auth_token}"} + return {} end end end From 203d871f8d3385a95855e9b735e5f0057a107852 Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Thu, 15 Aug 2024 15:12:48 -0400 Subject: [PATCH 5/9] Change result parsing for Neptune format --- lib/neo4j/http/results.rb | 77 ++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/lib/neo4j/http/results.rb b/lib/neo4j/http/results.rb index 5f152bd..21c8e19 100644 --- a/lib/neo4j/http/results.rb +++ b/lib/neo4j/http/results.rb @@ -3,26 +3,69 @@ module Neo4j module Http class Results - # Example result set: - # [{"columns"=>["n"], - # "data"=> - # [{"row"=>[{"name"=>"Foo", "uuid"=>"8c7dcfda-d848-4937-a91a-2e6debad2dd6"}], - # "meta"=>[{"id"=>242, "type"=>"node", "deleted"=>false}]}]}] - # + # And return result from this gem + # [{"u1"=> + # {"specialty"=>"Colon & Rectal Surgery", + # "city"=>"Staten Island", + # "credentials"=>"DO", + # "profile_url"=>"https://doximity.qa-skaur-1.doximity-staging.services/profiles/f3dff658-3c79-4227-878b-8cf56a6d32c9", + # "last_name"=>"test", + # "lon"=>-74.1780014038086, + # "state"=>"New York", + # "uuid"=>"f3dff658-3c79-4227-878b-8cf56a6d32c9", + # "legacy_elasticsearch_user_id"=>5532386, + # "first_name"=>"samm", + # "lat"=>40.61940002441406, + # "subspecialties"=>"General Colon & Rectal Surgery, Clinical Pharmacology, Clinical Informatics", + # "_neo4j_meta_data"=>{"id"=>0, "type"=>"node", "deleted"=>false}}}, + # {"u1"=> + # {"specialty"=>"Other MD/DO", + # "city"=>"Marina", + # "profile_url"=>"https://doximity.qa-skaur-1.doximity-staging.services/profiles/f40908ec-2f4e-4a49-97bc-68e70f341f9d", + # "credentials"=>"MD", + # "last_name"=>"kaur", + # "lon"=>-121.80216979980469, + # "state"=>"California", + # "uuid"=>"f40908ec-2f4e-4a49-97bc-68e70f341f9d", + # "legacy_elasticsearch_user_id"=>5532349, + # "first_name"=>"sandip", + # "lat"=>36.68439865112305, + # "subspecialties"=>"Other MD/DO", + # "_neo4j_meta_data"=>{"id"=>1, "type"=>"node", "deleted"=>false}}}] + + # Neptune Example result set + # { + # "results": [{ + # "n1": { + # "~id": "7b3caa7b-4187-4689-be1f-8d8c924a23df", + # "~entityType": "node", + # "~labels": ["User"], + # "~properties": { + # "uuid": "abc123" + # } + # } + # }, { + # "n1": { + # "~id": "28b2bc2a-03d4-415d-9f37-bd4b9420e449", + # "~entityType": "node", + # "~labels": ["User"], + # "~properties": { + # "uuid": "def456" + # } + # } + # }] def self.parse(results) - columns = results["columns"] - data = results["data"] + results.map do |result| + entries = {} + result.each_pair do |key, value| + meta = { + "id" => value["~id"], + "type" => value["~entityType"], + } - data.map do |result| - row = result["row"] || [] - meta = result["meta"] || [] - compacted_data = row.each_with_index.map do |attributes, index| - row_meta = meta[index] || {} - attributes["_neo4j_meta_data"] = row_meta if attributes.is_a?(Hash) - attributes + entries[key] = value["~properties"].merge({ "_neo4j_meta_data" => meta }) end - - columns.zip(compacted_data).to_h.with_indifferent_access + entries end end end From 006150cc02bbaaef21f5f12512122742bfbb90c1 Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Fri, 16 Aug 2024 17:59:21 -0400 Subject: [PATCH 6/9] Change error handling - only one error is surfaced --- lib/neo4j/http/cypher_client.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/neo4j/http/cypher_client.rb b/lib/neo4j/http/cypher_client.rb index 321aee4..ce4a805 100644 --- a/lib/neo4j/http/cypher_client.rb +++ b/lib/neo4j/http/cypher_client.rb @@ -52,10 +52,9 @@ def check_errors!(cypher, response, parameters) # "message":"It is not allowed to refer to variables in LIMIT (line 1, column 54 (offset: 53))"} body = response.body || {} - errors = body.fetch("message", []) - return body.fetch("results", {}) unless errors.present? + error = body.fetch("message", []) + return body.fetch("results", {}) unless error.present? - error = errors.first raise_error(error, cypher, parameters) end From 92ff1e7b96a9a44aa0b4f4882bbe698cdc6d628c Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Fri, 16 Aug 2024 17:59:49 -0400 Subject: [PATCH 7/9] Handle non-hash results --- lib/neo4j/http/results.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/neo4j/http/results.rb b/lib/neo4j/http/results.rb index 21c8e19..ca6908c 100644 --- a/lib/neo4j/http/results.rb +++ b/lib/neo4j/http/results.rb @@ -58,12 +58,15 @@ def self.parse(results) results.map do |result| entries = {} result.each_pair do |key, value| - meta = { - "id" => value["~id"], - "type" => value["~entityType"], - } - - entries[key] = value["~properties"].merge({ "_neo4j_meta_data" => meta }) + if value.is_a?(Hash) + meta = { + "id" => value["~id"], + "type" => value["~entityType"], + } + entries[key] = (value["~properties"] || {}).merge({ "_neo4j_meta_data" => meta }).with_indifferent_access + else + entries[key] = value + end end entries end From 98971913fdb2988ec34256abd7c623818809410c Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Wed, 21 Aug 2024 11:40:19 -0400 Subject: [PATCH 8/9] Change how we parse ~properties In Neo4j the properties of a node or relationship were at the level as the id, and type keys. In Neptune these properties are stored in a hash under a "~properties" key. We hoist these values to be at the same level as Neo4j so our query results are formatted the same way --- lib/neo4j/http/results.rb | 79 ++++------------- spec/neo4j/http/results_spec.rb | 146 ++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 64 deletions(-) create mode 100644 spec/neo4j/http/results_spec.rb diff --git a/lib/neo4j/http/results.rb b/lib/neo4j/http/results.rb index ca6908c..2207060 100644 --- a/lib/neo4j/http/results.rb +++ b/lib/neo4j/http/results.rb @@ -3,72 +3,23 @@ module Neo4j module Http class Results - # And return result from this gem - # [{"u1"=> - # {"specialty"=>"Colon & Rectal Surgery", - # "city"=>"Staten Island", - # "credentials"=>"DO", - # "profile_url"=>"https://doximity.qa-skaur-1.doximity-staging.services/profiles/f3dff658-3c79-4227-878b-8cf56a6d32c9", - # "last_name"=>"test", - # "lon"=>-74.1780014038086, - # "state"=>"New York", - # "uuid"=>"f3dff658-3c79-4227-878b-8cf56a6d32c9", - # "legacy_elasticsearch_user_id"=>5532386, - # "first_name"=>"samm", - # "lat"=>40.61940002441406, - # "subspecialties"=>"General Colon & Rectal Surgery, Clinical Pharmacology, Clinical Informatics", - # "_neo4j_meta_data"=>{"id"=>0, "type"=>"node", "deleted"=>false}}}, - # {"u1"=> - # {"specialty"=>"Other MD/DO", - # "city"=>"Marina", - # "profile_url"=>"https://doximity.qa-skaur-1.doximity-staging.services/profiles/f40908ec-2f4e-4a49-97bc-68e70f341f9d", - # "credentials"=>"MD", - # "last_name"=>"kaur", - # "lon"=>-121.80216979980469, - # "state"=>"California", - # "uuid"=>"f40908ec-2f4e-4a49-97bc-68e70f341f9d", - # "legacy_elasticsearch_user_id"=>5532349, - # "first_name"=>"sandip", - # "lat"=>36.68439865112305, - # "subspecialties"=>"Other MD/DO", - # "_neo4j_meta_data"=>{"id"=>1, "type"=>"node", "deleted"=>false}}}] - - # Neptune Example result set - # { - # "results": [{ - # "n1": { - # "~id": "7b3caa7b-4187-4689-be1f-8d8c924a23df", - # "~entityType": "node", - # "~labels": ["User"], - # "~properties": { - # "uuid": "abc123" - # } - # } - # }, { - # "n1": { - # "~id": "28b2bc2a-03d4-415d-9f37-bd4b9420e449", - # "~entityType": "node", - # "~labels": ["User"], - # "~properties": { - # "uuid": "def456" - # } - # } - # }] def self.parse(results) - results.map do |result| - entries = {} - result.each_pair do |key, value| - if value.is_a?(Hash) - meta = { - "id" => value["~id"], - "type" => value["~entityType"], - } - entries[key] = (value["~properties"] || {}).merge({ "_neo4j_meta_data" => meta }).with_indifferent_access - else - entries[key] = value - end + if results.is_a?(Array) + # Recuse on arrays + return results.map { |a| self.parse(a) } + elsif results.is_a?(Hash) + # Recurse on hashes + # Hoist ~properties key + new_hash = {} + results = results.except("~properties").merge(results["~properties"]) if results.key?("~properties") + + results.each_pair do |k,v| + new_hash[k] = self.parse(v) end - entries + return new_hash + else + # Primative value + return results end end end diff --git a/spec/neo4j/http/results_spec.rb b/spec/neo4j/http/results_spec.rb new file mode 100644 index 0000000..9530791 --- /dev/null +++ b/spec/neo4j/http/results_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require "spec_helper" + +# Examples taken from +# https://docs.aws.amazon.com/neptune/latest/userguide/access-graph-opencypher-queries.html +RSpec.describe Neo4j::Http::Results, type: :uses_neo4j do + it "returns node response" do + results = JSON.parse %q( + { + "results": [ + { + "a": { + "~id": "22", + "~entityType": "node", + "~labels": [ + "airport" + ], + "~properties": { + "desc": "Seattle-Tacoma", + "lon": -122.30899810791, + "runways": 3, + "type": "airport", + "country": "US", + "region": "US-WA", + "lat": 47.4490013122559, + "elev": 432, + "city": "Seattle", + "icao": "KSEA", + "code": "SEA", + "longest": 11901 + } + } + } + ] + } + ) + + output = described_class.parse(results["results"]) + expected = { + "desc" => "Seattle-Tacoma", + "lon" => -122.30899810791, + "runways" => 3, + "type" => "airport", + "country" => "US", + "region" => "US-WA", + "lat" => 47.4490013122559, + "elev" => 432, + "city" => "Seattle", + "icao" => "KSEA", + "code" => "SEA", + "longest" => 11901, + "~id" => "22", + "~entityType" => "node", + "~labels" => [ + "airport" + ], + } + expect(output[0]["a"]).to match(expected) + end + + it "returns relationship response" do + results = JSON.parse %q( + { + "results": [ + { + "r": { + "~id": "7389", + "~entityType": "relationship", + "~start": "22", + "~end": "151", + "~type": "route", + "~properties": { + "dist": 956 + } + } + } + ] + } + ) + + output = described_class.parse(results["results"]) + expected = { + "dist"=>956, + "~end" => "151", + "~entityType" => "relationship", + "~id" => "7389", + "~start" => "22", + "~type" => "route", + } + expect(output[0]["r"]).to eq(expected) + end + + it "returns value response" do + results = JSON.parse %q( + { + "results": [ + { + "count(a)": 121 + } + ] + } + ) + + output = described_class.parse(results["results"]) + expect(output[0]["count(a)"]).to eq(121) + end + + it "returns user network query results" do + results = [ + { + "colleagues" => [ + [{ + "~id" => "457e8e96-3cbc-4122-97a5-4dc9fe32dbc4", + "~entityType" => "node", + "~labels" => ["User"], + "~properties" => { + "name" => "Dolores Abernathy", "uuid" => "USER", "specialty" => "Immunology", "city" => "Westworld", "state" => "Utah" + } + }, { + "~id" => "387fb59e-9e7d-4507-ba17-7de691d3e13e", + "~entityType" => "relationship", + "~start" => "457e8e96-3cbc-4122-97a5-4dc9fe32dbc4", + "~end" => "48e1f64c-48d2-421f-8b10-38ebd58df761", + "~type" => "COLLEAGUES_WITH", + "~properties" => { + "state" => "invited" + } + }, { + "~id" => "48e1f64c-48d2-421f-8b10-38ebd58df761", + "~entityType" => "node", + "~labels" => ["User"], + "~properties" => { + "name" => "Maeve Millay", "uuid" => "OTHER-USER", "specialty" => "Immunology", "city" => "Westworld", "state" => "Utah" + } + }] + ], "medschool_classmates" => nil, "co_residents" => nil, "co_workers" => nil, "investigator_of" => nil, "full_name" => "Maeve Millay", "co_fellows" => nil, "co_authors" => nil, "paschool_classmates" => nil, "user_uuid" => "OTHER-USER" + } + ] + + output = described_class.parse(results) + expect(output[0]["colleagues"][0][0]["name"]).to eq("Dolores Abernathy") + expect(output[0]["colleagues"][0][1]["state"]).to eq("invited") + expect(output[0]["colleagues"][0][2]["name"]).to eq("Maeve Millay") + end +end From c92f2977fe02fa6ac81436a6c5703025b782ed8c Mon Sep 17 00:00:00 2001 From: Ben Simpson Date: Wed, 21 Aug 2024 11:50:26 -0400 Subject: [PATCH 9/9] Filter out the other ~ properties --- lib/neo4j/http/results.rb | 2 +- spec/neo4j/http/results_spec.rb | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/neo4j/http/results.rb b/lib/neo4j/http/results.rb index 2207060..98f86c7 100644 --- a/lib/neo4j/http/results.rb +++ b/lib/neo4j/http/results.rb @@ -11,7 +11,7 @@ def self.parse(results) # Recurse on hashes # Hoist ~properties key new_hash = {} - results = results.except("~properties").merge(results["~properties"]) if results.key?("~properties") + results = results.select {|k, _| k[0] != "~" }.merge(results["~properties"]) if results.key?("~properties") results.each_pair do |k,v| new_hash[k] = self.parse(v) diff --git a/spec/neo4j/http/results_spec.rb b/spec/neo4j/http/results_spec.rb index 9530791..cbebdc5 100644 --- a/spec/neo4j/http/results_spec.rb +++ b/spec/neo4j/http/results_spec.rb @@ -50,11 +50,6 @@ "icao" => "KSEA", "code" => "SEA", "longest" => 11901, - "~id" => "22", - "~entityType" => "node", - "~labels" => [ - "airport" - ], } expect(output[0]["a"]).to match(expected) end @@ -82,11 +77,6 @@ output = described_class.parse(results["results"]) expected = { "dist"=>956, - "~end" => "151", - "~entityType" => "relationship", - "~id" => "7389", - "~start" => "22", - "~type" => "route", } expect(output[0]["r"]).to eq(expected) end