From d2a9c4394f6f253927c266c6bfa5eb95bbcb42da Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 24 Oct 2023 16:13:47 +0200 Subject: [PATCH 1/8] fix: Arel::Nodes::SelectStatement --- lib/core_extensions/arel/nodes/select_statement.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core_extensions/arel/nodes/select_statement.rb b/lib/core_extensions/arel/nodes/select_statement.rb index d7f3807e..2784d027 100644 --- a/lib/core_extensions/arel/nodes/select_statement.rb +++ b/lib/core_extensions/arel/nodes/select_statement.rb @@ -4,8 +4,8 @@ module Nodes module SelectStatement attr_accessor :settings - def initialize - super + def initialize(relation = nil) + super(relation) @settings = nil end From f33abdce871513dc04c521092827bf34b6d73b1d Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 24 Oct 2023 16:20:05 +0200 Subject: [PATCH 2/8] fix: Type mapping for rails 7+ multi database --- .../connection_adapters/clickhouse_adapter.rb | 126 ++++++++++-------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index 88540e4d..690b8663 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -14,37 +14,44 @@ require 'openssl' module ActiveRecord - class Base - class << self - # Establishes a connection to the database that's used by all Active Record objects - def clickhouse_connection(config) - config = config.symbolize_keys - - if config[:connection] - connection = { - connection: config[:connection] - } - else - port = config[:port] || 8123 - connection = { - host: config[:host] || 'localhost', - port: port, - ssl: config[:ssl].present? ? config[:ssl] : port == 443, - sslca: config[:sslca], - read_timeout: config[:read_timeout], - write_timeout: config[:write_timeout], - keep_alive_timeout: config[:keep_alive_timeout] - } - end - - if config.key?(:database) - database = config[:database] - else - raise ArgumentError, 'No database specified. Missing argument: database.' - end + module ConnectionHandling # :nodoc: + def clickhouse_adapdated_class + ConnectionAdapters::ClickhouseAdapter + end - ConnectionAdapters::ClickhouseAdapter.new(logger, connection, { user: config[:username], password: config[:password], database: database }.compact, config) - end + # Establishes a connection to the database that's used by all Active Record objects + def clickhouse_connection(config) + config = config.symbolize_keys + + if config[:connection] + connection = { + connection: config[:connection] + } + else + port = config[:port] || 8123 + connection = { + host: config[:host] || 'localhost', + port: port, + ssl: config[:ssl].present? ? config[:ssl] : port == 443, + sslca: config[:sslca], + read_timeout: config[:read_timeout], + write_timeout: config[:write_timeout], + keep_alive_timeout: config[:keep_alive_timeout] + } + end + + if config.key?(:database) + database = config[:database] + else + raise ArgumentError, 'No database specified. Missing argument: database.' + end + + clickhouse_adapdated_class.new( + logger, + connection, + { user: config[:username], password: config[:password], database: database }.compact, + config + ) end end @@ -117,6 +124,36 @@ class ClickhouseAdapter < AbstractAdapter include Clickhouse::SchemaStatements + class << self + private + + def initialize_type_map(m) # :nodoc: + super + register_class_with_limit m, %r(String), Type::String + register_class_with_limit m, 'Date', Clickhouse::OID::Date + register_class_with_precision m, %r(datetime)i, Clickhouse::OID::DateTime + + register_class_with_limit m, %r(Int8), Type::Integer + register_class_with_limit m, %r(Int16), Type::Integer + register_class_with_limit m, %r(Int32), Type::Integer + register_class_with_limit m, %r(Int64), Type::Integer + register_class_with_limit m, %r(Int128), Type::Integer + register_class_with_limit m, %r(Int256), Type::Integer + + register_class_with_limit m, %r(UInt8), Type::UnsignedInteger + register_class_with_limit m, %r(UInt16), Type::UnsignedInteger + register_class_with_limit m, %r(UInt32), Type::UnsignedInteger + register_class_with_limit m, %r(UInt64), Type::UnsignedInteger + register_class_with_limit m, %r(UInt256), Type::UnsignedInteger + # register_class_with_limit m, %r(Array), Clickhouse::OID::Array + m.register_type(%r(Array)) do |sql_type| + Clickhouse::OID::Array.new(sql_type) + end + end + end + + TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) } + # Initializes and connects a Clickhouse adapter. def initialize(logger, connection_parameters, config, full_config) super(nil, logger) @@ -150,6 +187,10 @@ def arel_visitor # :nodoc: Arel::Visitors::Clickhouse.new(self) end + def type_map + self.class::TYPE_MAP + end + def native_database_types #:nodoc: NATIVE_DATABASE_TYPES end @@ -189,31 +230,6 @@ def extract_precision(sql_type) $1.to_i if sql_type =~ /\((\d+)(,\s?\d+)?\)/ end - def initialize_type_map(m) # :nodoc: - super - register_class_with_limit m, %r(String), Type::String - register_class_with_limit m, 'Date', Clickhouse::OID::Date - register_class_with_precision m, %r(datetime)i, Clickhouse::OID::DateTime - - register_class_with_limit m, %r(Int8), Type::Integer - register_class_with_limit m, %r(Int16), Type::Integer - register_class_with_limit m, %r(Int32), Type::Integer - register_class_with_limit m, %r(Int64), Type::Integer - register_class_with_limit m, %r(Int128), Type::Integer - register_class_with_limit m, %r(Int256), Type::Integer - - register_class_with_limit m, %r(UInt8), Type::UnsignedInteger - register_class_with_limit m, %r(UInt16), Type::UnsignedInteger - register_class_with_limit m, %r(UInt32), Type::UnsignedInteger - register_class_with_limit m, %r(UInt64), Type::UnsignedInteger - #register_class_with_limit m, %r(UInt128), Type::UnsignedInteger #not implemnted in clickhouse - register_class_with_limit m, %r(UInt256), Type::UnsignedInteger - # register_class_with_limit m, %r(Array), Clickhouse::OID::Array - m.register_type(%r(Array)) do |sql_type| - Clickhouse::OID::Array.new(sql_type) - end - end - def _quote(value) case value when Array From 905cd0f14ef6034681c8f010b73995d3361253ad Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 31 Oct 2023 12:59:24 +0100 Subject: [PATCH 3/8] feat(map): First attempt to add Map type support --- .../connection_adapters/clickhouse/oid/map.rb | 58 +++++++++++++++++++ .../clickhouse/schema_definitions.rb | 7 +++ .../connection_adapters/clickhouse_adapter.rb | 38 ++++++++++++ lib/clickhouse-activerecord/schema_dumper.rb | 6 ++ 4 files changed, 109 insertions(+) create mode 100644 lib/active_record/connection_adapters/clickhouse/oid/map.rb diff --git a/lib/active_record/connection_adapters/clickhouse/oid/map.rb b/lib/active_record/connection_adapters/clickhouse/oid/map.rb new file mode 100644 index 00000000..287db1a6 --- /dev/null +++ b/lib/active_record/connection_adapters/clickhouse/oid/map.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters + module Clickhouse + module OID # :nodoc: + class Map < Type::Value # :nodoc: + attr_reader :key_type, :value_type + + def initialize(sql_type) + types = sql_type.match(/Map\((.+)\,\s?(.+)\)/).captures + + @key_type = cast_type(types.first) + @value_type = cast_type(types.last) + end + + def type + :map + end + + def cast(value) + value + end + + def deserialize(value) + return value if value.is_a?(Hash) + + JSON.parse(value) + end + + + def serialize(value) + return '{}' if value.nil? + + "#{value.map { |key, value| "'#{key}': '#{value}'" }.join(' ')}" + end + + private + + def cast_type(type) + return type if type.nil? + + case type + when /U?Int\d+/ + :integer + when /DateTime/ + :datetime + when /Date/ + :date + else + :string + end + end + end + end + end + end +end diff --git a/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb b/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb index 54b89593..5bb5f24e 100644 --- a/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +++ b/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb @@ -94,6 +94,13 @@ def enum(*args, **options) args.each { |name| column(name, kind, **options.except(:limit)) } end + + def map(*args, **options) + key_type = options[:key_type].to_s.camelize + value_type = options[:value_type].to_s.camelize + + args.each { |name| column(name, :"Map(#{key_type}, #{value_type})", **options.except(:limit, :key_type, :value_type)) } + end end end end diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index 690b8663..ce5087a5 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -6,6 +6,7 @@ require 'active_record/connection_adapters/clickhouse/oid/array' require 'active_record/connection_adapters/clickhouse/oid/date' require 'active_record/connection_adapters/clickhouse/oid/date_time' +require 'active_record/connection_adapters/clickhouse/oid/map' require 'active_record/connection_adapters/clickhouse/oid/big_integer' require 'active_record/connection_adapters/clickhouse/schema_definitions' require 'active_record/connection_adapters/clickhouse/schema_creation' @@ -87,7 +88,38 @@ def is_view=(value) module ConnectionAdapters class ClickhouseColumn < Column + def key_type + return nil unless type == :map + cast_type(map_types.first) + end + + def value_type + return nil unless type == :map + + cast_type(map_types.last) + end + + private + + def map_types + sql_type_metadata.sql_type.match(/Map\((.+)\,\s?(.+)\)/).captures + end + + def cast_type(type) + return type if type.nil? + + case type + when /U?Int\d+/ + :integer + when /DateTime/ + :datetime + when /Date/ + :date + else + :string + end + end end class ClickhouseAdapter < AbstractAdapter @@ -120,6 +152,8 @@ class ClickhouseAdapter < AbstractAdapter uint64: { name: 'UInt64' }, # uint128: { name: 'UInt128' }, not yet implemented in clickhouse uint256: { name: 'UInt256' }, + + map: { name: 'Map' } }.freeze include Clickhouse::SchemaStatements @@ -149,6 +183,10 @@ def initialize_type_map(m) # :nodoc: m.register_type(%r(Array)) do |sql_type| Clickhouse::OID::Array.new(sql_type) end + + m.register_type(%r(Map)) do |sql_type| + Clickhouse::OID::Map.new(sql_type) + end end end diff --git a/lib/clickhouse-activerecord/schema_dumper.rb b/lib/clickhouse-activerecord/schema_dumper.rb index 94981c6f..6badba18 100644 --- a/lib/clickhouse-activerecord/schema_dumper.rb +++ b/lib/clickhouse-activerecord/schema_dumper.rb @@ -152,6 +152,12 @@ def prepare_column_options(column) spec = {} spec[:unsigned] = schema_unsigned(column) spec[:array] = schema_array(column) + + if column.type == :map + spec[:key_type] = "\"#{column.key_type}\"" + spec[:value_type] = "\"#{column.value_type}\"" + end + spec.merge(super).compact end end From 136ca092fb18e92f9f05100ffc9e8dd1a81633f8 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Mon, 27 Nov 2023 11:17:06 +0100 Subject: [PATCH 4/8] fix: Map type serializer --- lib/active_record/connection_adapters/clickhouse/oid/map.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/active_record/connection_adapters/clickhouse/oid/map.rb b/lib/active_record/connection_adapters/clickhouse/oid/map.rb index 287db1a6..a2d49358 100644 --- a/lib/active_record/connection_adapters/clickhouse/oid/map.rb +++ b/lib/active_record/connection_adapters/clickhouse/oid/map.rb @@ -8,7 +8,7 @@ class Map < Type::Value # :nodoc: attr_reader :key_type, :value_type def initialize(sql_type) - types = sql_type.match(/Map\((.+)\,\s?(.+)\)/).captures + types = sql_type.match(/Map\((.+),\s?(.+)\)/).captures @key_type = cast_type(types.first) @value_type = cast_type(types.last) @@ -28,11 +28,10 @@ def deserialize(value) JSON.parse(value) end - def serialize(value) return '{}' if value.nil? - "#{value.map { |key, value| "'#{key}': '#{value}'" }.join(' ')}" + "{#{value.map { |key, value| "'#{key}': '#{value}'" }.join(' ')}}" end private From 6502be10678b4b68cb3a82b9686118536b0a9082 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Mon, 27 Nov 2023 11:26:55 +0100 Subject: [PATCH 5/8] fix: Map type serializer --- lib/active_record/connection_adapters/clickhouse/oid/map.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/active_record/connection_adapters/clickhouse/oid/map.rb b/lib/active_record/connection_adapters/clickhouse/oid/map.rb index a2d49358..6c6a1879 100644 --- a/lib/active_record/connection_adapters/clickhouse/oid/map.rb +++ b/lib/active_record/connection_adapters/clickhouse/oid/map.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'yaml' + module ActiveRecord module ConnectionAdapters module Clickhouse @@ -25,13 +27,13 @@ def cast(value) def deserialize(value) return value if value.is_a?(Hash) - JSON.parse(value) + YAML.safe_load(value) end def serialize(value) return '{}' if value.nil? - "{#{value.map { |key, value| "'#{key}': '#{value}'" }.join(' ')}}" + "{#{value.map { |k, v| "'#{k}': '#{v}'" }.join(', ')}}" end private From d095013674cf3eba1458e28f178ea6e48e1a7dbb Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Thu, 13 Jun 2024 14:46:34 +0200 Subject: [PATCH 6/8] Ignore vscode subfolder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 42b720a5..6c837f74 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ /spec/reports/ /tmp/ *.gem +.vscode/ ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm From 5e82649289e0b90a43e85e3b0a8c63f617b9ba7a Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Mon, 8 Jul 2024 16:53:11 +0200 Subject: [PATCH 7/8] misc: Add support of array in map --- .../connection_adapters/clickhouse/oid/map.rb | 22 ++++++++++++++++++- .../connection_adapters/clickhouse_adapter.rb | 4 ++++ lib/clickhouse-activerecord/schema_dumper.rb | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/active_record/connection_adapters/clickhouse/oid/map.rb b/lib/active_record/connection_adapters/clickhouse/oid/map.rb index 6c6a1879..64d55760 100644 --- a/lib/active_record/connection_adapters/clickhouse/oid/map.rb +++ b/lib/active_record/connection_adapters/clickhouse/oid/map.rb @@ -33,7 +33,8 @@ def deserialize(value) def serialize(value) return '{}' if value.nil? - "{#{value.map { |k, v| "'#{k}': '#{v}'" }.join(', ')}}" + res = value.map { |k, v| "#{quote(k, key_type)}: #{quote(v, value_type)}" }.join(', ') + "{#{res}}" end private @@ -48,10 +49,29 @@ def cast_type(type) :datetime when /Date/ :date + when /Array\(.*\)/ + type else :string end end + + def quote(value, type) + case cast_type(type) + when :string + "'#{value}'" + when :integer + value + when :datetime, :date + "'#{value.iso8601}'" + when /Array\(.*\)/ + byebug + sub_type = type.match(/Array\((.+)\)/).captures.first + "[#{value.map { |v| quote(v, sub_type) }.join(', ')}]" + else + value + end + end end end end diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index 8c98cf2c..8e036726 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -112,6 +112,8 @@ def cast_type(type) :datetime when /Date/ :date + when /Array/ + type else :string end @@ -267,6 +269,8 @@ def quote(value) case value when Array '[' + value.map { |v| quote(v) }.join(', ') + ']' + when Hash + '{' + value.map { |k, v| "#{quote(k)}: #{quote(v)}" }.join(', ') + '}' else super end diff --git a/lib/clickhouse-activerecord/schema_dumper.rb b/lib/clickhouse-activerecord/schema_dumper.rb index cbd5a2e6..91ff6039 100644 --- a/lib/clickhouse-activerecord/schema_dumper.rb +++ b/lib/clickhouse-activerecord/schema_dumper.rb @@ -175,6 +175,7 @@ def prepare_column_options(column) if column.type == :map spec[:key_type] = "\"#{column.key_type}\"" spec[:value_type] = "\"#{column.value_type}\"" + spec[:array] = nil end spec.merge(super).compact From 3c4b369a34fb5d27f2cf5d30c87b96ce5a940fc3 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 9 Jul 2024 15:37:14 +0200 Subject: [PATCH 8/8] misc: Remove byebug --- lib/active_record/connection_adapters/clickhouse/oid/map.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/active_record/connection_adapters/clickhouse/oid/map.rb b/lib/active_record/connection_adapters/clickhouse/oid/map.rb index 64d55760..a4629d11 100644 --- a/lib/active_record/connection_adapters/clickhouse/oid/map.rb +++ b/lib/active_record/connection_adapters/clickhouse/oid/map.rb @@ -65,7 +65,6 @@ def quote(value, type) when :datetime, :date "'#{value.iso8601}'" when /Array\(.*\)/ - byebug sub_type = type.match(/Array\((.+)\)/).captures.first "[#{value.map { |v| quote(v, sub_type) }.join(', ')}]" else