From 45b5eaab796bee1c75b82c19610e5923122bfbf1 Mon Sep 17 00:00:00 2001 From: Patrick Pichler Date: Fri, 2 Feb 2024 07:33:26 +0100 Subject: [PATCH] Add clickhouse database support This adds initial support for the clickhouse database system. Keywords are currently based upon postgres, as the clickhouse docs do not provide a good overview of all allowed keywords. fix #53 --- dialect/clickhouse.go | 455 +++++++++++++++++++++++++++ dialect/keyword.go | 5 + go.mod | 14 + go.sum | 60 ++++ internal/database/clickhouse.go | 381 ++++++++++++++++++++++ internal/database/clickhouse_test.go | 58 ++++ internal/database/config.go | 28 +- internal/database/mssql.go | 4 +- internal/database/mysql.go | 3 +- internal/database/postgresql.go | 9 +- internal/handler/execute_command.go | 2 + 11 files changed, 1010 insertions(+), 9 deletions(-) create mode 100644 dialect/clickhouse.go create mode 100644 internal/database/clickhouse.go create mode 100644 internal/database/clickhouse_test.go diff --git a/dialect/clickhouse.go b/dialect/clickhouse.go new file mode 100644 index 0000000..e0e0067 --- /dev/null +++ b/dialect/clickhouse.go @@ -0,0 +1,455 @@ +package dialect + +// TODO(patrick.pichler): figure out real keywords as those are copied over from postgres +var clickhouseKeywords = []string{ + "ABORT", + "ABSOLUTE", + "ACCESS", + "ACTION", + "ADD", + "ADMIN", + "AFTER", + "AGGREGATE", + "ALL", + "ALSO", + "ALTER", + "ALWAYS", + "ANALYSE", + "ANALYZE", + "AND", + "ANY", + "ARRAY", + "AS", + "ASC", + "ASSERTION", + "ASSIGNMENT", + "ASYMMETRIC", + "AT", + "ATTACH", + "ATTRIBUTE", + "AUTHORIZATION", + "BACKWARD", + "BEFORE", + "BEGIN", + "BETWEEN", + "BIGINT", + "BINARY", + "BIT", + "BOOLEAN", + "BOTH", + "BY", + "CACHE", + "CALL", + "CALLED", + "CASCADE", + "CASCADED", + "CASE", + "CAST", + "CATALOG", + "CHAIN", + "CHAR", + "CHARACTER", + "CHARACTERISTICS", + "CHECK", + "CHECKPOINT", + "CLASS", + "CLOSE", + "CLUSTER", + "COALESCE", + "COLLATE", + "COLLATION", + "COLUMN", + "COLUMNS", + "COMMENT", + "COMMENTS", + "COMMIT", + "COMMITTED", + "CONCURRENTLY", + "CONFIGURATION", + "CONFLICT", + "CONNECTION", + "CONSTRAINT", + "CONSTRAINTS", + "CONTENT", + "CONTINUE", + "CONVERSION", + "COPY", + "COST", + "CREATE", + "CROSS", + "CSV", + "CUBE", + "CURRENT", + "CURRENT_CATALOG", + "CURRENT_DATE", + "CURRENT_ROLE", + "CURRENT_SCHEMA", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_USER", + "CURSOR", + "CYCLE", + "DATA", + "DATABASE", + "DAY", + "DEALLOCATE", + "DEC", + "DECIMAL", + "DECLARE", + "DEFAULT", + "DEFAULTS", + "DEFERRABLE", + "DEFERRED", + "DEFINER", + "DELETE", + "DELIMITER", + "DELIMITERS", + "DEPENDS", + "DESC", + "DETACH", + "DICTIONARY", + "DISABLE", + "DISCARD", + "DISTINCT", + "DO", + "DOCUMENT", + "DOMAIN", + "DOUBLE", + "DROP", + "EACH", + "ELSE", + "ENABLE", + "ENCODING", + "ENCRYPTED", + "END", + "ENUM", + "ESCAPE", + "EVENT", + "EXCEPT", + "EXCLUDE", + "EXCLUDING", + "EXCLUSIVE", + "EXECUTE", + "EXISTS", + "EXPLAIN", + "EXPRESSION", + "EXTENSION", + "EXTERNAL", + "EXTRACT", + "FALSE", + "FAMILY", + "FETCH", + "FILTER", + "FIRST", + "FLOAT", + "FOLLOWING", + "FOR", + "FORCE", + "FOREIGN", + "FORWARD", + "FREEZE", + "FROM", + "FULL", + "FUNCTION", + "FUNCTIONS", + "GENERATED", + "GLOBAL", + "GRANT", + "GRANTED", + "GREATEST", + "GROUP", + "GROUPING", + "GROUPS", + "HANDLER", + "HAVING", + "HEADER", + "HOLD", + "HOUR", + "IDENTITY", + "IF", + "ILIKE", + "IMMEDIATE", + "IMMUTABLE", + "IMPLICIT", + "IMPORT", + "IN", + "INCLUDE", + "INCLUDING", + "INCREMENT", + "INDEX", + "INDEXES", + "INHERIT", + "INHERITS", + "INITIALLY", + "INLINE", + "INNER", + "INOUT", + "INPUT", + "INSENSITIVE", + "INSERT", + "INSTEAD", + "INT", + "INTEGER", + "INTERSECT", + "INTERVAL", + "INTO", + "INVOKER", + "IS", + "ISNULL", + "ISOLATION", + "JOIN", + "KEY", + "LABEL", + "LANGUAGE", + "LARGE", + "LAST", + "LATERAL", + "LEADING", + "LEAKPROOF", + "LEAST", + "LEFT", + "LEVEL", + "LIKE", + "LIMIT", + "LISTEN", + "LOAD", + "LOCAL", + "LOCALTIME", + "LOCALTIMESTAMP", + "LOCATION", + "LOCK", + "LOCKED", + "LOGGED", + "MAPPING", + "MATCH", + "MATERIALIZED", + "MAXVALUE", + "METHOD", + "MINUTE", + "MINVALUE", + "MODE", + "MONTH", + "MOVE", + "NAME", + "NAMES", + "NATIONAL", + "NATURAL", + "NCHAR", + "NEW", + "NEXT", + "NFC", + "NFD", + "NFKC", + "NFKD", + "NO", + "NONE", + "NORMALIZE", + "NORMALIZED", + "NOT", + "NOTHING", + "NOTIFY", + "NOTNULL", + "NOWAIT", + "NULL", + "NULLIF", + "NULLS", + "NUMERIC", + "OBJECT", + "OF", + "OFF", + "OFFSET", + "OIDS", + "OLD", + "ON", + "ONLY", + "OPERATOR", + "OPTION", + "OPTIONS", + "OR", + "ORDER", + "ORDINALITY", + "OTHERS", + "OUT", + "OUTER", + "OVER", + "OVERLAPS", + "OVERLAY", + "OVERRIDING", + "OWNED", + "OWNER", + "PARALLEL", + "PARSER", + "PARTIAL", + "PARTITION", + "PASSING", + "PASSWORD", + "PLACING", + "PLANS", + "POLICY", + "POSITION", + "PRECEDING", + "PRECISION", + "PREPARE", + "PREPARED", + "PRESERVE", + "PRIMARY", + "PRIOR", + "PRIVILEGES", + "PROCEDURAL", + "PROCEDURE", + "PROCEDURES", + "PROGRAM", + "PUBLICATION", + "QUOTE", + "RANGE", + "READ", + "REAL", + "REASSIGN", + "RECHECK", + "RECURSIVE", + "REF", + "REFERENCES", + "REFERENCING", + "REFRESH", + "REINDEX", + "RELATIVE", + "RELEASE", + "RENAME", + "REPEATABLE", + "REPLACE", + "REPLICA", + "RESET", + "RESTART", + "RESTRICT", + "RETURNING", + "RETURNS", + "REVOKE", + "RIGHT", + "ROLE", + "ROLLBACK", + "ROLLUP", + "ROUTINE", + "ROUTINES", + "ROW", + "ROWS", + "RULE", + "SAVEPOINT", + "SCHEMA", + "SCHEMAS", + "SCROLL", + "SEARCH", + "SECOND", + "SECURITY", + "SELECT", + "SEQUENCE", + "SEQUENCES", + "SERIALIZABLE", + "SERVER", + "SESSION", + "SESSION_USER", + "SET", + "SETOF", + "SETS", + "SHARE", + "SHOW", + "SIMILAR", + "SIMPLE", + "SKIP", + "SMALLINT", + "SNAPSHOT", + "SOME", + "SQL", + "STABLE", + "STANDALONE", + "START", + "STATEMENT", + "STATISTICS", + "STDIN", + "STDOUT", + "STORAGE", + "STORED", + "STRICT", + "STRIP", + "SUBSCRIPTION", + "SUBSTRING", + "SUPPORT", + "SYMMETRIC", + "SYSID", + "SYSTEM", + "TABLE", + "TABLES", + "TABLESAMPLE", + "TABLESPACE", + "TEMP", + "TEMPLATE", + "TEMPORARY", + "TEXT", + "THEN", + "TIES", + "TIME", + "TIMESTAMP", + "TO", + "TRAILING", + "TRANSACTION", + "TRANSFORM", + "TREAT", + "TRIGGER", + "TRIM", + "TRUE", + "TRUNCATE", + "TRUSTED", + "TYPE", + "TYPES", + "UESCAPE", + "UNBOUNDED", + "UNCOMMITTED", + "UNENCRYPTED", + "UNION", + "UNIQUE", + "UNKNOWN", + "UNLISTEN", + "UNLOGGED", + "UNTIL", + "UPDATE", + "USER", + "USING", + "VACUUM", + "VALID", + "VALIDATE", + "VALIDATOR", + "VALUE", + "VALUES", + "VARCHAR", + "VARIADIC", + "VARYING", + "VERBOSE", + "VERSION", + "VIEW", + "VIEWS", + "VOLATILE", + "WHEN", + "WHERE", + "WHITESPACE", + "WINDOW", + "WITH", + "WITHIN", + "WITHOUT", + "WORK", + "WRAPPER", + "WRITE", + "XML", + "XMLATTRIBUTES", + "XMLCONCAT", + "XMLELEMENT", + "XMLEXISTS", + "XMLFOREST", + "XMLNAMESPACES", + "XMLPARSE", + "XMLPI", + "XMLROOT", + "XMLSERIALIZE", + "XMLTABLE", + "YEAR", + "YES", + "ZONE", +} diff --git a/dialect/keyword.go b/dialect/keyword.go index b5cc38a..17fb0b2 100644 --- a/dialect/keyword.go +++ b/dialect/keyword.go @@ -389,6 +389,7 @@ const ( DatabaseDriverOracle DatabaseDriver = "oracle" DatabaseDriverH2 DatabaseDriver = "h2" DatabaseDriverVertica DatabaseDriver = "vertica" + DatabaseDriverClickhouse DatabaseDriver = "clickhouse" ) func DataBaseKeywords(driver DatabaseDriver) []string { @@ -413,6 +414,8 @@ func DataBaseKeywords(driver DatabaseDriver) []string { return h2Keywords case DatabaseDriverVertica: return verticaKeywords + case DatabaseDriverClickhouse: + return clickhouseKeywords default: return sqliteKeywords } @@ -440,6 +443,8 @@ func DataBaseFunctions(driver DatabaseDriver) []string { return []string{} case DatabaseDriverVertica: return verticaReservedWords + case DatabaseDriverClickhouse: + return []string{} default: return []string{} } diff --git a/go.mod b/go.mod index 73df020..23b2ec0 100644 --- a/go.mod +++ b/go.mod @@ -24,13 +24,19 @@ require ( ) require ( + github.com/ClickHouse/ch-go v0.58.2 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.17.1 // indirect + github.com/andybalholm/brotli v1.0.6 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/elastic/go-sysinfo v1.11.2 // indirect github.com/elastic/go-windows v1.0.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.6.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/godror/knownpb v0.1.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.1 // indirect github.com/jackc/pgio v1.0.0 // indirect @@ -39,18 +45,26 @@ require ( github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/paulmach/orb v0.10.0 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 61700bc..c6cfdbc 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,23 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbL github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= +github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.17.1 h1:ZCmAYWpu75IyEi7+Yrs/uaAjiCGY5wfW5kXo64exkX4= +github.com/ClickHouse/clickhouse-go/v2 v2.17.1/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ= github.com/CodinGame/h2go v0.6.1 h1:xCPVmnhNhtiPQK6gka4O8SmmiyEaQBrO70qH4keq79g= github.com/CodinGame/h2go v0.6.1/go.mod h1:c41riTYIVgAtEbNWl0k5/HNeOw6xLd15WFV1opNteeI= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/UNO-SOFT/zlog v0.8.1 h1:TEFkGJHtUfTRgMkLZiAjLSHALjwSBdw6/zByMC5GJt4= github.com/UNO-SOFT/zlog v0.8.1/go.mod h1:yqFOjn3OhvJ4j7ArJqQNA+9V+u6t9zSAyIZdWdMweWc= +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -37,6 +46,10 @@ github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= +github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= @@ -59,11 +72,18 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -121,7 +141,11 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -148,6 +172,7 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/oklog/ulid/v2 v2.0.2 h1:r4fFzBm+bv0wNKNh5eXTwU7i85y5x+uwkxCUTNVQqLc= github.com/oklog/ulid/v2 v2.0.2/go.mod h1:mtBL0Qe/0HAx6/a4Z30qxVIAL1eQDweXq5lxOEiwQ68= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -156,6 +181,11 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= +github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -175,9 +205,13 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -193,19 +227,32 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/urfave/cli/v2 v2.27.0 h1:uNs1K8JwTFL84X68j5Fjny6hfANh9nTlJ6dRtZAFAHY= github.com/urfave/cli/v2 v2.27.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/vertica/vertica-sql-go v1.3.3 h1:fL+FKEAEy5ONmsvya2WH5T8bhkvY27y/Ik3ReR2T+Qw= github.com/vertica/vertica-sql-go v1.3.3/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -237,6 +284,8 @@ golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGb golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -244,6 +293,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -253,6 +304,9 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -266,6 +320,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -299,6 +354,8 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= @@ -306,7 +363,10 @@ golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/database/clickhouse.go b/internal/database/clickhouse.go new file mode 100644 index 0000000..0f84707 --- /dev/null +++ b/internal/database/clickhouse.go @@ -0,0 +1,381 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "log" + "net" + "net/url" + "strconv" + "strings" + + clickhouse "github.com/ClickHouse/clickhouse-go/v2" + "github.com/sqls-server/sqls/dialect" + "golang.org/x/crypto/ssh" +) + +func init() { + RegisterOpen("clickhouse", clickhouseOpen) + RegisterFactory("clickhouse", NewClickhouseRepository) +} + +func clickhouseOpen(dbConnCfg *DBConfig) (*DBConnection, error) { + var ( + conn *sql.DB + sshConn *ssh.Client + ) + + dsn, err := genClickhouseDsn(dbConnCfg) + + if err != nil { + return nil, err + } + + if dbConnCfg.SSHCfg != nil { + dbConn, dbSSHConn, err := openClickhouseViaSSH(dsn, dbConnCfg.SSHCfg) + if err != nil { + return nil, err + } + conn = dbConn + sshConn = dbSSHConn + } else { + dbConn, err := sql.Open("clickhouse", dsn) + if err != nil { + return nil, err + } + conn = dbConn + } + + if err = conn.Ping(); err != nil { + return nil, err + } + + conn.SetMaxIdleConns(DefaultMaxIdleConns) + conn.SetMaxOpenConns(DefaultMaxOpenConns) + + return &DBConnection{ + Conn: conn, + SSHConn: sshConn, + }, nil +} + +func openClickhouseViaSSH(dsn string, sshCfg *SSHConfig) (*sql.DB, *ssh.Client, error) { + sshConfig, err := sshCfg.ClientConfig() + if err != nil { + return nil, nil, err + } + sshConn, err := ssh.Dial("tcp", sshCfg.Endpoint(), sshConfig) + if err != nil { + return nil, nil, fmt.Errorf("cannot ssh dial, %w", err) + } + + conf, err := clickhouse.ParseDSN(dsn) + if err != nil { + return nil, nil, err + } + conf.DialContext = func(ctx context.Context, addr string) (net.Conn, error) { + return sshConn.DialContext(ctx, "tcp", addr) + } + + conn := clickhouse.OpenDB(conf) + + return conn, sshConn, nil +} + +func genClickhouseDsn(dbConfig *DBConfig) (string, error) { + if dbConfig.DataSourceName != "" { + return dbConfig.DataSourceName, nil + } + + u := url.URL{} + + switch dbConfig.Proto { + case ProtoTCP: + u.Scheme = "clickhouse" + + case ProtoHTTP: + u.Scheme = "http" + + case ProtoUDP, ProtoUnix: + return "", fmt.Errorf("unsupported protocol %s", dbConfig.Proto) + } + + if dbConfig.Passwd == "" { + u.User = url.User(dbConfig.User) + } else { + u.User = url.UserPassword(dbConfig.User, dbConfig.Passwd) + } + + u.Host = fmt.Sprintf("%s:%d", dbConfig.Host, dbConfig.Port) + + if dbConfig.DBName != "" { + u.Path = dbConfig.DBName + } + + values := u.Query() + + for k, v := range dbConfig.Params { + values.Set(k, v) + } + + u.RawQuery = values.Encode() + + return u.String(), nil +} + +func genClickhouseConfig(dbConfig *DBConfig) (*clickhouse.Options, error) { + if dbConfig.DataSourceName != "" { + return clickhouse.ParseDSN(dbConfig.DataSourceName) + } + + cfg := &clickhouse.Options{} + + cfg.Auth.Username = dbConfig.User + cfg.Auth.Password = dbConfig.Passwd + cfg.Auth.Database = dbConfig.DBName + + switch dbConfig.Proto { + case ProtoTCP, ProtoHTTP: + host, port := dbConfig.Host, dbConfig.Port + if host == "" { + host = "127.0.0.1" + } + if port == 0 { + port = 8443 + } + cfg.Addr = append(cfg.Addr, host+":"+strconv.Itoa(port)) + + if dbConfig.Proto == ProtoTCP { + cfg.Protocol = clickhouse.Native + } else { + cfg.Protocol = clickhouse.HTTP + } + case ProtoUnix, ProtoUDP: + default: + return nil, fmt.Errorf("default addr for network %s unknown", dbConfig.Proto) + } + + cfg.Settings = parseClickhouseSettings(dbConfig.Params) + + return nil, nil +} + +func parseClickhouseSettings(params map[string]string) clickhouse.Settings { + result := clickhouse.Settings{} + + for k, v := range params { + switch p := strings.ToLower(v); p { + case "true": + result[k] = int(1) + case "false": + result[k] = int(0) + default: + if n, err := strconv.Atoi(p); err == nil { + result[k] = n + } else { + result[k] = p + } + } + } + + return result +} + +type clickhouseSQLDBRepository struct { + Conn *sql.DB +} + +func NewClickhouseRepository(conn *sql.DB) DBRepository { + return &clickhouseSQLDBRepository{Conn: conn} +} + +func (db *clickhouseSQLDBRepository) CurrentDatabase(ctx context.Context) (string, error) { + row := db.Conn.QueryRowContext(ctx, "SELECT currentDatabase()") + var database string + if err := row.Scan(&database); err != nil { + return "", err + } + return database, nil +} + +func (db *clickhouseSQLDBRepository) CurrentSchema(ctx context.Context) (string, error) { + return db.CurrentDatabase(ctx) +} + +func (db *clickhouseSQLDBRepository) Databases(ctx context.Context) ([]string, error) { + rows, err := db.Conn.QueryContext(ctx, "SHOW databases") + if err != nil { + return nil, err + } + defer rows.Close() + databases := []string{} + for rows.Next() { + var database string + if err := rows.Scan(&database); err != nil { + return nil, err + } + databases = append(databases, database) + } + return databases, nil +} + +func (db *clickhouseSQLDBRepository) DescribeDatabaseTable(ctx context.Context) ([]*ColumnDesc, error) { + rows, err := db.Conn.QueryContext( + ctx, + ` +SELECT c.table_schema, + c.table_name, + c.column_name, + c.column_type, + CASE c.is_nullable + WHEN 1 THEN 'YES' + ELSE 'NO' + END, + CASE cu.constraint_name + WHEN 'PRIMARY' THEN 'YES' + ELSE 'NO' + END, + c.column_default, + '' +FROM information_schema.columns c + LEFT JOIN (SELECT icu.table_schema, + icu.table_name, + icu.column_name, + icu.constraint_name + FROM information_schema.key_column_usage icu + WHERE icu.constraint_name = 'PRIMARY') cu using ( + table_schema, table_name, column_name) +WHERE ( c.table_schema = currentDatabase() + OR c.table_schema = '' ) + AND c.table_name NOT LIKE '%inner%' +`) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + tableInfos := []*ColumnDesc{} + for rows.Next() { + var tableInfo ColumnDesc + err := rows.Scan( + &tableInfo.Schema, + &tableInfo.Table, + &tableInfo.Name, + &tableInfo.Type, + &tableInfo.Null, + &tableInfo.Key, + &tableInfo.Default, + &tableInfo.Extra, + ) + if err != nil { + return nil, err + } + tableInfos = append(tableInfos, &tableInfo) + } + return tableInfos, nil +} + +func (db *clickhouseSQLDBRepository) DescribeDatabaseTableBySchema(ctx context.Context, schemaName string) ([]*ColumnDesc, error) { + rows, err := db.Conn.QueryContext( + ctx, + ` +SELECT c.table_schema, + c.table_name, + c.column_name, + c.column_type, + CASE c.is_nullable + WHEN 1 THEN 'YES' + ELSE 'NO' + END, + CASE cu.constraint_name + WHEN 'PRIMARY' THEN 'YES' + ELSE 'NO' + END, + c.column_default, + '' +FROM information_schema.columns c + LEFT JOIN (SELECT icu.table_schema, + icu.table_name, + icu.column_name, + icu.constraint_name + FROM information_schema.key_column_usage icu + WHERE icu.constraint_name = 'PRIMARY') cu using ( + table_schema, table_name, column_name) +WHERE ( c.table_schema = ? + OR c.table_schema = '' ) + AND c.table_name NOT LIKE '%inner%' +`, schemaName) + if err != nil { + log.Println("schema", schemaName, err.Error()) + return nil, err + } + defer rows.Close() + tableInfos := []*ColumnDesc{} + for rows.Next() { + var tableInfo ColumnDesc + err := rows.Scan( + &tableInfo.Schema, + &tableInfo.Table, + &tableInfo.Name, + &tableInfo.Type, + &tableInfo.Null, + &tableInfo.Key, + &tableInfo.Default, + &tableInfo.Extra, + ) + if err != nil { + return nil, err + } + tableInfos = append(tableInfos, &tableInfo) + } + return tableInfos, nil +} + +func (*clickhouseSQLDBRepository) DescribeForeignKeysBySchema(ctx context.Context, schemaName string) ([]*ForeignKey, error) { + // clickhouse doesn't support foreign keys + return nil, nil +} + +func (*clickhouseSQLDBRepository) Driver() dialect.DatabaseDriver { + return dialect.DatabaseDriverClickhouse +} + +func (db *clickhouseSQLDBRepository) Exec(ctx context.Context, query string) (sql.Result, error) { + return db.Conn.ExecContext(ctx, query) +} + +func (db *clickhouseSQLDBRepository) Query(ctx context.Context, query string) (*sql.Rows, error) { + return db.Conn.QueryContext(ctx, query) +} + +func (db *clickhouseSQLDBRepository) SchemaTables(ctx context.Context) (map[string][]string, error) { + rows, err := db.Conn.QueryContext( + ctx, + ` + SELECT table_schema, table_name + FROM information_schema.tables + ORDER BY table_schema, table_name + `) + if err != nil { + return nil, err + } + defer rows.Close() + databaseTables := map[string][]string{} + for rows.Next() { + var schema, table string + if err := rows.Scan(&schema, &table); err != nil { + return nil, err + } + + if arr, ok := databaseTables[schema]; ok { + databaseTables[schema] = append(arr, table) + } else { + databaseTables[schema] = []string{table} + } + } + return databaseTables, nil +} + +func (db *clickhouseSQLDBRepository) Schemas(ctx context.Context) ([]string, error) { + return db.Databases(ctx) +} diff --git a/internal/database/clickhouse_test.go b/internal/database/clickhouse_test.go new file mode 100644 index 0000000..320d8c7 --- /dev/null +++ b/internal/database/clickhouse_test.go @@ -0,0 +1,58 @@ +package database + +import "testing" + +func TestGenClickhouseDsn(t *testing.T) { + type testCase struct { + name string + connCfg *DBConfig + want string + wantErr bool + } + + tests := []testCase{ + { + name: "use datasource name", + connCfg: &DBConfig{ + DataSourceName: "clickhouse://user:pwd@localhost:9001", + Driver: "clickhouse", + }, + want: "clickhouse://user:pwd@localhost:9001", + wantErr: false, + }, + { + name: "use config properties", + connCfg: &DBConfig{ + Alias: "", + DataSourceName: "", + Driver: "clickhouse", + Proto: "tcp", + User: "test", + Passwd: "secure", + Host: "localhost", + Port: 9001, + Path: "", + DBName: "default", + Params: map[string]string{ + "dial_timeout": "200ms", + }, + }, + want: "clickhouse://test:secure@localhost:9001/default?dial_timeout=200ms", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := genClickhouseDsn(tt.connCfg) + + if (err != nil) != tt.wantErr { + t.Errorf("genClickhouseDsn() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + + } +} diff --git a/internal/database/config.go b/internal/database/config.go index 91b1623..a95306e 100644 --- a/internal/database/config.go +++ b/internal/database/config.go @@ -15,6 +15,7 @@ const ( ProtoTCP Proto = "tcp" ProtoUDP Proto = "udp" ProtoUnix Proto = "unix" + ProtoHTTP Proto = "http" ) type DBConfig struct { @@ -54,7 +55,7 @@ func (c *DBConfig) Validate() error { return errors.New("required: connections[].user") } switch c.Proto { - case ProtoTCP, ProtoUDP: + case ProtoTCP, ProtoUDP, ProtoHTTP: if c.Host == "" { return errors.New("required: connections[].host") } @@ -87,7 +88,7 @@ func (c *DBConfig) Validate() error { if c.Host == "" { return errors.New("required: connections[].host") } - case ProtoUDP, ProtoUnix: + case ProtoUDP, ProtoUnix, ProtoHTTP: default: return errors.New("invalid: connections[].proto") } @@ -113,6 +114,29 @@ func (c *DBConfig) Validate() error { return errors.New("required: connections[].DBName") } } + case dialect.DatabaseDriverClickhouse: + if c.DataSourceName == "" && c.Proto == "" { + return errors.New("required: connections[].dataSourceName or connections[].proto") + } + + if c.DataSourceName == "" && c.Proto != "" { + if c.User == "" { + return errors.New("required: connections[].user") + } + switch c.Proto { + case ProtoTCP, ProtoHTTP: + if c.Host == "" { + return errors.New("required: connections[].host") + } + case ProtoUDP, ProtoUnix: + default: + return errors.New("invalid: connections[].proto") + } + if c.SSHCfg != nil { + return c.SSHCfg.Validate() + } + } + default: return errors.New("invalid: connections[].driver") } diff --git a/internal/database/mssql.go b/internal/database/mssql.go index 102357e..60611f2 100644 --- a/internal/database/mssql.go +++ b/internal/database/mssql.go @@ -304,7 +304,7 @@ func (db *MssqlDBRepository) DescribeForeignKeysBySchema(ctx context.Context, sc rows, err := db.Conn.QueryContext( ctx, ` - SELECT fk.name, + SELECT fk.name, src_tbl.name, src_col.name, dst_tbl.name, @@ -360,7 +360,7 @@ func genMssqlConfig(connCfg *DBConfig) (string, error) { } q.Set("server", host) q.Set("port", strconv.Itoa(port)) - case ProtoUDP, ProtoUnix: + case ProtoUDP, ProtoUnix, ProtoHTTP: default: return "", fmt.Errorf("default addr for network %s unknown", connCfg.Proto) } diff --git a/internal/database/mysql.go b/internal/database/mysql.go index f2c799b..c8fb57b 100644 --- a/internal/database/mysql.go +++ b/internal/database/mysql.go @@ -116,6 +116,7 @@ func genMysqlConfig(connCfg *DBConfig) (*mysql.Config, error) { } cfg.Addr = connCfg.Path cfg.Net = string(connCfg.Proto) + case ProtoHTTP: default: return nil, fmt.Errorf("default addr for network %s unknown", connCfg.Proto) } @@ -176,7 +177,7 @@ func (db *MySQLDBRepository) SchemaTables(ctx context.Context) (map[string][]str rows, err := db.Conn.QueryContext( ctx, ` - SELECT + SELECT TABLE_SCHEMA, TABLE_NAME FROM diff --git a/internal/database/postgresql.go b/internal/database/postgresql.go index 611d6cd..2d08c2f 100644 --- a/internal/database/postgresql.go +++ b/internal/database/postgresql.go @@ -196,12 +196,12 @@ func (db *PostgreSQLDBRepository) Tables(ctx context.Context) ([]string, error) ctx, ` SELECT - table_name + table_name FROM - information_schema.tables + information_schema.tables WHERE - table_type = 'BASE TABLE' - AND table_schema NOT IN ('pg_catalog', 'information_schema') + table_type = 'BASE TABLE' + AND table_schema NOT IN ('pg_catalog', 'information_schema') ORDER BY table_name `) @@ -414,6 +414,7 @@ func genPostgresConfig(connCfg *DBConfig) (string, error) { q.Set("port", strconv.Itoa(port)) case ProtoUnix: q.Set("host", connCfg.Path) + case ProtoHTTP: default: return "", fmt.Errorf("default addr for network %s unknown", connCfg.Proto) } diff --git a/internal/handler/execute_command.go b/internal/handler/execute_command.go index 58f457d..7ba0d9c 100644 --- a/internal/handler/execute_command.go +++ b/internal/handler/execute_command.go @@ -323,6 +323,8 @@ func (s *Server) showConnections(ctx context.Context, params lsp.ExecuteCommandP desc = fmt.Sprintf("udp(%s:%d)/%s", conn.Host, conn.Port, conn.DBName) case database.ProtoUnix: desc = fmt.Sprintf("unix(%s)/%s", conn.Path, conn.DBName) + case database.ProtoHTTP: + desc = fmt.Sprintf("http(%s:%d)/%s", conn.Host, conn.Port, conn.DBName) } } res := fmt.Sprintf("%d %s %s %s", i+1, conn.Driver, conn.Alias, desc)