From 4ed7a997e441cd588673c96de8a73c2bfb16098e Mon Sep 17 00:00:00 2001 From: big-vi Date: Tue, 11 Jul 2023 09:21:29 +0530 Subject: [PATCH 1/4] avro schema ui and validation --- db/db.go | 2 +- server/memphis_handlers_schemas.go | 20 +++++++++---- ui_src/package.json | 2 ++ .../schema/components/createSchema/index.js | 30 ++++++++++++++----- .../schema/components/schemaDetails/index.js | 20 +++++++++++-- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/db/db.go b/db/db.go index 06e5daf2b..95bcb98ea 100644 --- a/db/db.go +++ b/db/db.go @@ -248,7 +248,7 @@ func createTables(MetadataDbClient MetadataStorage) error { END $$;` schemasTable := ` - CREATE TYPE enum_type AS ENUM ('json', 'graphql', 'protobuf'); + CREATE TYPE enum_type AS ENUM ('json', 'graphql', 'protobuf', 'avro'); CREATE TABLE IF NOT EXISTS schemas( id SERIAL NOT NULL, name VARCHAR NOT NULL, diff --git a/server/memphis_handlers_schemas.go b/server/memphis_handlers_schemas.go index ca2cdae6d..118e03e78 100644 --- a/server/memphis_handlers_schemas.go +++ b/server/memphis_handlers_schemas.go @@ -30,6 +30,7 @@ import ( "github.com/graph-gophers/graphql-go" "github.com/jhump/protoreflect/desc/protoparse" "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/hamba/avro/v2" ) type SchemasHandler struct{ S *Server } @@ -75,6 +76,14 @@ func validateGraphqlSchemaContent(schemaContent string) error { return nil } +func validateAvroSchemaContent(schemaContent string) error { + _, err := avro.Parse(schemaContent) + if err != nil { + return err + } + return nil +} + func generateProtobufDescriptor(schemaName string, schemaVersionNum int, schemaContent string) ([]byte, error) { filename := fmt.Sprintf("%v_%v.proto", schemaName, schemaVersionNum) descFilename := fmt.Sprintf("%v_%v_desc", schemaName, schemaVersionNum) @@ -113,13 +122,9 @@ func validateSchemaName(schemaName string) error { func validateSchemaType(schemaType string) error { invalidTypeErrStr := "unsupported schema type" invalidTypeErr := errors.New(invalidTypeErrStr) - invalidSupportTypeErrStr := "avro is not supported at this time" - invalidSupportTypeErr := errors.New(invalidSupportTypeErrStr) - if schemaType == "protobuf" || schemaType == "json" || schemaType == "graphql" { + if schemaType == "protobuf" || schemaType == "json" || schemaType == "graphql" || schemaType == "avro" { return nil - } else if schemaType == "avro" { - return invalidSupportTypeErr } else { return invalidTypeErr } @@ -147,7 +152,10 @@ func validateSchemaContent(schemaContent, schemaType string) error { return err } case "avro": - break + err := validateAvroSchemaContent(schemaContent) + if err != nil { + return err + } } return nil } diff --git a/ui_src/package.json b/ui_src/package.json index d91932d82..e68b40848 100644 --- a/ui_src/package.json +++ b/ui_src/package.json @@ -13,6 +13,7 @@ "ajv": "^8.11.2", "ajv-draft-04": "^1.0.0", "antd": "4.23.1", + "avro-js": "^1.11.2", "axios": "^0.25.0", "buffer": "^6.0.3", "chart.js": "^2.9.4", @@ -41,6 +42,7 @@ "reaflow": "5.0.6", "recharts": "^2.1.16", "sass": "^1.49.0", + "util": "^0.12.5", "uuid": "^9.0.0", "web-vitals": "^2.1.4", "rdk":"6.4.4" diff --git a/ui_src/src/domain/schema/components/createSchema/index.js b/ui_src/src/domain/schema/components/createSchema/index.js index 80da479ae..1a58097a3 100644 --- a/ui_src/src/domain/schema/components/createSchema/index.js +++ b/ui_src/src/domain/schema/components/createSchema/index.js @@ -43,6 +43,7 @@ import { Context } from '../../../../hooks/store'; import Input from '../../../../components/Input'; import Modal from '../../../../components/modal'; import AttachStationModal from '../attachStationModal'; +const avro = require('avro-js') loader.init(); loader.config({ monaco }); @@ -73,15 +74,14 @@ const schemaTypes = [ }, { id: 4, - value: 'avro', - label: 'Avro (Coming soon)', + value: 'Avro', + label: 'Avro (New)', description: ( The popular. Apache Avro™ is the leading serialization format for record data, and first choice for streaming data pipelines. It offers excellent schema evolution. - ), - disabled: true + ) } ]; @@ -96,15 +96,15 @@ message Test { }` }, Avro: { - language: 'avro', + language: 'json', // Avro stores the data definition in JSON format. value: `{ "type": "record", "namespace": "com.example", - "name": "test-schema", + "name": "test_schema", "fields": [ { "name": "username", "type": "string", "default": "-2" }, - { "name": "age", "type": "int", "default": "none" }, - { "name": "phone", "type": "int", "default": "NONE" }, + { "name": "age", "type": "int" }, + { "name": "phone", "type": "long" }, { "name": "country", "type": "string", "default": "NONE" } ] }` @@ -355,6 +355,17 @@ function CreateSchema({ createNew }) { } }; + const validateAvroSchema = (value) => { + try { + avro.parse(value); + setValidateSuccess(''); + setValidateError(''); + } catch (error) { + setValidateSuccess(''); + setValidateError('Your schema is invalid'); + } + }; + const checkContent = (value) => { const { type } = formFields; if (value === ' ' || value === '') { @@ -368,6 +379,8 @@ function CreateSchema({ createNew }) { validateJsonSchema(value); } else if (type === 'GraphQL') { validateGraphQlSchema(value); + } else if (type === 'Avro') { + validateAvroSchema(value); } } }; @@ -536,6 +549,7 @@ function CreateSchema({ createNew }) { {formFields?.type === 'Protobuf' && schemaContentEditor} {formFields?.type === 'Json' && schemaContentEditor} {formFields?.type === 'GraphQL' && schemaContentEditor} + {formFields?.type === 'Avro' && schemaContentEditor}
{validateError && } diff --git a/ui_src/src/domain/schema/components/schemaDetails/index.js b/ui_src/src/domain/schema/components/schemaDetails/index.js index 1412020bd..7020da40e 100644 --- a/ui_src/src/domain/schema/components/schemaDetails/index.js +++ b/ui_src/src/domain/schema/components/schemaDetails/index.js @@ -48,6 +48,7 @@ import OverflowTip from '../../../../components/tooltip/overflowtip'; import { validate, parse, buildASTSchema } from 'graphql'; import SegmentButton from '../../../../components/segmentButton'; import AttachStationModal from '../attachStationModal'; +const avro = require('avro-js') loader.init(); loader.config({ monaco }); @@ -258,6 +259,17 @@ function SchemaDetails({ schemaName, closeDrawer }) { } }; + const validateAvroSchema = (value) => { + try { + avro.parse(value); + setValidateSuccess(''); + setValidateError(''); + } catch (error) { + setValidateSuccess(''); + setValidateError('Your schema is invalid'); + } + }; + const checkContent = (value) => { const { type } = schemaDetails; if (value === ' ' || value === '') { @@ -271,6 +283,8 @@ function SchemaDetails({ schemaName, closeDrawer }) { validateJsonSchema(value); } else if (type === 'graphql') { validateGraphQlSchema(value); + } else if (type === 'avro') { + validateAvroSchema(value); } } }; @@ -441,7 +455,7 @@ function SchemaDetails({ schemaName, closeDrawer }) { fontSize: '14px', fontFamily: 'Inter' }} - language={schemaDetails?.type === 'protobuf' ? 'proto' : schemaDetails?.type} + language={schemaDetails?.type === 'protobuf' ? 'proto' : schemaDetails?.type === 'avro' ? 'json' : schemaDetails?.type} height="calc(100% - 104px)" defaultValue={versionSelected?.schema_content} value={newVersion} @@ -465,7 +479,7 @@ function SchemaDetails({ schemaName, closeDrawer }) { fontSize: '14px', fontFamily: 'Inter' }} - language={schemaDetails?.type === 'protobuf' ? 'proto' : schemaDetails?.type} + language={schemaDetails?.type === 'protobuf' ? 'proto' : schemaDetails?.type === 'avro' ? 'json' : schemaDetails?.type} height="calc(100% - 100px)" value={versionSelected?.schema_content} /> @@ -473,7 +487,7 @@ function SchemaDetails({ schemaName, closeDrawer }) { {isDiff === 'Yes' && ( Date: Wed, 12 Jul 2023 07:43:15 +0530 Subject: [PATCH 2/4] commiting go.mod and go.sum files --- go.mod | 2 ++ go.sum | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/go.mod b/go.mod index 3eec88d69..677047984 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/golang-jwt/jwt/v4 v4.5.0 github.com/graph-gophers/graphql-go v1.5.0 + github.com/hamba/avro/v2 v2.12.0 github.com/jackc/pgx/v5 v5.3.1 github.com/santhosh-tekuri/jsonschema/v5 v5.1.0 github.com/slack-go/slack v0.11.4 @@ -94,6 +95,7 @@ require ( github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/memphisdev/memphis.go v1.0.4 + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 2eee9cd50..2a7fad133 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.5.0 h1:fDqblo50TEpD0LY7RXk/LFVYEVqo3+tXMNMPSVXA1yc= github.com/graph-gophers/graphql-go v1.5.0/go.mod h1:YtmJZDLbF1YYNrlNAuiO5zAStUWc3XZT07iGsVqe1Os= +github.com/hamba/avro/v2 v2.12.0 h1:QZvbrfOfHQ7kZnlxRdwRU0opSf9ZrqlzpKzJuIUjIjU= +github.com/hamba/avro/v2 v2.12.0/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK8fJ7Jo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -280,6 +282,8 @@ github.com/memphisdev/memphis.go v1.0.4 h1:cftOPl+XRf3zdnY8egLny3NVhtFwwaqEsN67z github.com/memphisdev/memphis.go v1.0.4/go.mod h1:VsFe2Wrght9LnzP2JWRA+tDeJS/12+3xxREWYieV6FQ= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= From 92dcdb2ca7540e019073654fa3a0cf7e1aef801e Mon Sep 17 00:00:00 2001 From: big-vi Date: Wed, 12 Jul 2023 08:58:53 +0530 Subject: [PATCH 3/4] alter schema table with new enum value for backward compatibility --- db/db.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/db.go b/db/db.go index 95bcb98ea..94b5ddde6 100644 --- a/db/db.go +++ b/db/db.go @@ -244,6 +244,7 @@ func createTables(MetadataDbClient MetadataStorage) error { ALTER TABLE schemas DROP CONSTRAINT IF EXISTS name; ALTER TABLE schemas DROP CONSTRAINT IF EXISTS schemas_name_tenant_name_key; ALTER TABLE schemas ADD CONSTRAINT schemas_name_tenant_name_key UNIQUE(name, tenant_name); + ALTER TYPE enum_type ADD VALUE 'avro'; END IF; END $$;` From 236a5d7dd23f26a2595611caad820c6302903b2c Mon Sep 17 00:00:00 2001 From: big-vi Date: Wed, 26 Jul 2023 08:10:24 +0530 Subject: [PATCH 4/4] avro schema fixes fixed ui filter removed util module better error handling --- server/memphis_handlers_schemas.go | 2 +- server/memphis_handlers_user_mgmt.go | 2 +- ui_src/package.json | 1 - ui_src/src/domain/schema/components/createSchema/index.js | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/memphis_handlers_schemas.go b/server/memphis_handlers_schemas.go index 118e03e78..3139aefe8 100644 --- a/server/memphis_handlers_schemas.go +++ b/server/memphis_handlers_schemas.go @@ -79,7 +79,7 @@ func validateGraphqlSchemaContent(schemaContent string) error { func validateAvroSchemaContent(schemaContent string) error { _, err := avro.Parse(schemaContent) if err != nil { - return err + return fmt.Errorf("your Avro file is invalid: %v", err.Error()) } return nil } diff --git a/server/memphis_handlers_user_mgmt.go b/server/memphis_handlers_user_mgmt.go index 8efc09399..e8b4dadfb 100644 --- a/server/memphis_handlers_user_mgmt.go +++ b/server/memphis_handlers_user_mgmt.go @@ -738,7 +738,7 @@ func (umh UserMgmtHandler) GetFilterDetails(c *gin.Context) { return } - schemaType := []string{"protobuf", "json", "graphql"} + schemaType := []string{"protobuf", "json", "graphql", "avro"} usage := []string{"used", "not used"} c.IndentedJSON(200, gin.H{"tags": tags, "users": users, "type": schemaType, "usage": usage}) return diff --git a/ui_src/package.json b/ui_src/package.json index e68b40848..c495600a3 100644 --- a/ui_src/package.json +++ b/ui_src/package.json @@ -42,7 +42,6 @@ "reaflow": "5.0.6", "recharts": "^2.1.16", "sass": "^1.49.0", - "util": "^0.12.5", "uuid": "^9.0.0", "web-vitals": "^2.1.4", "rdk":"6.4.4" diff --git a/ui_src/src/domain/schema/components/createSchema/index.js b/ui_src/src/domain/schema/components/createSchema/index.js index 1a58097a3..7525b8044 100644 --- a/ui_src/src/domain/schema/components/createSchema/index.js +++ b/ui_src/src/domain/schema/components/createSchema/index.js @@ -75,7 +75,7 @@ const schemaTypes = [ { id: 4, value: 'Avro', - label: 'Avro (New)', + label: 'Avro', description: ( The popular. Apache Avro™ is the leading serialization format for record data, and first choice for streaming data pipelines. It offers excellent schema