-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[fastserde] Logical types are now supported for avro version 1.9+ (#516)
A modelData parameter (GenericData or SpecificData instance) is passed to FastSerdeBase, FastSerdeCache and everything else in-between. The behavior is unchanged if the modelData is not passed in.
- Loading branch information
1 parent
917b1b9
commit 73acd3c
Showing
61 changed files
with
2,240 additions
and
262 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 38 additions & 0 deletions
38
fastserde/avro-fastserde-tests-common/src/test/avro/logicalTypesDefined.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| { | ||
| "type": "record", | ||
| "name": "FastSerdeLogicalTypesDefined", | ||
| "namespace": "com.linkedin.avro.fastserde.generated.avro", | ||
| "doc": "Used in tests to confirm fast-serde supports logical-types", | ||
| "fields": [ | ||
| { | ||
| "name": "timeMillisField", | ||
| "type": { | ||
| "type": "int", | ||
| "logicalType": "time-millis" | ||
| } | ||
| }, | ||
| { | ||
| "name": "dateField", | ||
| "type": { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| }, | ||
| { | ||
| "name": "arrayOfUnionOfDateAndTimestampMillis", | ||
| "type": { | ||
| "type": "array", | ||
| "items": [ | ||
| { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| }, | ||
| { | ||
| "type": "long", | ||
| "logicalType": "timestamp-millis" | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } |
187 changes: 187 additions & 0 deletions
187
fastserde/avro-fastserde-tests-common/src/test/avro/logicalTypesTest1.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,187 @@ | ||
| { | ||
| "type": "record", | ||
| "name": "FastSerdeLogicalTypesTest1", | ||
| "namespace": "com.linkedin.avro.fastserde.generated.avro", | ||
| "doc": "Used in tests to confirm fast-serde supports logical-types", | ||
| "fields": [ | ||
| { | ||
| "name": "unionOfArrayAndMap", | ||
| "type": [ | ||
| { | ||
| "type": "array", | ||
| "items": { | ||
| "type": "int", | ||
| "logicalType": "time-millis" | ||
| } | ||
| }, | ||
| { | ||
| "type": "map", | ||
| "values": { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "mapOfUnionsOfDateAndTimestampMillis", | ||
| "type": { | ||
| "type": "map", | ||
| "values": [ | ||
| { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| }, | ||
| { | ||
| "type": "long", | ||
| "logicalType": "timestamp-millis" | ||
| } | ||
| ] | ||
| } | ||
| }, | ||
| { | ||
| "name": "timestampMillisMap", | ||
| "type": { | ||
| "type": "map", | ||
| "values": { | ||
| "type": "long", | ||
| "logicalType": "timestamp-millis" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "name": "nullableArrayOfDates", | ||
| "type": [ | ||
| "null", | ||
| { | ||
| "type": "array", | ||
| "items": { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| } | ||
| ], | ||
| "default": null | ||
| }, | ||
| { | ||
| "name": "arrayOfDates", | ||
| "type": { | ||
| "type": "array", | ||
| "items": { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "name": "unionOfDecimalOrDate", | ||
| "type": [ | ||
| { | ||
| "type": "bytes", | ||
| "logicalType": "decimal", | ||
| "precision": 4, | ||
| "scale": 2 | ||
| }, | ||
| { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "uuidField", | ||
| "type": { | ||
| "type": "string", | ||
| "logicalType": "uuid" | ||
| } | ||
| }, | ||
| { | ||
| "name": "timestampMillisField", | ||
| "type": { | ||
| "type": "long", | ||
| "logicalType": "timestamp-millis" | ||
| } | ||
| }, | ||
| { | ||
| "name": "timestampMicrosField", | ||
| "type": { | ||
| "type": "long", | ||
| "logicalType": "timestamp-micros" | ||
| } | ||
| }, | ||
| { | ||
| "name": "timeMillisField", | ||
| "type": { | ||
| "type": "int", | ||
| "logicalType": "time-millis" | ||
| } | ||
| }, | ||
| { | ||
| "name": "timeMicrosField", | ||
| "type": { | ||
| "type": "long", | ||
| "logicalType": "time-micros" | ||
| } | ||
| }, | ||
| { | ||
| "name": "dateField", | ||
| "type": { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| } | ||
| }, | ||
| { | ||
| "name": "nestedLocalTimestampMillis", | ||
| "type": { | ||
| "name": "LocalTimestampRecord", | ||
| "type": "record", | ||
| "fields": [ | ||
| { | ||
| "name": "nestedTimestamp", | ||
| "type": { | ||
| "type": "long", | ||
| "logicalType": "local-timestamp-millis" | ||
| } | ||
| }, | ||
| { | ||
| "name": "nullableNestedTimestamp", | ||
| "type": [ | ||
| "null", | ||
| { | ||
| "type": "long", | ||
| "logicalType": "local-timestamp-millis" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "nullableUnionOfDateAndLocalTimestamp", | ||
| "type": [ | ||
| "null", | ||
| { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| }, | ||
| { | ||
| "type": "long", | ||
| "logicalType": "local-timestamp-millis" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "unionOfDateAndLocalTimestamp", | ||
| "type": [ | ||
| { | ||
| "type": "int", | ||
| "logicalType": "date" | ||
| }, | ||
| { | ||
| "type": "long", | ||
| "logicalType": "local-timestamp-millis" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } |
34 changes: 34 additions & 0 deletions
34
fastserde/avro-fastserde-tests-common/src/test/avro/logicalTypesUndefined.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| { | ||
| "type": "record", | ||
| "name": "FastSerdeLogicalTypesUndefined", | ||
| "namespace": "com.linkedin.avro.fastserde.generated.avro", | ||
| "doc": "Used in tests to confirm fast-serde supports logical-types", | ||
| "fields": [ | ||
| { | ||
| "name": "timeMillisField", | ||
| "type": { | ||
| "type": "int" | ||
| } | ||
| }, | ||
| { | ||
| "name": "dateField", | ||
| "type": { | ||
| "type": "int" | ||
| } | ||
| }, | ||
| { | ||
| "name": "arrayOfUnionOfDateAndTimestampMillis", | ||
| "type": { | ||
| "type": "array", | ||
| "items": [ | ||
| { | ||
| "type": "int" | ||
| }, | ||
| { | ||
| "type": "long" | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } |
Oops, something went wrong.