From f10722a96b1fa0160778b50eb7fb7490c8ecfb87 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 26 Sep 2024 19:16:46 -0400 Subject: [PATCH] cleanup --- .github/workflows/go-ci.yml | 2 +- .github/workflows/go-integration.yml | 2 +- go.sum | 4 ++-- schema.go | 2 +- table/arrow_utils.go | 8 +++++++- table/snapshots.go | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 3f90c29..578cb8c 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -39,7 +39,7 @@ jobs: strategy: fail-fast: false matrix: - go: [ '1.21', '1.22' ] + go: [ '1.22', '1.23' ] os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest' ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/go-integration.yml b/.github/workflows/go-integration.yml index b736ef3..7dd5e75 100644 --- a/.github/workflows/go-integration.yml +++ b/.github/workflows/go-integration.yml @@ -42,7 +42,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.22 + go-version: 1.23 cache: true cache-dependency-path: go.sum diff --git a/go.sum b/go.sum index d7cd9dc..55d0cc0 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= diff --git a/schema.go b/schema.go index 4748184..7ea7757 100644 --- a/schema.go +++ b/schema.go @@ -20,11 +20,11 @@ package iceberg import ( "encoding/json" "fmt" + "maps" "strings" "sync" "sync/atomic" - "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) diff --git a/table/arrow_utils.go b/table/arrow_utils.go index 7c07720..6104fc6 100644 --- a/table/arrow_utils.go +++ b/table/arrow_utils.go @@ -26,11 +26,17 @@ import ( "github.com/apache/iceberg-go" ) +// constants to look for as Keys in Arrow field metadata const ( - ArrowFieldDocKey = "doc" + ArrowFieldDocKey = "doc" + // Arrow schemas that are generated from the Parquet library will utilize + // this key to identify the field id of the source Parquet field. + // We use this when converting to Iceberg to provide field IDs ArrowParquetFieldIDKey = "PARQUET:field_id" ) +// ArrowSchemaVisitor is an interface that can be implemented and used to +// call VisitArrowSchema for iterating type ArrowSchemaVisitor[T any] interface { Schema(*arrow.Schema, T) T Struct(*arrow.StructType, []T) T diff --git a/table/snapshots.go b/table/snapshots.go index 26dc8d2..c880d7d 100644 --- a/table/snapshots.go +++ b/table/snapshots.go @@ -21,11 +21,11 @@ import ( "encoding/json" "errors" "fmt" + "maps" "strconv" "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/io" - "golang.org/x/exp/maps" ) type Operation string