snapshot: start working through issue #19

app/index/app.go

@@ -6,20 +6,26 @@ import (
 	"fmt"
 	"log"
 	"runtime"
+	"slices"
 
 	"github.com/aaronland/go-sqlite/v2"
 	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/whosonfirst/go-reader"
+	sql_tables "github.com/whosonfirst/go-whosonfirst-sql/tables"
 	"github.com/whosonfirst/go-whosonfirst-sqlite-features-index/v2"
 	"github.com/whosonfirst/go-whosonfirst-sqlite-features/v2/tables"
 	sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index/v4"
 )
 
+const index_alt_all string = "*"
+
 func Run(ctx context.Context, logger *log.Logger) error {
 	fs := DefaultFlagSet()
 	return RunWithFlagSet(ctx, fs, logger)
 }
 
+// To do: Add RunWithOptions...
+
 func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) error {
 
 	flagset.Parse(fs)
@@ -63,7 +69,6 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		}()
 
 	} else {
-
 		defer db.Close(ctx)
 	}
 
@@ -83,15 +88,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		geojson_opts, err := tables.DefaultGeoJSONTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geojson' table options because %s", err)
+			return fmt.Errorf("failed to create '%s' table options because %s", sql_tables.GEOJSON_TABLE_NAME, err)
 		}
 
-		geojson_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.GEOJSON_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			geojson_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewGeoJSONTableWithDatabaseAndOptions(ctx, db, geojson_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geojson' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.GEOJSON_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, gt)
@@ -102,7 +111,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		t, err := tables.NewSupersedesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'supersedes' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.SUPERSEDES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, t)
@@ -116,7 +125,11 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 			return fmt.Errorf("failed to create 'rtree' table options because %s", err)
 		}
 
-		rtree_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.RTREE_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			rtree_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewRTreeTableWithDatabaseAndOptions(ctx, db, rtree_opts)
 
@@ -135,7 +148,11 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 			return fmt.Errorf("failed to create 'properties' table options because %s", err)
 		}
 
-		properties_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.PROPERTIES_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			properties_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewPropertiesTableWithDatabaseAndOptions(ctx, db, properties_opts)
 
@@ -151,15 +168,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		spr_opts, err := tables.DefaultSPRTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("Failed to create 'spr' table options because %v", err)
+			return fmt.Errorf("Failed to create '%s' table options because %v", sql_tables.SPR_TABLE_NAME, err)
 		}
 
-		spr_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.SPR_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			spr_opts.IndexAltFiles = true
+		}
 
 		st, err := tables.NewSPRTableWithDatabaseAndOptions(ctx, db, spr_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'spr' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.SPR_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, st)
@@ -170,7 +191,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		nm, err := tables.NewNamesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'names' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.NAMES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, nm)
@@ -181,7 +202,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		an, err := tables.NewAncestorsTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'ancestors' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.ANCESTORS_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, an)
@@ -192,7 +213,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		cn, err := tables.NewConcordancesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'concordances' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.CONCORDANCES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, cn)
@@ -206,15 +227,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		geometries_opts, err := tables.DefaultGeometriesTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geometries' table options because %v", err)
+			return fmt.Errorf("failed to create '%s' table options because %v", sql_tables.GEOMETRIES_TABLE_NAME, err)
 		}
 
-		geometries_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.CONCORDANCES_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			geometries_opts.IndexAltFiles = true
+		}
 
 		gm, err := tables.NewGeometriesTableWithDatabaseAndOptions(ctx, db, geometries_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geometries' table because %v", err)
+			return fmt.Errorf("failed to create '%s' table because %v", sql_tables.CONCORDANCES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, gm)
@@ -226,6 +251,8 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 
 	if search {
 
+		// ALT FILES...
+
 		st, err := tables.NewSearchTableWithDatabase(ctx, db)
 
 		if err != nil {

app/index/flags.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/sfomuseum/go-flags/flagset"
+	"github.com/sfomuseum/go-flags/multi"
 	"github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter"
 )
 
@@ -34,6 +35,8 @@ var optimize bool
 var alt_files bool
 var strict_alt_files bool
 
+var index_alt multi.MultiString
+
 var index_relations bool
 var relations_uri string
 
@@ -68,7 +71,9 @@ func DefaultFlagSet() *flag.FlagSet {
 	fs.BoolVar(&timings, "timings", false, "Display timings during and after indexing")
 	fs.BoolVar(&optimize, "optimize", true, "Attempt to optimize the database before closing connection")
 
-	fs.BoolVar(&alt_files, "index-alt-files", false, "Index alt geometries")
+	fs.BoolVar(&alt_files, "index-alt-files", false, "Index alt geometries. This flag is deprecated, please use -index-alt=TABLE,TABLE,etc. instead. To index alt geometries in all the applicable tables use -index-alt=*")
+	fs.Var(&index_alt, "index-alt", "Zero or more table names where alt geometry files should be indexed.")
+
 	fs.BoolVar(&strict_alt_files, "strict-alt-files", true, "Be strict when indexing alt geometries")
 
 	fs.BoolVar(&index_relations, "index-relations", false, "Index the records related to a feature, specifically wof:belongsto, wof:depicts and wof:involves. Alt files for relations are not indexed at this time.")

go.mod

@@ -1,19 +1,20 @@
 module github.com/whosonfirst/go-whosonfirst-sqlite-features-index/v2
 
-go 1.21.3
+go 1.22
 
 require (
 	github.com/aaronland/go-sqlite-mattn v0.0.3
 	github.com/aaronland/go-sqlite-modernc v0.0.3
 	github.com/aaronland/go-sqlite/v2 v2.2.0
 	github.com/sfomuseum/go-flags v0.10.0
-	github.com/tidwall/gjson v1.17.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/whosonfirst/go-reader v1.0.2
 	github.com/whosonfirst/go-reader-http v0.3.1
 	github.com/whosonfirst/go-whosonfirst-feature v0.0.27
 	github.com/whosonfirst/go-whosonfirst-iterate-git/v2 v2.1.4
 	github.com/whosonfirst/go-whosonfirst-iterate/v2 v2.3.4
-	github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2
+	github.com/whosonfirst/go-whosonfirst-sql v0.0.3
+	github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3
 	github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0
 	github.com/whosonfirst/go-whosonfirst-uri v1.3.0
 )
@@ -55,7 +56,6 @@ require (
 	github.com/whosonfirst/go-whosonfirst-names v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-sources v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7 // indirect
-	github.com/whosonfirst/go-whosonfirst-sql v0.0.2 // indirect
 	github.com/whosonfirst/walk v0.0.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.mongodb.org/mongo-driver v1.11.4 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -143,10 +143,10 @@ github.com/whosonfirst/go-whosonfirst-sources v0.1.0 h1:JuKLa6KWke22jBfJ1pM9WQHo
 github.com/whosonfirst/go-whosonfirst-sources v0.1.0/go.mod h1:EUMHyGzUmqPPxlMmOp+28BFeoBdxxE0HCKRd67lkqGM=
 github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7 h1:1j4IMuVua4/NhqKm2ke16h2V5Z2XiLoOdXbjDrfZb0E=
 github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7/go.mod h1:I2IqsIutlDrvYp37Xzwk4WGq+m/gLh+gEifiLmH0bac=
-github.com/whosonfirst/go-whosonfirst-sql v0.0.2 h1:YkGkDxgeZiu4lhWFu4I/2f9uczgmwpW4wNV67xoRKAo=
-github.com/whosonfirst/go-whosonfirst-sql v0.0.2/go.mod h1:sQXkJTPfnxXbOPTCCxGjUydfQqATfEyO4xyxj2pEX3E=
-github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2 h1:ppfP8QKV6LMxnTeHgGE2ObJ+Txh99cHXjakDi0ohqHQ=
-github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2/go.mod h1:8QuXkCVjJ1wrCESwF2Rr3PvN/4yD6Zkw4QWLSdEesmg=
+github.com/whosonfirst/go-whosonfirst-sql v0.0.3 h1:yyAdKO8bjuhnFsV0HHeAprd7HExpZo54/7K2+10kHSg=
+github.com/whosonfirst/go-whosonfirst-sql v0.0.3/go.mod h1:sQXkJTPfnxXbOPTCCxGjUydfQqATfEyO4xyxj2pEX3E=
+github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3 h1:tDt4MkomZKljI7Q8IYbB7L2XfKy69Y73/qJL/BzbzyY=
+github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3/go.mod h1:TUDGyZW2nIWZ4gB7nJ0eTt0+p8izYXhyhoiOf1fOFFY=
 github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0 h1:JEDFIFN7bgQJTDAE1Bc0dYbnWUKF4M8Vsjr4AxsqI/A=
 github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0/go.mod h1:ZM85QUfMKGRywkWmsT3SfmzMG2vKEzPmQdj3II08IGU=
 github.com/whosonfirst/go-whosonfirst-uri v1.3.0 h1:LYOVLqP9rWQxauYVkdw65j5LZxEi8OK0GHh/qCEpX4g=