Change structure to separate application from library (#4)

* Change structure to separate application from library

Move application to cmd/main.go and update documentation
Move a few things into library code and name it a new package "tileutils"

* update setup-go action

* cmd/baremaps-exporter folder

* change structure per feedback

* fix test

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Setup golang
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ vars.GO_VERSION }}
       - name: Install staticcheck

README.md

@@ -32,7 +32,7 @@ Go must be installed, version 1.20 or later.
 Then:
 
 ```
-go install github.com/flightaware/baremaps-exporter
+go install github.com/flightaware/baremaps-exporter/cmd/baremaps-exporter
 ```
 
 Now you're ready to go.
@@ -41,7 +41,7 @@ Now you're ready to go.
 
 Run the application with:
 ```
-go run github.com/flightaware/baremaps-exporter --help
+go run github.com/flightaware/baremaps-exporter/cmd/baremaps-exporter --help
 ```
 
 All of the options:
@@ -68,7 +68,7 @@ Options:
 
 Typical usage:
 ```
-go run github.com/flightaware/baremaps-exporter -o ./tiles/ -d 'postgres://baremaps:baremaps@localhost:5432/baremaps' tiles.json
+go run github.com/flightaware/baremaps-exporter/cmd/baremaps-exporter -o ./tiles/ -d 'postgres://baremaps:baremaps@localhost:5432/baremaps' tiles.json
 ```
 
 ## LICENSE

main.go → cmd/baremaps-exporter/main.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/flightaware/baremaps-exporter/pkg/tileutils"
+
 	"github.com/alexflint/go-arg"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/twpayne/go-mbtiles"
@@ -40,22 +42,48 @@ func (Args) Description() string {
 	return "export baremaps-compatible tilesets from a postgis server"
 }
 
-type TileCoords struct {
-	Z int
-	X int
-	Y int
+type WorkerParams struct {
+	Num             int                      // worker number
+	Wg              *sync.WaitGroup          // waitgroup to signal when completed
+	Args            Args                     // input args
+	TileList        []tileutils.TileCoords   // coords that this worker should process
+	QueryMap        tileutils.ZoomLayerInfo  // a map of the queries relevant at each zoom level
+	GzipCompression bool                     // true if gzip compression should be used
+	Writer          tileutils.TileWriter     // writer to use for output
+	BulkWriter      tileutils.TileBulkWriter // bulk writer if available
+	Pool            *pgxpool.Pool            // postgres connection pool
 }
 
-type WorkerParams struct {
-	Num             int             // worker number
-	Wg              *sync.WaitGroup // waitgroup to signal when completed
-	Args            Args            // input args
-	TileList        []TileCoords    // coords that this worker should process
-	QueryMap        ZoomLayerInfo   // a map of the queries relevant at each zoom level
-	GzipCompression bool            // true if gzip compression should be used
-	Writer          TileWriter      // writer to use for output
-	BulkWriter      TileBulkWriter  // bulk writer if available
-	Pool            *pgxpool.Pool   // postgres connection pool
+// newWriters creates a TileWriter and TileBulkWriter based on the input arguments
+func newWriters(args Args, tj *tileutils.TileJSON) (writer tileutils.TileWriter, bulkWriter tileutils.TileBulkWriter, close func(), err error) {
+	var mbWriter *tileutils.MbTilesWriter
+	if args.Output == "" {
+		writer = &tileutils.DummyWriter{}
+		return
+	}
+	if args.MbTiles {
+		mbWriter = &tileutils.MbTilesWriter{
+			Filename: args.Output,
+		}
+		writer = mbWriter
+		bulkWriter = mbWriter
+		writer, close, err = mbWriter.New()
+		if err != nil {
+			return
+		}
+		meta := tileutils.CreateMetadata(tj, tileutils.CreateMetadataOptions{
+			Filename: args.TileJSON,
+			Version:  args.Version,
+			Format:   tileutils.MbTilesFormatPbf,
+		})
+		err = mbWriter.BulkWriteMetadata(meta)
+		return
+	}
+	writer = &tileutils.FileWriter{
+		Path: args.Output,
+	}
+	writer, close, err = writer.New()
+	return
 }
 
 func connectWithRetries(pool *pgxpool.Pool, numRetries int) (*pgxpool.Conn, error) {
@@ -147,7 +175,7 @@ func tileWorker(params WorkerParams) {
 			continue
 		}
 		if params.GzipCompression {
-			compressed, err := gzip(mvtTile)
+			compressed, err := tileutils.Gzip(mvtTile)
 			if err != nil {
 				fmt.Printf("error compressing tile: %v\n", err)
 			}
@@ -212,7 +240,7 @@ func main() {
 	}
 
 	// read tilejson
-	tileJSON, tileMap, err := ParseTileJSON(args.TileJSON)
+	tileJSON, tileMap, err := tileutils.ParseTileJSON(args.TileJSON)
 	if err != nil {
 		panic(err)
 	}
@@ -249,9 +277,9 @@ func main() {
 	tileJSON.MinZoom = zooms[0]
 	tileJSON.MaxZoom = zooms[len(zooms)-1]
 
-	tiles := ListTiles(zooms, tileJSON)
+	tiles := tileutils.ListTiles(zooms, tileJSON)
 	if args.TilesFile != "" {
-		extraTiles, err := tilesFromFile(args.TilesFile)
+		extraTiles, err := tileutils.TilesFromFile(args.TilesFile)
 		if err != nil {
 			panic(err)
 		}
@@ -261,7 +289,7 @@ func main() {
 	tileLen := len(tiles)
 	fmt.Printf("number of tiles: %d\n", tileLen)
 
-	writer, bulkWriter, close, err := NewWriters(args, tileJSON)
+	writer, bulkWriter, close, err := newWriters(args, tileJSON)
 	if err != nil {
 		panic(err)
 	}
@@ -271,7 +299,7 @@ func main() {
 		numWorkers = tileLen
 	}
 	// round robin the tiles so workers are hitting similar geospatial entries and zoom at the same time
-	rrTiles := RoundRobinTiles(tiles, numWorkers)
+	rrTiles := tileutils.RoundRobinTiles(tiles, numWorkers)
 	for i := 0; i < numWorkers; i++ {
 		wg.Add(1)
 		workerTiles := rrTiles[i]

gzip.go → pkg/tileutils/gzip.go

@@ -1,13 +1,13 @@
-package main
+package tileutils
 
 import (
 	"bytes"
 
 	gziplib "github.com/klauspost/compress/gzip"
 )
 
-// gzip is a utility function to zip up a tile for storage
-func gzip(data []byte) ([]byte, error) {
+// Gzip is a utility function to zip up a tile for storage
+func Gzip(data []byte) ([]byte, error) {
 	var buf bytes.Buffer
 	g, err := gziplib.NewWriterLevel(&buf, gziplib.BestCompression)
 	if err != nil {

gzip_test.go → pkg/tileutils/gzip_test.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"bytes"
@@ -12,7 +12,7 @@ import (
 
 func TestGzip(t *testing.T) {
 	data := []byte{1, 2, 3, 4, 5}
-	output, err := gzip(data)
+	output, err := Gzip(data)
 	assert.Nil(t, err)
 
 	// decode the gzip and make sure it is the same as the original input

mbtiles.go → pkg/tileutils/mbtiles.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"encoding/json"

tilejson.go → pkg/tileutils/tilejson.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"encoding/json"

tilejson_test.go → pkg/tileutils/tilejson_test.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"testing"

tileutils.go → pkg/tileutils/tileutils.go

@@ -1,4 +1,5 @@
-package main
+// Package tileutils provides types and functions for working with slippy map tiles, MBTiles, and TileJSON
+package tileutils
 
 import (
 	"fmt"
@@ -19,6 +20,12 @@ func latToY(lat float64, zoom int) int {
 	return int(math.Floor((1 - math.Log(math.Tan(latRad)+1/math.Cos(latRad))/math.Pi) / 2 * n))
 }
 
+type TileCoords struct {
+	Z int
+	X int
+	Y int
+}
+
 // BoundingBox is a lat/lon set of coordinates for a bounding box
 type BoundingBox struct {
 	Left   float64
@@ -101,8 +108,8 @@ func RoundRobinTiles(input []TileCoords, numWorkers int) [][]TileCoords {
 	return out
 }
 
-// tilesFromFile reads the tile coordinates to generate from a file
-func tilesFromFile(filename string) ([]TileCoords, error) {
+// TilesFromFile reads the tile coordinates to generate from a file
+func TilesFromFile(filename string) ([]TileCoords, error) {
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read file (%s): %w", filename, err)

tileutils_test.go → pkg/tileutils/tileutils_test.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"fmt"

utils.go → pkg/tileutils/utils.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"encoding/json"

utils_test.go → pkg/tileutils/utils_test.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"os"

writers.go → pkg/tileutils/writers.go

@@ -1,4 +1,4 @@
-package main
+package tileutils
 
 import (
 	"fmt"
@@ -152,35 +152,3 @@ func (w *MbTilesWriter) New() (TileWriter, func(), error) {
 		},
 		nil
 }
-
-// NewWriters creates a TileWriter and TileBulkWriter based on the input arguments
-func NewWriters(args Args, tj *TileJSON) (writer TileWriter, bulkWriter TileBulkWriter, close func(), err error) {
-	var mbWriter *MbTilesWriter
-	if args.Output == "" {
-		writer = &DummyWriter{}
-		return
-	}
-	if args.MbTiles {
-		mbWriter = &MbTilesWriter{
-			Filename: args.Output,
-		}
-		writer = mbWriter
-		bulkWriter = mbWriter
-		writer, close, err = mbWriter.New()
-		if err != nil {
-			return
-		}
-		meta := CreateMetadata(tj, CreateMetadataOptions{
-			Filename: args.TileJSON,
-			Version:  args.Version,
-			Format:   MbTilesFormatPbf,
-		})
-		err = mbWriter.BulkWriteMetadata(meta)
-		return
-	}
-	writer = &FileWriter{
-		Path: args.Output,
-	}
-	writer, close, err = writer.New()
-	return
-}