feat: package registry support (#98)

Fixes #8

pkgmgr/config.go

@@ -30,6 +30,8 @@ type Config struct {
 	Logger              *slog.Logger
 	Template            *Template
 	RequiredPackageTags []string
+	RegistryUrl         string
+	RegistryDir         string
 }
 
 func NewDefaultConfig() (Config, error) {

pkgmgr/pkgmgr.go

@@ -56,8 +56,12 @@ func (p *PackageManager) init() error {
 	if err := p.state.Load(); err != nil {
 		return fmt.Errorf("failed to load state: %s", err)
 	}
-	// TODO: replace this with syncing a repo and reading from disk
-	p.availablePackages = append(p.availablePackages, RegistryPackages...)
+	// Get available packages from configured registry
+	if registryPkgs, err := registryPackages(p.config); err != nil {
+		return err
+	} else {
+		p.availablePackages = registryPkgs[:]
+	}
 	// Setup templating
 	p.initTemplate()
 	return nil

pkgmgr/registry.go

@@ -14,7 +14,22 @@
 
 package pkgmgr
 
-var RegistryPackages = []Package{
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+var registryBuiltinPackages = []Package{
 	{
 		Name:        "cardano-node",
 		Version:     "8.7.3",
@@ -169,3 +184,136 @@ docker run --rm -ti ghcr.io/blinklabs-io/mithril-client:0.7.0-1 $@
 		},
 	},
 }
+
+func registryPackages(cfg Config) ([]Package, error) {
+	if cfg.RegistryDir != "" {
+		return registryPackagesDir(cfg)
+	} else if cfg.RegistryUrl != "" {
+		return registryPackagesUrl(cfg)
+	} else {
+		return registryBuiltinPackages[:], nil
+	}
+}
+
+func registryPackagesDir(cfg Config) ([]Package, error) {
+	tmpFs := os.DirFS("/").(fs.ReadFileFS)
+	return registryPackagesFs(cfg, tmpFs)
+}
+
+func registryPackagesFs(cfg Config, filesystem fs.ReadFileFS) ([]Package, error) {
+	var ret []Package
+	err := fs.WalkDir(
+		filesystem,
+		cfg.RegistryDir,
+		func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			// Skip dirs
+			if d.IsDir() {
+				// Skip all files inside dot-dirs
+				if strings.HasPrefix(d.Name(), `.`) && d.Name() != `.` {
+					return fs.SkipDir
+				}
+				return nil
+			}
+			// Skip non-YAML files based on file extension
+			if filepath.Ext(path) != ".yaml" && filepath.Ext(path) != ".yml" {
+				return nil
+			}
+			// Try to parse YAML file as package
+			fileData, err := filesystem.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			var tmpPkg Package
+			if err := yaml.Unmarshal(fileData, &tmpPkg); err != nil {
+				cfg.Logger.Warn(
+					fmt.Sprintf(
+						"failed to load %q as package: %s",
+						path,
+						err,
+					),
+				)
+				return nil
+			}
+			ret = append(ret, tmpPkg)
+			return nil
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func registryPackagesUrl(cfg Config) ([]Package, error) {
+	cachePath := filepath.Join(
+		cfg.CacheDir,
+		"registry",
+	)
+	// Check age of existing cache
+	stat, err := os.Stat(cachePath)
+	if err != nil {
+		if err != fs.ErrNotExist {
+			return nil, err
+		}
+	}
+	// Fetch and extract registry ZIP into cache if it doesn't exist or is too old
+	if err == fs.ErrNotExist ||
+		stat.ModTime().Before(time.Now().Add(-24*time.Hour)) {
+		// Fetch registry ZIP
+		resp, err := http.Get(cfg.RegistryUrl)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		respBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		zipData := bytes.NewReader(respBody)
+		zipReader, err := zip.NewReader(
+			zipData,
+			int64(zipData.Len()),
+		)
+		if err != nil {
+			return nil, err
+		}
+		// Clear out existing cache files
+		if err := os.RemoveAll(cachePath); err != nil {
+			return nil, err
+		}
+		if err := os.MkdirAll(cachePath, fs.ModePerm); err != nil {
+			return nil, err
+		}
+		// Extract files from ZIP into cache path
+		for _, zipFile := range zipReader.File {
+			outPath := filepath.Join(
+				cachePath,
+				zipFile.Name,
+			)
+			// Create parent dir
+			if err := os.MkdirAll(filepath.Dir(outPath), fs.ModePerm); err != nil {
+				return nil, err
+			}
+			// Read file bytes
+			zf, err := zipFile.Open()
+			if err != nil {
+				return nil, err
+			}
+			zfData, err := io.ReadAll(zf)
+			if err != nil {
+				return nil, err
+			}
+			zf.Close()
+			// Write file
+			if err := os.WriteFile(outPath, zfData, fs.ModePerm); err != nil {
+				return nil, err
+			}
+		}
+	}
+	// Process cache dir
+	cfg.RegistryDir = cachePath
+	return registryPackagesDir(cfg)
+}

pkgmgr/registry_test.go

@@ -0,0 +1,68 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkgmgr
+
+import (
+	"log/slog"
+	"reflect"
+	"testing"
+	"testing/fstest"
+)
+
+func TestRegistryPackagesFs(t *testing.T) {
+	testRegistryDir := "test/registry/dir"
+	testFs := fstest.MapFS{
+		// This file is outside the registry dir and should not get processed
+		"some/random/file.yaml": {},
+		"test/registry/dir/packageA/packageA-1.2.3.yaml": {
+			Data: []byte("name: packageA\nversion: 1.2.3"),
+		},
+		"test/registry/dir/packageA/packageA-2.3.4.yaml": {
+			Data: []byte("name: packageA\nversion: 2.3.4"),
+		},
+		"test/registry/dir/packageB/packageB-3.4.5.yml": {
+			Data: []byte("name: packageB\nversion: 3.4.5"),
+		},
+		// This file should get ignored without a YAML extension
+		"test/registry/dir/some.file": {
+			Data: []byte("name: packageC\nversion: 4.5.6"),
+		},
+	}
+	testExpectedPkgs := []Package{
+		{
+			Name:    "packageA",
+			Version: "1.2.3",
+		},
+		{
+			Name:    "packageA",
+			Version: "2.3.4",
+		},
+		{
+			Name:    "packageB",
+			Version: "3.4.5",
+		},
+	}
+	cfg := Config{
+		RegistryDir: testRegistryDir,
+		Logger:      slog.Default(),
+	}
+	pkgs, err := registryPackagesFs(cfg, testFs)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if !reflect.DeepEqual(pkgs, testExpectedPkgs) {
+		t.Fatalf("did not get expected packages\n  got: %#v\n  expected: %#v", pkgs, testExpectedPkgs)
+	}
+}