Better configuration (#288)

* Better config * print to log * Upgrade golang.org/x/net * Tidy go mods * export WebpConfig * Add TestConvertRawToJPG * Add TestConvertRawToJPG * Update Integration Tests UA * Add README for supporting NEF --------- Co-authored-by: Benny <[email protected]>
webp-sh · Nov 5, 2023 · 543af24 · 543af24
1 parent 4a90b18
commit 543af24
Show file tree

Hide file tree

Showing 8 changed files with 115 additions and 22 deletions.
diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
@@ -32,7 +32,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |
@@ -61,7 +61,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |

diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@
 
 This is a Server based on Golang, which allows you to serve WebP images on the fly.
 
-Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC
+Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF
 
 > e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp` format without changing the URL.
 

diff --git a/config/config.go b/config/config.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -27,7 +28,7 @@ const (
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
   "IMG_MAP": {},
-  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg"],
+  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","nef"],
   "ENABLE_AVIF": false,
   "ENABLE_EXTRA_PARAMS": false
 }`
@@ -58,8 +59,8 @@ var (
 	ShowVersion    bool
 	ProxyMode      bool
 	Prefetch       bool
-	Config         jsonFile
-	Version        = "0.9.12"
+	Config         = NewWebPConfig()
+	Version        = "0.10.1"
 	WriteLock      = cache.New(5*time.Minute, 10*time.Minute)
 	RemoteRaw      = "./remote-raw"
 	Metadata       = "./metadata"
@@ -72,7 +73,7 @@ type MetaFile struct {
 	Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes
 }
 
-type jsonFile struct {
+type WebpConfig struct {
 	Host              string            `json:"HOST"`
 	Port              string            `json:"PORT"`
 	ImgPath           string            `json:"IMG_PATH"`
@@ -84,6 +85,20 @@ type jsonFile struct {
 	EnableExtraParams bool              `json:"ENABLE_EXTRA_PARAMS"`
 }
 
+func NewWebPConfig() *WebpConfig {
+	return &WebpConfig{
+		Host:              "0.0.0.0",
+		Port:              "3333",
+		ImgPath:           "./pics",
+		Quality:           80,
+		AllowedTypes:      []string{"jpg", "png", "jpeg", "bmp", "svg", "nef"},
+		ImageMap:          map[string]string{},
+		ExhaustPath:       "./exhaust",
+		EnableAVIF:        false,
+		EnableExtraParams: false,
+	}
+}
+
 func init() {
 	flag.StringVar(&ConfigPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)")
 	flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert image to WebP format.")
@@ -103,6 +118,57 @@ func LoadConfig() {
 	_ = jsonObject.Close()
 	switchProxyMode()
 	Config.ImageMap = parseImgMap(Config.ImageMap)
+
+	// Read from ENV for override
+	if os.Getenv("WEBP_HOST") != "" {
+		Config.Host = os.Getenv("WEBP_HOST")
+	}
+	if os.Getenv("WEBP_PORT") != "" {
+		Config.Port = os.Getenv("WEBP_PORT")
+	}
+	if os.Getenv("WEBP_IMG_PATH") != "" {
+		Config.ImgPath = os.Getenv("WEBP_IMG_PATH")
+	}
+	if os.Getenv("WEBP_EXHAUST_PATH") != "" {
+		Config.ExhaustPath = os.Getenv("WEBP_EXHAUST_PATH")
+	}
+	if os.Getenv("WEBP_QUALITY") != "" {
+		quality, err := strconv.Atoi(os.Getenv("WEBP_QUALITY"))
+		if err != nil {
+			log.Warnf("WEBP_QUALITY is not a valid integer, using value in config.json %d", Config.Quality)
+		} else {
+			Config.Quality = quality
+		}
+	}
+	if os.Getenv("WEBP_ALLOWED_TYPES") != "" {
+		Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",")
+	}
+	if os.Getenv("WEBP_ENABLE_AVIF") != "" {
+		enableAVIF := os.Getenv("WEBP_ENABLE_AVIF")
+		if enableAVIF == "true" {
+			Config.EnableAVIF = true
+		} else if enableAVIF == "false" {
+			Config.EnableAVIF = false
+		} else {
+			log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF)
+		}
+	}
+	if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" {
+		enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS")
+		if enableExtraParams == "true" {
+			Config.EnableExtraParams = true
+		} else if enableExtraParams == "false" {
+			Config.EnableExtraParams = false
+		} else {
+			log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams)
+		}
+	}
+	if os.Getenv("WEBP_IMG_MAP") != "" {
+		// TODO
+	}
+
+	log.Debugln("Config init complete")
+	log.Debugln("Config", Config)
 }
 
 func parseImgMap(imgMap map[string]string) map[string]string {

diff --git a/encoder/jpgconvert.go → encoder/rawconvert.go b/encoder/jpgconvert.go → encoder/rawconvert.go
diff --git a/encoder/rawconvert_test.go b/encoder/rawconvert_test.go
@@ -0,0 +1,26 @@
+package encoder
+
+import (
+	"testing"
+)
+
+func TestConvertRawToJPG(t *testing.T) {
+	testCases := []struct {
+		rawPath        string
+		optimizedPath  string
+		expectedResult string
+		expectedStatus bool
+	}{
+		// blackbird.NEF is from https://github.com/jewright/nef-to-jpg/blob/main/photoconverter/Sample-Images/blackbird.NEF
+		{"../pics/blackbird.NEF", "../exhaust_test/", "../exhaust_test/blackbird.NEF_extracted.jpg", true},
+		{"../pics/big.jpg", "../exhaust_test/", "../pics/big.jpg", false},
+	}
+
+	for _, tc := range testCases {
+		result, status := ConvertRawToJPG(tc.rawPath, tc.optimizedPath)
+
+		if result != tc.expectedResult || status != tc.expectedStatus {
+			t.Errorf("ConvertRawToJPG(%s, %s) => (%s, %t), expected (%s, %t)", tc.rawPath, tc.optimizedPath, result, status, tc.expectedResult, tc.expectedStatus)
+		}
+	}
+}
diff --git a/go.mod b/go.mod
@@ -1,11 +1,11 @@
 module webp_server_go
 
-go 1.20
+go 1.21
 
 require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/davidbyttow/govips/v2 v2.13.0
-	github.com/gofiber/fiber/v2 v2.49.2
+	github.com/gofiber/fiber/v2 v2.50.0
 	github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/jeremytorres/rawparser v1.0.2
@@ -30,9 +30,9 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	golang.org/x/image v0.5.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -9,8 +9,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM=
 github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw=
-github.com/gofiber/fiber/v2 v2.49.2 h1:ONEN3/Vc+dUCxxDgZZwpqvhISgHqb+bu+isBiEyKEQs=
-github.com/gofiber/fiber/v2 v2.49.2/go.mod h1:gNsKnyrmfEWFpJxQAV0qvW6l70K1dZGno12oLtukcts=
+github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3aAzCLddcoxzboIrOm6/FVRXjp/5JM=
@@ -73,8 +73,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -86,19 +86,20 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

diff --git a/pics/blackbird.NEF b/pics/blackbird.NEF