diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index f2010178c..68328c98f 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -32,7 +32,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{} - name: Get container RAM stats run: | @@ -61,7 +61,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{} - name: Get container RAM stats run: | diff --git a/README.md b/README.md index 1aea77620..1f5cf812b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This is a Server based on Golang, which allows you to serve WebP images on the fly. -Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC +Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF > e.g When you visit `https://your.website/pics/tsuki.jpg`,it will serve as `image/webp` format without changing the URL. diff --git a/config/config.go b/config/config.go index cb4798914..4d89c827c 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "runtime" + "strconv" "strings" "time" @@ -27,7 +28,7 @@ const ( "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", "IMG_MAP": {}, - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg"], + "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","nef"], "ENABLE_AVIF": false, "ENABLE_EXTRA_PARAMS": false }` @@ -58,8 +59,8 @@ var ( ShowVersion bool ProxyMode bool Prefetch bool - Config jsonFile - Version = "0.9.12" + Config = NewWebPConfig() + Version = "0.10.1" WriteLock = cache.New(5*time.Minute, 10*time.Minute) RemoteRaw = "./remote-raw" Metadata = "./metadata" @@ -72,7 +73,7 @@ type MetaFile struct { Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes } -type jsonFile struct { +type WebpConfig struct { Host string `json:"HOST"` Port string `json:"PORT"` ImgPath string `json:"IMG_PATH"` @@ -84,6 +85,20 @@ type jsonFile struct { EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` } +func NewWebPConfig() *WebpConfig { + return &WebpConfig{ + Host: "0.0.0.0", + Port: "3333", + ImgPath: "./pics", + Quality: 80, + AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "svg", "nef"}, + ImageMap: map[string]string{}, + ExhaustPath: "./exhaust", + EnableAVIF: false, + EnableExtraParams: false, + } +} + func init() { flag.StringVar(&ConfigPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert image to WebP format.") @@ -103,6 +118,57 @@ func LoadConfig() { _ = jsonObject.Close() switchProxyMode() Config.ImageMap = parseImgMap(Config.ImageMap) + + // Read from ENV for override + if os.Getenv("WEBP_HOST") != "" { + Config.Host = os.Getenv("WEBP_HOST") + } + if os.Getenv("WEBP_PORT") != "" { + Config.Port = os.Getenv("WEBP_PORT") + } + if os.Getenv("WEBP_IMG_PATH") != "" { + Config.ImgPath = os.Getenv("WEBP_IMG_PATH") + } + if os.Getenv("WEBP_EXHAUST_PATH") != "" { + Config.ExhaustPath = os.Getenv("WEBP_EXHAUST_PATH") + } + if os.Getenv("WEBP_QUALITY") != "" { + quality, err := strconv.Atoi(os.Getenv("WEBP_QUALITY")) + if err != nil { + log.Warnf("WEBP_QUALITY is not a valid integer, using value in config.json %d", Config.Quality) + } else { + Config.Quality = quality + } + } + if os.Getenv("WEBP_ALLOWED_TYPES") != "" { + Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",") + } + if os.Getenv("WEBP_ENABLE_AVIF") != "" { + enableAVIF := os.Getenv("WEBP_ENABLE_AVIF") + if enableAVIF == "true" { + Config.EnableAVIF = true + } else if enableAVIF == "false" { + Config.EnableAVIF = false + } else { + log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF) + } + } + if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" { + enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") + if enableExtraParams == "true" { + Config.EnableExtraParams = true + } else if enableExtraParams == "false" { + Config.EnableExtraParams = false + } else { + log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams) + } + } + if os.Getenv("WEBP_IMG_MAP") != "" { + // TODO + } + + log.Debugln("Config init complete") + log.Debugln("Config", Config) } func parseImgMap(imgMap map[string]string) map[string]string { diff --git a/encoder/jpgconvert.go b/encoder/rawconvert.go similarity index 100% rename from encoder/jpgconvert.go rename to encoder/rawconvert.go diff --git a/encoder/rawconvert_test.go b/encoder/rawconvert_test.go new file mode 100644 index 000000000..33277dbb7 --- /dev/null +++ b/encoder/rawconvert_test.go @@ -0,0 +1,26 @@ +package encoder + +import ( + "testing" +) + +func TestConvertRawToJPG(t *testing.T) { + testCases := []struct { + rawPath string + optimizedPath string + expectedResult string + expectedStatus bool + }{ + // blackbird.NEF is from https://github.com/jewright/nef-to-jpg/blob/main/photoconverter/Sample-Images/blackbird.NEF + {"../pics/blackbird.NEF", "../exhaust_test/", "../exhaust_test/blackbird.NEF_extracted.jpg", true}, + {"../pics/big.jpg", "../exhaust_test/", "../pics/big.jpg", false}, + } + + for _, tc := range testCases { + result, status := ConvertRawToJPG(tc.rawPath, tc.optimizedPath) + + if result != tc.expectedResult || status != tc.expectedStatus { + t.Errorf("ConvertRawToJPG(%s, %s) => (%s, %t), expected (%s, %t)", tc.rawPath, tc.optimizedPath, result, status, tc.expectedResult, tc.expectedStatus) + } + } +} diff --git a/go.mod b/go.mod index 4323bac87..85ccf2321 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module webp_server_go -go 1.20 +go 1.21 require ( github.com/cespare/xxhash v1.1.0 github.com/davidbyttow/govips/v2 v2.13.0 - github.com/gofiber/fiber/v2 v2.49.2 + github.com/gofiber/fiber/v2 v2.50.0 github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c github.com/jeremytorres/rawparser v1.0.2 @@ -30,9 +30,9 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 76e4fc674..8c931061d 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM= github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw= -github.com/gofiber/fiber/v2 v2.49.2 h1:ONEN3/Vc+dUCxxDgZZwpqvhISgHqb+bu+isBiEyKEQs= -github.com/gofiber/fiber/v2 v2.49.2/go.mod h1:gNsKnyrmfEWFpJxQAV0qvW6l70K1dZGno12oLtukcts= +github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= +github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3aAzCLddcoxzboIrOm6/FVRXjp/5JM= @@ -73,8 +73,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -86,19 +86,20 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/pics/blackbird.NEF b/pics/blackbird.NEF new file mode 100644 index 000000000..4f4e28deb Binary files /dev/null and b/pics/blackbird.NEF differ