WebP Server Go 0.13.0 (#367)

* Init 0.13.0

* Copy image to exhaust if type match

* Correctly handle Firefox requests

* render webp on brower without support for heic

* silent curl

* Add support for Meta request

* Always return supported image format

* bump deps

* install libvips in ci runtime (#370)

* install libvips in ci runtime

* Run CI on every PR

* CI test in Docker

* Update Dockerfile.CI

* bump deps version

* uppercase AS

* disable export two functions

.dockerignore

@@ -1,10 +1,8 @@
 .github/*
 exhaust/*
-pics/*
 remote-raw/*
 scripts/*
 .git
 .dockerignore
 README.md
-LICENSE
-Makefile
+LICENSE

.github/workflows/CI.yaml

@@ -1,35 +1,14 @@
 name: CI check on every PR
 on:
   pull_request:
-    branches:
-      - master
     paths-ignore:
       - '**.md'
       - 'Makefile'
       - 'config.json'
 
 jobs:
-  ci:
-    name: CI check on every push and PR
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.23'
-
-      - name: Setup necessary packages
-        run: |
-          sudo apt update && sudo apt install libvips-dev -y
-
-      - name: run test cases
-        run: make && make test
-
   image-test:
-    name: Check for image build and CVE
+    name: Check for image build, testing and CVE
     runs-on: ubuntu-latest
     permissions: 
       pull-requests: write 
@@ -42,25 +21,23 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Cache Docker layers
-        uses: actions/cache@v2
+      - name: Make test
+        uses: docker/build-push-action@v6
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+          context: .
+          file: ./Dockerfile.CI
+          load: true
+          tags: |
+            ghcr.io/${{ github.event.repository.full_name }}:latest
 
-      - name: Build and load image
-        uses: docker/build-push-action@v3
+      - name: Build image test
+        uses: docker/build-push-action@v6
         with:
           context: .
           load: true
           tags: |
             ghcr.io/${{ github.event.repository.full_name }}:latest
 
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
-
       - name: Install trivy
         run: |
           wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -

.github/workflows/integration-test.yaml

@@ -22,7 +22,7 @@ jobs:
           sudo apt-get install -y docker-compose
 
       - name: Build image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -37,7 +37,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |
@@ -56,7 +56,7 @@ jobs:
           sudo apt-get install -y docker-compose
           
       - name: Build image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -71,7 +71,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |

.github/workflows/release_docker_image.yaml

@@ -57,7 +57,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Build and push latest images
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23-bookworm as builder
+FROM golang:1.23-bookworm AS builder
 
 ARG IMG_PATH=/opt/pics
 ARG EXHAUST_PATH=/opt/exhaust

Dockerfile.CI

@@ -0,0 +1,11 @@
+FROM golang:1.23-bookworm AS builder
+
+ARG IMG_PATH=/opt/pics
+ARG EXHAUST_PATH=/opt/exhaust
+RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build
+COPY go.mod /build
+RUN cd /build && go mod download
+
+COPY . /build
+
+RUN cd /build && make && make test

config/config.go

@@ -43,26 +43,40 @@ const (
 )
 
 var (
-	ConfigPath         string
-	Jobs               int
-	DumpSystemd        bool
-	DumpConfig         bool
-	ShowVersion        bool
-	ProxyMode          bool
-	Prefetch           bool // Prefech in go-routine, with WebP Server Go launch normally
-	PrefetchForeground bool // Standalone prefetch, prefetch and exit
-	Config             = NewWebPConfig()
-	Version            = "0.12.3"
-	WriteLock          = cache.New(5*time.Minute, 10*time.Minute)
-	ConvertLock        = cache.New(5*time.Minute, 10*time.Minute)
-	LocalHostAlias     = "local"
-	RemoteCache        *cache.Cache
+	ConfigPath          string
+	Jobs                int
+	DumpSystemd         bool
+	DumpConfig          bool
+	ShowVersion         bool
+	ProxyMode           bool
+	Prefetch            bool // Prefech in go-routine, with WebP Server Go launch normally
+	PrefetchForeground  bool // Standalone prefetch, prefetch and exit
+	AllowNonImage       bool
+	Config              = NewWebPConfig()
+	Version             = "0.13.0"
+	WriteLock           = cache.New(5*time.Minute, 10*time.Minute)
+	ConvertLock         = cache.New(5*time.Minute, 10*time.Minute)
+	LocalHostAlias      = "local"
+	RemoteCache         *cache.Cache
+	DefaultAllowedTypes = []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp", "avif", "jxl"} // Default allowed image types
 )
 
+type ImageMeta struct {
+	Width      int    `json:"width"`
+	Height     int    `json:"height"`
+	Format     string `json:"format"`
+	Size       int    `json:"size"`
+	NumPages   int    `json:"num_pages"`
+	Blurhash   string `json:"blurhash"`
+	Colorspace string `json:"colorspace"`
+}
+
 type MetaFile struct {
 	Id       string `json:"id"`       // hash of below path️, also json file name id.webp
 	Path     string `json:"path"`     // local: path with width and height, proxy: full url
 	Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes
+
+	ImageMeta
 }
 
 type WebpConfig struct {
@@ -94,12 +108,15 @@ type WebpConfig struct {
 }
 
 func NewWebPConfig() *WebpConfig {
+	// Copy DefaultAllowedTypes to avoid modification
+	defaultAllowedTypes := make([]string, len(DefaultAllowedTypes))
+	copy(defaultAllowedTypes, DefaultAllowedTypes)
 	return &WebpConfig{
 		Host:          "0.0.0.0",
 		Port:          "3333",
 		ImgPath:       "./pics",
 		Quality:       80,
-		AllowedTypes:  []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
+		AllowedTypes:  defaultAllowedTypes,
 		ConvertTypes:  []string{"webp"},
 		ImageMap:      map[string]string{},
 		ExhaustPath:   "./exhaust",

encoder/encoder.go

@@ -33,6 +33,14 @@ func init() {
 	intMinusOne.Set(-1)
 }
 
+func loadImage(filename string) (*vips.ImageRef, error) {
+	img, err := vips.LoadImageFromFile(filename, &vips.ImportParams{
+		FailOnError: boolFalse,
+		NumPages:    intMinusOne,
+	})
+	return img, err
+}
+
 func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) {
 	// Wait for the conversion to complete and return the converted image
 	retryDelay := 100 * time.Millisecond // Initial retry delay
@@ -122,30 +130,40 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
 	}
 
 	// Image is only opened here
-	img, err := vips.LoadImageFromFile(rawPath, &vips.ImportParams{
-		FailOnError: boolFalse,
-		NumPages:    intMinusOne,
-	})
-	if err != nil {
-		log.Warnf("Can't open source image: %v", err)
-		return err
-	}
+	img, err := loadImage(rawPath)
 	defer img.Close()
 
 	// Pre-process image(auto rotate, resize, etc.)
 	err = preProcessImage(img, imageType, extraParams)
 	if err != nil {
 		log.Warnf("Can't pre-process source image: %v", err)
-		return err
 	}
 
+	// If image is already in the target format, just copy it
+	imageFormat := img.Format()
+
 	switch imageType {
 	case "webp":
-		err = webpEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeWEBP {
+			log.Infof("Image is already in WebP format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = webpEncoder(img, rawPath, optimizedPath)
+		}
 	case "avif":
-		err = avifEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeAVIF {
+			log.Infof("Image is already in AVIF format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = avifEncoder(img, rawPath, optimizedPath)
+		}
 	case "jxl":
-		err = jxlEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeJXL {
+			log.Infof("Image is already in JXL format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = jxlEncoder(img, rawPath, optimizedPath)
+		}
 	}
 
 	return err

encoder/prefetch.go

@@ -33,9 +33,18 @@ func PrefetchImages() {
 			if info.IsDir() {
 				return nil
 			}
-			if !helper.CheckAllowedType(picAbsPath) {
+			// Only convert files with image extensions, use smaller of config.DefaultAllowedTypes and config.Config.AllowedTypes
+			if helper.CheckAllowedExtension(picAbsPath) {
+				// File type is allowed by user, check if it is an image
+				if helper.CheckImageExtension(picAbsPath) {
+					// File is an image, continue
+				} else {
+					return nil
+				}
+			} else {
 				return nil
 			}
+
 			// RawImagePath string, ImgFilename string, reqURI string
 			metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
 			avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)

encoder/process.go

@@ -114,6 +114,7 @@ func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
 
 	img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
 		FailOnError: boolFalse,
+		NumPages:    intMinusOne,
 	})
 	if err != nil {
 		log.Warnf("Could not load %s: %s", raw, err)
@@ -152,13 +153,18 @@ func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.Ex
 		}
 	}
 
-	// Auto rotate
-	err := img.AutoRotate()
-	if err != nil {
-		return err
-	}
 	if config.Config.EnableExtraParams {
-		err = resizeImage(img, extraParams)
+		err := resizeImage(img, extraParams)
+		if err != nil {
+			return err
+		}
+	}
+	// Skip auto rotate for GIF/WebP
+	if img.Format() == vips.ImageTypeGIF || img.Format() == vips.ImageTypeWEBP {
+		return nil
+	} else {
+		// Auto rotate
+		err := img.AutoRotate()
 		if err != nil {
 			return err
 		}

go.mod

@@ -3,12 +3,14 @@ module webp_server_go
 go 1.23
 
 require (
+	github.com/buckket/go-blurhash v1.1.0
 	github.com/cespare/xxhash v1.1.0
 	github.com/davidbyttow/govips/v2 v2.15.0
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/jeremytorres/rawparser v1.0.2
+	github.com/mileusna/useragent v1.3.5
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/schollz/progressbar/v3 v3.17.1
 	github.com/sirupsen/logrus v1.9.3
@@ -30,10 +32,10 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.27.0 // indirect
-	golang.org/x/term v0.26.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )