diff --git a/Dockerfile b/Dockerfile index 3e3c881..3060cc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ### # Client ### -FROM node:16-alpine3.14 as node-builder +FROM node:18-alpine as node-builder WORKDIR /ui # install deps @@ -17,7 +17,7 @@ RUN npm run build ### # Server ### -FROM golang:1.17-alpine AS go-builder +FROM golang:1-alpine AS go-builder # RUN apk add --no-cache gcc libffi-dev musl-dev libjpeg-turbo-dev WORKDIR /go/src/app @@ -30,6 +30,7 @@ RUN go mod download COPY *.go ./ COPY defaults.yaml ./ COPY internal ./internal +COPY io ./io COPY db ./db COPY fonts ./fonts # RUN go install -tags libjpeg . @@ -41,9 +42,9 @@ RUN go install -tags embedstatic . ### # Runtime ### -FROM alpine:3.14 +FROM alpine:latest # RUN apk add --no-cache exiftool>12.06-r0 libjpeg-turbo -RUN apk add --no-cache exiftool>12.06-r0 +RUN apk add --no-cache exiftool ffmpeg COPY --from=go-builder /go/bin/ /app diff --git a/Dockerfile-goreleaser b/Dockerfile-goreleaser index 56113b7..b0b6e6d 100644 --- a/Dockerfile-goreleaser +++ b/Dockerfile-goreleaser @@ -1,5 +1,5 @@ -FROM alpine:3.14 -RUN apk add --no-cache exiftool>12.06-r0 +FROM alpine:latest +RUN apk add --no-cache exiftool ffmpeg WORKDIR /app COPY photofield ./ diff --git a/api.yaml b/api.yaml index 582927f..0d2bd10 100644 --- a/api.yaml +++ b/api.yaml @@ -624,10 +624,11 @@ components: TaskType: type: string enum: - - INDEX - - LOAD_META - - LOAD_COLOR - - LOAD_AI + - INDEX_FILES + - INDEX_METADATA + - INDEX_CONTENTS + - INDEX_CONTENTS_COLOR + - INDEX_CONTENTS_AI CollectionId: type: string diff --git a/db/migrations-thumbs/000001_init.down.sql b/db/migrations-thumbs/000001_init.down.sql new file mode 100644 index 0000000..0c5180f --- /dev/null +++ b/db/migrations-thumbs/000001_init.down.sql @@ -0,0 +1 @@ +DROP TABLE thumb256; diff --git a/db/migrations-thumbs/000001_init.up.sql b/db/migrations-thumbs/000001_init.up.sql new file mode 100644 index 0000000..b177b3f --- /dev/null +++ b/db/migrations-thumbs/000001_init.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE thumb256 ( + id INTEGER PRIMARY KEY, + created_at_unix INTEGER, + data BLOB +); diff --git a/defaults.yaml b/defaults.yaml index 7c75a9f..7f22540 100644 --- a/defaults.yaml +++ b/defaults.yaml @@ -69,7 +69,10 @@ media: max_size: 256Mi # File extensions to index on the file system - extensions: [".jpg", ".jpeg", ".png", ".mp4"] + extensions: [ + ".jpg", ".jpeg", ".png", ".avif", ".bmp", ".pam", ".ppm", ".jxl", ".exr", ".cr2", ".dng", + ".mp4", + ] # Used to extract dates from file names as a heuristic in case of missing or # metadata or metadata yet to be loaded. @@ -77,68 +80,111 @@ media: date_formats: ["20060201_150405"] images: # Extensions to use to understand a file to be an image - extensions: [".jpg", ".jpeg", ".png", ".gif"] + # extensions: [".jpg", ".jpeg", ".png", ".gif"] + extensions: [".jpg", ".jpeg", ".png", ".avif", ".bmp", ".pam", ".ppm", ".jxl", ".exr", ".cr2", ".dng"] videos: extensions: [".mp4"] - # Pre-generated thumbnail configuration, these thumbnails will be used to - # greatly speed up the rendering - thumbnails: - - # name: Short thumbnail type name - # - # path: Path template where to find the thumbnail. - # {{.Dir}} is replaced by the parent directory of the original photo - # {{.Filename}} is replaced by the original photo filename - # - # fit: Aspect ratio fit of the thumbnail in case it doesn't match the - # original photo. - # - # INSIDE - # The thumbnail size is the maximum size of each dimension, so in case - # of different aspect ratios, one dimension will always be smaller. - # - # OUTSIDE - # The thumbnail size is the minimum size of each dimension, so in case - # of different aspect ratios, one dimension will always be bigger. - # - # ORIGINAL - # The size of the thumbnail is equal the size of the original. Mostly - # useful for transcoded or differently encoded files. - # - # width - # height: Predefined thumbnail dimensions used to pick the most - # appropriately-sized thumbnail for the zoom level. - # - # extra_cost: Additional weight to add when picking the closest thumbnail. - # Higher number means that other thumbnails will be preferred - # if they exist. - - # - # Embedded JPEG thumbnail - # - - name: exif-thumb - exif: ThumbnailImage + # + # Media source configuration + # + # Configures the different ways to load originals, photo variants, + # thumbnails, transcoded images, etc. For each photo, during loading or + # rendering, the most appropriate source is selected. The criteria include + # desired and expected dimensions, expected loading time, and possibly others. + # + # As a result, this configuration can have a large effect on the speed of + # the application. + # + # The following source types are supported: + # SQLITE, GOEXIF, THUMB, IMAGE, FFMPEG + # + # Common properties include: + # + # type: Case-insensitive source type (see above) + # + # extensions: File extensions supported by this source - the source will be + # skipped during selection if unsupported + # + # width, height: Expected dimensions of the provided images. These can be + # approximate in case it's not possible to know them upfront. + # Used during source selection to decide which source to pick. + # Some sources can have either hard-coded dimensions (e.g. sqlite) + # or assume the dimensions of the source file instead (e.g. image). + # + # fit: Aspect ratio fit of the provided image in case it doesn't match the + # original file. Not supported by all sources. + # + # INSIDE + # The thumbnail size is the maximum size of each dimension, so in case + # of different aspect ratios, one dimension will always be smaller. + # + # OUTSIDE + # The thumbnail size is the minimum size of each dimension, so in case + # of different aspect ratios, one dimension will always be bigger. + # + # ORIGINAL + # The size of the thumbnail is equal the size of the original. Mostly + # useful for transcoded or differently encoded files. + # + # + # Additional per-source properties are: + # + # SQLITE - internal thumbnail database + # type: sqlite + # path: File path relative to the data dir of the sqlite database containing + # the thumbnails + # + # GOEXIF - thumbnails embedded in JPEG EXIF metadata + # type: goexif + # + # IMAGE - native image loading + # type: image + # width, height: Resize the image after loading (can be slow) + # extensions: Supported source file extensions + # + # THUMB - pregenerated thumbnail files + # name: Short thumbnail type name + # path: Path template where to find the thumbnail + # {{.Dir}} is replaced by the parent directory of the original photo + # {{.Filename}} is replaced by the original photo filename + # + # FFMPEG - transcoded images via FFmpeg + # width, height: Dimensions to which the source media should be resized + # fit: The aspect ratio fit to use while resizing + # path: Path to the FFmpeg binary, uses the one in PATH if not set + # + sources: + + # Internal thumbnail database + - type: sqlite + path: photofield.thumbs.db + + # Embedded JPEG thumbnails + - type: goexif extensions: [".jpg", ".jpeg"] - fit: INSIDE - width: 120 - height: 120 - # It's expensive to extract, so this makes it more of a last resort, but - # still a lot better than loading the original photo. - extra_cost: 10 - + width: 256 + height: 256 + fit: "INSIDE" + + # Native image decoding + - type: image + extensions: [".jpg", ".jpeg", ".png"] + # # Synology Moments / Photo Station thumbnails # - name: S + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_S.jpg" extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] - fit: INSIDE + fit: "INSIDE" width: 120 height: 120 - name: SM + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_SM.jpg" extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] fit: OUTSIDE @@ -146,6 +192,7 @@ media: height: 240 - name: M + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_M.jpg" extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] fit: OUTSIDE @@ -153,6 +200,7 @@ media: height: 320 - name: B + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_B.jpg" extensions: [".jpg", ".jpeg", ".png", ".gif"] fit: INSIDE @@ -160,6 +208,7 @@ media: height: 640 - name: XL + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_XL.jpg" extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] fit: OUTSIDE @@ -170,6 +219,7 @@ media: # Synology Moments / Photo Station video variants # - name: FM + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_FILM_M.mp4" extensions: [".mp4"] fit: OUTSIDE @@ -177,6 +227,89 @@ media: height: 720 - name: H264 + type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_FILM_H264.mp4" extensions: [".mp4"] fit: ORIGINAL + + # + # FFmpeg on-the-fly decoding + # + - type: ffmpeg + width: 256 + height: 256 + fit: INSIDE + + - type: ffmpeg + width: 1280 + height: 1280 + fit: INSIDE + + - type: ffmpeg + width: 4096 + height: 4096 + fit: INSIDE + + + # These sources are used for handling small thumbnails specifically for + # specific purposes. + thumbnail: + + # Thumbnail sources used for extracting colors and AI embeddings + # 200 - 300px is likely ideal as it's small enough to process quickly, + # but big enough to retain some details. + # + # The sources here are ordered and the first source that returns + # a valid image is used. + sources: + # Internal thumbnail database + - type: sqlite + path: photofield.thumbs.db + + # Synology Moments / Photo Station thumbnail + - name: SM + type: thumb + path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_SM.jpg" + extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] + fit: OUTSIDE + width: 240 + height: 240 + + # Embedded JPEG thumbnails + - type: goexif + extensions: [".jpg", ".jpeg"] + width: 256 + height: 256 + fit: "INSIDE" + + # Synology Moments / Photo Station thumbnail + - name: S + type: thumb + path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_S.jpg" + extensions: [".jpg", ".jpeg", ".png", ".gif", ".mp4"] + fit: "INSIDE" + width: 120 + height: 120 + + + # If a thumbnail is not found among the sources above, + # it is generated with the first working generator. + generators: + + # FFmpeg decoding + - type: ffmpeg + width: 256 + height: 256 + fit: INSIDE + + # Native decoding (resized to 256px x 256px) + - type: image + extensions: [".jpg", ".jpeg", ".png"] + width: 256 + height: 256 + + # The sink is used to save the generated thumbnail above + # so that it persists and can be reused while rendering. + sink: + type: sqlite + path: photofield.thumbs.db diff --git a/docker-compose.yaml b/docker-compose.yaml index ebacffb..0124e49 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,6 +10,7 @@ services: - "traefik.http.services.photofield.loadbalancer.server.port=8080" volumes: - ./data/configuration.docker.yaml:/app/data/configuration.yaml:ro + - ./photos:/photos:ro restart: "no" prometheus: diff --git a/docker/grafana/dashboards/photofield.json b/docker/grafana/dashboards/photofield.json index 0120a55..9e6bc7a 100644 --- a/docker/grafana/dashboards/photofield.json +++ b/docker/grafana/dashboards/photofield.json @@ -15,7 +15,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1654108832702, + "iteration": 1674068728123, "links": [], "panels": [ { @@ -275,13 +275,82 @@ "type": "stat" }, { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateTurbo", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "Prometheus", + "description": "", "gridPos": { - "h": 1, + "h": 7, "w": 24, "x": 0, "y": 14 }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 166, + "legend": { + "show": false + }, + "maxDataPoints": 100, + "pluginVersion": "8.0.6", + "repeat": "source", + "repeatDirection": "v", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(pf_source_latency_bucket{source=\"$source\"}[10s])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Latency for $source", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, "id": 56, "title": "System", "type": "row" @@ -315,7 +384,7 @@ "h": 5, "w": 6, "x": 0, - "y": 15 + "y": 50 }, "id": 38, "options": { @@ -375,7 +444,7 @@ "h": 5, "w": 6, "x": 6, - "y": 15 + "y": 50 }, "id": 25, "options": { @@ -431,7 +500,7 @@ "h": 5, "w": 12, "x": 12, - "y": 15 + "y": 50 }, "id": 33, "options": { @@ -500,7 +569,7 @@ "h": 5, "w": 12, "x": 0, - "y": 20 + "y": 55 }, "id": 37, "options": { @@ -568,7 +637,7 @@ "h": 5, "w": 12, "x": 12, - "y": 20 + "y": 55 }, "id": 35, "options": { @@ -632,7 +701,7 @@ "h": 6, "w": 12, "x": 0, - "y": 25 + "y": 60 }, "id": 75, "options": { @@ -700,7 +769,7 @@ "h": 6, "w": 12, "x": 12, - "y": 25 + "y": 60 }, "id": 92, "options": { @@ -741,7 +810,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 66 }, "id": 12, "panels": [ @@ -1192,7 +1261,7 @@ { "allValue": null, "current": { - "selected": true, + "selected": false, "text": [ "All" ], @@ -1287,7 +1356,7 @@ { "allValue": null, "current": { - "selected": true, + "selected": false, "text": "/scenes/{scene_id}/tiles", "value": "/scenes/{scene_id}/tiles" }, @@ -1310,6 +1379,33 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "label_values(source)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "source", + "options": [], + "query": { + "query": "label_values(source)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -1336,5 +1432,5 @@ "timezone": "", "title": "Photofield", "uid": "9sQ5hGGnk", - "version": 6 + "version": 7 } \ No newline at end of file diff --git a/go.mod b/go.mod index 26c30c4..857a3f6 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/tdewolff/canvas v0.0.0-20200504121106-e2600b35c365 github.com/x448/float16 v0.8.4 golang.org/x/image v0.0.0-20191214001246-9130b4cfad52 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c zombiezen.com/go/sqlite v0.10.1 ) @@ -46,6 +47,7 @@ require ( github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.7 // indirect github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gosimple/unidecode v1.0.0 // indirect diff --git a/go.sum b/go.sum index b03e721..9307701 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -780,6 +781,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/image/cache.go b/internal/image/cache.go index 5e5420a..d1e4566 100644 --- a/internal/image/cache.go +++ b/internal/image/cache.go @@ -1,11 +1,7 @@ package image import ( - "image" - "log" "photofield/internal/metrics" - "reflect" - "time" "unsafe" "github.com/dgraph-io/ristretto" @@ -48,102 +44,6 @@ func newInfoCache() InfoCache { } } -type ImageCache struct { - cache *ristretto.Cache -} - -type ImageLoader interface { - Acquire(key string, path string, thumbnail *Thumbnail) (image.Image, Info, error) - Release(key string) -} - -func newImageCache(caches Caches) ImageCache { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e6, // number of keys to track frequency of - MaxCost: caches.Image.MaxSizeBytes(), // maximum cost of cache - BufferItems: 64, // number of keys per Get buffer - Metrics: true, - Cost: func(value interface{}) int64 { - imageRef := value.(imageRef) - img := imageRef.image - if img == nil { - return 1 - } - switch img := img.(type) { - - case *image.YCbCr: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Y))*int64(unsafe.Sizeof(img.Y[0])) + - int64(cap(img.Cb))*int64(unsafe.Sizeof(img.Cb[0])) + - int64(cap(img.Cr))*int64(unsafe.Sizeof(img.Cr[0])) - - case *image.Gray: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) - - case *image.NRGBA: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) - - case *image.RGBA: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) - - case *image.CMYK: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) - - case *image.Paletted: - return int64(unsafe.Sizeof(*img)) + - int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + - int64(cap(img.Palette))*int64(unsafe.Sizeof(img.Pix[0])) - - case nil: - return 1 - - default: - log.Printf("Unable to compute cost, unsupported image format %v", reflect.TypeOf(img)) - // Fallback image size (10MB) - return 10000000 - } - }, - }) - if err != nil { - panic(err) - } - metrics.AddRistretto("image_cache", cache) - return ImageCache{ - cache: cache, - } -} - -func (c *ImageCache) GetOrLoad(path string, thumbnail *Thumbnail, loader ImageLoader) (image.Image, Info, error) { - key := path - if thumbnail != nil { - key += "|thumbnail|" + thumbnail.Name - } - - value, found := c.cache.Get(key) - if found { - imageRef := value.(imageRef) - return imageRef.image, imageRef.info, imageRef.err - } - - image, info, err := loader.Acquire(key, path, thumbnail) - imageRef := imageRef{ - image: image, - info: info, - err: err, - } - c.cache.SetWithTTL(key, imageRef, 0, 10*time.Minute) - loader.Release(key) - return image, info, err -} - -func (c *ImageCache) Delete(path string) { - c.cache.Del(path) -} - func newFileExistsCache() *ristretto.Cache { cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: 1e7, // number of keys to track frequency of (10M). diff --git a/internal/image/color.go b/internal/image/color.go new file mode 100644 index 0000000..b33e07a --- /dev/null +++ b/internal/image/color.go @@ -0,0 +1,25 @@ +package image + +import ( + "image" + "image/color" + + "github.com/EdlinOrg/prominentcolor" +) + +func extractProminentColor(img image.Image) (color.RGBA, error) { + centroids, err := prominentcolor.KmeansWithAll(1, img, prominentcolor.ArgumentDefault, prominentcolor.DefaultSize, prominentcolor.GetDefaultMasks()) + if err != nil { + centroids, err = prominentcolor.KmeansWithAll(1, img, prominentcolor.ArgumentDefault, prominentcolor.DefaultSize, make([]prominentcolor.ColorBackgroundMask, 0)) + if err != nil { + return color.RGBA{}, err + } + } + promColor := centroids[0] + return color.RGBA{ + A: 0xFF, + R: uint8(promColor.Color.R), + G: uint8(promColor.Color.G), + B: uint8(promColor.Color.B), + }, nil +} diff --git a/internal/image/database.go b/internal/image/database.go index 2ab12c4..d238fcd 100644 --- a/internal/image/database.go +++ b/internal/image/database.go @@ -153,7 +153,7 @@ func (source *Database) migrate(migrations embed.FS) { if dirty { dirtystr = " (dirty)" } - log.Printf("database version %v%s, migrating if needed", version, dirtystr) + log.Printf("cache database version %v%s, migrating if needed", version, dirtystr) err = m.Up() if err != nil && err != migrate.ErrNoChange { @@ -602,14 +602,24 @@ func (source *Database) WaitForCommit() { defer source.transactionMutex.RUnlock() } -func (source *Database) DeleteNonexistent(dir string, m map[string]struct{}) { +func (source *Database) ListNonexistent(dir string, paths map[string]struct{}) <-chan IdPath { source.WaitForCommit() - // TODO delete prefixes - for path := range source.ListPaths([]string{dir}, 0) { - _, exists := m[path] - if !exists { - source.Write(path, Info{}, Delete) + out := make(chan IdPath, 1000) + go func() { + for ip := range source.ListIdPaths([]string{dir}, 0) { + _, exists := paths[ip.Path] + if !exists { + out <- ip + } } + close(out) + }() + return out +} + +func (source *Database) DeleteNonexistent(dir string, paths map[string]struct{}) { + for ip := range source.ListNonexistent(dir, paths) { + source.Write(ip.Path, Info{}, Delete) } } @@ -850,6 +860,72 @@ func (source *Database) ListPaths(dirs []string, limit int) <-chan string { return out } +func (source *Database) ListIdPaths(dirs []string, limit int) <-chan IdPath { + out := make(chan IdPath, 10000) + go func() { + defer metrics.Elapsed("list id paths sqlite")() + + conn := source.pool.Get(nil) + defer source.pool.Put(conn) + + sql := ` + SELECT infos.id, str || filename as path + FROM infos + JOIN prefix ON path_prefix_id == prefix.id + WHERE path_prefix_id IN ( + SELECT id + FROM prefix + WHERE + ` + + for i := range dirs { + sql += `str LIKE ? ` + if i < len(dirs)-1 { + sql += "OR " + } + } + + sql += ` + ) + ` + + if limit > 0 { + sql += `LIMIT ? ` + } + + sql += ";" + + stmt := conn.Prep(sql) + bindIndex := 1 + defer stmt.Reset() + + for _, dir := range dirs { + stmt.BindText(bindIndex, dir+"%") + bindIndex++ + } + + if limit > 0 { + stmt.BindInt64(bindIndex, (int64)(limit)) + } + + for { + if exists, err := stmt.Step(); err != nil { + log.Printf("Error listing files: %s\n", err.Error()) + } else if !exists { + break + } + ip := IdPath{ + Id: ImageId(stmt.ColumnInt64(0)), + Path: stmt.ColumnText(1), + } + out <- ip + } + + close(out) + }() + return out +} + func (source *Database) ListIds(dirs []string, limit int, missingEmbedding bool) <-chan ImageId { out := make(chan ImageId, 10000) go func() { @@ -925,3 +1001,162 @@ func (source *Database) ListIds(dirs []string, limit int, missingEmbedding bool) }() return out } + +func (source *Database) ListMissing(dirs []string, limit int, opts Missing) <-chan MissingInfo { + out := make(chan MissingInfo, 1000) + go func() { + defer metrics.Elapsed("list missing sqlite")() + + conn := source.pool.Get(nil) + defer source.pool.Put(conn) + + sql := ` + SELECT infos.id, str || filename as path` + + type condition struct { + inputs []string + output string + } + + conds := make([]condition, 0) + if opts.Metadata { + conds = append(conds, condition{ + inputs: []string{ + "width", + "height", + "orientation", + "created_at_unix", + }, + output: "missing_metadata", + }) + } + if opts.Color { + conds = append(conds, condition{ + inputs: []string{"color"}, + output: "missing_color", + }) + } + if opts.Embedding { + conds = append(conds, condition{ + inputs: []string{"file_id"}, + output: "missing_embedding", + }) + } + + for _, c := range conds { + sql += `, + ` + for i, input := range c.inputs { + sql += fmt.Sprintf("%s IS NULL ", input) + if i < len(c.inputs)-1 { + sql += "OR " + } + } + sql += fmt.Sprintf("AS %s", c.output) + } + + sql += ` + FROM infos + INNER JOIN prefix ON prefix.id = path_prefix_id + ` + + if opts.Embedding { + sql += ` + LEFT JOIN clip_emb ON clip_emb.file_id = infos.id + ` + } + + sql += ` + WHERE + path_prefix_id IN ( + SELECT id + FROM prefix + WHERE + ` + + for i := range dirs { + sql += `str LIKE ? ` + if i < len(dirs)-1 { + sql += "OR " + } + } + + sql += ` + ) + ` + + if len(conds) > 0 { + sql += ` + AND ( + ` + + for i, c := range conds { + sql += fmt.Sprintf("%s ", c.output) + if i < len(conds)-1 { + sql += `OR + ` + } + } + // for i, c := range conds { + // for j, input := range c.inputs { + // sql += fmt.Sprintf("%s IS NULL ", input) + // if j < len(c.inputs)-1 { + // sql += "OR " + // } + // } + // if i < len(conds)-1 { + // sql += "OR " + // } + // sql += ` + // ` + // } + sql += ` + ) + ` + } + + if limit > 0 { + sql += `LIMIT ? ` + } + + sql += ";" + + stmt := conn.Prep(sql) + bindIndex := 1 + defer stmt.Reset() + + for _, dir := range dirs { + stmt.BindText(bindIndex, dir+"%") + bindIndex++ + } + + if limit > 0 { + stmt.BindInt64(bindIndex, (int64)(limit)) + } + + for { + if exists, err := stmt.Step(); err != nil { + log.Printf("Error listing files: %s\n", err.Error()) + } else if !exists { + break + } + r := MissingInfo{ + Id: (ImageId)(stmt.ColumnInt64(0)), + Path: stmt.ColumnText(1), + } + i := 2 + if opts.Color { + r.Color = stmt.ColumnBool(i) + i++ + } + if opts.Embedding { + r.Embedding = stmt.ColumnBool(i) + i++ + } + out <- r + } + + close(out) + }() + return out +} diff --git a/internal/image/image.go b/internal/image/image.go deleted file mode 100644 index 631d5e0..0000000 --- a/internal/image/image.go +++ /dev/null @@ -1,167 +0,0 @@ -package image - -import ( - "image" - "image/color" - "io" - "os" - "photofield/internal/codec" - "strings" - "time" - - "github.com/EdlinOrg/prominentcolor" -) - -type Size = image.Point - -type loadingImage struct { - imageRef imageRef - loaded chan struct{} -} - -type imageRef struct { - err error - info Info - image image.Image -} - -func (source *Source) Exists(path string) bool { - value, found := source.fileExistsCache.Get(path) - if found { - return value.(bool) - } - _, err := os.Stat(path) - - exists := !os.IsNotExist(err) - source.fileExistsCache.SetWithTTL(path, exists, 1, 1*time.Minute) - return exists -} - -func (source *Source) decode(path string, reader io.ReadSeeker) (image.Image, error) { - lower := strings.ToLower(path) - if strings.HasSuffix(lower, "jpg") || strings.HasSuffix(lower, "jpeg") { - image, err := codec.DecodeJpeg(reader) - return image, err - } - - image, _, err := image.Decode(reader) - return image, err -} - -func (source *Source) LoadImage(path string) (image.Image, error) { - // fmt.Printf("loading %s\n", path) - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - return source.decode(path, file) -} - -func (source *Source) Acquire(key string, path string, thumbnail *Thumbnail) (image.Image, Info, error) { - // log.Printf("%v acquire, %v\n", key, source.imagesLoadingCount) - source.imagesLoadingCount++ - loading := &loadingImage{} - loading.loaded = make(chan struct{}) - stored, loaded := source.imagesLoading.LoadOrStore(key, loading) - if loaded { - loading = stored.(*loadingImage) - // log.Printf("%v blocking on channel\n", key) - <-loading.loaded - // log.Printf("%v channel unblocked\n", key) - imageRef := loading.imageRef - return imageRef.image, Info{}, imageRef.err - } else { - // log.Printf("%v not found, loading, mutex locked\n", key) - var image image.Image - info := Info{} - var err error - if thumbnail == nil { - // log.Printf("%v loading\n", key) - image, err = source.LoadImage(path) - } else { - thumbnailPath := thumbnail.GetPath(path) - if thumbnailPath != "" { - // log.Printf("%v loading thumbnail path %v\n", key, thumbnailPath) - image, err = source.LoadImage(thumbnailPath) - } else { - // log.Printf("%v loading embedded %v\n", key, thumbnail.Exif) - image, info, err = source.decoder.DecodeImage(path, thumbnail.Exif) - } - } - imageRef := imageRef{ - image: image, - err: err, - } - loading.imageRef = imageRef - // log.Printf("%v loaded, closing channel\n", key) - close(loading.loaded) - return image, info, err - } -} - -func (source *Source) Release(key string) { - source.imagesLoading.Delete(key) - source.imagesLoadingCount-- - // log.Printf("%v loaded, map entry deleted\n", key) - // log.Printf("%v release, %v\n", key, source.imagesLoadingCount) -} - -func (source *Source) GetImage(path string) (image.Image, Info, error) { - return source.imageCache.GetOrLoad(path, nil, source) -} - -func (source *Source) GetImageOrThumbnail(path string, thumbnail *Thumbnail) (image.Image, Info, error) { - return source.imageCache.GetOrLoad(path, thumbnail, source) -} - -func (source *Source) OpenSmallestThumbnail(path string, minSize int) (*os.File, error) { - for i := range source.Thumbnails { - thumbnail := &source.Thumbnails[i] - if thumbnail.Width < minSize || thumbnail.Height < minSize { - continue - } - thumbnailPath := thumbnail.GetPath(path) - file, err := os.Open(thumbnailPath) - if err != nil { - continue - } - return file, nil - } - return os.Open(path) -} - -func (source *Source) LoadSmallestImage(path string) (image.Image, error) { - for i := range source.Thumbnails { - thumbnail := &source.Thumbnails[i] - thumbnailPath := thumbnail.GetPath(path) - file, err := os.Open(thumbnailPath) - if err != nil { - continue - } - defer file.Close() - return source.decode(thumbnailPath, file) - } - return source.LoadImage(path) -} - -func (source *Source) LoadImageColor(path string) (color.RGBA, error) { - colorImage, err := source.LoadSmallestImage(path) - if err != nil { - return color.RGBA{}, err - } - centroids, err := prominentcolor.KmeansWithAll(1, colorImage, prominentcolor.ArgumentDefault, prominentcolor.DefaultSize, prominentcolor.GetDefaultMasks()) - if err != nil { - centroids, err = prominentcolor.KmeansWithAll(1, colorImage, prominentcolor.ArgumentDefault, prominentcolor.DefaultSize, make([]prominentcolor.ColorBackgroundMask, 0)) - if err != nil { - return color.RGBA{}, err - } - } - promColor := centroids[0] - return color.RGBA{ - A: 0xFF, - R: uint8(promColor.Color.R), - G: uint8(promColor.Color.G), - B: uint8(promColor.Color.B), - }, nil -} diff --git a/internal/image/indexContents.go b/internal/image/indexContents.go new file mode 100644 index 0000000..87f7125 --- /dev/null +++ b/internal/image/indexContents.go @@ -0,0 +1,136 @@ +package image + +import ( + "bytes" + "context" + "fmt" + "image" + goio "io" + "log" + "photofield/internal/clip" + "photofield/io" + "time" +) + +func (source *Source) indexContents(in <-chan interface{}) { + ctx := context.TODO() + for elem := range in { + + for source.metadataQueue.Length() > 0 { + time.Sleep(1 * time.Second) + } + + m := elem.(MissingInfo) + id := io.ImageId(m.Id) + path := m.Path + + done := false + for _, src := range source.thumbnailSources { + src.Reader(ctx, id, path, func(rs goio.ReadSeeker, err error) { + if err != nil { + return + } + + // log.Printf("index contents source %s path %s\n", src.(io.Source).Name(), path) + source.indexContentsReader(ctx, m, src, nil, rs) + done = true + }) + if done { + break + } + } + + // Generate thumbnail if none loaded + if !done { + // log.Printf("index contents generate %s\n", path) + img, rs, err := source.indexContentsGenerate(ctx, id, path) + if err != nil { + log.Println("Unable to generate image thumbnail", err) + continue + } + source.indexContentsReader(ctx, m, nil, img, rs) + } + } +} + +func (source *Source) indexContentsReader(ctx context.Context, m MissingInfo, src io.ReadDecoder, img image.Image, rs goio.ReadSeeker) { + var err error + if m.Color { + // Decode image if needed + if img == nil && rs != nil { + img, err = source.indexContentsDecode(ctx, src, rs) + if err != nil { + log.Println("Unable to decode image thumbnail", err) + } + } + + // Extract colors + if img != nil { + color, err := extractProminentColor(img) + if err != nil { + log.Println("Unable to extract image color", err, m.Path) + } else { + info := Info{} + info.SetColorRGBA(color) + source.database.Write(m.Path, info, UpdateColor) + source.imageInfoCache.Delete(m.Id) + } + } + } + + // Extract AI embedding + if m.Embedding && rs != nil { + embedding, err := source.Clip.EmbedImageReader(rs) + if err != clip.ErrNotAvailable { + if err != nil { + fmt.Println("Unable to get image embedding", err, m.Path) + } else { + source.database.WriteAI(m.Id, embedding) + } + } + } +} + +func (source *Source) indexContentsGenerate(ctx context.Context, id io.ImageId, path string) (image.Image, *bytes.Reader, error) { + errs := make([]error, 0) + for _, gen := range source.thumbnailGenerators { + // Generate thumbnail + r := gen.Get(ctx, id, path) + if r.Image == nil || r.Error != nil { + errs = append(errs, r.Error) + continue + } + + // Save thumbnail + var b bytes.Buffer + ok := source.thumbnailSink.SetWithBuffer(ctx, id, path, &b, r) + if !ok { + return r.Image, nil, fmt.Errorf("unable to save %s", path) + } + + // Return encoded bytes + rd := bytes.NewReader(b.Bytes()) + return r.Image, rd, nil + } + + e := "" + for _, err := range errs { + e += err.Error() + " " + } + return nil, nil, fmt.Errorf("all generators failed: %s: %s", e, path) +} + +func (source *Source) indexContentsDecode(ctx context.Context, d io.Decoder, rs goio.ReadSeeker) (image.Image, error) { + if d == nil { + return nil, fmt.Errorf("unable to decode, missing decoder") + } + r := d.Decode(ctx, rs) + if r.Error != nil { + return nil, fmt.Errorf("unable to decode %w", r.Error) + } + _, err := rs.Seek(0, goio.SeekStart) + if err != nil { + return nil, fmt.Errorf("unable to seek to start %w", err) + } + return r.Image, nil +} diff --git a/internal/image/index.go b/internal/image/indexFiles.go similarity index 100% rename from internal/image/index.go rename to internal/image/indexFiles.go diff --git a/internal/image/indexMetadata.go b/internal/image/indexMetadata.go new file mode 100644 index 0000000..a35d154 --- /dev/null +++ b/internal/image/indexMetadata.go @@ -0,0 +1,22 @@ +package image + +import ( + "fmt" +) + +func (source *Source) indexMetadata(in <-chan interface{}) { + for elem := range in { + m := elem.(MissingInfo) + id := m.Id + path := m.Path + + var info Info + err := source.decoder.DecodeInfo(path, &info) + if err != nil { + fmt.Println("Unable to load image info meta", err, path) + continue + } + source.database.Write(path, info, UpdateMeta) + source.imageInfoCache.Delete(id) + } +} diff --git a/internal/image/info.go b/internal/image/info.go index a3f77d8..5835950 100644 --- a/internal/image/info.go +++ b/internal/image/info.go @@ -2,10 +2,13 @@ package image import ( "fmt" + "image" "image/color" "time" ) +type Size = image.Point + type Info struct { Width, Height int DateTime time.Time diff --git a/internal/image/queue.go b/internal/image/queue.go deleted file mode 100644 index ead2a7d..0000000 --- a/internal/image/queue.go +++ /dev/null @@ -1,37 +0,0 @@ -package image - -func (source *Source) LoadInfo(path string) (Info, error) { - var info Info - var err error - err = source.decoder.DecodeInfo(path, &info) - if err != nil { - return info, err - } - - color, err := source.LoadImageColor(path) - if err != nil { - return info, err - } - info.SetColorRGBA(color) - - return info, nil -} - -func (source *Source) LoadInfoMeta(path string) (Info, error) { - var info Info - err := source.decoder.DecodeInfo(path, &info) - if err != nil { - return info, err - } - return info, nil -} - -func (source *Source) LoadInfoColor(path string) (Info, error) { - var info Info - color, err := source.LoadImageColor(path) - if err != nil { - return info, err - } - info.SetColorRGBA(color) - return info, nil -} diff --git a/internal/image/search.go b/internal/image/search.go index 0b66435..823bdde 100644 --- a/internal/image/search.go +++ b/internal/image/search.go @@ -63,6 +63,11 @@ func (source *Source) ListSimilar(dirs []string, embedding clip.Embedding, optio out := make(chan SimilarityInfo, 1000) go func() { defer metrics.Elapsed("list similar")() + defer close(out) + + if embedding == nil { + return + } // Prepare search term embedding similars := make([]similar, 0, 1000) @@ -152,8 +157,6 @@ func (source *Source) ListSimilar(dirs []string, embedding clip.Embedding, optio } } done() - - close(out) }() return out } diff --git a/internal/image/source.go b/internal/image/source.go index 1de1904..7ab239c 100644 --- a/internal/image/source.go +++ b/internal/image/source.go @@ -1,19 +1,28 @@ package image import ( + "context" "embed" "errors" + "fmt" "log" "path/filepath" "strings" - "sync" + + goio "io" "photofield/internal/clip" "photofield/internal/metrics" "photofield/internal/queue" + "photofield/io" + "photofield/io/ffmpeg" + ioristretto "photofield/io/ristretto" + "photofield/io/sqlite" "github.com/dgraph-io/ristretto" "github.com/docker/go-units" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ErrNotFound = errors.New("not found") @@ -32,11 +41,39 @@ func IdsToUint32(ids <-chan ImageId) <-chan uint32 { return out } +func MissingInfoToInterface(c <-chan MissingInfo) <-chan interface{} { + out := make(chan interface{}) + go func() { + for m := range c { + out <- interface{}(m) + } + close(out) + }() + return out +} + type SourcedInfo struct { Id ImageId Info } +type Missing struct { + Metadata bool + Color bool + Embedding bool +} + +type IdPath struct { + Id ImageId + Path string +} + +type MissingInfo struct { + Id ImageId + Path string + Missing +} + type SimilarityInfo struct { SourcedInfo Similarity float32 @@ -70,8 +107,8 @@ type Caches struct { } type Config struct { - DatabasePath string - AI clip.AI + DataDir string + AI clip.AI ExifToolCount int `json:"exif_tool_count"` SkipLoadInfo bool `json:"skip_load_info"` @@ -79,11 +116,12 @@ type Config struct { ConcurrentColorLoads int `json:"concurrent_color_loads"` ConcurrentAILoads int `json:"concurrent_ai_loads"` - ListExtensions []string `json:"extensions"` - DateFormats []string `json:"date_formats"` - Images FileConfig `json:"images"` - Videos FileConfig `json:"videos"` - Thumbnails []Thumbnail `json:"thumbnails"` + ListExtensions []string `json:"extensions"` + DateFormats []string `json:"date_formats"` + Images FileConfig `json:"images"` + Videos FileConfig `json:"videos"` + Sources SourceConfigs `json:"sources"` + Thumbnail ThumbnailConfig `json:"thumbnail"` Caches Caches `json:"caches"` } @@ -95,64 +133,111 @@ type FileConfig struct { type Source struct { Config + Sources io.Sources + SourcesLatencyHistogram *prometheus.HistogramVec + decoder *Decoder database *Database imageInfoCache InfoCache - imageCache ImageCache pathCache PathCache fileExistsCache *ristretto.Cache - imagesLoading sync.Map - imagesLoadingCount int + metadataQueue queue.Queue + contentsQueue queue.Queue - MetaQueue queue.Queue - ColorQueue queue.Queue + thumbnailSources []io.ReadDecoder + thumbnailGenerators io.Sources + thumbnailSink *sqlite.Source - Clip clip.Clip - AIQueue queue.Queue + Clip clip.Clip } -func NewSource(config Config, migrations embed.FS) *Source { +func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *Source { source := Source{} source.Config = config source.decoder = NewDecoder(config.ExifToolCount) - source.database = NewDatabase(config.DatabasePath, migrations) + source.database = NewDatabase(filepath.Join(config.DataDir, "photofield.cache.db"), migrations) source.imageInfoCache = newInfoCache() - source.imageCache = newImageCache(config.Caches) source.fileExistsCache = newFileExistsCache() source.pathCache = newPathCache() + source.SourcesLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Name: "source_latency", + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + }, + []string{"source"}, + ) + + env := SourceEnvironment{ + FFmpegPath: ffmpeg.FindPath(), + Migrations: migrationsThumbs, + ImageCache: ioristretto.New(), + DataDir: config.DataDir, + } + + // Sources used for rendering + srcs, err := config.Sources.NewSources(&env) + if err != nil { + log.Fatalf("failed to create sources: %s", err) + } + source.Sources = srcs + + // Further sources should not be cached + env.ImageCache = nil + + tsrcs, err := config.Thumbnail.Sources.NewSources(&env) + if err != nil { + log.Fatalf("failed to create thumbnail sources: %s", err) + } + for _, s := range tsrcs { + rd, ok := s.(io.ReadDecoder) + if !ok { + log.Fatalf("source %s does not implement io.ReadDecoder", s.Name()) + } + source.thumbnailSources = append(source.thumbnailSources, rd) + } + + gens, err := config.Thumbnail.Generators.NewSources(&env) + if err != nil { + log.Fatalf("failed to create thumbnail generators: %s", err) + } + source.thumbnailGenerators = gens + + sink, err := config.Thumbnail.Sink.NewSource(&env) + if err != nil { + log.Fatalf("failed to create thumbnail sink: %s", err) + } + sqliteSink, ok := sink.(*sqlite.Source) + if !ok { + log.Fatalf("thumbnail sink %s is not a sqlite source", sink.Name()) + } + source.thumbnailSink = sqliteSink + if config.SkipLoadInfo { log.Printf("skipping load info") } else { - source.MetaQueue = queue.Queue{ - ID: "load_meta", - Name: "load meta", - Worker: source.loadInfosMeta, + source.metadataQueue = queue.Queue{ + ID: "index_metadata", + Name: "index metadata", + Worker: source.indexMetadata, WorkerCount: config.ConcurrentMetaLoads, } - go source.MetaQueue.Run() - - source.ColorQueue = queue.Queue{ - ID: "load_color", - Name: "load color", - Worker: source.loadInfosColor, - WorkerCount: config.ConcurrentColorLoads, - } - go source.ColorQueue.Run() + go source.metadataQueue.Run() source.Clip = config.AI - if config.AI.Available() { - source.AIQueue = queue.Queue{ - ID: "load_ai", - Name: "load ai", - Worker: source.loadInfosAI, - WorkerCount: 8, - } - go source.AIQueue.Run() + // } + + source.contentsQueue = queue.Queue{ + ID: "index_contents", + Name: "index contents", + Worker: source.indexContents, + WorkerCount: 8, } + go source.contentsQueue.Run() + } return &source @@ -209,7 +294,71 @@ func (source *Source) ListMissingEmbeddingIds(dirs []string, maxPhotos int) <-ch return source.database.ListIds(dirs, maxPhotos, true) } +func (source *Source) ListMissingMetadata(dirs []string, maxPhotos int, force Missing) <-chan MissingInfo { + for i := range dirs { + dirs[i] = filepath.FromSlash(dirs[i]) + } + opts := Missing{ + Metadata: true, + } + if force.Metadata { + opts = Missing{} + } + out := make(chan MissingInfo) + go func() { + for m := range source.database.ListMissing(dirs, maxPhotos, opts) { + m.Metadata = m.Metadata || force.Metadata + out <- m + } + close(out) + }() + return out +} + +func (source *Source) ListMissingContents(dirs []string, maxPhotos int, force Missing) <-chan MissingInfo { + for i := range dirs { + dirs[i] = filepath.FromSlash(dirs[i]) + } + opts := Missing{ + Color: true, + Embedding: source.AI.Available(), + } + if force.Color || force.Embedding { + opts = Missing{} + } + out := make(chan MissingInfo) + go func() { + for m := range source.database.ListMissing(dirs, maxPhotos, opts) { + m.Color = m.Color || force.Color + m.Embedding = m.Embedding || force.Embedding + out <- m + } + close(out) + }() + return out +} + func (source *Source) ListInfos(dirs []string, options ListOptions) <-chan SourcedInfo { + for i := range dirs { + dirs[i] = filepath.FromSlash(dirs[i]) + } + out := make(chan SourcedInfo, 1000) + go func() { + defer metrics.Elapsed("list infos")() + + infos := source.database.List(dirs, options) + for info := range infos { + // if info.NeedsMeta() || info.NeedsColor() { + // info.Info = source.GetInfo(info.Id) + // } + out <- info.SourcedInfo + } + close(out) + }() + return out +} + +func (source *Source) ListInfosWithExistence(dirs []string, options ListOptions) <-chan SourcedInfo { for i := range dirs { dirs[i] = filepath.FromSlash(dirs[i]) } @@ -254,13 +403,20 @@ func (source *Source) IndexFiles(dir string, max int, counter chan<- int) { // time.Sleep(10 * time.Millisecond) counter <- 1 } - source.database.DeleteNonexistent(dir, indexed) + for ip := range source.database.ListNonexistent(dir, indexed) { + source.database.Write(ip.Path, Info{}, Delete) + source.thumbnailSink.Delete(uint32(ip.Id)) + } source.database.SetIndexed(dir) source.database.WaitForCommit() } -func (source *Source) IndexAI(dirs []string, maxPhotos int) { - source.AIQueue.AppendChan(IdsToUint32(source.ListMissingEmbeddingIds(dirs, maxPhotos))) +func (source *Source) IndexMetadata(dirs []string, maxPhotos int, force Missing) { + source.metadataQueue.AppendItems(MissingInfoToInterface(source.ListMissingMetadata(dirs, maxPhotos, force))) +} + +func (source *Source) IndexContents(dirs []string, maxPhotos int, force Missing) { + source.contentsQueue.AppendItems(MissingInfoToInterface(source.ListMissingContents(dirs, maxPhotos, force))) } func (source *Source) GetDir(dir string) Info { @@ -277,20 +433,35 @@ func (source *Source) GetDirsCount(dirs []string) int { return count } -func (source *Source) GetApplicableThumbnails(path string) []Thumbnail { - thumbs := make([]Thumbnail, 0, len(source.Thumbnails)) - pathExt := strings.ToLower(filepath.Ext(path)) - for _, t := range source.Thumbnails { - supported := false - for _, ext := range t.Extensions { - if pathExt == ext { - supported = true - break - } +func (source *Source) GetImageReader(id ImageId, sourceName string, fn func(r goio.ReadSeeker, err error)) { + ctx := context.TODO() + path, err := source.GetImagePath(id) + if err != nil { + fn(nil, err) + return + } + found := false + for _, s := range source.Sources { + if s.Name() != sourceName { + continue } - if supported { - thumbs = append(thumbs, t) + r, ok := s.(io.Reader) + if !ok { + continue } + r.Reader(ctx, io.ImageId(id), path, func(r goio.ReadSeeker, err error) { + // println(id, sourceName, s.Name(), r, ok, err) + if err != nil { + return + } + found = true + fn(r, nil) + }) + if found { + break + } + } + if !found { + fn(nil, fmt.Errorf("unable to find image %d using %s", id, sourceName)) } - return thumbs } diff --git a/internal/image/sourceConfig.go b/internal/image/sourceConfig.go new file mode 100644 index 0000000..4ff11c0 --- /dev/null +++ b/internal/image/sourceConfig.go @@ -0,0 +1,147 @@ +package image + +import ( + "embed" + "fmt" + "path/filepath" + "photofield/io" + "photofield/io/cached" + "photofield/io/ffmpeg" + "photofield/io/filtered" + "photofield/io/goexif" + "photofield/io/goimage" + "photofield/io/ristretto" + "photofield/io/sqlite" + "photofield/io/thumb" + "strings" + + "github.com/goccy/go-yaml" +) + +const ( + SourceTypeNone = "" + SourceTypeSqlite = "SQLITE" + SourceTypeGoexif = "GOEXIF" + SourceTypeThumb = "THUMB" + SourceTypeImage = "IMAGE" + SourceTypeFFmpeg = "FFMPEG" +) + +// SourceType is the type of a source (e.g. SQLITE, THUMB, IMAGE, FFMPEG) +type SourceType string + +func (t *SourceType) UnmarshalYAML(b []byte) error { + var s string + if err := yaml.Unmarshal(b, &s); err != nil { + return err + } + *t = SourceType(strings.ToUpper(s)) + return nil +} + +type SourceConfig struct { + Name string `json:"name"` + Type SourceType `json:"type"` + Path string `json:"path"` + Width int `json:"width"` + Height int `json:"height"` + Fit io.AspectRatioFit `json:"fit"` + Extensions []string `json:"extensions"` +} + +type ThumbnailConfig struct { + Sources SourceConfigs `json:"sources"` + Generators SourceConfigs `json:"generators"` + Sink SourceConfig `json:"sink"` +} + +// SourceEnvironment is the environment for creating sources +type SourceEnvironment struct { + DataDir string + FFmpegPath string + Migrations embed.FS + ImageCache *ristretto.Ristretto + Databases map[string]*sqlite.Source +} + +func (c SourceConfig) NewSource(env *SourceEnvironment) (io.Source, error) { + switch c.Type { + + case SourceTypeSqlite: + existing, ok := env.Databases[c.Path] + if ok { + return existing, nil + } + s := sqlite.New( + filepath.Join(env.DataDir, c.Path), + env.Migrations, + ) + if env.Databases == nil { + env.Databases = make(map[string]*sqlite.Source) + } + env.Databases[c.Path] = s + return s, nil + + case SourceTypeGoexif: + return goexif.Exif{ + Width: c.Width, + Height: c.Height, + }, nil + + case SourceTypeThumb: + return thumb.New( + c.Name, + c.Path, + c.Fit, + c.Width, + c.Height, + ), nil + + case SourceTypeImage: + return goimage.Image{ + Width: c.Width, + Height: c.Height, + }, nil + + case SourceTypeFFmpeg: + return ffmpeg.FFmpeg{ + Path: env.FFmpegPath, + Width: c.Width, + Height: c.Height, + Fit: c.Fit, + }, nil + + default: + return nil, fmt.Errorf("unknown source type: %s", c.Type) + } +} + +type SourceConfigs []SourceConfig + +// NewSources creates a list of sources from the configuration +// and adds caching and filtering layers if needed +func (cfgs SourceConfigs) NewSources(env *SourceEnvironment) ([]io.Source, error) { + var sources []io.Source + for _, c := range cfgs { + s, err := c.NewSource(env) + if err != nil { + return nil, err + } + if env.ImageCache != nil { + // Add caching layer + s = &cached.Cached{ + Source: s, + Cache: *env.ImageCache, + } + } + // Add filtering layer + if len(c.Extensions) > 0 { + s = &filtered.Filtered{ + Source: s, + Extensions: c.Extensions, + } + } + sources = append(sources, s) + } + return sources, nil +} diff --git a/internal/image/sourceInfo.go b/internal/image/sourceInfo.go index 1b0bd4b..0feff90 100644 --- a/internal/image/sourceInfo.go +++ b/internal/image/sourceInfo.go @@ -9,70 +9,6 @@ import ( "time" ) -func (source *Source) loadInfosMeta(ids <-chan uint32) { - for iduint := range ids { - id := ImageId(iduint) - path, err := source.GetImagePath(id) - if err != nil { - fmt.Println("Unable to find image path", err, path) - continue - } - info, err := source.LoadInfoMeta(path) - if err != nil { - fmt.Println("Unable to load image info meta", err, path) - continue - } - source.database.Write(path, info, UpdateMeta) - source.imageInfoCache.Delete(id) - } -} - -func (source *Source) loadInfosColor(ids <-chan uint32) { - for iduint := range ids { - id := ImageId(iduint) - path, err := source.GetImagePath(id) - if err != nil { - fmt.Println("Unable to find image path", err, path) - continue - } - info, err := source.LoadInfoColor(path) - if err != nil { - fmt.Println("Unable to load image info color", err, path) - continue - } - source.database.Write(path, info, UpdateColor) - source.imageInfoCache.Delete(id) - } -} - -func (source *Source) loadInfosAI(ids <-chan uint32) { - for iduint := range ids { - id := ImageId(iduint) - path, err := source.GetImagePath(id) - if err != nil { - fmt.Println("Unable to find image path", err, path) - continue - } - - minSize := 200 - f, err := source.OpenSmallestThumbnail(path, minSize) - if err != nil { - fmt.Println("Unable to load smallest image", err, path) - continue - } - - embedding, err := source.Clip.EmbedImageReader(f) - f.Close() - - if err != nil { - fmt.Println("Unable to get image embedding", err, path) - continue - } - - source.database.WriteAI(id, embedding) - } -} - func (source *Source) heuristicFromPath(path string) (Info, error) { var info Info @@ -131,18 +67,6 @@ func (source *Source) GetInfo(id ImageId) Info { } } - startTime = time.Now() - needsColor := result.NeedsColor() - if needsMeta || needsColor { - if needsMeta { - source.MetaQueue.Append(uint32(id)) - } - if needsColor { - source.ColorQueue.Append(uint32(id)) - } - } - addPendingMs := time.Since(startTime).Milliseconds() - if found && !needsMeta { return info } @@ -170,7 +94,7 @@ func (source *Source) GetInfo(id ImageId) Info { logging = totalMs > 1000 if logging { - log.Printf("image info %5d ms get cache, %5d ms get db, %5d ms add pending, %5d ms get heuristic, %5d ms set cache\n", cacheGetMs, dbGetMs, addPendingMs, heuristicGetMs, cacheSetMs) + log.Printf("image info %5d ms get cache, %5d ms get db, %5d ms get heuristic, %5d ms set cache\n", cacheGetMs, dbGetMs, heuristicGetMs, cacheSetMs) } return info } diff --git a/internal/image/thumbnail.go b/internal/image/thumbnail.go deleted file mode 100644 index d508c77..0000000 --- a/internal/image/thumbnail.go +++ /dev/null @@ -1,106 +0,0 @@ -package image - -import ( - "bytes" - "image" - "math" - "path/filepath" - "text/template" -) - -type PhotoTemplateData struct { - Dir string - Filename string -} - -type Thumbnail struct { - Name string `json:"name"` - PathTemplateRaw string `json:"path"` - PathTemplate *template.Template - Exif string `json:"exif"` - Extensions []string `json:"extensions"` - - SizeTypeRaw string `json:"fit"` - SizeType ThumbnailSizeType - - Width int `json:"width"` - Height int `json:"height"` - ExtraCost int `json:"extra_cost"` -} - -type ThumbnailSizeType int32 - -const ( - FitOutside ThumbnailSizeType = iota - FitInside ThumbnailSizeType = iota - OriginalSize ThumbnailSizeType = iota -) - -func (thumbnail *Thumbnail) Init() { - if thumbnail.PathTemplateRaw != "" { - var err error - thumbnail.PathTemplate, err = template.New("").Parse(thumbnail.PathTemplateRaw) - if err != nil { - panic(err) - } - } else if thumbnail.Exif != "" { - // No setup required - } else { - panic("thumbnail path or exif name must be specified") - } - - switch thumbnail.SizeTypeRaw { - case "INSIDE": - thumbnail.SizeType = FitInside - case "OUTSIDE": - thumbnail.SizeType = FitOutside - case "ORIGINAL": - thumbnail.SizeType = OriginalSize - default: - panic("Unsupported thumbnail fit: " + thumbnail.SizeTypeRaw) - } -} - -func (thumbnail *Thumbnail) GetPath(originalPath string) string { - if thumbnail.PathTemplate == nil { - return "" - } - var rendered bytes.Buffer - dir, filename := filepath.Split(originalPath) - err := thumbnail.PathTemplate.Execute(&rendered, PhotoTemplateData{ - Dir: dir, - Filename: filename, - }) - if err != nil { - panic(err) - } - return rendered.String() -} - -func (thumbnail *Thumbnail) Fit(originalSize image.Point) image.Point { - thumbWidth, thumbHeight := float64(thumbnail.Width), float64(thumbnail.Height) - thumbRatio := thumbWidth / thumbHeight - originalWidth, originalHeight := float64(originalSize.X), float64(originalSize.Y) - originalRatio := originalWidth / originalHeight - switch thumbnail.SizeType { - case FitInside: - if thumbRatio < originalRatio { - thumbHeight = thumbWidth / originalRatio - } else { - thumbWidth = thumbHeight * originalRatio - } - case FitOutside: - if thumbRatio > originalRatio { - thumbHeight = thumbWidth / originalRatio - } else { - thumbWidth = thumbHeight * originalRatio - } - case OriginalSize: - thumbWidth = originalWidth - thumbHeight = originalHeight - } - return image.Point{ - X: int(math.Round(thumbWidth)), - Y: int(math.Round(thumbHeight)), - } -} diff --git a/internal/layout/album.go b/internal/layout/album.go index 590429e..12086f4 100644 --- a/internal/layout/album.go +++ b/internal/layout/album.go @@ -117,6 +117,9 @@ func LayoutAlbum(layout Layout, collection collection.Collection, scene *render. break } + // path, _ := source.GetImagePath(info.Id) + // println(path, info.Width, info.Height) + photoTime := info.DateTime elapsed := photoTime.Sub(lastPhotoTime) if elapsed > 1*time.Hour { diff --git a/internal/layout/common.go b/internal/layout/common.go index 6537a83..e08095c 100644 --- a/internal/layout/common.go +++ b/internal/layout/common.go @@ -1,11 +1,14 @@ package layout import ( + "context" "fmt" "log" "path/filepath" "photofield/internal/image" "photofield/internal/render" + "photofield/io" + "sort" "strings" "time" ) @@ -51,10 +54,11 @@ type PhotoRegionSource struct { } type RegionThumbnail struct { - Name string `json:"name"` - Width int `json:"width"` - Height int `json:"height"` - Filename string `json:"filename"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + Width int `json:"width"` + Height int `json:"height"` + Filename string `json:"filename"` } type PhotoRegionData struct { @@ -76,35 +80,51 @@ func (regionSource PhotoRegionSource) getRegionFromPhoto(id int, photo *render.P originalPath := photo.GetPath(source) info := source.GetInfo(photo.Id) - originalSize := image.Size{ + + originalSize := io.Size{ X: info.Width, Y: info.Height, } isVideo := source.IsSupportedVideo(originalPath) - extension := strings.ToLower(filepath.Ext(originalPath)) + extension := filepath.Ext(originalPath) filename := filepath.Base(originalPath) + basename := strings.TrimSuffix(filename, extension) - thumbnailTemplates := source.GetApplicableThumbnails(originalPath) var thumbnails []RegionThumbnail - for i := range thumbnailTemplates { - thumbnail := &thumbnailTemplates[i] - thumbnailPath := thumbnail.GetPath(originalPath) - if source.Exists(thumbnailPath) { - thumbnailSize := thumbnail.Fit(originalSize) - basename := strings.TrimSuffix(filename, extension) - thumbnailFilename := fmt.Sprintf( - "%s_%s%s", - basename, thumbnail.Name, filepath.Ext(thumbnailPath), - ) - thumbnails = append(thumbnails, RegionThumbnail{ - Name: thumbnail.Name, - Width: thumbnailSize.X, - Height: thumbnailSize.Y, - Filename: thumbnailFilename, - }) + + for _, s := range source.Sources { + if !s.Exists(context.TODO(), io.ImageId(id), originalPath) { + continue } + size := s.Size(originalSize) + ext := s.Ext() + if ext == "" { + ext = extension + } + filename := fmt.Sprintf( + "%s_%s%s", + basename, s.Name(), ext, + ) + thumbnails = append(thumbnails, RegionThumbnail{ + Name: s.Name(), + DisplayName: s.DisplayName(), + Width: size.X, + Height: size.Y, + Filename: filename, + }) } + sort.Slice(thumbnails, func(i, j int) bool { + a := &thumbnails[i] + b := &thumbnails[j] + aa := a.Width * a.Height + bb := b.Width * b.Height + if aa != bb { + return aa < bb + } + return a.Name < b.Name + }) + return render.Region{ Id: id, Bounds: photo.Sprite.Rect, @@ -216,6 +236,8 @@ func addSectionToScene(section *Section, scene *render.Scene, bounds render.Rect float64(photo.Size.Y), ) + // println(photo.GetPath(source), photo.Sprite.Rect.String(), bounds.X, bounds.Y, x, y, config.ImageHeight, photo.Size.X, photo.Size.Y) + row = append(row, photo) x += imageWidth + config.ImageSpacing diff --git a/internal/layout/search.go b/internal/layout/search.go index c6f9cf8..16b12c1 100644 --- a/internal/layout/search.go +++ b/internal/layout/search.go @@ -13,10 +13,6 @@ import ( func LayoutSearch(layout Layout, collection collection.Collection, scene *render.Scene, source *image.Source) { - if scene.SearchEmbedding == nil { - return - } - limit := collection.Limit infos := collection.GetSimilar(source, scene.SearchEmbedding, image.ListOptions{ diff --git a/internal/openapi/api.gen.go b/internal/openapi/api.gen.go index 3a5d1f6..0e6edd6 100644 --- a/internal/openapi/api.gen.go +++ b/internal/openapi/api.gen.go @@ -27,13 +27,15 @@ const ( // Defines values for TaskType. const ( - TaskTypeINDEX TaskType = "INDEX" + TaskTypeINDEXCONTENTS TaskType = "INDEX_CONTENTS" - TaskTypeLOADAI TaskType = "LOAD_AI" + TaskTypeINDEXCONTENTSAI TaskType = "INDEX_CONTENTS_AI" - TaskTypeLOADCOLOR TaskType = "LOAD_COLOR" + TaskTypeINDEXCONTENTSCOLOR TaskType = "INDEX_CONTENTS_COLOR" - TaskTypeLOADMETA TaskType = "LOAD_META" + TaskTypeINDEXFILES TaskType = "INDEX_FILES" + + TaskTypeINDEXMETADATA TaskType = "INDEX_METADATA" ) // Bounds defines model for Bounds. diff --git a/internal/queue/queue.go b/internal/queue/queue.go index f847a53..16897c1 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -14,7 +14,7 @@ type Queue struct { queue *queue.Queue ID string Name string - Worker func(<-chan uint32) + Worker func(<-chan interface{}) WorkerCount int } @@ -35,24 +35,27 @@ func (q *Queue) Run() { logging := false - ids := make(chan uint32) - defer close(ids) + items := make(chan interface{}) + defer close(items) if q.WorkerCount == 0 { q.WorkerCount = 1 } for i := 0; i < q.WorkerCount; i++ { - go q.Worker(ids) + if q.Worker != nil { + go q.Worker(items) + } } for { - id := q.queue.Pop().(uint32) - if id == 0 { - log.Printf("%s queue stopping\n", q.Name) - return + if q.Worker != nil { + item := q.queue.Pop() + if item == nil { + log.Printf("%s queue stopping\n", q.Name) + return + } + items <- item } - - ids <- id doneCounter.Inc() now := time.Now() @@ -79,23 +82,23 @@ func (q *Queue) Run() { if logging { // log.Printf("image info load for id %5d, %5d pending, %5d ms get file, %5d ms set db, %5d ms set cache\n", id, len(backlog), fileGetMs, dbSetMs, cacheSetMs) - log.Printf("%s queue id %5d, %5d pending\n", q.Name, id, q.queue.Length()) + log.Printf("%s queue %5d pending\n", q.Name, q.queue.Length()) } } } -func (q *Queue) Append(id uint32) { +func (q *Queue) Length() int { if q.queue == nil { - return + return 0 } - q.queue.Append(id) + return q.queue.Length() } -func (q *Queue) AppendChan(ids <-chan uint32) { +func (q *Queue) AppendItems(items <-chan interface{}) { if q.queue == nil { return } - for id := range ids { - q.queue.Append(id) + for item := range items { + q.queue.Append(item) } } diff --git a/internal/render/bitmap.go b/internal/render/bitmap.go index a876f3c..27c728f 100644 --- a/internal/render/bitmap.go +++ b/internal/render/bitmap.go @@ -22,26 +22,66 @@ type Bitmap struct { Orientation image.Orientation } -func (bitmap *Bitmap) Draw(rimg draw.Image, c *canvas.Context, scales Scales, source *image.Source) error { - if bitmap.Sprite.IsVisible(c, scales) { - image, _, err := source.GetImage(bitmap.Path) - if err != nil { - return err - } - - bitmap.DrawImage(rimg, image, c) +func fitInside(cw float64, ch float64, w float64, h float64) (float64, float64) { + r := w / h + cr := cw / ch + if r < cr { + h = w / cr + } else { + w = h * cr } - return nil + return w, h } +// func fitCenterInside(c Rect, r Rect) Rect { +// ar := r.W / r.H +// car := c.W / c.H +// if ar < car { +// h := r.W / car +// r.Y = h +// } else { +// r.W = r.H * car +// } +// return r +// } + func (bitmap *Bitmap) DrawImage(rimg draw.Image, img goimage.Image, c *canvas.Context) { bounds := img.Bounds() + model := bitmap.Sprite.Rect.GetMatrixFitBoundsRotate(bounds, bitmap.Orientation) - // modelTopLeft := model.Dot(canvas.Point{X: 0, Y: 0}) - // modelBottomRight := model.Dot(canvas.Point{X: float64(bounds.Max.X), Y: float64(bounds.Max.Y)}) m := c.View().Mul(model) + + // bar := bitmap.Sprite.Rect.W / bitmap.Sprite.Rect.H + // iar := float64(bounds.Dx()) / float64(bounds.Dy()) + // if math.Abs(bar-iar) > 1e-3 { + // // println(bitmap.Sprite.Rect.String(), bounds.Dx(), bounds.Dy()) + // // modelTopLeft := model.Dot(canvas.Point{X: 0, Y: 0}) + // // modelBottomRight := model.Dot(canvas.Point{X: float64(bounds.Max.X), Y: float64(bounds.Max.Y)}) + // // renderImageFastCropped(rimg, img, m, bitmap.Sprite.Rect, modelTopLeft, modelBottomRight) + // // w, h := fitInside( + // // bitmap.Sprite.Rect.W, + // // bitmap.Sprite.Rect.H, + // // float64(bounds.Dx()), + // // float64(bounds.Dy()), + // // ) + // // b := bounds + // // b.Min.X = + + // img := Rect{ + // X: float64(bounds.Min.X), + // Y: float64(bounds.Min.Y), + // W: float64(bounds.Dx()), + // H: float64(bounds.Dy()), + // } + + // model := bitmap.Sprite.Rect.GetMatrixFitBoundsRotate(b, bitmap.Orientation) + // m := c.View().Mul(model) + + // renderImageFastBounds(rimg, img, m, b) + // return + // } + renderImageFast(rimg, img, m) - // renderImageFastCropped(rimg, img, m, bitmap.Sprite.Rect, modelTopLeft, modelBottomRight) } func renderImageFast(rimg draw.Image, img goimage.Image, m canvas.Matrix) { @@ -55,6 +95,16 @@ func renderImageFast(rimg draw.Image, img goimage.Image, m canvas.Matrix) { draw.ApproxBiLinear.Transform(rimg, aff3, img, bounds, draw.Src, nil) } +func renderImageFastBounds(rimg draw.Image, img goimage.Image, m canvas.Matrix, bounds goimage.Rectangle) { + origin := m.Dot(canvas.Point{X: 0, Y: float64(bounds.Size().Y)}) + h := float64(rimg.Bounds().Size().Y) + aff3 := f64.Aff3{ + m[0][0], -m[0][1], origin.X, + -m[1][0], m[1][1], h - origin.Y, + } + draw.ApproxBiLinear.Transform(rimg, aff3, img, bounds, draw.Src, nil) +} + // TODO finish implementation func renderImageFastCropped(rimg draw.Image, img goimage.Image, m canvas.Matrix, crop Rect, modelTopLeft canvas.Point, modelBottomRight canvas.Point) { bounds := img.Bounds() @@ -78,7 +128,7 @@ func renderImageFastCropped(rimg draw.Image, img goimage.Image, m canvas.Matrix, } println(bounds.String(), "crop", crop.String(), "model", model.String()) - // bounds = bounds.Inset(10) + bounds = bounds.Inset(10) // bounds = draw.ApproxBiLinear.Transform(rimg, aff3, img, bounds, draw.Src, nil) } diff --git a/internal/render/photo.go b/internal/render/photo.go index eafc088..d46e836 100644 --- a/internal/render/photo.go +++ b/internal/render/photo.go @@ -1,10 +1,12 @@ package render import ( + "context" "fmt" - "math" + "log" "photofield/internal/image" - "sort" + "photofield/io" + "time" "github.com/tdewolff/canvas" ) @@ -14,20 +16,6 @@ type Photo struct { Sprite Sprite } -type Variant struct { - Thumbnail *image.Thumbnail - Orientation image.Orientation - ZoomDist float64 -} - -func (variant Variant) String() string { - name := "original" - if variant.Thumbnail != nil { - name = variant.Thumbnail.Name - } - return fmt.Sprintf("%0.2f %v", variant.ZoomDist, name) -} - func (photo *Photo) GetSize(source *image.Source) image.Size { info := source.GetInfo(photo.Id) return image.Size{X: info.Width, Y: info.Height} @@ -40,7 +28,7 @@ func (photo *Photo) GetInfo(source *image.Source) image.Info { func (photo *Photo) GetPath(source *image.Source) string { path, err := source.GetImagePath(photo.Id) if err != nil { - panic("Unable to get photo path") + log.Fatalf("Unable to get photo path for id %v", photo.Id) } return path } @@ -53,41 +41,6 @@ func (photo *Photo) Place(x float64, y float64, width float64, height float64, s photo.Sprite.PlaceFit(x, y, width, height, imageWidth, imageHeight) } -func (photo *Photo) getBestVariants(config *Render, scene *Scene, c *canvas.Context, scales Scales, source *image.Source, originalPath string) []Variant { - - originalInfo := photo.GetInfo(source) - originalSize := originalInfo.Size() - originalZoomDist := math.Inf(1) - if source.IsSupportedImage(originalPath) { - originalZoomDist = photo.Sprite.Rect.GetPixelZoomDist(c, originalSize) - } - - thumbnails := source.GetApplicableThumbnails(originalPath) - variants := make([]Variant, 1+len(thumbnails)) - variants[0] = Variant{ - Thumbnail: nil, - Orientation: originalInfo.Orientation, - ZoomDist: originalZoomDist, - } - - for i := range thumbnails { - thumbnail := &thumbnails[i] - thumbSize := thumbnail.Fit(originalSize) - variants[1+i] = Variant{ - Thumbnail: thumbnail, - ZoomDist: photo.Sprite.Rect.GetPixelZoomDist(c, thumbSize) + float64(thumbnail.ExtraCost), - } - } - - sort.Slice(variants, func(i, j int) bool { - a := variants[i] - b := variants[j] - return a.ZoomDist < b.ZoomDist - }) - - return variants -} - func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales Scales, source *image.Source) { pixelArea := photo.Sprite.Rect.GetPixelArea(c, image.Size{X: 1, Y: 1}) @@ -105,33 +58,35 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales drawn := false path := photo.GetPath(source) - variants := photo.getBestVariants(config, scene, c, scales, source, path) - for _, variant := range variants { - // text := fmt.Sprintf("index %d zd %4.2f %s", index, bitmapAtZoom.ZoomDist, bitmap.Path) - // println(text) - bitmap := Bitmap{ - Sprite: photo.Sprite, - Orientation: variant.Orientation, + info := source.GetInfo(photo.Id) + size := info.Size() + rsize := photo.Sprite.Rect.RenderedSize(c, size) + + sources := source.Sources.EstimateCost(io.Size(size), io.Size(rsize)) + sources.Sort() + for i, s := range sources { + if drawn { + break } + start := time.Now() + r := s.Get(context.TODO(), io.ImageId(photo.Id), path) + elapsed := time.Since(start).Microseconds() - img, _, err := source.GetImageOrThumbnail(path, variant.Thumbnail) - if err != nil { + img, err := r.Image, r.Error + if img == nil || err != nil { continue } - if variant.Thumbnail != nil { - bounds := img.Bounds() - imgWidth := float64(bounds.Max.X - bounds.Min.X) - imgHeight := float64(bounds.Max.Y - bounds.Min.Y) - imgAspect := imgWidth / imgHeight - imgAspectRotated := 1 / imgAspect - rectAspect := bitmap.Sprite.Rect.W / bitmap.Sprite.Rect.H - // In case the image dimensions don't match expected aspect ratio, - // assume a 90 CCW rotation - if math.Abs(rectAspect-imgAspect) > math.Abs(rectAspect-imgAspectRotated) { - bitmap.Orientation = image.Rotate90 - } + if r.Orientation == io.SourceInfoOrientation { + r.Orientation = io.Orientation(info.Orientation) + } + + source.SourcesLatencyHistogram.WithLabelValues(s.Name()).Observe(float64(elapsed)) + + bitmap := Bitmap{ + Sprite: photo.Sprite, + Orientation: image.Orientation(r.Orientation), } bitmap.DrawImage(config.CanvasImage, img, c) @@ -147,16 +102,16 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales } if config.DebugThumbnails { - bounds := img.Bounds() - text := fmt.Sprintf("%dx%d %s", bounds.Size().X, bounds.Size().Y, variant.String()) + size := img.Bounds().Size() + text := fmt.Sprintf("%dx%d %d %4f\n%s", size.X, size.Y, i, s.Cost, s.Name()) font := scene.Fonts.Debug - font.Color = canvas.Lime - bitmap.Sprite.DrawText(config, c, scales, &font, text) + font.Color = canvas.Yellow + s := bitmap.Sprite + s.Rect.Y -= 20 + s.DrawText(config, c, scales, &font, text) } break - - // bitmap.Sprite.DrawText(c, scales, &scene.Fonts.Debug, text) } if !drawn { diff --git a/internal/render/rect.go b/internal/render/rect.go index 037fd2b..8d401bb 100644 --- a/internal/render/rect.go +++ b/internal/render/rect.go @@ -3,6 +3,7 @@ package render import ( "fmt" goimage "image" + "math" "photofield/internal/image" "github.com/tdewolff/canvas" @@ -244,3 +245,17 @@ func (rect Rect) GetPixelZoomDist(c *canvas.Context, size image.Size) float64 { return -zoom } } + +func (rect Rect) RenderedSize(c *canvas.Context, size image.Size) image.Size { + r := canvas.Rect{ + X: 0, + Y: 0, + W: rect.W, + H: rect.W * float64(size.Y) / float64(size.X), + } + t := r.Transform(c.View()) + return image.Size{ + X: int(math.Round(t.W)), + Y: int(math.Round(t.H)), + } +} diff --git a/internal/render/sprite.go b/internal/render/sprite.go index b624021..acd8e84 100644 --- a/internal/render/sprite.go +++ b/internal/render/sprite.go @@ -1,6 +1,10 @@ package render -import "github.com/tdewolff/canvas" +import ( + "math" + + "github.com/tdewolff/canvas" +) type Sprite struct { Rect Rect @@ -14,6 +18,9 @@ func (sprite *Sprite) PlaceFitHeight( contentHeight float64, ) { scale := fitHeight / contentHeight + if math.IsNaN(scale) || math.IsInf(scale, 0) { + scale = 1 + } sprite.Rect = Rect{ X: x, diff --git a/io/cached/cached.go b/io/cached/cached.go new file mode 100644 index 0000000..58a22bd --- /dev/null +++ b/io/cached/cached.go @@ -0,0 +1,92 @@ +package cached + +import ( + "context" + "fmt" + "photofield/io" + "photofield/io/ristretto" + "time" + + goio "io" + + "golang.org/x/sync/singleflight" +) + +type Cached struct { + Source io.Source + Cache ristretto.Ristretto + loading singleflight.Group +} + +func (c *Cached) Name() string { + return c.Source.Name() +} + +func (c *Cached) DisplayName() string { + return c.Source.DisplayName() +} + +func (c *Cached) Ext() string { + return c.Source.Ext() +} + +func (c *Cached) Size(size io.Size) io.Size { + return c.Source.Size(size) +} + +func (c *Cached) GetDurationEstimate(size io.Size) time.Duration { + return c.Source.GetDurationEstimate(size) +} + +func (c *Cached) Rotate() bool { + return false +} + +func (c *Cached) Exists(ctx context.Context, id io.ImageId, path string) bool { + return c.Source.Exists(ctx, id, path) +} + +func (c *Cached) Get(ctx context.Context, id io.ImageId, path string) io.Result { + r := c.Cache.GetWithName(ctx, id, c.Source.Name()) + // fmt.Printf("%v %v\n", r.Image, r.Error) + if r.Image != nil || r.Error != nil { + // fmt.Printf("%v cache found\n", id) + // println("found in cache") + return r + } + // r = c.Source.Get(ctx, id, path) + r = c.load(ctx, id, path) + // fmt.Printf("%v cache load end\n", id) + // c.Ristretto.SetWithName(ctx, id, c.Source.Name(), r) + // fmt.Printf("%v cache set\n", id) + // println("saved to cache", s) + return r +} + +func (c *Cached) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + r, ok := c.Source.(io.Reader) + if !ok { + fn(nil, fmt.Errorf("reader not supported by %s", c.Source.Name())) + return + } + r.Reader(ctx, id, path, fn) +} + +func (c *Cached) load(ctx context.Context, id io.ImageId, path string) io.Result { + key := fmt.Sprintf("%d", id) + // fmt.Printf("%v cache load begin %v\n", id, key) + ri, _, _ := c.loading.Do(key, func() (interface{}, error) { + // fmt.Printf("%p %v %s %v cache get begin\n", c, c.Source, c.Source.Name(), id) + r := c.Source.Get(ctx, id, path) + // fmt.Printf("%p %v %s %v cache get end\n", c, c.Source, c.Source.Name(), id) + c.Cache.SetWithName(ctx, id, c.Source.Name(), r) + // fmt.Printf("%v cache set\n", id) + return r, nil + }) + // fmt.Printf("%v cache load end %v\n", id, key) + return ri.(io.Result) +} + +func (c *Cached) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} diff --git a/io/exiftool/exiftool.go b/io/exiftool/exiftool.go new file mode 100644 index 0000000..9e671b4 --- /dev/null +++ b/io/exiftool/exiftool.go @@ -0,0 +1,65 @@ +package exiftool + +import ( + "bytes" + "context" + "fmt" + "image" + "photofield/io" + "time" + + "image/jpeg" + + "github.com/mostlygeek/go-exiftool" +) + +type Exif struct { + Tag string `json:"tag"` + exifTool *exiftool.Pool +} + +func New(tag string) *Exif { + e := Exif{ + Tag: tag, + } + exifTool, err := exiftool.NewPool( + "exiftool", 4, + "-n", // Machine-readable values + "-S", // Short tag names with no padding + ) + e.exifTool = exifTool + if err != nil { + panic(err) + } + return &e +} + +func (e Exif) Name() string { + return fmt.Sprintf("exiftool-%s", e.Tag) +} + +func (e Exif) Size(size io.Size) io.Size { + return io.Size{X: 120, Y: 120}.Fit(size, io.FitInside) +} + +func (e Exif) GetDurationEstimate(size io.Size) time.Duration { + return 17 * time.Millisecond +} + +func (e Exif) Get(ctx context.Context, id io.ImageId, path string) (image.Image, error) { + b, err := e.decodeBytes(path) + if err != nil { + return nil, err + } + + r := bytes.NewReader(b) + return jpeg.Decode(r) +} + +func (e Exif) Set(ctx context.Context, id io.ImageId, path string, img image.Image, err error) bool { + return false +} + +func (e Exif) decodeBytes(path string) ([]byte, error) { + return e.exifTool.ExtractFlags(path, "-b", "-"+e.Tag) +} diff --git a/io/ffmpeg/ffmpeg.go b/io/ffmpeg/ffmpeg.go new file mode 100644 index 0000000..0db3264 --- /dev/null +++ b/io/ffmpeg/ffmpeg.go @@ -0,0 +1,329 @@ +package ffmpeg + +import ( + "bytes" + "context" + "fmt" + "image" + "log" + "os/exec" + "photofield/io" + "strconv" + "time" + + goio "io" + + "image/jpeg" +) + +// type ForceOriginalAspectRatio string + +// const ( +// Decrease ForceOriginalAspectRatio = "decrease" +// Increase ForceOriginalAspectRatio = "increase" +// ) + +// type Fit uint8 + +// const ( +// FitOutside Fit = iota + 1 +// FitInside +// ) + +var ( + ErrMissingBinary = fmt.Errorf("ffmpeg binary not found") +) + +type FFmpeg struct { + Path string + Width int + Height int + Fit io.AspectRatioFit +} + +func FindPath() string { + path, err := exec.LookPath("ffmpeg") + if err != nil { + log.Printf("ffmpeg not found: %s\n", err.Error()) + return "" + } + log.Printf("ffmpeg found at %s\n", path) + return path +} + +func (f FFmpeg) Name() string { + var fit string + switch f.Fit { + case io.FitInside: + fit = "in" + case io.FitOutside: + fit = "out" + case io.OriginalSize: + fit = "orig" + } + found := "" + if f.Path == "" { + found = " (N/A)" + } + return fmt.Sprintf("ffmpeg-%dx%d-%s%s", f.Width, f.Height, fit, found) +} + +func (f FFmpeg) DisplayName() string { + return "FFmpeg JPEG" +} + +func (f FFmpeg) Ext() string { + return ".jpg" +} + +func (f FFmpeg) Size(size io.Size) io.Size { + return io.Size{X: f.Width, Y: f.Height}.Fit(size, f.Fit) +} + +func (f FFmpeg) GetDurationEstimate(size io.Size) time.Duration { + // return 30 * time.Nanosecond * time.Duration(size.Area()) + return 30 * time.Nanosecond * time.Duration(size.Area()) +} + +func (f FFmpeg) Rotate() bool { + return false +} + +func (f FFmpeg) ForceOriginalAspectRatio() string { + switch f.Fit { + case io.FitInside: + return "decrease" + case io.FitOutside: + return "increase" + } + return "unknown" +} + +func (f FFmpeg) FilterGraph() string { + if f.Fit == io.OriginalSize { + return "null" + } + foar := f.ForceOriginalAspectRatio() + return fmt.Sprintf( + "scale='min(iw,%d)':'min(ih,%d)':force_original_aspect_ratio=%s", + // "scale_npp='min(iw,%d)':'min(ih,%d)':force_original_aspect_ratio=%s", + f.Width, f.Height, foar, + ) +} + +func (f FFmpeg) Exists(ctx context.Context, id io.ImageId, path string) bool { + return true +} + +func (f FFmpeg) Get(ctx context.Context, id io.ImageId, path string) io.Result { + if f.Path == "" { + return io.Result{Error: ErrMissingBinary} + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + cmd := exec.CommandContext( + ctx, + f.Path, + "-hide_banner", + "-loglevel", "error", + "-i", path, + "-vframes", "1", + "-vf", f.FilterGraph(), + // "-q:v", "2", + // "-f", "image2pipe", // jpeg + "-c:v", "pam", + "-f", "rawvideo", + "-pix_fmt", "rgba", + "-an", // no audio + "-", + ) + + // println(cmd.String()) + b, err := cmd.Output() + err = formatErr(err, "ffmpeg") + if err != nil { + return io.Result{Error: err} + } + + pam, err := readPAM(b) + if err != nil { + return io.Result{Error: err} + } + + if pam.Depth != 4 { + return io.Result{Error: fmt.Errorf("unexpected depth %d", pam.Depth)} + } + + if pam.MaxValue != 255 { + return io.Result{Error: fmt.Errorf("unexpected max value %d", pam.MaxValue)} + } + + if pam.Width < 0 || pam.Height < 0 { + return io.Result{Error: fmt.Errorf("unexpected size %d x %d", pam.Width, pam.Height)} + } + + rgba := image.RGBA{ + Pix: pam.Bytes, + Stride: 4 * pam.Width, + Rect: image.Rect(0, 0, pam.Width, pam.Height), + } + return io.Result{Image: image.Image(&rgba)} +} + +func (f FFmpeg) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + r := f.Get(ctx, id, path) + if r.Error != nil { + fn(nil, r.Error) + return + } + var buf bytes.Buffer + err := jpeg.Encode(&buf, r.Image, nil) + if err != nil { + fn(nil, err) + return + } + rd := bytes.NewReader(buf.Bytes()) + fn(rd, nil) +} + +func (f FFmpeg) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} + +var pam_prefix_magic = []byte("P7\n") +var pam_header_end = []byte("ENDHDR\n") + +type metadata struct { + Streams []struct { + Index int `json:"index"` + CodecName string `json:"codec_name"` + CodecLongName string `json:"codec_long_name"` + CodecType string `json:"codec_type"` + CodecTagString string `json:"codec_tag_string"` + CodecTag string `json:"codec_tag"` + Width int `json:"width"` + Height int `json:"height"` + CodedWidth int `json:"coded_width"` + CodedHeight int `json:"coded_height"` + ClosedCaptions int `json:"closed_captions"` + FilmGrain int `json:"film_grain"` + HasBFrames int `json:"has_b_frames"` + SampleAspectRatio string `json:"sample_aspect_ratio"` + DisplayAspectRatio string `json:"display_aspect_ratio"` + PixFmt string `json:"pix_fmt"` + Level int `json:"level"` + ColorRange string `json:"color_range"` + Refs int `json:"refs"` + RFrameRate string `json:"r_frame_rate"` + AvgFrameRate string `json:"avg_frame_rate"` + TimeBase string `json:"time_base"` + NbReadFrames string `json:"nb_read_frames"` + Disposition struct { + Default int `json:"default"` + Dub int `json:"dub"` + Original int `json:"original"` + Comment int `json:"comment"` + Lyrics int `json:"lyrics"` + Karaoke int `json:"karaoke"` + Forced int `json:"forced"` + HearingImpaired int `json:"hearing_impaired"` + VisualImpaired int `json:"visual_impaired"` + CleanEffects int `json:"clean_effects"` + AttachedPic int `json:"attached_pic"` + TimedThumbnails int `json:"timed_thumbnails"` + Captions int `json:"captions"` + Descriptions int `json:"descriptions"` + Metadata int `json:"metadata"` + Dependent int `json:"dependent"` + StillImage int `json:"still_image"` + } `json:"disposition"` + } `json:"streams"` + Format struct { + Filename string `json:"filename"` + NbStreams int `json:"nb_streams"` + NbPrograms int `json:"nb_programs"` + FormatName string `json:"format_name"` + FormatLongName string `json:"format_long_name"` + Size string `json:"size"` + ProbeScore int `json:"probe_score"` + } `json:"format"` +} + +func formatErr(err error, cmdName string) error { + if exiterr, ok := err.(*exec.ExitError); ok { + return fmt.Errorf( + "%s error (exit code %d)\n%s", + cmdName, exiterr.ExitCode(), exiterr.Stderr, + ) + } + return err +} + +func readInt(buf *bytes.Buffer, delim byte) (int, error) { + s, err := buf.ReadString(delim) + if err != nil { + return 0, fmt.Errorf("unable to read: %w", err) + } + return strconv.Atoi(s[:len(s)-1]) +} + +type pamImage struct { + Width int + Height int + Depth int + MaxValue int + TupleType string + Bytes []byte +} + +func readPAM(b []byte) (pamImage, error) { + + var img pamImage + + if !bytes.HasPrefix(b, pam_prefix_magic) { + return img, fmt.Errorf("expected magic prefix %v", pam_prefix_magic) + } + + b = b[len(pam_prefix_magic):] + header := b[:256] + buf := bytes.NewBuffer(header) + + for { + key, err := buf.ReadString(' ') + if err != nil { + return img, err + } + key = key[:len(key)-1] + switch key { + case "WIDTH": + img.Width, err = readInt(buf, '\n') + case "HEIGHT": + img.Height, err = readInt(buf, '\n') + case "DEPTH": + img.Depth, err = readInt(buf, '\n') + case "MAXVAL": + img.MaxValue, err = readInt(buf, '\n') + case "TUPLTYPE": + img.TupleType, err = buf.ReadString('\n') + default: + return img, fmt.Errorf("unexpected key: %s", key) + } + if err != nil { + return img, err + } + if key == "TUPLTYPE" { + end := buf.Bytes() + if !bytes.HasPrefix(end, pam_header_end) { + return img, fmt.Errorf("expected end of header marker") + } + start := len(header) - buf.Len() + len(pam_header_end) + // println("len", buf.Len(), start) + // fmt.Printf(">%s<\n", b[start:100]) + img.Bytes = b[start:] + break + } + } + return img, nil +} diff --git a/io/filtered/filtered.go b/io/filtered/filtered.go new file mode 100644 index 0000000..87080b3 --- /dev/null +++ b/io/filtered/filtered.go @@ -0,0 +1,89 @@ +package filtered + +import ( + "context" + "fmt" + "path/filepath" + "photofield/io" + "strings" + "time" + + goio "io" +) + +type Filtered struct { + Source io.Source + Extensions []string +} + +func (f *Filtered) Name() string { + return f.Source.Name() +} + +func (f *Filtered) DisplayName() string { + return f.Source.DisplayName() +} + +func (f *Filtered) Ext() string { + return f.Source.Ext() +} + +func (f *Filtered) Size(size io.Size) io.Size { + return f.Source.Size(size) +} + +func (f *Filtered) GetDurationEstimate(size io.Size) time.Duration { + return f.Source.GetDurationEstimate(size) +} + +func (f *Filtered) Rotate() bool { + return f.Source.Rotate() +} + +func (f *Filtered) SupportsExtension(path string) bool { + if len(f.Extensions) == 0 { + return true + } + ext := strings.ToLower(filepath.Ext(path)) + for _, e := range f.Extensions { + if ext == e { + return true + } + } + return false +} + +func (f *Filtered) Exists(ctx context.Context, id io.ImageId, path string) bool { + if !f.SupportsExtension(path) { + return false + } + return f.Source.Exists(ctx, id, path) +} + +func (f *Filtered) Get(ctx context.Context, id io.ImageId, path string) io.Result { + if !f.SupportsExtension(path) { + return io.Result{Error: fmt.Errorf("extension not supported")} + } + return f.Source.Get(ctx, id, path) +} + +func (f *Filtered) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + if !f.SupportsExtension(path) { + fn(nil, fmt.Errorf("extension not supported")) + return + } + r, ok := f.Source.(io.Reader) + if !ok { + fn(nil, fmt.Errorf("reader not supported by %s", f.Source.Name())) + return + } + r.Reader(ctx, id, path, fn) +} + +func (f *Filtered) Decode(ctx context.Context, r goio.Reader) io.Result { + d, ok := f.Source.(io.Decoder) + if !ok { + return io.Result{Error: fmt.Errorf("decoder not supported by %s", f.Source.Name())} + } + return d.Decode(ctx, r) +} diff --git a/io/goexif/goexif.go b/io/goexif/goexif.go new file mode 100644 index 0000000..0cd0707 --- /dev/null +++ b/io/goexif/goexif.go @@ -0,0 +1,178 @@ +package goexif + +import ( + "bytes" + "context" + "os" + "photofield/io" + "strconv" + "time" + + goio "io" + + "image/jpeg" + + "github.com/rwcarlsen/goexif/exif" +) + +type Exif struct { + // Tag string `json:"tag"` + Width int `json:"width"` + Height int `json:"height"` + Fit io.AspectRatioFit +} + +func (e Exif) Name() string { + return "goexif" +} + +func (e Exif) DisplayName() string { + return "EXIF Thumbnail" +} + +func (e Exif) Ext() string { + return ".jpg" +} + +func (e Exif) Size(size io.Size) io.Size { + return io.Size{X: e.Width, Y: e.Height}.Fit(size, e.Fit) +} + +func (e Exif) GetDurationEstimate(size io.Size) time.Duration { + // return 862 * time.Microsecond // SSD + // return 10 * time.Millisecond // SSD - real world + return 100 * time.Millisecond // penalized + // return 930 * time.Microsecond // HDD +} + +func (e Exif) Rotate() bool { + return false +} + +func (e Exif) Exists(ctx context.Context, id io.ImageId, path string) bool { + exists := false + e.Reader(ctx, id, path, func(r goio.ReadSeeker, err error) { + if r != nil || err == nil { + exists = true + } + }) + return exists +} + +func (e Exif) Get(ctx context.Context, id io.ImageId, path string) io.Result { + b, o, err := load(path) + if err != nil { + return io.Result{Error: err} + } + + r := bytes.NewReader(b) + img, err := jpeg.Decode(r) + return io.Result{ + Image: img, + Orientation: o, + Error: err, + } +} + +func (e Exif) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + b, _, err := load(path) + if err != nil { + fn(nil, err) + return + } + + fn(bytes.NewReader(b), nil) +} + +func (e Exif) Decode(ctx context.Context, r goio.Reader) io.Result { + img, err := jpeg.Decode(r) + return io.Result{ + Image: img, + Orientation: io.Normal, + Error: err, + } +} + +func (e Exif) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} + +func load(path string) ([]byte, io.Orientation, error) { + f, err := os.Open(path) + if err != nil { + return nil, io.Normal, err + } + defer f.Close() + + x, err := exif.Decode(f) + if err != nil { + return nil, io.Normal, err + } + + b, err := x.JpegThumbnail() + o := getOrientation(x) + return b, o, err +} + +func getOrientation(x *exif.Exif) io.Orientation { + i, err := getTagInt(x, exif.Orientation) + if err != nil { + return io.Normal + } + return io.Orientation(i) + // s, err := getTagString(x, exif.Orientation) + // if err != nil { + // println(err.Error()) + // return io.Normal + // } + // return parseOrientation(s) +} + +func getTagString(x *exif.Exif, name exif.FieldName) (string, error) { + t, err := x.Get(exif.Orientation) + if err != nil { + return "", err + } + + s, err := t.StringVal() + if err != nil { + return "", err + } + + return s, nil +} + +func getTagInt(x *exif.Exif, name exif.FieldName) (int, error) { + t, err := x.Get(exif.Orientation) + if err != nil { + return 0, err + } + i, err := t.Int(0) + if err != nil { + return i, err + } + return i, nil +} + +func parseOrientation(orientation string) io.Orientation { + n, err := strconv.Atoi(orientation) + if err != nil || n < 1 || n > 8 { + return io.Normal + } + return io.Orientation(n) +} + +// func getOrientationFromRotation(rotation string) io.Orientation { +// switch rotation { +// case "0": +// return Normal +// case "90": +// return Rotate90 +// case "180": +// return Rotate180 +// case "270": +// return Rotate270 +// default: +// return Normal +// } +// } diff --git a/io/goimage/goimage.go b/io/goimage/goimage.go new file mode 100644 index 0000000..530106e --- /dev/null +++ b/io/goimage/goimage.go @@ -0,0 +1,129 @@ +package goimage + +import ( + "context" + "image" + "os" + "photofield/io" + "time" + + goio "io" + + _ "image/jpeg" + _ "image/png" + + "golang.org/x/image/draw" +) + +type Image struct { + Width int + Height int + Decoder func(goio.Reader) (image.Image, error) +} + +func (o Image) Name() string { + return "original" +} + +func (o Image) DisplayName() string { + return "Original" +} + +func (o Image) Ext() string { + return "" +} + +func (o Image) Resized() bool { + return o.Width != 0 && o.Height != 0 +} + +func (o Image) Size(size io.Size) io.Size { + if o.Resized() { + return io.Size{ + X: o.Width, + Y: o.Height, + } + } + return size +} + +func (o Image) GetDurationEstimate(size io.Size) time.Duration { + return 30 * time.Nanosecond * time.Duration(size.Area()) +} + +func (o Image) Rotate() bool { + return true +} + +func resize(img image.Image, maxWidth, maxHeight int) image.Image { + origW := img.Bounds().Size().X + origH := img.Bounds().Size().Y + aspectRatio := float64(origW) / float64(origH) + + desiredW := maxWidth + desiredH := maxHeight + if float64(desiredW)/float64(desiredH) > aspectRatio { + desiredW = int(float64(desiredH) * aspectRatio) + } else { + desiredH = int(float64(desiredW) / aspectRatio) + } + resized := image.NewRGBA(image.Rect(0, 0, desiredW, desiredH)) + draw.ApproxBiLinear.Scale(resized, resized.Bounds(), img, img.Bounds(), draw.Src, nil) + return resized +} + +func (o Image) Exists(ctx context.Context, id io.ImageId, path string) bool { + return true +} + +func (o Image) Get(ctx context.Context, id io.ImageId, path string) io.Result { + f, err := os.Open(path) + if err != nil { + return io.Result{Error: err} + } + defer f.Close() + + var img image.Image + if o.Decoder != nil { + img, err = o.Decoder(f) + } else { + img, _, err = image.Decode(f) + } + + if o.Resized() && err == nil { + img = resize(img, o.Width, o.Height) + } + + return io.Result{ + Image: img, + Error: err, + Orientation: io.SourceInfoOrientation, + } +} + +func (o Image) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + f, err := os.Open(path) + if err != nil { + fn(nil, err) + return + } + defer f.Close() + + fn(f, nil) +} + +func (o Image) Decode(ctx context.Context, r goio.Reader) io.Result { + img, _, err := image.Decode(r) + if o.Resized() && err == nil { + img = resize(img, o.Width, o.Height) + } + return io.Result{ + Image: img, + Error: err, + Orientation: io.SourceInfoOrientation, + } +} + +func (o Image) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} diff --git a/io/io.go b/io/io.go new file mode 100644 index 0000000..1bb8c87 --- /dev/null +++ b/io/io.go @@ -0,0 +1,176 @@ +package io + +import ( + "context" + "fmt" + "image" + "io" + "math" + "sort" + "strings" + "time" + + "github.com/goccy/go-yaml" +) + +type AspectRatioFit int32 + +func (f *AspectRatioFit) UnmarshalYAML(b []byte) error { + var s string + if err := yaml.Unmarshal(b, &s); err != nil { + return err + } + switch strings.ToUpper(s) { + default: + *f = OriginalSize + case "INSIDE": + *f = FitInside + case "OUTSIDE": + *f = FitOutside + } + return nil +} + +const ( + OriginalSize AspectRatioFit = iota + FitOutside AspectRatioFit = iota + FitInside AspectRatioFit = iota +) + +type Orientation int8 + +// All rotations are counter-clockwise +const ( + Normal Orientation = 1 + MirrorHorizontal Orientation = 2 + Rotate180 Orientation = 3 + MirrorVertical Orientation = 4 + MirrorHorizontalRotate270 Orientation = 5 + Rotate90 Orientation = 6 + MirrorHorizontalRotate90 Orientation = 7 + Rotate270 Orientation = 8 + SourceInfoOrientation Orientation = 127 +) + +type ImageId uint32 + +type Size image.Point + +func (s Size) String() string { + return fmt.Sprintf("%d x %d", s.X, s.Y) +} + +func (s Size) Area() int64 { + return int64(s.X) * int64(s.Y) +} + +func (s Size) Fit(original Size, fit AspectRatioFit) Size { + if fit == OriginalSize { + return original + } + tw, th := float64(s.X), float64(s.Y) + ar := tw / th + ow, oh := float64(original.X), float64(original.Y) + oar := ow / oh + switch fit { + case FitInside: + if ar < oar { + th = tw / oar + } else { + tw = th * oar + } + case FitOutside: + if ar > oar { + th = tw / oar + } else { + tw = th * oar + } + } + return Size{ + X: int(math.Round(tw)), + Y: int(math.Round(th)), + } +} + +type Result struct { + Image image.Image + Orientation Orientation + Error error +} + +type Source interface { + Name() string + DisplayName() string + Ext() string + Size(original Size) Size + Rotate() bool + GetDurationEstimate(original Size) time.Duration + Exists(ctx context.Context, id ImageId, path string) bool + Get(ctx context.Context, id ImageId, path string) Result +} + +type Sink interface { + Set(ctx context.Context, id ImageId, path string, r Result) bool +} + +type Reader interface { + Reader(ctx context.Context, id ImageId, path string, fn func(r io.ReadSeeker, err error)) +} + +type Decoder interface { + Decode(ctx context.Context, r io.Reader) Result +} + +type ReadDecoder interface { + Reader + Decoder +} + +type Sources []Source + +func (sources Sources) EstimateCost(original Size, target Size) SourceCosts { + targetArea := target.Area() + costs := make([]SourceCost, len(sources)) + for i := range sources { + s := sources[i] + ssize := s.Size(original) + if ssize.X == 0 && ssize.Y == 0 { + ssize = target + } + sarea := ssize.Area() + sizecost := math.Abs(float64(targetArea)-float64(sarea)) * 0.001 + if targetArea > sarea { + // areacost = math.Sqrt(float64(targetArea)-float64(sarea)) * 3 + // areacost = math.Sqrt(float64(targetArea)-float64(sarea)) * 3 + sizecost *= 7 + } + // dx := float64(target.X - ssize.X) + // dy := float64(target.Y - ssize.Y) + // sizecost := math.Sqrt(dx*dx + dy*dy) + dur := s.GetDurationEstimate(original) + durcost := math.Pow(float64(dur.Microseconds()), 1) * 0.003 + // durcost := float64(dur.Microseconds()) * 0.001 + cost := sizecost + durcost + // fmt.Printf("%4d %30s %12s %12s %12s %12d %12f %10s %12f %12f\n", i, s.Name(), original, target, ssize, sarea, sizecost, dur, durcost, cost) + costs[i] = SourceCost{ + Source: s, + Cost: cost, + } + } + return costs +} + +type SourceCost struct { + Source + Cost float64 +} + +type SourceCosts []SourceCost + +func (costs SourceCosts) Sort() { + sort.Slice(costs, func(i, j int) bool { + a := costs[i] + b := costs[j] + return a.Cost < b.Cost + }) +} diff --git a/io/mutex/mutex.go b/io/mutex/mutex.go new file mode 100644 index 0000000..e95574e --- /dev/null +++ b/io/mutex/mutex.go @@ -0,0 +1,59 @@ +package mutex + +import ( + "context" + "fmt" + "photofield/io" + "sync" + "time" +) + +type Mutex struct { + Source io.Source + loading sync.Map +} + +type loadingResult struct { + result io.Result + loaded chan struct{} +} + +func (m Mutex) Name() string { + return fmt.Sprintf("%s (mutex)", m.Source.Name()) +} + +func (m Mutex) Size(size io.Size) io.Size { + return m.Source.Size(size) +} + +func (m Mutex) GetDurationEstimate(size io.Size) time.Duration { + return m.Source.GetDurationEstimate(size) +} + +func (m Mutex) Rotate() bool { + return false +} + +func (m Mutex) Get(ctx context.Context, id io.ImageId, path string) io.Result { + loading := &loadingResult{} + loading.loaded = make(chan struct{}) + key := id + stored, loaded := m.loading.LoadOrStore(key, loading) + if loaded { + loading = stored.(*loadingResult) + fmt.Printf("%v blocking on channel\n", key) + <-loading.loaded + fmt.Printf("%v channel unblocked\n", key) + return loading.result + } + + fmt.Printf("%v not found, loading, mutex locked\n", key) + loading.result = m.Source.Get(ctx, id, path) + fmt.Printf("%v loaded, closing channel\n", key) + close(loading.loaded) + return loading.result +} + +func (m Mutex) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} diff --git a/io/ristretto/ristretto.go b/io/ristretto/ristretto.go new file mode 100644 index 0000000..2e6f4dd --- /dev/null +++ b/io/ristretto/ristretto.go @@ -0,0 +1,180 @@ +package ristretto + +import ( + "context" + "fmt" + "image" + "log" + "photofield/internal/metrics" + "photofield/io" + "reflect" + "time" + "unsafe" + + _ "image/jpeg" + _ "image/png" + + drist "github.com/dgraph-io/ristretto" + "github.com/dgraph-io/ristretto/z" +) + +type Ristretto struct { + cache *drist.Cache +} + +type IdWithSize struct { + Id io.ImageId + Size io.Size +} + +type IdWithName struct { + Id io.ImageId + Name string +} + +func (ids IdWithSize) String() string { + return fmt.Sprintf("%6d %4d %4d", ids.Id, ids.Size.X, ids.Size.Y) +} + +func New() *Ristretto { + maxSizeBytes := int64(256000000) + cache, err := drist.NewCache(&drist.Config{ + NumCounters: 1e6, // number of keys to track frequency of + MaxCost: maxSizeBytes, // maximum cost of cache + BufferItems: 64, // number of keys per Get buffer + Metrics: true, + Cost: cost, + KeyToHash: keyToHash, + }) + if err != nil { + panic(err) + } + metrics.AddRistretto("image_cache", cache) + return &Ristretto{ + cache: cache, + } +} + +func keyToHash(key interface{}) (uint64, uint64) { + switch k := key.(type) { + case IdWithSize: + a, b := uint64(k.Id), (uint64(k.Size.X)<<32)|(uint64(k.Size.Y)) + fmt.Printf("%x %x\n", a, b) + return uint64(k.Id), (uint64(k.Size.X) << 32) | (uint64(k.Size.Y)) + case IdWithName: + // a, b := uint64(k.Id), z.MemHashString(k.Name) + // fmt.Printf("%x %x\n", a, b) + // b = 0 + // return a, b + str := fmt.Sprintf("%d %s", k.Id, k.Name) + return z.KeyToHash(str) + } + + return z.KeyToHash(key) + // ids, ok := key.(IdWithSize) + // if ok { + + // } + // return +} + +func (r Ristretto) Name() string { + return "ristretto" +} + +func (r Ristretto) Size(size io.Size) io.Size { + return io.Size{} +} + +func (r Ristretto) GetDurationEstimate(size io.Size) time.Duration { + return 80 * time.Nanosecond +} + +func (r Ristretto) Get(ctx context.Context, id io.ImageId, path string) io.Result { + value, found := r.cache.Get(uint32(id)) + if found { + return value.(io.Result) + } + return io.Result{} +} + +func (r Ristretto) GetWithSize(ctx context.Context, ids IdWithSize) io.Result { + value, found := r.cache.Get(ids) + if found { + return value.(io.Result) + } + return io.Result{} +} + +func (r Ristretto) GetWithName(ctx context.Context, id io.ImageId, name string) io.Result { + idn := IdWithName{ + Id: id, + Name: name, + } + value, found := r.cache.Get(idn) + if found { + return value.(io.Result) + } + return io.Result{} +} + +func (r Ristretto) SetWithName(ctx context.Context, id io.ImageId, name string, v io.Result) bool { + idn := IdWithName{ + Id: id, + Name: name, + } + return r.cache.SetWithTTL(idn, v, 0, 10*time.Minute) +} + +func (r Ristretto) Set(ctx context.Context, id io.ImageId, path string, v io.Result) bool { + return r.cache.SetWithTTL(uint32(id), v, 0, 10*time.Minute) +} + +func (r Ristretto) SetWithSize(ctx context.Context, ids IdWithSize, v io.Result) bool { + return r.cache.SetWithTTL(ids, v, 0, 10*time.Minute) +} + +func cost(value interface{}) int64 { + r := value.(io.Result) + img := r.Image + if img == nil { + return 1 + } + switch img := img.(type) { + + case *image.YCbCr: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Y))*int64(unsafe.Sizeof(img.Y[0])) + + int64(cap(img.Cb))*int64(unsafe.Sizeof(img.Cb[0])) + + int64(cap(img.Cr))*int64(unsafe.Sizeof(img.Cr[0])) + + case *image.Gray: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + + case *image.NRGBA: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + + case *image.RGBA: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + + case *image.CMYK: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + + case *image.Paletted: + return int64(unsafe.Sizeof(*img)) + + int64(cap(img.Pix))*int64(unsafe.Sizeof(img.Pix[0])) + + int64(cap(img.Palette))*int64(unsafe.Sizeof(img.Pix[0])) + + case nil: + return 1 + + default: + log.Printf("Unable to compute cost, unsupported image format %v", reflect.TypeOf(img)) + // Fallback image size (10MB) + return 10000000 + } +} diff --git a/io/sqlite/sqlite.go b/io/sqlite/sqlite.go new file mode 100644 index 0000000..5ad7e38 --- /dev/null +++ b/io/sqlite/sqlite.go @@ -0,0 +1,445 @@ +package sqlite + +import ( + "bufio" + "bytes" + "context" + "embed" + "fmt" + "image/jpeg" + "log" + "net/http" + "path/filepath" + "photofield/internal/metrics" + "photofield/io" + "time" + + goio "io" + + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/httpfs" +) + +var ( + ErrNotFound = fmt.Errorf("image not found") +) + +type Source struct { + path string + pool *sqlitex.Pool + pending chan Thumb +} + +type Thumb struct { + Id uint32 + Bytes []byte +} + +func (s *Source) Name() string { + return "sqlite" +} + +func (s *Source) DisplayName() string { + return "Internal thumbnail" +} + +func (s *Source) Ext() string { + return ".jpg" +} + +func (s *Source) GetDurationEstimate(size io.Size) time.Duration { + return 879 * time.Microsecond // SSD + // return 958 * time.Microsecond // HDD +} + +func (s *Source) Rotate() bool { + return false +} + +func (s *Source) Size(size io.Size) io.Size { + return io.Size{X: 256, Y: 256}.Fit(size, io.FitInside) +} + +func New(path string, migrations embed.FS) *Source { + + var err error + + source := Source{ + path: path, + } + source.migrate(migrations) + + poolSize := 10 + source.pool, err = sqlitex.Open(source.path, 0, poolSize) + if err != nil { + panic(err) + } + conns := make([]*sqlite.Conn, poolSize) + for i := 0; i < poolSize; i++ { + conns[i] = source.pool.Get(context.Background()) + setPragma(conns[i], "synchronous", "NORMAL") + assertPragma(conns[i], "synchronous", 1) + } + for i := 0; i < poolSize; i++ { + source.pool.Put(conns[i]) + } + + source.pending = make(chan Thumb, 100) + go source.writePending() + + return &source +} + +func setPragma(conn *sqlite.Conn, name string, value interface{}) error { + sql := fmt.Sprintf("PRAGMA %s = %v;", name, value) + return sqlitex.ExecuteTransient(conn, sql, &sqlitex.ExecOptions{}) +} + +func assertPragma(conn *sqlite.Conn, name string, value interface{}) error { + sql := fmt.Sprintf("PRAGMA %s;", name) + return sqlitex.ExecuteTransient(conn, sql, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { + expected := fmt.Sprintf("%v", value) + actual := stmt.GetText(name) + if expected != actual { + return fmt.Errorf("unable to initialize %s to %v, got back %v", name, expected, actual) + } + return nil + }, + }) +} + +func (s *Source) init() { + flags := sqlite.OpenReadWrite | + sqlite.OpenCreate | + sqlite.OpenURI | + sqlite.OpenNoMutex + + conn, err := sqlite.OpenConn(s.path, flags) + if err != nil { + panic(err) + } + defer conn.Close() + + // Optimized for 256px jpeg thumbnails + pageSize := 16384 + err = setPragma(conn, "page_size", pageSize) + if err != nil { + panic(err) + } + + // Vacuum to apply page size + err = sqlitex.ExecuteTransient(conn, "VACUUM;", &sqlitex.ExecOptions{}) + if err != nil { + panic(err) + } + + err = assertPragma(conn, "page_size", pageSize) + if err != nil { + panic(err) + } +} + +func (s *Source) migrate(migrations embed.FS) { + dbsource, err := httpfs.New(http.FS(migrations), "db/migrations-thumbs") + if err != nil { + panic(err) + } + url := fmt.Sprintf("sqlite://%v", filepath.ToSlash(s.path)) + m, err := migrate.NewWithSourceInstance( + "migrations-thumbs", + dbsource, + url, + ) + if err != nil { + panic(err) + } + + version, dirty, err := m.Version() + if err != nil && err != migrate.ErrNilVersion { + panic(err) + } + + if err == migrate.ErrNilVersion { + s.init() + } + + dirtystr := "" + if dirty { + dirtystr = " (dirty)" + } + log.Printf("thumbs database version %v%s, migrating if needed", version, dirtystr) + + err = m.Up() + if err != nil && err != migrate.ErrNoChange { + panic(err) + } + + serr, derr := m.Close() + if serr != nil { + panic(serr) + } + if derr != nil { + panic(derr) + } +} + +func (s *Source) Write(id uint32, bytes []byte) error { + s.pending <- Thumb{ + Id: id, + Bytes: bytes, + } + return nil +} + +func (s *Source) Delete(id uint32) error { + s.pending <- Thumb{ + Id: id, + Bytes: nil, + } + return nil +} + +func (s *Source) writePending() { + c := s.pool.Get(context.Background()) + defer s.pool.Put(c) + + insert := c.Prep(` + INSERT OR REPLACE INTO thumb256(id, created_at_unix, data) + VALUES (?, ?, ?);`) + defer insert.Reset() + + delete := c.Prep(` + DELETE FROM thumb256 WHERE id = ?;`) + defer delete.Reset() + + lastCommit := time.Now() + lastOptimize := time.Time{} + inTransaction := false + + for t := range s.pending { + if !inTransaction { + // s.transactionMutex.Lock() + err := sqlitex.Execute(c, "BEGIN TRANSACTION;", nil) + if err != nil { + panic(err) + } + inTransaction = true + } + + now := time.Now() + + if t.Bytes == nil { + delete.BindInt64(1, int64(t.Id)) + _, err := delete.Step() + if err != nil { + log.Printf("Unable to delete image %d: %s\n", t.Id, err) + } + delete.Reset() + } else { + insert.BindInt64(1, int64(t.Id)) + insert.BindInt64(2, now.Unix()) + insert.BindBytes(3, t.Bytes) + _, err := insert.Step() + if err != nil { + log.Printf("Unable to insert image %d: %s\n", t.Id, err) + } + insert.Reset() + } + + sinceLastCommitSeconds := time.Since(lastCommit).Seconds() + if inTransaction && (sinceLastCommitSeconds >= 10 || len(s.pending) == 0) { + err := sqlitex.Execute(c, "COMMIT;", nil) + lastCommit = time.Now() + if err != nil { + panic(err) + } + + if time.Since(lastOptimize).Hours() >= 1 { + lastOptimize = time.Now() + log.Println("database optimizing") + optimizeDone := metrics.Elapsed("database optimize") + err = sqlitex.Execute(c, "PRAGMA optimize;", nil) + if err != nil { + panic(err) + } + optimizeDone() + } + + // s.transactionMutex.Unlock() + inTransaction = false + } + } +} + +func (s *Source) Exists(ctx context.Context, id io.ImageId, path string) bool { + exists := false + s.Reader(ctx, id, path, func(r goio.ReadSeeker, err error) { + if r != nil && err == nil { + exists = true + } + }) + return exists +} + +func (s *Source) Get(ctx context.Context, id io.ImageId, path string) io.Result { + c := s.pool.Get(ctx) + defer s.pool.Put(c) + + stmt := c.Prep(` + SELECT data + FROM thumb256 + WHERE id == ?;`) + defer stmt.Reset() + + stmt.BindInt64(1, int64(id)) + + exists, err := stmt.Step() + if err != nil { + return io.Result{Error: fmt.Errorf("unable to execute query: %w", err)} + } + if !exists { + return io.Result{} + } + + r := stmt.ColumnReader(0) + return s.Decode(ctx, r) +} + +func (s *Source) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + c := s.pool.Get(ctx) + defer s.pool.Put(c) + + stmt := c.Prep(` + SELECT data + FROM thumb256 + WHERE id == ?;`) + defer stmt.Reset() + + stmt.BindInt64(1, int64(id)) + + exists, err := stmt.Step() + if err != nil { + fn(nil, fmt.Errorf("unable to execute query: %w", err)) + return + } + if !exists { + fn(nil, ErrNotFound) + return + } + + r := stmt.ColumnReader(0) + fn(r, nil) +} + +func (s *Source) Decode(ctx context.Context, r goio.Reader) io.Result { + img, err := jpeg.Decode(r) + if err != nil { + return io.Result{Error: fmt.Errorf("unable to decode image: %w", err)} + } + return io.Result{ + Image: img, + Error: err, + } +} + +func (s *Source) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + var b bytes.Buffer + return s.SetWithBuffer(ctx, id, path, &b, r) +} + +func (s *Source) SetWithBuffer(ctx context.Context, id io.ImageId, path string, b *bytes.Buffer, r io.Result) bool { + w := bufio.NewWriter(b) + s.Encode(ctx, r, w) + s.Write(uint32(id), b.Bytes()) + return true +} + +func (s *Source) Encode(ctx context.Context, r io.Result, w goio.Writer) bool { + if r.Image == nil || r.Error != nil { + return false + } + bounds := r.Image.Bounds() + if bounds.Dx() > 256 || bounds.Dy() > 256 { + return false + } + + jpeg.Encode(w, r.Image, &jpeg.Options{ + Quality: 70, + }) + return true +} + +// func (s *Source) migrate(migrations embed.FS) { +// dbsource, err := httpfs.New(http.FS(migrations), "db/migrations") +// if err != nil { +// panic(err) +// } +// url := fmt.Sprintf("sqlite://%v", filepath.ToSlash(s.path)) +// m, err := migrate.NewWithSourceInstance( +// "migrations", +// dbsource, +// url, +// ) +// if err != nil { +// panic(err) +// } + +// version, dirty, err := m.Version() +// if err != nil && err != migrate.ErrNilVersion { +// panic(err) +// } + +// dirtystr := "" +// if dirty { +// dirtystr = " (dirty)" +// } +// log.Printf("database version %v%s, migrating if needed", version, dirtystr) + +// err = m.Up() +// if err != nil && err != migrate.ErrNoChange { +// panic(err) +// } + +// serr, derr := m.Close() +// if serr != nil { +// panic(serr) +// } +// if derr != nil { +// panic(derr) +// } +// } + +// pool, err := sqlitex.Open(path.Join(dir, "test/photofield.thumbs.db"), 0, 10) +// if err != nil { +// panic(err) +// } +// c := pool.Get(context.Background()) +// defer pool.Put(c) + +// stmt := c.Prep(` +// SELECT data +// FROM thumb256 +// WHERE id == ?;`) +// defer stmt.Reset() + +// maxid := int64(1000000) + +// for i := 0; i < b.N; i++ { +// id := 1 + rand.Int63n(maxid) +// stmt.BindInt64(1, id) +// exists, err := stmt.Step() +// if err != nil { +// b.Error(err) +// } +// if !exists { +// b.Errorf("id not found: %d", id) +// } +// r := stmt.ColumnReader(1) +// io.ReadAll(r) +// stmt.Reset() +// } diff --git a/io/sqlite/sqlite_test.go b/io/sqlite/sqlite_test.go new file mode 100644 index 0000000..b1271cb --- /dev/null +++ b/io/sqlite/sqlite_test.go @@ -0,0 +1,33 @@ +package sqlite + +import ( + "context" + "embed" + "os" + "path" + "testing" +) + +var dir = "../../../photos/" + +func TestRoundtrip(t *testing.T) { + p := path.Join(dir, "test/P1110220-ffmpeg-256-cjpeg-70.jpg") + bytes, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + + s := New(path.Join(dir, "test/photofield.thumbs.db"), embed.FS{}) + + id := uint32(1) + + s.Write(id, bytes) + img, err := s.Load(context.Background(), id) + if err != nil { + t.Fatal(err) + } + b := img.Bounds() + if b.Dx() != 256 || b.Dy() != 171 { + t.Errorf("unexpected size %d x %d", b.Dx(), b.Dy()) + } +} diff --git a/io/thumb/thumb.go b/io/thumb/thumb.go new file mode 100644 index 0000000..e7664d4 --- /dev/null +++ b/io/thumb/thumb.go @@ -0,0 +1,160 @@ +package thumb + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + "time" + + goio "io" + + "image/jpeg" + "image/png" + + "photofield/io" + "photofield/io/goimage" +) + +// type Fit uint8 +type Template = *template.Template + +// const ( +// FitOutside Fit = iota + 1 +// FitInside +// OriginalSize +// ) + +// var ( +// FitValue = map[string]uint8{ +// "INSIDE": uint8(FitInside), +// "OUTSIDE": uint8(FitOutside), +// "ORIGINAL": uint8(OriginalSize), +// } +// ) + +type TemplateData struct { + Dir string + Filename string +} + +// func (f *ThumbFit) UnmarshalJSON(data []byte) error { +// var s string + +// if err = json.Unmarshal(data, &s); err != nil { +// return err +// } + +// println(s) + +// // return json.Unmarshal(data, &) +// } + +type Thumb struct { + ThumbName string `json:"name"` + + PathTemplate Template + + Fit io.AspectRatioFit `json:"fit"` + + Width int `json:"width"` + Height int `json:"height"` + + goimage goimage.Image +} + +func New( + name string, + pathTemplate string, + fit io.AspectRatioFit, + Width int, + Height int, +) *Thumb { + t := &Thumb{ + ThumbName: name, + PathTemplate: template.Must(template.New("").Parse(pathTemplate)), + Fit: fit, + Width: Width, + Height: Height, + } + + // Optimized jpeg/png case, case insensitive + ext := strings.ToLower(filepath.Ext(pathTemplate)) + + switch ext { + case ".jpg", ".jpeg": + t.goimage.Decoder = jpeg.Decode + + case ".png": + t.goimage.Decoder = png.Decode + } + + return t +} + +func (t Thumb) Name() string { + return fmt.Sprintf("thumb-%dx%d-%s", t.Width, t.Height, t.ThumbName) +} + +func (t Thumb) DisplayName() string { + return "Pregenerated thumbnail" +} + +func (t Thumb) Ext() string { + return filepath.Ext(t.resolvePath("")) +} + +func (t Thumb) Rotate() bool { + return true +} + +func (t Thumb) Size(size io.Size) io.Size { + return io.Size{X: t.Width, Y: t.Height}.Fit(size, t.Fit) +} + +func (t Thumb) GetDurationEstimate(size io.Size) time.Duration { + return 31 * time.Nanosecond * time.Duration(t.Size(size).Area()) +} + +func (t *Thumb) resolvePath(originalPath string) string { + if t.PathTemplate == nil { + return "" + } + var rendered bytes.Buffer + dir, filename := filepath.Split(originalPath) + err := t.PathTemplate.Execute(&rendered, TemplateData{ + Dir: dir, + Filename: filename, + }) + if err != nil { + panic(err) + } + return rendered.String() +} + +func (t Thumb) Exists(ctx context.Context, id io.ImageId, path string) bool { + _, err := os.Stat(t.resolvePath(path)) + return !errors.Is(err, os.ErrNotExist) +} + +func (t Thumb) Get(ctx context.Context, id io.ImageId, path string) io.Result { + r := t.goimage.Get(ctx, id, t.resolvePath(path)) + r.Orientation = io.Normal + return r +} + +func (t Thumb) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + t.goimage.Reader(ctx, id, t.resolvePath(path), fn) +} + +func (t Thumb) Decode(ctx context.Context, r goio.Reader) io.Result { + return t.goimage.Decode(ctx, r) +} + +func (t Thumb) Set(ctx context.Context, id io.ImageId, path string, r io.Result) bool { + return false +} diff --git a/io_test.go b/io_test.go new file mode 100644 index 0000000..89ed253 --- /dev/null +++ b/io_test.go @@ -0,0 +1,275 @@ +package main + +import ( + "context" + "embed" + "fmt" + gio "io" + "math/rand" + "os" + "path" + "photofield/io" + "photofield/io/ffmpeg" + "photofield/io/goexif" + "photofield/io/goimage" + "photofield/io/ristretto" + "photofield/io/sqlite" + "photofield/io/thumb" + "testing" + "time" + + "zombiezen.com/go/sqlite/sqlitex" +) + +var dir = "photos/" + +// var dir = "E:/photos/" + +var cache = ristretto.New() +var goimg goimage.Image + +var ffmpegPath = ffmpeg.FindPath() + +var sources = io.Sources{ + // cache, + sqlite.New(path.Join(dir, "../data/photofield.thumbs.db"), embed.FS{}), + goexif.Exif{}, + thumb.New( + "S", + "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_S.jpg", + io.FitInside, + 120, + 120, + ), + thumb.New( + "SM", + "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_SM.jpg", + io.FitOutside, + 240, + 240, + ), + thumb.New( + "M", + "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_M.jpg", + io.FitOutside, + 320, + 320, + ), + thumb.New( + "B", + "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_B.jpg", + io.FitInside, + 640, + 640, + ), + thumb.New( + "XL", + "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_XL.jpg", + io.FitOutside, + 1280, + 1280, + ), + goimg, + ffmpeg.FFmpeg{ + Path: ffmpegPath, + Width: 128, + Height: 128, + Fit: io.FitInside, + }, + ffmpeg.FFmpeg{ + Path: ffmpegPath, + Width: 256, + Height: 256, + Fit: io.FitInside, + }, + ffmpeg.FFmpeg{ + Path: ffmpegPath, + Width: 512, + Height: 512, + Fit: io.FitInside, + }, + ffmpeg.FFmpeg{ + Path: ffmpegPath, + Width: 1280, + Height: 1280, + Fit: io.FitInside, + }, + ffmpeg.FFmpeg{ + Path: ffmpegPath, + Width: 4096, + Height: 4096, + Fit: io.FitInside, + }, +} + +var files = []struct { + name string + path string +}{ + {name: "P1110220", path: "test/P1110220.JPG"}, + + // {name: "logo", path: "formats/logo.png"}, + // {name: "P1110220", path: "formats/P1110220.jpg"}, + // {name: "palette", path: "formats/i_palettes01_04.png"}, + // {name: "cow", path: "formats/cow.avif"}, +} + +func BenchmarkSources(b *testing.B) { + ctx := context.Background() + for _, bm := range files { + bm := bm + p := path.Join(dir, bm.path) + id := io.ImageId(1) + r := goimg.Get(ctx, id, p) + for i := 0; i < 100; i++ { + cache.Set(ctx, id, p, r) + time.Sleep(1 * time.Millisecond) + r = cache.Get(ctx, id, p) + if r.Image != nil || r.Error != nil { + break + } + } + + b.Run(bm.name, func(b *testing.B) { + for _, l := range sources { + r := l.Get(ctx, id, p) + img := r.Image + err := r.Error + if err != nil { + b.Error(err) + } + if img == nil { + b.Errorf("image not found: %d %s", id, p) + continue + } else { + b.Logf("size: %d x %d", img.Bounds().Dx(), img.Bounds().Dy()) + } + + // b.ReportMetric(float64(img.Bounds().Dx()), "px") + + b.Run(fmt.Sprintf("%s-%dx%d", l.Name(), img.Bounds().Dx(), img.Bounds().Dy()), func(b *testing.B) { + for i := 0; i < b.N; i++ { + r := l.Get(ctx, id, p) + if r.Error != nil { + b.Error(r.Error) + } + } + }) + } + }) + } +} + +func TestCost(t *testing.T) { + cases := []struct { + zoom int + o io.Size + size io.Size + name string + }{ + // {zoom: -1, size: io.Size{X: 1, Y: 1}, name: "bla"}, + {zoom: 0, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 120, Y: 80}, name: "thumb-120x120-S"}, + {zoom: 1, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 240, Y: 160}, name: "sqlite"}, + {zoom: 2, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 480, Y: 320}, name: "thumb-320x320-M"}, + {zoom: 3, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 960, Y: 640}, name: "thumb-1280x1280-XL"}, + {zoom: 4, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 1920, Y: 1280}, name: "thumb-1280x1280-XL"}, + {zoom: 5, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 3840, Y: 2560}, name: "ffmpeg-4096x4096-in"}, + {zoom: 6, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 7680, Y: 5120}, name: "image"}, + {zoom: 7, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 15360, Y: 10240}, name: "image"}, + {zoom: 8, o: io.Size{X: 5472, Y: 3648}, size: io.Size{X: 30720, Y: 20480}, name: "image"}, + } + for _, c := range cases { + t.Run(fmt.Sprintf("%dx%d-%d", c.o.X, c.o.Y, c.zoom), func(t *testing.T) { + costs := sources.EstimateCost(c.o, c.size) + costs.Sort() + for i, c := range costs { + t.Logf("%4d %6f %s\n", i, c.Cost, c.Name()) + } + if costs[0].Name() != c.name { + t.Errorf("unexpected smallest source %s", costs[0].Name()) + } + }) + } +} + +func TestCostSmallest(t *testing.T) { + costs := sources.EstimateCost(io.Size{X: 5472, Y: 3648}, io.Size{X: 1, Y: 1}) + costs.Sort() + for i, c := range costs { + fmt.Printf("%4d %6f %s\n", i, c.Cost, c.Name()) + } + if costs[0].Name() != "thumb-120x120-S" { + t.Errorf("unexpected smallest source %s", costs[0].Name()) + } +} + +func BenchmarkSqlite(b *testing.B) { + + pool, err := sqlitex.Open(path.Join(dir, "test/photofield.thumbs.db"), 0, 10) + if err != nil { + panic(err) + } + c := pool.Get(context.Background()) + defer pool.Put(c) + + stmt := c.Prep(` + SELECT data + FROM thumb256 + WHERE id == ?;`) + defer stmt.Reset() + + maxid := int64(1000000) + + for i := 0; i < b.N; i++ { + id := 1 + rand.Int63n(maxid) + stmt.BindInt64(1, id) + exists, err := stmt.Step() + if err != nil { + b.Error(err) + } + if !exists { + b.Errorf("id not found: %d", id) + } + r := stmt.ColumnReader(1) + gio.ReadAll(r) + stmt.Reset() + } +} + +func BenchmarkFile(b *testing.B) { + + maxid := int64(1000000) + + for i := 0; i < b.N; i++ { + id := 1 + rand.Int63n(maxid) + path := path.Join(dir, fmt.Sprintf("test/thumb/%d.jpg", id)) + _, err := os.ReadFile(path) + if err != nil { + b.Error(err) + } + } +} + +// func BenchmarkThumbs(b *testing.B) { +// l := GoImage{} +// ctx := context.Background() +// for _, bc := range files { +// bc := bc // capture range variable +// b. +// // t.Run(tc.Name, func(t *testing.T) { +// // t.Parallel() +// // ... +// // }) +// } +// // for i := 0; i < b.N; i++ { +// // // l.Load(ctx, "photos/formats/logo.png") +// // // l.Load(ctx, "P1110220.JPG") +// // // _, err := l.Load(ctx, "C:/w/photofield/photos/formats/carina-nebula-high-resolution_52259221868_o.png") +// // _, err := l.Load(ctx, "C:/w/photofield/photos/formats/logo.png") +// // // _, err := l.Load(ctx, "C:/w/photofield/photos/formats/P1110220.jpg") +// // if err != nil { +// // b.Error(err) +// // } +// // // time.Sleep(100 * time.Millisecond) +// // } +// } diff --git a/justfile b/justfile index fedea60..ec55e94 100644 --- a/justfile +++ b/justfile @@ -34,6 +34,12 @@ db-add migration_file_name: db *args: migrate -database sqlite://data/photofield.cache.db -path db/migrations {{args}} +dbt-add migration_file_name: + migrate create -ext sql -dir db/migrations-thumbs -seq {{migration_file_name}} + +dbt *args: + migrate -database sqlite://data/photofield.thumbs.db -path db/migrations-thumbs {{args}} + api-codegen: oapi-codegen -generate="types,chi-server" -package=openapi api.yaml > internal/openapi/api.gen.go diff --git a/main.go b/main.go index c6156fd..5548a38 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" "io" @@ -67,6 +66,9 @@ var defaults AppConfig //go:embed db/migrations var migrations embed.FS +//go:embed db/migrations-thumbs +var migrationsThumbs embed.FS + //go:embed fonts/Roboto/Roboto-Regular.ttf var robotoRegular []byte @@ -90,10 +92,7 @@ var imageSource *image.Source var sceneSource *scene.SceneSource var collections []collection.Collection -var indexTasks sync.Map -var loadMetaOffset int64 -var loadColorOffset int64 -var loadAIOffset int64 +var globalTasks sync.Map var tileRequestsOut chan struct{} var tileRequests []TileRequest @@ -124,6 +123,18 @@ type Task struct { CollectionId string `json:"collection_id"` Done int `json:"done"` Pending int `json:"pending,omitempty"` + Offset int `json:"-"` + Queue string `json:"-"` +} + +func (t *Task) Counter() chan<- int { + counter := make(chan int, 10) + go func() { + for add := range counter { + t.Done += add + } + }() + return counter } type TileWriter func(w io.Writer) error @@ -237,11 +248,11 @@ func getCollectionById(id string) *collection.Collection { return nil } -func getIndexTask(collection *collection.Collection) Task { +func newFileIndexTask(collection *collection.Collection) Task { return Task{ - Type: string(openapi.TaskTypeINDEX), - Id: fmt.Sprintf("index-%v", collection.Id), - Name: fmt.Sprintf("Indexing %v", collection.Name), + Type: string(openapi.TaskTypeINDEXFILES), + Id: fmt.Sprintf("index-files-%v", collection.Id), + Name: fmt.Sprintf("Indexing files %v", collection.Name), CollectionId: collection.Id, Done: 0, } @@ -468,69 +479,42 @@ func gatherIntFromMetric(value *int, metric *io_prometheus_client.MetricFamily, func (*Api) GetTasks(w http.ResponseWriter, r *http.Request, params openapi.GetTasksParams) { - tasks := make([]Task, 0) - - if params.Type == nil || *params.Type == openapi.TaskTypeINDEX { - indexTasks.Range(func(key, value interface{}) bool { - task := value.(Task) - if params.CollectionId == nil || task.CollectionId == string(*params.CollectionId) { - tasks = append(tasks, task) - } - return true - }) - } - - loadMetaTask := Task{ - Type: string(openapi.TaskTypeLOADMETA), - Id: "load-meta", - Name: "Extracting metadata", - } - loadColorTask := Task{ - Type: string(openapi.TaskTypeLOADCOLOR), - Id: "load-color", - Name: "Extracting colors", - } - loadAITask := Task{ - Type: string(openapi.TaskTypeLOADAI), - Id: "load-ai", - Name: "Comprehending photos (AI)", - } - metrics, err := prometheus.DefaultGatherer.Gather() if err != nil { problem(w, r, http.StatusInternalServerError, "Unable to gather metrics") return } - for _, metric := range metrics { - gatherIntFromMetric(&loadMetaTask.Pending, metric, "pf_load_meta_pending") - gatherIntFromMetric(&loadMetaTask.Done, metric, "pf_load_meta_done") - gatherIntFromMetric(&loadColorTask.Pending, metric, "pf_load_color_pending") - gatherIntFromMetric(&loadColorTask.Done, metric, "pf_load_color_done") - gatherIntFromMetric(&loadAITask.Pending, metric, "pf_load_ai_pending") - gatherIntFromMetric(&loadAITask.Done, metric, "pf_load_ai_done") - } - if loadMetaTask.Pending > 0 { - offset := atomic.LoadInt64(&loadMetaOffset) - loadMetaTask.Done -= int(offset) - tasks = append(tasks, loadMetaTask) - } else { - atomic.StoreInt64(&loadMetaOffset, int64(loadMetaTask.Done)) - } - if loadColorTask.Pending > 0 { - offset := atomic.LoadInt64(&loadColorOffset) - loadColorTask.Done -= int(offset) - tasks = append(tasks, loadColorTask) - } else { - atomic.StoreInt64(&loadColorOffset, int64(loadColorTask.Done)) - } - if loadAITask.Pending > 0 { - offset := atomic.LoadInt64(&loadAIOffset) - loadAITask.Done -= int(offset) - tasks = append(tasks, loadAITask) - } else { - atomic.StoreInt64(&loadAIOffset, int64(loadAITask.Done)) - } + tasks := make([]Task, 0) + globalTasks.Range(func(key, value interface{}) bool { + t := value.(Task) + + add := true + if params.Type != nil && t.Type != string(*params.Type) { + add = false + } + if params.CollectionId != nil && t.CollectionId != string(*params.CollectionId) { + add = false + } + + if t.Queue != "" { + for _, m := range metrics { + gatherIntFromMetric(&t.Pending, m, fmt.Sprintf("pf_%s_pending", t.Queue)) + gatherIntFromMetric(&t.Done, m, fmt.Sprintf("pf_%s_done", t.Queue)) + } + if t.Pending > 0 { + t.Done -= t.Offset + } else { + t.Offset = t.Done + globalTasks.Store(t.Id, t) + add = false + } + } + if add { + tasks = append(tasks, t) + } + return true + }) sort.Slice(tasks, func(i, j int) bool { a := tasks[i] @@ -560,42 +544,45 @@ func (*Api) PostTasks(w http.ResponseWriter, r *http.Request) { switch data.Type { - case openapi.TaskTypeINDEX: - task := getIndexTask(collection) - stored, loaded := indexTasks.LoadOrStore(collection.Id, task) - task = stored.(Task) - if loaded { + case openapi.TaskTypeINDEXFILES: + task, existing := indexCollection(collection) + if existing { respond(w, r, http.StatusConflict, task) } else { respond(w, r, http.StatusAccepted, task) - indexCollection(collection) } - case openapi.TaskTypeLOADMETA: - imageSource.MetaQueue.AppendChan(collection.GetIdsUint32(imageSource)) - task := Task{ - Id: fmt.Sprintf("load-meta-%v", collection.Id), - CollectionId: collection.Id, - Name: fmt.Sprintf("Extracting metadata from %v", collection.Name), - } + case openapi.TaskTypeINDEXMETADATA: + imageSource.IndexMetadata(collection.Dirs, collection.IndexLimit, image.Missing{ + Metadata: true, + }) + stored, _ := globalTasks.Load("index-metadata") + task := stored.(Task) respond(w, r, http.StatusAccepted, task) - case openapi.TaskTypeLOADCOLOR: - imageSource.ColorQueue.AppendChan(collection.GetIdsUint32(imageSource)) - task := Task{ - Id: fmt.Sprintf("load-color-%v", collection.Id), - CollectionId: collection.Id, - Name: fmt.Sprintf("Extracting colors from %v", collection.Name), - } + case openapi.TaskTypeINDEXCONTENTS: + imageSource.IndexContents(collection.Dirs, collection.IndexLimit, image.Missing{ + Color: true, + Embedding: true, + }) + stored, _ := globalTasks.Load("index-contents") + task := stored.(Task) respond(w, r, http.StatusAccepted, task) - case openapi.TaskTypeLOADAI: - imageSource.AIQueue.AppendChan(collection.GetIdsUint32(imageSource)) - task := Task{ - Id: fmt.Sprintf("load-ai-%v", collection.Id), - CollectionId: collection.Id, - Name: fmt.Sprintf("Comprehending (AI) %v", collection.Name), - } + case openapi.TaskTypeINDEXCONTENTSCOLOR: + imageSource.IndexContents(collection.Dirs, collection.IndexLimit, image.Missing{ + Color: true, + }) + stored, _ := globalTasks.Load("index-contents") + task := stored.(Task) + respond(w, r, http.StatusAccepted, task) + + case openapi.TaskTypeINDEXCONTENTSAI: + imageSource.IndexContents(collection.Dirs, collection.IndexLimit, image.Missing{ + Embedding: true, + }) + stored, _ := globalTasks.Load("index-contents") + task := stored.(Task) respond(w, r, http.StatusAccepted, task) default: @@ -779,33 +766,13 @@ func (*Api) GetFilesIdOriginalFilename(w http.ResponseWriter, r *http.Request, i } func (*Api) GetFilesIdVariantsSizeFilename(w http.ResponseWriter, r *http.Request, id openapi.FileIdPathParam, size openapi.SizePathParam, filename openapi.FilenamePathParam) { - - imagePath, err := imageSource.GetImagePath(image.ImageId(id)) - if err == image.ErrNotFound { - problem(w, r, http.StatusNotFound, "Path not found") - return - } - - path := "" - thumbnails := imageSource.GetApplicableThumbnails(imagePath) - for i := range thumbnails { - thumbnail := thumbnails[i] - candidatePath := thumbnail.GetPath(imagePath) - if !imageSource.Exists(candidatePath) { - continue - } - if thumbnail.Name != string(size) { - continue + imageSource.GetImageReader(image.ImageId(id), string(size), func(rs io.ReadSeeker, err error) { + if err != nil { + problem(w, r, http.StatusBadRequest, err.Error()) + return } - path = candidatePath - } - - if path == "" || !imageSource.Exists(path) { - problem(w, r, http.StatusNotFound, "Variant not found") - return - } - - http.ServeFile(w, r, path) + http.ServeContent(w, r, string(filename), time.Time{}, rs) + }) } func AddPrefix(prefix string) func(next http.Handler) http.Handler { @@ -854,46 +821,29 @@ func expandCollections(collections *[]collection.Collection) { *collections = expanded } -func indexCollections(collections *[]collection.Collection) (ok bool) { - go func() { - for _, collection := range *collections { - counter := make(chan int, 10) - go func(id string, counter chan int) { - task := getIndexTask(&collection) - for add := range counter { - task.Done += add - indexTasks.Store(id, task) - } - indexTasks.Delete(id) - }(collection.Id, counter) - for _, dir := range collection.Dirs { - imageSource.IndexFiles(dir, collection.IndexLimit, counter) - } - close(counter) - } - }() - return true -} +func indexCollection(collection *collection.Collection) (task Task, existing bool) { + task = newFileIndexTask(collection) + stored, existing := globalTasks.LoadOrStore(task.Id, task) + task = stored.(Task) + if existing { + return + } + + counter := task.Counter() -func indexCollection(collection *collection.Collection) { - counter := make(chan int, 10) - go func() { - task := getIndexTask(collection) - for add := range counter { - task.Done += add - indexTasks.Store(collection.Id, task) - } - indexTasks.Delete(collection.Id) - }() go func() { - log.Printf("indexing %s\n", collection.Id) + log.Printf("indexing files %s\n", collection.Id) for _, dir := range collection.Dirs { - log.Printf("indexing %s %s\n", collection.Id, dir) + log.Printf("indexing files %s dir %s\n", collection.Id, dir) imageSource.IndexFiles(dir, collection.IndexLimit, counter) } - imageSource.IndexAI(collection.Dirs, collection.IndexLimit) + // imageSource.IndexAI(collection.Dirs, collection.IndexLimit) + imageSource.IndexMetadata(collection.Dirs, collection.IndexLimit, image.Missing{}) + imageSource.IndexContents(collection.Dirs, collection.IndexLimit, image.Missing{}) + globalTasks.Delete(task.Id) close(counter) }() + return } func loadConfiguration(path string) AppConfig { @@ -922,10 +872,6 @@ func loadConfiguration(path string) AppConfig { } } - for i := range appConfig.Media.Thumbnails { - appConfig.Media.Thumbnails[i].Init() - } - appConfig.Media.AI = appConfig.AI return appConfig @@ -1019,7 +965,7 @@ func main() { configurationPath := filepath.Join(dataDir, "configuration.yaml") appConfig := loadConfiguration(configurationPath) - appConfig.Media.DatabasePath = filepath.Join(dataDir, "photofield.cache.db") + appConfig.Media.DataDir = dataDir if len(appConfig.Collections) > 0 { defaultSceneConfig.Collection = appConfig.Collections[0] @@ -1029,7 +975,7 @@ func main() { defaultSceneConfig.Render = appConfig.Render tileRequestConfig = appConfig.TileRequests - imageSource = image.NewSource(appConfig.Media, migrations) + imageSource = image.NewSource(appConfig.Media, migrations, migrationsThumbs) defer imageSource.Close() if *vacuumPtr { @@ -1071,6 +1017,22 @@ func main() { log.Printf(" %v - %v files indexed %v ago", collection.Name, collection.IndexedCount, indexedAgo) } + metadataTask := Task{ + Type: string(openapi.TaskTypeINDEXMETADATA), + Id: "index-metadata", + Name: "Indexing metadata", + Queue: "index_metadata", + } + globalTasks.Store(metadataTask.Id, metadataTask) + + contentsTask := Task{ + Type: string(openapi.TaskTypeINDEXCONTENTS), + Id: "index-contents", + Name: "Indexing contents", + Queue: "index_contents", + } + globalTasks.Store(contentsTask.Id, contentsTask) + // renderSample(defaultSceneConfig.Config, sceneSource.GetScene(defaultSceneConfig, imageSource)) addr, exists := os.LookupEnv("PHOTOFIELD_ADDRESS") diff --git a/ui/src/App.vue b/ui/src/App.vue index a4f79de..bc0c925 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -49,9 +49,9 @@ @@ -147,7 +147,6 @@ export default { immersive: false, collectionMenuOpen: false, scrollbar: null, - scene: null, scenes: [], viewerTasks: null, searchActive: false, @@ -262,6 +261,9 @@ export default { } return null; }, + scrollScene() { + return this.scenes?.find(scene => scene.name == "Scroll"); + }, }, methods: { toggleFocus() { @@ -273,7 +275,7 @@ export default { this.recreateEvent.emit(); }, async reindex() { - await createTask("INDEX", this.collection?.id); + await createTask("INDEX_FILES", this.collection?.id); await this.remoteTasksUpdateUntilDone(); this.recreateScene(); }, diff --git a/ui/src/components/CollectionDebug.vue b/ui/src/components/CollectionDebug.vue index e54b018..6e120f1 100644 --- a/ui/src/components/CollectionDebug.vue +++ b/ui/src/components/CollectionDebug.vue @@ -1,11 +1,12 @@