diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5e01a8..b0611da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,29 +16,29 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version: 1.19 - name: Set up Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: '16' - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v4 with: distribution: goreleaser version: latest diff --git a/README.md b/README.md index 00f3789..d83b001 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ Photofield is a photo viewer built to mainly push the limits of what is possible in terms of the number of photos visible at the same time and at the speed at which they are displayed. The goal is to be as fast or faster than Google Photos on commodity hardware while displaying more photos at the same time. It is -non-invasive and at this point meant to be used to complement other photo -gallery software. +non-invasive and can be used either completely standalone or complementing other +photo gallery software. ### Features @@ -86,13 +86,16 @@ layouts. able to search for photo contents using words like "beach sunset", "a couple kissing", or "cat eyes". ![semantic search for "cat eyes"](docs/assets/semantic-search.jpg) -* **Reuse of existing thumbnails**. Do you have hundreds of gigabytes of - existing thumbnails from an existing system? Me too! Let's just reuse those. - Here are the currently supported thumbnail sources: +* **Flexible media/thumbnail system**. Do you have hundreds of gigabytes of existing + thumbnails from an existing system? Me too! Let's reuse those. Don't have any? + No worries, they will be generated automatically to speed up display. Here are + the currently supported thumbnail sources: + * Bespoke SQLite thumbnail database - `photofield.thumbs.db`. * Synology Moments / Photo Station auto-generated thumbnails in `@eaDir`. - * Embedded JPEG thumbnails (`ThumbnailImage` Exif tag). - * Limited support for extension via `thumbnails` section of - the [Configuration]. + * Embedded JPEG thumbnails - `ThumbnailImage` Exif tag. + * Native Go [image](https://pkg.go.dev/image) package. + * FFmpeg on-the-fly conversion - thumbnails and full sized variants. + * Configurable via the `sources` section of the [Configuration]. * Please [open an issue] for other systems, bonus points for an idea on how to integrate! * **Single file binary**. Thanks to [Go] and [GoReleaser], all the dependencies @@ -112,7 +115,6 @@ transcoding supported right now. ### Limitations -* **No thumbnail generation**. Only pre-generated thumbnails are supported. * **No photo details (yet)**. There is no way to show metadata of a photo in the UI at this point. * **Not optimized for many clients**. As a lot of the normally client-side @@ -172,7 +174,7 @@ default. For further configuration, create a `configuration.yaml` in the services: photofield: - image: ghcr.io/smilyorg/photofield + image: ghcr.io/smilyorg/photofield:latest ports: - 8080:8080 volumes: diff --git a/api.yaml b/api.yaml index 0d2bd10..a6d1db6 100644 --- a/api.yaml +++ b/api.yaml @@ -101,6 +101,11 @@ paths: schema: $ref: "#/components/schemas/LayoutType" + - name: sort + in: query + schema: + $ref: "#/components/schemas/Sort" + - name: search in: query schema: @@ -189,6 +194,15 @@ paths: required: true schema: $ref: "#/components/schemas/TileCoord" + + - name: sources + in: query + schema: + type: array + items: + type: string + style: form + explode: false - name: debug_overdraw in: query @@ -620,6 +634,8 @@ components: $ref: "#/components/schemas/LayoutType" search: $ref: "#/components/schemas/Search" + sort: + $ref: "#/components/schemas/Sort" TaskType: type: string @@ -675,6 +691,10 @@ components: minimum: 0 example: 0 + Sources: + type: array + + Bounds: type: object properties: @@ -694,6 +714,9 @@ components: Search: type: string + Sort: + type: string + LayoutType: type: string enum: diff --git a/defaults.yaml b/defaults.yaml index 7f22540..0c1c06c 100644 --- a/defaults.yaml +++ b/defaults.yaml @@ -155,26 +155,52 @@ media: # fit: The aspect ratio fit to use while resizing # path: Path to the FFmpeg binary, uses the one in PATH if not set # + source_types: + sqlite: + path: photofield.thumbs.db + cost: + time: 5ms + + goexif: + extensions: [".jpg", ".jpeg"] + width: 256 + height: 256 + fit: "INSIDE" + cost: + time: 30ms # 15ms + extra cost + + image: + extensions: [".jpg", ".jpeg", ".png"] + cost: + time_per_original_megapixel: 90ms + + thumb: + fit: "INSIDE" + cost: + time_per_resized_megapixel: 140ms + + ffmpeg: + fit: "INSIDE" + cost: + time_per_original_megapixel: 220ms + + sources: # Internal thumbnail database - type: sqlite - path: photofield.thumbs.db + # Embedded JPEG thumbnails - type: goexif - extensions: [".jpg", ".jpeg"] - width: 256 - height: 256 - fit: "INSIDE" # Native image decoding - type: image - extensions: [".jpg", ".jpeg", ".png"] - # + + # # Synology Moments / Photo Station thumbnails - # + # - name: S type: thumb path: "{{.Dir}}@eaDir/{{.Filename}}/SYNOPHOTO_THUMB_S.jpg" @@ -264,7 +290,6 @@ media: sources: # Internal thumbnail database - type: sqlite - path: photofield.thumbs.db # Synology Moments / Photo Station thumbnail - name: SM @@ -277,10 +302,6 @@ media: # Embedded JPEG thumbnails - type: goexif - extensions: [".jpg", ".jpeg"] - width: 256 - height: 256 - fit: "INSIDE" # Synology Moments / Photo Station thumbnail - name: S @@ -304,7 +325,6 @@ media: # Native decoding (resized to 256px x 256px) - type: image - extensions: [".jpg", ".jpeg", ".png"] width: 256 height: 256 @@ -312,4 +332,3 @@ media: # so that it persists and can be reused while rendering. sink: type: sqlite - path: photofield.thumbs.db diff --git a/docker-compose.yaml b/docker-compose.yaml index 0124e49..22e05ba 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,13 @@ services: volumes: - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + pyroscope: + image: "pyroscope/pyroscope:latest" + ports: + - "4040:4040" + command: + - "server" + grafana: build: ./docker/grafana/ environment: diff --git a/docker/grafana/Dockerfile b/docker/grafana/Dockerfile index e330aef..d6c0ebc 100644 --- a/docker/grafana/Dockerfile +++ b/docker/grafana/Dockerfile @@ -1,3 +1,3 @@ -FROM grafana/grafana:8.0.6 +FROM grafana/grafana:9.4.7 COPY provisioning /provisioning COPY dashboards /var/lib/grafana/dashboards diff --git a/docker/grafana/dashboards/photofield.json b/docker/grafana/dashboards/photofield.json index 9e6bc7a..ca3b1eb 100644 --- a/docker/grafana/dashboards/photofield.json +++ b/docker/grafana/dashboards/photofield.json @@ -3,25 +3,33 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "iteration": 1674068728123, "links": [], + "liveNow": false, "panels": [ { "cards": { - "cardPadding": 0, - "cardRound": null + "cardPadding": 0 }, "color": { "cardColor": "#FADE2A", @@ -31,7 +39,25 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 24, @@ -46,10 +72,51 @@ "show": false }, "maxDataPoints": 100, - "pluginVersion": "8.0.6", + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#FADE2A", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Turbo", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.4.7", "reverseYBuckets": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "sum(increase(pf_http_latency_bucket{path=\"$path\"}[10s])) by (le)", "format": "heatmap", @@ -68,35 +135,546 @@ "xAxis": { "show": true }, - "xBucketNumber": null, - "xBucketSize": null, "yAxis": { - "decimals": null, "format": "s", "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null + "show": true }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yBucketBound": "auto" }, { - "datasource": "Prometheus", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, + "id": 233, + "panels": [], + "title": "Sources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 214, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "interval": "", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Image Load Latency (p90)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 230, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "interval": "", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Image Load Latency per Original Megapixel (p90)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 231, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "interval": "", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Image Load Latency per Resized Megapixel (p90)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 249, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{source}}", + "range": false, + "refId": "A" + } + ], + "title": "Image Load Latency (p90)", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 250, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{source}}", + "range": false, + "refId": "A" + } + ], + "title": "Image Load Latency per Original Megapixel (p90)", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 251, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{source}}", + "range": false, + "refId": "A" + } + ], + "title": "Image Load Latency per Resized Megapixel (p90)", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, "id": 143, - "title": "Loading", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "refId": "A" + } + ], + "title": "Index", "type": "row" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -124,7 +702,7 @@ "h": 5, "w": 12, "x": 0, - "y": 9 + "y": 26 }, "id": 93, "options": { @@ -142,23 +720,31 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "repeatDirection": "h", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, - "expr": "label_replace({__name__=~\"pf_load_${loader}_pending\"}, \"name\", \"$1\", \"__name__\", \"pf_load_(.*)_pending\")", + "expr": "label_replace({__name__=~\"pf_index_${indexer}_pending\"}, \"name\", \"$1\", \"__name__\", \"pf_index_(.*)_pending\")", "hide": false, "interval": "", "legendFormat": "{{name}}", + "range": true, "refId": "B" } ], - "title": "Pending Loads", + "title": "Pending", "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -182,7 +768,7 @@ "h": 5, "w": 6, "x": 12, - "y": 9 + "y": 26 }, "id": 109, "options": { @@ -200,12 +786,15 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", - "repeat": null, + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, - "expr": "rate(pf_load_meta_done[10s])", + "expr": "rate(pf_index_metadata_done[10s])", "hide": false, "instant": false, "interval": "", @@ -213,11 +802,14 @@ "refId": "B" } ], - "title": "Meta Load Rate", + "title": "Metadata Indexing Rate", "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -241,7 +833,7 @@ "h": 5, "w": 6, "x": 18, - "y": 9 + "y": 26 }, "id": 126, "options": { @@ -259,11 +851,15 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, - "expr": "rate(pf_load_color_done[10s])", + "expr": "rate(pf_index_contents_done[10s])", "hide": false, "instant": false, "interval": "", @@ -271,92 +867,38 @@ "refId": "B" } ], - "title": "Color Load Rate", + "title": "Contents Indexing Rate", "type": "stat" }, { - "cards": { - "cardPadding": null, - "cardRound": null + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateTurbo", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "Prometheus", - "description": "", "gridPos": { - "h": 7, + "h": 1, "w": 24, "x": 0, - "y": 14 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 166, - "legend": { - "show": false + "y": 31 }, - "maxDataPoints": 100, - "pluginVersion": "8.0.6", - "repeat": "source", - "repeatDirection": "v", - "reverseYBuckets": false, + "id": 56, "targets": [ { - "exemplar": true, - "expr": "sum(increase(pf_source_latency_bucket{source=\"$source\"}[10s])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "refId": "A" } ], - "title": "Latency for $source", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "datasource": "Prometheus", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 56, "title": "System", "type": "row" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -384,7 +926,7 @@ "h": 5, "w": 6, "x": 0, - "y": 50 + "y": 32 }, "id": 38, "options": { @@ -402,9 +944,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(process_cpu_seconds_total[2s])", "interval": "", @@ -416,7 +962,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -444,7 +993,7 @@ "h": 5, "w": 6, "x": 6, - "y": 50 + "y": 32 }, "id": 25, "options": { @@ -462,9 +1011,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "go_memstats_alloc_bytes", "interval": "", @@ -476,7 +1029,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -500,7 +1056,7 @@ "h": 5, "w": 12, "x": 12, - "y": 50 + "y": 32 }, "id": 33, "options": { @@ -518,9 +1074,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(go_memstats_mallocs_total[10s])", "hide": false, @@ -529,6 +1089,10 @@ "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(go_memstats_frees_total[10s])", "hide": false, @@ -541,7 +1105,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -569,7 +1136,7 @@ "h": 5, "w": 12, "x": 0, - "y": 55 + "y": 37 }, "id": 37, "options": { @@ -587,9 +1154,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "go_goroutines", "interval": "", @@ -597,6 +1168,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "go_threads", "hide": false, @@ -609,7 +1184,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -637,7 +1215,7 @@ "h": 5, "w": 12, "x": 12, - "y": 55 + "y": 37 }, "id": 35, "options": { @@ -655,9 +1233,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "go_gc_duration_seconds{quantile=~\"0.75|1\"}", "hide": false, @@ -671,7 +1253,10 @@ "type": "stat" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -701,11 +1286,13 @@ "h": 6, "w": 12, "x": 0, - "y": 60 + "y": 42 }, "id": 75, "options": { "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, "orientation": "auto", "reduceOptions": { "calcs": [ @@ -717,9 +1304,13 @@ "showUnfilled": true, "text": {} }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "label_replace({__name__=~\"pf_${cache}_cache_ratio\"}, \"name\", \"$1\", \"__name__\", \"pf_(.*)_cache_ratio\")", "format": "time_series", @@ -730,6 +1321,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "{__name__=~\"pf_${cache}_cache_cost_active\"}", "hide": true, @@ -744,7 +1339,10 @@ "type": "bargauge" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -769,7 +1367,7 @@ "h": 6, "w": 12, "x": 12, - "y": 60 + "y": 42 }, "id": 92, "options": { @@ -787,9 +1385,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "9.4.7", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "label_replace({__name__=~\"pf_${cache}_cache_cost_active\"}, \"name\", \"$1\", \"__name__\", \"pf_(.*)_cache_cost_active\")", "hide": false, @@ -805,17 +1407,23 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 66 + "y": 48 }, "id": 12, "panels": [ { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -856,8 +1464,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -889,7 +1496,7 @@ "h": 6, "w": 12, "x": 0, - "y": 23 + "y": 47 }, "id": 13, "options": { @@ -898,7 +1505,8 @@ "last" ], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single" @@ -906,6 +1514,10 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "pf_${cache}_cache_ratio", "hide": false, @@ -914,6 +1526,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "pf_${cache}_cache_cost_active", "hide": false, @@ -927,7 +1543,10 @@ "type": "timeseries" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -966,8 +1585,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -982,14 +1600,15 @@ "h": 6, "w": 12, "x": 12, - "y": 23 + "y": 47 }, "id": 17, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single" @@ -997,6 +1616,10 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_sets_dropped[1m])", "interval": "", @@ -1004,6 +1627,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_sets_rejected[1m])", "hide": false, @@ -1012,6 +1639,10 @@ "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_gets_dropped[1m])", "hide": false, @@ -1020,6 +1651,10 @@ "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_gets_kept[1m])", "hide": false, @@ -1032,7 +1667,10 @@ "type": "timeseries" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -1071,8 +1709,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1087,14 +1724,15 @@ "h": 6, "w": 12, "x": 0, - "y": 29 + "y": 53 }, "id": 14, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single" @@ -1102,6 +1740,10 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_keys_added[1m])", "interval": "", @@ -1109,6 +1751,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "-rate(pf_${cache}_cache_keys_evicted[1m])", "hide": false, @@ -1117,6 +1763,10 @@ "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_keys_updated[1m])", "hide": false, @@ -1129,7 +1779,10 @@ "type": "timeseries" }, { - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "fieldConfig": { "defaults": { "color": { @@ -1168,8 +1821,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1210,14 +1862,15 @@ "h": 6, "w": 12, "x": 12, - "y": 29 + "y": 53 }, "id": 15, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single" @@ -1225,6 +1878,10 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "rate(pf_${cache}_cache_cost_added[1m])", "instant": false, @@ -1233,6 +1890,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "exemplar": true, "expr": "-rate(pf_${cache}_cache_cost_evicted[1m])", "hide": false, @@ -1248,18 +1909,27 @@ } ], "repeat": "cache", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "refId": "A" + } + ], "title": "$cache cache", "type": "row" } ], "refresh": "2s", - "schemaVersion": 30, + "revision": 1, + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [ { - "allValue": null, "current": { "selected": false, "text": [ @@ -1269,11 +1939,8 @@ "$__all" ] }, - "description": null, - "error": null, "hide": 0, "includeAll": true, - "label": null, "multi": true, "name": "cache", "options": [ @@ -1297,26 +1964,20 @@ "text": "image_info", "value": "image_info" }, - { - "selected": false, - "text": "file_exists", - "value": "file_exists" - }, { "selected": false, "text": "path", "value": "path" } ], - "query": "scene,image,image_info,file_exists,path", + "query": "scene,image,image_info,path", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -1324,13 +1985,11 @@ "$__all" ] }, - "description": null, - "error": null, "hide": 0, "includeAll": true, - "label": null, + "label": "", "multi": true, - "name": "loader", + "name": "indexer", "options": [ { "selected": true, @@ -1339,31 +1998,31 @@ }, { "selected": false, - "text": "meta", - "value": "meta" + "text": "metadata", + "value": "metadata" }, { "selected": false, - "text": "color", - "value": "color" + "text": "contents", + "value": "contents" } ], - "query": "meta,color", + "query": "metadata,contents", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { - "allValue": null, "current": { "selected": false, "text": "/scenes/{scene_id}/tiles", "value": "/scenes/{scene_id}/tiles" }, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, "definition": "label_values(path)", - "description": null, - "error": null, "hide": 0, "includeAll": false, "label": "", @@ -1381,20 +2040,25 @@ "type": "query" }, { - "allValue": null, + "allValue": "", "current": { "selected": false, - "text": "All", - "value": "$__all" + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" }, - "datasource": "Prometheus", "definition": "label_values(source)", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "", - "multi": false, + "multi": true, "name": "source", "options": [], "query": { @@ -1410,7 +2074,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-15m", "to": "now" }, "timepicker": { @@ -1432,5 +2096,6 @@ "timezone": "", "title": "Photofield", "uid": "9sQ5hGGnk", - "version": 7 + "version": 13, + "weekStart": "" } \ No newline at end of file diff --git a/docker/prometheus/Dockerfile b/docker/prometheus/Dockerfile index c12c06a..5fc2ddc 100644 --- a/docker/prometheus/Dockerfile +++ b/docker/prometheus/Dockerfile @@ -1,2 +1,2 @@ -FROM prom/prometheus:v2.28.1 +FROM prom/prometheus:v2.43.0 COPY prometheus.yml /etc/prometheus/ diff --git a/go.mod b/go.mod index 857a3f6..5d712f2 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 + github.com/pyroscope-io/client v0.7.0 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/sheerun/queue v1.0.1 github.com/tdewolff/canvas v0.0.0-20200504121106-e2600b35c365 @@ -64,6 +65,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect + github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tdewolff/minify/v2 v2.7.1-0.20200112204046-70870d25a935 // indirect github.com/tdewolff/parse/v2 v2.4.2 // indirect diff --git a/go.sum b/go.sum index 9307701..33d822b 100644 --- a/go.sum +++ b/go.sum @@ -560,6 +560,10 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk= +github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= +github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE= +github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= diff --git a/internal/image/cache.go b/internal/image/cache.go index d1e4566..eaeb7d4 100644 --- a/internal/image/cache.go +++ b/internal/image/cache.go @@ -44,20 +44,6 @@ func newInfoCache() InfoCache { } } -func newFileExistsCache() *ristretto.Cache { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e7, // number of keys to track frequency of (10M). - MaxCost: 1 << 24, // maximum cost of cache (16MB). - BufferItems: 64, // number of keys per Get buffer. - Metrics: true, - }) - if err != nil { - panic(err) - } - metrics.AddRistretto("file_exists_cache", cache) - return cache -} - type PathCache struct { cache *ristretto.Cache } diff --git a/internal/image/source.go b/internal/image/source.go index 7ab239c..c5e0231 100644 --- a/internal/image/source.go +++ b/internal/image/source.go @@ -19,7 +19,6 @@ import ( ioristretto "photofield/io/ristretto" "photofield/io/sqlite" - "github.com/dgraph-io/ristretto" "github.com/docker/go-units" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -120,6 +119,7 @@ type Config struct { DateFormats []string `json:"date_formats"` Images FileConfig `json:"images"` Videos FileConfig `json:"videos"` + SourceTypes SourceTypeMap `json:"source_types"` Sources SourceConfigs `json:"sources"` Thumbnail ThumbnailConfig `json:"thumbnail"` @@ -133,15 +133,16 @@ type FileConfig struct { type Source struct { Config - Sources io.Sources - SourcesLatencyHistogram *prometheus.HistogramVec + Sources io.Sources + SourceLatencyHistogram *prometheus.HistogramVec + SourcePerOriginalMegapixelLatencyHistogram *prometheus.HistogramVec + SourcePerResizedMegapixelLatencyHistogram *prometheus.HistogramVec decoder *Decoder database *Database - imageInfoCache InfoCache - pathCache PathCache - fileExistsCache *ristretto.Cache + imageInfoCache InfoCache + pathCache PathCache metadataQueue queue.Queue contentsQueue queue.Queue @@ -159,10 +160,9 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S source.decoder = NewDecoder(config.ExifToolCount) source.database = NewDatabase(filepath.Join(config.DataDir, "photofield.cache.db"), migrations) source.imageInfoCache = newInfoCache() - source.fileExistsCache = newFileExistsCache() source.pathCache = newPathCache() - source.SourcesLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + source.SourceLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.Namespace, Name: "source_latency", Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, @@ -170,11 +170,28 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S []string{"source"}, ) + source.SourcePerOriginalMegapixelLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Name: "source_per_original_megapixel_latency", + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + }, + []string{"source"}, + ) + + source.SourcePerResizedMegapixelLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Name: "source_per_resized_megapixel_latency", + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + }, + []string{"source"}, + ) + env := SourceEnvironment{ - FFmpegPath: ffmpeg.FindPath(), - Migrations: migrationsThumbs, - ImageCache: ioristretto.New(), - DataDir: config.DataDir, + SourceTypes: config.SourceTypes, + FFmpegPath: ffmpeg.FindPath(), + Migrations: migrationsThumbs, + ImageCache: ioristretto.New(), + DataDir: config.DataDir, } // Sources used for rendering diff --git a/internal/image/sourceConfig.go b/internal/image/sourceConfig.go index 4ff11c0..b292468 100644 --- a/internal/image/sourceConfig.go +++ b/internal/image/sourceConfig.go @@ -6,6 +6,7 @@ import ( "path/filepath" "photofield/io" "photofield/io/cached" + "photofield/io/configured" "photofield/io/ffmpeg" "photofield/io/filtered" "photofield/io/goexif" @@ -16,6 +17,7 @@ import ( "strings" "github.com/goccy/go-yaml" + "github.com/imdario/mergo" ) const ( @@ -41,6 +43,7 @@ func (t *SourceType) UnmarshalYAML(b []byte) error { type SourceConfig struct { Name string `json:"name"` + Cost configured.Cost `json:"cost"` Type SourceType `json:"type"` Path string `json:"path"` Width int `json:"width"` @@ -49,6 +52,22 @@ type SourceConfig struct { Extensions []string `json:"extensions"` } +type SourceTypeMap map[SourceType]SourceConfig + +func (smt *SourceTypeMap) UnmarshalYAML(b []byte) error { + var m map[SourceType]SourceConfig + if err := yaml.Unmarshal(b, &m); err != nil { + return err + } + *smt = make(map[SourceType]SourceConfig) + for st, sc := range m { + st = SourceType(strings.ToUpper(string(st))) + sc.Type = st + (*smt)[st] = sc + } + return nil +} + type ThumbnailConfig struct { Sources SourceConfigs `json:"sources"` Generators SourceConfigs `json:"generators"` @@ -57,14 +76,26 @@ type ThumbnailConfig struct { // SourceEnvironment is the environment for creating sources type SourceEnvironment struct { - DataDir string - FFmpegPath string - Migrations embed.FS - ImageCache *ristretto.Ristretto - Databases map[string]*sqlite.Source + SourceTypes SourceTypeMap + DataDir string + FFmpegPath string + Migrations embed.FS + ImageCache *ristretto.Ristretto + Databases map[string]*sqlite.Source } func (c SourceConfig) NewSource(env *SourceEnvironment) (io.Source, error) { + // Merge the source config with the source type config + if st, ok := env.SourceTypes[c.Type]; ok { + // println("merging source config with source type config", c.Type, st.Type, st.Cost.Time, st.Cost.TimePerResizedMegapixel, st.Cost.TimePerOriginalMegapixel) + err := mergo.Merge(&c, &st) + if err != nil { + return nil, err + } + } + + var s io.Source + switch c.Type { case SourceTypeSqlite: @@ -72,48 +103,76 @@ func (c SourceConfig) NewSource(env *SourceEnvironment) (io.Source, error) { if ok { return existing, nil } - s := sqlite.New( + if c.Path == "" { + return nil, fmt.Errorf("missing path for SQLITE source") + } + sq := sqlite.New( filepath.Join(env.DataDir, c.Path), env.Migrations, ) if env.Databases == nil { env.Databases = make(map[string]*sqlite.Source) } - env.Databases[c.Path] = s - return s, nil + env.Databases[c.Path] = sq + s = sq case SourceTypeGoexif: - return goexif.Exif{ + s = goexif.Exif{ Width: c.Width, Height: c.Height, - }, nil + } case SourceTypeThumb: - return thumb.New( + s = thumb.New( c.Name, c.Path, c.Fit, c.Width, c.Height, - ), nil + ) case SourceTypeImage: - return goimage.Image{ + s = goimage.Image{ Width: c.Width, Height: c.Height, - }, nil + } case SourceTypeFFmpeg: - return ffmpeg.FFmpeg{ + s = ffmpeg.FFmpeg{ Path: env.FFmpegPath, Width: c.Width, Height: c.Height, Fit: c.Fit, - }, nil + } default: return nil, fmt.Errorf("unknown source type: %s", c.Type) } + + if env.ImageCache != nil { + // Add caching layer + s = &cached.Cached{ + Source: s, + Cache: *env.ImageCache, + } + } + // Add filtering layer + if len(c.Extensions) > 0 { + s = &filtered.Filtered{ + Source: s, + Extensions: c.Extensions, + } + } + + s = configured.New( + c.Name, + c.Cost, + s, + ) + + // println(s.Name(), c.Cost.Time.String(), c.Cost.TimePerOriginalMegapixel.String(), c.Cost.TimePerResizedMegapixel.String()) + + return s, nil } type SourceConfigs []SourceConfig @@ -127,20 +186,6 @@ func (cfgs SourceConfigs) NewSources(env *SourceEnvironment) ([]io.Source, error if err != nil { return nil, err } - if env.ImageCache != nil { - // Add caching layer - s = &cached.Cached{ - Source: s, - Cache: *env.ImageCache, - } - } - // Add filtering layer - if len(c.Extensions) > 0 { - s = &filtered.Filtered{ - Source: s, - Extensions: c.Extensions, - } - } sources = append(sources, s) } return sources, nil diff --git a/internal/layout/album.go b/internal/layout/album.go index 12086f4..f42c4e9 100644 --- a/internal/layout/album.go +++ b/internal/layout/album.go @@ -77,7 +77,7 @@ func LayoutAlbum(layout Layout, collection collection.Collection, scene *render. limit := collection.Limit infos := collection.GetInfos(source, image.ListOptions{ - OrderBy: image.DateAsc, + OrderBy: image.ListOrder(layout.Order), Limit: limit, }) diff --git a/internal/layout/common.go b/internal/layout/common.go index e08095c..9fb6960 100644 --- a/internal/layout/common.go +++ b/internal/layout/common.go @@ -24,8 +24,28 @@ const ( Strip Type = "STRIP" ) +type Order int + +const ( + None Order = iota + DateAsc Order = iota + DateDesc Order = iota +) + +func OrderFromSort(s string) Order { + switch s { + case "+date": + return DateAsc + case "-date": + return DateDesc + default: + return None + } +} + type Layout struct { - Type Type `json:"type"` + Type Type `json:"type"` + Order Order `json:"order"` ViewportWidth float64 ViewportHeight float64 ImageHeight float64 diff --git a/internal/layout/strip.go b/internal/layout/strip.go index 47478b0..96cc921 100644 --- a/internal/layout/strip.go +++ b/internal/layout/strip.go @@ -26,7 +26,7 @@ func LayoutStrip(layout Layout, collection collection.Collection, scene *render. ) } else { infos = collection.GetInfos(source, image.ListOptions{ - OrderBy: image.DateAsc, + OrderBy: image.ListOrder(layout.Order), Limit: limit, }) } diff --git a/internal/layout/timeline.go b/internal/layout/timeline.go index c15340e..5b9100a 100644 --- a/internal/layout/timeline.go +++ b/internal/layout/timeline.go @@ -72,7 +72,7 @@ func LayoutTimeline(layout Layout, collection collection.Collection, scene *rend limit := collection.Limit infos := collection.GetInfos(source, image.ListOptions{ - OrderBy: image.DateDesc, + OrderBy: image.ListOrder(layout.Order), Limit: limit, }) diff --git a/internal/layout/wall.go b/internal/layout/wall.go index 8f86223..ed115bd 100644 --- a/internal/layout/wall.go +++ b/internal/layout/wall.go @@ -13,7 +13,7 @@ import ( func LayoutWall(layout Layout, collection collection.Collection, scene *render.Scene, source *image.Source) { infos := collection.GetInfos(source, image.ListOptions{ - OrderBy: image.DateAsc, + OrderBy: image.ListOrder(layout.Order), Limit: collection.Limit, }) @@ -34,6 +34,9 @@ func LayoutWall(layout Layout, collection collection.Collection, scene *render.S photoCount := len(section.infos) edgeCount := int(math.Sqrt(float64(photoCount))) + if edgeCount < 1 { + edgeCount = 1 + } scene.Bounds.W = layout.ViewportWidth cols := edgeCount @@ -51,7 +54,7 @@ func LayoutWall(layout Layout, collection collection.Collection, scene *render.S rows := int(math.Ceil(float64(photoCount) / float64(cols))) - scene.Bounds.H = math.Ceil(float64(rows)) * (imageHeight + layoutConfig.LineSpacing) + scene.Bounds.H = float64(rows) * (imageHeight + layoutConfig.LineSpacing) sceneMargin := 10. layoutConfig.ImageHeight = imageHeight diff --git a/internal/openapi/api.gen.go b/internal/openapi/api.gen.go index 0e6edd6..34580e9 100644 --- a/internal/openapi/api.gen.go +++ b/internal/openapi/api.gen.go @@ -126,6 +126,7 @@ type SceneParams struct { ImageHeight *ImageHeight `json:"image_height,omitempty"` Layout LayoutType `json:"layout"` Search *Search `json:"search,omitempty"` + Sort *Sort `json:"sort,omitempty"` ViewportHeight ViewportHeight `json:"viewport_height"` ViewportWidth ViewportWidth `json:"viewport_width"` } @@ -133,6 +134,9 @@ type SceneParams struct { // Search defines model for Search. type Search string +// Sort defines model for Sort. +type Sort string + // Task defines model for Task. type Task struct { CollectionId *CollectionId `json:"collection_id,omitempty"` @@ -179,6 +183,7 @@ type GetScenesParams struct { ViewportHeight *ViewportHeight `json:"viewport_height,omitempty"` ImageHeight *ImageHeight `json:"image_height,omitempty"` Layout *LayoutType `json:"layout,omitempty"` + Sort *Sort `json:"sort,omitempty"` Search *Search `json:"search,omitempty"` } @@ -206,6 +211,7 @@ type GetScenesSceneIdTilesParams struct { Zoom int `json:"zoom"` X TileCoord `json:"x"` Y TileCoord `json:"y"` + Sources *[]string `json:"sources,omitempty"` DebugOverdraw *bool `json:"debug_overdraw,omitempty"` DebugThumbnails *bool `json:"debug_thumbnails,omitempty"` } @@ -516,6 +522,17 @@ func (siw *ServerInterfaceWrapper) GetScenes(w http.ResponseWriter, r *http.Requ return } + // ------------- Optional query parameter "sort" ------------- + if paramValue := r.URL.Query().Get("sort"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", true, false, "sort", r.URL.Query(), ¶ms.Sort) + if err != nil { + http.Error(w, fmt.Sprintf("Invalid format for parameter sort: %s", err), http.StatusBadRequest) + return + } + // ------------- Optional query parameter "search" ------------- if paramValue := r.URL.Query().Get("search"); paramValue != "" { @@ -838,6 +855,17 @@ func (siw *ServerInterfaceWrapper) GetScenesSceneIdTiles(w http.ResponseWriter, return } + // ------------- Optional query parameter "sources" ------------- + if paramValue := r.URL.Query().Get("sources"); paramValue != "" { + + } + + err = runtime.BindQueryParameter("form", false, false, "sources", r.URL.Query(), ¶ms.Sources) + if err != nil { + http.Error(w, fmt.Sprintf("Invalid format for parameter sources: %s", err), http.StatusBadRequest) + return + } + // ------------- Optional query parameter "debug_overdraw" ------------- if paramValue := r.URL.Query().Get("debug_overdraw"); paramValue != "" { diff --git a/internal/render/photo.go b/internal/render/photo.go index d46e836..44a1205 100644 --- a/internal/render/photo.go +++ b/internal/render/photo.go @@ -63,7 +63,11 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales size := info.Size() rsize := photo.Sprite.Rect.RenderedSize(c, size) - sources := source.Sources.EstimateCost(io.Size(size), io.Size(rsize)) + srcs := source.Sources + if config.Sources != nil { + srcs = config.Sources + } + sources := srcs.EstimateCost(io.Size(size), io.Size(rsize)) sources.Sort() for i, s := range sources { if drawn { @@ -71,19 +75,22 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales } start := time.Now() r := s.Get(context.TODO(), io.ImageId(photo.Id), path) - elapsed := time.Since(start).Microseconds() + elapsed := time.Since(start) img, err := r.Image, r.Error if img == nil || err != nil { continue } + name := s.Name() + source.SourceLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed.Microseconds())) + source.SourcePerOriginalMegapixelLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed) * 1e6 / (float64(size.X) * float64(size.Y))) + source.SourcePerResizedMegapixelLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed) * 1e6 / float64(s.EstimatedArea)) + if r.Orientation == io.SourceInfoOrientation { r.Orientation = io.Orientation(info.Orientation) } - source.SourcesLatencyHistogram.WithLabelValues(s.Name()).Observe(float64(elapsed)) - bitmap := Bitmap{ Sprite: photo.Sprite, Orientation: image.Orientation(r.Orientation), diff --git a/internal/render/scene.go b/internal/render/scene.go index ef35847..90810b3 100644 --- a/internal/render/scene.go +++ b/internal/render/scene.go @@ -11,6 +11,7 @@ import ( "photofield/internal/clip" "photofield/internal/image" + "photofield/io" ) type Render struct { @@ -18,8 +19,10 @@ type Render struct { MaxSolidPixelArea float64 `json:"max_solid_pixel_area"` BackgroundColor color.Color `json:"background_color"` LogDraws bool - DebugOverdraw bool - DebugThumbnails bool + + Sources io.Sources + DebugOverdraw bool + DebugThumbnails bool Zoom int CanvasImage draw.Image diff --git a/internal/scene/sceneSource.go b/internal/scene/sceneSource.go index 948123e..126e5af 100644 --- a/internal/scene/sceneSource.go +++ b/internal/scene/sceneSource.go @@ -215,9 +215,17 @@ func sceneConfigEqual(a SceneConfig, b SceneConfig) bool { return false } - return a.Layout.Type != "" && + if a.Layout.Type != "" && b.Layout.Type != "" && - a.Layout.Type == b.Layout.Type + a.Layout.Type != b.Layout.Type { + return false + } + + if a.Layout.Order != b.Layout.Order { + return false + } + + return true } func (source *SceneSource) GetScenesWithConfig(config SceneConfig) []*render.Scene { diff --git a/io/configured/configured.go b/io/configured/configured.go new file mode 100644 index 0000000..61c2a14 --- /dev/null +++ b/io/configured/configured.go @@ -0,0 +1,113 @@ +package configured + +import ( + "context" + "fmt" + "photofield/io" + "time" + + goio "io" + + "github.com/goccy/go-yaml" +) + +type Cost struct { + Time Duration `json:"time"` + TimePerOriginalMegapixel Duration `json:"time_per_original_megapixel"` + TimePerResizedMegapixel Duration `json:"time_per_resized_megapixel"` +} + +type Duration time.Duration + +func (d *Duration) UnmarshalYAML(b []byte) error { + var s string + if err := yaml.Unmarshal(b, &s); err != nil { + return err + } + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(dur) + return nil +} + +func (d Duration) MarshalYAML() (interface{}, error) { + return d.String(), nil +} + +func (d Duration) String() string { + return time.Duration(d).String() +} + +type Configured struct { + NameStr string + Cost Cost + Source io.Source +} + +func New(name string, cost Cost, source io.Source) *Configured { + c := Configured{ + NameStr: name, + Cost: cost, + Source: source, + } + if c.NameStr == "" { + c.NameStr = c.Source.Name() + } + return &c +} + +func (c *Configured) Name() string { + return c.NameStr +} + +func (c *Configured) DisplayName() string { + return c.Source.DisplayName() +} + +func (c *Configured) Ext() string { + return c.Source.Ext() +} + +func (c *Configured) Size(size io.Size) io.Size { + return c.Source.Size(size) +} + +func (c *Configured) GetDurationEstimate(original io.Size) time.Duration { + resized := c.Size(original) + t := c.Cost.Time + tomp := c.Cost.TimePerOriginalMegapixel + trmp := c.Cost.TimePerResizedMegapixel + d := Duration(t + (tomp*Duration(original.Area())+trmp*Duration(resized.Area()))/1e6) + return time.Duration(d) +} + +func (c *Configured) Rotate() bool { + return c.Source.Rotate() +} + +func (c *Configured) Exists(ctx context.Context, id io.ImageId, path string) bool { + return c.Source.Exists(ctx, id, path) +} + +func (c *Configured) Get(ctx context.Context, id io.ImageId, path string) io.Result { + return c.Source.Get(ctx, id, path) +} + +func (c *Configured) Reader(ctx context.Context, id io.ImageId, path string, fn func(r goio.ReadSeeker, err error)) { + r, ok := c.Source.(io.Reader) + if !ok { + fn(nil, fmt.Errorf("reader not supported by %s", c.Source.Name())) + return + } + r.Reader(ctx, id, path, fn) +} + +func (c *Configured) Decode(ctx context.Context, r goio.Reader) io.Result { + d, ok := c.Source.(io.Decoder) + if !ok { + return io.Result{Error: fmt.Errorf("decoder not supported by %s", c.Source.Name())} + } + return d.Decode(ctx, r) +} diff --git a/io/io.go b/io/io.go index 1bb8c87..3abe0e5 100644 --- a/io/io.go +++ b/io/io.go @@ -32,9 +32,9 @@ func (f *AspectRatioFit) UnmarshalYAML(b []byte) error { } const ( - OriginalSize AspectRatioFit = iota - FitOutside AspectRatioFit = iota - FitInside AspectRatioFit = iota + OriginalSize AspectRatioFit = iota + 1 + FitOutside + FitInside ) type Orientation int8 @@ -153,8 +153,10 @@ func (sources Sources) EstimateCost(original Size, target Size) SourceCosts { cost := sizecost + durcost // fmt.Printf("%4d %30s %12s %12s %12s %12d %12f %10s %12f %12f\n", i, s.Name(), original, target, ssize, sarea, sizecost, dur, durcost, cost) costs[i] = SourceCost{ - Source: s, - Cost: cost, + Source: s, + EstimatedArea: sarea, + EstimatedDuration: dur, + Cost: cost, } } return costs @@ -162,7 +164,9 @@ func (sources Sources) EstimateCost(original Size, target Size) SourceCosts { type SourceCost struct { Source - Cost float64 + EstimatedArea int64 + EstimatedDuration time.Duration + Cost float64 } type SourceCosts []SourceCost diff --git a/io/sqlite/sqlite.go b/io/sqlite/sqlite.go index 5ad7e38..0f60420 100644 --- a/io/sqlite/sqlite.go +++ b/io/sqlite/sqlite.go @@ -52,8 +52,8 @@ func (s *Source) Ext() string { } func (s *Source) GetDurationEstimate(size io.Size) time.Duration { - return 879 * time.Microsecond // SSD - // return 958 * time.Microsecond // HDD + // return 879 * time.Microsecond // SSD + return 958 * time.Microsecond // HDD } func (s *Source) Rotate() bool { @@ -147,7 +147,7 @@ func (s *Source) init() { func (s *Source) migrate(migrations embed.FS) { dbsource, err := httpfs.New(http.FS(migrations), "db/migrations-thumbs") if err != nil { - panic(err) + log.Fatalf("failed to create migrate source: %v", err) } url := fmt.Sprintf("sqlite://%v", filepath.ToSlash(s.path)) m, err := migrate.NewWithSourceInstance( @@ -156,7 +156,7 @@ func (s *Source) migrate(migrations embed.FS) { url, ) if err != nil { - panic(err) + log.Fatalf("failed to create migrate instance for %s: %v", s.path, err) } version, dirty, err := m.Version() diff --git a/io/sqlite/sqlite_test.go b/io/sqlite/sqlite_test.go index b1271cb..6396ff6 100644 --- a/io/sqlite/sqlite_test.go +++ b/io/sqlite/sqlite_test.go @@ -5,6 +5,7 @@ import ( "embed" "os" "path" + "photofield/io" "testing" ) @@ -22,11 +23,11 @@ func TestRoundtrip(t *testing.T) { id := uint32(1) s.Write(id, bytes) - img, err := s.Load(context.Background(), id) - if err != nil { - t.Fatal(err) + r := s.Get(context.Background(), io.ImageId(id), p) + if r.Error != nil { + t.Fatal(r.Error) } - b := img.Bounds() + b := r.Image.Bounds() if b.Dx() != 256 || b.Dy() != 171 { t.Errorf("unexpected size %d x %d", b.Dx(), b.Dy()) } diff --git a/io/thumb/thumb.go b/io/thumb/thumb.go index e7664d4..674638c 100644 --- a/io/thumb/thumb.go +++ b/io/thumb/thumb.go @@ -117,7 +117,7 @@ func (t Thumb) Size(size io.Size) io.Size { } func (t Thumb) GetDurationEstimate(size io.Size) time.Duration { - return 31 * time.Nanosecond * time.Duration(t.Size(size).Area()) + return 31 * time.Nanosecond * time.Duration(size.Area()) } func (t *Thumb) resolvePath(originalPath string) string { diff --git a/main.go b/main.go index 5548a38..96c1aa0 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "sort" "strings" "sync" @@ -38,6 +39,7 @@ import ( "github.com/imdario/mergo" "github.com/joho/godotenv" "github.com/lpar/gzipped" + "github.com/pyroscope-io/client/pyroscope" "github.com/tdewolff/canvas" "github.com/tdewolff/canvas/rasterizer" @@ -57,6 +59,7 @@ import ( "photofield/internal/openapi" "photofield/internal/render" "photofield/internal/scene" + pfio "photofield/io" ) //go:embed defaults.yaml @@ -342,27 +345,39 @@ func (*Api) PostScenes(w http.ResponseWriter, r *http.Request) { } sceneConfig := defaultSceneConfig + + collection := getCollectionById(string(data.CollectionId)) + if collection == nil { + problem(w, r, http.StatusBadRequest, "Collection not found") + return + } + sceneConfig.Collection = *collection + sceneConfig.Layout.ViewportWidth = float64(data.ViewportWidth) sceneConfig.Layout.ViewportHeight = float64(data.ViewportHeight) sceneConfig.Layout.ImageHeight = 0 if data.ImageHeight != nil { sceneConfig.Layout.ImageHeight = float64(*data.ImageHeight) } + if sceneConfig.Collection.Layout != "" { + sceneConfig.Layout.Type = layout.Type(sceneConfig.Collection.Layout) + } if data.Layout != "" { sceneConfig.Layout.Type = layout.Type(data.Layout) } + if data.Sort != nil { + sceneConfig.Layout.Order = layout.OrderFromSort(string(*data.Sort)) + if sceneConfig.Layout.Order == layout.None { + problem(w, r, http.StatusBadRequest, "Invalid sort") + return + } + } if data.Search != nil { sceneConfig.Scene.Search = string(*data.Search) if sceneConfig.Layout.Type != layout.Strip { sceneConfig.Layout.Type = layout.Search } } - collection := getCollectionById(string(data.CollectionId)) - if collection == nil { - problem(w, r, http.StatusBadRequest, "Collection not found") - return - } - sceneConfig.Collection = *collection scene := sceneSource.Add(sceneConfig, imageSource) @@ -384,6 +399,13 @@ func (*Api) GetScenes(w http.ResponseWriter, r *http.Request, params openapi.Get if params.Layout != nil { sceneConfig.Layout.Type = layout.Type(*params.Layout) } + if params.Sort != nil { + sceneConfig.Layout.Order = layout.OrderFromSort(string(*params.Sort)) + if sceneConfig.Layout.Order == layout.None { + problem(w, r, http.StatusBadRequest, "Invalid sort") + return + } + } if params.Search != nil { sceneConfig.Scene.Search = string(*params.Search) if sceneConfig.Layout.Type != layout.Strip { @@ -641,6 +663,26 @@ func GetScenesSceneIdTilesImpl(w http.ResponseWriter, r *http.Request, sceneId o render := defaultSceneConfig.Render render.TileSize = params.TileSize + if params.Sources != nil { + render.Sources = make(pfio.Sources, len(*params.Sources)) + for _, src := range imageSource.Sources { + for i, name := range *params.Sources { + if src.Name() == name { + if render.Sources[i] != nil { + problem(w, r, http.StatusBadRequest, "Duplicate source") + return + } + render.Sources[i] = src + } + } + } + for _, src := range render.Sources { + if src == nil { + problem(w, r, http.StatusBadRequest, "Unknown source") + return + } + } + } if params.DebugOverdraw != nil { render.DebugOverdraw = *params.DebugOverdraw } @@ -937,7 +979,6 @@ func IndexHTML() func(next http.Handler) http.Handler { } func main() { - startupTime = time.Now() versionPtr := flag.Bool("version", false, "print version and exit") @@ -954,6 +995,38 @@ func main() { loadEnv() + if os.Getenv("PYROSCOPE_HOST") != "" { + log.Printf("pyroscope enabled at %s", os.Getenv("PYROSCOPE_HOST")) + + // These 2 lines are only required if you're using mutex or block profiling + // Read the explanation below for how to set these rates: + runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(5) + + pyroscope.Start(pyroscope.Config{ + ApplicationName: "photofield", + ServerAddress: os.Getenv("PYROSCOPE_HOST"), + Logger: nil, + AuthToken: os.Getenv("PYROSCOPE_AUTH_TOKEN"), + Tags: map[string]string{"hostname": os.Getenv("HOSTNAME")}, + ProfileTypes: []pyroscope.ProfileType{ + // these profile types are enabled by default: + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + + // these profile types are optional: + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + }) + } + if err := yaml.Unmarshal(defaultsYaml, &defaults); err != nil { panic(err) } diff --git a/ui/src/App.vue b/ui/src/App.vue index bc0c925..f06eca0 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -154,15 +154,6 @@ export default { }, setup(props) { const collectionId = toRef(props, "collectionId"); - const layoutOptions = computed(() => { - return [ - { label: `Default`, value: "DEFAULT" }, - { label: "Album", value: "ALBUM" }, - { label: "Timeline", value: "TIMELINE" }, - { label: "Wall", value: "WALL" }, - ] - }) - const router = useRouter(); const route = useRoute(); const query = computed(() => route.query); @@ -206,7 +197,6 @@ export default { goHome, query, setQuery, - layoutOptions, remoteTasks, remoteTasksUpdateUntilDone, indexTasks, diff --git a/ui/src/api.js b/ui/src/api.js index f5ed1df..a0e6d5c 100644 --- a/ui/src/api.js +++ b/ui/src/api.js @@ -100,7 +100,7 @@ export function getTileUrl(sceneId, level, x, y, tileSize, backgroundColor, extr y, ...extraParams, }; - let url = `${host}/scenes/${sceneId}/tiles?${qs.stringify(params)}`; + let url = `${host}/scenes/${sceneId}/tiles?${qs.stringify(params, { arrayFormat: "comma" })}`; return url; } @@ -142,6 +142,7 @@ export function useApi(getUrl, config) { export function useScene({ collectionId, layout, + sort, imageHeight, viewport, search, @@ -152,6 +153,7 @@ export function useScene({ viewport?.height?.value && { layout: layout.value, + sort: sort.value, image_height: imageHeight?.value || undefined, collection_id: collectionId.value, viewport_width: viewport.width.value, diff --git a/ui/src/components/CollectionView.vue b/ui/src/components/CollectionView.vue index 7b257a1..ea466da 100644 --- a/ui/src/components/CollectionView.vue +++ b/ui/src/components/CollectionView.vue @@ -6,6 +6,7 @@ :interactive="!stripVisible" :collectionId="collectionId" :layout="layout" + :sort="sort" :imageHeight="imageHeight" :search="search" :debug="debug" @@ -22,6 +23,7 @@ :class="{ visible: stripVisible }" :interactive="stripVisible" :collectionId="collectionId" + :sort="sort" :regionId="transitionRegionId || regionId" :search="search" :debug="debug" @@ -42,6 +44,7 @@ import { useRoute, useRouter } from 'vue-router'; import StripViewer from './StripViewer.vue'; import ScrollViewer from './ScrollViewer.vue'; +import { useApi } from '../api'; const props = defineProps([ "collectionId", @@ -94,8 +97,21 @@ const scenes = computed(() => { }); watch(scenes, scenes => emit("scenes", scenes)); +const { data: collection } = useApi( + () => collectionId.value && `/collections/${collectionId.value}` +); + const layout = computed(() => { - return route.query.layout; + return route.query.layout || collection.value?.layout || undefined; +}) + +const sort = computed(() => { + switch (layout.value) { + case "TIMELINE": + return "-date"; + default: + return "+date"; + } }) const imageHeight = computed(() => { diff --git a/ui/src/components/ScrollViewer.vue b/ui/src/components/ScrollViewer.vue index 1c69bd4..d861a19 100644 --- a/ui/src/components/ScrollViewer.vue +++ b/ui/src/components/ScrollViewer.vue @@ -78,6 +78,7 @@ const props = defineProps({ collectionId: String, regionId: String, layout: String, + sort: String, imageHeight: Number, search: String, debug: Object, @@ -100,6 +101,7 @@ const { collectionId, scrollbar, layout, + sort, imageHeight, search, debug, @@ -112,6 +114,7 @@ const lastView = ref(null); const { scene, recreate: recreateScene, filesPerSecond } = useScene({ layout, + sort, collectionId, imageHeight, viewport, diff --git a/ui/src/components/StripViewer.vue b/ui/src/components/StripViewer.vue index 0d2c2b0..562d4dc 100644 --- a/ui/src/components/StripViewer.vue +++ b/ui/src/components/StripViewer.vue @@ -83,6 +83,7 @@ const props = defineProps({ interactive: Boolean, collectionId: String, regionId: String, + sort: String, search: String, debug: Object, fullpage: Boolean, @@ -101,6 +102,7 @@ const { interactive, regionId, collectionId, + sort, search, debug, } = toRefs(props); @@ -120,6 +122,7 @@ const viewport = useViewport(container); const { scene, recreate: recreateScene } = useScene({ layout: ref("STRIP"), + sort, collectionId, viewport, search,