Merge pull request #65 from ipfs/migrate-gateway

Extract Gateway Code From Kubo
See ipfs/kubo#8524

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# Code owners are automatically requested for review when someone opens a pull
+# request that modifies code that they own. Code owners are not automatically
+# requested to review draft pull requests.
+
+# HTTP Gateway
+gateway/  @lidel @hacdias

.github/workflows/gateway-sharness.yml

@@ -0,0 +1,48 @@
+name: Gateway Sharness
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths: ['gateway/**']
+  push:
+    branches: ['main']
+    paths: ['gateway/**']
+
+jobs:
+  sharness:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.1
+      - name: Checkout go-libipfs
+        uses: actions/checkout@v3
+        with:
+          path: go-libipfs
+      - name: Checkout Kubo
+        uses: actions/checkout@v3
+        with:
+          repository: ipfs/kubo
+          path: kubo
+      - name: Install Missing Tools
+        run: sudo apt install -y socat net-tools fish libxml2-utils
+      - name: Restore Go Cache
+        uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - name: Replace go-libipfs in Kubo go.mod
+        run: |
+          go mod edit -replace=github.com/ipfs/go-libipfs=../go-libipfs
+          go mod tidy
+          cat go.mod
+        working-directory: kubo
+      - name: Install sharness dependencies
+        run: make test_sharness_deps
+        working-directory: kubo
+      - name: Run Sharness Tests
+        run: find . -maxdepth 1 -name "*gateway*.sh" -print0 | xargs -0 -I {} bash -c "echo {}; {}"
+        working-directory: kubo/test/sharness

gateway/README.md

@@ -0,0 +1,36 @@
+# IPFS Gateway
+
+> A reference implementation of HTTP Gateway Specifications.
+
+## Documentation
+
+* Go Documentation: https://pkg.go.dev/github.com/ipfs/go-libipfs/gateway
+* Gateway Specification: https://github.com/ipfs/specs/tree/main/http-gateways#readme
+* Types of HTTP Gateways: https://docs.ipfs.tech/how-to/address-ipfs-on-web/#http-gateways
+## Example
+
+```go
+// Initialize your headers and apply the default headers.
+headers := map[string][]string{}
+gateway.AddAccessControlHeaders(headers)
+
+conf := gateway.Config{
+  Writable: false,
+  Headers:  headers,
+}
+
+// Initialize a NodeAPI interface for both an online and offline versions.
+// The offline version should not make any network request for missing content.
+ipfs := ...
+offlineIPFS := ...
+
+// Create http mux and setup path gateway handler.
+mux := http.NewServeMux()
+gwHandler := gateway.NewHandler(conf, ipfs, offlineIPFS)
+mux.Handle("/ipfs/", gwHandler)
+mux.Handle("/ipns/", gwHandler)
+
+// Start the server on :8080 and voilá! You have a basic IPFS gateway running
+// in http://localhost:8080.
+_ = http.ListenAndServe(":8080", mux)
+```

gateway/assets/README.md

@@ -0,0 +1,27 @@
+# Required Assets for the Gateway
+
+> DAG and Directory HTML for HTTP gateway
+
+## Updating
+
+When making updates to the templates, please note the following:
+
+1. Make your changes to the (human-friendly) source documents in the `src` directory.
+2. Before testing or releasing, go to `assets/` and run `go generate .`.
+
+## Testing
+
+1. Make sure you have [Go](https://golang.org/dl/) installed
+2. Start the test server, which lives in its own directory:
+
+```bash
+> cd test
+> go run .
+```
+
+This will listen on [`localhost:3000`](http://localhost:3000/) and reload the template every time you refresh the page. Here you have two pages:
+
+- [`localhost:3000/dag`](http://localhost:3000/dag) for the DAG template preview; and
+- [`localhost:3000/directory`](http://localhost:3000/directory) for the Directory template preview.
+
+If you get a "no such file or directory" error upon trying `go run .`, make sure you ran `go generate .` to generate the minified artifact that the test is looking for.

gateway/assets/assets.go

@@ -0,0 +1,203 @@
+//go:generate ./build.sh
+package assets
+
+import (
+	"embed"
+	"io"
+	"io/fs"
+	"net"
+	"strconv"
+
+	"html/template"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/cespare/xxhash"
+
+	ipfspath "github.com/ipfs/go-path"
+)
+
+//go:embed dag-index.html directory-index.html knownIcons.txt
+var asset embed.FS
+
+// AssetHash a non-cryptographic hash of all embedded assets
+var AssetHash string
+
+var (
+	DirectoryTemplate *template.Template
+	DagTemplate       *template.Template
+)
+
+func init() {
+	initAssetsHash()
+	initTemplates()
+}
+
+func initAssetsHash() {
+	sum := xxhash.New()
+	err := fs.WalkDir(asset, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		file, err := asset.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		_, err = io.Copy(sum, file)
+		return err
+	})
+	if err != nil {
+		panic("error creating asset sum: " + err.Error())
+	}
+
+	AssetHash = strconv.FormatUint(sum.Sum64(), 32)
+}
+
+func initTemplates() {
+	knownIconsBytes, err := asset.ReadFile("knownIcons.txt")
+	if err != nil {
+		panic(err)
+	}
+	knownIcons := make(map[string]struct{})
+	for _, ext := range strings.Split(strings.TrimSuffix(string(knownIconsBytes), "\n"), "\n") {
+		knownIcons[ext] = struct{}{}
+	}
+
+	// helper to guess the type/icon for it by the extension name
+	iconFromExt := func(name string) string {
+		ext := path.Ext(name)
+		_, ok := knownIcons[ext]
+		if !ok {
+			// default blank icon
+			return "ipfs-_blank"
+		}
+		return "ipfs-" + ext[1:] // slice of the first dot
+	}
+
+	// custom template-escaping function to escape a full path, including '#' and '?'
+	urlEscape := func(rawUrl string) string {
+		pathURL := url.URL{Path: rawUrl}
+		return pathURL.String()
+	}
+
+	// Directory listing template
+	dirIndexBytes, err := asset.ReadFile("directory-index.html")
+	if err != nil {
+		panic(err)
+	}
+
+	DirectoryTemplate = template.Must(template.New("dir").Funcs(template.FuncMap{
+		"iconFromExt": iconFromExt,
+		"urlEscape":   urlEscape,
+	}).Parse(string(dirIndexBytes)))
+
+	// DAG Index template
+	dagIndexBytes, err := asset.ReadFile("dag-index.html")
+	if err != nil {
+		panic(err)
+	}
+
+	DagTemplate = template.Must(template.New("dir").Parse(string(dagIndexBytes)))
+}
+
+type DagTemplateData struct {
+	Path      string
+	CID       string
+	CodecName string
+	CodecHex  string
+}
+
+type DirectoryTemplateData struct {
+	GatewayURL  string
+	DNSLink     bool
+	Listing     []DirectoryItem
+	Size        string
+	Path        string
+	Breadcrumbs []Breadcrumb
+	BackLink    string
+	Hash        string
+}
+
+type DirectoryItem struct {
+	Size      string
+	Name      string
+	Path      string
+	Hash      string
+	ShortHash string
+}
+
+type Breadcrumb struct {
+	Name string
+	Path string
+}
+
+func Breadcrumbs(urlPath string, dnslinkOrigin bool) []Breadcrumb {
+	var ret []Breadcrumb
+
+	p, err := ipfspath.ParsePath(urlPath)
+	if err != nil {
+		// No assets.Breadcrumbs, fallback to bare Path in template
+		return ret
+	}
+	segs := p.Segments()
+	contentRoot := segs[1]
+	for i, seg := range segs {
+		if i == 0 {
+			ret = append(ret, Breadcrumb{Name: seg})
+		} else {
+			ret = append(ret, Breadcrumb{
+				Name: seg,
+				Path: "/" + strings.Join(segs[0:i+1], "/"),
+			})
+		}
+	}
+
+	// Drop the /ipns/<fqdn> prefix from assets.Breadcrumb Paths when directory
+	// listing on a DNSLink website (loaded due to Host header in HTTP
+	// request).  Necessary because the hostname most likely won't have a
+	// public gateway mounted.
+	if dnslinkOrigin {
+		prefix := "/ipns/" + contentRoot
+		for i, crumb := range ret {
+			if strings.HasPrefix(crumb.Path, prefix) {
+				ret[i].Path = strings.Replace(crumb.Path, prefix, "", 1)
+			}
+		}
+		// Make contentRoot assets.Breadcrumb link to the website root
+		ret[1].Path = "/"
+	}
+
+	return ret
+}
+
+func ShortHash(hash string) string {
+	if len(hash) <= 8 {
+		return hash
+	}
+	return (hash[0:4] + "\u2026" + hash[len(hash)-4:])
+}
+
+// helper to detect DNSLink website context
+// (when hostname from gwURL is matching /ipns/<fqdn> in path)
+func HasDNSLinkOrigin(gwURL string, path string) bool {
+	if gwURL != "" {
+		fqdn := stripPort(strings.TrimPrefix(gwURL, "//"))
+		return strings.HasPrefix(path, "/ipns/"+fqdn)
+	}
+	return false
+}
+
+func stripPort(hostname string) string {
+	host, _, err := net.SplitHostPort(hostname)
+	if err == nil {
+		return host
+	}
+	return hostname
+}

gateway/assets/build.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -euo pipefail
+
+function build() {
+  rm -f $1
+  sed '/<link rel=\"stylesheet\"/d' ./src/$1 > ./base-html.html
+  (echo "<style>" && cat ./src/icons.css ./src/style.css | tr -d "\t\n\r" && echo && echo "</style>") > ./minified-wrapped-style.html
+  sed '/<\/title>/ r ./minified-wrapped-style.html' ./base-html.html > ./$1
+  rm ./base-html.html && rm ./minified-wrapped-style.html
+}
+
+build "directory-index.html"
+build "dag-index.html"