fix: functions to parse URLs and handle URL file paths (#219)

This code was first created for PR #557 in oathkeeper.
It should probably be in the shared ory/x project instead of there,
so that is why this code is checked in here.

Co-authored-by: zepatrik <[email protected]>
Co-authored-by: Patrik <[email protected]>
Co-authored-by: hackerman <[email protected]>

.github/workflows/windows_test.yml

@@ -0,0 +1,18 @@
+name: Windows go test
+
+on:
+ pull_request:
+ branches:
+ - master
+ push:
+ branches:
+ - '*'
+
+jobs:
+ test:
+ runs-on: windows-latest
+ steps:
+ - uses: actions/checkout@v2
+ - run: |
+ go test -failfast -timeout=20m $(go list ./... | grep -v viperx | grep -v watcherx | grep -v sqlcon)
+ shell: bash

urlx/parse.go

@@ -2,13 +2,54 @@ package urlx
 
 import (
  "net/url"
+ "regexp"
+ "strings"
 
  "github.com/ory/x/logrusx"
 )
 
+// winPathRegex is a regex for [DRIVE-LETTER]:
+var winPathRegex = regexp.MustCompile("^[A-Za-z]:.*")
+
+// Parse parses rawURL into a URL structure with special handling for file:// URLs
+// File URLs with relative paths (file://../file, ../file) will be returned as a
+// url.URL object without the Scheme set to "file". This is because the file
+// scheme doesn't support relative paths. Make sure to check for
+// both "file" or "" (an empty string) in URL.Scheme if you are looking for
+// a file path.
+// Use the companion function GetURLFilePath() to get a file path suitable
+// for the current operaring system.
+func Parse(rawURL string) (*url.URL, error) {
+ lcRawURL := strings.ToLower(rawURL)
+ if strings.HasPrefix(lcRawURL, "file:///") {
+ return url.Parse(rawURL)
+ }
+
+ // Normally the first part after file:// is a hostname, but since
+ // this is often misused we interpret the URL like a normal path
+ // by removing the "file://" from the beginning (if it exists)
+ rawURL = trimPrefixIC(rawURL, "file://")
+
+ if winPathRegex.MatchString(rawURL) {
+ // Windows path
+ return url.Parse("file:///" + rawURL)
+ }
+
+ if strings.HasPrefix(lcRawURL, "\\\\") {
+ // Windows UNC path
+ // We extract the hostname and create an appropriate file:// URL
+ // based on the hostname and the path
+ host, path := extractUNCPathParts(rawURL)
+ // It is safe to replace the \ with / here because this is POSIX style path
+ return url.Parse("file://" + host + strings.ReplaceAll(path, "\\", "/"))
+ }
+
+ return url.Parse(rawURL)
+}
+
 // ParseOrPanic parses a url or panics.
 func ParseOrPanic(in string) *url.URL {
- out, err := url.Parse(in)
+ out, err := Parse(in)
  if err != nil {
  panic(err.Error())
  }
@@ -17,7 +58,7 @@ func ParseOrPanic(in string) *url.URL {
 
 // ParseOrFatal parses a url or fatals.
 func ParseOrFatal(l *logrusx.Logger, in string) *url.URL {
- out, err := url.Parse(in)
+ out, err := Parse(in)
  if err != nil {
  l.WithError(err).Fatalf("Unable to parse url: %s", in)
  }
@@ -41,3 +82,22 @@ func ParseRequestURIOrFatal(l *logrusx.Logger, in string) *url.URL {
  }
  return out
 }
+
+func extractUNCPathParts(uncPath string) (host, path string) {
+ parts := strings.Split(strings.TrimPrefix(uncPath, "\\\\"), "\\")
+ host = parts[0]
+ if len(parts) > 0 {
+ path = "\\" + strings.Join(parts[1:], "\\")
+ }
+ return host, path
+}
+
+// trimPrefixIC returns s without the provided leading prefix string using
+// case insensitive matching.
+// If s doesn't start with prefix, s is returned unchanged.
+func trimPrefixIC(s, prefix string) string {
+ if strings.HasPrefix(strings.ToLower(s), prefix) {
+ return s[len(prefix):]
+ }
+ return s
+}

urlx/parse_test.go

@@ -0,0 +1,82 @@
+package urlx
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParseURL(t *testing.T) {
+ type testData struct {
+ urlStr string
+ expectedPath string
+ expectedStr string
+ }
+ var testURLs = []testData{
+ {"File:///home/test/file1.txt", "/home/test/file1.txt", "file:///home/test/file1.txt"},
+ {"fIle:/home/test/file2.txt", "/home/test/file2.txt", "file:///home/test/file2.txt"},
+ {"fiLe:///../test/update/file3.txt", "/../test/update/file3.txt", "file:///../test/update/file3.txt"},
+ {"filE://../test/update/file4.txt", "../test/update/file4.txt", "../test/update/file4.txt"},
+ {"file://C:/users/test/file5.txt", "/C:/users/test/file5.txt", "file:///C:/users/test/file5.txt"}, // We expect a initial / in the path because this is a Windows absolute path
+ {"file:///C:/users/test/file6.txt", "/C:/users/test/file6.txt", "file:///C:/users/test/file6.txt"}, // --//--
+ {"file://file7.txt", "file7.txt", "file7.txt"},
+ {"file://path/file8.txt", "path/file8.txt", "path/file8.txt"},
+ {"file://C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "/C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "file:///C:%5CUsers%5CRUNNER~1%5CAppData%5CLocal%5CTemp%5C9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json"},
+ {"file:///C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "/C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "file:///C:%5CUsers%5CRUNNER~1%5CAppData%5CLocal%5CTemp%5C9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json"},
+ {"file://C:\\Users\\path with space\\file.txt", "/C:\\Users\\path with space\\file.txt", "file:///C:%5CUsers%5Cpath%20with%20space%5Cfile.txt"},
+ {"file8b.txt", "file8b.txt", "file8b.txt"},
+ {"../file9.txt", "../file9.txt", "../file9.txt"},
+ {"./file9b.txt", "./file9b.txt", "./file9b.txt"},
+ {"file://./file9c.txt", "./file9c.txt", "./file9c.txt"},
+ {"file://./folder/.././file9d.txt", "./folder/.././file9d.txt", "./folder/.././file9d.txt"},
+ {"..\\file10.txt", "..\\file10.txt", "..%5Cfile10.txt"},
+ {"C:\\file11.txt", "/C:\\file11.txt", "file:///C:%5Cfile11.txt"},
+ {"\\\\hostname\\share\\file12.txt", "/share/file12.txt", "file://hostname/share/file12.txt"},
+ {"\\\\", "/", "file:///"},
+ {"\\\\hostname", "/", "file://hostname/"},
+ {"\\\\hostname\\", "/", "file://hostname/"},
+ {"file:///home/test/file 13.txt", "/home/test/file 13.txt", "file:///home/test/file%2013.txt"},
+ {"file:///home/test/file%2014.txt", "/home/test/file 14.txt", "file:///home/test/file%2014.txt"},
+ {"http://server:80/test/file%2015.txt", "/test/file 15.txt", "http://server:80/test/file%2015.txt"},
+ {"file:///dir/file\\ with backslash", "/dir/file\\ with backslash", "file:///dir/file%5C%20with%20backslash"},
+ {"file://dir/file\\ with backslash", "dir/file\\ with backslash", "dir/file%5C%20with%20backslash"},
+ {"file:///dir/file with windows path forbidden chars \\<>:\"|%3F*", "/dir/file with windows path forbidden chars \\<>:\"|?*", "file:///dir/file%20with%20windows%20path%20forbidden%20chars%20%5C%3C%3E:%22%7C%3F%2A"},
+ {"file://dir/file with windows path forbidden chars \\<>:\"|%3F*", "dir/file with windows path forbidden chars \\<>:\"|?*", "dir/file%20with%20windows%20path%20forbidden%20chars%20%5C%3C%3E:%22%7C%3F%2A"},
+ {"file:///path/file?query=1", "/path/file", "file:///path/file?query=1"},
+ {"http://host:80/path/file?query=1", "/path/file", "http://host:80/path/file?query=1"},
+ {"file://////C:/file.txt", "////C:/file.txt", "file://////C:/file.txt"},
+ {"file://////C:\\file.txt", "////C:\\file.txt", "file://////C:%5Cfile.txt"},
+ }
+
+ for _, td := range testURLs {
+ u, err := Parse(td.urlStr)
+ assert.NoError(t, err)
+ if err != nil {
+ continue
+ }
+ assert.Equal(t, td.expectedPath, u.Path, "expected path for %s", td.urlStr)
+ assert.Equal(t, td.expectedStr, u.String(), "expected URL string for %s", td.urlStr)
+ }
+ _, err := Parse("://")
+ assert.Error(t, err)
+ _, err = Parse("://host:80/file")
+ assert.Error(t, err)
+ _, err = Parse(":///path/file")
+ assert.Error(t, err)
+}
+
+func TestTrimPrefixIC(t *testing.T) {
+ for _, td := range []struct {
+ s string
+ prefix string
+ expected string
+ }{
+ {"file://test", "file://", "test"},
+ {"FILE://test", "file://", "test"},
+ {"FiLe://test", "file://", "test"},
+ {"http://test", "file://", "http://test"},
+ {"files://test", "file://", "files://test"},
+ } {
+ assert.Equal(t, td.expected, trimPrefixIC(td.s, td.prefix))
+ }
+}

urlx/path.go

@@ -0,0 +1,15 @@
+// +build !windows
+
+package urlx
+
+import (
+ "net/url"
+)
+
+// GetURLFilePath returns the path of a URL that is compatible with the runtime os filesystem
+func GetURLFilePath(u *url.URL) string {
+ if u == nil {
+ return ""
+ }
+ return u.Path
+}

urlx/path_test.go

@@ -0,0 +1,71 @@
+package urlx
+
+import (
+ "runtime"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetURLFilePath(t *testing.T) {
+ type testData struct {
+ urlStr string
+ expectedUnix string
+ expectedWindows string
+ shouldSucceed bool
+ }
+ var testURLs = []testData{
+ {"File:///home/test/file1.txt", "/home/test/file1.txt", "\\home\\test\\file1.txt", true},
+ {"fIle:/home/test/file2.txt", "/home/test/file2.txt", "\\home\\test\\file2.txt", true},
+ {"fiLe:///../test/update/file3.txt", "/../test/update/file3.txt", "\\..\\test\\update\\file3.txt", true},
+ {"filE://../test/update/file4.txt", "../test/update/file4.txt", "..\\test\\update\\file4.txt", true},
+ {"file://C:/users/test/file5.txt", "/C:/users/test/file5.txt", "C:\\users\\test\\file5.txt", true},
+ {"file:///C:/users/test/file5b.txt", "/C:/users/test/file5b.txt", "C:\\users\\test\\file5b.txt", true},
+ {"file://anotherhost/share/users/test/file6.txt", "/share/users/test/file6.txt", "\\\\anotherhost\\share\\users\\test\\file6.txt", false}, // this is not supported
+ {"file://file7.txt", "file7.txt", "file7.txt", true},
+ {"file://path/file8.txt", "path/file8.txt", "path\\file8.txt", true},
+ {"file://C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "/C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343\\access-rules.json", true},
+ {"file:///C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "/C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343/access-rules.json", "C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\9ccf9f68-121c-451a-8a73-2aa360925b5a386398343\\access-rules.json", true},
+ {"file8.txt", "file8.txt", "file8.txt", true},
+ {"../file9.txt", "../file9.txt", "..\\file9.txt", true},
+ {"./file9b.txt", "./file9b.txt", ".\\file9b.txt", true},
+ {"file://./file9c.txt", "./file9c.txt", ".\\file9c.txt", true},
+ {"file://./folder/.././file9d.txt", "./folder/.././file9d.txt", ".\\folder\\..\\.\\file9d.txt", true},
+ {"..\\file10.txt", "..\\file10.txt", "..\\file10.txt", true},
+ {"C:\\file11.txt", "/C:\\file11.txt", "C:\\file11.txt", true},
+ {"\\\\hostname\\share\\file12.txt", "/share/file12.txt", "\\\\hostname\\share\\file12.txt", true},
+ {"file:///home/test/file 13.txt", "/home/test/file 13.txt", "\\home\\test\\file 13.txt", true},
+ {"file:///home/test/file%2014.txt", "/home/test/file 14.txt", "\\home\\test\\file 14.txt", true},
+ {"http://server:80/test/file%2015.txt", "/test/file 15.txt", "/test/file 15.txt", true},
+ {"file:///dir/file\\ with backslash", "/dir/file\\ with backslash", "\\dir\\file\\ with backslash", true},
+ {"file://dir/file\\ with backslash", "dir/file\\ with backslash", "dir\\file\\ with backslash", true},
+ {"file:///dir/file with windows path forbidden chars \\<>:\"|%3F*", "/dir/file with windows path forbidden chars \\<>:\"|?*", "\\dir\\file with windows path forbidden chars \\<>:\"|?*", true},
+ {"file://dir/file with windows path forbidden chars \\<>:\"|%3F*", "dir/file with windows path forbidden chars \\<>:\"|?*", "dir\\file with windows path forbidden chars \\<>:\"|?*", true},
+ {"file:///path/file?query=1", "/path/file", "\\path\\file", true},
+ {"http://host:80/path/file?query=1", "/path/file", "/path/file", true},
+ {"file://////C:/file.txt", "////C:/file.txt", "C:\\file.txt", true},
+ {"file://////C:\\file.txt", "////C:\\file.txt", "C:\\file.txt", true},
+ }
+ for _, td := range testURLs {
+ u, err := Parse(td.urlStr)
+ assert.NoError(t, err)
+ if err != nil {
+ continue
+ }
+ p := GetURLFilePath(u)
+ if runtime.GOOS == "windows" {
+ if td.shouldSucceed {
+ assert.Equal(t, td.expectedWindows, p)
+ } else {
+ assert.NotEqual(t, td.expectedWindows, p)
+ }
+ } else {
+ if td.shouldSucceed {
+ assert.Equal(t, td.expectedUnix, p)
+ } else {
+ assert.NotEqual(t, td.expectedUnix, p)
+ }
+ }
+ }
+ assert.Empty(t, GetURLFilePath(nil))
+}

urlx/path_windows.go

@@ -0,0 +1,33 @@
+// +build windows
+
+package urlx
+
+import (
+ "net/url"
+ "path/filepath"
+ "strings"
+)
+
+// GetURLFilePath returns the path of a URL that is compatible with the runtime os filesystem
+func GetURLFilePath(u *url.URL) string {
+ if u == nil {
+ return ""
+ }
+ if !(u.Scheme == "file" || u.Scheme == "") {
+ return u.Path
+ }
+
+ fPath := u.Path
+ if u.Host != "" {
+ // Make UNC Path
+ fPath = "\\\\" + u.Host + filepath.FromSlash(fPath)
+ return fPath
+ }
+ fPathTrimmed := strings.TrimLeft(fPath, "/")
+ if winPathRegex.MatchString(fPathTrimmed) {
+ // On Windows we should remove the initial path separator in case this
+ // is a normal path (for example: "\c:\" -> "c:\"")
+ fPath = fPathTrimmed
+ }
+ return filepath.FromSlash(fPath)
+}