From d51849bf18a32de34c35d75c118caef5294223e8 Mon Sep 17 00:00:00 2001 From: Mike Crute Date: Sun, 8 Oct 2023 12:35:38 -0700 Subject: httputil: add header parsing from godoc --- httputil/buster.go | 95 +++++++++++++ httputil/buster_test.go | 29 ++++ httputil/header/header.go | 298 +++++++++++++++++++++++++++++++++++++++++ httputil/header/header_test.go | 139 +++++++++++++++++++ httputil/httputil.go | 25 ++++ httputil/negotiate.go | 79 +++++++++++ httputil/negotiate_test.go | 71 ++++++++++ httputil/respbuf.go | 58 ++++++++ httputil/static.go | 265 ++++++++++++++++++++++++++++++++++++ httputil/static_test.go | 175 ++++++++++++++++++++++++ httputil/transport.go | 87 ++++++++++++ httputil/transport_test.go | 126 +++++++++++++++++ 12 files changed, 1447 insertions(+) create mode 100644 httputil/buster.go create mode 100644 httputil/buster_test.go create mode 100644 httputil/header/header.go create mode 100644 httputil/header/header_test.go create mode 100644 httputil/httputil.go create mode 100644 httputil/negotiate.go create mode 100644 httputil/negotiate_test.go create mode 100644 httputil/respbuf.go create mode 100644 httputil/static.go create mode 100644 httputil/static_test.go create mode 100644 httputil/transport.go create mode 100644 httputil/transport_test.go diff --git a/httputil/buster.go b/httputil/buster.go new file mode 100644 index 0000000..beab151 --- /dev/null +++ b/httputil/buster.go @@ -0,0 +1,95 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + "sync" +) + +type busterWriter struct { + headerMap http.Header + status int + io.Writer +} + +func (bw *busterWriter) Header() http.Header { + return bw.headerMap +} + +func (bw *busterWriter) WriteHeader(status int) { + bw.status = status +} + +// CacheBusters maintains a cache of cache busting tokens for static resources served by Handler. +type CacheBusters struct { + Handler http.Handler + + mu sync.Mutex + tokens map[string]string +} + +func sanitizeTokenRune(r rune) rune { + if r <= ' ' || r >= 127 { + return -1 + } + // Convert percent encoding reserved characters to '-'. + if strings.ContainsRune("!#$&'()*+,/:;=?@[]", r) { + return '-' + } + return r +} + +// Get returns the cache busting token for path. If the token is not already +// cached, Get issues a HEAD request on handler and uses the response ETag and +// Last-Modified headers to compute a token. +func (cb *CacheBusters) Get(path string) string { + cb.mu.Lock() + if cb.tokens == nil { + cb.tokens = make(map[string]string) + } + token, ok := cb.tokens[path] + cb.mu.Unlock() + if ok { + return token + } + + w := busterWriter{ + Writer: ioutil.Discard, + headerMap: make(http.Header), + } + r := &http.Request{URL: &url.URL{Path: path}, Method: "HEAD"} + cb.Handler.ServeHTTP(&w, r) + + if w.status == 200 { + token = w.headerMap.Get("Etag") + if token == "" { + token = w.headerMap.Get("Last-Modified") + } + token = strings.Trim(token, `" `) + token = strings.Map(sanitizeTokenRune, token) + } + + cb.mu.Lock() + cb.tokens[path] = token + cb.mu.Unlock() + + return token +} + +// AppendQueryParam appends the token as a query parameter to path. +func (cb *CacheBusters) AppendQueryParam(path string, name string) string { + token := cb.Get(path) + if token == "" { + return path + } + return path + "?" + name + "=" + token +} diff --git a/httputil/buster_test.go b/httputil/buster_test.go new file mode 100644 index 0000000..eb44cf4 --- /dev/null +++ b/httputil/buster_test.go @@ -0,0 +1,29 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "net/http" + "testing" +) + +func TestCacheBusters(t *testing.T) { + cbs := &CacheBusters{Handler: http.FileServer(http.Dir("."))} + + token := cbs.Get("/buster_test.go") + if token == "" { + t.Errorf("could not extract token from http.FileServer") + } + + var ss StaticServer + cbs = &CacheBusters{Handler: ss.FileHandler("buster_test.go")} + + token = cbs.Get("/xxx") + if token == "" { + t.Errorf("could not extract token from StaticServer FileHandler") + } +} diff --git a/httputil/header/header.go b/httputil/header/header.go new file mode 100644 index 0000000..0f1572e --- /dev/null +++ b/httputil/header/header.go @@ -0,0 +1,298 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// Package header provides functions for parsing HTTP headers. +package header + +import ( + "net/http" + "strings" + "time" +) + +// Octet types from RFC 2616. +var octetTypes [256]octetType + +type octetType byte + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +// Copy returns a shallow copy of the header. +func Copy(header http.Header) http.Header { + h := make(http.Header) + for k, vs := range header { + h[k] = vs + } + return h +} + +var timeLayouts = []string{"Mon, 02 Jan 2006 15:04:05 GMT", time.RFC850, time.ANSIC} + +// ParseTime parses the header as time. The zero value is returned if the +// header is not present or there is an error parsing the +// header. +func ParseTime(header http.Header, key string) time.Time { + if s := header.Get(key); s != "" { + for _, layout := range timeLayouts { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC() + } + } + } + return time.Time{} +} + +// ParseList parses a comma separated list of values. Commas are ignored in +// quoted strings. Quoted values are not unescaped or unquoted. Whitespace is +// trimmed. +func ParseList(header http.Header, key string) []string { + var result []string + for _, s := range header[http.CanonicalHeaderKey(key)] { + begin := 0 + end := 0 + escape := false + quote := false + for i := 0; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + end = i + 1 + case quote: + switch b { + case '\\': + escape = true + case '"': + quote = false + } + end = i + 1 + case b == '"': + quote = true + end = i + 1 + case octetTypes[b]&isSpace != 0: + if begin == end { + begin = i + 1 + end = begin + } + case b == ',': + if begin < end { + result = append(result, s[begin:end]) + } + begin = i + 1 + end = begin + default: + end = i + 1 + } + } + if begin < end { + result = append(result, s[begin:end]) + } + } + return result +} + +// ParseValueAndParams parses a comma separated list of values with optional +// semicolon separated name-value pairs. Content-Type and Content-Disposition +// headers are in this format. +func ParseValueAndParams(header http.Header, key string) (value string, params map[string]string) { + params = make(map[string]string) + s := header.Get(key) + value, s = expectTokenSlash(s) + if value == "" { + return + } + value = strings.ToLower(value) + s = skipSpace(s) + for strings.HasPrefix(s, ";") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +// AcceptSpec describes an Accept* header. +type AcceptSpec struct { + Value string + Q float64 +} + +// ParseAccept parses Accept* headers. +func ParseAccept(header http.Header, key string) (specs []AcceptSpec) { +loop: + for _, s := range header[key] { + for { + var spec AcceptSpec + spec.Value, s = expectTokenSlash(s) + if spec.Value == "" { + continue loop + } + spec.Q = 1.0 + s = skipSpace(s) + if strings.HasPrefix(s, ";") { + s = skipSpace(s[1:]) + if !strings.HasPrefix(s, "q=") { + continue loop + } + spec.Q, s = expectQuality(s[2:]) + if spec.Q < 0.0 { + continue loop + } + } + specs = append(specs, spec) + s = skipSpace(s) + if !strings.HasPrefix(s, ",") { + continue loop + } + s = skipSpace(s[1:]) + } + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenSlash(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + b := s[i] + if (octetTypes[b]&isToken == 0) && b != '/' { + break + } + } + return s[:i], s[i:] +} + +func expectQuality(s string) (q float64, rest string) { + switch { + case len(s) == 0: + return -1, "" + case s[0] == '0': + q = 0 + case s[0] == '1': + q = 1 + default: + return -1, "" + } + s = s[1:] + if !strings.HasPrefix(s, ".") { + return q, s + } + s = s[1:] + i := 0 + n := 0 + d := 1 + for ; i < len(s); i++ { + b := s[i] + if b < '0' || b > '9' { + break + } + n = n*10 + int(b) - '0' + d *= 10 + } + return q + float64(n)/float64(d), s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/httputil/header/header_test.go b/httputil/header/header_test.go new file mode 100644 index 0000000..97c8685 --- /dev/null +++ b/httputil/header/header_test.go @@ -0,0 +1,139 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package header + +import ( + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +var getHeaderListTests = []struct { + s string + l []string +}{ + {s: `a`, l: []string{`a`}}, + {s: `a, b , c `, l: []string{`a`, `b`, `c`}}, + {s: `a,, b , , c `, l: []string{`a`, `b`, `c`}}, + {s: `a,b,c`, l: []string{`a`, `b`, `c`}}, + {s: ` a b, c d `, l: []string{`a b`, `c d`}}, + {s: `"a, b, c", d `, l: []string{`"a, b, c"`, "d"}}, + {s: `","`, l: []string{`","`}}, + {s: `"\""`, l: []string{`"\""`}}, + {s: `" "`, l: []string{`" "`}}, +} + +func TestGetHeaderList(t *testing.T) { + for _, tt := range getHeaderListTests { + header := http.Header{"Foo": {tt.s}} + if l := ParseList(header, "foo"); !cmp.Equal(tt.l, l) { + t.Errorf("ParseList for %q = %q, want %q", tt.s, l, tt.l) + } + } +} + +var parseValueAndParamsTests = []struct { + s string + value string + params map[string]string +}{ + {`text/html`, "text/html", map[string]string{}}, + {`text/html `, "text/html", map[string]string{}}, + {`text/html ; `, "text/html", map[string]string{}}, + {`tExt/htMl`, "text/html", map[string]string{}}, + {`tExt/htMl; fOO=";"; hellO=world`, "text/html", map[string]string{ + "hello": "world", + "foo": `;`, + }}, + {`text/html; foo=bar, hello=world`, "text/html", map[string]string{"foo": "bar"}}, + {`text/html ; foo=bar `, "text/html", map[string]string{"foo": "bar"}}, + {`text/html ;foo=bar `, "text/html", map[string]string{"foo": "bar"}}, + {`text/html; foo="b\ar"`, "text/html", map[string]string{"foo": "bar"}}, + {`text/html; foo="bar\"baz\"qux"`, "text/html", map[string]string{"foo": `bar"baz"qux`}}, + {`text/html; foo="b,ar"`, "text/html", map[string]string{"foo": "b,ar"}}, + {`text/html; foo="b;ar"`, "text/html", map[string]string{"foo": "b;ar"}}, + {`text/html; FOO="bar"`, "text/html", map[string]string{"foo": "bar"}}, + {`form-data; filename="file.txt"; name=file`, "form-data", map[string]string{"filename": "file.txt", "name": "file"}}, +} + +func TestParseValueAndParams(t *testing.T) { + for _, tt := range parseValueAndParamsTests { + header := http.Header{"Content-Type": {tt.s}} + value, params := ParseValueAndParams(header, "Content-Type") + if value != tt.value { + t.Errorf("%q, value=%q, want %q", tt.s, value, tt.value) + } + if !cmp.Equal(params, tt.params) { + t.Errorf("%q, param=%#v, want %#v", tt.s, params, tt.params) + } + } +} + +var parseTimeValidTests = []string{ + "Sun, 06 Nov 1994 08:49:37 GMT", + "Sunday, 06-Nov-94 08:49:37 GMT", + "Sun Nov 6 08:49:37 1994", +} + +var parseTimeInvalidTests = []string{ + "junk", +} + +func TestParseTime(t *testing.T) { + expected := time.Date(1994, 11, 6, 8, 49, 37, 0, time.UTC) + for _, s := range parseTimeValidTests { + header := http.Header{"Date": {s}} + actual := ParseTime(header, "Date") + if actual != expected { + t.Errorf("GetTime(%q)=%v, want %v", s, actual, expected) + } + } + for _, s := range parseTimeInvalidTests { + header := http.Header{"Date": {s}} + actual := ParseTime(header, "Date") + if !actual.IsZero() { + t.Errorf("GetTime(%q) did not return zero", s) + } + } +} + +var parseAcceptTests = []struct { + s string + expected []AcceptSpec +}{ + {"text/html", []AcceptSpec{{"text/html", 1}}}, + {"text/html; q=0", []AcceptSpec{{"text/html", 0}}}, + {"text/html; q=0.0", []AcceptSpec{{"text/html", 0}}}, + {"text/html; q=1", []AcceptSpec{{"text/html", 1}}}, + {"text/html; q=1.0", []AcceptSpec{{"text/html", 1}}}, + {"text/html; q=0.1", []AcceptSpec{{"text/html", 0.1}}}, + {"text/html;q=0.1", []AcceptSpec{{"text/html", 0.1}}}, + {"text/html, text/plain", []AcceptSpec{{"text/html", 1}, {"text/plain", 1}}}, + {"text/html; q=0.1, text/plain", []AcceptSpec{{"text/html", 0.1}, {"text/plain", 1}}}, + {"iso-8859-5, unicode-1-1;q=0.8,iso-8859-1", []AcceptSpec{{"iso-8859-5", 1}, {"unicode-1-1", 0.8}, {"iso-8859-1", 1}}}, + {"iso-8859-1", []AcceptSpec{{"iso-8859-1", 1}}}, + {"*", []AcceptSpec{{"*", 1}}}, + {"da, en-gb;q=0.8, en;q=0.7", []AcceptSpec{{"da", 1}, {"en-gb", 0.8}, {"en", 0.7}}}, + {"da, q, en-gb;q=0.8", []AcceptSpec{{"da", 1}, {"q", 1}, {"en-gb", 0.8}}}, + {"image/png, image/*;q=0.5", []AcceptSpec{{"image/png", 1}, {"image/*", 0.5}}}, + + // bad cases + {"value1; q=0.1.2", []AcceptSpec{{"value1", 0.1}}}, + {"da, en-gb;q=foo", []AcceptSpec{{"da", 1}}}, +} + +func TestParseAccept(t *testing.T) { + for _, tt := range parseAcceptTests { + header := http.Header{"Accept": {tt.s}} + actual := ParseAccept(header, "Accept") + if !cmp.Equal(actual, tt.expected) { + t.Errorf("ParseAccept(h, %q)=%v, want %v", tt.s, actual, tt.expected) + } + } +} diff --git a/httputil/httputil.go b/httputil/httputil.go new file mode 100644 index 0000000..a03717c --- /dev/null +++ b/httputil/httputil.go @@ -0,0 +1,25 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// Package httputil is a toolkit for the Go net/http package. +package httputil + +import ( + "net" + "net/http" +) + +// StripPort removes the port specification from an address. +func StripPort(s string) string { + if h, _, err := net.SplitHostPort(s); err == nil { + s = h + } + return s +} + +// Error defines a type for a function that accepts a ResponseWriter for +// a Request with the HTTP status code and error. +type Error func(w http.ResponseWriter, r *http.Request, status int, err error) diff --git a/httputil/negotiate.go b/httputil/negotiate.go new file mode 100644 index 0000000..6af3e4c --- /dev/null +++ b/httputil/negotiate.go @@ -0,0 +1,79 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "github.com/golang/gddo/httputil/header" + "net/http" + "strings" +) + +// NegotiateContentEncoding returns the best offered content encoding for the +// request's Accept-Encoding header. If two offers match with equal weight and +// then the offer earlier in the list is preferred. If no offers are +// acceptable, then "" is returned. +func NegotiateContentEncoding(r *http.Request, offers []string) string { + bestOffer := "identity" + bestQ := -1.0 + specs := header.ParseAccept(r.Header, "Accept-Encoding") + for _, offer := range offers { + for _, spec := range specs { + if spec.Q > bestQ && + (spec.Value == "*" || spec.Value == offer) { + bestQ = spec.Q + bestOffer = offer + } + } + } + if bestQ == 0 { + bestOffer = "" + } + return bestOffer +} + +// NegotiateContentType returns the best offered content type for the request's +// Accept header. If two offers match with equal weight, then the more specific +// offer is preferred. For example, text/* trumps */*. If two offers match +// with equal weight and specificity, then the offer earlier in the list is +// preferred. If no offers match, then defaultOffer is returned. +func NegotiateContentType(r *http.Request, offers []string, defaultOffer string) string { + bestOffer := defaultOffer + bestQ := -1.0 + bestWild := 3 + specs := header.ParseAccept(r.Header, "Accept") + for _, offer := range offers { + for _, spec := range specs { + switch { + case spec.Q == 0.0: + // ignore + case spec.Q < bestQ: + // better match found + case spec.Value == "*/*": + if spec.Q > bestQ || bestWild > 2 { + bestQ = spec.Q + bestWild = 2 + bestOffer = offer + } + case strings.HasSuffix(spec.Value, "/*"): + if strings.HasPrefix(offer, spec.Value[:len(spec.Value)-1]) && + (spec.Q > bestQ || bestWild > 1) { + bestQ = spec.Q + bestWild = 1 + bestOffer = offer + } + default: + if spec.Value == offer && + (spec.Q > bestQ || bestWild > 0) { + bestQ = spec.Q + bestWild = 0 + bestOffer = offer + } + } + } + } + return bestOffer +} diff --git a/httputil/negotiate_test.go b/httputil/negotiate_test.go new file mode 100644 index 0000000..24bf4be --- /dev/null +++ b/httputil/negotiate_test.go @@ -0,0 +1,71 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil_test + +import ( + "github.com/golang/gddo/httputil" + "net/http" + "testing" +) + +var negotiateContentEncodingTests = []struct { + s string + offers []string + expect string +}{ + {"", []string{"identity", "gzip"}, "identity"}, + {"*;q=0", []string{"identity", "gzip"}, ""}, + {"gzip", []string{"identity", "gzip"}, "gzip"}, +} + +func TestNegotiateContentEnoding(t *testing.T) { + for _, tt := range negotiateContentEncodingTests { + r := &http.Request{Header: http.Header{"Accept-Encoding": {tt.s}}} + actual := httputil.NegotiateContentEncoding(r, tt.offers) + if actual != tt.expect { + t.Errorf("NegotiateContentEncoding(%q, %#v)=%q, want %q", tt.s, tt.offers, actual, tt.expect) + } + } +} + +var negotiateContentTypeTests = []struct { + s string + offers []string + defaultOffer string + expect string +}{ + {"text/html, */*;q=0", []string{"x/y"}, "", ""}, + {"text/html, */*", []string{"x/y"}, "", "x/y"}, + {"text/html, image/png", []string{"text/html", "image/png"}, "", "text/html"}, + {"text/html, image/png", []string{"image/png", "text/html"}, "", "image/png"}, + {"text/html, image/png; q=0.5", []string{"image/png"}, "", "image/png"}, + {"text/html, image/png; q=0.5", []string{"text/html"}, "", "text/html"}, + {"text/html, image/png; q=0.5", []string{"foo/bar"}, "", ""}, + {"text/html, image/png; q=0.5", []string{"image/png", "text/html"}, "", "text/html"}, + {"text/html, image/png; q=0.5", []string{"text/html", "image/png"}, "", "text/html"}, + {"text/html;q=0.5, image/png", []string{"image/png"}, "", "image/png"}, + {"text/html;q=0.5, image/png", []string{"text/html"}, "", "text/html"}, + {"text/html;q=0.5, image/png", []string{"image/png", "text/html"}, "", "image/png"}, + {"text/html;q=0.5, image/png", []string{"text/html", "image/png"}, "", "image/png"}, + {"image/png, image/*;q=0.5", []string{"image/jpg", "image/png"}, "", "image/png"}, + {"image/png, image/*;q=0.5", []string{"image/jpg"}, "", "image/jpg"}, + {"image/png, image/*;q=0.5", []string{"image/jpg", "image/gif"}, "", "image/jpg"}, + {"image/png, image/*", []string{"image/jpg", "image/gif"}, "", "image/jpg"}, + {"image/png, image/*", []string{"image/gif", "image/jpg"}, "", "image/gif"}, + {"image/png, image/*", []string{"image/gif", "image/png"}, "", "image/png"}, + {"image/png, image/*", []string{"image/png", "image/gif"}, "", "image/png"}, +} + +func TestNegotiateContentType(t *testing.T) { + for _, tt := range negotiateContentTypeTests { + r := &http.Request{Header: http.Header{"Accept": {tt.s}}} + actual := httputil.NegotiateContentType(r, tt.offers, tt.defaultOffer) + if actual != tt.expect { + t.Errorf("NegotiateContentType(%q, %#v, %q)=%q, want %q", tt.s, tt.offers, tt.defaultOffer, actual, tt.expect) + } + } +} diff --git a/httputil/respbuf.go b/httputil/respbuf.go new file mode 100644 index 0000000..051af21 --- /dev/null +++ b/httputil/respbuf.go @@ -0,0 +1,58 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "bytes" + "net/http" + "strconv" +) + +// ResponseBuffer is the current response being composed by its owner. +// It implements http.ResponseWriter and io.WriterTo. +type ResponseBuffer struct { + buf bytes.Buffer + status int + header http.Header +} + +// Write implements the http.ResponseWriter interface. +func (rb *ResponseBuffer) Write(p []byte) (int, error) { + return rb.buf.Write(p) +} + +// WriteHeader implements the http.ResponseWriter interface. +func (rb *ResponseBuffer) WriteHeader(status int) { + rb.status = status +} + +// Header implements the http.ResponseWriter interface. +func (rb *ResponseBuffer) Header() http.Header { + if rb.header == nil { + rb.header = make(http.Header) + } + return rb.header +} + +// WriteTo implements the io.WriterTo interface. +func (rb *ResponseBuffer) WriteTo(w http.ResponseWriter) error { + for k, v := range rb.header { + w.Header()[k] = v + } + if rb.buf.Len() > 0 { + w.Header().Set("Content-Length", strconv.Itoa(rb.buf.Len())) + } + if rb.status != 0 { + w.WriteHeader(rb.status) + } + if rb.buf.Len() > 0 { + if _, err := w.Write(rb.buf.Bytes()); err != nil { + return err + } + } + return nil +} diff --git a/httputil/static.go b/httputil/static.go new file mode 100644 index 0000000..6610dde --- /dev/null +++ b/httputil/static.go @@ -0,0 +1,265 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "bytes" + "crypto/sha1" + "errors" + "fmt" + "github.com/golang/gddo/httputil/header" + "io" + "io/ioutil" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// StaticServer serves static files. +type StaticServer struct { + // Dir specifies the location of the directory containing the files to serve. + Dir string + + // MaxAge specifies the maximum age for the cache control and expiration + // headers. + MaxAge time.Duration + + // Error specifies the function used to generate error responses. If Error + // is nil, then http.Error is used to generate error responses. + Error Error + + // MIMETypes is a map from file extensions to MIME types. + MIMETypes map[string]string + + mu sync.Mutex + etags map[string]string +} + +func (ss *StaticServer) resolve(fname string) string { + if path.IsAbs(fname) { + panic("Absolute path not allowed when creating a StaticServer handler") + } + dir := ss.Dir + if dir == "" { + dir = "." + } + fname = filepath.FromSlash(fname) + return filepath.Join(dir, fname) +} + +func (ss *StaticServer) mimeType(fname string) string { + ext := path.Ext(fname) + var mimeType string + if ss.MIMETypes != nil { + mimeType = ss.MIMETypes[ext] + } + if mimeType == "" { + mimeType = mime.TypeByExtension(ext) + } + if mimeType == "" { + mimeType = "application/octet-stream" + } + return mimeType +} + +func (ss *StaticServer) openFile(fname string) (io.ReadCloser, int64, string, error) { + f, err := os.Open(fname) + if err != nil { + return nil, 0, "", err + } + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, 0, "", err + } + const modeType = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice + if fi.Mode()&modeType != 0 { + f.Close() + return nil, 0, "", errors.New("not a regular file") + } + return f, fi.Size(), ss.mimeType(fname), nil +} + +// FileHandler returns a handler that serves a single file. The file is +// specified by a slash separated path relative to the static server's Dir +// field. +func (ss *StaticServer) FileHandler(fileName string) http.Handler { + id := fileName + fileName = ss.resolve(fileName) + return &staticHandler{ + ss: ss, + id: func(_ string) string { return id }, + open: func(_ string) (io.ReadCloser, int64, string, error) { return ss.openFile(fileName) }, + } +} + +// DirectoryHandler returns a handler that serves files from a directory tree. +// The directory is specified by a slash separated path relative to the static +// server's Dir field. +func (ss *StaticServer) DirectoryHandler(prefix, dirName string) http.Handler { + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + idBase := dirName + dirName = ss.resolve(dirName) + return &staticHandler{ + ss: ss, + id: func(p string) string { + if !strings.HasPrefix(p, prefix) { + return "." + } + return path.Join(idBase, p[len(prefix):]) + }, + open: func(p string) (io.ReadCloser, int64, string, error) { + if !strings.HasPrefix(p, prefix) { + return nil, 0, "", errors.New("request url does not match directory prefix") + } + p = p[len(prefix):] + return ss.openFile(filepath.Join(dirName, filepath.FromSlash(p))) + }, + } +} + +// FilesHandler returns a handler that serves the concatentation of the +// specified files. The files are specified by slash separated paths relative +// to the static server's Dir field. +func (ss *StaticServer) FilesHandler(fileNames ...string) http.Handler { + + // todo: cache concatenated files on disk and serve from there. + + mimeType := ss.mimeType(fileNames[0]) + var buf []byte + var openErr error + + for _, fileName := range fileNames { + p, err := ioutil.ReadFile(ss.resolve(fileName)) + if err != nil { + openErr = err + buf = nil + break + } + buf = append(buf, p...) + } + + id := strings.Join(fileNames, " ") + + return &staticHandler{ + ss: ss, + id: func(_ string) string { return id }, + open: func(p string) (io.ReadCloser, int64, string, error) { + return ioutil.NopCloser(bytes.NewReader(buf)), int64(len(buf)), mimeType, openErr + }, + } +} + +type staticHandler struct { + id func(fname string) string + open func(p string) (io.ReadCloser, int64, string, error) + ss *StaticServer +} + +func (h *staticHandler) error(w http.ResponseWriter, r *http.Request, status int, err error) { + http.Error(w, http.StatusText(status), status) +} + +func (h *staticHandler) etag(p string) (string, error) { + id := h.id(p) + + h.ss.mu.Lock() + if h.ss.etags == nil { + h.ss.etags = make(map[string]string) + } + etag := h.ss.etags[id] + h.ss.mu.Unlock() + + if etag != "" { + return etag, nil + } + + // todo: if a concurrent goroutine is calculating the hash, then wait for + // it instead of computing it again here. + + rc, _, _, err := h.open(p) + if err != nil { + return "", err + } + + defer rc.Close() + + w := sha1.New() + _, err = io.Copy(w, rc) + if err != nil { + return "", err + } + + etag = fmt.Sprintf(`"%x"`, w.Sum(nil)) + + h.ss.mu.Lock() + h.ss.etags[id] = etag + h.ss.mu.Unlock() + + return etag, nil +} + +func (h *staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p := path.Clean(r.URL.Path) + if p != r.URL.Path { + http.Redirect(w, r, p, 301) + return + } + + etag, err := h.etag(p) + if err != nil { + h.error(w, r, http.StatusNotFound, err) + return + } + + maxAge := h.ss.MaxAge + if maxAge == 0 { + maxAge = 24 * time.Hour + } + if r.FormValue("v") != "" { + maxAge = 365 * 24 * time.Hour + } + + cacheControl := fmt.Sprintf("public, max-age=%d", maxAge/time.Second) + + for _, e := range header.ParseList(r.Header, "If-None-Match") { + if e == etag { + w.Header().Set("Cache-Control", cacheControl) + w.Header().Set("Etag", etag) + w.WriteHeader(http.StatusNotModified) + return + } + } + + rc, cl, ct, err := h.open(p) + if err != nil { + h.error(w, r, http.StatusNotFound, err) + return + } + defer rc.Close() + + w.Header().Set("Cache-Control", cacheControl) + w.Header().Set("Etag", etag) + if ct != "" { + w.Header().Set("Content-Type", ct) + } + if cl != 0 { + w.Header().Set("Content-Length", strconv.FormatInt(cl, 10)) + } + w.WriteHeader(http.StatusOK) + if r.Method != "HEAD" { + io.Copy(w, rc) + } +} diff --git a/httputil/static_test.go b/httputil/static_test.go new file mode 100644 index 0000000..a258d52 --- /dev/null +++ b/httputil/static_test.go @@ -0,0 +1,175 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil_test + +import ( + "crypto/sha1" + "encoding/hex" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "testing" + "time" + + "github.com/golang/gddo/httputil" + "github.com/google/go-cmp/cmp" +) + +var ( + testHash = computeTestHash() + testEtag = `"` + testHash + `"` + testContentLength = computeTestContentLength() +) + +func mustParseURL(urlStr string) *url.URL { + u, err := url.Parse(urlStr) + if err != nil { + panic(err) + } + return u +} + +func computeTestHash() string { + p, err := ioutil.ReadFile("static_test.go") + if err != nil { + panic(err) + } + w := sha1.New() + w.Write(p) + return hex.EncodeToString(w.Sum(nil)) +} + +func computeTestContentLength() string { + info, err := os.Stat("static_test.go") + if err != nil { + panic(err) + } + return strconv.FormatInt(info.Size(), 10) +} + +var fileServerTests = []*struct { + name string // test name for log + ss *httputil.StaticServer + r *http.Request + header http.Header // expected response headers + status int // expected response status + empty bool // true if response body not expected. +}{ + { + name: "get", + ss: &httputil.StaticServer{MaxAge: 3 * time.Second}, + r: &http.Request{ + URL: mustParseURL("/dir/static_test.go"), + Method: "GET", + }, + status: http.StatusOK, + header: http.Header{ + "Etag": {testEtag}, + "Cache-Control": {"public, max-age=3"}, + "Content-Length": {testContentLength}, + "Content-Type": {"application/octet-stream"}, + }, + }, + { + name: "get .", + ss: &httputil.StaticServer{Dir: ".", MaxAge: 3 * time.Second}, + r: &http.Request{ + URL: mustParseURL("/dir/static_test.go"), + Method: "GET", + }, + status: http.StatusOK, + header: http.Header{ + "Etag": {testEtag}, + "Cache-Control": {"public, max-age=3"}, + "Content-Length": {testContentLength}, + "Content-Type": {"application/octet-stream"}, + }, + }, + { + name: "get with ?v=", + ss: &httputil.StaticServer{MaxAge: 3 * time.Second}, + r: &http.Request{ + URL: mustParseURL("/dir/static_test.go?v=xxxxx"), + Method: "GET", + }, + status: http.StatusOK, + header: http.Header{ + "Etag": {testEtag}, + "Cache-Control": {"public, max-age=31536000"}, + "Content-Length": {testContentLength}, + "Content-Type": {"application/octet-stream"}, + }, + }, + { + name: "head", + ss: &httputil.StaticServer{MaxAge: 3 * time.Second}, + r: &http.Request{ + URL: mustParseURL("/dir/static_test.go"), + Method: "HEAD", + }, + status: http.StatusOK, + header: http.Header{ + "Etag": {testEtag}, + "Cache-Control": {"public, max-age=3"}, + "Content-Length": {testContentLength}, + "Content-Type": {"application/octet-stream"}, + }, + empty: true, + }, + { + name: "if-none-match", + ss: &httputil.StaticServer{MaxAge: 3 * time.Second}, + r: &http.Request{ + URL: mustParseURL("/dir/static_test.go"), + Method: "GET", + Header: http.Header{"If-None-Match": {testEtag}}, + }, + status: http.StatusNotModified, + header: http.Header{ + "Cache-Control": {"public, max-age=3"}, + "Etag": {testEtag}, + }, + empty: true, + }, +} + +func testStaticServer(t *testing.T, f func(*httputil.StaticServer) http.Handler) { + for _, tt := range fileServerTests { + w := httptest.NewRecorder() + + h := f(tt.ss) + h.ServeHTTP(w, tt.r) + + if w.Code != tt.status { + t.Errorf("%s, status=%d, want %d", tt.name, w.Code, tt.status) + } + + if !cmp.Equal(w.HeaderMap, tt.header) { + t.Errorf("%s\n\theader=%v,\n\twant %v", tt.name, w.HeaderMap, tt.header) + } + + empty := w.Body.Len() == 0 + if empty != tt.empty { + t.Errorf("%s empty=%v, want %v", tt.name, empty, tt.empty) + } + } +} + +func TestFileHandler(t *testing.T) { + testStaticServer(t, func(ss *httputil.StaticServer) http.Handler { return ss.FileHandler("static_test.go") }) +} + +func TestDirectoryHandler(t *testing.T) { + testStaticServer(t, func(ss *httputil.StaticServer) http.Handler { return ss.DirectoryHandler("/dir", ".") }) +} + +func TestFilesHandler(t *testing.T) { + testStaticServer(t, func(ss *httputil.StaticServer) http.Handler { return ss.FilesHandler("static_test.go") }) +} diff --git a/httputil/transport.go b/httputil/transport.go new file mode 100644 index 0000000..fdad3b4 --- /dev/null +++ b/httputil/transport.go @@ -0,0 +1,87 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// This file implements a http.RoundTripper that authenticates +// requests issued against api.github.com endpoint. + +package httputil + +import ( + "net/http" + "net/url" +) + +// AuthTransport is an implementation of http.RoundTripper that authenticates +// with the GitHub API. +// +// When both a token and client credentials are set, the latter is preferred. +type AuthTransport struct { + UserAgent string + GithubToken string + GithubClientID string + GithubClientSecret string + Base http.RoundTripper +} + +// RoundTrip implements the http.RoundTripper interface. +func (t *AuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + var reqCopy *http.Request + if t.UserAgent != "" { + reqCopy = copyRequest(req) + reqCopy.Header.Set("User-Agent", t.UserAgent) + } + if req.URL.Host == "api.github.com" && req.URL.Scheme == "https" { + switch { + case t.GithubClientID != "" && t.GithubClientSecret != "": + if reqCopy == nil { + reqCopy = copyRequest(req) + } + if reqCopy.URL.RawQuery == "" { + reqCopy.URL.RawQuery = "client_id=" + t.GithubClientID + "&client_secret=" + t.GithubClientSecret + } else { + reqCopy.URL.RawQuery += "&client_id=" + t.GithubClientID + "&client_secret=" + t.GithubClientSecret + } + case t.GithubToken != "": + if reqCopy == nil { + reqCopy = copyRequest(req) + } + reqCopy.Header.Set("Authorization", "token "+t.GithubToken) + } + } + if reqCopy != nil { + return t.base().RoundTrip(reqCopy) + } + return t.base().RoundTrip(req) +} + +// CancelRequest cancels an in-flight request by closing its connection. +func (t *AuthTransport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(req *http.Request) + } + if cr, ok := t.base().(canceler); ok { + cr.CancelRequest(req) + } +} + +func (t *AuthTransport) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func copyRequest(req *http.Request) *http.Request { + req2 := new(http.Request) + *req2 = *req + req2.URL = new(url.URL) + *req2.URL = *req.URL + req2.Header = make(http.Header, len(req.Header)) + for k, s := range req.Header { + req2.Header[k] = append([]string(nil), s...) + } + return req2 +} diff --git a/httputil/transport_test.go b/httputil/transport_test.go new file mode 100644 index 0000000..4b36118 --- /dev/null +++ b/httputil/transport_test.go @@ -0,0 +1,126 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "bytes" + "io/ioutil" + "net/http" + "net/url" + "testing" +) + +func TestTransportGithubAuth(t *testing.T) { + tests := []struct { + name string + + url string + token string + clientID string + clientSecret string + + queryClientID string + queryClientSecret string + authorization string + }{ + { + name: "Github token", + url: "https://api.github.com/", + token: "xyzzy", + authorization: "token xyzzy", + }, + { + name: "Github client ID/secret", + url: "https://api.github.com/", + clientID: "12345", + clientSecret: "xyzzy", + queryClientID: "12345", + queryClientSecret: "xyzzy", + }, + { + name: "non-Github site does not have token headers", + url: "http://www.example.com/", + token: "xyzzy", + }, + { + name: "non-Github site does not have client ID/secret headers", + url: "http://www.example.com/", + clientID: "12345", + clientSecret: "xyzzy", + }, + { + name: "Github token not sent over HTTP", + url: "http://api.github.com/", + token: "xyzzy", + }, + { + name: "Github client ID/secret not sent over HTTP", + url: "http://api.github.com/", + clientID: "12345", + clientSecret: "xyzzy", + }, + { + name: "Github token not sent over schemeless", + url: "//api.github.com/", + token: "xyzzy", + }, + { + name: "Github client ID/secret not sent over schemeless", + url: "//api.github.com/", + clientID: "12345", + clientSecret: "xyzzy", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var ( + query url.Values + authHeader string + ) + client := &http.Client{ + Transport: &AuthTransport{ + Base: roundTripFunc(func(r *http.Request) { + query = r.URL.Query() + authHeader = r.Header.Get("Authorization") + }), + GithubToken: test.token, + GithubClientID: test.clientID, + GithubClientSecret: test.clientSecret, + }, + } + _, err := client.Get(test.url) + if err != nil { + t.Fatal(err) + } + if got := query.Get("client_id"); got != test.queryClientID { + t.Errorf("url query client_id = %q; want %q", got, test.queryClientID) + } + if got := query.Get("client_secret"); got != test.queryClientSecret { + t.Errorf("url query client_secret = %q; want %q", got, test.queryClientSecret) + } + if authHeader != test.authorization { + t.Errorf("header Authorization = %q; want %q", authHeader, test.authorization) + } + }) + } +} + +type roundTripFunc func(r *http.Request) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + f(r) + return &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Body: ioutil.NopCloser(bytes.NewReader(nil)), + ContentLength: 0, + Request: r, + }, nil +} -- cgit v1.2.3