package main import ( "encoding/json" "github.com/lox/httpcache" "github.com/pkg/errors" "net" "net/http" "net/url" "strings" "time" ) type CautiousHTTPClient interface { Get(string) (*http.Response, error) GetJSON(string, interface{}) error GetJSONExpires(string, interface{}) (time.Duration, error) } type cautiousHttpClient struct { allowHttp bool client *http.Client } // allowHttp is UNSAFE and technically validates the spec but it does make it // easier to work in dev so leaving it in for now func NewCautiousHTTPClient(allowHttp bool) (CautiousHTTPClient, error) { CautiousTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 1 * time.Second, KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 1 * time.Second, ExpectContinueTimeout: 1 * time.Second, ResponseHeaderTimeout: 10 * time.Second, MaxResponseHeaderBytes: 500000, // .5 MB } return &cautiousHttpClient{ allowHttp: allowHttp, client: &http.Client{ Transport: CautiousTransport, Timeout: 30 * time.Second, }, }, nil } func (c *cautiousHttpClient) Get(gurl string) (*http.Response, error) { u, err := url.Parse(gurl) if err != nil { return nil, errors.WithStack(err) } if u.Scheme != "https" && !c.allowHttp { return nil, errors.Errorf("URL for GET must be secure") } r, err := c.client.Get(u.String()) if err != nil { return nil, errors.WithStack(err) } r.Body = http.MaxBytesReader(nil, r.Body, 1000000) return r, nil } func (c *cautiousHttpClient) GetJSON(url string, rv interface{}) error { r, err := c.Get(url) if err != nil { return errors.WithStack(err) } defer r.Body.Close() d := json.NewDecoder(r.Body) err = d.Decode(rv) if err != nil { return errors.WithStack(err) } return nil } func (c *cautiousHttpClient) GetJSONExpires(url string, rv interface{}) (time.Duration, error) { r, err := c.Get(url) if err != nil { return time.Duration(0), errors.WithStack(err) } defer r.Body.Close() res := httpcache.NewResource(r.StatusCode, nil, r.Header) d := json.NewDecoder(r.Body) err = d.Decode(rv) if err != nil { return time.Duration(0), errors.WithStack(err) } return refreshAfter(res), nil } type JSONURL struct { *url.URL } func (u *JSONURL) AsURL() *url.URL { return u.URL } func (u *JSONURL) UnmarshalJSON(data []byte) error { d := strings.Trim(string(data), "\"") pu, err := url.Parse(d) if err != nil { return errors.WithStack(err) } u.URL = pu return nil } func refreshAfter(res *httpcache.Resource) time.Duration { maxAge, err := res.MaxAge(false) if err != nil { return time.Duration(0) } age, err := res.Age() if err != nil { return time.Duration(0) } if hFresh := res.HeuristicFreshness(); hFresh > maxAge { maxAge = hFresh } return maxAge - age }