sq/libsq/source/fetcher/fetcher.go
Neil O'Toole 98e91bae51
Bunch of linting issues (#96)
* linting

* linting

* linting

* linting

* linting

* cleaned up readme trailing newlines
2021-09-12 16:14:30 -06:00

111 lines
2.4 KiB
Go

// Package fetcher provides a mechanism for fetching files
// from URLs.
package fetcher
import (
"context"
"crypto/tls"
"io"
"net/http"
"net/url"
"time"
"golang.org/x/net/context/ctxhttp"
"github.com/neilotoole/sq/libsq/core/errz"
"github.com/neilotoole/sq/libsq/core/stringz"
)
// Config parameterizes Fetcher behavior.
type Config struct {
// Timeout is the request timeout.
Timeout time.Duration
// Skip verification of insecure transports.
InsecureSkipVerify bool
}
// Fetcher can fetch files from URLs. If field Config is nil,
// defaults are used. At this time, only HTTP/HTTPS is supported,
// but it's possible other schemes (such as FTP) will be
// supported in future.
type Fetcher struct {
Config *Config
}
// Fetch writes the body of the document at fileURL to w.
func (f *Fetcher) Fetch(ctx context.Context, fileURL string, w io.Writer) error {
return fetchHTTP(ctx, f.Config, fileURL, w)
}
func httpClient(cfg *Config) *http.Client {
var client = *http.DefaultClient
var tr *http.Transport
if client.Transport == nil {
tr = (http.DefaultTransport.(*http.Transport)).Clone()
} else {
tr = (client.Transport.(*http.Transport)).Clone()
}
if tr.TLSClientConfig == nil {
tr.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
} else {
tr.TLSClientConfig = tr.TLSClientConfig.Clone()
}
if cfg != nil {
tr.TLSClientConfig.InsecureSkipVerify = cfg.InsecureSkipVerify
client.Timeout = cfg.Timeout
}
client.Transport = tr
return &client
}
func fetchHTTP(ctx context.Context, cfg *Config, fileURL string, w io.Writer) error {
c := httpClient(cfg)
req, err := http.NewRequest("GET", fileURL, nil)
if err != nil {
return err
}
resp, err := ctxhttp.Do(ctx, c, req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
_ = resp.Body.Close()
return errz.Errorf("http: returned non-200 status code (%s) from: %s", resp.Status, fileURL)
}
_, err = io.Copy(w, resp.Body)
if err != nil {
_ = resp.Body.Close()
return errz.Wrapf(err, "http: failed to read body from: %s", fileURL)
}
return errz.Err(resp.Body.Close())
}
// Schemes is the set of supported schemes.
func (f *Fetcher) Schemes() []string {
return []string{"http", "https"}
}
// Supported returns true if loc is a supported URL.
func (f *Fetcher) Supported(loc string) bool {
u, err := url.ParseRequestURI(loc)
if err != nil {
return false
}
if stringz.InSlice(f.Schemes(), u.Scheme) {
return true
}
return false
}