sq/libsq/source/fetcher/fetcher.go

// Package fetcher provides a mechanism for fetching files
// from URLs.
package fetcher

import (
	"context"
	"crypto/tls"
	"io"
	"net/http"
	"net/url"
	"time"

	"golang.org/x/net/context/ctxhttp"

	"github.com/neilotoole/sq/libsq/core/errz"
	"github.com/neilotoole/sq/libsq/core/stringz"
)

// Config parameterizes Fetcher behavior.
type Config struct {
	// Timeout is the request timeout.
	Timeout time.Duration

	// Skip verification of insecure transports.
	InsecureSkipVerify bool
}

// Fetcher can fetch files from URLs. If field Config is nil,
// defaults are used. At this time, only HTTP/HTTPS is supported,
// but it's possible other schemes (such as FTP) will be
// supported in future.
type Fetcher struct {
	Config *Config
}

// Fetch writes the body of the document at fileURL to w.
func (f *Fetcher) Fetch(ctx context.Context, fileURL string, w io.Writer) error {
	return fetchHTTP(ctx, f.Config, fileURL, w)
}

func httpClient(cfg *Config) *http.Client {
	var client = *http.DefaultClient

	var tr *http.Transport
	if client.Transport == nil {
		tr = (http.DefaultTransport.(*http.Transport)).Clone()
	} else {
		tr = (client.Transport.(*http.Transport)).Clone()
	}

	if tr.TLSClientConfig == nil {
		tr.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
	} else {
		tr.TLSClientConfig = tr.TLSClientConfig.Clone()
	}

	if cfg != nil {
		tr.TLSClientConfig.InsecureSkipVerify = cfg.InsecureSkipVerify
		client.Timeout = cfg.Timeout
	}

	client.Transport = tr

	return &client
}

func fetchHTTP(ctx context.Context, cfg *Config, fileURL string, w io.Writer) error {
	c := httpClient(cfg)
	req, err := http.NewRequest("GET", fileURL, nil)
	if err != nil {
		return err
	}

	resp, err := ctxhttp.Do(ctx, c, req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		_ = resp.Body.Close()
		return errz.Errorf("http: returned non-200 status code (%s) from: %s", resp.Status, fileURL)
	}

	_, err = io.Copy(w, resp.Body)
	if err != nil {
		_ = resp.Body.Close()
		return errz.Wrapf(err, "http: failed to read body from: %s", fileURL)
	}

	return errz.Err(resp.Body.Close())
}

// Schemes is the set of supported schemes.
func (f *Fetcher) Schemes() []string {
	return []string{"http", "https"}
}

// Supported returns true if loc is a supported URL.
func (f *Fetcher) Supported(loc string) bool {
	u, err := url.ParseRequestURI(loc)
	if err != nil {
		return false
	}

	if stringz.InSlice(f.Schemes(), u.Scheme) {
		return true
	}

	return false
}