package files import ( "context" "net/http" "path/filepath" "time" "github.com/neilotoole/streamcache" "github.com/neilotoole/sq/libsq/core/ioz" "github.com/neilotoole/sq/libsq/core/ioz/checksum" "github.com/neilotoole/sq/libsq/core/ioz/httpz" "github.com/neilotoole/sq/libsq/core/options" "github.com/neilotoole/sq/libsq/files/internal/downloader" "github.com/neilotoole/sq/libsq/source" ) var ( OptHTTPRequestTimeout = options.NewDuration( "http.request.timeout", nil, time.Second*10, "HTTP/S request initial response timeout duration", `How long to wait for initial response from a HTTP/S endpoint before timeout occurs. Reading the body of the response, such as a large HTTP file download, is not affected by this option. Example: 500ms or 3s. Contrast with http.response.timeout.`, options.TagSource, ) OptHTTPResponseTimeout = options.NewDuration( "http.response.timeout", nil, 0, "HTTP/S request completion timeout duration", `How long to wait for the entire HTTP transaction to complete. This includes reading the body of the response, such as a large HTTP file download. Typically this is set to 0, indicating no timeout. Contrast with http.request.timeout.`, options.TagSource, ) OptHTTPSInsecureSkipVerify = options.NewBool( "https.insecure-skip-verify", nil, false, "Skip HTTPS TLS verification", "Skip HTTPS TLS verification. Useful when downloading against self-signed certs.", options.TagSource, ) OptDownloadContinueOnError = downloader.OptContinueOnError OptDownloadCache = downloader.OptCache ) // maybeStartDownload starts a download for src if one is not already in progress // or completed. If there's a download in progress, dlStream returns non-nil. // If the file is already downloaded to disk (and is valid/fresh), dlFile // returns non-empty and contains the absolute path to the downloaded file. // Otherwise, a new download is started (on a spawned goroutine), and the // stream returned from the downloader is added to Files.streams. On successful // download and cache update completion, the stream is removed from Files.streams // and the path to the cached file is added to Files.downloadedFiles. // // If arg checkFresh is false, and there's already a cached download on disk, // then the cached file is returned immediately, and no download is started. // // It is guaranteed that one (and only one) of the returned values will be non-nil. // REVISIT: look into use of checkFresh? func (fs *Files) maybeStartDownload(ctx context.Context, src *source.Source, checkFresh bool) (dlFile string, dlStream *streamcache.Stream, err error, ) { var ok bool // If the file has just been downloaded, just return it. It doesn't // matter about checkFresh. if dlFile, ok = fs.downloadedFiles[src.Handle]; ok { return dlFile, nil, nil } // If there's already a download in progress, then we can just return // that stream. if dlStream, ok = fs.streams[src.Handle]; ok { // A download stream is always fresh, so we // can ignore checkFresh here. return "", dlStream, nil } dldr, err := fs.downloaderFor(ctx, src) if err != nil { return "", nil, err } if !checkFresh { // If we don't care about freshness, check if the download is // already on disk. If so, Downloader.CacheFile will return the // path to the cached file. dlFile, err = dldr.CacheFile(ctx) if err == nil && dlFile != "" { // The file is already on disk, so we can just return it. // REVISIT: Should we add dlFile to the downloadFiles map? return dlFile, nil, err } } dlFile, dlStream, err = dldr.Get(ctx) switch { case err != nil: return "", nil, err case dlFile != "": // The file is already on disk, so we can just return it. fs.downloadedFiles[src.Handle] = dlFile return dlFile, nil, nil default: // A new download stream was created. Append it to Files.streams, // and return the stream. fs.streams[src.Handle] = dlStream return "", dlStream, nil } } // downloadPaths returns the paths for src's download cache dir and // cache body file. It is not guaranteed that the returned paths exist. func (fs *Files) downloadPaths(src *source.Source) (dlDir, dlFile string, err error) { var cacheDir string cacheDir, err = fs.CacheDirFor(src) if err != nil { return "", dlFile, err } // Note: we depend on internal knowledge of the downloader impl here, // which is not great. It might be better to implement a function // in pkg downloader. dlDir = filepath.Join(cacheDir, "download", checksum.Sum([]byte(src.Location))) dlFile = filepath.Join(dlDir, "main", "body") return dlDir, dlFile, nil } // downloaderFor returns the downloader.Downloader for src, creating // and caching it if necessary. func (fs *Files) downloaderFor(ctx context.Context, src *source.Source) (*downloader.Downloader, error) { dl, ok := fs.downloaders[src.Handle] if ok { return dl, nil } dlDir, _, err := fs.downloadPaths(src) if err != nil { return nil, err } if err = ioz.RequireDir(dlDir); err != nil { return nil, err } c := fs.httpClientFor(ctx, src) if dl, err = downloader.New(src.Handle, c, src.Location, dlDir); err != nil { return nil, err } fs.downloaders[src.Handle] = dl return dl, nil } func (fs *Files) httpClientFor(ctx context.Context, src *source.Source) *http.Client { o := options.Merge(options.FromContext(ctx), src.Options) return httpz.NewClient(httpz.DefaultUserAgent, httpz.OptRequestTimeout(OptHTTPRequestTimeout.Get(o)), httpz.OptResponseTimeout(OptHTTPResponseTimeout.Get(o)), httpz.OptInsecureSkipVerify(OptHTTPSInsecureSkipVerify.Get(o)), ) }