// Package http provides a filesystem interface using golang.org/net/http//// It treats HTML pages served from the endpoint as directory// listings, and includes any links found as files.package http import (	"context"	"errors"	"fmt"	"io"	"mime"	"net/http"	"net/url"	"path"	"strings"	"sync"	"time" 	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/lib/rest"	"golang.org/x/net/html") var (	errorReadOnly = errors.New("http remotes are read only")	timeUnset     = time.Unix(0, 0)) func init() {	fsi := &fs.RegInfo{		Name:        "http",		Description: "HTTP",		NewFs:       NewFs,		CommandHelp: commandHelp,		Options: []fs.Option{{			Name:     "url",			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",			Required: true,		}, {			Name: "headers",			Help: `Set HTTP headers for all transactions. Use this to set additional HTTP headers for all transactions. The input format is comma separated list of key,value pairs.  Standard[CSV encoding](https://godoc.org/encoding/csv) may be used. For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'. You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,			Default:  fs.CommaSepList{},			Advanced: true,		}, {			Name: "no_slash",			Help: `Set this if the site doesn't end directories with /. Use this if your target website does not use / on the end ofdirectories. A / on the end of a path is how rclone normally tells the differencebetween files and directories.  If this flag is set, then rclone willtreat all files with Content-Type: text/html as directories and readURLs from them rather than downloading them. Note that this may cause rclone to confuse genuine HTML files withdirectories.`,			Default:  false,			Advanced: true,		}, {			Name: "no_head",			Help: `Don't use HEAD requests. HEAD requests are mainly used to find file sizes in dir listing.If your site is being very slow to load then you can try this option.Normally rclone does a HEAD request for each potential file in adirectory listing to: - find its size- check it really exists- check to see if it is a directory If you set this option, rclone will not do the HEAD request. This will meanthat directory listings are much quicker, but rclone won't have the times orsizes of any files, and some files that don't exist may be in the listing.`,			Default:  false,			Advanced: true,		}, {			Name:    "no_escape",			Help:    "Do not escape URL metacharacters in path names.",			Default: false,		}},	}	fs.Register(fsi)} // Options defines the configuration for this backendtype Options struct {	Endpoint string          `config:"url"`	NoSlash  bool            `config:"no_slash"`	NoHead   bool            `config:"no_head"`	Headers  fs.CommaSepList `config:"headers"`	NoEscape bool            `config:"no_escape"`} // Fs stores the interface to the remote HTTP filestype Fs struct {	name        string	root        string	features    *fs.Features   // optional features	opt         Options        // options for this backend	ci          *fs.ConfigInfo // global config	endpoint    *url.URL	endpointURL string // endpoint as a string	httpClient  *http.Client} // Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)type Object struct {	fs          *Fs	remote      string	size        int64	modTime     time.Time	contentType string} // statusError returns an error if the res contained an errorfunc statusError(res *http.Response, err error) error {	if err != nil {		return err	}	if res.StatusCode < 200 || res.StatusCode > 299 {		_ = res.Body.Close()		return fmt.Errorf("HTTP Error: %s", res.Status)	}	return nil} // getFsEndpoint decides if url is to be considered a file or directory,// and returns a proper endpoint url to use for the fs.func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Options) (string, bool) {	// If url ends with '/' it is already a proper url always assumed to be a directory.	if url[len(url)-1] == '/' {		return url, false	} 	// If url does not end with '/' we send a HEAD request to decide	// if it is directory or file, and if directory appends the missing	// '/', or if file returns the directory url to parent instead.	createFileResult := func() (string, bool) {		fs.Debugf(nil, "If path is a directory you must add a trailing '/'")		parent, _ := path.Split(url)		return parent, true	}	createDirResult := func() (string, bool) {		fs.Debugf(nil, "To avoid the initial HEAD request add a trailing '/' to the path")		return url + "/", false	} 	// If HEAD requests are not allowed we just have to assume it is a file.	if opt.NoHead {		fs.Debugf(nil, "Assuming path is a file as --http-no-head is set")		return createFileResult()	} 	// Use a client which doesn't follow redirects so the server	// doesn't redirect http://host/dir to http://host/dir/	noRedir := *client	noRedir.CheckRedirect = func(req *http.Request, via []*http.Request) error {		return http.ErrUseLastResponse	}	req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)	if err != nil {		fs.Debugf(nil, "Assuming path is a file as HEAD request could not be created: %v", err)		return createFileResult()	}	addHeaders(req, opt)	res, err := noRedir.Do(req)	if err != nil {		fs.Debugf(nil, "Assuming path is a file as HEAD request could not be sent: %v", err)		return createFileResult()	}	if res.StatusCode == http.StatusNotFound {		fs.Debugf(nil, "Assuming path is a directory as HEAD response is it does not exist as a file (%s)", res.Status)		return createDirResult()	}	if res.StatusCode == http.StatusMovedPermanently ||		res.StatusCode == http.StatusFound ||		res.StatusCode == http.StatusSeeOther ||		res.StatusCode == http.StatusTemporaryRedirect ||		res.StatusCode == http.StatusPermanentRedirect {		redir := res.Header.Get("Location")		if redir != "" {			if redir[len(redir)-1] == '/' {				fs.Debugf(nil, "Assuming path is a directory as HEAD response is redirect (%s) to a path that ends with '/': %s", res.Status, redir)				return createDirResult()			}			fs.Debugf(nil, "Assuming path is a file as HEAD response is redirect (%s) to a path that does not end with '/': %s", res.Status, redir)			return createFileResult()		}		fs.Debugf(nil, "Assuming path is a file as HEAD response is redirect (%s) but no location header", res.Status)		return createFileResult()	}	if res.StatusCode < 200 || res.StatusCode > 299 {		// Example is 403 (http.StatusForbidden) for servers not allowing HEAD requests.		fs.Debugf(nil, "Assuming path is a file as HEAD response is an error (%s)", res.Status)		return createFileResult()	} 	fs.Debugf(nil, "Assuming path is a file as HEAD response is success (%s)", res.Status)	return createFileResult()} // Make the http connection with optfunc (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err error) {	if len(opt.Headers)%2 != 0 {		return false, errors.New("odd number of headers supplied")	} 	if !strings.HasSuffix(opt.Endpoint, "/") {		opt.Endpoint += "/"	} 	// Parse the endpoint and stick the root onto it	base, err := url.Parse(opt.Endpoint)	if err != nil {		return false, err	}	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))	if err != nil {		return false, err	} 	client := fshttp.NewClient(ctx) 	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)	fs.Debugf(nil, "Root: %s", endpoint)	u, err = url.Parse(endpoint)	if err != nil {		return false, err	} 	// Update f with the new parameters	f.httpClient = client	f.endpoint = u	f.endpointURL = u.String() 	if isFile {		// Correct root if definitely pointing to a file		f.root = path.Dir(f.root)		if f.root == "." || f.root == "/" {			f.root = ""		}	}	return isFile, nil} // NewFs creates a new Fs object from the name and root. It connects to// the host specified in the config file.func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {	// Parse config into Options struct	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	} 	ci := fs.GetConfig(ctx)	f := &Fs{		name: name,		root: root,		opt:  *opt,		ci:   ci,	}	f.features = (&fs.Features{		CanHaveEmptyDirectories: true,	}).Fill(ctx, f) 	// Make the http connection	isFile, err := f.httpConnection(ctx, opt)	if err != nil {		return nil, err	} 	if isFile {		// return an error with an fs which points to the parent		return f, fs.ErrorIsFile	} 	if !strings.HasSuffix(f.endpointURL, "/") {		return nil, errors.New("internal error: url doesn't end with /")	} 	return f, nil} // Name returns the configured name of the file systemfunc (f *Fs) Name() string {	return f.name} // Root returns the root for the filesystemfunc (f *Fs) Root() string {	return f.root} // String returns the URL for the filesystemfunc (f *Fs) String() string {	return f.endpointURL} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // Precision is the remote http file system's modtime precision, which we have no way of knowing. We estimate at 1sfunc (f *Fs) Precision() time.Duration {	return time.Second} // NewObject creates a new remote http file objectfunc (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	err := o.head(ctx)	if err != nil {		return nil, err	}	return o, nil} // Join's the remote onto the base URLfunc (f *Fs) url(remote string) string {	trimmedRemote := strings.TrimLeft(remote, "/") // remove leading "/" since we always have it in f.endpointURL	if f.opt.NoEscape {		// Directly concatenate without escaping, no_escape behavior		return f.endpointURL + trimmedRemote	}	// Default behavior	return f.endpointURL + rest.URLPathEscape(trimmedRemote)} // Errors returned by parseNamevar (	errURLJoinFailed     = errors.New("URLJoin failed")	errFoundQuestionMark = errors.New("found ? in URL")	errHostMismatch      = errors.New("host mismatch")	errSchemeMismatch    = errors.New("scheme mismatch")	errNotUnderRoot      = errors.New("not under root")	errNameIsEmpty       = errors.New("name is empty")	errNameContainsSlash = errors.New("name contains /")) // parseName turns a name as found in the page into a remote path or returns an errorfunc parseName(base *url.URL, name string) (string, error) {	// make URL absolute	u, err := rest.URLJoin(base, name)	if err != nil {		return "", errURLJoinFailed	}	// check it doesn't have URL parameters	uStr := u.String()	if strings.Contains(uStr, "?") {		return "", errFoundQuestionMark	}	// check that this is going back to the same host and scheme	if base.Host != u.Host {		return "", errHostMismatch	}	if base.Scheme != u.Scheme {		return "", errSchemeMismatch	}	// check has path prefix	if !strings.HasPrefix(u.Path, base.Path) {		return "", errNotUnderRoot	}	// calculate the name relative to the base	name = u.Path[len(base.Path):]	// mustn't be empty	if name == "" {		return "", errNameIsEmpty	}	// mustn't contain a / - we are looking for a single level directory	slash := strings.Index(name, "/")	if slash >= 0 && slash != len(name)-1 {		return "", errNameContainsSlash	}	return name, nil} // Parse turns HTML for a directory into names// base should be the base URL to resolve any relative names fromfunc parse(base *url.URL, in io.Reader) (names []string, err error) {	doc, err := html.Parse(in)	if err != nil {		return nil, err	}	var (		walk func(*html.Node)		seen = make(map[string]struct{})	)	walk = func(n *html.Node) {		if n.Type == html.ElementNode && n.Data == "a" {			for _, a := range n.Attr {				if a.Key == "href" {					name, err := parseName(base, a.Val)					if err == nil {						if _, found := seen[name]; !found {							names = append(names, name)							seen[name] = struct{}{}						}					}					break				}			}		}		for c := n.FirstChild; c != nil; c = c.NextSibling {			walk(c)		}	}	walk(doc)	return names, nil} // Adds the configured headers to the request if anyfunc addHeaders(req *http.Request, opt *Options) {	for i := 0; i < len(opt.Headers); i += 2 {		key := opt.Headers[i]		value := opt.Headers[i+1]		req.Header.Add(key, value)	}} // Adds the configured headers to the request if anyfunc (f *Fs) addHeaders(req *http.Request) {	addHeaders(req, &f.opt)} // Read the directory passed infunc (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error) {	URL := f.url(dir)	u, err := url.Parse(URL)	if err != nil {		return nil, fmt.Errorf("failed to readDir: %w", err)	}	if !strings.HasSuffix(URL, "/") {		return nil, fmt.Errorf("internal error: readDir URL %q didn't end in /", URL)	}	// Do the request	req, err := http.NewRequestWithContext(ctx, "GET", URL, nil)	if err != nil {		return nil, fmt.Errorf("readDir failed: %w", err)	}	f.addHeaders(req)	res, err := f.httpClient.Do(req)	if err == nil {		defer fs.CheckClose(res.Body, &err)		if res.StatusCode == http.StatusNotFound {			return nil, fs.ErrorDirNotFound		}	}	err = statusError(res, err)	if err != nil {		return nil, fmt.Errorf("failed to readDir: %w", err)	} 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]	switch contentType {	case "text/html":		names, err = parse(u, res.Body)		if err != nil {			return nil, fmt.Errorf("readDir: %w", err)		}	default:		return nil, fmt.Errorf("can't parse content type %q", contentType)	}	return names, nil} // List the objects and directories in dir into entries.  The// entries can be returned in any order but should be for a// complete directory.//// dir should be "" to list the root, and should not have// trailing slashes.//// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {	if !strings.HasSuffix(dir, "/") && dir != "" {		dir += "/"	}	names, err := f.readDir(ctx, dir)

Lines 1–500 of 781. Load more