fs/backend_config.go - master - githubmirror/rclone

fs/backend_config.go
615 строк19 KB
Nick Craig-Wood
fs: fix passwords and tokens appearing in the debug log during rclone config
16 июл 2026, 18:11
16 июл 2026, 18:118b812ff
// Structures and utilities for backend config
//
//

package fs

import (
	"context"
	"errors"
	"fmt"
	"slices"
	"strconv"
	"strings"
	"sync"

	"github.com/rclone/rclone/fs/config/configmap"
)

const (
	// ConfigToken is the key used to store the token under
	ConfigToken = "token"

	// ConfigKeyEphemeralPrefix marks config keys which shouldn't be stored in the config file
	ConfigKeyEphemeralPrefix = "config_"
)

// ConfigOAuth should be called to do the OAuth
//
// set in lib/oauthutil to avoid a circular import
var ConfigOAuth func(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (*ConfigOut, error)

// ConfigIn is passed to the Config function for an Fs
//
// The interactive config system for backends is state based. This is
// so that different frontends to the config can be attached, eg over
// the API or web page.
//
// Each call to the config system supplies ConfigIn which tells the
// system what to do. Each will return a ConfigOut which gives a
// question to ask the user and a state to return to. There is one
// special question which allows the backends to do OAuth.
//
// The ConfigIn contains a State which the backend should act upon and
// a Result from the previous question to the user.
//
// If ConfigOut is nil or ConfigOut.State == "" then the process is
// deemed to have finished. If there is no Option in ConfigOut then
// the next state will be called immediately. This is wrapped in
// ConfigGoto and ConfigResult.
//
// Backends should keep no state in memory - if they need to persist
// things between calls it should be persisted in the config file.
// Things can also be persisted in the state using the StatePush and
// StatePop utilities here.
//
// The utilities here are convenience methods for different kinds of
// questions and responses.
//
// Where the questions ask for a name then this should start with
// "config_" to show it is an ephemeral config input rather than the
// actual value stored in the config file. Names beginning with
// "config_fs_" are reserved for internal use.
//
// State names starting with "*" are reserved for internal use.
//
// Note that in the bin directory there is a python program called
// "config.py" which shows how this interface should be used.
type ConfigIn struct {
	State  string // State to run
	Result string // Result from previous Option
}

// ConfigOut is returned from Config function for an Fs
//
// State is the state for the next call to Config
// OAuth is a special value set by oauthutil.ConfigOAuth
// Error is displayed to the user before asking a question
// Result is passed to the next call to Config if Option/OAuth isn't set
type ConfigOut struct {
	State  string  // State to jump to after this
	Option *Option // Option to query user about
	OAuth  any     `json:"-"` // Do OAuth if set
	Error  string  // error to be displayed to the user
	Result string  // if Option/OAuth not set then this is passed to the next state
}

// ConfigInputOptional asks the user for a string which may be empty
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
func ConfigInputOptional(state string, name string, help string) (*ConfigOut, error) {
	return &ConfigOut{
		State: state,
		Option: &Option{
			Name:    name,
			Help:    help,
			Default: "",
		},
	}, nil
}

// ConfigInput asks the user for a non-empty string
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
func ConfigInput(state string, name string, help string) (*ConfigOut, error) {
	out, _ := ConfigInputOptional(state, name, help)
	out.Option.Required = true
	return out, nil
}

// ConfigPassword asks the user for a password
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
func ConfigPassword(state string, name string, help string) (*ConfigOut, error) {
	out, _ := ConfigInputOptional(state, name, help)
	out.Option.IsPassword = true
	return out, nil
}

// ConfigGoto goes to the next state with empty Result
//
// state should be the next state required
func ConfigGoto(state string) (*ConfigOut, error) {
	return &ConfigOut{
		State: state,
	}, nil
}

// ConfigResult goes to the next state with result given
//
// state should be the next state required
// result should be the result for the next state
func ConfigResult(state, result string) (*ConfigOut, error) {
	return &ConfigOut{
		State:  state,
		Result: result,
	}, nil
}

// ConfigError shows the error to the user and goes to the state passed in
//
// state should be the next state required
// Error should be the error shown to the user
func ConfigError(state string, Error string) (*ConfigOut, error) {
	return &ConfigOut{
		State: state,
		Error: Error,
	}, nil
}

// ConfigConfirm returns a ConfigOut structure which asks a Yes/No question
//
// state should be the next state required
// Default should be the default state
// name is the config name for this item
// help should be the help shown to the user
func ConfigConfirm(state string, Default bool, name string, help string) (*ConfigOut, error) {
	return &ConfigOut{
		State: state,
		Option: &Option{
			Name:    name,
			Help:    help,
			Default: Default,
			Examples: []OptionExample{{
				Value: "true",
				Help:  "Yes",
			}, {
				Value: "false",
				Help:  "No",
			}},
			Exclusive: true,
		},
	}, nil
}

// ConfigChooseExclusiveFixed returns a ConfigOut structure which has a list of
// items to choose from.
//
// Possible items must be supplied as a fixed list.
//
// User is required to supply a value, and is restricted to the specified list,
// i.e. free text input is not allowed.
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
// items should be the items in the list
//
// It chooses the first item to be the default.
// If there are no items then it will return an error.
// If there is only one item it will short cut to the next state.
func ConfigChooseExclusiveFixed(state string, name string, help string, items []OptionExample) (*ConfigOut, error) {
	if len(items) == 0 {
		return nil, fmt.Errorf("no items found in: %s", help)
	}
	choose := &ConfigOut{
		State: state,
		Option: &Option{
			Name:      name,
			Help:      help,
			Examples:  items,
			Exclusive: true,
		},
	}
	choose.Option.Default = choose.Option.Examples[0].Value
	if len(items) == 1 {
		// short circuit asking the question if only one entry
		choose.Result = choose.Option.Examples[0].Value
		choose.Option = nil
	}
	return choose, nil
}

// ConfigChooseExclusive returns a ConfigOut structure which has a list of
// items to choose from.
//
// Possible items are retrieved from a supplied function.
//
// User is required to supply a value, and is restricted to the specified list,
// i.e. free text input is not allowed.
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
// n should be the number of items in the list
// getItem should return the items (value, help)
//
// It chooses the first item to be the default.
// If there are no items then it will return an error.
// If there is only one item it will short cut to the next state.
func ConfigChooseExclusive(state string, name string, help string, n int, getItem func(i int) (itemValue string, itemHelp string)) (*ConfigOut, error) {
	items := make(OptionExamples, n)
	for i := range items {
		items[i].Value, items[i].Help = getItem(i)
	}
	return ConfigChooseExclusiveFixed(state, name, help, items)
}

// ConfigChooseFixed returns a ConfigOut structure which has a list of
// suggested items.
//
// Suggested items must be supplied as a fixed list.
//
// User is required to supply a value, but is not restricted to the specified
// list, i.e. free text input is accepted.
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
// items should be the items in the list
//
// It chooses the first item to be the default.
func ConfigChooseFixed(state string, name string, help string, items []OptionExample) (*ConfigOut, error) {
	choose := &ConfigOut{
		State: state,
		Option: &Option{
			Name:     name,
			Help:     help,
			Examples: items,
			Required: true,
		},
	}
	if len(choose.Option.Examples) > 0 {
		choose.Option.Default = choose.Option.Examples[0].Value
	}
	return choose, nil
}

// ConfigChoose returns a ConfigOut structure which has a list of suggested
// items.
//
// Suggested items are retrieved from a supplied function.
//
// User is required to supply a value, but is not restricted to the specified
// list, i.e. free text input is accepted.
//
// state should be the next state required
// name is the config name for this item
// help should be the help shown to the user
// n should be the number of items in the list
// getItem should return the items (value, help)
//
// It chooses the first item to be the default.
func ConfigChoose(state string, name string, help string, n int, getItem func(i int) (itemValue string, itemHelp string)) (*ConfigOut, error) {
	items := make(OptionExamples, n)
	for i := range items {
		items[i].Value, items[i].Help = getItem(i)
	}
	return ConfigChooseFixed(state, name, help, items)
}

// StatePush pushes a new values onto the front of the config string
func StatePush(state string, values ...string) string {
	for i := range values {
		values[i] = strings.ReplaceAll(values[i], ",", "，") // replace comma with unicode wide version
	}
	if state != "" {
		values = append(values[:len(values):len(values)], state)
	}
	return strings.Join(values, ",")
}

type configOAuthKeyType struct{}

// OAuth key for config
var configOAuthKey = configOAuthKeyType{}

// ConfigOAuthOnly marks the ctx so that the Config will stop after
// finding an OAuth
func ConfigOAuthOnly(ctx context.Context) context.Context {
	return context.WithValue(ctx, configOAuthKey, struct{}{})
}

// Return true if ctx is marked as ConfigOAuthOnly
func isConfigOAuthOnly(ctx context.Context) bool {
	return ctx.Value(configOAuthKey) != nil
}

// StatePop pops a state from the front of the config string
// It returns the new state and the value popped
func StatePop(state string) (newState string, value string) {
	comma := strings.IndexRune(state, ',')
	if comma < 0 {
		return "", state
	}
	value, newState = state[:comma], state[comma+1:]
	value = strings.ReplaceAll(value, "，", ",") // replace unicode wide comma with comma
	return newState, value
}

// BackendConfig calls the config for the backend in ri
//
// It wraps any OAuth transactions as necessary so only straight
// forward config questions are emitted
func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, choices configmap.Getter, in ConfigIn) (out *ConfigOut, err error) {
	for {
		out, err = backendConfigStep(ctx, name, m, ri, choices, in)
		if err != nil {
			break
		}
		if out == nil || out.State == "" {
			// finished
			break
		}
		if out.Option != nil {
			// question to ask user
			break
		}
		if out.Error != "" {
			// error to show user
			break
		}
		// non terminal state, but no question to ask or error to show - loop here
		in = ConfigIn{
			State:  out.State,
			Result: out.Result,
		}
	}
	return out, err
}

// ConfigAll should be passed in as the initial state to run the
// entire config
const ConfigAll = "*all"

// Run the config state machine for the normal config
func configAll(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
	if len(ri.Options) == 0 {
		return ConfigGoto("*postconfig")
	}

	// States are encoded
	//
	//     *all-ACTION,NUMBER,ADVANCED
	//
	// Where NUMBER is the current state, ADVANCED is a flag true or false
	// to say whether we are asking about advanced config and
	// ACTION is what the state should be doing next.
	stateParams, state := StatePop(in.State)
	stateParams, stateNumber := StatePop(stateParams)
	_, stateAdvanced := StatePop(stateParams)

	optionNumber := 0
	advanced := stateAdvanced == "true"
	if stateNumber != "" {
		optionNumber, err = strconv.Atoi(stateNumber)
		if err != nil {
			return nil, fmt.Errorf("internal error: bad state number: %w", err)
		}
	}

	// Detect if reached the end of the questions
	if optionNumber == len(ri.Options) {
		if ri.Options.HasAdvanced() {
			return ConfigConfirm("*all-advanced", false, "config_fs_advanced", "Edit advanced config?")
		}
		return ConfigGoto("*postconfig")
	} else if optionNumber < 0 || optionNumber > len(ri.Options) {
		return nil, errors.New("internal error: option out of range")
	}

	// Make the next state
	newState := func(state string, i int, advanced bool) string {
		return StatePush("", state, fmt.Sprint(i), fmt.Sprint(advanced))
	}

	// Find the current option
	option := &ri.Options[optionNumber]

	switch state {
	case "*all":
		// If option is hidden or doesn't match advanced setting then skip it
		if option.Hide&OptionHideConfigurator != 0 || option.Advanced != advanced {
			return ConfigGoto(newState("*all", optionNumber+1, advanced))
		}

		// Skip this question if it isn't the correct provider
		provider, _ := m.Get(ConfigProvider)
		if !MatchProvider(option.Provider, provider) {
			return ConfigGoto(newState("*all", optionNumber+1, advanced))
		}

		out = &ConfigOut{
			State:  newState("*all-set", optionNumber, advanced),
			Option: option,
		}

		// Filter examples by provider if necessary
		if provider != "" && len(option.Examples) > 0 {
			optionCopy := option.Copy()
			optionCopy.Examples = OptionExamples{}
			for _, example := range option.Examples {
				if MatchProvider(example.Provider, provider) {
					optionCopy.Examples = append(optionCopy.Examples, example)
				}
			}
			out.Option = optionCopy
		}

		return out, nil
	case "*all-set":
		// Set the value if not different to current
		// Note this won't set blank values in the config file
		// if the default is blank
		currentValue, _ := m.Get(option.Name)
		if currentValue != in.Result {
			m.Set(option.Name, in.Result)
		}
		// Find the next question
		return ConfigGoto(newState("*all", optionNumber+1, advanced))
	case "*all-advanced":
		// Reply to edit advanced question
		if in.Result == "true" {
			return ConfigGoto(newState("*all", 0, true))
		}
		return ConfigGoto("*postconfig")
	}
	return nil, fmt.Errorf("internal error: bad state %q", state)
}

// redactDumpAuthWarnOnce makes sure the --dump auth warning is only shown once
var redactDumpAuthWarnOnce sync.Once

// RedactValue makes value safe for inclusion in the debug log.
//
// Non-empty values are replaced with "XXX" as they may contain
// secrets such as passwords or tokens, unless --dump auth is in use
// in which case the value is returned quoted.
func RedactValue(ci *ConfigInfo, value string) string {
	if value == "" || ci.Dump&DumpAuth != 0 {
		return fmt.Sprintf("%q", value)
	}
	return "XXX"
}

// RedactOptionValue is like RedactValue except that value is only
// redacted if opt is unknown (nil), a password or sensitive.
func RedactOptionValue(ci *ConfigInfo, opt *Option, value string) string {
	if opt == nil || opt.IsPassword || opt.Sensitive {
		return RedactValue(ci, value)
	}
	return fmt.Sprintf("%q", value)
}

// redactConfigOut renders out for the debug log, redacting any
// values which may contain secrets.
func redactConfigOut(ci *ConfigInfo, out *ConfigOut) string {
	if out == nil {
		return "<nil>"
	}
	var b strings.Builder
	fmt.Fprintf(&b, "{State:%q", out.State)
	if out.Option != nil {
		fmt.Fprintf(&b, " Option:%s=%s", out.Option.Name, RedactOptionValue(ci, out.Option, out.Option.String()))
	}
	if out.OAuth != nil {
		b.WriteString(" OAuth:set")
	}
	if out.Error != "" {
		fmt.Fprintf(&b, " Error:%q", out.Error)
	}
	fmt.Fprintf(&b, " Result:%s}", RedactValue(ci, out.Result))
	return b.String()
}

func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, choices configmap.Getter, in ConfigIn) (out *ConfigOut, err error) {
	ci := GetConfig(ctx)
	if ci.Dump&DumpAuth != 0 {
		redactDumpAuthWarnOnce.Do(func() {
			Logf(nil, "--dump auth is in use - debug output may contain secrets such as passwords and tokens")
		})
	}
	Debugf(name, "config in: state=%q, result=%s", in.State, RedactValue(ci, in.Result))
	defer func() {
		Debugf(name, "config out: out=%s, err=%v", redactConfigOut(ci, out), err)
	}()

	switch {
	case strings.HasPrefix(in.State, ConfigAll):
		// Do all config
		out, err = configAll(ctx, name, m, ri, in)
	case strings.HasPrefix(in.State, "*oauth"):
		// Do internal oauth states
		out, err = ConfigOAuth(ctx, name, m, ri, in)
	case strings.HasPrefix(in.State, "*postconfig"):
		// Do the post config starting from state ""
		in.State = ""
		return backendConfigStep(ctx, name, m, ri, choices, in)
	case strings.HasPrefix(in.State, "*"):
		err = fmt.Errorf("unknown internal state %q", in.State)
	default:
		// Otherwise pass to backend
		if ri.Config == nil {
			return nil, nil
		}
		out, err = ri.Config(ctx, name, m, in)
	}
	if err != nil {
		return nil, err
	}
	switch {
	case out == nil:
	case out.OAuth != nil:
		// If this is an OAuth state the deal with it here
		returnState := out.State
		// If rclone authorize, stop after doing oauth
		if isConfigOAuthOnly(ctx) {
			Debugf(nil, "OAuth only is set - overriding return state")
			returnState = ""
		}
		// Run internal state, saving the input so we can recall the state
		return ConfigGoto(StatePush("", "*oauth", returnState, in.State, in.Result))
	case out.Option != nil:
		if out.Option.Name == "" {
			return nil, errors.New("internal error: no name set in Option")
		}
		// If override value is set in the choices then use that
		if result, ok := choices.Get(out.Option.Name); ok {
			Debugf(nil, "Override value found, choosing value %s for state %q", RedactOptionValue(ci, out.Option, result), out.State)
			return ConfigResult(out.State, result)
		}
		// If AutoConfirm is set, choose the default value
		if ci.AutoConfirm {
			result := fmt.Sprint(out.Option.Default)
			Debugf(nil, "Auto confirm is set, choosing default %s for state %q, override by setting config parameter %q", RedactOptionValue(ci, out.Option, result), out.State, out.Option.Name)
			return ConfigResult(out.State, result)
		}
		// If fs.ConfigEdit is set then make the default value
		// in the config the current value.
		if result, ok := choices.Get(ConfigEdit); ok && result == "true" {
			if value, ok := m.Get(out.Option.Name); ok {
				newOption := out.Option.Copy()
				oldValue := newOption.Value
				err = newOption.Set(value)
				if err != nil {
					Errorf(nil, "Failed to set %q from %s - using default: %v", out.Option.Name, RedactOptionValue(ci, newOption, value), err)
				} else {
					newOption.Default = newOption.Value
					newOption.Value = oldValue
					out.Option = newOption
				}
			}
		}
	}
	return out, nil
}

// MatchProvider returns true if provider matches the providerConfig string.
//
// The providerConfig string can either be a list of providers to
// match, or if it starts with "!" it will be a list of providers not
// to match.
//
// If either providerConfig or provider is blank then it will return true
func MatchProvider(providerConfig, provider string) bool {
	if providerConfig == "" || provider == "" {
		return true
	}
	negate := false
	if strings.HasPrefix(providerConfig, "!") {
		providerConfig = providerConfig[1:]
		negate = true
	}
	providers := strings.Split(providerConfig, ",")
	matched := slices.Contains(providers, provider)
	if negate {
		return !matched
	}
	return matched
}