// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package cmd import (	"bufio"	"bytes"	"context"	"crypto/tls"	"crypto/x509"	"encoding/gob"	"encoding/pem"	"errors"	"fmt"	"io/ioutil"	"math/rand"	"net"	"net/http"	"net/url"	"os"	"path"	"path/filepath"	"runtime"	"sort"	"strconv"	"strings"	"syscall"	"time" 	"github.com/dustin/go-humanize"	fcolor "github.com/fatih/color"	"github.com/go-openapi/loads"	"github.com/inconshreveable/mousetrap"	dns2 "github.com/miekg/dns"	"github.com/minio/cli"	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"	consoleCerts "github.com/minio/console/pkg/certs"	"github.com/minio/console/restapi"	"github.com/minio/console/restapi/operations"	"github.com/minio/kes"	"github.com/minio/madmin-go"	"github.com/minio/minio-go/v7"	"github.com/minio/minio-go/v7/pkg/credentials"	"github.com/minio/minio-go/v7/pkg/set"	"github.com/minio/minio/internal/auth"	"github.com/minio/minio/internal/color"	"github.com/minio/minio/internal/config"	"github.com/minio/minio/internal/handlers"	"github.com/minio/minio/internal/kms"	"github.com/minio/minio/internal/logger"	"github.com/minio/pkg/certs"	"github.com/minio/pkg/console"	"github.com/minio/pkg/ellipses"	"github.com/minio/pkg/env"	xnet "github.com/minio/pkg/net"	"github.com/rs/dnscache") // serverDebugLog will enable debug printingvar serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn var (	shardDiskTimeDelta     time.Duration	defaultAWSCredProvider []credentials.Provider) func init() {	if runtime.GOOS == "windows" {		if mousetrap.StartedByExplorer() {			fmt.Printf("Don't double-click %s\n", os.Args[0])			fmt.Println("You need to open cmd.exe/PowerShell and run it from the command line")			fmt.Println("Refer to the docs here on how to run it as a Windows Service https://github.com/minio/minio-service/tree/master/windows")			fmt.Println("Press the Enter Key to Exit")			fmt.Scanln()			os.Exit(1)		}	} 	rand.Seed(time.Now().UTC().UnixNano()) 	logger.Init(GOPATH, GOROOT)	logger.RegisterError(config.FmtError) 	initGlobalContext() 	options := dnscache.ResolverRefreshOptions{		ClearUnused:      true,		PersistOnFailure: false,	} 	t, _ := minioVersionToReleaseTime(Version)	if !t.IsZero() {		globalVersionUnix = uint64(t.Unix())	} 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != "" 	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() 	// Call to refresh will refresh names in cache. If you pass true, it will also	// remove cached names not looked up since the last call to Refresh. It is a good idea	// to call this method on a regular interval.	go func() {		var t *time.Ticker		if containers {			t = time.NewTicker(1 * time.Minute)		} else {			t = time.NewTicker(10 * time.Minute)		}		defer t.Stop()		for {			select {			case <-t.C:				globalDNSCache.RefreshWithOptions(options)			case <-GlobalContext.Done():				return			}		}	}() 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{		PassHost:     true,		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),		Logger: func(err error) {			if err != nil && !errors.Is(err, context.Canceled) {				logger.LogIf(GlobalContext, err)			}		},	}) 	console.SetColor("Debug", fcolor.New()) 	gob.Register(StorageErr(""))	gob.Register(madmin.TimeInfo{})	gob.Register(map[string]interface{}{}) 	defaultAWSCredProvider = []credentials.Provider{		&credentials.IAM{			Client: &http.Client{				Transport: NewGatewayHTTPTransport(),			},		},	} 	var err error	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))	if err != nil {		shardDiskTimeDelta = 1 * time.Minute	} 	// All minio-go API operations shall be performed only once,	// another way to look at this is we are turning off retries.	minio.MaxRetry = 1} const consolePrefix = "CONSOLE_" func minioConfigToConsoleFeatures() {	os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID)	os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID)	if globalMinioEndpoint != "" {		os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)	} else {		os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])	}	if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {		os.Setenv("CONSOLE_LOG_QUERY_URL", value)		if value := env.Get("MINIO_LOG_QUERY_AUTH_TOKEN", ""); value != "" {			os.Setenv("CONSOLE_LOG_QUERY_AUTH_TOKEN", value)		}	}	// pass the console subpath configuration	if value := env.Get(config.EnvMinIOBrowserRedirectURL, ""); value != "" {		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))		if subPath != SlashSeparator {			os.Setenv("CONSOLE_SUBPATH", subPath)		}	}	// Enable if prometheus URL is set.	if value := env.Get("MINIO_PROMETHEUS_URL", ""); value != "" {		os.Setenv("CONSOLE_PROMETHEUS_URL", value)		if value := env.Get("MINIO_PROMETHEUS_JOB_ID", "minio-job"); value != "" {			os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value)			// Support additional labels for more granular filtering.			if value := env.Get("MINIO_PROMETHEUS_EXTRA_LABELS", ""); value != "" {				os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value)			}		}	}	// Enable if LDAP is enabled.	if globalLDAPConfig.Enabled {		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)	}	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))	if globalSubnetConfig.License != "" {		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)	}	if globalSubnetConfig.APIKey != "" {		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)	}	if globalSubnetConfig.ProxyURL != nil {		os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())	}} func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))	for name, cfg := range globalOpenIDConfig.ProviderCfgs {		callback := getConsoleEndpoints()[0] + "/oauth_callback"		if cfg.RedirectURI != "" {			callback = cfg.RedirectURI		}		m[name] = consoleoauth2.ProviderConfig{			URL:                     cfg.URL.String(),			DisplayName:             cfg.DisplayName,			ClientID:                cfg.ClientID,			ClientSecret:            cfg.ClientSecret,			HMACSalt:                globalDeploymentID,			HMACPassphrase:          cfg.ClientID,			Scopes:                  strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","),			Userinfo:                cfg.ClaimUserinfo,			RedirectCallbackDynamic: cfg.RedirectURIDynamic,			RedirectCallback:        callback,		}	}	return m} func initConsoleServer() (*restapi.Server, error) {	// unset all console_ environment variables.	for _, cenv := range env.List(consolePrefix) {		os.Unsetenv(cenv)	} 	// enable all console environment variables	minioConfigToConsoleFeatures() 	// set certs dir to minio directory	consoleCerts.GlobalCertsDir = &consoleCerts.ConfigDir{		Path: globalCertsDir.Get(),	}	consoleCerts.GlobalCertsCADir = &consoleCerts.ConfigDir{		Path: globalCertsCADir.Get(),	} 	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)	if err != nil {		return nil, err	} 	api := operations.NewConsoleAPI(swaggerSpec) 	if !serverDebugLog {		// Disable console logging if server debug log is not enabled		noLog := func(string, ...interface{}) {} 		restapi.LogInfo = noLog		restapi.LogError = noLog		api.Logger = noLog	} 	// Pass in console application config. This needs to happen before the	// ConfigureAPI() call.	restapi.GlobalMinIOConfig = restapi.MinIOConfig{		OpenIDProviders: buildOpenIDConsoleConfig(),	} 	server := restapi.NewServer(api)	// register all APIs	server.ConfigureAPI() 	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts 	consolePort, _ := strconv.Atoi(globalMinioConsolePort) 	server.Host = globalMinioConsoleHost	server.Port = consolePort	restapi.Port = globalMinioConsolePort	restapi.Hostname = globalMinioConsoleHost 	if globalIsTLS {		// If TLS certificates are provided enforce the HTTPS.		server.EnabledListeners = []string{"https"}		server.TLSPort = consolePort		// Need to store tls-port, tls-host un config variables so secure.middleware can read from there		restapi.TLSPort = globalMinioConsolePort		restapi.Hostname = globalMinioConsoleHost	} 	return server, nil} func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {	if strings.HasPrefix(name, "gateway") {		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")			logger.Fatal(uiErr, "Unable to start gateway with SSE")		}	} 	globalCompressConfigMu.Lock()	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {		logger.Fatal(errInvalidArgument,			"Compression support is requested but '%s' does not support compression", name)	}	globalCompressConfigMu.Unlock()} // Check for updates and print a notification messagefunc checkUpdate(mode string) {	updateURL := minioReleaseInfoURL	if runtime.GOOS == globalWindowsOSName {		updateURL = minioReleaseWindowsInfoURL	} 	u, err := url.Parse(updateURL)	if err != nil {		return	} 	// Its OK to ignore any errors during doUpdate() here.	crTime, err := GetCurrentReleaseTime()	if err != nil {		return	} 	_, lrTime, err := getLatestReleaseTime(u, 2*time.Second, mode)	if err != nil {		return	} 	var older time.Duration	var downloadURL string	if lrTime.After(crTime) {		older = lrTime.Sub(crTime)		downloadURL = getDownloadURL(releaseTimeToReleaseTag(lrTime))	} 	updateMsg := prepareUpdateMessage(downloadURL, older)	if updateMsg == "" {		return	} 	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))} func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {	var dir string	var dirSet bool 	switch {	case ctx.IsSet(option):		dir = ctx.String(option)		dirSet = true	case ctx.GlobalIsSet(option):		dir = ctx.GlobalString(option)		dirSet = true		// cli package does not expose parent's option option.  Below code is workaround.		if dir == "" || dir == getDefaultDir() {			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.			if ctx.Parent().GlobalIsSet(option) {				dir = ctx.Parent().GlobalString(option)				dirSet = true			}		}	default:		// Neither local nor global option is provided.  In this case, try to use		// default directory.		dir = getDefaultDir()		if dir == "" {			logger.FatalIf(errInvalidArgument, "%s option must be provided", option)		}	} 	if dir == "" {		logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)	} 	// Disallow relative paths, figure out absolute paths.	dirAbs, err := filepath.Abs(dir)	logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir) 	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir) 	return &ConfigDir{path: dirAbs}, dirSet} func handleCommonCmdArgs(ctx *cli.Context) {	// Get "json" flag from command line argument and	// enable json and quite modes if json flag is turned on.	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")	if globalCLIContext.JSON {		logger.EnableJSON()	} 	// Get quiet flag from command line argument.	globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")	if globalCLIContext.Quiet {		logger.EnableQuiet()	} 	// Get anonymous flag from command line argument.	globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")	if globalCLIContext.Anonymous {		logger.EnableAnonymous()	} 	// Fetch address option	addr := ctx.GlobalString("address")	if addr == "" || addr == ":"+GlobalMinioDefaultPort {		addr = ctx.String("address")	} 	// Fetch console address option	consoleAddr := ctx.GlobalString("console-address")	if consoleAddr == "" {		consoleAddr = ctx.String("console-address")	} 	if consoleAddr == "" {		p, err := xnet.GetFreePort()		if err != nil {			logger.FatalIf(err, "Unable to get free port for console on the host")		}		consoleAddr = net.JoinHostPort("", p.String())	} 	if _, _, err := net.SplitHostPort(consoleAddr); err != nil {		logger.FatalIf(err, "Unable to start listening on console port")	} 	if consoleAddr == addr {		logger.FatalIf(errors.New("--console-address cannot be same as --address"), "Unable to start the server")	} 	globalMinioHost, globalMinioPort = mustSplitHostPort(addr)	globalMinioConsoleHost, globalMinioConsolePort = mustSplitHostPort(consoleAddr) 	if globalMinioPort == globalMinioConsolePort {		logger.FatalIf(errors.New("--console-address port cannot be same as --address port"), "Unable to start the server")	} 	globalMinioAddr = addr 	// Check "no-compat" flag from command line argument.	globalCLIContext.StrictS3Compat = true	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {		globalCLIContext.StrictS3Compat = false	} 	// Set all config, certs and CAs directories.	var configSet, certsSet bool	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get) 	// Remove this code when we deprecate and remove config-dir.	// This code is to make sure we inherit from the config-dir	// option if certs-dir is not provided.	if !certsSet && configSet {		globalCertsDir = &ConfigDir{path: filepath.Join(globalConfigDir.Get(), certsDir)}	} 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)} 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())} type envKV struct {	Key   string	Value string	Skip  bool} func (e envKV) String() string {	if e.Skip {		return ""	}	return fmt.Sprintf("%s=%s", e.Key, e.Value)} func parsEnvEntry(envEntry string) (envKV, error) {	envEntry = strings.TrimSpace(envEntry)	if envEntry == "" {		// Skip all empty lines

Lines 1–500 of 1008. Load more