// ParseServerTLS tries to parse out valid server TLS options from a Viper.
// The cert/key files are relative to the config file used to populate the instance
// of viper.
func ParseServerTLS(configuration *viper.Viper, tlsRequired bool) (*tls.Config, error) {
// unmarshalling into objects does not seem to pick up env vars
tlsOpts := tlsconfig.Options{
CertFile: GetPathRelativeToConfig(configuration, "server.tls_cert_file"),
KeyFile: GetPathRelativeToConfig(configuration, "server.tls_key_file"),
CAFile: GetPathRelativeToConfig(configuration, "server.client_ca_file"),
}
if tlsOpts.CAFile != "" {
tlsOpts.ClientAuth = tls.RequireAndVerifyClientCert
}
if !tlsRequired {
cert, key, ca := tlsOpts.CertFile, tlsOpts.KeyFile, tlsOpts.CAFile
if cert == "" && key == "" && ca == "" {
return nil, nil
}
if (cert == "" && key != "") || (cert != "" && key == "") || (cert == "" && key == "" && ca != "") {
return nil, fmt.Errorf(
"either include both a cert and key file, or no TLS information at all to disable TLS")
}
}
return tlsconfig.Server(tlsOpts)
}
// the config can provide all the TLS information necessary - the root ca file,
// the tls client files - they are all relative to the directory of the config
// file, and not the cwd, or absolute paths
func TestConfigFileTLSCanBeRelativeToConfigOrAbsolute(t *testing.T) {
// Set up server that with a self signed cert
var err error
// add a handler for getting the root
m := &recordingMetaStore{MemStorage: *storage.NewMemStorage()}
s := httptest.NewUnstartedServer(setupServerHandler(m))
s.TLS, err = tlsconfig.Server(tlsconfig.Options{
CertFile: "../../fixtures/notary-server.crt",
KeyFile: "../../fixtures/notary-server.key",
CAFile: "../../fixtures/root-ca.crt",
ClientAuth: tls.RequireAndVerifyClientCert,
})
require.NoError(t, err)
s.StartTLS()
defer s.Close()
tempDir, err := ioutil.TempDir("", "config-test")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
configFile, err := os.Create(filepath.Join(tempDir, "config.json"))
require.NoError(t, err)
fmt.Fprintf(configFile, `{
"remote_server": {
"url": "%s",
"root_ca": "root-ca.crt",
"tls_client_cert": "%s",
"tls_client_key": "notary-server.key"
}
}`, s.URL, filepath.Join(tempDir, "notary-server.crt"))
configFile.Close()
// copy the certs to be relative to the config directory
for _, fname := range []string{"notary-server.crt", "notary-server.key", "root-ca.crt"} {
content, err := ioutil.ReadFile(filepath.Join("../../fixtures", fname))
require.NoError(t, err)
require.NoError(t, ioutil.WriteFile(filepath.Join(tempDir, fname), content, 0766))
}
// set a config file, so it doesn't check ~/.notary/config.json by default,
// and execute a random command so that the flags are parsed
cmd := NewNotaryCommand()
cmd.SetArgs([]string{"-c", configFile.Name(), "-d", tempDir, "list", "repo"})
cmd.SetOutput(new(bytes.Buffer)) // eat the output
err = cmd.Execute()
require.Error(t, err, "there was no repository, so list should have failed")
require.NotContains(t, err.Error(), "TLS", "there was no TLS error though!")
// validate that we actually managed to connect and attempted to download the root though
require.Len(t, m.gotten, 1)
require.Equal(t, m.gotten[0], "repo.root")
}
// Whatever TLS config is in the config file can be overridden by the command line
// TLS flags, which are relative to the CWD (not the config) or absolute
func TestConfigFileOverridenByCmdLineFlags(t *testing.T) {
// Set up server that with a self signed cert
var err error
// add a handler for getting the root
m := &recordingMetaStore{MemStorage: *storage.NewMemStorage()}
s := httptest.NewUnstartedServer(setupServerHandler(m))
s.TLS, err = tlsconfig.Server(tlsconfig.Options{
CertFile: "../../fixtures/notary-server.crt",
KeyFile: "../../fixtures/notary-server.key",
CAFile: "../../fixtures/root-ca.crt",
ClientAuth: tls.RequireAndVerifyClientCert,
})
require.NoError(t, err)
s.StartTLS()
defer s.Close()
tempDir := tempDirWithConfig(t, fmt.Sprintf(`{
"remote_server": {
"url": "%s",
"root_ca": "nope",
"tls_client_cert": "nope",
"tls_client_key": "nope"
}
}`, s.URL))
defer os.RemoveAll(tempDir)
configFile := filepath.Join(tempDir, "config.json")
// set a config file, so it doesn't check ~/.notary/config.json by default,
// and execute a random command so that the flags are parsed
cwd, err := os.Getwd()
require.NoError(t, err)
cmd := NewNotaryCommand()
cmd.SetArgs([]string{
"-c", configFile, "-d", tempDir, "list", "repo",
"--tlscacert", "../../fixtures/root-ca.crt",
"--tlscert", filepath.Clean(filepath.Join(cwd, "../../fixtures/notary-server.crt")),
"--tlskey", "../../fixtures/notary-server.key"})
cmd.SetOutput(new(bytes.Buffer)) // eat the output
err = cmd.Execute()
require.Error(t, err, "there was no repository, so list should have failed")
require.NotContains(t, err.Error(), "TLS", "there was no TLS error though!")
// validate that we actually managed to connect and attempted to download the root though
require.Len(t, m.gotten, 1)
require.Equal(t, m.gotten[0], "repo.root")
}
// the config can provide all the TLS information necessary - the root ca file,
// the tls client files - they are all relative to the directory of the config
// file, and not the cwd
func TestConfigFileTLSCannotBeRelativeToCWD(t *testing.T) {
// Set up server that with a self signed cert
var err error
// add a handler for getting the root
m := &recordingMetaStore{MemStorage: *storage.NewMemStorage()}
s := httptest.NewUnstartedServer(setupServerHandler(m))
s.TLS, err = tlsconfig.Server(tlsconfig.Options{
CertFile: "../../fixtures/notary-server.crt",
KeyFile: "../../fixtures/notary-server.key",
CAFile: "../../fixtures/root-ca.crt",
ClientAuth: tls.RequireAndVerifyClientCert,
})
require.NoError(t, err)
s.StartTLS()
defer s.Close()
// test that a config file with certs that are relative to the cwd fail
tempDir := tempDirWithConfig(t, fmt.Sprintf(`{
"remote_server": {
"url": "%s",
"root_ca": "../../fixtures/root-ca.crt",
"tls_client_cert": "../../fixtures/notary-server.crt",
"tls_client_key": "../../fixtures/notary-server.key"
}
}`, s.URL))
defer os.RemoveAll(tempDir)
configFile := filepath.Join(tempDir, "config.json")
// set a config file, so it doesn't check ~/.notary/config.json by default,
// and execute a random command so that the flags are parsed
cmd := NewNotaryCommand()
cmd.SetArgs([]string{"-c", configFile, "-d", tempDir, "list", "repo"})
cmd.SetOutput(new(bytes.Buffer)) // eat the output
err = cmd.Execute()
require.Error(t, err, "expected a failure due to TLS")
require.Contains(t, err.Error(), "TLS", "should have been a TLS error")
// validate that we failed to connect and attempt any downloads at all
require.Len(t, m.gotten, 0)
}
func (cli *DaemonCli) start(opts daemonOptions) (err error) {
stopc := make(chan bool)
defer close(stopc)
// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf
opts.common.SetDefaultOptions(opts.flags)
if opts.common.TrustKey == "" {
opts.common.TrustKey = filepath.Join(
getDaemonConfDir(),
cliflags.DefaultTrustKeyFile)
}
if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
return err
}
cli.configFile = &opts.configFile
cli.flags = opts.flags
if cli.Config.Debug {
utils.EnableDebug()
}
if utils.ExperimentalBuild() {
logrus.Warn("Running experimental build")
}
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: jsonlog.RFC3339NanoFixed,
DisableColors: cli.Config.RawLogs,
})
if err := setDefaultUmask(); err != nil {
return fmt.Errorf("Failed to set umask: %v", err)
}
if len(cli.LogConfig.Config) > 0 {
if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
return fmt.Errorf("Failed to set log opts: %v", err)
}
}
if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}
defer func() {
if err := pf.Remove(); err != nil {
logrus.Error(err)
}
}()
}
serverConfig := &apiserver.Config{
Logging: true,
SocketGroup: cli.Config.SocketGroup,
Version: dockerversion.Version,
EnableCors: cli.Config.EnableCors,
CorsHeaders: cli.Config.CorsHeaders,
}
if cli.Config.TLS {
tlsOptions := tlsconfig.Options{
CAFile: cli.Config.CommonTLSOptions.CAFile,
CertFile: cli.Config.CommonTLSOptions.CertFile,
KeyFile: cli.Config.CommonTLSOptions.KeyFile,
}
if cli.Config.TLSVerify {
// server requires and verifies client's certificate
tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(tlsOptions)
if err != nil {
return err
}
serverConfig.TLSConfig = tlsConfig
}
if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}
api := apiserver.New(serverConfig)
cli.api = api
for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
}
protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
}
proto := protoAddrParts[0]
addr := protoAddrParts[1]
// It's a bad idea to bind to TCP without tlsverify.
if proto == "tcp" && (serverConfig.TLSConfig == nil || serverConfig.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert) {
logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
}
ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
return err
}
ls = wrapListeners(proto, ls)
// If we're binding to a TCP port, make sure that a container doesn't try to use it.
if proto == "tcp" {
if err := allocateDaemonPort(addr); err != nil {
return err
}
}
logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
api.Accept(addr, ls...)
}
if err := migrateKey(); err != nil {
return err
}
// FIXME: why is this down here instead of with the other TrustKey logic above?
cli.TrustKeyPath = opts.common.TrustKey
registryService := registry.NewService(cli.Config.ServiceOptions)
containerdRemote, err := libcontainerd.New(cli.getLibcontainerdRoot(), cli.getPlatformRemoteOptions()...)
if err != nil {
return err
}
signal.Trap(func() {
cli.stop()
<-stopc // wait for daemonCli.start() to return
})
d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote)
if err != nil {
return fmt.Errorf("Error starting daemon: %v", err)
}
name, _ := os.Hostname()
c, err := cluster.New(cluster.Config{
Root: cli.Config.Root,
Name: name,
Backend: d,
NetworkSubnetsProvider: d,
DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr,
RuntimeRoot: cli.getSwarmRunRoot(),
})
if err != nil {
logrus.Fatalf("Error creating cluster component: %v", err)
}
// Restart all autostart containers which has a swarm endpoint
// and is not yet running now that we have successfully
// initialized the cluster.
d.RestartSwarmContainers()
logrus.Info("Daemon has completed initialization")
logrus.WithFields(logrus.Fields{
"version": dockerversion.Version,
"commit": dockerversion.GitCommit,
"graphdriver": d.GraphDriverName(),
}).Info("Docker daemon")
cli.d = d
// initMiddlewares needs cli.d to be populated. Dont change this init order.
cli.initMiddlewares(api, serverConfig)
initRouter(api, d, c)
cli.setupConfigReloadTrap()
// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
serveAPIWait := make(chan error)
go api.Wait(serveAPIWait)
// after the daemon is done setting up we can notify systemd api
notifySystem()
// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
errAPI := <-serveAPIWait
c.Cleanup()
shutdownDaemon(d)
containerdRemote.Cleanup()
if errAPI != nil {
return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
}
return nil
}
// CmdDaemon is the daemon command, called the raw arguments after `docker daemon`.
func (cli *DaemonCli) CmdDaemon(args ...string) error {
// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf
if !commonFlags.FlagSet.IsEmpty() || !clientFlags.FlagSet.IsEmpty() {
// deny `docker -D daemon`
illegalFlag := getGlobalFlag()
fmt.Fprintf(os.Stderr, "invalid flag '-%s'.\nSee 'docker daemon --help'.\n", illegalFlag.Names[0])
os.Exit(1)
} else {
// allow new form `docker daemon -D`
flag.Merge(cli.flags, commonFlags.FlagSet)
}
configFile := cli.flags.String([]string{daemonConfigFileFlag}, defaultDaemonConfigFile, "Daemon configuration file")
cli.flags.ParseFlags(args, true)
commonFlags.PostParse()
if commonFlags.TrustKey == "" {
commonFlags.TrustKey = filepath.Join(getDaemonConfDir(), defaultTrustKeyFile)
}
cliConfig, err := loadDaemonCliConfig(cli.Config, cli.flags, commonFlags, *configFile)
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
cli.Config = cliConfig
if cli.Config.Debug {
utils.EnableDebug()
}
if utils.ExperimentalBuild() {
logrus.Warn("Running experimental build")
}
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: jsonlog.RFC3339NanoFixed,
DisableColors: cli.Config.RawLogs,
})
if err := setDefaultUmask(); err != nil {
logrus.Fatalf("Failed to set umask: %v", err)
}
if len(cli.LogConfig.Config) > 0 {
if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
logrus.Fatalf("Failed to set log opts: %v", err)
}
}
var pfile *pidfile.PIDFile
if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
logrus.Fatalf("Error starting daemon: %v", err)
}
pfile = pf
defer func() {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}()
}
serverConfig := &apiserver.Config{
AuthorizationPluginNames: cli.Config.AuthorizationPlugins,
Logging: true,
SocketGroup: cli.Config.SocketGroup,
Version: dockerversion.Version,
}
serverConfig = setPlatformServerConfig(serverConfig, cli.Config)
if cli.Config.TLS {
tlsOptions := tlsconfig.Options{
CAFile: cli.Config.CommonTLSOptions.CAFile,
CertFile: cli.Config.CommonTLSOptions.CertFile,
KeyFile: cli.Config.CommonTLSOptions.KeyFile,
}
if cli.Config.TLSVerify {
// server requires and verifies client's certificate
tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(tlsOptions)
if err != nil {
logrus.Fatal(err)
}
serverConfig.TLSConfig = tlsConfig
}
if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}
api := apiserver.New(serverConfig)
for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
logrus.Fatalf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
}
protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr)
}
l, err := listeners.Init(protoAddrParts[0], protoAddrParts[1], serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
api.Accept(protoAddrParts[1], l...)
}
if err := migrateKey(); err != nil {
logrus.Fatal(err)
}
cli.TrustKeyPath = commonFlags.TrustKey
registryService := registry.NewService(cli.Config.ServiceOptions)
d, err := daemon.NewDaemon(cli.Config, registryService)
if err != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Error starting daemon: %v", err)
}
logrus.Info("Daemon has completed initialization")
logrus.WithFields(logrus.Fields{
"version": dockerversion.Version,
"commit": dockerversion.GitCommit,
"execdriver": d.ExecutionDriver().Name(),
"graphdriver": d.GraphDriverName(),
}).Info("Docker daemon")
initRouter(api, d)
reload := func(config *daemon.Config) {
if err := d.Reload(config); err != nil {
logrus.Errorf("Error reconfiguring the daemon: %v", err)
return
}
if config.IsValueSet("debug") {
debugEnabled := utils.IsDebugEnabled()
switch {
case debugEnabled && !config.Debug: // disable debug
utils.DisableDebug()
api.DisableProfiler()
case config.Debug && !debugEnabled: // enable debug
utils.EnableDebug()
api.EnableProfiler()
}
}
}
setupConfigReloadTrap(*configFile, cli.flags, reload)
// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
serveAPIWait := make(chan error)
go api.Wait(serveAPIWait)
signal.Trap(func() {
api.Close()
<-serveAPIWait
shutdownDaemon(d, 15)
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
})
// after the daemon is done setting up we can notify systemd api
notifySystem()
// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
errAPI := <-serveAPIWait
shutdownDaemon(d, 15)
if errAPI != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Shutting down due to ServeAPI error: %v", errAPI)
}
return nil
}
// CmdDaemon is the daemon command, called the raw arguments after `docker daemon`.
func (cli *DaemonCli) CmdDaemon(args ...string) error {
// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf
if !commonFlags.FlagSet.IsEmpty() || !clientFlags.FlagSet.IsEmpty() {
// deny `docker -D daemon`
illegalFlag := getGlobalFlag()
fmt.Fprintf(os.Stderr, "invalid flag '-%s'.\nSee 'docker daemon --help'.\n", illegalFlag.Names[0])
os.Exit(1)
} else {
// allow new form `docker daemon -D`
flag.Merge(daemonFlags, commonFlags.FlagSet)
}
daemonFlags.ParseFlags(args, true)
commonFlags.PostParse()
if commonFlags.TrustKey == "" {
commonFlags.TrustKey = filepath.Join(getDaemonConfDir(), defaultTrustKeyFile)
}
if utils.ExperimentalBuild() {
logrus.Warn("Running experimental build")
}
logrus.SetFormatter(&logrus.TextFormatter{TimestampFormat: jsonlog.RFC3339NanoFixed})
if err := setDefaultUmask(); err != nil {
logrus.Fatalf("Failed to set umask: %v", err)
}
if len(cli.LogConfig.Config) > 0 {
if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
logrus.Fatalf("Failed to set log opts: %v", err)
}
}
var pfile *pidfile.PIDFile
if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
logrus.Fatalf("Error starting daemon: %v", err)
}
pfile = pf
defer func() {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}()
}
serverConfig := &apiserver.Config{
AuthZPluginNames: cli.Config.AuthZPlugins,
Logging: true,
Version: dockerversion.Version,
}
serverConfig = setPlatformServerConfig(serverConfig, cli.Config)
defaultHost := opts.DefaultHost
if commonFlags.TLSOptions != nil {
if !commonFlags.TLSOptions.InsecureSkipVerify {
// server requires and verifies client's certificate
commonFlags.TLSOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(*commonFlags.TLSOptions)
if err != nil {
logrus.Fatal(err)
}
serverConfig.TLSConfig = tlsConfig
defaultHost = opts.DefaultTLSHost
}
if len(commonFlags.Hosts) == 0 {
commonFlags.Hosts = make([]string, 1)
}
for i := 0; i < len(commonFlags.Hosts); i++ {
var err error
if commonFlags.Hosts[i], err = opts.ParseHost(defaultHost, commonFlags.Hosts[i]); err != nil {
logrus.Fatalf("error parsing -H %s : %v", commonFlags.Hosts[i], err)
}
}
for _, protoAddr := range commonFlags.Hosts {
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr)
}
serverConfig.Addrs = append(serverConfig.Addrs, apiserver.Addr{Proto: protoAddrParts[0], Addr: protoAddrParts[1]})
}
api, err := apiserver.New(serverConfig)
if err != nil {
logrus.Fatal(err)
}
if err := migrateKey(); err != nil {
logrus.Fatal(err)
}
cli.TrustKeyPath = commonFlags.TrustKey
registryService := registry.NewService(cli.registryOptions)
d, err := daemon.NewDaemon(cli.Config, registryService)
if err != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Error starting daemon: %v", err)
}
logrus.Info("Daemon has completed initialization")
logrus.WithFields(logrus.Fields{
"version": dockerversion.Version,
"commit": dockerversion.GitCommit,
"execdriver": d.ExecutionDriver().Name(),
"graphdriver": d.GraphDriverName(),
}).Info("Docker daemon")
api.InitRouters(d)
// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
serveAPIWait := make(chan error)
go func() {
if err := api.ServeAPI(); err != nil {
logrus.Errorf("ServeAPI error: %v", err)
serveAPIWait <- err
return
}
serveAPIWait <- nil
}()
signal.Trap(func() {
api.Close()
<-serveAPIWait
shutdownDaemon(d, 15)
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
})
// after the daemon is done setting up we can notify systemd api
notifySystem()
// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
errAPI := <-serveAPIWait
shutdownDaemon(d, 15)
if errAPI != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Shutting down due to ServeAPI error: %v", errAPI)
}
return nil
}
func (cli *DaemonCli) start() {
// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf
flags := flag.CommandLine
cli.commonFlags.PostParse()
if cli.commonFlags.TrustKey == "" {
cli.commonFlags.TrustKey = filepath.Join(getDaemonConfDir(), cliflags.DefaultTrustKeyFile)
}
cliConfig, err := loadDaemonCliConfig(cli.Config, flags, cli.commonFlags, *cli.configFile)
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
cli.Config = cliConfig
if cli.Config.Debug {
utils.EnableDebug()
}
if utils.ExperimentalBuild() {
logrus.Warn("Running experimental build")
}
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: jsonlog.RFC3339NanoFixed,
DisableColors: cli.Config.RawLogs,
})
if err := setDefaultUmask(); err != nil {
logrus.Fatalf("Failed to set umask: %v", err)
}
if len(cli.LogConfig.Config) > 0 {
if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
logrus.Fatalf("Failed to set log opts: %v", err)
}
}
var pfile *pidfile.PIDFile
if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
logrus.Fatalf("Error starting daemon: %v", err)
}
pfile = pf
defer func() {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}()
}
serverConfig := &apiserver.Config{
Logging: true,
SocketGroup: cli.Config.SocketGroup,
Version: dockerversion.Version,
}
serverConfig = setPlatformServerConfig(serverConfig, cli.Config)
if cli.Config.TLS {
tlsOptions := tlsconfig.Options{
CAFile: cli.Config.CommonTLSOptions.CAFile,
CertFile: cli.Config.CommonTLSOptions.CertFile,
KeyFile: cli.Config.CommonTLSOptions.KeyFile,
}
if cli.Config.TLSVerify {
// server requires and verifies client's certificate
tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(tlsOptions)
if err != nil {
logrus.Fatal(err)
}
serverConfig.TLSConfig = tlsConfig
}
if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}
api := apiserver.New(serverConfig)
for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
logrus.Fatalf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
}
protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr)
}
proto := protoAddrParts[0]
addr := protoAddrParts[1]
// It's a bad idea to bind to TCP without tlsverify.
if proto == "tcp" && (serverConfig.TLSConfig == nil || serverConfig.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert) {
logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
}
l, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
logrus.Fatal(err)
}
// If we're binding to a TCP port, make sure that a container doesn't try to use it.
if proto == "tcp" {
if err := allocateDaemonPort(addr); err != nil {
logrus.Fatal(err)
}
}
logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
api.Accept(protoAddrParts[1], l...)
}
if err := migrateKey(); err != nil {
logrus.Fatal(err)
}
cli.TrustKeyPath = cli.commonFlags.TrustKey
registryService := registry.NewService(cli.Config.ServiceOptions)
containerdRemote, err := libcontainerd.New(cli.getLibcontainerdRoot(), cli.getPlatformRemoteOptions()...)
if err != nil {
logrus.Fatal(err)
}
d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote)
if err != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Error starting daemon: %v", err)
}
logrus.Info("Daemon has completed initialization")
logrus.WithFields(logrus.Fields{
"version": dockerversion.Version,
"commit": dockerversion.GitCommit,
"graphdriver": d.GraphDriverName(),
}).Info("Docker daemon")
cli.initMiddlewares(api, serverConfig)
initRouter(api, d)
reload := func(config *daemon.Config) {
if err := d.Reload(config); err != nil {
logrus.Errorf("Error reconfiguring the daemon: %v", err)
return
}
if config.IsValueSet("debug") {
debugEnabled := utils.IsDebugEnabled()
switch {
case debugEnabled && !config.Debug: // disable debug
utils.DisableDebug()
api.DisableProfiler()
case config.Debug && !debugEnabled: // enable debug
utils.EnableDebug()
api.EnableProfiler()
}
}
}
setupConfigReloadTrap(*cli.configFile, flags, reload)
// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
serveAPIWait := make(chan error)
go api.Wait(serveAPIWait)
signal.Trap(func() {
api.Close()
<-serveAPIWait
shutdownDaemon(d, 15)
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
})
// after the daemon is done setting up we can notify systemd api
notifySystem()
// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
errAPI := <-serveAPIWait
shutdownDaemon(d, 15)
containerdRemote.Cleanup()
if errAPI != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Shutting down due to ServeAPI error: %v", errAPI)
}
}
// CmdDaemon is the daemon command, called the raw arguments after `docker daemon`.
func (cli *DaemonCli) CmdDaemon(args ...string) error {
// warn from uuid package when running the daemon
uuid.Loggerf = logrus.Warnf
//调整一下daemon的启动方式
if !commonFlags.FlagSet.IsEmpty() || !clientFlags.FlagSet.IsEmpty() {
// deny `docker -D daemon`
illegalFlag := getGlobalFlag()
fmt.Fprintf(os.Stderr, "invalid flag '-%s'.\nSee 'docker daemon --help'.\n", illegalFlag.Names[0])
os.Exit(1)
} else {
// allow new form `docker daemon -D`
flag.Merge(cli.flags, commonFlags.FlagSet)
}
configFile := cli.flags.String([]string{daemonConfigFileFlag}, defaultDaemonConfigFile, "Daemon configuration file")
//匹配配置参数
cli.flags.ParseFlags(args, true)
//配置参数生效
commonFlags.PostParse()
if commonFlags.TrustKey == "" {
commonFlags.TrustKey = filepath.Join(getDaemonConfDir(), defaultTrustKeyFile)
}
cliConfig, err := loadDaemonCliConfig(cli.Config, cli.flags, commonFlags, *configFile)
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
cli.Config = cliConfig
if cli.Config.Debug {
utils.EnableDebug()
}
if utils.ExperimentalBuild() {
logrus.Warn("Running experimental build")
}
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: jsonlog.RFC3339NanoFixed,
DisableColors: cli.Config.RawLogs,
})
if err := setDefaultUmask(); err != nil {
logrus.Fatalf("Failed to set umask: %v", err)
}
if len(cli.LogConfig.Config) > 0 {
if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
logrus.Fatalf("Failed to set log opts: %v", err)
}
}
var pfile *pidfile.PIDFile
if cli.Pidfile != "" {
pf, err := pidfile.New(cli.Pidfile)
if err != nil {
logrus.Fatalf("Error starting daemon: %v", err)
}
pfile = pf
defer func() {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}()
}
//定义apiserver的配置，包括认证、日志输出、版本等。
serverConfig := &apiserver.Config{
AuthorizationPluginNames: cli.Config.AuthorizationPlugins,
Logging: true,
SocketGroup: cli.Config.SocketGroup,
Version: dockerversion.Version,
}
serverConfig = setPlatformServerConfig(serverConfig, cli.Config)
if cli.Config.TLS {
tlsOptions := tlsconfig.Options{
CAFile: cli.Config.CommonTLSOptions.CAFile,
CertFile: cli.Config.CommonTLSOptions.CertFile,
KeyFile: cli.Config.CommonTLSOptions.KeyFile,
}
if cli.Config.TLSVerify {
// server requires and verifies client's certificate
tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
}
tlsConfig, err := tlsconfig.Server(tlsOptions)
if err != nil {
logrus.Fatal(err)
}
serverConfig.TLSConfig = tlsConfig
}
if len(cli.Config.Hosts) == 0 {
cli.Config.Hosts = make([]string, 1)
}
//定义一个新的apiserver。
//apiServer是一个这样的结构(api/server/server.go)：
/*
type Server struct {
cfg *Config
servers []*HTTPServer
routers []router.Router
authZPlugins []authorization.Plugin
routerSwapper *routerSwapper
}
*/
api := apiserver.New(serverConfig)
for i := 0; i < len(cli.Config.Hosts); i++ {
var err error
if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
logrus.Fatalf("error parsin
g -H %s : %v", cli.Config.Hosts[i], err)
}
protoAddr := cli.Config.Hosts[i]
protoAddrParts := strings.SplitN(protoAddr, "://", 2)
if len(protoAddrParts) != 2 {
logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr)
}
l, err := listeners.Init(protoAddrParts[0], protoAddrParts[1], serverConfig.SocketGroup, serverConfig.TLSConfig)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
//初始化api的servers数组，里面放着的都是httpserver类型。此时也没有具体的运行什么
api.Accept(protoAddrParts[1], l...)
}
if err := migrateKey(); err != nil {
logrus.Fatal(err)
}
cli.TrustKeyPath = commonFlags.TrustKey
//创建镜像仓库服务
registryService := registry.NewService(cli.Config.ServiceOptions)
//初始化libcontainer。比如在linux中，就会调用libcontainerd/remote_linux.go中的New方法。
containerdRemote, err := libcontainerd.New(filepath.Join(cli.Config.ExecRoot, "libcontainerd"), cli.getPlatformRemoteOptions()...)
if err != nil {
logrus.Fatal(err)
}
//初始化守护进程使得能够服务。需要输入仓库服务和libcontainerd服务的参数。
//返回的d是Daemon类型：
/*
type Daemon struct {
ID string
repository string
containers container.Store
execCommands *exec.Store
referenceStore reference.Store
downloadManager *xfer.LayerDownloadManager
uploadManager *xfer.LayerUploadManager
distributionMetadataStore dmetadata.Store
trustKey libtrust.PrivateKey
idIndex *truncindex.TruncIndex
configStore *Config
statsCollector *statsCollector
defaultLogConfig containertypes.LogConfig
RegistryService *registry.Service
EventsService *events.Events
netController libnetwork.NetworkController
volumes *store.VolumeStore
discoveryWatcher discoveryReloader
root string
seccompEnabled bool
shutdown bool
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
layerStore layer.Store
imageStore image.Store
nameIndex *registrar.Registrar
linkIndex *linkIndex
containerd libcontainerd.Client
defaultIsolation containertypes.Isolation // Default isolation mode on Windows
}
*/
d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote)
if err != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Error starting daemon: %v", err)
}
logrus.Info("Daemon has completed initialization")
logrus.WithFields(logrus.Fields{
"version": dockerversion.Version,
"commit": dockerversion.GitCommit,
"graphdriver": d.GraphDriverName(),
}).Info("Docker daemon")
//初始化http的路由，这个路由设计的非常易懂，所有的路由及处理函数的映射关系
//请见api/server/router/文件夹中的内容。有类似这样的内容：
//router.NewPostRoute("/containers/create", r.postContainersCreate),
//其中，对应的处理函数postContainersCreate在api/server/router/container/container_routes.go
//但是，实际上这个函数也不做具体的事情，他交给backend去做，就是daemon去做
/*
ccr, err := s.backend.ContainerCreate(types.ContainerCreateConfig{
Name: name,
Config: config,
HostConfig: hostConfig,
NetworkingConfig: networkingConfig,
AdjustCPUShares: adjustCPUShares,
})
*/
//其中的ContainerCreate在
initRouter(api, d)
reload := func(config *daemon.Config) {
if err := d.Reload(config); err != nil {
logrus.Errorf("Error reconfiguring the daemon: %v", err)
return
}
if config.IsValueSet("debug") {
debugEnabled := utils.IsDebugEnabled()
switch {
case debugEnabled && !config.Debug: // disable debug
utils.DisableDebug()
api.DisableProfiler()
case config.Debug && !debugEnabled: // enable debug
utils.EnableDebug()
api.EnableProfiler()
}
}
}
setupConfigReloadTrap(*configFile, cli.flags, reload)
// The serve API routine never exits unless an error occurs
// We need to start it as a goroutine and wait on it so
// daemon doesn't exit
//设置一个传输apiServer状态的通道
serveAPIWait := make(chan error)
//重新开启一个goroutine作为httpServer。
//具体的请查看api/server/server.go中的方法func (s *Server) serveAPI() error
go api.Wait(serveAPIWait)
signal.Trap(func() {
api.Close()
<-serveAPIWait
shutdownDaemon(d, 15)
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
})
// after the daemon is done setting up we can notify systemd api
notifySystem()
// Daemon is fully initialized and handling API traffic
// Wait for serve API to complete
//<-表示接受通道值，只有当通道中有值的时候，才会返回。
//也就是说主线程一直在等待api.wait的goroutine启动apiServer之后的返回才会进行。
errAPI := <-serveAPIWait
//当接收到返回（返回就是错误了），开始清理进程。
shutdownDaemon(d, 15)
containerdRemote.Cleanup()
if errAPI != nil {
if pfile != nil {
if err := pfile.Remove(); err != nil {
logrus.Error(err)
}
}
logrus.Fatalf("Shutting down due to ServeAPI error: %v", errAPI)
}
return nil
}