func (a *HostAgent) removeInstance(stateID string, ctr *docker.Container) {
rc, err := ctr.Wait(time.Second)
if err != nil || rc != 0 || glog.GetVerbosity() > 0 {
// TODO: output of docker logs is potentially very large
// this should be implemented another way, perhaps a docker attach
// or extend docker to give last N seconds
if output, err := exec.Command("docker", "logs", "--tail", "10000", ctr.ID).CombinedOutput(); err != nil {
glog.Errorf("Could not get logs for container %s", ctr.ID)
} else {
glog.Warningf("Last 10000 lines of container %s:\n %s", ctr.ID, string(output))
}
}
if ctr.IsRunning() {
glog.Errorf("Instance %s (%s) is still running, killing container")
ctr.Kill()
}
if err := ctr.Delete(true); err != nil {
glog.Errorf("Could not remove instance %s (%s): %s", stateID, ctr.ID, err)
}
glog.Infof("Service state %s (%s) receieved exit code %d", stateID, ctr.ID, rc)
}
func setLogging(ctx *cli.Context) error {
if ctx.GlobalBool("master") || ctx.GlobalBool("agent") {
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
glog.SetLogstashType("serviced-" + hostname)
}
if ctx.IsSet("logtostderr") {
glog.SetToStderr(ctx.GlobalBool("logtostderr"))
}
if ctx.IsSet("alsologtostderr") {
glog.SetAlsoToStderr(ctx.GlobalBool("alsologtostderr"))
}
glog.SetLogstashURL(ctx.GlobalString("logstashurl"))
if ctx.IsSet("v") {
glog.SetVerbosity(ctx.GlobalInt("v"))
}
if ctx.IsSet("stderrthreshold") {
if err := glog.SetStderrThreshold(ctx.GlobalString("stderrthreshold")); err != nil {
return err
}
}
if ctx.IsSet("vmodule") {
if err := glog.SetVModule(ctx.GlobalString("vmodule")); err != nil {
return err
}
}
if ctx.IsSet("log_backtrace_at") {
if err := glog.SetTraceLocation(ctx.GlobalString("log_backtrace_at")); err != nil {
return err
}
}
// Listen for SIGUSR1 and, when received, toggle the log level between
// 0 and 2. If the log level is anything but 0, we set it to 0, and on
// subsequent signals, set it to 2.
go func() {
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGUSR1)
for {
<-signalChan
glog.Infof("Received signal SIGUSR1")
if glog.GetVerbosity() == 0 {
glog.SetVerbosity(2)
} else {
glog.SetVerbosity(0)
}
glog.Infof("Log level changed to %v", glog.GetVerbosity())
}
}()
return nil
}
