func ClusterServices() (ECSServices, error) { var log = logger.New("ns=ClusterServices") services := ECSServices{} lsres, err := ECS().ListServices(&ecs.ListServicesInput{ Cluster: aws.String(os.Getenv("CLUSTER")), }) if err != nil { log.Log("at=ListServices err=%q", err) return services, err } dsres, err := ECS().DescribeServices(&ecs.DescribeServicesInput{ Cluster: aws.String(os.Getenv("CLUSTER")), Services: lsres.ServiceArns, }) if err != nil { log.Log("at=ListServices err=%q", err) return services, err } for i := 0; i < len(dsres.Services); i++ { services = append(services, dsres.Services[i]) } return services, nil }
func ws(at string, handler ApiWebsocketFunc) websocket.Handler { return websocket.Handler(func(ws *websocket.Conn) { log := logger.New("ns=kernel").At(at).Start() if !passwordCheck(ws.Request()) { ws.Write([]byte("ERROR: invalid authorization\n")) return } if !versionCheck(ws.Request()) { ws.Write([]byte("client outdated, please update with `convox update`\n")) return } err := handler(ws) if err != nil { ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err))) logError(log, err) return } log.Log("state=success") }) }
func subscribeCloudWatchLogsStream(group, stream string, startTime time.Time, output chan []byte, quit chan bool) { log := logger.New("at=subscribe-cloudwatch").Start() fmt.Printf("subscribeCloudWatchLogsStream group=%s stream=%s startTime=%s\n", group, stream, startTime) startTimeMs := startTime.Unix() * 1000 // ms since epoch req := cloudwatchlogs.GetLogEventsInput{ LogGroupName: aws.String(group), LogStreamName: aws.String(stream), } for { select { case <-quit: log.Log("qutting") return default: req.StartTime = &startTimeMs res, err := CloudWatchLogs().GetLogEvents(&req) if err != nil { fmt.Printf("err3 %+v\n", err) return } for _, event := range res.Events { output <- []byte(fmt.Sprintf("%s\n", string(*event.Message))) startTimeMs = *event.Timestamp + 1 } time.Sleep(1000 * time.Millisecond) } } }
// Get latest convergence state and send notifications func monitorConverged(lastConverged bool, lastEventAt time.Time) (bool, ecs.ServiceEvent) { log := logger.New("ns=services_monitor") services, err := models.ClusterServices() if err != nil { log.Log("fn=monitorConverged err=%q", err) return lastConverged, ecs.ServiceEvent{ CreatedAt: aws.Time(lastEventAt), } } converged := services.IsConverged() events := services.EventsSince(lastEventAt) log.Log("fn=monitorConverged converged=%t events=%d lastEventAt=%q", converged, len(events), lastEventAt) if events.HasCapacityWarning() { models.NotifyError("rack:capacity", fmt.Errorf(events.CapacityWarning()), map[string]string{ "rack": os.Getenv("RACK"), }) } if converged != lastConverged { models.NotifySuccess("rack:converge", map[string]string{ "rack": os.Getenv("RACK"), "converged": fmt.Sprintf("%t", converged), }) } return converged, services.LastEvent() }
func api(at string, handler ApiHandlerFunc) http.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request) { log := logger.New("ns=kernel").At(at).Start() if !passwordCheck(r) { rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`) rw.WriteHeader(401) rw.Write([]byte("invalid authorization")) return } if !versionCheck(r) { rw.WriteHeader(403) rw.Write([]byte("client outdated, please update with `convox update`")) return } err := handler(rw, r) if err != nil { rw.WriteHeader(err.Code()) RenderError(rw, err) logError(log, err) return } log.Log("state=success") } }
func Notify(name, status string, data map[string]string) error { if PauseNotifications { return nil } log := logger.New("ns=kernel") data["rack"] = os.Getenv("RACK") event := &client.NotifyEvent{ Action: name, Status: status, Data: data, Timestamp: time.Now().UTC(), } message, err := json.Marshal(event) if err != nil { return err } params := &sns.PublishInput{ Message: aws.String(string(message)), // Required Subject: aws.String(name), TargetArn: aws.String(NotificationTopic), } resp, err := SNS().Publish(params) if err != nil { return err } log.At("Notfiy").Log("message-id=%q", *resp.MessageId) return nil }
func SNSProxy(w http.ResponseWriter, r *http.Request) { log := logger.New("ns=kernel").At("SNSProxy") defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } var payload map[string]string err = json.Unmarshal(body, &payload) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } url := r.FormValue("endpoint") resp, err := http.Post(url, "application/json", strings.NewReader(payload["Message"])) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } log.Log("proxied=true status=%s", resp.Status) w.Write([]byte("ok")) }
func StartCluster() { var log = logger.New("ns=cluster_monitor") defer recoverWith(func(err error) { helpers.Error(log, err) }) for _ = range time.Tick(5 * time.Minute) { log.Log("tick") instances := Instances{} err := instances.describeASG() if err != nil { log.Error(err) continue } err = instances.describeECS() if err != nil { log.Error(err) continue } // TODO: Add an instances.testDocker() call to the mission critical path // Test if ASG Instance is registered and connected in ECS cluster for _, i := range instances { if !i.ASG { // TODO: Rogue instance?! Terminate? continue } if !i.ECS { // Not registered or not connected => set Unhealthy _, err := models.AutoScaling().SetInstanceHealth( &autoscaling.SetInstanceHealthInput{ HealthStatus: aws.String("Unhealthy"), InstanceId: aws.String(i.Id), ShouldRespectGracePeriod: aws.Bool(true), }, ) i.Unhealthy = true if err != nil { log.Error(err) continue } } } log.Log(instances.log()) } }
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { defer recoverWith(func(err error) { log := logger.New("ns=kernel").At("panic") helpers.Error(log, err) http.Error(rw, err.Error(), http.StatusInternalServerError) }) next(rw, r) }
func StartImages() { var log = logger.New("ns=app_images") if os.Getenv("DEVELOPMENT") == "true" { return } maxRetries := 5 var err error for i := 0; i < maxRetries; i++ { err := dockerLogin() if err == nil { break } time.Sleep(30 * time.Second) } if err != nil { return } apps, err := models.ListApps() if err != nil { log.Error(err) return } for _, app := range apps { a, err := models.GetApp(app.Name) if err != nil { log.Error(err) continue } for key, value := range a.Parameters { if strings.HasSuffix(key, "Image") { log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value)) data, err := exec.Command("docker", "pull", value).CombinedOutput() if err != nil { fmt.Printf("%+v\n", string(data)) log.Error(err) continue } } } } }
func autoscaleRack() { log := logger.New("ns=workers.autoscale at=autoscaleRack") capacity, err := provider.CapacityGet() if err != nil { log.Log("fn=models.GetSystemCapacity err=%q", err) return } log.Log("autoscale=%t", autoscale) if !autoscale { return } system, err := provider.SystemGet() if err != nil { log.Log("fn=models.GetSystem err=%q", err) return } // calaculate instance requirements based on total process memory needed divided by the memory // on an individual instance instances := int(math.Ceil(float64(capacity.ProcessMemory) / float64(capacity.InstanceMemory))) // instance count cant be less than 2 if instances < 2 { instances = 2 } // instance count must be at least maxconcurrency+1 if instances < (int(capacity.ProcessWidth) + 1) { instances = int(capacity.ProcessWidth) + 1 } log.Log("process.memory=%d instance.memory=%d instances=%d change=%d", capacity.ProcessMemory, capacity.InstanceMemory, instances, (instances - system.Count)) // if no change then exit if system.Count == instances { return } system.Count = instances err = provider.SystemSave(*system) if err != nil { log.Log("fn=system.Save err=%q", err) return } }
func StartHeartbeat() { log := logger.New("ns=heartbeat") defer recoverWith(func(err error) { helpers.Error(log, err) }) helpers.TrackEvent("kernel-heartbeat", "") for _ = range time.Tick(1 * time.Hour) { helpers.TrackEvent("kernel-heartbeat", "") } }
func dockerLogin() error { var log = logger.New("ns=app_images") log.Log("cmd=%q", fmt.Sprintf("docker login -e [email protected] -u convox -p ***** %s", os.Getenv("REGISTRY_HOST"))) data, err := exec.Command("docker", "login", "-e", "*****@*****.**", "-u", "convox", "-p", os.Getenv("PASSWORD"), os.Getenv("REGISTRY_HOST")).CombinedOutput() if err != nil { fmt.Printf("%+v\n", string(data)) log.Error(err) } return err }
func subscribeKinesisShard(stream, shard string, output chan []byte, quit chan bool) { log := logger.New("at=subscribe-kinesis").Start() ireq := &kinesis.GetShardIteratorInput{ ShardId: aws.String(shard), ShardIteratorType: aws.String("LATEST"), StreamName: aws.String(stream), } ires, err := Kinesis().GetShardIterator(ireq) if err != nil { log.Error(err) return } iter := *ires.ShardIterator for { select { case <-quit: log.Log("qutting") return default: greq := &kinesis.GetRecordsInput{ ShardIterator: aws.String(iter), } gres, err := Kinesis().GetRecords(greq) if err != nil { fmt.Printf("err3 %+v\n", err) // panic(err) return } iter = *gres.NextShardIterator for _, record := range gres.Records { output <- []byte(fmt.Sprintf("%s\n", string(record.Data))) } time.Sleep(500 * time.Millisecond) } } }
// Get initial convergence state func checkConverged() (bool, ecs.ServiceEvent) { log := logger.New("ns=services_monitor") services, err := models.ClusterServices() if err != nil { log.Log("fn=checkConverged err=%q", err) return true, ecs.ServiceEvent{ CreatedAt: aws.Time(time.Now()), } } converged := services.IsConverged() lastEvent := services.LastEvent() log.Log("fn=checkConverged converged=%t lastEventAt=%q", converged, lastEvent.CreatedAt) return converged, lastEvent }
func SNSConfirm(w http.ResponseWriter, r *http.Request) { log := logger.New("ns=kernel").At("SNSConfirm") defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } var payload map[string]string err = json.Unmarshal(body, &payload) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } params := &sns.ConfirmSubscriptionInput{ Token: aws.String(payload["Token"]), TopicArn: aws.String(payload["TopicArn"]), } resp, err := models.SNS().ConfirmSubscription(params) if err != nil { log.Error(err) http.Error(w, err.Error(), 500) return } log.Log("confirmed=true subscriptionArn=%q", *resp.SubscriptionArn) w.Write([]byte("ok")) }
func New(ns string, ignore []string) *Nlogger { return &Nlogger{ignore: ignore, log: logger.New(ns)} }
// Set up rack instance Docker environment for builds // Log into all configured private registries // Pull down latest images for all apps func StartImages() { var log = logger.New("ns=app_images") if os.Getenv("DEVELOPMENT") == "true" { return } // doing this in development updates a ~/.docker file and causes a rerun loop models.LoginRegistries() maxRetries := 5 var err error for i := 0; i < maxRetries; i++ { err = models.DockerLogin(docker.AuthConfiguration{ Email: "*****@*****.**", Username: "******", Password: os.Getenv("PASSWORD"), ServerAddress: os.Getenv("REGISTRY_HOST"), }) if err == nil { break } time.Sleep(30 * time.Second) } if err != nil { return } apps, err := models.ListApps() if err != nil { log.Error(err) return } for _, app := range apps { a, err := models.GetApp(app.Name) if err != nil { log.Error(err) continue } for key, value := range a.Parameters { if strings.HasSuffix(key, "Image") { log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value)) data, err := exec.Command("docker", "pull", value).CombinedOutput() if err != nil { fmt.Printf("%+v\n", string(data)) log.Error(err) continue } } } } }
func subscribeRDS(prefix, id string, output chan []byte, quit chan bool) { // Get latest log file details via pagination tokens details := rds.DescribeDBLogFilesDetails{} marker := "" log := logger.New("at=subscribe-kinesis").Start() for { params := &rds.DescribeDBLogFilesInput{ DBInstanceIdentifier: aws.String(id), MaxRecords: aws.Int64(100), } if marker != "" { params.Marker = aws.String(marker) } res, err := RDS().DescribeDBLogFiles(params) if err != nil { panic(err) } if res.Marker == nil { files := res.DescribeDBLogFiles details = *files[len(files)-1] break } marker = *res.Marker } // Get last 50 log lines params := &rds.DownloadDBLogFilePortionInput{ DBInstanceIdentifier: aws.String(id), LogFileName: aws.String(*details.LogFileName), NumberOfLines: aws.Int64(50), } res, err := RDS().DownloadDBLogFilePortion(params) if err != nil { panic(err) } output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData)) params.Marker = aws.String(*res.Marker) for { select { case <-quit: log.Log("qutting") return default: res, err := RDS().DownloadDBLogFilePortion(params) if err != nil { panic(err) } if *params.Marker != *res.Marker { params.Marker = aws.String(*res.Marker) output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData)) } time.Sleep(1000 * time.Millisecond) } } }