func ClusterServices() (ECSServices, error) {
var log = logger.New("ns=ClusterServices")
services := ECSServices{}
lsres, err := ECS().ListServices(&ecs.ListServicesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
})
if err != nil {
log.Log("at=ListServices err=%q", err)
return services, err
}
dsres, err := ECS().DescribeServices(&ecs.DescribeServicesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
Services: lsres.ServiceArns,
})
if err != nil {
log.Log("at=ListServices err=%q", err)
return services, err
}
for i := 0; i < len(dsres.Services); i++ {
services = append(services, dsres.Services[i])
}
return services, nil
}
func ws(at string, handler ApiWebsocketFunc) websocket.Handler {
return websocket.Handler(func(ws *websocket.Conn) {
log := logger.New("ns=kernel").At(at).Start()
if !passwordCheck(ws.Request()) {
ws.Write([]byte("ERROR: invalid authorization\n"))
return
}
if !versionCheck(ws.Request()) {
ws.Write([]byte("client outdated, please update with `convox update`\n"))
return
}
err := handler(ws)
if err != nil {
ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err)))
logError(log, err)
return
}
log.Log("state=success")
})
}
func subscribeCloudWatchLogsStream(group, stream string, startTime time.Time, output chan []byte, quit chan bool) {
log := logger.New("at=subscribe-cloudwatch").Start()
fmt.Printf("subscribeCloudWatchLogsStream group=%s stream=%s startTime=%s\n", group, stream, startTime)
startTimeMs := startTime.Unix() * 1000 // ms since epoch
req := cloudwatchlogs.GetLogEventsInput{
LogGroupName: aws.String(group),
LogStreamName: aws.String(stream),
}
for {
select {
case <-quit:
log.Log("qutting")
return
default:
req.StartTime = &startTimeMs
res, err := CloudWatchLogs().GetLogEvents(&req)
if err != nil {
fmt.Printf("err3 %+v\n", err)
return
}
for _, event := range res.Events {
output <- []byte(fmt.Sprintf("%s\n", string(*event.Message)))
startTimeMs = *event.Timestamp + 1
}
time.Sleep(1000 * time.Millisecond)
}
}
}
// Get latest convergence state and send notifications
func monitorConverged(lastConverged bool, lastEventAt time.Time) (bool, ecs.ServiceEvent) {
log := logger.New("ns=services_monitor")
services, err := models.ClusterServices()
if err != nil {
log.Log("fn=monitorConverged err=%q", err)
return lastConverged, ecs.ServiceEvent{
CreatedAt: aws.Time(lastEventAt),
}
}
converged := services.IsConverged()
events := services.EventsSince(lastEventAt)
log.Log("fn=monitorConverged converged=%t events=%d lastEventAt=%q", converged, len(events), lastEventAt)
if events.HasCapacityWarning() {
models.NotifyError("rack:capacity", fmt.Errorf(events.CapacityWarning()), map[string]string{
"rack": os.Getenv("RACK"),
})
}
if converged != lastConverged {
models.NotifySuccess("rack:converge", map[string]string{
"rack": os.Getenv("RACK"),
"converged": fmt.Sprintf("%t", converged),
})
}
return converged, services.LastEvent()
}
func api(at string, handler ApiHandlerFunc) http.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) {
log := logger.New("ns=kernel").At(at).Start()
if !passwordCheck(r) {
rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`)
rw.WriteHeader(401)
rw.Write([]byte("invalid authorization"))
return
}
if !versionCheck(r) {
rw.WriteHeader(403)
rw.Write([]byte("client outdated, please update with `convox update`"))
return
}
err := handler(rw, r)
if err != nil {
rw.WriteHeader(err.Code())
RenderError(rw, err)
logError(log, err)
return
}
log.Log("state=success")
}
}
func Notify(name, status string, data map[string]string) error {
if PauseNotifications {
return nil
}
log := logger.New("ns=kernel")
data["rack"] = os.Getenv("RACK")
event := &client.NotifyEvent{
Action: name,
Status: status,
Data: data,
Timestamp: time.Now().UTC(),
}
message, err := json.Marshal(event)
if err != nil {
return err
}
params := &sns.PublishInput{
Message: aws.String(string(message)), // Required
Subject: aws.String(name),
TargetArn: aws.String(NotificationTopic),
}
resp, err := SNS().Publish(params)
if err != nil {
return err
}
log.At("Notfiy").Log("message-id=%q", *resp.MessageId)
return nil
}
func SNSProxy(w http.ResponseWriter, r *http.Request) {
log := logger.New("ns=kernel").At("SNSProxy")
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
var payload map[string]string
err = json.Unmarshal(body, &payload)
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
url := r.FormValue("endpoint")
resp, err := http.Post(url, "application/json", strings.NewReader(payload["Message"]))
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
log.Log("proxied=true status=%s", resp.Status)
w.Write([]byte("ok"))
}
func StartCluster() {
var log = logger.New("ns=cluster_monitor")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
instances := Instances{}
err := instances.describeASG()
if err != nil {
log.Error(err)
continue
}
err = instances.describeECS()
if err != nil {
log.Error(err)
continue
}
// TODO: Add an instances.testDocker() call to the mission critical path
// Test if ASG Instance is registered and connected in ECS cluster
for _, i := range instances {
if !i.ASG {
// TODO: Rogue instance?! Terminate?
continue
}
if !i.ECS {
// Not registered or not connected => set Unhealthy
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceId: aws.String(i.Id),
ShouldRespectGracePeriod: aws.Bool(true),
},
)
i.Unhealthy = true
if err != nil {
log.Error(err)
continue
}
}
}
log.Log(instances.log())
}
}
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
defer recoverWith(func(err error) {
log := logger.New("ns=kernel").At("panic")
helpers.Error(log, err)
http.Error(rw, err.Error(), http.StatusInternalServerError)
})
next(rw, r)
}
func StartImages() {
var log = logger.New("ns=app_images")
if os.Getenv("DEVELOPMENT") == "true" {
return
}
maxRetries := 5
var err error
for i := 0; i < maxRetries; i++ {
err := dockerLogin()
if err == nil {
break
}
time.Sleep(30 * time.Second)
}
if err != nil {
return
}
apps, err := models.ListApps()
if err != nil {
log.Error(err)
return
}
for _, app := range apps {
a, err := models.GetApp(app.Name)
if err != nil {
log.Error(err)
continue
}
for key, value := range a.Parameters {
if strings.HasSuffix(key, "Image") {
log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
data, err := exec.Command("docker", "pull", value).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
continue
}
}
}
}
}
func autoscaleRack() {
log := logger.New("ns=workers.autoscale at=autoscaleRack")
capacity, err := provider.CapacityGet()
if err != nil {
log.Log("fn=models.GetSystemCapacity err=%q", err)
return
}
log.Log("autoscale=%t", autoscale)
if !autoscale {
return
}
system, err := provider.SystemGet()
if err != nil {
log.Log("fn=models.GetSystem err=%q", err)
return
}
// calaculate instance requirements based on total process memory needed divided by the memory
// on an individual instance
instances := int(math.Ceil(float64(capacity.ProcessMemory) / float64(capacity.InstanceMemory)))
// instance count cant be less than 2
if instances < 2 {
instances = 2
}
// instance count must be at least maxconcurrency+1
if instances < (int(capacity.ProcessWidth) + 1) {
instances = int(capacity.ProcessWidth) + 1
}
log.Log("process.memory=%d instance.memory=%d instances=%d change=%d", capacity.ProcessMemory, capacity.InstanceMemory, instances, (instances - system.Count))
// if no change then exit
if system.Count == instances {
return
}
system.Count = instances
err = provider.SystemSave(*system)
if err != nil {
log.Log("fn=system.Save err=%q", err)
return
}
}
func StartHeartbeat() {
log := logger.New("ns=heartbeat")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
helpers.TrackEvent("kernel-heartbeat", "")
for _ = range time.Tick(1 * time.Hour) {
helpers.TrackEvent("kernel-heartbeat", "")
}
}
func dockerLogin() error {
var log = logger.New("ns=app_images")
log.Log("cmd=%q", fmt.Sprintf("docker login -e [email protected] -u convox -p ***** %s", os.Getenv("REGISTRY_HOST")))
data, err := exec.Command("docker", "login", "-e", "*****@*****.**", "-u", "convox", "-p", os.Getenv("PASSWORD"), os.Getenv("REGISTRY_HOST")).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
}
return err
}
func subscribeKinesisShard(stream, shard string, output chan []byte, quit chan bool) {
log := logger.New("at=subscribe-kinesis").Start()
ireq := &kinesis.GetShardIteratorInput{
ShardId: aws.String(shard),
ShardIteratorType: aws.String("LATEST"),
StreamName: aws.String(stream),
}
ires, err := Kinesis().GetShardIterator(ireq)
if err != nil {
log.Error(err)
return
}
iter := *ires.ShardIterator
for {
select {
case <-quit:
log.Log("qutting")
return
default:
greq := &kinesis.GetRecordsInput{
ShardIterator: aws.String(iter),
}
gres, err := Kinesis().GetRecords(greq)
if err != nil {
fmt.Printf("err3 %+v\n", err)
// panic(err)
return
}
iter = *gres.NextShardIterator
for _, record := range gres.Records {
output <- []byte(fmt.Sprintf("%s\n", string(record.Data)))
}
time.Sleep(500 * time.Millisecond)
}
}
}
// Get initial convergence state
func checkConverged() (bool, ecs.ServiceEvent) {
log := logger.New("ns=services_monitor")
services, err := models.ClusterServices()
if err != nil {
log.Log("fn=checkConverged err=%q", err)
return true, ecs.ServiceEvent{
CreatedAt: aws.Time(time.Now()),
}
}
converged := services.IsConverged()
lastEvent := services.LastEvent()
log.Log("fn=checkConverged converged=%t lastEventAt=%q", converged, lastEvent.CreatedAt)
return converged, lastEvent
}
func SNSConfirm(w http.ResponseWriter, r *http.Request) {
log := logger.New("ns=kernel").At("SNSConfirm")
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
var payload map[string]string
err = json.Unmarshal(body, &payload)
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
params := &sns.ConfirmSubscriptionInput{
Token: aws.String(payload["Token"]),
TopicArn: aws.String(payload["TopicArn"]),
}
resp, err := models.SNS().ConfirmSubscription(params)
if err != nil {
log.Error(err)
http.Error(w, err.Error(), 500)
return
}
log.Log("confirmed=true subscriptionArn=%q", *resp.SubscriptionArn)
w.Write([]byte("ok"))
}
func New(ns string, ignore []string) *Nlogger {
return &Nlogger{ignore: ignore, log: logger.New(ns)}
}
// Set up rack instance Docker environment for builds
// Log into all configured private registries
// Pull down latest images for all apps
func StartImages() {
var log = logger.New("ns=app_images")
if os.Getenv("DEVELOPMENT") == "true" {
return
}
// doing this in development updates a ~/.docker file and causes a rerun loop
models.LoginRegistries()
maxRetries := 5
var err error
for i := 0; i < maxRetries; i++ {
err = models.DockerLogin(docker.AuthConfiguration{
Email: "*****@*****.**",
Username: "******",
Password: os.Getenv("PASSWORD"),
ServerAddress: os.Getenv("REGISTRY_HOST"),
})
if err == nil {
break
}
time.Sleep(30 * time.Second)
}
if err != nil {
return
}
apps, err := models.ListApps()
if err != nil {
log.Error(err)
return
}
for _, app := range apps {
a, err := models.GetApp(app.Name)
if err != nil {
log.Error(err)
continue
}
for key, value := range a.Parameters {
if strings.HasSuffix(key, "Image") {
log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
data, err := exec.Command("docker", "pull", value).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
continue
}
}
}
}
}
func subscribeRDS(prefix, id string, output chan []byte, quit chan bool) {
// Get latest log file details via pagination tokens
details := rds.DescribeDBLogFilesDetails{}
marker := ""
log := logger.New("at=subscribe-kinesis").Start()
for {
params := &rds.DescribeDBLogFilesInput{
DBInstanceIdentifier: aws.String(id),
MaxRecords: aws.Int64(100),
}
if marker != "" {
params.Marker = aws.String(marker)
}
res, err := RDS().DescribeDBLogFiles(params)
if err != nil {
panic(err)
}
if res.Marker == nil {
files := res.DescribeDBLogFiles
details = *files[len(files)-1]
break
}
marker = *res.Marker
}
// Get last 50 log lines
params := &rds.DownloadDBLogFilePortionInput{
DBInstanceIdentifier: aws.String(id),
LogFileName: aws.String(*details.LogFileName),
NumberOfLines: aws.Int64(50),
}
res, err := RDS().DownloadDBLogFilePortion(params)
if err != nil {
panic(err)
}
output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))
params.Marker = aws.String(*res.Marker)
for {
select {
case <-quit:
log.Log("qutting")
return
default:
res, err := RDS().DownloadDBLogFilePortion(params)
if err != nil {
panic(err)
}
if *params.Marker != *res.Marker {
params.Marker = aws.String(*res.Marker)
output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))
}
time.Sleep(1000 * time.Millisecond)
}
}
}