Example #1
0
func ClusterServices() (ECSServices, error) {
	var log = logger.New("ns=ClusterServices")

	services := ECSServices{}

	lsres, err := ECS().ListServices(&ecs.ListServicesInput{
		Cluster: aws.String(os.Getenv("CLUSTER")),
	})

	if err != nil {
		log.Log("at=ListServices err=%q", err)
		return services, err
	}

	dsres, err := ECS().DescribeServices(&ecs.DescribeServicesInput{
		Cluster:  aws.String(os.Getenv("CLUSTER")),
		Services: lsres.ServiceArns,
	})

	if err != nil {
		log.Log("at=ListServices err=%q", err)
		return services, err
	}

	for i := 0; i < len(dsres.Services); i++ {
		services = append(services, dsres.Services[i])
	}

	return services, nil
}
Example #2
0
func ws(at string, handler ApiWebsocketFunc) websocket.Handler {
	return websocket.Handler(func(ws *websocket.Conn) {
		log := logger.New("ns=kernel").At(at).Start()

		if !passwordCheck(ws.Request()) {
			ws.Write([]byte("ERROR: invalid authorization\n"))
			return
		}

		if !versionCheck(ws.Request()) {
			ws.Write([]byte("client outdated, please update with `convox update`\n"))
			return
		}

		err := handler(ws)

		if err != nil {
			ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err)))
			logError(log, err)
			return
		}

		log.Log("state=success")
	})
}
Example #3
0
func subscribeCloudWatchLogsStream(group, stream string, startTime time.Time, output chan []byte, quit chan bool) {
	log := logger.New("at=subscribe-cloudwatch").Start()
	fmt.Printf("subscribeCloudWatchLogsStream group=%s stream=%s startTime=%s\n", group, stream, startTime)

	startTimeMs := startTime.Unix() * 1000 // ms since epoch

	req := cloudwatchlogs.GetLogEventsInput{
		LogGroupName:  aws.String(group),
		LogStreamName: aws.String(stream),
	}

	for {
		select {
		case <-quit:
			log.Log("qutting")
			return
		default:
			req.StartTime = &startTimeMs

			res, err := CloudWatchLogs().GetLogEvents(&req)

			if err != nil {
				fmt.Printf("err3 %+v\n", err)
				return
			}

			for _, event := range res.Events {
				output <- []byte(fmt.Sprintf("%s\n", string(*event.Message)))
				startTimeMs = *event.Timestamp + 1
			}

			time.Sleep(1000 * time.Millisecond)
		}
	}
}
Example #4
0
// Get latest convergence state and send notifications
func monitorConverged(lastConverged bool, lastEventAt time.Time) (bool, ecs.ServiceEvent) {
	log := logger.New("ns=services_monitor")

	services, err := models.ClusterServices()

	if err != nil {
		log.Log("fn=monitorConverged err=%q", err)

		return lastConverged, ecs.ServiceEvent{
			CreatedAt: aws.Time(lastEventAt),
		}
	}

	converged := services.IsConverged()
	events := services.EventsSince(lastEventAt)

	log.Log("fn=monitorConverged converged=%t events=%d lastEventAt=%q", converged, len(events), lastEventAt)

	if events.HasCapacityWarning() {
		models.NotifyError("rack:capacity", fmt.Errorf(events.CapacityWarning()), map[string]string{
			"rack": os.Getenv("RACK"),
		})
	}

	if converged != lastConverged {
		models.NotifySuccess("rack:converge", map[string]string{
			"rack":      os.Getenv("RACK"),
			"converged": fmt.Sprintf("%t", converged),
		})
	}

	return converged, services.LastEvent()
}
Example #5
0
func api(at string, handler ApiHandlerFunc) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		log := logger.New("ns=kernel").At(at).Start()

		if !passwordCheck(r) {
			rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`)
			rw.WriteHeader(401)
			rw.Write([]byte("invalid authorization"))
			return
		}

		if !versionCheck(r) {
			rw.WriteHeader(403)
			rw.Write([]byte("client outdated, please update with `convox update`"))
			return
		}

		err := handler(rw, r)

		if err != nil {
			rw.WriteHeader(err.Code())
			RenderError(rw, err)
			logError(log, err)
			return
		}

		log.Log("state=success")
	}
}
Example #6
0
func Notify(name, status string, data map[string]string) error {
	if PauseNotifications {
		return nil
	}

	log := logger.New("ns=kernel")
	data["rack"] = os.Getenv("RACK")

	event := &client.NotifyEvent{
		Action:    name,
		Status:    status,
		Data:      data,
		Timestamp: time.Now().UTC(),
	}

	message, err := json.Marshal(event)
	if err != nil {
		return err
	}

	params := &sns.PublishInput{
		Message:   aws.String(string(message)), // Required
		Subject:   aws.String(name),
		TargetArn: aws.String(NotificationTopic),
	}
	resp, err := SNS().Publish(params)

	if err != nil {
		return err
	}

	log.At("Notfiy").Log("message-id=%q", *resp.MessageId)

	return nil
}
Example #7
0
func SNSProxy(w http.ResponseWriter, r *http.Request) {
	log := logger.New("ns=kernel").At("SNSProxy")

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	var payload map[string]string
	err = json.Unmarshal(body, &payload)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	url := r.FormValue("endpoint")
	resp, err := http.Post(url, "application/json", strings.NewReader(payload["Message"]))
	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	log.Log("proxied=true status=%s", resp.Status)
	w.Write([]byte("ok"))
}
Example #8
0
func StartCluster() {
	var log = logger.New("ns=cluster_monitor")

	defer recoverWith(func(err error) {
		helpers.Error(log, err)
	})

	for _ = range time.Tick(5 * time.Minute) {
		log.Log("tick")

		instances := Instances{}

		err := instances.describeASG()

		if err != nil {
			log.Error(err)
			continue
		}

		err = instances.describeECS()

		if err != nil {
			log.Error(err)
			continue
		}

		// TODO: Add an instances.testDocker() call to the mission critical path

		// Test if ASG Instance is registered and connected in ECS cluster
		for _, i := range instances {
			if !i.ASG {
				// TODO: Rogue instance?! Terminate?
				continue
			}

			if !i.ECS {
				// Not registered or not connected => set Unhealthy
				_, err := models.AutoScaling().SetInstanceHealth(
					&autoscaling.SetInstanceHealthInput{
						HealthStatus:             aws.String("Unhealthy"),
						InstanceId:               aws.String(i.Id),
						ShouldRespectGracePeriod: aws.Bool(true),
					},
				)

				i.Unhealthy = true

				if err != nil {
					log.Error(err)
					continue
				}
			}
		}

		log.Log(instances.log())
	}
}
Example #9
0
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer recoverWith(func(err error) {
		log := logger.New("ns=kernel").At("panic")
		helpers.Error(log, err)
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	})

	next(rw, r)
}
Example #10
0
func StartImages() {
	var log = logger.New("ns=app_images")

	if os.Getenv("DEVELOPMENT") == "true" {
		return
	}

	maxRetries := 5
	var err error

	for i := 0; i < maxRetries; i++ {
		err := dockerLogin()

		if err == nil {
			break
		}

		time.Sleep(30 * time.Second)
	}

	if err != nil {
		return
	}

	apps, err := models.ListApps()

	if err != nil {
		log.Error(err)
		return
	}

	for _, app := range apps {
		a, err := models.GetApp(app.Name)

		if err != nil {
			log.Error(err)
			continue
		}

		for key, value := range a.Parameters {
			if strings.HasSuffix(key, "Image") {

				log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
				data, err := exec.Command("docker", "pull", value).CombinedOutput()

				if err != nil {
					fmt.Printf("%+v\n", string(data))
					log.Error(err)
					continue
				}
			}
		}
	}
}
Example #11
0
func autoscaleRack() {
	log := logger.New("ns=workers.autoscale at=autoscaleRack")

	capacity, err := provider.CapacityGet()

	if err != nil {
		log.Log("fn=models.GetSystemCapacity err=%q", err)
		return
	}

	log.Log("autoscale=%t", autoscale)

	if !autoscale {
		return
	}

	system, err := provider.SystemGet()

	if err != nil {
		log.Log("fn=models.GetSystem err=%q", err)
		return
	}

	// calaculate instance requirements based on total process memory needed divided by the memory
	// on an individual instance
	instances := int(math.Ceil(float64(capacity.ProcessMemory) / float64(capacity.InstanceMemory)))

	// instance count cant be less than 2
	if instances < 2 {
		instances = 2
	}

	// instance count must be at least maxconcurrency+1
	if instances < (int(capacity.ProcessWidth) + 1) {
		instances = int(capacity.ProcessWidth) + 1
	}

	log.Log("process.memory=%d instance.memory=%d instances=%d change=%d", capacity.ProcessMemory, capacity.InstanceMemory, instances, (instances - system.Count))

	// if no change then exit
	if system.Count == instances {
		return
	}

	system.Count = instances

	err = provider.SystemSave(*system)

	if err != nil {
		log.Log("fn=system.Save err=%q", err)
		return
	}
}
Example #12
0
func StartHeartbeat() {
	log := logger.New("ns=heartbeat")
	defer recoverWith(func(err error) {
		helpers.Error(log, err)
	})

	helpers.TrackEvent("kernel-heartbeat", "")

	for _ = range time.Tick(1 * time.Hour) {
		helpers.TrackEvent("kernel-heartbeat", "")
	}
}
Example #13
0
func dockerLogin() error {
	var log = logger.New("ns=app_images")

	log.Log("cmd=%q", fmt.Sprintf("docker login -e [email protected] -u convox -p ***** %s", os.Getenv("REGISTRY_HOST")))
	data, err := exec.Command("docker", "login", "-e", "*****@*****.**", "-u", "convox", "-p", os.Getenv("PASSWORD"), os.Getenv("REGISTRY_HOST")).CombinedOutput()

	if err != nil {
		fmt.Printf("%+v\n", string(data))
		log.Error(err)
	}

	return err
}
Example #14
0
func subscribeKinesisShard(stream, shard string, output chan []byte, quit chan bool) {
	log := logger.New("at=subscribe-kinesis").Start()

	ireq := &kinesis.GetShardIteratorInput{
		ShardId:           aws.String(shard),
		ShardIteratorType: aws.String("LATEST"),
		StreamName:        aws.String(stream),
	}

	ires, err := Kinesis().GetShardIterator(ireq)

	if err != nil {
		log.Error(err)
		return
	}

	iter := *ires.ShardIterator

	for {
		select {
		case <-quit:
			log.Log("qutting")
			return
		default:
			greq := &kinesis.GetRecordsInput{
				ShardIterator: aws.String(iter),
			}
			gres, err := Kinesis().GetRecords(greq)

			if err != nil {
				fmt.Printf("err3 %+v\n", err)
				// panic(err)
				return
			}

			iter = *gres.NextShardIterator

			for _, record := range gres.Records {
				output <- []byte(fmt.Sprintf("%s\n", string(record.Data)))
			}

			time.Sleep(500 * time.Millisecond)
		}
	}
}
Example #15
0
// Get initial convergence state
func checkConverged() (bool, ecs.ServiceEvent) {
	log := logger.New("ns=services_monitor")

	services, err := models.ClusterServices()

	if err != nil {
		log.Log("fn=checkConverged err=%q", err)

		return true, ecs.ServiceEvent{
			CreatedAt: aws.Time(time.Now()),
		}
	}

	converged := services.IsConverged()
	lastEvent := services.LastEvent()

	log.Log("fn=checkConverged converged=%t lastEventAt=%q", converged, lastEvent.CreatedAt)

	return converged, lastEvent
}
Example #16
0
func SNSConfirm(w http.ResponseWriter, r *http.Request) {
	log := logger.New("ns=kernel").At("SNSConfirm")

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	var payload map[string]string
	err = json.Unmarshal(body, &payload)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	params := &sns.ConfirmSubscriptionInput{
		Token:    aws.String(payload["Token"]),
		TopicArn: aws.String(payload["TopicArn"]),
	}
	resp, err := models.SNS().ConfirmSubscription(params)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), 500)
		return
	}

	log.Log("confirmed=true subscriptionArn=%q", *resp.SubscriptionArn)
	w.Write([]byte("ok"))
}
Example #17
0
func New(ns string, ignore []string) *Nlogger {
	return &Nlogger{ignore: ignore, log: logger.New(ns)}
}
Example #18
0
// Set up rack instance Docker environment for builds
// Log into all configured private registries
// Pull down latest images for all apps
func StartImages() {
	var log = logger.New("ns=app_images")

	if os.Getenv("DEVELOPMENT") == "true" {
		return
	}

	// doing this in development updates a ~/.docker file and causes a rerun loop
	models.LoginRegistries()

	maxRetries := 5
	var err error

	for i := 0; i < maxRetries; i++ {
		err = models.DockerLogin(docker.AuthConfiguration{
			Email:         "*****@*****.**",
			Username:      "******",
			Password:      os.Getenv("PASSWORD"),
			ServerAddress: os.Getenv("REGISTRY_HOST"),
		})

		if err == nil {
			break
		}

		time.Sleep(30 * time.Second)
	}

	if err != nil {
		return
	}

	apps, err := models.ListApps()

	if err != nil {
		log.Error(err)
		return
	}

	for _, app := range apps {
		a, err := models.GetApp(app.Name)

		if err != nil {
			log.Error(err)
			continue
		}

		for key, value := range a.Parameters {
			if strings.HasSuffix(key, "Image") {

				log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
				data, err := exec.Command("docker", "pull", value).CombinedOutput()

				if err != nil {
					fmt.Printf("%+v\n", string(data))
					log.Error(err)
					continue
				}
			}
		}
	}
}
Example #19
0
func subscribeRDS(prefix, id string, output chan []byte, quit chan bool) {
	// Get latest log file details via pagination tokens
	details := rds.DescribeDBLogFilesDetails{}
	marker := ""
	log := logger.New("at=subscribe-kinesis").Start()

	for {
		params := &rds.DescribeDBLogFilesInput{
			DBInstanceIdentifier: aws.String(id),
			MaxRecords:           aws.Int64(100),
		}

		if marker != "" {
			params.Marker = aws.String(marker)
		}

		res, err := RDS().DescribeDBLogFiles(params)

		if err != nil {
			panic(err)
		}

		if res.Marker == nil {
			files := res.DescribeDBLogFiles
			details = *files[len(files)-1]

			break
		}

		marker = *res.Marker
	}

	// Get last 50 log lines
	params := &rds.DownloadDBLogFilePortionInput{
		DBInstanceIdentifier: aws.String(id),
		LogFileName:          aws.String(*details.LogFileName),
		NumberOfLines:        aws.Int64(50),
	}

	res, err := RDS().DownloadDBLogFilePortion(params)

	if err != nil {
		panic(err)
	}

	output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))

	params.Marker = aws.String(*res.Marker)

	for {
		select {
		case <-quit:
			log.Log("qutting")
			return
		default:
			res, err := RDS().DownloadDBLogFilePortion(params)

			if err != nil {
				panic(err)
			}

			if *params.Marker != *res.Marker {
				params.Marker = aws.String(*res.Marker)

				output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))
			}

			time.Sleep(1000 * time.Millisecond)
		}
	}
}