Esempio n. 1
0
File: api.go Progetto: gmelika/rack
func ws(at string, handler ApiWebsocketFunc) websocket.Handler {
	return websocket.Handler(func(ws *websocket.Conn) {
		log := logger.New("ns=api.controllers").At(at).Start()

		if !passwordCheck(ws.Request()) {
			ws.Write([]byte("ERROR: invalid authorization\n"))
			return
		}

		if !versionCheck(ws.Request()) {
			ws.Write([]byte("client outdated, please update with `convox update`\n"))
			return
		}

		err := handler(ws)

		if err != nil {
			log.Error(err)
			ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err)))
			return
		}

		log.Success()
	})
}
Esempio n. 2
0
File: api.go Progetto: gmelika/rack
func api(at string, handler ApiHandlerFunc) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		log := logger.New("ns=api.controllers").At(at).Start()

		if !passwordCheck(r) {
			log.Errorf("invalid authorization")
			rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`)
			rw.WriteHeader(401)
			rw.Write([]byte("invalid authorization"))
			return
		}

		if !versionCheck(r) {
			log.Errorf("invalid version")
			rw.WriteHeader(403)
			rw.Write([]byte("client outdated, please update with `convox update`"))
			return
		}

		err := handler(rw, r)

		if err != nil {
			log.Error(err)
			rw.WriteHeader(err.Code())
			RenderError(rw, err)
			return
		}

		log.Success()
	}
}
Esempio n. 3
0
File: web.go Progetto: gmelika/rack
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer recoverWith(func(err error) {
		log := logger.New("ns=kernel").At("panic")
		helpers.Error(log, err)
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	})

	next(rw, r)
}
Esempio n. 4
0
func StartCluster() {
	var log = logger.New("ns=cluster_monitor")

	defer recoverWith(func(err error) {
		helpers.Error(log, err)
	})

	for range time.Tick(5 * time.Minute) {
		log.Logf("tick")

		instances := Instances{}

		err := instances.describeASG()
		if err != nil {
			log.Error(err)
			continue
		}

		err = instances.describeECS()
		if err != nil {
			log.Error(err)
			continue
		}

		// Test if ASG Instance is registered and connected in ECS cluster
		for k, i := range instances {
			if !i.ASG {
				// TODO: Rogue instance?! Terminate?
				continue
			}

			if !i.ECS {
				// Not registered or not connected => set Unhealthy
				_, err := models.AutoScaling().SetInstanceHealth(
					&autoscaling.SetInstanceHealthInput{
						HealthStatus:             aws.String("Unhealthy"),
						InstanceId:               aws.String(i.Id),
						ShouldRespectGracePeriod: aws.Bool(true),
					},
				)

				i.Unhealthy = true
				instances[k] = i

				if err != nil {
					log.Error(err)
					continue
				}

				// log for humans
				fmt.Printf("who=\"convox/monitor\" what=\"marked instance %s unhealthy\" why=\"ECS reported agent disconnected\"\n", i.Id)
			}
		}

		log.Logf(instances.log())
	}
}
Esempio n. 5
0
func StartHeartbeat() {
	log := logger.New("ns=heartbeat")

	defer recoverWith(func(err error) {
		helpers.Error(log, err)
	})

	heartbeat()

	for range time.Tick(1 * time.Hour) {
		heartbeat()
	}
}
Esempio n. 6
0
File: aws.go Progetto: gmelika/rack
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/convox/logger"
)

var (
	customTopic       = os.Getenv("CUSTOM_TOPIC")
	notificationTopic = os.Getenv("NOTIFICATION_TOPIC")
	sortableTime      = "20060102.150405.000000000"
)

// Logger is a package-wide logger
var Logger = logger.New("ns=provider.aws")

type AWSProvider struct {
	Region   string
	Endpoint string
	Access   string
	Secret   string
	Token    string

	Cluster           string
	Development       bool
	DockerImageAPI    string
	DynamoBuilds      string
	DynamoReleases    string
	NotificationHost  string
	NotificationTopic string
Esempio n. 7
0
func autoscaleRack() {
	log := logger.New("ns=workers.autoscale").At("autoscaleRack")

	// do nothing unless autoscaling is on
	if !autoscale {
		return
	}

	capacity, err := models.Provider().CapacityGet()
	if err != nil {
		log.Error(err)
		return
	}

	system, err := models.Provider().SystemGet()
	if err != nil {
		log.Error(err)
		return
	}

	log.Logf("status=%q", system.Status)

	// only allow running and converging status through
	switch system.Status {
	case "running", "converging":
	default:
		return
	}

	// start with the current count
	desired := 0

	// calculate instances required to statisfy cpu reservations plus one for breathing room
	if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired {
		log = log.Replace("reason", "cpu")
		desired = c
	}

	// calculate instances required to statisfy memory reservations plus one for breathing room
	if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired {
		log = log.Replace("reason", "memory")
		desired = c
	}

	// instance count cant be less than 2
	if desired < 2 {
		log = log.Replace("reason", "minimum")
		desired = 2
	}

	// instance count must be at least maxconcurrency+1
	if c := int(capacity.ProcessWidth) + 1; c > desired {
		log = log.Replace("reason", "width")
		desired = c
	}

	// if no change then exit
	if system.Count == desired {
		return
	}

	// ok to start multiple
	// when shutting down go one at a time but only if current status is "running"
	if desired < system.Count {
		if system.Status == "running" {
			system.Count--
		}
	} else {
		system.Count = desired
	}

	log.Logf("change=%d", (desired - system.Count))

	err = models.Provider().SystemSave(*system)
	if err != nil {
		log.Error(err)
		return
	}
}
Esempio n. 8
0
	"time"

	"github.com/convox/logger"
)

type Cache map[string]map[string]*CacheItem

type CacheItem struct {
	Item    interface{}
	Expires time.Time
}

var (
	cache = Cache{}
	lock  = sync.Mutex{}
	log   = logger.New("ns=api.cache")
)

func Get(collection string, key interface{}) interface{} {
	lock.Lock()
	defer lock.Unlock()

	if os.Getenv("PROVIDER") == "test" {
		return nil
	}

	hash, err := hashKey(key)

	if err != nil {
		log.Logf("fn=get collection=%q key=%q status=error error=%q", collection, hash, err)
		return nil
Esempio n. 9
0
	"github.com/aws/aws-sdk-go/service/acm"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/convox/logger"
)

// Logger is a package-wide logger
var Logger = logger.New("ns=api.models")

var SortableTime = "20060102.150405.000000000"

func awsError(err error) string {
	if ae, ok := err.(awserr.Error); ok {
		return ae.Code()
	}

	return ""
}

func awsConfig() *aws.Config {
	config := &aws.Config{
		Credentials: credentials.NewStaticCredentials(os.Getenv("AWS_ACCESS"), os.Getenv("AWS_SECRET"), ""),
	}
Esempio n. 10
0
func New(ns string, ignore []string) *Nlogger {
	return &Nlogger{ignore: ignore, log: logger.New(ns)}
}
Esempio n. 11
0
func autoscaleRack() {
	log := logger.New("ns=workers.autoscale").At("autoscaleRack")

	// do nothing unless autoscaling is on
	if !autoscale {
		return
	}

	capacity, err := models.Provider().CapacityGet()
	if err != nil {
		log.Error(err)
		return
	}

	system, err := models.Provider().SystemGet()
	if err != nil {
		log.Error(err)
		return
	}

	log.Logf("status=%q", system.Status)
	if system.Status != "running" {
		return
	}

	// start with the current count
	desired := 0

	// calculate instances required to statisfy cpu reservations plus one for breathing room
	if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired {
		log = log.Replace("reason", "cpu")
		desired = c
	}

	// calculate instances required to statisfy memory reservations plus one for breathing room
	if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired {
		log = log.Replace("reason", "memory")
		desired = c
	}

	// instance count cant be less than 2
	if desired < 2 {
		log = log.Replace("reason", "minimum")
		desired = 2
	}

	// instance count must be at least maxconcurrency+1
	if c := int(capacity.ProcessWidth) + 1; c > desired {
		log = log.Replace("reason", "width")
		desired = c
	}

	// if no change then exit
	if system.Count == desired {
		return
	}

	log.Logf("change=%d", (desired - system.Count))

	system.Count = desired

	err = models.Provider().SystemSave(*system)
	if err != nil {
		log.Error(err)
		return
	}
}