func ws(at string, handler ApiWebsocketFunc) websocket.Handler {
return websocket.Handler(func(ws *websocket.Conn) {
log := logger.New("ns=api.controllers").At(at).Start()
if !passwordCheck(ws.Request()) {
ws.Write([]byte("ERROR: invalid authorization\n"))
return
}
if !versionCheck(ws.Request()) {
ws.Write([]byte("client outdated, please update with `convox update`\n"))
return
}
err := handler(ws)
if err != nil {
log.Error(err)
ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err)))
return
}
log.Success()
})
}
func api(at string, handler ApiHandlerFunc) http.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) {
log := logger.New("ns=api.controllers").At(at).Start()
if !passwordCheck(r) {
log.Errorf("invalid authorization")
rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`)
rw.WriteHeader(401)
rw.Write([]byte("invalid authorization"))
return
}
if !versionCheck(r) {
log.Errorf("invalid version")
rw.WriteHeader(403)
rw.Write([]byte("client outdated, please update with `convox update`"))
return
}
err := handler(rw, r)
if err != nil {
log.Error(err)
rw.WriteHeader(err.Code())
RenderError(rw, err)
return
}
log.Success()
}
}
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
defer recoverWith(func(err error) {
log := logger.New("ns=kernel").At("panic")
helpers.Error(log, err)
http.Error(rw, err.Error(), http.StatusInternalServerError)
})
next(rw, r)
}
func StartCluster() {
var log = logger.New("ns=cluster_monitor")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
for range time.Tick(5 * time.Minute) {
log.Logf("tick")
instances := Instances{}
err := instances.describeASG()
if err != nil {
log.Error(err)
continue
}
err = instances.describeECS()
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
for k, i := range instances {
if !i.ASG {
// TODO: Rogue instance?! Terminate?
continue
}
if !i.ECS {
// Not registered or not connected => set Unhealthy
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceId: aws.String(i.Id),
ShouldRespectGracePeriod: aws.Bool(true),
},
)
i.Unhealthy = true
instances[k] = i
if err != nil {
log.Error(err)
continue
}
// log for humans
fmt.Printf("who=\"convox/monitor\" what=\"marked instance %s unhealthy\" why=\"ECS reported agent disconnected\"\n", i.Id)
}
}
log.Logf(instances.log())
}
}
func StartHeartbeat() {
log := logger.New("ns=heartbeat")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
heartbeat()
for range time.Tick(1 * time.Hour) {
heartbeat()
}
}
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/sns"
"github.com/convox/logger"
)
var (
customTopic = os.Getenv("CUSTOM_TOPIC")
notificationTopic = os.Getenv("NOTIFICATION_TOPIC")
sortableTime = "20060102.150405.000000000"
)
// Logger is a package-wide logger
var Logger = logger.New("ns=provider.aws")
type AWSProvider struct {
Region string
Endpoint string
Access string
Secret string
Token string
Cluster string
Development bool
DockerImageAPI string
DynamoBuilds string
DynamoReleases string
NotificationHost string
NotificationTopic string
func autoscaleRack() {
log := logger.New("ns=workers.autoscale").At("autoscaleRack")
// do nothing unless autoscaling is on
if !autoscale {
return
}
capacity, err := models.Provider().CapacityGet()
if err != nil {
log.Error(err)
return
}
system, err := models.Provider().SystemGet()
if err != nil {
log.Error(err)
return
}
log.Logf("status=%q", system.Status)
// only allow running and converging status through
switch system.Status {
case "running", "converging":
default:
return
}
// start with the current count
desired := 0
// calculate instances required to statisfy cpu reservations plus one for breathing room
if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired {
log = log.Replace("reason", "cpu")
desired = c
}
// calculate instances required to statisfy memory reservations plus one for breathing room
if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired {
log = log.Replace("reason", "memory")
desired = c
}
// instance count cant be less than 2
if desired < 2 {
log = log.Replace("reason", "minimum")
desired = 2
}
// instance count must be at least maxconcurrency+1
if c := int(capacity.ProcessWidth) + 1; c > desired {
log = log.Replace("reason", "width")
desired = c
}
// if no change then exit
if system.Count == desired {
return
}
// ok to start multiple
// when shutting down go one at a time but only if current status is "running"
if desired < system.Count {
if system.Status == "running" {
system.Count--
}
} else {
system.Count = desired
}
log.Logf("change=%d", (desired - system.Count))
err = models.Provider().SystemSave(*system)
if err != nil {
log.Error(err)
return
}
}
"time"
"github.com/convox/logger"
)
type Cache map[string]map[string]*CacheItem
type CacheItem struct {
Item interface{}
Expires time.Time
}
var (
cache = Cache{}
lock = sync.Mutex{}
log = logger.New("ns=api.cache")
)
func Get(collection string, key interface{}) interface{} {
lock.Lock()
defer lock.Unlock()
if os.Getenv("PROVIDER") == "test" {
return nil
}
hash, err := hashKey(key)
if err != nil {
log.Logf("fn=get collection=%q key=%q status=error error=%q", collection, hash, err)
return nil
"github.com/aws/aws-sdk-go/service/acm"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/sns"
"github.com/convox/logger"
)
// Logger is a package-wide logger
var Logger = logger.New("ns=api.models")
var SortableTime = "20060102.150405.000000000"
func awsError(err error) string {
if ae, ok := err.(awserr.Error); ok {
return ae.Code()
}
return ""
}
func awsConfig() *aws.Config {
config := &aws.Config{
Credentials: credentials.NewStaticCredentials(os.Getenv("AWS_ACCESS"), os.Getenv("AWS_SECRET"), ""),
}
func New(ns string, ignore []string) *Nlogger {
return &Nlogger{ignore: ignore, log: logger.New(ns)}
}
func autoscaleRack() {
log := logger.New("ns=workers.autoscale").At("autoscaleRack")
// do nothing unless autoscaling is on
if !autoscale {
return
}
capacity, err := models.Provider().CapacityGet()
if err != nil {
log.Error(err)
return
}
system, err := models.Provider().SystemGet()
if err != nil {
log.Error(err)
return
}
log.Logf("status=%q", system.Status)
if system.Status != "running" {
return
}
// start with the current count
desired := 0
// calculate instances required to statisfy cpu reservations plus one for breathing room
if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired {
log = log.Replace("reason", "cpu")
desired = c
}
// calculate instances required to statisfy memory reservations plus one for breathing room
if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired {
log = log.Replace("reason", "memory")
desired = c
}
// instance count cant be less than 2
if desired < 2 {
log = log.Replace("reason", "minimum")
desired = 2
}
// instance count must be at least maxconcurrency+1
if c := int(capacity.ProcessWidth) + 1; c > desired {
log = log.Replace("reason", "width")
desired = c
}
// if no change then exit
if system.Count == desired {
return
}
log.Logf("change=%d", (desired - system.Count))
system.Count = desired
err = models.Provider().SystemSave(*system)
if err != nil {
log.Error(err)
return
}
}