func ws(at string, handler ApiWebsocketFunc) websocket.Handler { return websocket.Handler(func(ws *websocket.Conn) { log := logger.New("ns=api.controllers").At(at).Start() if !passwordCheck(ws.Request()) { ws.Write([]byte("ERROR: invalid authorization\n")) return } if !versionCheck(ws.Request()) { ws.Write([]byte("client outdated, please update with `convox update`\n")) return } err := handler(ws) if err != nil { log.Error(err) ws.Write([]byte(fmt.Sprintf("ERROR: %v\n", err))) return } log.Success() }) }
func api(at string, handler ApiHandlerFunc) http.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request) { log := logger.New("ns=api.controllers").At(at).Start() if !passwordCheck(r) { log.Errorf("invalid authorization") rw.Header().Set("WWW-Authenticate", `Basic realm="Convox System"`) rw.WriteHeader(401) rw.Write([]byte("invalid authorization")) return } if !versionCheck(r) { log.Errorf("invalid version") rw.WriteHeader(403) rw.Write([]byte("client outdated, please update with `convox update`")) return } err := handler(rw, r) if err != nil { log.Error(err) rw.WriteHeader(err.Code()) RenderError(rw, err) return } log.Success() } }
func recovery(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { defer recoverWith(func(err error) { log := logger.New("ns=kernel").At("panic") helpers.Error(log, err) http.Error(rw, err.Error(), http.StatusInternalServerError) }) next(rw, r) }
func StartCluster() { var log = logger.New("ns=cluster_monitor") defer recoverWith(func(err error) { helpers.Error(log, err) }) for range time.Tick(5 * time.Minute) { log.Logf("tick") instances := Instances{} err := instances.describeASG() if err != nil { log.Error(err) continue } err = instances.describeECS() if err != nil { log.Error(err) continue } // Test if ASG Instance is registered and connected in ECS cluster for k, i := range instances { if !i.ASG { // TODO: Rogue instance?! Terminate? continue } if !i.ECS { // Not registered or not connected => set Unhealthy _, err := models.AutoScaling().SetInstanceHealth( &autoscaling.SetInstanceHealthInput{ HealthStatus: aws.String("Unhealthy"), InstanceId: aws.String(i.Id), ShouldRespectGracePeriod: aws.Bool(true), }, ) i.Unhealthy = true instances[k] = i if err != nil { log.Error(err) continue } // log for humans fmt.Printf("who=\"convox/monitor\" what=\"marked instance %s unhealthy\" why=\"ECS reported agent disconnected\"\n", i.Id) } } log.Logf(instances.log()) } }
func StartHeartbeat() { log := logger.New("ns=heartbeat") defer recoverWith(func(err error) { helpers.Error(log, err) }) heartbeat() for range time.Tick(1 * time.Hour) { heartbeat() } }
"github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sns" "github.com/convox/logger" ) var ( customTopic = os.Getenv("CUSTOM_TOPIC") notificationTopic = os.Getenv("NOTIFICATION_TOPIC") sortableTime = "20060102.150405.000000000" ) // Logger is a package-wide logger var Logger = logger.New("ns=provider.aws") type AWSProvider struct { Region string Endpoint string Access string Secret string Token string Cluster string Development bool DockerImageAPI string DynamoBuilds string DynamoReleases string NotificationHost string NotificationTopic string
func autoscaleRack() { log := logger.New("ns=workers.autoscale").At("autoscaleRack") // do nothing unless autoscaling is on if !autoscale { return } capacity, err := models.Provider().CapacityGet() if err != nil { log.Error(err) return } system, err := models.Provider().SystemGet() if err != nil { log.Error(err) return } log.Logf("status=%q", system.Status) // only allow running and converging status through switch system.Status { case "running", "converging": default: return } // start with the current count desired := 0 // calculate instances required to statisfy cpu reservations plus one for breathing room if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired { log = log.Replace("reason", "cpu") desired = c } // calculate instances required to statisfy memory reservations plus one for breathing room if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired { log = log.Replace("reason", "memory") desired = c } // instance count cant be less than 2 if desired < 2 { log = log.Replace("reason", "minimum") desired = 2 } // instance count must be at least maxconcurrency+1 if c := int(capacity.ProcessWidth) + 1; c > desired { log = log.Replace("reason", "width") desired = c } // if no change then exit if system.Count == desired { return } // ok to start multiple // when shutting down go one at a time but only if current status is "running" if desired < system.Count { if system.Status == "running" { system.Count-- } } else { system.Count = desired } log.Logf("change=%d", (desired - system.Count)) err = models.Provider().SystemSave(*system) if err != nil { log.Error(err) return } }
"time" "github.com/convox/logger" ) type Cache map[string]map[string]*CacheItem type CacheItem struct { Item interface{} Expires time.Time } var ( cache = Cache{} lock = sync.Mutex{} log = logger.New("ns=api.cache") ) func Get(collection string, key interface{}) interface{} { lock.Lock() defer lock.Unlock() if os.Getenv("PROVIDER") == "test" { return nil } hash, err := hashKey(key) if err != nil { log.Logf("fn=get collection=%q key=%q status=error error=%q", collection, hash, err) return nil
"github.com/aws/aws-sdk-go/service/acm" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sns" "github.com/convox/logger" ) // Logger is a package-wide logger var Logger = logger.New("ns=api.models") var SortableTime = "20060102.150405.000000000" func awsError(err error) string { if ae, ok := err.(awserr.Error); ok { return ae.Code() } return "" } func awsConfig() *aws.Config { config := &aws.Config{ Credentials: credentials.NewStaticCredentials(os.Getenv("AWS_ACCESS"), os.Getenv("AWS_SECRET"), ""), }
func New(ns string, ignore []string) *Nlogger { return &Nlogger{ignore: ignore, log: logger.New(ns)} }
func autoscaleRack() { log := logger.New("ns=workers.autoscale").At("autoscaleRack") // do nothing unless autoscaling is on if !autoscale { return } capacity, err := models.Provider().CapacityGet() if err != nil { log.Error(err) return } system, err := models.Provider().SystemGet() if err != nil { log.Error(err) return } log.Logf("status=%q", system.Status) if system.Status != "running" { return } // start with the current count desired := 0 // calculate instances required to statisfy cpu reservations plus one for breathing room if c := int(math.Ceil(float64(capacity.ProcessCPU)/float64(capacity.InstanceCPU))) + 1; c > desired { log = log.Replace("reason", "cpu") desired = c } // calculate instances required to statisfy memory reservations plus one for breathing room if c := int(math.Ceil(float64(capacity.ProcessMemory)/float64(capacity.InstanceMemory))) + 1; c > desired { log = log.Replace("reason", "memory") desired = c } // instance count cant be less than 2 if desired < 2 { log = log.Replace("reason", "minimum") desired = 2 } // instance count must be at least maxconcurrency+1 if c := int(capacity.ProcessWidth) + 1; c > desired { log = log.Replace("reason", "width") desired = c } // if no change then exit if system.Count == desired { return } log.Logf("change=%d", (desired - system.Count)) system.Count = desired err = models.Provider().SystemSave(*system) if err != nil { log.Error(err) return } }