func pullAppImages() {
var log = logger.New("ns=app_images")
if os.Getenv("DEVELOPMENT") == "true" {
return
}
maxRetries := 5
var err error
for i := 0; i < maxRetries; i++ {
err := dockerLogin()
if err == nil {
break
}
time.Sleep(30 * time.Second)
}
if err != nil {
return
}
apps, err := models.ListApps()
if err != nil {
log.Error(err)
return
}
for _, app := range apps {
a, err := models.GetApp(app.Name)
if err != nil {
log.Error(err)
continue
}
for key, value := range a.Parameters {
if strings.HasSuffix(key, "Image") {
log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
data, err := exec.Command("docker", "pull", value).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
continue
}
}
}
}
}
func dockerLogin() error {
var log = logger.New("ns=app_images")
log.Log("cmd=%q", fmt.Sprintf("docker login -e [email protected] -u convox -p ***** %s", os.Getenv("REGISTRY_HOST")))
data, err := exec.Command("docker", "login", "-e", "*****@*****.**", "-u", "convox", "-p", os.Getenv("PASSWORD"), os.Getenv("REGISTRY_HOST")).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
}
return err
}
func pullAppImages() {
if os.Getenv("DEVELOPMENT") == "true" {
return
}
var log = logger.New("ns=app_images")
apps, err := models.ListApps()
if err != nil {
log.Error(err)
return
}
log.Log("cmd=%q", fmt.Sprintf("docker login -e [email protected] -u convox -p ***** %s", os.Getenv("REGISTRY_HOST")))
data, err := exec.Command("docker", "login", "-e", "*****@*****.**", "-u", "convox", "-p", os.Getenv("PASSWORD"), os.Getenv("REGISTRY_HOST")).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
return
}
for _, app := range apps {
a, err := models.GetApp(app.Name)
if err != nil {
log.Error(err)
continue
}
for key, value := range a.Parameters {
if strings.HasSuffix(key, "Image") {
log.Log("cmd=%q", fmt.Sprintf("docker pull %s", value))
data, err := exec.Command("docker", "pull", value).CombinedOutput()
if err != nil {
fmt.Printf("%+v\n", string(data))
log.Error(err)
continue
}
}
}
}
}
func processesLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=processes").At(at)
}
func releasesLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=releases").At(at)
}
func appsLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=apps").At(at)
}
func servicesLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=services").At(at)
}
func buildsLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=builds").At(at)
}
func startClusterMonitor() {
var log = logger.New("ns=cluster_monitor")
Tick:
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
// Ger Rack InstanceCount Parameter
instanceCount := 0
res, err := models.CloudFormation().DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String(os.Getenv("RACK")),
},
)
if err != nil {
log.Error(err)
continue
}
for _, p := range res.Stacks[0].Parameters {
if *p.ParameterKey == "InstanceCount" {
c, err := strconv.Atoi(*p.ParameterValue)
if err != nil {
log.Error(err)
break Tick
}
instanceCount = c
break
}
}
// List and Describe ECS Container Instances
ires, err := models.ECS().ListContainerInstances(
&ecs.ListContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
},
)
if err != nil {
log.Error(err)
continue
}
dres, err := models.ECS().DescribeContainerInstances(
&ecs.DescribeContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
ContainerInstances: ires.ContainerInstanceARNs,
},
)
if err != nil {
log.Error(err)
continue
}
cInstanceIds := make([]string, 0)
cInstanceConnections := make(map[string]bool)
for _, i := range dres.ContainerInstances {
cInstanceConnections[*i.EC2InstanceID] = *i.AgentConnected
if *i.AgentConnected {
cInstanceIds = append(cInstanceIds, *i.EC2InstanceID)
}
}
// Get and Describe Rack ASG Resource
resources, err := models.ListResources(os.Getenv("RACK"))
ares, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
aInstanceIds := []string{}
uInstanceIds := []string{}
for _, i := range ares.AutoScalingGroups[0].Instances {
if connected, exists := cInstanceConnections[*i.InstanceID]; connected && exists {
aInstanceIds = append(aInstanceIds, *i.InstanceID)
} else {
// Not registered or not connected => set Unhealthy
if *i.LifecycleState == "InService" {
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceID: aws.String(*i.InstanceID),
ShouldRespectGracePeriod: aws.Boolean(true),
},
)
if err != nil {
log.Error(err)
continue
}
uInstanceIds = append(uInstanceIds, *i.InstanceID)
}
}
}
sort.Strings(aInstanceIds)
sort.Strings(cInstanceIds)
sort.Strings(uInstanceIds)
log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
}
}
func New(ns string, ignore []string) *Nlogger {
return &Nlogger{ignore: ignore, log: logger.New(ns)}
}
func systemLogger(at string) *logger.Logger {
return logger.New("ns=kernel cn=system").At(at)
}