func GetPrices() { // prevent shadowing var err error log.Info("Downloading prices") pricesMap, err = prices.DownloadPricesMap(prices.Ec2PricingUrl) if err != nil { log.Error(fmt.Sprintf("Error getting prices: %s", err.Error())) return } log.Info("Successfully downloaded prices") }
func getVolumes() chan *reaperaws.Volume { ch := make(chan *reaperaws.Volume) go func() { volumeCh := reaperaws.AllVolumes() regionSums := make(map[reapable.Region]int) volumeSizeSums := make(map[reapable.Region]map[int64]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for volume := range volumeCh { // make the map if it is not initialized if volumeSizeSums[volume.Region()] == nil { volumeSizeSums[volume.Region()] = make(map[int64]int) } regionSums[volume.Region()]++ if isWhitelisted(volume) { whitelistedCount[volume.Region()]++ } volumeSizeSums[volume.Region()][*volume.Size]++ if matchesFilters(volume) { filteredCount[volume.Region()]++ } ch <- volume } for region, sum := range regionSums { log.Info("Found %d total volumes in %s", sum, region) } go func() { for region, regionMap := range volumeSizeSums { for volumeType, volumeSizeSum := range regionMap { err := reaperevents.NewStatistic("reaper.volumes.total", float64(volumeSizeSum), []string{fmt.Sprintf("region:%s,volumesize:%d", region, volumeType)}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.volumes.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s,volumesize:%d", region, volumeType)}) if err != nil { log.Error(err.Error()) } } err := reaperevents.NewStatistic("reaper.volumes.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region)}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
func registerReapable(a reaperevents.Reapable) { // update the internal state if time.Now().After(a.ReaperState().Until) { // if we updated the state, mark it as having been updated a.SetUpdated(a.IncrementState()) } log.Info("Reapable resource discovered: %s.", a.ReapableDescription()) reapables.Put(a.Region(), a.ID(), a) }
func main() { // config and events are vars in the reaper package // they NEED to be set before a reaper.Reaper can be initialized reaper.SetConfig(&config) reaperevents.SetEvents(&eventReporters) if config.DryRun { log.Info("Dry run mode enabled, no events will be triggered. Enable Extras in Notifications for per-event DryRun notifications.") reaperevents.SetDryRun(config.DryRun) } // Ready() NEEDS to be called after BOTH SetConfig() and SetEvents() // it uses those values to set individual EventReporter config values // and to init the Reapables map reaper.Ready() // sets the config variable in Reaper's AWS package // this also NEEDS to be set before a Reaper can be started reaperaws.SetConfig(&config.AWS) // single instance of reaper reapRunner := reaper.NewReaper() // Run the reaper process reapRunner.Start() // run the HTTP server api := reaper.NewHTTPApi(config.HTTP) if err := api.Serve(); err != nil { log.Error(err.Error()) } else { // HTTP server successfully started c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill) // waiting for an Interrupt or Kill signal // this channel blocks until it receives one sig := <-c log.Info("Got signal %s, stopping services", sig.String()) log.Info("Stopping HTTP") api.Stop() log.Info("Stopping reaper runner") reapRunner.Stop() } }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *AutoScalingGroup) Terminate() (bool, error) { log.Info("Terminating AutoScalingGroup %s", a.ReapableDescriptionTiny()) as := autoscaling.New(session.New(&aws.Config{Region: aws.String(a.Region().String())})) input := &autoscaling.DeleteAutoScalingGroupInput{ AutoScalingGroupName: aws.String(a.ID().String()), } _, err := as.DeleteAutoScalingGroup(input) if err != nil { log.Error("could not delete AutoScalingGroup ", a.ReapableDescriptionTiny()) return false, err } return true, nil }
// newReapableEvent is a method of EventReporter func (e *ReaperEvent) newReapableEvent(r Reapable, tags []string) error { if e.Config.shouldTriggerFor(r) { var err error switch e.Config.Mode { case "Stop": _, err = r.Stop() log.Info("ReaperEvent: Stopping ", r.ReapableDescriptionShort()) NewEvent("Reaper: Stopping ", r.ReapableDescriptionShort(), nil, []string{}) NewCountStatistic("reaper.reapables.stopped", []string{r.ReapableDescriptionTiny()}) case "Terminate": _, err = r.Terminate() log.Info("ReaperEvent: Terminating ", r.ReapableDescriptionShort()) NewEvent("Reaper: Terminating ", r.ReapableDescriptionShort(), nil, []string{}) NewCountStatistic("reaper.reapables.terminated", []string{r.ReapableDescriptionTiny()}) default: log.Error(fmt.Sprintf("Invalid %s Mode %s", e.Config.Name, e.Config.Mode)) } if err != nil { return err } } return nil }
// newReapableEvent is a method of EventReporter func (e *Tagger) newReapableEvent(r Reapable, tags []string) error { if r.ReaperState().Until.IsZero() { log.Warning("Uninitialized time value for %s!", r.ReapableDescription()) } if e.Config.shouldTriggerFor(r) { log.Info("Tagging %s with %s", r.ReapableDescriptionTiny(), r.ReaperState().State.String()) _, err := r.Save(r.ReaperState()) if err != nil { return err } } return nil }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *SecurityGroup) Terminate() (bool, error) { log.Info("Terminating SecurityGroup ", a.ReapableDescriptionTiny()) api := ec2.New(sess, aws.NewConfig().WithRegion(string(a.Region()))) input := &ec2.DeleteSecurityGroupInput{ GroupName: aws.String(a.ID().String()), } _, err := api.DeleteSecurityGroup(input) if err != nil { log.Error("could not delete SecurityGroup ", a.ReapableDescriptionTiny()) return false, err } return false, nil }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *Cloudformation) Terminate() (bool, error) { log.Info("Terminating Cloudformation %s", a.ReapableDescriptionTiny()) as := cloudformation.New(sess, aws.NewConfig().WithRegion(a.Region().String())) input := &cloudformation.DeleteStackInput{ StackName: aws.String(a.ID().String()), } _, err := as.DeleteStack(input) if err != nil { log.Error("could not delete Cloudformation ", a.ReapableDescriptionTiny()) return false, err } return false, nil }
func getSecurityGroups() chan *reaperaws.SecurityGroup { ch := make(chan *reaperaws.SecurityGroup) go func() { securityGroupCh := reaperaws.AllSecurityGroups() regionSums := make(map[reapable.Region]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for sg := range securityGroupCh { regionSums[sg.Region()]++ if isWhitelisted(sg) { whitelistedCount[sg.Region()]++ } if matchesFilters(sg) { filteredCount[sg.Region()]++ } ch <- sg } for region, sum := range regionSums { log.Info("Found %d total SecurityGroups in %s", sum, region) } go func() { for region, regionSum := range regionSums { err := reaperevents.NewStatistic("reaper.securitygroups.total", float64(regionSum), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.securitygroups.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.securitygroups.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
func (a *AutoScalingGroup) scaleToSize(size int64, minSize int64) (bool, error) { log.Info("Scaling AutoScalingGroup %s to size %d.", a.ReapableDescriptionTiny(), size) as := autoscaling.New(session.New(&aws.Config{Region: aws.String(a.Region().String())})) input := &autoscaling.UpdateAutoScalingGroupInput{ AutoScalingGroupName: aws.String(a.ID().String()), DesiredCapacity: &size, MinSize: &minSize, } _, err := as.UpdateAutoScalingGroup(input) if err != nil { log.Error("could not update AutoScalingGroup ", a.ReapableDescriptionTiny()) return false, err } return true, nil }
// cloudformationResources returns a chan of CloudformationResources, sourced from the AWS API // there is rate limiting in the AWS API for CloudformationResources, so we delay // this is skippable with the CLI flag -withoutCloudformationResources func cloudformationResources(region, id string) chan *cloudformation.StackResource { ch := make(chan *cloudformation.StackResource) if config.WithoutCloudformationResources { close(ch) return ch } api := cloudformation.New(sess, aws.NewConfig().WithRegion(region)) go func() { <-timeout // this query can fail, so we retry didRetry := false input := &cloudformation.DescribeStackResourcesInput{StackName: &id} // initial query resp, err := api.DescribeStackResources(input) for err != nil { sleepTime := 2*time.Second + time.Duration(rand.Intn(2000))*time.Millisecond if err != nil { // this error is annoying and will come up all the time... so you can disable it if strings.Split(err.Error(), ":")[0] == "Throttling" && log.Extras() { log.Warning("StackResources: %s (retrying %s after %ds)", err.Error(), id, sleepTime*1.0/time.Second) } else if strings.Split(err.Error(), ":")[0] != "Throttling" { // any other errors log.Error(fmt.Sprintf("StackResources: %s (retrying %s after %ds)", err.Error(), id, sleepTime*1.0/time.Second)) } } // wait a random amount of time... hopefully long enough to beat rate limiting time.Sleep(sleepTime) // retry query resp, err = api.DescribeStackResources(input) didRetry = true } if didRetry && log.Extras() { log.Info("Retry succeeded for %s!", id) } for _, resource := range resp.StackResources { ch <- resource } close(ch) }() return ch }
func getCloudformations() chan *reaperaws.Cloudformation { ch := make(chan *reaperaws.Cloudformation) go func() { cfs := reaperaws.AllCloudformations() regionSums := make(map[reapable.Region]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for cf := range cfs { if isWhitelisted(cf) { whitelistedCount[cf.Region()]++ } regionSums[cf.Region()]++ if matchesFilters(cf) { filteredCount[cf.Region()]++ } ch <- cf } for region, sum := range regionSums { log.Info("Found %d total Cloudformation Stacks in %s", sum, region) } go func() { for region, regionSum := range regionSums { err := reaperevents.NewStatistic("reaper.cloudformations.total", float64(regionSum), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.cloudformations.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.cloudformations.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
// Whitelist is a method of reapable.Whitelistable, which is embedded in reapable.Reapable func (a *AutoScalingGroup) Whitelist() (bool, error) { log.Info("Whitelisting AutoScalingGroup %s", a.ReapableDescriptionTiny()) api := autoscaling.New(session.New(&aws.Config{Region: aws.String(a.Region().String())})) createreq := &autoscaling.CreateOrUpdateTagsInput{ Tags: []*autoscaling.Tag{ &autoscaling.Tag{ ResourceId: aws.String(a.ID().String()), ResourceType: aws.String("auto-scaling-group"), PropagateAtLaunch: aws.Bool(false), Key: aws.String(config.WhitelistTag), Value: aws.String("true"), }, }, } _, err := api.CreateOrUpdateTags(createreq) return err == nil, err }
// this is a copy of the method from events.go EXCEPT // that it triggers whether or not the state was updated this run func (e *ReaperEventConfig) shouldTriggerFor(r Reapable) bool { if e.DryRun { if log.Extras() { log.Info("DryRun: Not triggering %s for %s", e.Name, r.ReapableDescriptionTiny()) } return false } triggering := false // if the reapable's state is set to trigger this EventReporter for _, trigger := range e.parseTriggers() { // if the reapable's state should trigger this event if trigger == r.ReaperState().State { triggering = true } } return triggering }
// newEvent is a method of EventReporter // newEvent reports an event to Datadog func (e *DatadogEvents) newEvent(title string, text string, fields map[string]string, tags []string) error { if e.Config.DryRun { if log.Extras() { log.Info("DryRun: Not reporting %s", title) } return nil } g, err := e.godspeed() if err != nil { return err } err = g.Event(title, text, fields, tags) if err != nil { return err } return nil }
// Stop is a method of reapable.Stoppable, which is embedded in reapable.Reapable func (a *Instance) Stop() (bool, error) { log.Info("Stopping Instance %s", a.ReapableDescriptionTiny()) api := ec2.New(sess, aws.NewConfig().WithRegion(string(a.Region()))) req := &ec2.StopInstancesInput{ InstanceIds: []*string{aws.String(a.ID().String())}, } resp, err := api.StopInstances(req) if err != nil { return false, err } if len(resp.StoppingInstances) != 1 { return false, fmt.Errorf("Instance %s could not be stopped.", a.ReapableDescriptionTiny()) } return true, nil }
func tagAutoScalingGroup(region reapable.Region, id reapable.ID, key, value string) (bool, error) { log.Info("Tagging AutoScalingGroup %s in %s with %s:%s", region.String(), id.String(), key, value) api := autoscaling.New(session.New(&aws.Config{Region: aws.String(region.String())})) createreq := &autoscaling.CreateOrUpdateTagsInput{ Tags: []*autoscaling.Tag{ &autoscaling.Tag{ ResourceId: aws.String(string(id)), ResourceType: aws.String("auto-scaling-group"), PropagateAtLaunch: aws.Bool(false), Key: aws.String(key), Value: aws.String(value), }, }, } _, err := api.CreateOrUpdateTags(createreq) if err != nil { return false, err } return true, nil }
// IncrementState updates the ReaperState of a Resource // returns a boolean of whether it was updated func (a *Resource) IncrementState() (updated bool) { var newState state.StateEnum until := time.Now() switch a.reaperState.State { default: fallthrough case state.InitialState: // set state to the FirstState newState = state.FirstState until = until.Add(config.Notifications.FirstStateDuration.Duration) case state.FirstState: // go to SecondState at the end of FirstState newState = state.SecondState until = until.Add(config.Notifications.SecondStateDuration.Duration) case state.SecondState: // go to ThirdState at the end of SecondState newState = state.ThirdState until = until.Add(config.Notifications.ThirdStateDuration.Duration) case state.ThirdState: // go to FinalState at the end of ThirdState newState = state.FinalState case state.FinalState: // keep same state newState = state.FinalState case state.IgnoreState: // keep same state newState = state.IgnoreState } if newState != a.reaperState.State { updated = true a.reaperState = state.NewStateWithUntilAndState(until, newState) log.Info("Updating state for %s. New state: %s.", a.ReapableDescriptionTiny(), newState.String()) } return updated }
// Unsave is a method of reapable.Saveable, which is embedded in reapable.Reapable // Unsave untags a Resource's reaperTag func (a *Resource) Unsave() (bool, error) { log.Info("Unsaving %s", a.ReapableDescriptionTiny()) return untag(a.Region().String(), a.ID().String(), reaperTag) }
func getInstances() chan *reaperaws.Instance { ch := make(chan *reaperaws.Instance) go func() { instanceCh := reaperaws.AllInstances() regionSums := make(map[reapable.Region]int) instanceTypeSums := make(map[reapable.Region]map[string]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for instance := range instanceCh { // make the map if it is not initialized if instanceTypeSums[instance.Region()] == nil { instanceTypeSums[instance.Region()] = make(map[string]int) } // don't count terminated or stopped instances if !instance.Terminated() && !instance.Stopped() { // increment InstanceType counter instanceTypeSums[instance.Region()][*instance.InstanceType]++ if isWhitelisted(instance) { whitelistedCount[instance.Region()]++ } } regionSums[instance.Region()]++ if matchesFilters(instance) { filteredCount[instance.Region()]++ } ch <- instance } for region, sum := range regionSums { log.Info("Found %d total Instances in %s", sum, region) } go func() { for region, regionMap := range instanceTypeSums { for instanceType, instanceTypeSum := range regionMap { if pricesMap != nil { price, ok := pricesMap[string(region)][instanceType] if ok { priceFloat, err := strconv.ParseFloat(price, 64) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.instances.totalcost", float64(instanceTypeSum)*priceFloat, []string{fmt.Sprintf("region:%s,instancetype:%s", region, instanceType), config.EventTag}) if err != nil { log.Error(err.Error()) } } else { // some instance types are priceless log.Error(fmt.Sprintf("No price for %s", instanceType)) } } err := reaperevents.NewStatistic("reaper.instances.total", float64(instanceTypeSum), []string{fmt.Sprintf("region:%s,instancetype:%s", region, instanceType), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.instances.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s,instancetype:%s", region, instanceType), config.EventTag}) if err != nil { log.Error(err.Error()) } } err := reaperevents.NewStatistic("reaper.instances.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
// Unsave is part of reapable.Saveable, which embedded in reapable.Reapable func (a *AutoScalingGroup) Unsave() (bool, error) { log.Info("Unsaving %s", a.ReapableDescriptionTiny()) return untagAutoScalingGroup(a.Region(), a.ID(), reaperTag) }
func getAutoScalingGroups() chan *reaperaws.AutoScalingGroup { ch := make(chan *reaperaws.AutoScalingGroup) go func() { asgCh := reaperaws.AllAutoScalingGroups() regionSums := make(map[reapable.Region]int) asgSizeSums := make(map[reapable.Region]map[int64]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for asg := range asgCh { // make the map if it is not initialized if asgSizeSums[asg.Region()] == nil { asgSizeSums[asg.Region()] = make(map[int64]int) } if asg.DesiredCapacity != nil { asgSizeSums[asg.Region()][*asg.DesiredCapacity]++ } if isWhitelisted(asg) { whitelistedCount[asg.Region()]++ } regionSums[asg.Region()]++ if matchesFilters(asg) { filteredCount[asg.Region()]++ } ch <- asg } for region, sum := range regionSums { log.Info("Found %d total AutoScalingGroups in %s", sum, region) } go func() { for region, regionMap := range asgSizeSums { for asgSize, asgSizeSum := range regionMap { err := reaperevents.NewStatistic("reaper.asgs.asgsizes", float64(asgSizeSum), []string{fmt.Sprintf("region:%s,asgsize:%d", region, asgSize), config.EventTag}) if err != nil { log.Error(err.Error()) } } } for region, regionSum := range regionSums { err := reaperevents.NewStatistic("reaper.asgs.total", float64(regionSum), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.asgs.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.asgs.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
// Run handles all reaping logic // conforms to the cron.Job interface func (r *Reaper) Run() { r.reap() // this is no longer true, but is roughly accurate log.Info("Sleeping for %s", config.Notifications.Interval.Duration.String()) }
// Unsave is part of reapable.Saveable, which embedded in reapable.Reapable // no op because we cannot tag cloudformations without updating the stack func (a *Cloudformation) Unsave() (bool, error) { log.Info("Unsaving %s", a.ReapableDescriptionTiny()) return false, nil }
// Save is a method of reapable.Saveable, which is embedded in reapable.Reapable // Save tags a Resource's reaperTag func (a *Resource) Save(reaperState *state.State) (bool, error) { log.Info("Saving %s", a.ReapableDescriptionTiny()) return tag(a.Region().String(), a.ID().String(), reaperTag, reaperState.String()) }
func init() { configFile := flag.String("config", "", "path to config file") withoutCloudformationResources := flag.Bool("withoutCloudformationResources", false, "disables dependency checking for Cloudformations (which is slow!)") useMozlog := flag.Bool("useMozlog", true, "set to false to disable mozlog output") flag.Parse() if *useMozlog { log.EnableMozlog() } // if no config file -> exit with error if *configFile == "" { log.Error("Config file is a required Argument. Specify with -config='filename'") os.Exit(1) } // if config file path specified, attempt to load it if c, err := reaper.LoadConfig(*configFile); err == nil { // catches panics loading config defer func() { if r := recover(); r != nil { log.Error("Invalid config ", r) os.Exit(1) } }() config = *c log.Info(fmt.Sprintf("Configuration loaded from %s", *configFile)) } else { // config not successfully loaded -> exit with error log.Error("toml", err) os.Exit(1) } // if log file path specified, attempt to load it if config.LogFile != "" { log.AddLogFile(config.LogFile) } // if DatadogStatistics EventReporter is enabled if config.Events.DatadogStatistics.Enabled { log.Info("DatadogStatistics EventReporter enabled.") eventReporters = append(eventReporters, reaperevents.NewDatadogStatistics(&config.Events.DatadogStatistics)) } // if DatadogEvents EventReporter is enabled if config.Events.DatadogEvents.Enabled { log.Info("DatadogEvents EventReporter enabled.") eventReporters = append(eventReporters, reaperevents.NewDatadogEvents(&config.Events.DatadogEvents)) } // if Email EventReporter is enabled if config.Events.Email.Enabled { log.Info("Email EventReporter enabled.") eventReporters = append(eventReporters, reaperevents.NewMailer(&config.Events.Email)) // these methods have pointer receivers log.Debug("SMTP Config: %s", &config.Events.Email) log.Debug("SMTP From: %s", &config.Events.Email.From) } // if Tagger EventReporter is enabled if config.Events.Tagger.Enabled { log.Info("Tagger EventReporter enabled.") eventReporters = append(eventReporters, reaperevents.NewTagger(&config.Events.Tagger)) } // if Reaper EventReporter is enabled if config.Events.Reaper.Enabled { log.Info("Reaper EventReporter enabled.") eventReporters = append(eventReporters, reaperevents.NewReaperEvent(&config.Events.Reaper)) } // if WhitelistTag is not set if config.WhitelistTag == "" { log.Error("WhitelistTag is empty, exiting") os.Exit(1) } else { log.Info("Using WhitelistTag '%s'", config.WhitelistTag) } if *withoutCloudformationResources { config.AWS.WithoutCloudformationResources = true } }