func (m *Master) addWorker(c chan<- *godo.Droplet) { var ( droplet *godo.Droplet err error ) // TODO find a better way to dynamically name the workers, create using a snapshot name := fmt.Sprintf("%s%d", m.workerConfig.NamePrefix, len(m.workers)+1) createRequest := &godo.DropletCreateRequest{ Name: name, Region: "tor1", Size: "512mb", PrivateNetworking: true, Image: godo.DropletCreateImage{ Slug: m.imageID, }, } if droplet, _, err = m.doClient.Droplets.Create(createRequest); err != nil { utils.Die("Couldn't create droplet: %s\n", err.Error()) } for { time.Sleep(m.pollInterval) if droplet, _, _ = m.doClient.Droplets.Get(droplet.ID); droplet.Status == "active" { break } fmt.Printf("Polling. Status: %s\n", droplet.Status) } fmt.Println("Droplet creation complete") c <- droplet }
func (m *Master) reload() { var ( out []byte err error ) if out, err = exec.Command("sh", "-c", m.command).Output(); err != nil { utils.Die("Error executing 'reload' command: %s", err.Error()) } fmt.Printf("Executed command. Output: '%s'\n", strings.TrimSpace(string(out))) }
func main() { host := flag.String("host", "", "the IP address and port") clientId := flag.Int64("id", 1, "the id of the node") flag.Parse() if *host == "" { utils.Die("No host address provided") } fmt.Printf("Starting client. Connecting to master at %s\n", *host) startNode(*host, fmt.Sprintf("%d", *clientId)) }
func getPrivateIP() string { i, err := net.InterfaceByName("eth1") if err != nil { utils.Die("Error getting interface eth1: %s\n", err.Error()) } addrs, err := i.Addrs() if err != nil { utils.Die("Error getting interface eth1 addresses: %s\n", err.Error()) } var ip string for _, a := range addrs { if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { if ipnet.IP.To4() != nil { ip = ipnet.IP.String() break } } } return ip }
func (m *Master) removeWorker(c chan<- bool) { // TODO implement logic to remove a worker only after all requests have finished processing // Delete the last droplet toDelete := m.workers[len(m.workers)-1] if _, err := m.doClient.Droplets.Delete(toDelete.droplet.ID); err != nil { utils.Die("Error deleting droplet: %s", err.Error()) } m.workers = m.workers[0 : len(m.workers)-1] c <- true }
func (m *Master) writeConfigFile() { var ( file *os.File temp *template.Template err error ) if temp, err = template.ParseFiles(m.balanceConfigTemplate); err != nil { utils.Die("Error reading in template: %s", err.Error()) } if file, err = os.OpenFile(m.balanceConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil { utils.Die("Error opening load balancer config file: %s", err.Error()) } // Get the IP addresses together type haproxyInfo struct { Addr string Weight int64 } ips := make(map[string]haproxyInfo) for _, worker := range m.workers { ips[worker.droplet.Name] = haproxyInfo{ worker.publicAddr, worker.weight, } } // Print out all of the objects fmt.Printf("Writing out new config for %d workers\n", len(m.workers)) for key, ip := range ips { fmt.Printf("%s: ip=%s, weight=%d\n", key, ip.Addr, ip.Weight) } // Write changes out to the template file if err = temp.Execute(file, ips); err != nil { utils.Die("Error writing template out to the file: %s", err.Error()) } }
func startNode(masterHost, name string) { var sock mangos.Socket var err error var msg []byte masterUrl := url.URL{Scheme: "tcp", Host: masterHost} ip := getPrivateIP() // Try to get new "respondent" socket if sock, err = respondent.NewSocket(); err != nil { utils.Die("Can't get new respondent socket: %s", err.Error()) } defer sock.Close() sock.AddTransport(tcp.NewTransport()) // Connect to master if err = sock.Dial(masterUrl.String()); err != nil { utils.Die("Can't dial on respondent socket: %s", err.Error()) } // Wait for a survey request and send responses for { if msg, err = sock.Recv(); err != nil { utils.Die("Cannot recv: %s", err.Error()) } fmt.Printf("Client(%s): Received \"%s\" survey request\n", name, string(msg)) var loadAvg *load.LoadAvgStat if loadAvg, err = load.LoadAvg(); err != nil { utils.Die("Cannot get load average: %s", err.Error()) } var cpuInfo []cpu.CPUInfoStat if cpuInfo, err = cpu.CPUInfo(); err != nil { utils.Die("Cannot get CPU info: %s", err.Error()) } fmt.Printf("CPU INFO len: %d\n", len(cpuInfo)) // Get the normalized CPU load avg := loadAvg.Load1 cores := int32(0) for _, info := range cpuInfo { fmt.Printf("Inner Cores: %d\n", info.Cores) cores += info.Cores } fmt.Printf("Load avg: %f\n", avg) fmt.Printf("Cores: %d\n", cores) avg = avg / float64(cores) fmt.Printf("Client(%s): Sending survey response\n", name) if err = sock.Send([]byte(fmt.Sprintf("%s,%f", ip, avg))); err != nil { utils.Die("Cannot send: %s", err.Error()) } } }
func main() { host := flag.String("host", "0.0.0.0:8000", "the IP address and port to bind to") command := flag.String("command", "", "the command to run after writing out the load balancer's new configuration file") balanceConfigTemplate := flag.String("balancetemplate", "", "the load balancer config file template to use") balanceConfigFile := flag.String("balanceconfig", "", "the load balancer config file to write to") workerConfigFile := flag.String("workerconfig", "", "the worker config file (JSON) to read from") digitalOceanToken := flag.String("token", "", "the Digital Ocean API token to use") digitalOceanImageID := flag.String("image", "", "the ID of the image to use when creating worker nodes") overloadedCpuThreshold := flag.Float64("overloaded", 0.7, "the average CPU usage threshold after which the nodes are considered overloaded") underusedCpuThreshold := flag.Float64("underused", 0.3, "the CPU usage threshold to consider a node as underutilized") minWorkers := flag.Int64("min", 1, "the minimum number of workers to have") maxWorkers := flag.Int64("max", 10, "the minimum number of workers to have") streamStatsd := flag.Bool("statsd", false, "a flag indicating whether or not to stream statsd stats") statsdAddr := flag.String("statsdaddr", "localhost:8125", "the address and port of the statsd server") statsdPrefix := flag.String("statsdprefix", "autoscaler.", "the statsd prefix to use") statsdInterval := flag.Int64("statsdinterval", 2, "the number of seconds to wait before flushing every batch of statsd stats") pollInterval := flag.Int64("pollinterval", 3, "the amount of time (in seconds) to wait between polling Digital Ocean for updates") cooldownInterval := flag.Int64("cooldowninterval", 15, "the amount of time (in seconds) to wait before making changes to workers after altering the worker set") surveyDeadline := flag.Int64("surveydeadline", 1, "the amount of time (in seconds) to wait to receive feedback from workers") queryInterval := flag.Int64("surveytimeout", 3, "the amount of time (in seconds) to leave between querying workers") changeWeights := flag.Bool("weights", true, "whether or not to use weights") scaleNodes := flag.Bool("autoscale", true, "whether or not to scale nodes up and down") flag.Parse() // Handle checking command line arguments if *command == "" { utils.Die("Missing -command flag") } else if *balanceConfigTemplate == "" { utils.Die("Missing -balancetemplate flag") } else if *balanceConfigFile == "" { utils.Die("Missing -balanceconfig flag") } else if *workerConfigFile == "" { utils.Die("Missing -workerconfig flag") } else if *digitalOceanToken == "" { utils.Die("Missing -token flag") } else if *digitalOceanImageID == "" { utils.Die("Missing -image flag") } else if *minWorkers <= 0 { utils.Die("The -min must be non-negative") } else if *maxWorkers <= 0 { utils.Die("The -max must be non-negative") } else if *maxWorkers < *minWorkers { utils.Die("Max number of workers must be greater than or equal to the min") } else if *streamStatsd && *statsdAddr == "" { utils.Die("Statsd streaming requested, but missing -statsdaddr flag") } // Read in the config file var workerConfig master.WorkerConfig jsonData, err := ioutil.ReadFile(*workerConfigFile) if err != nil { utils.Die("Error reading in config file: %s", err.Error()) } if err = json.Unmarshal(jsonData, &workerConfig); err != nil { utils.Die("Error parsing JSON in config file: %s", err.Error()) } if !*changeWeights { fmt.Println("NOT CHANGING WEIGHTS") } if !*scaleNodes { fmt.Println("NOT SCALING NODES") } // Start the master fmt.Printf("Starting master at %s\n", *host) var monitor *master.Master if !*streamStatsd { monitor = master.NewMaster( *host, &workerConfig, *command, *balanceConfigTemplate, *balanceConfigFile, *digitalOceanToken, *digitalOceanImageID, *overloadedCpuThreshold, *underusedCpuThreshold, *minWorkers, *maxWorkers, time.Duration(*pollInterval)*time.Second, time.Duration(*cooldownInterval)*time.Second, time.Duration(*surveyDeadline)*time.Second, time.Duration(*queryInterval)*time.Second, *scaleNodes, *changeWeights, ) } else { monitor = master.NewMasterWithStatsd( *host, &workerConfig, *command, *balanceConfigTemplate, *balanceConfigFile, *digitalOceanToken, *digitalOceanImageID, *overloadedCpuThreshold, *underusedCpuThreshold, *minWorkers, *maxWorkers, time.Duration(*pollInterval)*time.Second, time.Duration(*cooldownInterval)*time.Second, time.Duration(*surveyDeadline)*time.Second, time.Duration(*queryInterval)*time.Second, *scaleNodes, *changeWeights, *statsdAddr, *statsdPrefix, time.Duration(*statsdInterval)*time.Second, ) } defer monitor.CleanUp() monitor.MonitorWorkers() }
func NewMaster(host string, workerConfig *WorkerConfig, command, balanceConfigTemplate, balanceConfigFile, digitalOceanToken, digitalOceanImageID string, overloadedCpuThreshold, underusedCpuThreshold float64, minWorkers, maxWorkers int64, pollInterval, cooldownInterval, surveyDeadline, queryInterval time.Duration, scaleNodes, changeWeights bool) *Master { var err error bindUrl := url.URL{Scheme: "tcp", Host: host} // Set up the Digital Ocean client tokenSource := &TokenSource{ AccessToken: digitalOceanToken, } oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource) client := godo.NewClient(oauthClient) // Create a set containing the configured worker nodes workerSet := make(map[string]interface{}) for _, name := range workerConfig.DropletNames { workerSet[name] = nil } // Get a list of all of the droplets and filter out any irrelevant ones var workerDroplets, allDroplets []godo.Droplet if allDroplets, _, err = client.Droplets.List(&godo.ListOptions{ PerPage: 200, }); err != nil { utils.Die("Error getting the list of droplets: %s", err) } for _, droplet := range allDroplets { if _, contains := workerSet[droplet.Name]; contains { workerDroplets = append(workerDroplets, droplet) } } // Wrap the droplets for easier access to relevant information (public and private IP) var workers []*Worker for _, droplet := range workerDroplets { workers = append(workers, newWorker(droplet)) } return &Master{ url: bindUrl, scaleNodes: scaleNodes, changeWeights: changeWeights, workerConfig: workerConfig, workers: workers, command: command, balanceConfigTemplate: balanceConfigTemplate, balanceConfigFile: balanceConfigFile, overloadedCpuThreshold: overloadedCpuThreshold, underusedCpuThreshold: underusedCpuThreshold, minWorkers: minWorkers, maxWorkers: maxWorkers, token: tokenSource, imageID: digitalOceanImageID, doClient: client, pollInterval: pollInterval, cooldownInterval: cooldownInterval, surveyDeadline: surveyDeadline, queryInterval: queryInterval, } }
func (m *Master) queryWorkers(c chan<- float64) { var ( err error sock mangos.Socket ) // Try to get new "surveyor" socket if sock, err = surveyor.NewSocket(); err != nil { utils.Die("Can't get new surveyor socket: %s", err) } defer sock.Close() sock.AddTransport(tcp.NewTransport()) // Begin listening on the URL if err = sock.Listen(m.url.String()); err != nil { utils.Die("Can't listen on surveyor socket: %s", err.Error()) } // Set "deadline" for the survey and a timeout for receiving responses if err = sock.SetOption(mangos.OptionSurveyTime, m.surveyDeadline); err != nil { utils.Die("SetOption(mangos.OptionSurveyTime): %s", err.Error()) } if err = sock.SetOption(mangos.OptionRecvDeadline, m.surveyDeadline+(1*time.Second)); err != nil { utils.Die("SetOption(mangos.OptionRecvDeadline): %s", err.Error()) } for { fmt.Println("Sending master request") if err = sock.Send([]byte("CPU")); err != nil { utils.Die("Failed sending survey: %s", err.Error()) } loadAvgs := []float64{} for { var msg []byte if msg, err = sock.Recv(); err != nil { break } parts := strings.Split(string(msg), ",") if len(parts) != 2 { continue } ip := parts[0] loadAvgString := parts[1] // Find the corresponding droplet var worker *Worker for _, w := range m.workers { if w.privateAddr == ip { worker = w break } } if worker == nil { fmt.Printf("Message received from unknown worker '%s'. Skipping...\n", ip) continue } var loadAvg float64 if loadAvg, err = strconv.ParseFloat(string(loadAvgString), 64); err != nil { utils.Die("ParseFloat(): %s", err.Error()) } // Set their load average and append this worker's load average to the list worker.loadAvg = loadAvg loadAvgs = append(loadAvgs, loadAvg) } // Compute the average loadAvg var loadAvg float64 for _, avg := range loadAvgs { loadAvg += avg } loadAvg /= float64(len(loadAvgs)) // Send the load averages if !math.IsNaN(loadAvg) { c <- loadAvg } // Wait time.Sleep(m.queryInterval) } }