func (m *Master) addWorker(c chan<- *godo.Droplet) {
var (
droplet *godo.Droplet
err error
)
// TODO find a better way to dynamically name the workers, create using a snapshot
name := fmt.Sprintf("%s%d", m.workerConfig.NamePrefix, len(m.workers)+1)
createRequest := &godo.DropletCreateRequest{
Name: name,
Region: "tor1",
Size: "512mb",
PrivateNetworking: true,
Image: godo.DropletCreateImage{
Slug: m.imageID,
},
}
if droplet, _, err = m.doClient.Droplets.Create(createRequest); err != nil {
utils.Die("Couldn't create droplet: %s\n", err.Error())
}
for {
time.Sleep(m.pollInterval)
if droplet, _, _ = m.doClient.Droplets.Get(droplet.ID); droplet.Status == "active" {
break
}
fmt.Printf("Polling. Status: %s\n", droplet.Status)
}
fmt.Println("Droplet creation complete")
c <- droplet
}
func (m *Master) reload() {
var (
out []byte
err error
)
if out, err = exec.Command("sh", "-c", m.command).Output(); err != nil {
utils.Die("Error executing 'reload' command: %s", err.Error())
}
fmt.Printf("Executed command. Output: '%s'\n", strings.TrimSpace(string(out)))
}
func main() {
host := flag.String("host", "", "the IP address and port")
clientId := flag.Int64("id", 1, "the id of the node")
flag.Parse()
if *host == "" {
utils.Die("No host address provided")
}
fmt.Printf("Starting client. Connecting to master at %s\n", *host)
startNode(*host, fmt.Sprintf("%d", *clientId))
}
func getPrivateIP() string {
i, err := net.InterfaceByName("eth1")
if err != nil {
utils.Die("Error getting interface eth1: %s\n", err.Error())
}
addrs, err := i.Addrs()
if err != nil {
utils.Die("Error getting interface eth1 addresses: %s\n", err.Error())
}
var ip string
for _, a := range addrs {
if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
ip = ipnet.IP.String()
break
}
}
}
return ip
}
func (m *Master) removeWorker(c chan<- bool) {
// TODO implement logic to remove a worker only after all requests have finished processing
// Delete the last droplet
toDelete := m.workers[len(m.workers)-1]
if _, err := m.doClient.Droplets.Delete(toDelete.droplet.ID); err != nil {
utils.Die("Error deleting droplet: %s", err.Error())
}
m.workers = m.workers[0 : len(m.workers)-1]
c <- true
}
func (m *Master) writeConfigFile() {
var (
file *os.File
temp *template.Template
err error
)
if temp, err = template.ParseFiles(m.balanceConfigTemplate); err != nil {
utils.Die("Error reading in template: %s", err.Error())
}
if file, err = os.OpenFile(m.balanceConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil {
utils.Die("Error opening load balancer config file: %s", err.Error())
}
// Get the IP addresses together
type haproxyInfo struct {
Addr string
Weight int64
}
ips := make(map[string]haproxyInfo)
for _, worker := range m.workers {
ips[worker.droplet.Name] = haproxyInfo{
worker.publicAddr,
worker.weight,
}
}
// Print out all of the objects
fmt.Printf("Writing out new config for %d workers\n", len(m.workers))
for key, ip := range ips {
fmt.Printf("%s: ip=%s, weight=%d\n", key, ip.Addr, ip.Weight)
}
// Write changes out to the template file
if err = temp.Execute(file, ips); err != nil {
utils.Die("Error writing template out to the file: %s", err.Error())
}
}
func startNode(masterHost, name string) {
var sock mangos.Socket
var err error
var msg []byte
masterUrl := url.URL{Scheme: "tcp", Host: masterHost}
ip := getPrivateIP()
// Try to get new "respondent" socket
if sock, err = respondent.NewSocket(); err != nil {
utils.Die("Can't get new respondent socket: %s", err.Error())
}
defer sock.Close()
sock.AddTransport(tcp.NewTransport())
// Connect to master
if err = sock.Dial(masterUrl.String()); err != nil {
utils.Die("Can't dial on respondent socket: %s", err.Error())
}
// Wait for a survey request and send responses
for {
if msg, err = sock.Recv(); err != nil {
utils.Die("Cannot recv: %s", err.Error())
}
fmt.Printf("Client(%s): Received \"%s\" survey request\n", name, string(msg))
var loadAvg *load.LoadAvgStat
if loadAvg, err = load.LoadAvg(); err != nil {
utils.Die("Cannot get load average: %s", err.Error())
}
var cpuInfo []cpu.CPUInfoStat
if cpuInfo, err = cpu.CPUInfo(); err != nil {
utils.Die("Cannot get CPU info: %s", err.Error())
}
fmt.Printf("CPU INFO len: %d\n", len(cpuInfo))
// Get the normalized CPU load
avg := loadAvg.Load1
cores := int32(0)
for _, info := range cpuInfo {
fmt.Printf("Inner Cores: %d\n", info.Cores)
cores += info.Cores
}
fmt.Printf("Load avg: %f\n", avg)
fmt.Printf("Cores: %d\n", cores)
avg = avg / float64(cores)
fmt.Printf("Client(%s): Sending survey response\n", name)
if err = sock.Send([]byte(fmt.Sprintf("%s,%f", ip, avg))); err != nil {
utils.Die("Cannot send: %s", err.Error())
}
}
}
func main() {
host := flag.String("host", "0.0.0.0:8000", "the IP address and port to bind to")
command := flag.String("command", "", "the command to run after writing out the load balancer's new configuration file")
balanceConfigTemplate := flag.String("balancetemplate", "", "the load balancer config file template to use")
balanceConfigFile := flag.String("balanceconfig", "", "the load balancer config file to write to")
workerConfigFile := flag.String("workerconfig", "", "the worker config file (JSON) to read from")
digitalOceanToken := flag.String("token", "", "the Digital Ocean API token to use")
digitalOceanImageID := flag.String("image", "", "the ID of the image to use when creating worker nodes")
overloadedCpuThreshold := flag.Float64("overloaded", 0.7, "the average CPU usage threshold after which the nodes are considered overloaded")
underusedCpuThreshold := flag.Float64("underused", 0.3, "the CPU usage threshold to consider a node as underutilized")
minWorkers := flag.Int64("min", 1, "the minimum number of workers to have")
maxWorkers := flag.Int64("max", 10, "the minimum number of workers to have")
streamStatsd := flag.Bool("statsd", false, "a flag indicating whether or not to stream statsd stats")
statsdAddr := flag.String("statsdaddr", "localhost:8125", "the address and port of the statsd server")
statsdPrefix := flag.String("statsdprefix", "autoscaler.", "the statsd prefix to use")
statsdInterval := flag.Int64("statsdinterval", 2, "the number of seconds to wait before flushing every batch of statsd stats")
pollInterval := flag.Int64("pollinterval", 3, "the amount of time (in seconds) to wait between polling Digital Ocean for updates")
cooldownInterval := flag.Int64("cooldowninterval", 15, "the amount of time (in seconds) to wait before making changes to workers after altering the worker set")
surveyDeadline := flag.Int64("surveydeadline", 1, "the amount of time (in seconds) to wait to receive feedback from workers")
queryInterval := flag.Int64("surveytimeout", 3, "the amount of time (in seconds) to leave between querying workers")
changeWeights := flag.Bool("weights", true, "whether or not to use weights")
scaleNodes := flag.Bool("autoscale", true, "whether or not to scale nodes up and down")
flag.Parse()
// Handle checking command line arguments
if *command == "" {
utils.Die("Missing -command flag")
} else if *balanceConfigTemplate == "" {
utils.Die("Missing -balancetemplate flag")
} else if *balanceConfigFile == "" {
utils.Die("Missing -balanceconfig flag")
} else if *workerConfigFile == "" {
utils.Die("Missing -workerconfig flag")
} else if *digitalOceanToken == "" {
utils.Die("Missing -token flag")
} else if *digitalOceanImageID == "" {
utils.Die("Missing -image flag")
} else if *minWorkers <= 0 {
utils.Die("The -min must be non-negative")
} else if *maxWorkers <= 0 {
utils.Die("The -max must be non-negative")
} else if *maxWorkers < *minWorkers {
utils.Die("Max number of workers must be greater than or equal to the min")
} else if *streamStatsd && *statsdAddr == "" {
utils.Die("Statsd streaming requested, but missing -statsdaddr flag")
}
// Read in the config file
var workerConfig master.WorkerConfig
jsonData, err := ioutil.ReadFile(*workerConfigFile)
if err != nil {
utils.Die("Error reading in config file: %s", err.Error())
}
if err = json.Unmarshal(jsonData, &workerConfig); err != nil {
utils.Die("Error parsing JSON in config file: %s", err.Error())
}
if !*changeWeights {
fmt.Println("NOT CHANGING WEIGHTS")
}
if !*scaleNodes {
fmt.Println("NOT SCALING NODES")
}
// Start the master
fmt.Printf("Starting master at %s\n", *host)
var monitor *master.Master
if !*streamStatsd {
monitor = master.NewMaster(
*host,
&workerConfig,
*command,
*balanceConfigTemplate, *balanceConfigFile,
*digitalOceanToken, *digitalOceanImageID,
*overloadedCpuThreshold, *underusedCpuThreshold,
*minWorkers, *maxWorkers,
time.Duration(*pollInterval)*time.Second, time.Duration(*cooldownInterval)*time.Second,
time.Duration(*surveyDeadline)*time.Second, time.Duration(*queryInterval)*time.Second,
*scaleNodes, *changeWeights,
)
} else {
monitor = master.NewMasterWithStatsd(
*host,
&workerConfig,
*command,
*balanceConfigTemplate, *balanceConfigFile,
*digitalOceanToken, *digitalOceanImageID,
*overloadedCpuThreshold, *underusedCpuThreshold,
*minWorkers, *maxWorkers,
time.Duration(*pollInterval)*time.Second, time.Duration(*cooldownInterval)*time.Second,
time.Duration(*surveyDeadline)*time.Second, time.Duration(*queryInterval)*time.Second,
*scaleNodes, *changeWeights,
*statsdAddr, *statsdPrefix, time.Duration(*statsdInterval)*time.Second,
)
}
defer monitor.CleanUp()
monitor.MonitorWorkers()
}
func NewMaster(host string, workerConfig *WorkerConfig, command, balanceConfigTemplate, balanceConfigFile, digitalOceanToken, digitalOceanImageID string,
overloadedCpuThreshold, underusedCpuThreshold float64, minWorkers, maxWorkers int64, pollInterval, cooldownInterval, surveyDeadline, queryInterval time.Duration,
scaleNodes, changeWeights bool) *Master {
var err error
bindUrl := url.URL{Scheme: "tcp", Host: host}
// Set up the Digital Ocean client
tokenSource := &TokenSource{
AccessToken: digitalOceanToken,
}
oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
client := godo.NewClient(oauthClient)
// Create a set containing the configured worker nodes
workerSet := make(map[string]interface{})
for _, name := range workerConfig.DropletNames {
workerSet[name] = nil
}
// Get a list of all of the droplets and filter out any irrelevant ones
var workerDroplets, allDroplets []godo.Droplet
if allDroplets, _, err = client.Droplets.List(&godo.ListOptions{
PerPage: 200,
}); err != nil {
utils.Die("Error getting the list of droplets: %s", err)
}
for _, droplet := range allDroplets {
if _, contains := workerSet[droplet.Name]; contains {
workerDroplets = append(workerDroplets, droplet)
}
}
// Wrap the droplets for easier access to relevant information (public and private IP)
var workers []*Worker
for _, droplet := range workerDroplets {
workers = append(workers, newWorker(droplet))
}
return &Master{
url: bindUrl,
scaleNodes: scaleNodes,
changeWeights: changeWeights,
workerConfig: workerConfig,
workers: workers,
command: command,
balanceConfigTemplate: balanceConfigTemplate,
balanceConfigFile: balanceConfigFile,
overloadedCpuThreshold: overloadedCpuThreshold,
underusedCpuThreshold: underusedCpuThreshold,
minWorkers: minWorkers,
maxWorkers: maxWorkers,
token: tokenSource,
imageID: digitalOceanImageID,
doClient: client,
pollInterval: pollInterval,
cooldownInterval: cooldownInterval,
surveyDeadline: surveyDeadline,
queryInterval: queryInterval,
}
}
func (m *Master) queryWorkers(c chan<- float64) {
var (
err error
sock mangos.Socket
)
// Try to get new "surveyor" socket
if sock, err = surveyor.NewSocket(); err != nil {
utils.Die("Can't get new surveyor socket: %s", err)
}
defer sock.Close()
sock.AddTransport(tcp.NewTransport())
// Begin listening on the URL
if err = sock.Listen(m.url.String()); err != nil {
utils.Die("Can't listen on surveyor socket: %s", err.Error())
}
// Set "deadline" for the survey and a timeout for receiving responses
if err = sock.SetOption(mangos.OptionSurveyTime, m.surveyDeadline); err != nil {
utils.Die("SetOption(mangos.OptionSurveyTime): %s", err.Error())
}
if err = sock.SetOption(mangos.OptionRecvDeadline, m.surveyDeadline+(1*time.Second)); err != nil {
utils.Die("SetOption(mangos.OptionRecvDeadline): %s", err.Error())
}
for {
fmt.Println("Sending master request")
if err = sock.Send([]byte("CPU")); err != nil {
utils.Die("Failed sending survey: %s", err.Error())
}
loadAvgs := []float64{}
for {
var msg []byte
if msg, err = sock.Recv(); err != nil {
break
}
parts := strings.Split(string(msg), ",")
if len(parts) != 2 {
continue
}
ip := parts[0]
loadAvgString := parts[1]
// Find the corresponding droplet
var worker *Worker
for _, w := range m.workers {
if w.privateAddr == ip {
worker = w
break
}
}
if worker == nil {
fmt.Printf("Message received from unknown worker '%s'. Skipping...\n", ip)
continue
}
var loadAvg float64
if loadAvg, err = strconv.ParseFloat(string(loadAvgString), 64); err != nil {
utils.Die("ParseFloat(): %s", err.Error())
}
// Set their load average and append this worker's load average to the list
worker.loadAvg = loadAvg
loadAvgs = append(loadAvgs, loadAvg)
}
// Compute the average loadAvg
var loadAvg float64
for _, avg := range loadAvgs {
loadAvg += avg
}
loadAvg /= float64(len(loadAvgs))
// Send the load averages
if !math.IsNaN(loadAvg) {
c <- loadAvg
}
// Wait
time.Sleep(m.queryInterval)
}
}