func TestSendMetrics(t *testing.T) {
var tasks demand.Tasks
tasks.Tasks = make([]*demand.Task, 2)
tasks.Tasks[0] = &demand.Task{Name: "priority1", Demand: 8, Requested: 3, Running: 4}
tasks.Tasks[1] = &demand.Task{Name: "priority2", Demand: 2, Requested: 7, Running: 5}
globalT = t
for testIndex = range tests {
server := httptest.NewServer(websocket.Handler(testServerMetrics))
serverAddr := server.Listener.Addr().String()
ws, err := utils.InitWebSocket(serverAddr)
if err != nil {
t.Fatal("dialing", err)
}
SendMetrics(ws, "hello", &tasks)
ws.Close()
server.Close()
}
}
func TestServerMonitor(t *testing.T) {
var tasks demand.Tasks
tasks.Tasks = make([]*demand.Task, 2)
tasks.Tasks[0] = &demand.Task{Name: "priority1", Demand: 8, Requested: 3, Running: 4}
tasks.Tasks[1] = &demand.Task{Name: "priority2", Demand: 2, Requested: 7, Running: 5}
server := httptest.NewServer(websocket.Handler(testServerMetrics))
serverAddr := server.Listener.Addr().String()
ws, err := utils.InitWebSocket(serverAddr)
if err != nil {
t.Fatal("dialing", err)
}
s := NewServerMonitor(ws, "hello")
if s.userID != "hello" {
t.Fatal("Didn't set userID")
}
s.SendMetrics(&tasks)
ws.Close()
server.Close()
}
// For this simple prototype, Microscaling sits in a loop checking for demand changes every X milliseconds
func main() {
var err error
var tasks *demand.Tasks
st := getSettings()
// Sending an empty struct on this channel triggers the scheduler to make updates
demandUpdate := make(chan struct{}, 1)
s, err := getScheduler(st, demandUpdate)
if err != nil {
log.Errorf("Failed to get scheduler: %v", err)
return
}
tasks, err = getTasks(st)
if err != nil {
log.Errorf("Failed to get tasks: %v", err)
return
}
// Let the scheduler know about the task types.
for _, task := range tasks.Tasks {
err = s.InitScheduler(task)
if err != nil {
log.Errorf("Failed to start task %s: %v", task.Name, err)
return
}
}
// Check if there are already any of these containers running
err = s.CountAllTasks(tasks)
if err != nil {
log.Errorf("Failed to count containers. %v", err)
}
// Set the initial requested counts to match what's running
for name, task := range tasks.Tasks {
task.Requested = task.Running
tasks.Tasks[name] = task
}
// Prepare for cleanup when we receive an interrupt
closedown := make(chan os.Signal, 1)
signal.Notify(closedown, os.Interrupt)
signal.Notify(closedown, syscall.SIGTERM)
// Open a web socket to the server TODO!! This won't always be necessary if we're not sending metrics & calculating demand locally
ws, err := utils.InitWebSocket(st.microscalingAPI)
if err != nil {
log.Errorf("Failed to open web socket: %v", err)
return
}
de, err := getDemandEngine(st, ws)
if err != nil {
log.Errorf("Failed to get demand engine: %v", err)
return
}
go de.GetDemand(tasks, demandUpdate)
// Handle demand updates
go func() {
for range demandUpdate {
err = s.StopStartTasks(tasks)
if err != nil {
log.Errorf("Failed to stop / start tasks. %v", err)
}
}
// When the demandUpdate channel is closed, it's time to scale everything down to 0
cleanup(s, tasks)
}()
// Periodically read the current state of tasks
getMetricsTimeout := time.NewTicker(constGetMetricsTimeout * time.Millisecond)
go func() {
for _ = range getMetricsTimeout.C {
// Find out how many instances of each task are running
err = s.CountAllTasks(tasks)
if err != nil {
log.Errorf("Failed to count containers. %v", err)
}
}
}()
// Periodically send metrics to any monitors
monitors := getMonitors(st, ws)
if len(monitors) > 0 {
sendMetricsTimeout := time.NewTicker(constSendMetricsTimeout * time.Millisecond)
go func() {
for _ = range sendMetricsTimeout.C {
for _, m := range monitors {
err = m.SendMetrics(tasks)
if err != nil {
log.Errorf("Failed to send metrics. %v", err)
}
}
}
}()
}
// When we're asked to close down, we don't want to handle demand updates any more
<-closedown
log.Info("Clean up when ready")
// Give the scheduler a chance to do any necessary cleanup
s.Cleanup()
// The demand engine is responsible for closing the demandUpdate channel so that we stop
// doing scaling operations
de.StopDemand(demandUpdate)
exitWaitTimeout := time.NewTicker(constGetMetricsTimeout * time.Millisecond)
for _ = range exitWaitTimeout.C {
if tasks.Exited() {
log.Info("All finished")
break
}
}
}
