func main() {
ann := learn.Neural{}
ann.Init("fann.dat")
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for sig := range c {
// sig is a ^C, handle it
fmt.Println(sig)
ann.Save("fann.dat")
os.Exit(0)
}
}()
// Connect InfluxDB
influxDB, err := client.NewHTTPClient(client.HTTPConfig{
Addr: thoth.InfluxdbApi,
Username: username,
Password: password,
})
if err != nil {
panic(err)
}
round := 0
correct := 0
// TODO:it's need to calculate this for all RC
ann.Avg = profil.GetProfilLast(influxDB, "thoth", "eight-puzzle", "4d")
ann.StdDev = profil.GetProfilStdLast(influxDB, "thoth", "eight-puzzle", "4d")
fmt.Println("AVG ", ann.Avg)
fmt.Println("STDDEV ", ann.StdDev)
for {
// Get all user RCLen
RC := profil.GetUserRC()
RCLen := len(RC)
// Getting App Metric
for i := 0; i < RCLen; i++ {
round++
action := 0.0
replicas, err := profil.GetReplicas(RC[i].Namespace, RC[i].Name)
if err != nil {
panic(err)
}
fmt.Println("Replicas:", replicas)
// Check Resposne time & Label & Save WPI
var responseDay, response10Min float64
if responseDay, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "rtime", "1d"); err != nil {
panic(err)
log.Println(err)
}
fmt.Println("resDays : ", responseDay)
if response10Min, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "rtime", "5m"); err != nil {
fmt.Println("res10min : ", response10Min)
panic(err)
log.Println(err)
}
// Floor
responseDay = math.Floor(responseDay)
response10Min = math.Floor(response10Min)
fmt.Println("D", responseDay, " 10M", response10Min)
metrics := profil.GetAppResource(RC[i].Namespace, RC[i].Name)
var cpu10Min float64
if cpu10Min, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "cpu", "5m"); err != nil {
panic(err)
log.Println(err)
}
fmt.Println("CPU ", cpu10Min)
if cpu10Min > 70 {
fmt.Println("Response check")
// if response10Min > responseDay { // TODO:Need to check WPI too
// Save WPI
fmt.Println("Scale+1")
if err := profil.WriteRPI(influxDB, RC[i].Namespace, RC[i].Name, metrics.Request, replicas); err != nil {
panic(err)
log.Println(err)
}
// Scale +1
// TODO: Limit
if replicas < 10 {
action = 1
// if _, err := thoth.ScaleOutViaCli(replicas+1, RC[i].Namespace, RC[i].Name); err != nil {
// panic(err)
// }
}
// }
} else if replicas > 1 {
// = rpi/replicas
var rpiMax float64
if rpiMax, err = profil.GetAvgRPI(influxDB, RC[i].Namespace, RC[i].Name); err != nil {
rpiMax = -1
// TODO:Handler
//panic(err)
}
fmt.Println("WPI", rpiMax)
if rpiMax > 0 {
minReplicas := int(metrics.Request / int64(rpiMax)) // TODO: Ceil?
if minReplicas < replicas {
// Scale -1
fmt.Println("Scale-1")
action = -1
//if _, err := thoth.ScaleOutViaCli(replicas-1, RC[i].Namespace, RC[i].Name); err != nil {
// panic(err)
//}
}
}
}
// Normalize
resUsage10min := profil.GetProfilLast(influxDB, RC[i].Namespace, RC[i].Name, "10min")
fmt.Println("============================ FANN ============================")
// Training
ann.Train(resUsage10min, action)
// Run (Predict)
predict := ann.Run(resUsage10min)
if predict != 0 {
if predict == 1 {
if _, err := thoth.ScaleOutViaCli(replicas+1, RC[i].Namespace, RC[i].Name); err != nil {
fmt.Println(err)
}
} else if predict == -1 && replicas > 1 {
if _, err := thoth.ScaleOutViaCli(replicas-1, RC[i].Namespace, RC[i].Name); err != nil {
fmt.Println(err)
}
}
}
if predict == int(action) {
correct++
}
fmt.Println("Stats Round: ", round, " Correct:", correct, " Accuracy: ", (correct*100)/round, "%")
//-----------
}
fmt.Println("Sleep TODO:Change to 5 Minnn")
time.Sleep(300 * time.Second)
}
}
func main() {
// Connect InfluxDB
influxDB, err := client.NewHTTPClient(client.HTTPConfig{
Addr: thoth.InfluxdbApi,
Username: username,
Password: password,
})
if err != nil {
panic(err)
}
for {
// Get all user RC
RC := profil.GetUserRC()
RCLen := len(RC)
// Getting App Metric
for i := 0; i < RCLen; i++ {
replicas, err := profil.GetReplicas(RC[i].Namespace, RC[i].Name)
if err != nil {
panic(err)
}
fmt.Println(replicas)
// Check Resposne time & Label & Save WPI
var responseDay, response10Min float64
if responseDay, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "rtime", "1d"); err != nil {
panic(err)
log.Println(err)
}
if response10Min, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "rtime", "5m"); err != nil {
panic(err)
log.Println(err)
}
// Floor
responseDay = math.Floor(responseDay)
response10Min = math.Floor(response10Min)
fmt.Println("D", responseDay, " 10M", response10Min)
//metrics := profil.GetAppResource(RC[i].Namespace, RC[i].Name)
var cpu10Min float64
if cpu10Min, err = profil.GetProfilAvg(influxDB, RC[i].Namespace, RC[i].Name, "cpu", "5m"); err != nil {
panic(err)
log.Println(err)
}
fmt.Println("CPU ", cpu10Min)
qRepSpread, err := profil.QueryDB(influxDB, fmt.Sprint("SELECT spread(replicas) FROM "+RC[i].Namespace+" WHERE time > now() - 5m"))
if err != nil {
log.Fatal(err)
}
repSpread, err := strconv.ParseFloat(fmt.Sprint(qRepSpread[0].Series[0].Values[0][1]), 32)
if repSpread < 1 {
if cpu10Min > 70 {
//fmt.Println("Response check")
//if response10Min > responseDay { // TODO:Need to check WPI too
// Save WPI
fmt.Println("Scale+1")
//if err := profil.WriteRPI(influxDB, RC[i].Namespace, RC[i].Name, metrics.Request, replicas); err != nil {
// panic(err)
// log.Println(err)
//}
// Scale +1
// TODO: Limit
if replicas < 10 {
if _, err := thoth.ScaleOutViaCli(replicas+1, RC[i].Namespace, RC[i].Name); err != nil {
panic(err)
}
}
// }
} else if replicas > 1 {
// = rpi/replicas
//var rpiMax float64
//if rpiMax, err = profil.GetAvgRPI(influxDB, RC[i].Namespace, RC[i].Name); err != nil {
// rpiMax = -1
// TODO:Handler
//panic(err)
//}
//fmt.Println("WPI", rpiMax)
//if rpiMax > 0 {
// minReplicas := int(metrics.Request / int64(rpiMax)) // TODO: Ceil?
// if minReplicas < replicas {
// Scale -1
fmt.Println("Scale-1")
if _, err := thoth.ScaleOutViaCli(replicas-1, RC[i].Namespace, RC[i].Name); err != nil {
panic(err)
}
// }
//}
}
}
}
// -----Prediction-----
// Normalize
// Run (Predict)
// Label
//runFann()
//-----------
fmt.Println("Sleep TODO:Change to 5 Min")
time.Sleep(1 * time.Minute)
}
}
func main() {
// Connect InfluxDB
c, _ := client.NewHTTPClient(client.HTTPConfig{
Addr: thoth.InfluxdbApi,
Username: username,
Password: password,
})
//-------------------------------------------------------------
for {
// Get All RC&SVC name
runningPod := profil.GetAllRunningPod()
RC := profil.GetUserRC()
SVC := profil.GetUserSVC()
RCLen := len(RC)
statsMap := profil.GetHAProxyStats()
// Getting App resource usage
for i := 0; i < RCLen; i++ {
appName := RC[i].Namespace + "/" + RC[i].Name
value, exist := runningPod[appName]
if exist && value {
res := profil.GetAppResource(RC[i].Namespace, RC[i].Name)
replicas, err := profil.GetReplicas(RC[i].Namespace, RC[i].Name)
if err != nil {
log.Println(err)
}
tags := map[string]string{
"app": RC[i].Name,
}
svcSpec := SVC[appName]
fmt.Println(svcSpec)
// if svcSpec != nil {
fmt.Println(SVC[appName].Spec.Ports)
nodePort := SVC[appName].Spec.Ports[0].NodePort
vampPort := nodePort - 21000
// fmt.Println("VAMPStats ", vampStats.Spec.Ports[0].NodePort)
fmt.Println("Port ", vampPort)
// fmt.Println("VAMPStats ", statsMap[vampPort])
stats := statsMap[vampPort]
fields := map[string]interface{}{
"cpu": res.Cpu,
"memory": res.Memory,
"rps": stats.Request,
"rtime": stats.Response,
"r2xx": stats.Response2xx,
"r4xx": stats.Response4xx,
"r5xx": stats.Response5xx,
"r5xxroute": stats.Response5xxRoute,
"replicas": replicas,
}
fmt.Println(fields)
if err := profil.WritePoints(c, RC[i].Namespace, "s", tags, fields); err != nil {
panic(err)
}
// }
}
}
fmt.Println("Sleep\n")
time.Sleep(10 * time.Second)
}
//-------------------------------------------------------------
}
func main() {
agent := learn.QLearn{Gamma: 0.3}
agent.Init()
if err := agent.Load("ql.da"); err != nil {
fmt.Println("Load Fail", err)
}
agent.Epsilon = 0.0
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for sig := range c {
// sig is a ^C, handle it
fmt.Println(sig)
agent.Save("ql.da")
os.Exit(0)
}
}()
// Connect InfluxDB
influxDB, err := client.NewHTTPClient(client.HTTPConfig{
Addr: thoth.InfluxdbApi,
Username: username,
Password: password,
})
if err != nil {
panic(err)
}
firstRun := true
lastState := learn.State{}
lastAction := 0
var action int
for {
// Get all user RC
RC := profil.GetUserRC()
RCLen := len(RC)
// Getting App Metric
for i := 0; i < RCLen; i++ {
// TODO : Use model only with Eight-puzzle
if RC[i].Namespace == "thoth" && RC[i].Name == "eight-puzzle" {
replicas, err := profil.GetReplicas(RC[i].Namespace, RC[i].Name)
if err != nil {
panic(err)
}
res := profil.GetProfilLast(influxDB, RC[i].Namespace, RC[i].Name, "5m")
fmt.Println(res)
if !firstRun {
// Reward Last state
agent.Reward(lastState, lastAction, res)
}
agent.SetCurrentState(res["cpu"], res["memory"], res["rps"], res["rtime"], res["r5xx"], replicas)
action = agent.ChooseAction()
lastState = agent.CurrentState
firstRun = false
if action+replicas > 0 {
if _, err := thoth.ScaleOutViaCli(replicas+action, RC[i].Namespace, RC[i].Name); err != nil {
fmt.Println(err)
}
}
lastAction = action
fmt.Println(agent)
fmt.Println("C", agent.CurrentState.CPUH,
"M", agent.CurrentState.MemH,
"R", agent.CurrentState.RpsH,
"T", agent.CurrentState.RtimeH,
"5", agent.CurrentState.R5xxH)
}
fmt.Println(lastState)
fmt.Println(lastAction)
}
//-----------
fmt.Println("Sleep TODO:Change to 5 Min\n")
time.Sleep(60 * time.Second)
}
}