func configPluginRoutes() {
http.HandleFunc("/plugin/update", func(w http.ResponseWriter, r *http.Request) {
if !g.Config().Plugin.Enabled {
w.Write([]byte("plugin not enabled"))
return
}
dir := g.Config().Plugin.Dir
parentDir := file.Dir(dir)
file.InsureDir(parentDir)
if file.IsExist(dir) {
// git pull
cmd := exec.Command("git", "pull")
cmd.Dir = dir
err := cmd.Run()
if err != nil {
w.Write([]byte(fmt.Sprintf("git pull in dir:%s fail. error: %s", dir, err)))
return
}
} else {
// git clone
cmd := exec.Command("git", "clone", g.Config().Plugin.Git, file.Basename(dir))
cmd.Dir = parentDir
err := cmd.Run()
if err != nil {
w.Write([]byte(fmt.Sprintf("git clone in dir:%s fail. error: %s", parentDir, err)))
return
}
}
w.Write([]byte("success"))
})
http.HandleFunc("/plugin/reset", func(w http.ResponseWriter, r *http.Request) {
if !g.Config().Plugin.Enabled {
w.Write([]byte("plugin not enabled"))
return
}
dir := g.Config().Plugin.Dir
if file.IsExist(dir) {
cmd := exec.Command("git", "reset", "--hard")
cmd.Dir = dir
err := cmd.Run()
if err != nil {
w.Write([]byte(fmt.Sprintf("git reset --hard in dir:%s fail. error: %s", dir, err)))
return
}
}
w.Write([]byte("success"))
})
http.HandleFunc("/plugins", func(w http.ResponseWriter, r *http.Request) {
//TODO: not thread safe
RenderDataJson(w, plugins.Plugins)
})
}
func syncMinePlugins() {
var (
timestamp int64 = -1
pluginDirs []string
)
duration := time.Duration(g.Config().Heartbeat.Interval) * time.Second
for {
REST:
time.Sleep(duration)
hostname, err := g.Hostname()
if err != nil {
goto REST
}
req := model.AgentHeartbeatRequest{
Hostname: hostname,
}
var resp model.AgentPluginsResponse
err = g.HbsClient.Call("Agent.MinePlugins", req, &resp)
if err != nil {
log.Println("ERROR:", err)
goto REST
}
if resp.Timestamp <= timestamp {
goto REST
}
pluginDirs = resp.Plugins
timestamp = resp.Timestamp
if g.Config().Debug {
log.Println(&resp)
}
if len(pluginDirs) == 0 {
plugins.ClearAllPlugins()
}
desiredAll := make(map[string]*plugins.Plugin)
for _, p := range pluginDirs {
underOneDir := plugins.ListPlugins(strings.Trim(p, "/"))
for k, v := range underOneDir {
desiredAll[k] = v
}
}
plugins.DelNoUsePlugins(desiredAll)
plugins.AddNewPlugins(desiredAll)
}
}
func Collect() {
if !g.Config().Transfer.Enabled {
return
}
if g.Config().Transfer.Addr == "" {
return
}
for _, v := range funcs.Mappers {
go collect(int64(v.Interval), v.Fs)
}
}
func SyncMinePlugins() {
if !g.Config().Plugin.Enabled {
return
}
if !g.Config().Heartbeat.Enabled {
return
}
if g.Config().Heartbeat.Addr == "" {
return
}
go syncMinePlugins()
}
func configRunRoutes() {
http.HandleFunc("/run", func(w http.ResponseWriter, r *http.Request) {
if !g.Config().Http.Backdoor {
w.Write([]byte("/run disabled"))
return
}
if g.IsTrustable(r.RemoteAddr) {
if r.ContentLength == 0 {
http.Error(w, "body is blank", http.StatusBadRequest)
return
}
bs, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
body := string(bs)
out, err := sys.CmdOutBytes("sh", "-c", body)
if err != nil {
w.Write([]byte("exec fail: " + err.Error()))
return
}
w.Write(out)
} else {
w.Write([]byte("no privilege"))
}
})
}
func configAdminRoutes() {
http.HandleFunc("/exit", func(w http.ResponseWriter, r *http.Request) {
if g.IsTrustable(r.RemoteAddr) {
w.Write([]byte("exiting..."))
go func() {
time.Sleep(time.Second)
os.Exit(0)
}()
} else {
w.Write([]byte("no privilege"))
}
})
http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
if g.IsTrustable(r.RemoteAddr) {
g.ParseConfig(g.ConfigFile)
RenderDataJson(w, g.Config())
} else {
w.Write([]byte("no privilege"))
}
})
http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, file.SelfDir())
})
http.HandleFunc("/ips", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, g.TrustableIps())
})
}
func Start() {
if !g.Config().Http.Enabled {
return
}
addr := g.Config().Http.Listen
if addr == "" {
return
}
s := &http.Server{
Addr: addr,
MaxHeaderBytes: 1 << 30,
}
log.Println("listening", addr)
log.Fatalln(s.ListenAndServe())
}
func syncTrustableIps() {
duration := time.Duration(g.Config().Heartbeat.Interval) * time.Second
for {
time.Sleep(duration)
var ips string
err := g.HbsClient.Call("Agent.TrustableIps", model.NullRpcRequest{}, &ips)
if err != nil {
log.Println("ERROR: call Agent.TrustableIps fail", err)
continue
}
g.SetTrustableIps(ips)
}
}
func BuildMappers() {
interval := g.Config().Transfer.Interval
Mappers = []FuncsAndInterval{
FuncsAndInterval{
Fs: []func() []*model.MetricValue{
AgentMetrics,
CpuMetrics,
NetMetrics,
KernelMetrics,
LoadAvgMetrics,
MemMetrics,
DiskIOMetrics,
IOStatsMetrics,
NetstatMetrics,
ProcMetrics,
UdpMetrics,
},
Interval: interval,
},
FuncsAndInterval{
Fs: []func() []*model.MetricValue{
DeviceMetrics,
},
Interval: interval,
},
FuncsAndInterval{
Fs: []func() []*model.MetricValue{
PortMetrics,
SocketStatSummaryMetrics,
},
Interval: interval,
},
FuncsAndInterval{
Fs: []func() []*model.MetricValue{
DuMetrics,
},
Interval: interval,
},
FuncsAndInterval{
Fs: []func() []*model.MetricValue{
UrlMetrics,
},
Interval: interval,
},
}
}
func collect(sec int64, fns []func() []*model.MetricValue) {
for {
REST:
time.Sleep(time.Duration(sec) * time.Second)
hostname, err := g.Hostname()
if err != nil {
goto REST
}
mvs := []*model.MetricValue{}
ignoreMetrics := g.Config().IgnoreMetrics
for _, fn := range fns {
items := fn()
if items == nil {
continue
}
if len(items) == 0 {
continue
}
for _, mv := range items {
if b, ok := ignoreMetrics[mv.Metric]; ok && b {
continue
} else {
mvs = append(mvs, mv)
}
}
}
now := time.Now().Unix()
for j := 0; j < len(mvs); j++ {
mvs[j].Step = sec
mvs[j].Endpoint = hostname
mvs[j].Timestamp = now
}
g.SendToTransfer(mvs)
}
}
// key: sys/ntp/60_ntp.py
func ListPlugins(relativePath string) map[string]*Plugin {
ret := make(map[string]*Plugin)
if relativePath == "" {
return ret
}
dir := filepath.Join(g.Config().Plugin.Dir, relativePath)
if !file.IsExist(dir) || file.IsFile(dir) {
return ret
}
fs, err := ioutil.ReadDir(dir)
if err != nil {
log.Println("can not list files under", dir)
return ret
}
for _, f := range fs {
if f.IsDir() {
continue
}
filename := f.Name()
arr := strings.Split(filename, "_")
if len(arr) < 2 {
continue
}
// filename should be: $cycle_$xx
var cycle int
cycle, err = strconv.Atoi(arr[0])
if err != nil {
continue
}
fpath := filepath.Join(relativePath, filename)
plugin := &Plugin{FilePath: fpath, MTime: f.ModTime().Unix(), Cycle: cycle}
ret[fpath] = plugin
}
return ret
}
func syncBuiltinMetrics() {
var timestamp int64 = -1
var checksum string = "nil"
duration := time.Duration(g.Config().Heartbeat.Interval) * time.Second
for {
time.Sleep(duration)
var ports = []int64{}
var paths = []string{}
var procs = make(map[string]map[int]string)
var urls = make(map[string]string)
hostname, err := g.Hostname()
if err != nil {
continue
}
req := model.AgentHeartbeatRequest{
Hostname: hostname,
Checksum: checksum,
}
var resp model.BuiltinMetricResponse
err = g.HbsClient.Call("Agent.BuiltinMetrics", req, &resp)
if err != nil {
log.Println("ERROR:", err)
continue
}
if resp.Timestamp <= timestamp {
continue
}
if resp.Checksum == checksum {
continue
}
timestamp = resp.Timestamp
checksum = resp.Checksum
for _, metric := range resp.Metrics {
if metric.Metric == g.URL_CHECK_HEALTH {
arr := strings.Split(metric.Tags, ",")
if len(arr) != 2 {
continue
}
url := strings.Split(arr[0], "=")
if len(url) != 2 {
continue
}
stime := strings.Split(arr[1], "=")
if len(stime) != 2 {
continue
}
if _, err := strconv.ParseInt(stime[1], 10, 64); err == nil {
urls[url[1]] = stime[1]
} else {
log.Println("metric ParseInt timeout failed:", err)
}
}
if metric.Metric == g.NET_PORT_LISTEN {
arr := strings.Split(metric.Tags, "=")
if len(arr) != 2 {
continue
}
if port, err := strconv.ParseInt(arr[1], 10, 64); err == nil {
ports = append(ports, port)
} else {
log.Println("metrics ParseInt failed:", err)
}
continue
}
if metric.Metric == g.DU_BS {
arr := strings.Split(metric.Tags, "=")
if len(arr) != 2 {
continue
}
paths = append(paths, strings.TrimSpace(arr[1]))
continue
}
if metric.Metric == g.PROC_NUM {
arr := strings.Split(metric.Tags, ",")
tmpMap := make(map[int]string)
for i := 0; i < len(arr); i++ {
if strings.HasPrefix(arr[i], "name=") {
tmpMap[1] = strings.TrimSpace(arr[i][5:])
} else if strings.HasPrefix(arr[i], "cmdline=") {
tmpMap[2] = strings.TrimSpace(arr[i][8:])
}
}
procs[metric.Tags] = tmpMap
}
}
g.SetReportUrls(urls)
g.SetReportPorts(ports)
g.SetReportProcs(procs)
g.SetDuPaths(paths)
}
}
func ReportAgentStatus() {
if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
go reportAgentStatus(time.Duration(g.Config().Heartbeat.Interval) * time.Second)
}
}
func SyncBuiltinMetrics() {
if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
go syncBuiltinMetrics()
}
}
func SyncTrustableIps() {
if g.Config().Heartbeat.Enabled && g.Config().Heartbeat.Addr != "" {
go syncTrustableIps()
}
}
func syncBuiltinMetrics() {
var timestamp int64 = -1
var checksum string = "nil"
duration := time.Duration(g.Config().Heartbeat.Interval) * time.Second
for {
REST:
time.Sleep(duration)
var ports = []int64{}
var procs = make(map[string]map[int]string)
hostname, err := g.Hostname()
if err != nil {
goto REST
}
req := model.AgentHeartbeatRequest{
Hostname: hostname,
Checksum: checksum,
}
var resp model.BuiltinMetricResponse
err = g.HbsClient.Call("Agent.BuiltinMetrics", req, &resp)
if err != nil {
log.Println("ERROR:", err)
goto REST
}
if resp.Timestamp <= timestamp {
goto REST
}
if resp.Checksum == checksum {
goto REST
}
timestamp = resp.Timestamp
checksum = resp.Checksum
for _, metric := range resp.Metrics {
if metric.Metric == "net.port.listen" {
if !strings.Contains(metric.Tags, "=") {
// illegal
continue
}
if port, err := strconv.ParseInt(metric.Tags[5:], 10, 64); err == nil {
ports = append(ports, port)
}
continue
}
if metric.Metric == "proc.num" {
arr := strings.Split(metric.Tags, ",")
tmpMap := make(map[int]string)
for i := 0; i < len(arr); i++ {
if strings.HasPrefix(arr[i], "name=") {
tmpMap[1] = arr[i][5:]
} else if strings.HasPrefix(arr[i], "cmdline=") {
tmpMap[2] = arr[i][8:]
}
}
procs[metric.Tags] = tmpMap
}
}
g.SetReportPorts(ports)
g.SetReportProcs(procs)
}
}
func NetMetrics() []*model.MetricValue {
return CoreNetMetrics(g.Config().Collector.IfacePrefix)
}
func PluginRun(plugin *Plugin) {
timeout := plugin.Cycle*1000 - 500
fpath := filepath.Join(g.Config().Plugin.Dir, plugin.FilePath)
if !file.IsExist(fpath) {
log.Println("no such plugin:", fpath)
return
}
debug := g.Config().Debug
if debug {
log.Println(fpath, "running...")
}
cmd := exec.Command(fpath)
var stdout bytes.Buffer
cmd.Stdout = &stdout
var stderr bytes.Buffer
cmd.Stderr = &stderr
cmd.Start()
err, isTimeout := sys.CmdRunWithTimeout(cmd, time.Duration(timeout)*time.Millisecond)
errStr := stderr.String()
if errStr != "" {
logFile := filepath.Join(g.Config().Plugin.LogDir, plugin.FilePath+".stderr.log")
if _, err = file.WriteString(logFile, errStr); err != nil {
log.Printf("[ERROR] write log to %s fail, error: %s\n", logFile, err)
}
}
if isTimeout {
// has be killed
if err == nil && debug {
log.Println("[INFO] timeout and kill process", fpath, "successfully")
}
if err != nil {
log.Println("[ERROR] kill process", fpath, "occur error:", err)
}
return
}
if err != nil {
log.Println("[ERROR] exec plugin", fpath, "fail. error:", err)
return
}
// exec successfully
data := stdout.Bytes()
if len(data) == 0 {
if debug {
log.Println("[DEBUG] stdout of", fpath, "is blank")
}
return
}
var metrics []*model.MetricValue
err = json.Unmarshal(data, &metrics)
if err != nil {
log.Printf("[ERROR] json.Unmarshal stdout of %s fail. error:%s stdout: \n%s\n", fpath, err, stdout.String())
return
}
g.SendToTransfer(metrics)
}