func (e *Exporter) scrapeMetrics(json *gabs.Container, ch chan<- prometheus.Metric) {
elements, _ := json.ChildrenMap()
for key, element := range elements {
switch key {
case "message":
log.Errorf("Problem collecting metrics: %s\n", element.Data().(string))
return
case "version":
data := element.Data()
version, ok := data.(string)
if !ok {
log.Errorf(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for version\n", data))
} else {
gauge, _ := e.Gauges.Fetch("metrics_version", "Marathon metrics version", "version")
gauge.WithLabelValues(version).Set(1)
gauge.Collect(ch)
}
case "counters":
e.scrapeCounters(element)
case "gauges":
e.scrapeGauges(element)
case "histograms":
e.scrapeHistograms(element)
case "meters":
e.scrapeMeters(element)
case "timers":
e.scrapeTimers(element)
}
}
}
func testGetMultStatementOfGroup(t *testing.T, group *gabs.Container) {
db := initDatabase(t)
defer db.Close()
mart := initHandler(db)
ids := []string{
uuid.NewV4().String(),
uuid.NewV4().String(),
uuid.NewV4().String(),
uuid.NewV4().String(),
}
stmt, err := gabs.ParseJSON([]byte(statetmentAgentBoilerplate))
fatalIfError(t, err)
stmtSlice := []*gabs.Container{
appendAccount(t, stmt),
appendMbox(t, stmt),
appendMboxSHA1(t, stmt),
appendOpenID(t, stmt),
}
for i := 0; i < len(ids); i++ {
putStatement(t, mart, stmtSlice[i].String(), ids[i])
var s interface{}
err = json.Unmarshal([]byte(stmtSlice[i].Search("actor").String()), &s)
fatalIfError(t, err)
err = group.ArrayAppendP(s, "actor.member")
fatalIfError(t, err)
}
// construct query
v := &url.Values{}
//t.Log(group.Search("actor").String())
v.Add("agent", group.Search("actor").String())
resp := getStatement(t, mart, v)
//t.Log(string(resp))
respstmt, err := gabs.ParseJSON(resp)
fatalIfError(t, err)
cnt, err := respstmt.ArrayCount("statements")
fatalIfError(t, err)
if cnt != len(ids) {
t.Fatalf("Expected %d statements in response; got %d", len(ids), cnt)
}
children, err := respstmt.S("statements").Children()
fatalIfError(t, err)
for idx, stm := range children {
if id, ok := stm.Search("id").Data().(string); !ok || id != ids[idx] {
t.Fatalf("Got invalid order of statement array")
}
}
}
// Save guest configuration to path from JSON format.
func saveGuestConfig(config *gabs.Container, path string) (err error) {
s := config.String()
dst, err := os.Create(path)
if err != nil {
return
}
defer dst.Close()
_, err = dst.WriteString(s)
return
}
func appendOpenID(t *testing.T, stmt *gabs.Container) *gabs.Container {
stmt, err := gabs.ParseJSON([]byte(stmt.String()))
fatalIfError(t, err)
domainPrefix := uuid.NewV4().String()
openID := "http://" + domainPrefix + ".openid.example.com/"
_, err = stmt.SetP(openID, "actor.openid")
fatalIfError(t, err)
return stmt
}
func PublicUserString(token *gabs.Container) string {
s, err := gabs.ParseJSON([]byte(token.String()))
if err != nil {
return "{}"
}
s.SetP("", "privid")
s.SetP("", "alertaddress")
return s.String()
}
func (e *Exporter) scrapeTimers(json *gabs.Container) {
elements, _ := json.ChildrenMap()
for key, element := range elements {
new, err := e.scrapeTimer(key, element)
if err != nil {
log.Debug(err)
} else if new {
log.Infof("Added timer %q\n", key)
}
}
}
func appendMbox(t *testing.T, stmt *gabs.Container) *gabs.Container {
stmt, err := gabs.ParseJSON([]byte(stmt.String()))
fatalIfError(t, err)
domainPrefix := uuid.NewV4().String()
mbox := "mbox:test@" + domainPrefix + ".example.com"
_, err = stmt.SetP(mbox, "actor.mbox")
fatalIfError(t, err)
return stmt
}
func (e *Exporter) scrapeCounter(key string, json *gabs.Container) (bool, error) {
data := json.Path("count").Data()
count, ok := data.(float64)
if !ok {
return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for counter %s\n", data, key))
}
name := renameMetric(key)
help := fmt.Sprintf(counterHelp, key)
counter, new := e.Counters.Fetch(name, help)
counter.WithLabelValues().Set(count)
return new, nil
}
func (e *Exporter) scrapeGauge(key string, json *gabs.Container) (bool, error) {
data := json.Path("value").Data()
value, ok := data.(float64)
if !ok {
return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for gauge %s\n", data, key))
}
name := renameMetric(key)
help := fmt.Sprintf(gaugeHelp, key)
gauge, new := e.Gauges.Fetch(name, help)
gauge.WithLabelValues().Set(value)
return new, nil
}
func appendMboxSHA1(t *testing.T, stmt *gabs.Container) *gabs.Container {
stmt, err := gabs.ParseJSON([]byte(stmt.String()))
fatalIfError(t, err)
data := make([]byte, 16)
_, err = rand.Read(data)
fatalIfError(t, err)
mboxSHA1 := fmt.Sprintf("%x", sha1.Sum(data))
_, err = stmt.SetP(mboxSHA1, "actor.mbox_sha1sum")
fatalIfError(t, err)
return stmt
}
func (svc *Service) ExtractAndCheckToken(so socketio.Socket, g *gabs.Container) (string, bool) {
if g.Path("t").Data() == nil {
so.Emit("auth_error", "Missing Token")
return "", false
}
uid, err := svc.ValidateUserToken(nil, g.Path("t").Data().(string))
if err != nil {
so.Emit("auth_error", "Invalid Token")
return "", false
}
return uid, true
}
func (e *Exporter) scrapeApps(json *gabs.Container, ch chan<- prometheus.Metric) {
elements, _ := json.S("apps").Children()
states := map[string]string{
"running": "tasksRunning",
"staged": "tasksStaged",
"healthy": "tasksHealthy",
"unhealthy": "tasksUnhealthy",
"cpus": "cpus",
"mem_in_mb": "mem",
"disk_in_mb": "disk",
"gpus": "gpus",
"avg_uptime": "taskStats.startedAfterLastScaling.stats.lifeTime.averageSeconds",
}
name := "app_instances"
gauge, new := e.Gauges.Fetch(name, "Marathon app instance count", "app")
if new {
log.Infof("Added gauge %q\n", name)
}
gauge.Reset()
for _, app := range elements {
id := app.Path("id").Data().(string)
data := app.Path("instances").Data()
count, ok := data.(float64)
if !ok {
log.Debugf(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for number of app instances\n", data))
continue
}
gauge.WithLabelValues(id).Set(count)
for key, value := range states {
name := fmt.Sprintf("app_task_%s", key)
gauge, new := e.Gauges.Fetch(name, fmt.Sprintf("Marathon app task %s count", key), "app")
if new {
log.Infof("Added gauge %q\n", name)
}
data := app.Path(value).Data()
count, ok := data.(float64)
if !ok {
log.Debugf(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for number of \"%s\" tasks\n", data, key))
continue
}
gauge.WithLabelValues(id).Set(count)
}
}
}
func documentFromHit(hit *gabs.Container) *Document {
return &Document{
Index: hit.Path("_index").Data().(string),
Type: hit.Path("_type").Data().(string),
Id: hit.Path("_id").Data().(string),
source: []byte(hit.Path("_source").String()),
}
}
func (e *Exporter) scrapeHistogram(key string, json *gabs.Container) (bool, error) {
count, ok := json.Path("count").Data().(float64)
if !ok {
return false, errors.New(fmt.Sprintf("Bad historgram! %s has no count\n", key))
}
name := renameMetric(key)
help := fmt.Sprintf(histogramHelp, key)
counter, new := e.Counters.Fetch(name+"_count", help)
counter.WithLabelValues().Set(count)
percentiles, _ := e.Gauges.Fetch(name, help, "percentile")
max, _ := e.Gauges.Fetch(name+"_max", help)
mean, _ := e.Gauges.Fetch(name+"_mean", help)
min, _ := e.Gauges.Fetch(name+"_min", help)
stddev, _ := e.Gauges.Fetch(name+"_stddev", help)
properties, _ := json.ChildrenMap()
for key, property := range properties {
switch key {
case "p50", "p75", "p95", "p98", "p99", "p999":
if value, ok := property.Data().(float64); ok {
percentiles.WithLabelValues("0." + key[1:]).Set(value)
}
case "min":
if value, ok := property.Data().(float64); ok {
min.WithLabelValues().Set(value)
}
case "max":
if value, ok := property.Data().(float64); ok {
max.WithLabelValues().Set(value)
}
case "mean":
if value, ok := property.Data().(float64); ok {
mean.WithLabelValues().Set(value)
}
case "stddev":
if value, ok := property.Data().(float64); ok {
stddev.WithLabelValues().Set(value)
}
}
}
return new, nil
}
// TODO: Properly get commit count
func PushEvent(data *gabs.Container) {
repo, _ := data.S("repository", "full_name").Data().(string)
user, _ := data.S("pusher", "name").Data().(string)
gitio := GitioShort(data.S("head_commit", "url").Data().(string))
commits, _ := data.S("commits").Children()
cobj, _ := data.S("commits").ChildrenMap()
fmt.Printf("[!] Commit # %d\n", len(cobj))
commitlen := strconv.Itoa(len(cobj))
message <- "[" + repo + "] " + user + " pushed " + commitlen + " commits " + gitio
for _, commit := range commits {
hash := commit.S("id").Data().(string)[0:6]
msg := commit.S("message").Data().(string)
user := commit.S("author", "name").Data().(string)
message <- "[" + repo + "] " + user + " " + hash + " - " + msg
}
}
func (e *Exporter) scrapeMeter(key string, json *gabs.Container) (bool, error) {
count, ok := json.Path("count").Data().(float64)
if !ok {
return false, errors.New(fmt.Sprintf("Bad meter! %s has no count\n", key))
}
units, ok := json.Path("units").Data().(string)
if !ok {
return false, errors.New(fmt.Sprintf("Bad meter! %s has no units\n", key))
}
name := renameMetric(key)
help := fmt.Sprintf(meterHelp, key, units)
counter, new := e.Counters.Fetch(name+"_count", help)
counter.WithLabelValues().Set(count)
gauge, _ := e.Gauges.Fetch(name, help, "rate")
properties, _ := json.ChildrenMap()
for key, property := range properties {
if strings.Contains(key, "rate") {
if value, ok := property.Data().(float64); ok {
gauge.WithLabelValues(renameRate(key)).Set(value)
}
}
}
return new, nil
}
func testGetMultStatementOfActor(t *testing.T, stmt *gabs.Container) {
db := initDatabase(t)
defer db.Close()
mart := initHandler(db)
id1 := uuid.NewV4().String()
id2 := uuid.NewV4().String()
putStatement(t, mart, stmt.String(), id1)
putStatement(t, mart, stmt.String(), id2)
// construct query
v := &url.Values{}
v.Add("agent", stmt.Search("actor").String())
respstmt, err := gabs.ParseJSON(getStatement(t, mart, v))
fatalIfError(t, err)
cnt, err := respstmt.ArrayCount("statements")
fatalIfError(t, err)
if cnt != 2 {
t.Fatalf("Expected 2 statements in response; got %d", cnt)
}
s0, err := respstmt.ArrayElement(0, "statements")
fatalIfError(t, err)
if id, ok := s0.Search("id").Data().(string); !ok || id != id1 {
t.Fatalf("Got invalid order of statement array")
}
s1, err := respstmt.ArrayElement(1, "statements")
fatalIfError(t, err)
if id, ok := s1.Search("id").Data().(string); !ok || id != id2 {
t.Fatalf("Got invalid order of statement array")
}
}
func appendAccount(t *testing.T, stmt *gabs.Container) *gabs.Container {
stmt, err := gabs.ParseJSON([]byte(stmt.String()))
fatalIfError(t, err)
suffix := uuid.NewV4().String()
name := "name-example-" + suffix
homePage := "http://example.com"
_, err = stmt.SetP(name, "actor.account.name")
fatalIfError(t, err)
_, err = stmt.SetP(homePage, "actor.account.homePage")
fatalIfError(t, err)
return stmt
}
// signupHandler controls the signup and user profile update forms
func signupHandler(w http.ResponseWriter, r *http.Request) {
if strings.ToUpper(r.Method) == "POST" {
invite, ok := service.Invites[r.FormValue("invite")]
tokenenc := r.FormValue("token")
var token *gabs.Container
if ok {
delete(service.Invites, r.FormValue("invite"))
token, _ = assemble.CreateNewUserToken(
r.FormValue("nick"),
r.FormValue("name"),
r.FormValue("email"),
r.FormValue("phone"),
r.FormValue("url"),
r.FormValue("desc"),
r.FormValue("avatar"),
r.FormValue("alertaddress"))
} else if tokenenc != "" {
uid, tokerr := service.ValidateUserToken(nil, tokenenc)
if tokerr == nil {
privid := service.Users[uid].Token.Path("privid").Data().(string)
token, _ = assemble.CreateUpdatedUserToken(
r.FormValue("nick"),
r.FormValue("name"),
r.FormValue("email"),
r.FormValue("phone"),
r.FormValue("url"),
r.FormValue("desc"),
r.FormValue("avatar"),
r.FormValue("alertaddress"),
uid, privid)
log.Println(uid, "updated user token")
} else {
tokenenc = ""
}
}
if ok || tokenenc != "" {
competok := utils.Compress([]byte(token.String()))
etok, _ := utils.Encrypt(service.UserKey, competok.Bytes())
service.AddToRoom(nil, token.Path("uid").Data().(string), "lobby")
//TODO use templates
fmt.Fprintf(w, `<html>`)
fmt.Fprintf(w, `<meta http-equiv="refresh" content="10; url=/#%s">`, base64.StdEncoding.EncodeToString(etok))
if ok {
fmt.Fprintf(w, `<strong>SUCCESS!</strong> `+invite+`<br><br>`)
} else {
fmt.Fprintf(w, `<strong>Delete your old login bookmark and close the window or you will still be using your old profile!</strong><br><br>`)
}
//fmt.Fprintf(w, "Token (KEEP THIS SOMEWHERE SAFE OR SAVE THE LOGIN LINK): <br><textarea rows='10' cols='60'>%s</textarea><br><br>", base64.StdEncoding.EncodeToString(etok))
fmt.Fprintf(w, "<a href='/#%s' target='_blank'>Assemble Chat Login - CLICK HERE AND BOOKMARK THE CHAT!</a> <strong>DO NOT SHARE THIS LINK</strong>", base64.StdEncoding.EncodeToString(etok))
fmt.Fprintf(w, "<br>You will automatically enter the chat in 10 seconds...")
fmt.Fprintf(w, `</html>`)
} else {
fmt.Fprintf(w, `Invalid Invite ID or Token`)
}
} else {
fc, _ := ioutil.ReadFile("./static/signup.html")
if fc != nil {
fmt.Fprintf(w, string(fc[:]))
}
}
}
func SendMessage(cn, msg, cid string, container *gabs.Container) {
counter[cid]++
t := time.Now()
timestr := t.Format(time.RFC3339Nano)
container.Set(timestr, "timestamp")
container.Set(UserId, "userid")
container.Set(counter[cid], "counter")
container.Set(ClusterId, "clusterid")
container.Set(UUID, "uuid")
container.Set(IP, "ip")
container.Set(Hostname, "hostname")
ars := strings.Split(cn, " ")
if len(ars) == 3 {
container.Set(ars[0], "typename")
container.Set(ars[1], "taskid")
container.Set(ars[2], "ports")
container.Set(msg, "msg")
} else {
container.Set("", "msg")
}
}
func search(data *gabs.Container, filter tFilter, i int) (*gabs.Container, string) {
var ret string //returned value
filter.Path = filter.Path[i:]
//fmt.Println("I:::", i, filter.Path, reflect.TypeOf(data.Data()).String())
//fmt.Println(data)
// fmt.Println(data.Search(filter.Key).String())
if len(filter.Value) > 0 {
if data.Search(filter.Key).Data() == filter.Value {
ret = data.Search(filter.Ret).Data().(string)
if len(filter.Match) > 0 {
if regexp.MustCompile(filter.Match).MatchString(ret) {
if len(filter.Replace) > 0 {
ret = regexp.MustCompile(filter.Replace).ReplaceAllString(ret, "")
}
return data, ret
} else {
ret = ""
}
} else {
if len(filter.Replace) > 0 {
ret = regexp.MustCompile(filter.Replace).ReplaceAllString(ret, "")
}
return data, ret
}
}
}
for j, str := range filter.Path {
if data.Data() != nil {
if reflect.TypeOf(data.Data()).String() == "[]interface {}" {
children, err := data.Children()
if err != nil {
fmt.Println(err.Error())
}
for _, child := range children {
data, ret = search(child, filter, j)
if len(ret) > 0 {
return data, ret
}
}
} else {
data = data.Search(str)
}
}
// fmt.Println(str, data)
}
return data, ret
}
// Merge ("generate?") guest network configuration from host configuration.
func mergeNetConfig(netconfig *netConfig, guest *gabs.Container) (err error) {
if guest.ExistsP("hostname") {
log.Print("Guest already defines hostname, not overriden")
} else {
guest.SetP(netconfig.hostname, "hostname")
}
if guest.ExistsP("net") {
return fmt.Errorf("Guest already defines net configuration")
}
addrs, _ := guest.ArrayOfSizeP(1, "net.interfaces.vioif0.addrs")
addr, _ := addrs.ObjectI(0)
addr.Set("inet", "type")
addr.Set("static", "method")
addr.Set(netconfig.ipAddress, "addr")
gateways, _ := guest.ArrayOfSizeP(1, "net.gateways")
gw, _ := gateways.ObjectI(0)
gw.Set("inet", "type")
gw.Set(netconfig.gateway, "addr")
dns, _ := guest.ObjectP("net.dns")
dns.Set(netconfig.dnsServers, "nameservers")
dns.Set(netconfig.dnsSearch, "search")
return
}
// Merge guest volume/block device configuration with any volumes found in
// host container.
func mergeVolumeConfig(volumeconfig []volumeConfig, guest *gabs.Container) (err error) {
var mounts *gabs.Container
if guest.ExistsP("mount") {
mounts = guest.S("mount")
} else {
mounts, _ = guest.ObjectP("mount")
}
mmap, _ := mounts.ChildrenMap()
for _, mount := range mmap {
var source string
var path string
var ok bool
if source, ok = mount.Path("source").Data().(string); !ok {
continue
}
if path, ok = mount.Path("path").Data().(string); !ok {
continue
}
if source == "blk" && strings.HasPrefix(path, "/dev/ld") {
return fmt.Errorf("Guest configuration defines " +
"/dev/ld* block devices, cannot merge with " +
"host volumes")
}
}
blkIndex := 0
for _, v := range volumeconfig {
mp := "/" + v.name
obj, _ := guest.Object("mount", mp)
obj.Set("blk", "source")
obj.Set(fmt.Sprintf("/dev/ld%va", blkIndex), "path")
blkIndex += 1
}
return
}
func cleanToken(token *gabs.Container) {
token.SetP(html.EscapeString(token.Path("nick").Data().(string)), "nick")
token.SetP(html.EscapeString(token.Path("uid").Data().(string)), "uid")
token.SetP(html.EscapeString(token.Path("name").Data().(string)), "name")
token.SetP(html.EscapeString(token.Path("email").Data().(string)), "email")
token.SetP(html.EscapeString(token.Path("phone").Data().(string)), "phone")
token.SetP(html.EscapeString(token.Path("url").Data().(string)), "url")
token.SetP(html.EscapeString(token.Path("desc").Data().(string)), "desc")
token.SetP(html.EscapeString(token.Path("avatar").Data().(string)), "avatar")
}
func IssuesEvent(data *gabs.Container) {
action := data.S("action").Data().(string)
repo, _ := data.S("repository", "full_name").Data().(string)
user, _ := data.S("issue", "user", "login").Data().(string)
title, _ := data.S("issue", "title").Data().(string)
inum, _ := data.S("issue", "id").Data().(string)
gitio := GitioShort(data.S("issue", "html_url").Data().(string))
switch action {
case "opened":
message <- "[" + repo + "] " + user + " opened issue #" + inum + " \"" + title + "\" " + gitio
case "closed":
message <- "[" + repo + "] " + user + " closed issue #" + inum + " \"" + title + "\" " + gitio
case "reopened":
message <- "[" + repo + "] " + user + " reopened issue #" + inum + " \"" + title + "\" " + gitio
case "assigned":
assignee, _ := data.S("issue", "assignee", "login").Data().(string)
message <- "[" + repo + "] " + user + " assigned issue #" + inum + " \"" + title + "\" to " + assignee + " " + gitio
default:
// Ignore it
}
}
func PullRequestEvent(data *gabs.Container) {
action := data.S("action").Data().(string)
repo, _ := data.S("repository", "full_name").Data().(string)
user, _ := data.S("pull_request", "user", "login").Data().(string)
title, _ := data.S("pull_request", "title").Data().(string)
inum, _ := data.S("pull_request", "number").Data().(string)
gitio := GitioShort(data.S("pull_request", "html_url").Data().(string))
switch action {
case "opened":
message <- "[" + repo + "] " + user + " opened pull request #" + inum + " \"" + title + "\" " + gitio
case "closed":
if data.S("pull_request", "merged").Data().(bool) {
message <- "[" + repo + "] " + user + " merged pull request #" + inum + " \"" + title + "\" " + gitio
} else {
message <- "[" + repo + "] " + user + " closed pull request #" + inum + " \"" + title + "\" " + gitio
}
case "assigned":
assignee, _ := data.S("pull_request", "assignee", "login").Data().(string)
message <- "[" + repo + "] " + user + " assigned pull request #" + inum + " \"" + title + "\" to " + assignee + " " + gitio
default:
// Ignore it
}
}