/*
* Initializations
*/
func init() {
flag.Parse()
status.InputEventCount = expvar.NewInt("input_event_count")
status.OutputEventCount = expvar.NewInt("output_event_count")
status.ErrorCount = expvar.NewInt("error_count")
expvar.Publish("connection_status",
expvar.Func(func() interface{} {
res := make(map[string]interface{}, 0)
res["last_connect_time"] = status.LastConnectTime
res["last_error_text"] = status.LastConnectError
res["last_error_time"] = status.ErrorTime
if status.IsConnected {
res["connected"] = true
res["uptime"] = time.Now().Sub(status.LastConnectTime).Seconds()
} else {
res["connected"] = false
res["uptime"] = 0.0
}
return res
}))
expvar.Publish("uptime", expvar.Func(func() interface{} {
return time.Now().Sub(status.StartTime).Seconds()
}))
expvar.Publish("subscribed_events", expvar.Func(func() interface{} {
return config.EventTypes
}))
results = make(chan string, 100)
output_errors = make(chan error)
status.StartTime = time.Now()
}
func init() {
expvar.Publish(kapacitor.ClusterIDVarName, cidVar)
expvar.Publish(kapacitor.ServerIDVarName, sidVar)
expvar.Publish(kapacitor.HostVarName, hostVar)
expvar.Publish(kapacitor.ProductVarName, productVar)
expvar.Publish(kapacitor.VersionVarName, versionVar)
}
func StartDebugServer(address string, sink *lager.ReconfigurableSink, metrics Metrics) (ifrit.Process, error) {
expvar.Publish("numCPUS", expvar.Func(func() interface{} {
return metrics.NumCPU()
}))
expvar.Publish("numGoRoutines", expvar.Func(func() interface{} {
return metrics.NumGoroutine()
}))
expvar.Publish("loopDevices", expvar.Func(func() interface{} {
return metrics.LoopDevices()
}))
expvar.Publish("backingStores", expvar.Func(func() interface{} {
return metrics.BackingStores()
}))
expvar.Publish("depotDirs", expvar.Func(func() interface{} {
return metrics.DepotDirs()
}))
server := http_server.New(address, handler(sink))
p := ifrit.Invoke(server)
select {
case <-p.Ready():
case err := <-p.Wait():
return nil, err
}
return p, nil
}
func init() {
besticon.SetCacheMaxSize(128)
expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
}
func init() {
expvar.Publish("now", expvar.Func(func() interface{} {
return time.Now().Format("\"2006-01-02 15:04:05\"")
}))
stats = &Stats{}
expvar.Publish("stats", stats)
hits = expvar.NewMap("hits").Init()
}
func fanout(
pubc chan Line,
subc, unsubc chan Subscription,
histc <-chan HistoryRequest,
histSize int,
) {
var (
hist = NewHistory(histSize)
reg = registry{}
publishs = expvar.NewMap("publishs")
subscribes = expvar.NewMap("subscribes")
unsubscribes = expvar.NewMap("unsubscribes")
)
expvar.Publish("history", hist)
expvar.Publish("subscriptions", reg)
for {
select {
case l := <-pubc:
for _, s := range reg[l.Topic()] {
s.In() <- l
}
hist.Store(l)
publishs.Add(l.Topic(), 1)
case s := <-subc:
_, ok := reg[s.Topic()]
if !ok {
reg[s.Topic()] = map[string]Subscription{}
}
reg[s.Topic()][s.ID()] = s
subscribes.Add(s.Topic(), 1)
logf("fanout: subscribed %s\n", s.ID())
case s := <-unsubc:
subs, ok := reg[s.Topic()]
if !ok {
continue
}
_, ok = subs[s.ID()]
if !ok {
continue
}
delete(subs, s.ID())
unsubscribes.Add(s.Topic(), 1)
logf("fanout: unsubscribed %s\n", s.ID())
case req := <-histc:
req.Respond(hist.Get(req.Topic(), req.Size()))
}
}
}
func initDebug(mux *http.ServeMux) {
stats.prev = time.Now()
expvar.Publish("dcp", expvar.Func(dcp))
expvar.Publish("goroutines", expvar.Func(goroutines))
mux.Handle("/debug/vars/", http.HandlerFunc(expvarHandler))
mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
}
func main() {
flag.Parse()
if err := loadContext(); err != nil {
panic(err)
}
ctx := context()
expvar.Publish("config", &ctx.Config)
expvar.Publish("codemanager", expvar.Func(func() interface{} {
return context().InputManager
}))
expvar.Publish("queues", expvar.Func(func() interface{} {
return context().QueueManager
}))
expvar.Publish("inflight_runs", expvar.Func(func() interface{} {
return context().InflightMonitor
}))
cachePath := path.Join(ctx.Config.Grader.RuntimePath, "cache")
go ctx.InputManager.PreloadInputs(
cachePath,
grader.NewGraderCachedInputFactory(cachePath),
&sync.Mutex{},
)
// Database
db, err := sql.Open(
ctx.Config.Db.Driver,
ctx.Config.Db.DataSourceName,
)
if err != nil {
panic(err)
}
if err := db.Ping(); err != nil {
panic(err)
}
setupMetrics(ctx)
ctx.Log.Info("omegaUp grader started")
mux := http.DefaultServeMux
if ctx.Config.Grader.V1.Enabled {
registerV1CompatHandlers(mux, db)
go common.RunServer(&ctx.Config.TLS, mux, ctx.Config.Grader.V1.Port, *insecure)
mux = http.NewServeMux()
}
registerHandlers(mux, db)
common.RunServer(&ctx.Config.TLS, mux, ctx.Config.Grader.Port, *insecure)
}
func init() {
expvar.Publish("snmp_request_cache", expvar.Func(func() interface{} {
requests_mutex.Lock()
size := requests_cache.Size()
requests_mutex.Unlock()
return size
}))
expvar.Publish("snmp_bytes_cache", expvar.Func(func() interface{} {
bytes_mutex.Lock()
size := bytes_cache.Size()
bytes_mutex.Unlock()
return size
}))
}
// NewStatistics creates an expvar-based map. Within there "name" is the Measurement name, "tags" are the tags,
// and values are placed at the key "values".
// The "values" map is returned so that statistics can be set.
func NewStatistics(name string, tags map[string]string) *expvar.Map {
expvarMu.Lock()
defer expvarMu.Unlock()
key := uuid.NewV4().String()
m := &expvar.Map{}
m.Init()
expvar.Publish(key, m)
// Set the name
nameVar := &expvar.String{}
nameVar.Set(name)
m.Set("name", nameVar)
// Set the tags
tagsVar := &expvar.Map{}
tagsVar.Init()
for k, v := range tags {
value := &expvar.String{}
value.Set(v)
tagsVar.Set(k, value)
}
m.Set("tags", tagsVar)
// Create and set the values entry used for actual stats.
statMap := &expvar.Map{}
statMap.Init()
m.Set("values", statMap)
return statMap
}
func hook(m measurable.Measurable, event measurable.HookEvent) {
switch m.MsType() {
case measurable.CounterType, measurable.GaugeType:
// ok
default:
return // not supported
}
name := m.MsName()
switch event {
case measurable.RegisterEvent, measurable.RegisterCatchupEvent:
a := &adaptor{
measurable: m,
}
measurablesMutex.Lock()
defer measurablesMutex.Unlock()
measurables[name] = a
expvar.Publish(name, a)
case measurable.UnregisterEvent:
a := measurables[name]
if a != nil { // this should always be the case, but whatever
a.mutex.Lock()
defer a.mutex.Unlock()
a.measurable = nil
}
}
}
func NewCounters(name string) *Counters {
c := &Counters{counts: make(map[string]int64)}
if name != "" {
expvar.Publish(name, c)
}
return c
}
// Create a new simple moving percentile expvar.Var. It will be
// published under `name` and maintain `size` values for
// calculating the percentile.
//
// percentile must be between 0 and 1
//
// An empty name will cause it to not be published
func NewSimpleMovingPercentile(name string, percentile float64, size int) *SimpleMovingStat {
sm := new(SimpleMovingStat)
sm.size = size
sm.mutex = new(sync.Mutex)
sm.values = ring.New(size)
sm.calculate = func(s *SimpleMovingStat) float64 {
ary := make([]float64, 0)
s.values.Do(func(val interface{}) {
if val != nil {
ary = append(ary, val.(float64))
}
})
length := len(ary)
if length == 0 {
return 0.0
}
sort.Float64s(ary)
mid := int(float64(len(ary)) * percentile)
return ary[mid]
}
if name != "" {
expvar.Publish(name, sm)
}
return sm
}
func NewTimings(name string) *Timings {
t := &Timings{Histograms: make(map[string]*Histogram)}
if name != "" {
expvar.Publish(name, t)
}
return t
}
func main() {
// Initialise our configuration from flags
var nodeName = flag.String("name", REQUIRED, "Node network name and port, e.g. localhost:3000")
var gobName = flag.String("gob", "", "Alternative gob network name and port for clients, allowing clients to connect over a different physical interface to nodes.")
var httpName = flag.String("http", "", "Network name and port for the http ExpVar to listen on.")
var cborName = flag.String("cbor", "", "Network name and port for the CBOR RPC interface to listen on.")
var nodePath = flag.String("path", REQUIRED, "Node root path for configuration and log files")
var clusterID = flag.String("id", "", "Cluster ID that this node is part of")
flag.Parse()
if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
flag.PrintDefaults()
return
}
if *nodeName == REQUIRED {
log.Printf("name missing.\n")
flag.PrintDefaults()
return
}
if *nodePath == REQUIRED {
log.Printf("path missing.\n")
flag.PrintDefaults()
return
}
// Create our server
serverNode, err := server.NewServerNode(*nodeName, *gobName, *httpName, *cborName, *nodePath, *clusterID)
if err != nil {
log.Fatalf("Unable to start server due to errors.\n")
}
expvar.Publish("node", expvar.Func(serverNode.ExpVar))
//dataLog := &memlog.MemoryLog{}
// Start a listener to handle incoming requests from other peers
err = serverNode.ListenConnections()
if err != nil {
log.Fatalf("Error starting listener: %v\n", err)
}
// Setup signal handling to catch interrupts.
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
os.Interrupt,
os.Kill)
go func() {
<-sigc
serverNode.RequestShutdown("Request to terminate process detected.")
}()
serverNode.WaitForShutdown()
}
func init() {
raft.SetLogger(capnslog.NewPackageLogger("github.com/coreos/etcd", "raft"))
expvar.Publish("raft.status", expvar.Func(func() interface{} {
raftStatusMu.Lock()
defer raftStatusMu.Unlock()
return raftStatus()
}))
}
func init() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "Usage of inbucket [options] <conf file>:")
flag.PrintDefaults()
}
expvar.Publish("uptime", expvar.Func(uptime))
}
// StartSelfMonitor starts http server on random port and exports expvars.
//
// It tries 1024 ports, starting from startPort and registers some expvars if ok.
func StartSelfMonitor() (string, error) {
for port := startPort; port < startPort+1024; port++ {
bind := fmt.Sprintf("localhost:%d", port)
l, err := net.Listen("tcp", bind)
if err != nil {
continue
}
l.Close()
expvar.Publish("Goroutines", expvar.Func(goroutines))
expvar.Publish("Uptime", expvar.Func(uptime))
go http.ListenAndServe(bind, nil)
return bind, nil
}
return "", fmt.Errorf("no free ports found")
}
func init() {
expvar.NewString("goVersion").Set(runtime.Version())
expvar.NewString("iconVersion").Set(besticon.VersionString)
expvar.NewString("timeLastDeploy").Set(parseUnixTimeStamp(os.Getenv("DEPLOYED_AT")).String())
expvar.NewString("timeStartup").Set(time.Now().String())
expvar.Publish("timeCurrent", expvar.Func(func() interface{} { return time.Now() }))
}
func initExpvars() {
expvar.Publish("__binary_path", expvar.Func(func() interface{} { return &binaryPath }))
expvar.Publish("__config_path", expvar.Func(func() interface{} { return &configPath }))
expvar.Publish("_command_line", expvar.Func(func() interface{} { return &commandLine }))
expvar.Publish("_version", expvar.Func(func() interface{} { return VersionInfo }))
expvar.Publish("_hostname", expvar.Func(func() interface{} { return hostname }))
expvar.Publish("_config", expvar.Func(func() interface{} {
cf, err := structs.BadooStripSecretFields(config) // makes a copy of `config`
if err != nil {
return struct {
Error string `json:"error"`
}{err.Error()}
}
return cf
}))
expvar.Publish("_service-stats", expvar.Func(func() interface{} {
stats, err := GatherServiceStats()
if err != nil {
return struct {
Error string `json:"error"`
}{err.Error()}
}
return stats
}))
}
func init() {
expvar.Publish("kaman.metrics", expvar.Func(func() interface{} {
counters := Snapshot()
return map[string]interface{}{
"Counters": counters,
}
}))
}
func publishExtraVars() {
// By using sync.Once instead of an init() function, we don't clutter
// the app's expvar export unnecessarily, or risk colliding with it.
publishOnce.Do(func() {
expvar.Publish("Goroutines", expvar.Func(func() interface{} {
return runtime.NumGoroutine()
}))
})
}
func Run(address string, sink *lager.ReconfigurableSink) (ifrit.Process, error) {
expvar.Publish("numCPUS", expvar.Func(func() interface{} {
return int64(runtime.NumCPU())
}))
expvar.Publish("numGoRoutines", expvar.Func(func() interface{} {
return int64(runtime.NumGoroutine())
}))
server := http_server.New(address, handler(sink))
p := ifrit.Invoke(server)
select {
case <-p.Ready():
case err := <-p.Wait():
return nil, err
}
return p, nil
}
func (vg *varGroup) publish(name string, v expvar.Var) {
vg.Lock()
defer vg.Unlock()
expvar.Publish(name, v)
if vg.newVarHook != nil {
vg.newVarHook(name, v)
} else {
vg.vars[name] = v
}
}
// NewDerive initializes a new Derive, registers it with the "expvar" package
// and returns it. The initial value is zero.
func NewDerive(id api.Identifier) *Derive {
d := &Derive{
id: id,
value: 0,
}
Publish(d)
expvar.Publish(id.String(), d)
return d
}
// NewGauge initializes a new Gauge, registers it with the "expvar" package and
// returns it. The initial value is NaN.
func NewGauge(id api.Identifier) *Gauge {
g := &Gauge{
id: id,
value: api.Gauge(math.NaN()),
}
Publish(g)
expvar.Publish(id.String(), g)
return g
}
func NewStates(name string, labels []string, startTime time.Time, initialState int) *States {
s := &States{StateCount: len(labels), Labels: labels, CurrentState: initialState, CurrentStateStartTime: startTime, Durations: make([]time.Duration, len(labels)), Transitions: make([]int, len(labels))}
if initialState < 0 || initialState >= s.StateCount {
panic(fmt.Errorf("initialState out of range 0-%v: %v", s.StateCount, initialState))
}
if name != "" {
expvar.Publish(name, s)
}
return s
}
func startOutputs() error {
// Configure the specific output.
// Valid options are: 'udp', 'tcp', 'file', 's3'
var outputHandler OutputHandler
parameters := config.OutputParameters
switch config.OutputType {
case FileOutputType:
outputHandler = &FileOutput{}
case TCPOutputType:
outputHandler = &NetOutput{}
parameters = "tcp:" + parameters
case UDPOutputType:
outputHandler = &NetOutput{}
parameters = "udp:" + parameters
case S3OutputType:
outputHandler = &S3Output{}
default:
return errors.New(fmt.Sprintf("No valid output handler found (%d)", config.OutputType))
}
err := outputHandler.Initialize(parameters)
if err != nil {
return err
}
expvar.Publish("output_status", expvar.Func(func() interface{} {
ret := make(map[string]interface{})
ret[outputHandler.Key()] = outputHandler.Statistics()
switch config.OutputFormat {
case LEEFOutputFormat:
ret["format"] = "leef"
case JSONOutputFormat:
ret["format"] = "json"
}
switch config.OutputType {
case FileOutputType:
ret["type"] = "file"
case UDPOutputType:
ret["type"] = "net"
case TCPOutputType:
ret["type"] = "net"
case S3OutputType:
ret["type"] = "s3"
}
return ret
}))
log.Printf("Initialized output: %s\n", outputHandler.String())
return outputHandler.Go(results, output_errors)
}
func main() {
Whispers = whispers{metrics: make(map[string]*carbonmem.Whisper)}
flag.IntVar(&Whispers.windowSize, "w", 600, "window size")
flag.IntVar(&Whispers.epochSize, "e", 60, "epoch window size")
flag.IntVar(&Whispers.epoch0, "epoch0", 0, "epoch0")
flag.IntVar(&Whispers.prefix, "prefix", 0, "prefix nodes to shard on")
port := flag.Int("p", 8001, "port to listen on (http)")
gport := flag.Int("gp", 2003, "port to listen on (graphite)")
verbose := flag.Bool("v", false, "verbose logging")
logdir := flag.String("logdir", "/var/log/carbonmem/", "logging directory")
logtostdout := flag.Bool("stdout", false, "log also to stdout")
flag.Parse()
rl := rotatelogs.NewRotateLogs(
*logdir + "/carbonmem.%Y%m%d%H%M.log",
)
// Optional fields must be set afterwards
rl.LinkName = *logdir + "/carbonmem.log"
if *logtostdout {
log.SetOutput(io.MultiWriter(os.Stdout, rl))
} else {
log.SetOutput(rl)
}
expvar.NewString("BuildVersion").Set(BuildVersion)
log.Println("starting carbonmem", BuildVersion)
expvar.Publish("Whispers", expvar.Func(func() interface{} {
m := make(map[string]int)
Whispers.RLock()
for k, v := range Whispers.metrics {
m[k] = v.Len()
}
Whispers.RUnlock()
return m
}))
if Whispers.epoch0 == 0 {
Whispers.epoch0 = int(time.Now().Unix())
}
go graphiteServer(*gport)
http.HandleFunc("/metrics/find/", accessHandler(*verbose, findHandler))
http.HandleFunc("/render/", accessHandler(*verbose, renderHandler))
log.Println("http server starting on port", *port)
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
func RegisterCacheInvalidator() {
if CacheInvalidationProcessor != nil {
return
}
CacheInvalidationProcessor = NewInvalidationProcessor()
CacheInvalidationProcessor.states = estats.NewStates("", []string{
"Disabled",
"Enabled",
}, time.Now(), DISABLED)
expvar.Publish("CacheInvalidationProcessor", estats.StrFunc(func() string { return CacheInvalidationProcessor.statsJSON() }))
}