Esempio n. 1
0
func StartDebugServer(address string, sink *lager.ReconfigurableSink, metrics Metrics) (ifrit.Process, error) {
	expvar.Publish("numCPUS", expvar.Func(func() interface{} {
		return metrics.NumCPU()
	}))

	expvar.Publish("numGoRoutines", expvar.Func(func() interface{} {
		return metrics.NumGoroutine()
	}))

	expvar.Publish("loopDevices", expvar.Func(func() interface{} {
		return metrics.LoopDevices()
	}))

	expvar.Publish("backingStores", expvar.Func(func() interface{} {
		return metrics.BackingStores()
	}))

	expvar.Publish("depotDirs", expvar.Func(func() interface{} {
		return metrics.DepotDirs()
	}))

	server := http_server.New(address, handler(sink))
	p := ifrit.Invoke(server)
	select {
	case <-p.Ready():
	case err := <-p.Wait():
		return nil, err
	}
	return p, nil
}
Esempio n. 2
0
/*
 * Initializations
 */
func init() {
	flag.Parse()
	status.InputEventCount = expvar.NewInt("input_event_count")
	status.OutputEventCount = expvar.NewInt("output_event_count")
	status.ErrorCount = expvar.NewInt("error_count")

	expvar.Publish("connection_status",
		expvar.Func(func() interface{} {
			res := make(map[string]interface{}, 0)
			res["last_connect_time"] = status.LastConnectTime
			res["last_error_text"] = status.LastConnectError
			res["last_error_time"] = status.ErrorTime
			if status.IsConnected {
				res["connected"] = true
				res["uptime"] = time.Now().Sub(status.LastConnectTime).Seconds()
			} else {
				res["connected"] = false
				res["uptime"] = 0.0
			}

			return res
		}))
	expvar.Publish("uptime", expvar.Func(func() interface{} {
		return time.Now().Sub(status.StartTime).Seconds()
	}))
	expvar.Publish("subscribed_events", expvar.Func(func() interface{} {
		return config.EventTypes
	}))

	results = make(chan string, 100)
	output_errors = make(chan error)

	status.StartTime = time.Now()
}
Esempio n. 3
0
func newOriginAllower(blockedDomains []string, hostname string, gclog logClient, ns *expvar.Map) *originAllower {
	mu := &sync.RWMutex{}
	topKAllDomains := topk.New(100)
	topKOfflistDomains := topk.New(100)
	lifetime := new(expvar.Map).Init()
	ns.Set("lifetime", lifetime)
	lifetime.Set("top_all_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKAllDomains.Keys()
	}))
	lifetime.Set("top_offlist_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKOfflistDomains.Keys()
	}))

	oa := &originAllower{
		m:                  make(map[string]struct{}),
		ns:                 ns,
		hostname:           hostname,
		gclog:              gclog,
		mu:                 mu,
		topKAllDomains:     topKAllDomains,
		topKOfflistDomains: topKOfflistDomains,
	}
	for _, d := range blockedDomains {
		oa.m[d] = struct{}{}
	}
	return oa
}
Esempio n. 4
0
func init() {
	besticon.SetCacheMaxSize(128)

	expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
	expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
	expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
	expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
	expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
}
Esempio n. 5
0
func initDebug(mux *http.ServeMux) {
	stats.prev = time.Now()

	expvar.Publish("dcp", expvar.Func(dcp))
	expvar.Publish("goroutines", expvar.Func(goroutines))

	mux.Handle("/debug/vars/", http.HandlerFunc(expvarHandler))
	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
}
Esempio n. 6
0
func main() {
	flag.Parse()

	if err := loadContext(); err != nil {
		panic(err)
	}

	ctx := context()
	expvar.Publish("config", &ctx.Config)

	expvar.Publish("codemanager", expvar.Func(func() interface{} {
		return context().InputManager
	}))
	expvar.Publish("queues", expvar.Func(func() interface{} {
		return context().QueueManager
	}))
	expvar.Publish("inflight_runs", expvar.Func(func() interface{} {
		return context().InflightMonitor
	}))
	cachePath := path.Join(ctx.Config.Grader.RuntimePath, "cache")
	go ctx.InputManager.PreloadInputs(
		cachePath,
		grader.NewGraderCachedInputFactory(cachePath),
		&sync.Mutex{},
	)

	// Database
	db, err := sql.Open(
		ctx.Config.Db.Driver,
		ctx.Config.Db.DataSourceName,
	)
	if err != nil {
		panic(err)
	}
	if err := db.Ping(); err != nil {
		panic(err)
	}

	setupMetrics(ctx)
	ctx.Log.Info("omegaUp grader started")

	mux := http.DefaultServeMux
	if ctx.Config.Grader.V1.Enabled {
		registerV1CompatHandlers(mux, db)
		go common.RunServer(&ctx.Config.TLS, mux, ctx.Config.Grader.V1.Port, *insecure)
		mux = http.NewServeMux()
	}

	registerHandlers(mux, db)
	common.RunServer(&ctx.Config.TLS, mux, ctx.Config.Grader.Port, *insecure)
}
Esempio n. 7
0
func init() {
	expvar.Publish("snmp_request_cache", expvar.Func(func() interface{} {
		requests_mutex.Lock()
		size := requests_cache.Size()
		requests_mutex.Unlock()
		return size
	}))

	expvar.Publish("snmp_bytes_cache", expvar.Func(func() interface{} {
		bytes_mutex.Lock()
		size := bytes_cache.Size()
		bytes_mutex.Unlock()
		return size
	}))
}
Esempio n. 8
0
func NewRPCExt(name string, maxIdleConnections int, clientBuilder ClientBuilder) *RPCExt {
	r := &RPCExt{
		name:               name,
		clients:            make([]*rpc.Client, 0, maxIdleConnections),
		maxIdleConnections: maxIdleConnections,
		clientBuilder:      clientBuilder,
		closed:             false,

		statRequests:               metrics.NewCounter(),
		statEstablishedConnections: metrics.NewCounter(),
		statLiveConnections:        metrics.NewCounter(),
	}

	m := expvar.NewMap(name + "-rpc")
	m.Set("requests", r.statRequests)
	m.Set("connections.established", r.statEstablishedConnections)
	m.Set("connections.inuse", r.statLiveConnections)
	m.Set("connections.idle", expvar.Func(func() interface{} {
		r.mu.RLock()
		n := len(r.clients)
		r.mu.RUnlock()
		return n
	}))

	return r
}
Esempio n. 9
0
func init() {
	registry := expvar.Get("registry")
	if registry == nil {
		registry = expvar.NewMap("registry")
	}

	cache := registry.(*expvar.Map).Get("cache")
	if cache == nil {
		cache = &expvar.Map{}
		cache.(*expvar.Map).Init()
		registry.(*expvar.Map).Set("cache", cache)
	}

	storage := cache.(*expvar.Map).Get("storage")
	if storage == nil {
		storage = &expvar.Map{}
		storage.(*expvar.Map).Init()
		cache.(*expvar.Map).Set("storage", storage)
	}

	storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} {
		// no need for synchronous access: the increments are atomic and
		// during reading, we don't care if the data is up to date. The
		// numbers will always *eventually* be reported correctly.
		return blobStatterCacheMetrics
	}))
}
Esempio n. 10
0
func main() {
	// Initialise our configuration from flags

	var nodeName = flag.String("name", REQUIRED, "Node network name and port, e.g. localhost:3000")
	var gobName = flag.String("gob", "", "Alternative gob network name and port for clients, allowing clients to connect over a different physical interface to nodes.")
	var httpName = flag.String("http", "", "Network name and port for the http ExpVar to listen on.")
	var cborName = flag.String("cbor", "", "Network name and port for the CBOR RPC interface to listen on.")
	var nodePath = flag.String("path", REQUIRED, "Node root path for configuration and log files")
	var clusterID = flag.String("id", "", "Cluster ID that this node is part of")

	flag.Parse()

	if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
		flag.PrintDefaults()
		return
	}

	if *nodeName == REQUIRED {
		log.Printf("name missing.\n")
		flag.PrintDefaults()
		return
	}

	if *nodePath == REQUIRED {
		log.Printf("path missing.\n")
		flag.PrintDefaults()
		return
	}

	// Create our server
	serverNode, err := server.NewServerNode(*nodeName, *gobName, *httpName, *cborName, *nodePath, *clusterID)

	if err != nil {
		log.Fatalf("Unable to start server due to errors.\n")
	}

	expvar.Publish("node", expvar.Func(serverNode.ExpVar))

	//dataLog := &memlog.MemoryLog{}

	// Start a listener to handle incoming requests from other peers
	err = serverNode.ListenConnections()
	if err != nil {
		log.Fatalf("Error starting listener: %v\n", err)
	}

	// Setup signal handling to catch interrupts.
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		os.Interrupt,
		os.Kill)
	go func() {
		<-sigc
		serverNode.RequestShutdown("Request to terminate process detected.")
	}()

	serverNode.WaitForShutdown()

}
Esempio n. 11
0
// StartSelfMonitor starts http server on random port and exports expvars.
//
// It tries 1024 ports, starting from startPort and registers some expvars if ok.
func StartSelfMonitor() (string, error) {
	for port := startPort; port < startPort+1024; port++ {
		bind := fmt.Sprintf("localhost:%d", port)
		l, err := net.Listen("tcp", bind)
		if err != nil {
			continue
		}
		l.Close()

		expvar.Publish("Goroutines", expvar.Func(goroutines))
		expvar.Publish("Uptime", expvar.Func(uptime))
		go http.ListenAndServe(bind, nil)
		return bind, nil
	}

	return "", fmt.Errorf("no free ports found")
}
Esempio n. 12
0
func init() {
	expvar.NewString("goVersion").Set(runtime.Version())
	expvar.NewString("iconVersion").Set(besticon.VersionString)

	expvar.NewString("timeLastDeploy").Set(parseUnixTimeStamp(os.Getenv("DEPLOYED_AT")).String())
	expvar.NewString("timeStartup").Set(time.Now().String())
	expvar.Publish("timeCurrent", expvar.Func(func() interface{} { return time.Now() }))
}
Esempio n. 13
0
func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage of inbucket [options] <conf file>:")
		flag.PrintDefaults()
	}

	expvar.Publish("uptime", expvar.Func(uptime))
}
Esempio n. 14
0
func init() {
	raft.SetLogger(capnslog.NewPackageLogger("github.com/coreos/etcd", "raft"))
	expvar.Publish("raft.status", expvar.Func(func() interface{} {
		raftStatusMu.Lock()
		defer raftStatusMu.Unlock()
		return raftStatus()
	}))
}
Esempio n. 15
0
func initExpvars() {
	expvar.Publish("__binary_path", expvar.Func(func() interface{} { return &binaryPath }))
	expvar.Publish("__config_path", expvar.Func(func() interface{} { return &configPath }))
	expvar.Publish("_command_line", expvar.Func(func() interface{} { return &commandLine }))
	expvar.Publish("_version", expvar.Func(func() interface{} { return VersionInfo }))
	expvar.Publish("_hostname", expvar.Func(func() interface{} { return hostname }))

	expvar.Publish("_config", expvar.Func(func() interface{} {
		cf, err := structs.BadooStripSecretFields(config) // makes a copy of `config`
		if err != nil {
			return struct {
				Error string `json:"error"`
			}{err.Error()}
		}
		return cf
	}))

	expvar.Publish("_service-stats", expvar.Func(func() interface{} {
		stats, err := GatherServiceStats()
		if err != nil {
			return struct {
				Error string `json:"error"`
			}{err.Error()}
		}
		return stats
	}))
}
Esempio n. 16
0
func publishExtraVars() {
	// By using sync.Once instead of an init() function, we don't clutter
	// the app's expvar export unnecessarily, or risk colliding with it.
	publishOnce.Do(func() {
		expvar.Publish("Goroutines", expvar.Func(func() interface{} {
			return runtime.NumGoroutine()
		}))
	})
}
Esempio n. 17
0
func Run(address string, sink *lager.ReconfigurableSink) (ifrit.Process, error) {
	expvar.Publish("numCPUS", expvar.Func(func() interface{} {
		return int64(runtime.NumCPU())
	}))

	expvar.Publish("numGoRoutines", expvar.Func(func() interface{} {
		return int64(runtime.NumGoroutine())
	}))

	server := http_server.New(address, handler(sink))
	p := ifrit.Invoke(server)
	select {
	case <-p.Ready():
	case err := <-p.Wait():
		return nil, err
	}
	return p, nil
}
Esempio n. 18
0
func init() {
	expvar.Publish("kaman.metrics", expvar.Func(func() interface{} {
		counters := Snapshot()
		return map[string]interface{}{
			"Counters": counters,
		}
	}))

}
Esempio n. 19
0
func ListenAndServe(addr string) tsdb.Chan {
	ch := make(chan *tsdb.Point, MaxQueue)
	s := listen(addr)
	statQueue.Set("", expvar.Func(func() interface{} {
		return len(ch)
	}))
	go s.loop(ch)
	return ch
}
Esempio n. 20
0
func init() {
	rm := expvar.NewMap("retention")
	rm.Set("SecondsSinceScanCompleted", expvar.Func(secondsSinceRetentionScanCompleted))
	rm.Set("DeletesHist", expRetentionDeletesHist)
	rm.Set("DeletesTotal", expRetentionDeletesTotal)
	rm.Set("Period", expRetentionPeriod)
	rm.Set("RetainedHist", expRetainedHist)
	rm.Set("RetainedCurrent", expRetainedCurrent)
}
Esempio n. 21
0
func startOutputs() error {
	// Configure the specific output.
	// Valid options are: 'udp', 'tcp', 'file', 's3'
	var outputHandler OutputHandler

	parameters := config.OutputParameters

	switch config.OutputType {
	case FileOutputType:
		outputHandler = &FileOutput{}
	case TCPOutputType:
		outputHandler = &NetOutput{}
		parameters = "tcp:" + parameters
	case UDPOutputType:
		outputHandler = &NetOutput{}
		parameters = "udp:" + parameters
	case S3OutputType:
		outputHandler = &S3Output{}
	default:
		return errors.New(fmt.Sprintf("No valid output handler found (%d)", config.OutputType))
	}

	err := outputHandler.Initialize(parameters)
	if err != nil {
		return err
	}

	expvar.Publish("output_status", expvar.Func(func() interface{} {
		ret := make(map[string]interface{})
		ret[outputHandler.Key()] = outputHandler.Statistics()

		switch config.OutputFormat {
		case LEEFOutputFormat:
			ret["format"] = "leef"
		case JSONOutputFormat:
			ret["format"] = "json"
		}

		switch config.OutputType {
		case FileOutputType:
			ret["type"] = "file"
		case UDPOutputType:
			ret["type"] = "net"
		case TCPOutputType:
			ret["type"] = "net"
		case S3OutputType:
			ret["type"] = "s3"
		}

		return ret
	}))

	log.Printf("Initialized output: %s\n", outputHandler.String())
	return outputHandler.Go(results, output_errors)
}
Esempio n. 22
0
func main() {

	Whispers = whispers{metrics: make(map[string]*carbonmem.Whisper)}

	flag.IntVar(&Whispers.windowSize, "w", 600, "window size")
	flag.IntVar(&Whispers.epochSize, "e", 60, "epoch window size")
	flag.IntVar(&Whispers.epoch0, "epoch0", 0, "epoch0")
	flag.IntVar(&Whispers.prefix, "prefix", 0, "prefix nodes to shard on")

	port := flag.Int("p", 8001, "port to listen on (http)")
	gport := flag.Int("gp", 2003, "port to listen on (graphite)")
	verbose := flag.Bool("v", false, "verbose logging")
	logdir := flag.String("logdir", "/var/log/carbonmem/", "logging directory")
	logtostdout := flag.Bool("stdout", false, "log also to stdout")

	flag.Parse()

	rl := rotatelogs.NewRotateLogs(
		*logdir + "/carbonmem.%Y%m%d%H%M.log",
	)

	// Optional fields must be set afterwards
	rl.LinkName = *logdir + "/carbonmem.log"

	if *logtostdout {
		log.SetOutput(io.MultiWriter(os.Stdout, rl))
	} else {
		log.SetOutput(rl)
	}

	expvar.NewString("BuildVersion").Set(BuildVersion)
	log.Println("starting carbonmem", BuildVersion)

	expvar.Publish("Whispers", expvar.Func(func() interface{} {
		m := make(map[string]int)
		Whispers.RLock()
		for k, v := range Whispers.metrics {
			m[k] = v.Len()
		}
		Whispers.RUnlock()
		return m
	}))

	if Whispers.epoch0 == 0 {
		Whispers.epoch0 = int(time.Now().Unix())
	}

	go graphiteServer(*gport)

	http.HandleFunc("/metrics/find/", accessHandler(*verbose, findHandler))
	http.HandleFunc("/render/", accessHandler(*verbose, renderHandler))

	log.Println("http server starting on port", *port)
	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
Esempio n. 23
0
func init() {

	expvar.Publish("now", expvar.Func(func() interface{} {
		return time.Now().Format("\"2006-01-02 15:04:05\"")
	}))

	stats = &Stats{}
	expvar.Publish("stats", stats)

	hits = expvar.NewMap("hits").Init()
}
Esempio n. 24
0
File: album.go Progetto: dpup/dbps
func NewAlbum(folder string, dropbox *dropbox.Dropbox) *Album {
	a := &Album{folder: folder, dropbox: dropbox, cache: rcache.New(folder)}
	a.cache.RegisterFetcher(a.fetchOriginal)
	a.cache.RegisterFetcher(a.fetchThumbnail)

	expvar.Publish(fmt.Sprintf("photos (%s)", folder), expvar.Func(func() interface{} {
		return a.photoMap
	}))

	return a
}
Esempio n. 25
0
// admin-port entry point, once started never shutsdown.
func (p *Projector) mainAdminPort(reqch chan ap.Request) {
	p.admind.Register(reqVbmap)
	p.admind.Register(reqFailoverLog)
	p.admind.Register(reqMutationFeed)
	p.admind.Register(reqRestartVbuckets)
	p.admind.Register(reqShutdownVbuckets)
	p.admind.Register(reqAddBuckets)
	p.admind.Register(reqDelBuckets)
	p.admind.Register(reqAddInstances)
	p.admind.Register(reqDelInstances)
	p.admind.Register(reqRepairEndpoints)
	p.admind.Register(reqShutdownFeed)
	p.admind.Register(reqStats)
	p.admind.RegisterHTTPHandler("/stats", p.handleStats)
	p.admind.RegisterHTTPHandler("/settings", p.handleSettings)

	// debug pprof hanlders.
	blockHandler := pprof.Handler("block")
	grHandler := pprof.Handler("goroutine")
	hpHandler := pprof.Handler("heap")
	tcHandler := pprof.Handler("threadcreate")
	profHandler := pprof.Handler("profile")
	p.admind.RegisterHTTPHandler("/debug/pprof", pprof.Index)
	p.admind.RegisterHTTPHandler("/debug/pprof/block", blockHandler)
	p.admind.RegisterHTTPHandler("/debug/pprof/goroutine", grHandler)
	p.admind.RegisterHTTPHandler("/debug/pprof/heap", hpHandler)
	p.admind.RegisterHTTPHandler("/debug/pprof/threadcreate", tcHandler)
	p.admind.RegisterHTTPHandler("/debug/pprof/profile", profHandler)

	expvar.Publish("projector", expvar.Func(p.doStatistics))

	p.admind.Start()

loop:
	for {
		select {
		case req, ok := <-reqch:
			if ok == false {
				break loop
			}
			// a go-routine is spawned so that requests to
			// different feeds can be simultaneously executed.
			go p.handleRequest(req, angioToken)
			angioToken++
			if angioToken >= 0xFFFE {
				angioToken = 1
			}
		}
	}

	p.admind.Stop()
	logging.Infof("%v ... adminport stopped\n", p.logPrefix)
}
Esempio n. 26
0
// Var returns an expvar variable that shows all the current configuration variables and their
// current value
func (c *Distconf) Var() expvar.Var {
	return expvar.Func(func() interface{} {
		c.varsMutex.Lock()
		defer c.varsMutex.Unlock()

		m := make(map[string]interface{})
		for name, v := range c.registeredVars {
			m[name] = v.GenericGet()
		}
		return m
	})
}
Esempio n. 27
0
func main() {

	proto := flag.String("proto", "udp", "listen protocol")
	port := flag.Int("p", 12233, "listen port")
	debugPort := flag.Int("debugPort", 8080, "debug port")

	flag.Parse()

	go func() {
		err := http.ListenAndServe(":"+strconv.Itoa(*debugPort), nil)
		if err != nil {
			log.Fatal("Error from ListenAndServe:", err)
		}
	}()

	quit := make(chan struct{})
	done := make(chan struct{})

	// start workers
	var workerchs []chan []byte
	for _, dst := range flag.Args() {
		w := make(chan []byte, 100000)
		workerchs = append(workerchs, w)
		go worker(w, dst, quit, done)
	}

	Arena.arena = slab.NewArena(8192, 32*1024*1024, 2, nil)

	expvar.Publish("arenastats", expvar.Func(func() interface{} {
		m := make(map[string]int64)
		Arena.Stats(m)
		return m
	}))

	switch *proto {
	case "udp":
		go udpHandler(*port, workerchs)
	case "tcp":
		go tcpHandler(*port, workerchs)
	default:
		log.Fatal("unknown protocol", *proto)
	}

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt)
	<-sig
	log.Println("Shutting down..")
	close(quit)
	for i := 0; i < len(workerchs); i++ {
		<-done
	}
}
Esempio n. 28
0
func init() {
	startTime := time.Now().UTC()
	expvar.Publish("Uptime", expvar.Func(func() interface{} { return int64(time.Since(startTime)) }))

	expvar.Publish("Goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() }))

	// subscribe to the broker in order to report data about current graph
	c := broker.Get().Subscribe()

	// Instantiate a real, empty graph to ensure the interface type is never nil when it might be called
	var g system.CoreGraph = represent.NewGraph()
	go func() {
		for latest := range c {
			g = latest
		}
	}()
	expvar.Publish("MsgId", expvar.Func(func() interface{} { return g.MsgId() }))

	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()
}
Esempio n. 29
0
func init() {
	rand.Seed(time.Now().UnixNano())

	expvar.Publish(
		"file_descriptor_limit",
		expvar.Func(
			func() interface{} {
				n, _ := runtime.FDLimit()
				return n
			},
		),
	)
}
Esempio n. 30
0
func init() {
	registry := expvar.Get("registry")
	if registry == nil {
		registry = expvar.NewMap("registry")
	}

	pm := registry.(*expvar.Map).Get("proxy")
	if pm == nil {
		pm = &expvar.Map{}
		pm.(*expvar.Map).Init()
		registry.(*expvar.Map).Set("proxy", pm)
	}

	pm.(*expvar.Map).Set("blobs", expvar.Func(func() interface{} {
		return proxyMetrics.blobMetrics
	}))

	pm.(*expvar.Map).Set("manifests", expvar.Func(func() interface{} {
		return proxyMetrics.manifestMetrics
	}))

}