Esempio n. 1
0
func TestReporter(t *testing.T) {
	var (
		release   = "release"
		version   = "version"
		network   = "192.168.0.0/16"
		hostID    = "hostid"
		hostname  = "hostname"
		timestamp = time.Now()
		load      = report.Metrics{
			host.Load1:    report.MakeMetric().Add(timestamp, 1.0),
			host.Load5:    report.MakeMetric().Add(timestamp, 5.0),
			host.Load15:   report.MakeMetric().Add(timestamp, 15.0),
			host.CPUUsage: report.MakeMetric().Add(timestamp, 30.0).WithMax(100.0),
			host.MemUsage: report.MakeMetric().Add(timestamp, 60.0).WithMax(100.0),
		}
		uptime      = "278h55m43s"
		kernel      = "release version"
		_, ipnet, _ = net.ParseCIDR(network)
		localNets   = report.Networks([]*net.IPNet{ipnet})
	)

	mtime.NowForce(timestamp)
	defer mtime.NowReset()

	var (
		oldGetKernelVersion    = host.GetKernelVersion
		oldGetLoad             = host.GetLoad
		oldGetUptime           = host.GetUptime
		oldGetCPUUsagePercent  = host.GetCPUUsagePercent
		oldGetMemoryUsageBytes = host.GetMemoryUsageBytes
	)
	defer func() {
		host.GetKernelVersion = oldGetKernelVersion
		host.GetLoad = oldGetLoad
		host.GetUptime = oldGetUptime
		host.GetCPUUsagePercent = oldGetCPUUsagePercent
		host.GetMemoryUsageBytes = oldGetMemoryUsageBytes
	}()
	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
	host.GetLoad = func(time.Time) report.Metrics { return load }
	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
	host.GetCPUUsagePercent = func() (float64, float64) { return 30.0, 100.0 }
	host.GetMemoryUsageBytes = func() (float64, float64) { return 60.0, 100.0 }

	want := report.MakeReport()
	want.Host.AddNode(report.MakeHostNodeID(hostID), report.MakeNodeWith(map[string]string{
		host.Timestamp:     timestamp.UTC().Format(time.RFC3339Nano),
		host.HostName:      hostname,
		host.OS:            runtime.GOOS,
		host.Uptime:        uptime,
		host.KernelVersion: kernel,
	}).WithSets(report.Sets{
		host.LocalNetworks: report.MakeStringSet(network),
	}).WithMetrics(load))
	have, _ := host.NewReporter(hostID, hostname, localNets).Report()
	if !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Esempio n. 2
0
func TestReporter(t *testing.T) {
	var (
		release   = "release"
		version   = "version"
		network   = "192.168.0.0/16"
		hostID    = "hostid"
		now       = "now"
		hostname  = "hostname"
		timestamp = time.Now()
		load      = report.Metrics{
			host.Load1:  report.MakeMetric().Add(timestamp, 1.0),
			host.Load5:  report.MakeMetric().Add(timestamp, 5.0),
			host.Load15: report.MakeMetric().Add(timestamp, 15.0),
		}
		uptime      = "278h55m43s"
		kernel      = "release version"
		_, ipnet, _ = net.ParseCIDR(network)
		localNets   = report.Networks([]*net.IPNet{ipnet})
	)

	var (
		oldGetKernelVersion = host.GetKernelVersion
		oldGetLoad          = host.GetLoad
		oldGetUptime        = host.GetUptime
		oldNow              = host.Now
	)
	defer func() {
		host.GetKernelVersion = oldGetKernelVersion
		host.GetLoad = oldGetLoad
		host.GetUptime = oldGetUptime
		host.Now = oldNow
	}()
	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
	host.GetLoad = func() report.Metrics { return load }
	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
	host.Now = func() string { return now }

	want := report.MakeReport()
	want.Host.AddNode(report.MakeHostNodeID(hostID), report.MakeNodeWith(map[string]string{
		host.Timestamp:     now,
		host.HostName:      hostname,
		host.OS:            runtime.GOOS,
		host.Uptime:        uptime,
		host.KernelVersion: kernel,
	}).WithSets(report.Sets{
		host.LocalNetworks: report.MakeStringSet(network),
	}).WithMetrics(load))
	have, _ := host.NewReporter(hostID, hostname, localNets).Report()
	if !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Esempio n. 3
0
func TestReporter(t *testing.T) {
	oldInterfaceAddrs, oldNow, oldReadFile, oldUname := host.InterfaceAddrs, host.Now, host.ReadFile, host.Uname
	defer func() {
		host.InterfaceAddrs, host.Now, host.ReadFile, host.Uname = oldInterfaceAddrs, oldNow, oldReadFile, oldUname
	}()

	host.InterfaceAddrs = func() ([]net.Addr, error) {
		_, ipnet, _ := net.ParseCIDR(network)
		return []net.Addr{ipnet}, nil
	}

	host.Now = func() string { return now }

	host.ReadFile = func(filename string) ([]byte, error) {
		switch filename {
		case host.ProcUptime:
			return []byte(procUptime), nil
		case host.ProcLoad:
			return []byte(procLoad), nil
		default:
			panic(filename)
		}
	}

	host.Uname = func(uts *syscall.Utsname) error {
		uts.Release = string2c(release)
		uts.Version = string2c(version)
		return nil
	}

	r := host.NewReporter(hostid, hostname)
	have, _ := r.Report()
	want := report.MakeReport()
	want.Host.NodeMetadatas[report.MakeHostNodeID(hostid)] = report.NodeMetadata{
		host.Timestamp:     now,
		host.HostName:      hostname,
		host.LocalNetworks: network,
		host.OS:            runtime.GOOS,
		host.Load:          load,
		host.Uptime:        uptime,
		host.KernelVersion: kernel,
	}

	if !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Esempio n. 4
0
func TestReporter(t *testing.T) {
	var (
		release     = "release"
		version     = "version"
		network     = "192.168.0.0/16"
		hostID      = "hostid"
		now         = "now"
		hostname    = "hostname"
		load        = "0.59 0.36 0.29"
		uptime      = "278h55m43s"
		kernel      = "release version"
		_, ipnet, _ = net.ParseCIDR(network)
		localNets   = report.Networks([]*net.IPNet{ipnet})
	)

	var (
		oldGetKernelVersion = host.GetKernelVersion
		oldGetLoad          = host.GetLoad
		oldGetUptime        = host.GetUptime
		oldNow              = host.Now
	)
	defer func() {
		host.GetKernelVersion = oldGetKernelVersion
		host.GetLoad = oldGetLoad
		host.GetUptime = oldGetUptime
		host.Now = oldNow
	}()
	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
	host.GetLoad = func() string { return load }
	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
	host.Now = func() string { return now }

	want := report.MakeReport()
	want.Host.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.MakeNodeMetadataWith(map[string]string{
		host.Timestamp:     now,
		host.HostName:      hostname,
		host.LocalNetworks: network,
		host.OS:            runtime.GOOS,
		host.Load:          load,
		host.Uptime:        uptime,
		host.KernelVersion: kernel,
	})
	have, _ := host.NewReporter(hostID, hostname, localNets).Report()
	if !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Esempio n. 5
0
func TestReporter(t *testing.T) {
	var (
		oldGetKernelVersion = host.GetKernelVersion
		oldGetLoad          = host.GetLoad
		oldGetUptime        = host.GetUptime
		oldInterfaceAddrs   = host.InterfaceAddrs
		oldNow              = host.Now
	)
	defer func() {
		host.GetKernelVersion = oldGetKernelVersion
		host.GetLoad = oldGetLoad
		host.GetUptime = oldGetUptime
		host.InterfaceAddrs = oldInterfaceAddrs
		host.Now = oldNow
	}()
	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
	host.GetLoad = func() string { return load }
	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
	host.Now = func() string { return now }
	host.InterfaceAddrs = func() ([]net.Addr, error) { _, ipnet, _ := net.ParseCIDR(network); return []net.Addr{ipnet}, nil }

	want := report.MakeReport()
	want.Host.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.NewNodeMetadata(map[string]string{
		host.Timestamp:     now,
		host.HostName:      hostname,
		host.LocalNetworks: network,
		host.OS:            runtime.GOOS,
		host.Load:          load,
		host.Uptime:        uptime,
		host.KernelVersion: kernel,
	})
	r := host.NewReporter(hostID, hostname)
	have, _ := r.Report()
	if !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Esempio n. 6
0
// Main runs the probe
func probeMain() {
	var (
		targets         = []string{fmt.Sprintf("localhost:%d", xfer.AppPort)}
		token           = flag.String("token", "default-token", "probe token")
		httpListen      = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval     = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		spyProcs        = flag.Bool("processes", true, "report processes (needs root)")
		procRoot        = flag.String("proc.root", "/proc", "location of the proc filesystem")
		useConntrack    = flag.Bool("conntrack", true, "also use conntrack to track connections")
		insecure        = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
		logPrefix       = flag.String("log.prefix", "<probe>", "prefix for each log line")
		logLevel        = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")

		dockerEnabled  = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge   = flag.String("docker.bridge", "docker0", "the docker bridge name")

		kubernetesEnabled  = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
		kubernetesAPI      = flag.String("kubernetes.api", "", "Address of kubernetes master api")
		kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")

		weaveRouterAddr = flag.String("weave.router.addr", "127.0.0.1:6784", "IP address & port of the Weave router")
		weaveDNSTarget  = flag.String("weave.hostname", fmt.Sprintf("scope.weave.local:%d", xfer.AppPort), "Hostname to lookup in weaveDNS")
	)
	flag.Parse()

	setLogLevel(*logLevel)
	setLogFormatter(*logPrefix)

	// Setup in memory metrics sink
	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
	sig := metrics.DefaultInmemSignal(inm)
	defer sig.Stop()
	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)

	defer log.Info("probe exiting")

	if *spyProcs && os.Getegid() != 0 {
		log.Warn("-process=true, but that requires root to find everything")
	}

	rand.Seed(time.Now().UnixNano())
	probeID := strconv.FormatInt(rand.Int63(), 16)
	var (
		hostName = hostname.Get()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
	)
	log.Infof("probe starting, version %s, ID %s", version, probeID)
	go check()

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Infof("publishing to: %s", strings.Join(targets, ", "))

	probeConfig := appclient.ProbeConfig{
		Token:    *token,
		ProbeID:  probeID,
		Insecure: *insecure,
	}
	clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
		return appclient.NewAppClient(
			probeConfig, hostname, endpoint,
			xfer.ControlHandlerFunc(controls.HandleControlRequest),
		)
	})
	defer clients.Stop()

	resolver := appclient.NewResolver(targets, net.LookupIP, clients.Set)
	defer resolver.Stop()

	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
	scanner := procspy.NewConnectionScanner(processCache)

	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack, scanner)
	defer endpointReporter.Stop()

	p := probe.New(*spyInterval, *publishInterval, clients)
	p.AddTicker(processCache)
	p.AddReporter(
		endpointReporter,
		host.NewReporter(hostID, hostName),
		process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies),
	)
	p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Errorf("Docker: problem with bridge %s: %v", *dockerBridge, err)
		}
		if registry, err := docker.NewRegistry(*dockerInterval, clients); err == nil {
			defer registry.Stop()
			p.AddTagger(docker.NewTagger(registry, processCache))
			p.AddReporter(docker.NewReporter(registry, hostID, p))
		} else {
			log.Errorf("Docker: failed to start registry: %v", err)
		}
	}

	if *kubernetesEnabled {
		if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
			defer client.Stop()
			p.AddReporter(kubernetes.NewReporter(client))
		} else {
			log.Errorf("Kubernetes: failed to start client: %v", err)
			log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
		}
	}

	if *weaveRouterAddr != "" {
		client := weave.NewClient(sanitize.URL("http://", 6784, "")(*weaveRouterAddr))
		weave := overlay.NewWeave(hostID, client)
		defer weave.Stop()
		p.AddTagger(weave)
		p.AddReporter(weave)

		dockerBridgeIP, err := getFirstAddressOf(*dockerBridge)
		if err != nil {
			log.Println("Error getting docker bridge ip:", err)
		} else {
			weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
			weaveResolver := appclient.NewResolver([]string{*weaveDNSTarget}, weaveDNSLookup, clients.Set)
			defer weaveResolver.Stop()
		}
	}

	if *httpListen != "" {
		go func() {
			log.Infof("Profiling data being exported to %s", *httpListen)
			log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
			log.Infof("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
		}()
	}

	p.Start()
	defer p.Stop()

	common.SignalHandlerLoop()
}
Esempio n. 7
0
func main() {
	var (
		targets            = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
		token              = flag.String("token", "default-token", "probe token")
		httpListen         = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval    = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval        = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		spyProcs           = flag.Bool("processes", true, "report processes (needs root)")
		dockerEnabled      = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval     = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge       = flag.String("docker.bridge", "docker0", "the docker bridge name")
		kubernetesEnabled  = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
		kubernetesAPI      = flag.String("kubernetes.api", "http://localhost:8080", "Address of kubernetes master api")
		kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
		weaveRouterAddr    = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
		printVersion       = flag.Bool("version", false, "print version number and exit")
		useConntrack       = flag.Bool("conntrack", true, "also use conntrack to track connections")
		insecure           = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
		logPrefix          = flag.String("log.prefix", "<probe>", "prefix for each log line")
	)
	flag.Parse()

	if *printVersion {
		fmt.Println(version)
		return
	}

	// Setup in memory metrics sink
	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
	sig := metrics.DefaultInmemSignal(inm)
	defer sig.Stop()
	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)

	if !strings.HasSuffix(*logPrefix, " ") {
		*logPrefix += " "
	}
	log.SetPrefix(*logPrefix)

	defer log.Print("probe exiting")

	if *spyProcs && os.Getegid() != 0 {
		log.Printf("warning: -process=true, but that requires root to find everything")
	}

	rand.Seed(time.Now().UnixNano())
	probeID := strconv.FormatInt(rand.Int63(), 16)
	var (
		hostName = probe.Hostname()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
	)
	log.Printf("probe starting, version %s, ID %s", version, probeID)

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatal(err)
	}
	localNets := report.Networks{}
	for _, addr := range addrs {
		// Not all addrs are IPNets.
		if ipNet, ok := addr.(*net.IPNet); ok {
			localNets = append(localNets, ipNet)
		}
	}

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Printf("publishing to: %s", strings.Join(targets, ", "))

	factory := func(hostname, endpoint string) (string, xfer.Publisher, error) {
		id, publisher, err := xfer.NewHTTPPublisher(hostname, endpoint, *token, probeID, *insecure)
		if err != nil {
			return "", nil, err
		}
		return id, xfer.NewBackgroundPublisher(publisher), nil
	}

	publishers := xfer.NewMultiPublisher(factory)
	defer publishers.Stop()

	clients := xfer.NewMultiAppClient(xfer.ProbeConfig{
		Token:    *token,
		ProbeID:  probeID,
		Insecure: *insecure,
	}, xfer.ControlHandlerFunc(controls.HandleControlRequest), xfer.NewAppClient)
	defer clients.Stop()

	resolver := xfer.NewStaticResolver(targets, publishers.Set, clients.Set)
	defer resolver.Stop()

	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
	defer endpointReporter.Stop()

	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
	p := probe.New(*spyInterval, *publishInterval, publishers)
	p.AddTicker(processCache)
	p.AddReporter(
		endpointReporter,
		host.NewReporter(hostID, hostName, localNets),
		process.NewReporter(processCache, hostID),
	)
	p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
		}
		if registry, err := docker.NewRegistry(*dockerInterval); err == nil {
			defer registry.Stop()
			p.AddTagger(docker.NewTagger(registry, processCache))
			p.AddReporter(docker.NewReporter(registry, hostID, p))
		} else {
			log.Printf("Docker: failed to start registry: %v", err)
		}
	}

	if *kubernetesEnabled {
		if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
			defer client.Stop()
			p.AddReporter(kubernetes.NewReporter(client))
		} else {
			log.Printf("Kubernetes: failed to start client: %v", err)
		}
	}

	if *weaveRouterAddr != "" {
		weave := overlay.NewWeave(hostID, *weaveRouterAddr)
		defer weave.Stop()
		p.AddTicker(weave)
		p.AddTagger(weave)
		p.AddReporter(weave)
	}

	if *httpListen != "" {
		go func() {
			log.Printf("Profiling data being exported to %s", *httpListen)
			log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
			log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
		}()
	}

	p.Start()
	defer p.Stop()

	log.Printf("%s", <-interrupt())
}
Esempio n. 8
0
// Main runs the probe
func probeMain(flags probeFlags) {
	setLogLevel(flags.logLevel)
	setLogFormatter(flags.logPrefix)

	// Setup in memory metrics sink
	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
	sig := metrics.DefaultInmemSignal(inm)
	defer sig.Stop()
	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)

	defer log.Info("probe exiting")

	if flags.spyProcs && os.Getegid() != 0 {
		log.Warn("--probe.process=true, but that requires root to find everything")
	}

	rand.Seed(time.Now().UnixNano())
	var (
		probeID  = strconv.FormatInt(rand.Int63(), 16)
		hostName = hostname.Get()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
	)
	log.Infof("probe starting, version %s, ID %s", version, probeID)
	log.Infof("command line: %v", os.Args)
	checkpointFlags := map[string]string{}
	if flags.kubernetesEnabled {
		checkpointFlags["kubernetes_enabled"] = "true"
	}
	go check(checkpointFlags)

	var targets = []string{}
	if flags.token != "" {
		// service mode
		if len(flag.Args()) == 0 {
			targets = append(targets, defaultServiceHost)
		}
	} else if !flags.noApp {
		targets = append(targets, fmt.Sprintf("localhost:%d", xfer.AppPort))
	}
	targets = append(targets, flag.Args()...)
	log.Infof("publishing to: %s", strings.Join(targets, ", "))

	probeConfig := appclient.ProbeConfig{
		Token:        flags.token,
		ProbeVersion: version,
		ProbeID:      probeID,
		Insecure:     flags.insecure,
	}
	clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
		return appclient.NewAppClient(
			probeConfig, hostname, endpoint,
			xfer.ControlHandlerFunc(controls.HandleControlRequest),
		)
	})
	defer clients.Stop()

	dnsLookupFn := net.LookupIP
	if flags.resolver != "" {
		dnsLookupFn = appclient.LookupUsing(flags.resolver)
	}
	resolver := appclient.NewResolver(targets, dnsLookupFn, clients.Set)
	defer resolver.Stop()

	p := probe.New(flags.spyInterval, flags.publishInterval, clients)

	hostReporter := host.NewReporter(hostID, hostName, probeID, version, clients)
	defer hostReporter.Stop()
	p.AddReporter(hostReporter)
	p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID))

	var processCache *process.CachingWalker
	var scanner procspy.ConnectionScanner
	if flags.procEnabled {
		processCache = process.NewCachingWalker(process.NewWalker(flags.procRoot))
		scanner = procspy.NewConnectionScanner(processCache)
		p.AddTicker(processCache)
		p.AddReporter(process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies))
	}

	endpointReporter := endpoint.NewReporter(hostID, hostName, flags.spyProcs, flags.useConntrack, flags.procEnabled, scanner)
	defer endpointReporter.Stop()
	p.AddReporter(endpointReporter)

	if flags.dockerEnabled {
		// Don't add the bridge in Kubernetes since container IPs are global and
		// shouldn't be scoped
		if !flags.kubernetesEnabled {
			if err := report.AddLocalBridge(flags.dockerBridge); err != nil {
				log.Errorf("Docker: problem with bridge %s: %v", flags.dockerBridge, err)
			}
		}
		if registry, err := docker.NewRegistry(flags.dockerInterval, clients, true, hostID); err == nil {
			defer registry.Stop()
			if flags.procEnabled {
				p.AddTagger(docker.NewTagger(registry, processCache))
			}
			p.AddReporter(docker.NewReporter(registry, hostID, probeID, p))
		} else {
			log.Errorf("Docker: failed to start registry: %v", err)
		}
	}

	if flags.kubernetesEnabled {
		if client, err := kubernetes.NewClient(flags.kubernetesAPI, flags.kubernetesInterval); err == nil {
			defer client.Stop()
			reporter := kubernetes.NewReporter(client, clients, probeID, hostID, p)
			defer reporter.Stop()
			p.AddReporter(reporter)
			p.AddTagger(reporter)
		} else {
			log.Errorf("Kubernetes: failed to start client: %v", err)
			log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
		}
	}

	if flags.weaveAddr != "" {
		client := weave.NewClient(sanitize.URL("http://", 6784, "")(flags.weaveAddr))
		weave := overlay.NewWeave(hostID, client)
		defer weave.Stop()
		p.AddTagger(weave)
		p.AddReporter(weave)

		dockerBridgeIP, err := network.GetFirstAddressOf(flags.dockerBridge)
		if err != nil {
			log.Println("Error getting docker bridge ip:", err)
		} else {
			weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
			weaveResolver := appclient.NewResolver([]string{flags.weaveHostname}, weaveDNSLookup, clients.Set)
			defer weaveResolver.Stop()
		}
	}

	pluginRegistry, err := plugins.NewRegistry(
		flags.pluginsRoot,
		pluginAPIVersion,
		map[string]string{
			"probe_id":    probeID,
			"api_version": pluginAPIVersion,
		},
	)
	if err != nil {
		log.Errorf("plugins: problem loading: %v", err)
	} else {
		defer pluginRegistry.Close()
		p.AddReporter(pluginRegistry)
	}

	if flags.httpListen != "" {
		go func() {
			log.Infof("Profiling data being exported to %s", flags.httpListen)
			log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", flags.httpListen)
			log.Infof("Profiling endpoint %s terminated: %v", flags.httpListen, http.ListenAndServe(flags.httpListen, nil))
		}()
	}

	p.Start()
	defer p.Stop()

	common.SignalHandlerLoop()
}
Esempio n. 9
0
func main() {
	var (
		targets            = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
		token              = flag.String("token", "default-token", "probe token")
		httpListen         = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval    = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval        = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
		spyProcs           = flag.Bool("processes", true, "report processes (needs root)")
		dockerEnabled      = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval     = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge       = flag.String("docker.bridge", "docker0", "the docker bridge name")
		weaveRouterAddr    = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
		printVersion       = flag.Bool("version", false, "print version number and exit")
		useConntrack       = flag.Bool("conntrack", true, "also use conntrack to track connections")
		logPrefix          = flag.String("log.prefix", "<probe>", "prefix for each log line")
	)
	flag.Parse()

	if *printVersion {
		fmt.Println(version)
		return
	}

	if !strings.HasSuffix(*logPrefix, " ") {
		*logPrefix += " "
	}
	log.SetPrefix(*logPrefix)

	defer log.Print("probe exiting")

	if *spyProcs && os.Getegid() != 0 {
		log.Printf("warning: -process=true, but that requires root to find everything")
	}

	var (
		hostName = hostname()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
		probeID  = hostName // TODO(pb): does this need to be a random string instead?
	)
	log.Printf("probe starting, version %s, ID %s", version, probeID)

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatal(err)
	}
	localNets := report.Networks{}
	for _, addr := range addrs {
		// Not all addrs are IPNets.
		if ipNet, ok := addr.(*net.IPNet); ok {
			localNets = append(localNets, ipNet)
		}
	}

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Printf("publishing to: %s", strings.Join(targets, ", "))

	factory := func(endpoint string) (string, xfer.Publisher, error) {
		id, publisher, err := xfer.NewHTTPPublisher(endpoint, *token, probeID)
		if err != nil {
			return "", nil, err
		}
		return id, xfer.NewBackgroundPublisher(publisher), nil
	}

	publishers := xfer.NewMultiPublisher(factory)
	defer publishers.Stop()

	resolver := newStaticResolver(targets, publishers.Set)
	defer resolver.Stop()

	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
	defer endpointReporter.Stop()

	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))

	var (
		tickers   = []Ticker{processCache}
		reporters = []Reporter{endpointReporter, host.NewReporter(hostID, hostName, localNets), process.NewReporter(processCache, hostID)}
		taggers   = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
	)

	dockerTagger, dockerReporter, dockerRegistry := func() (*docker.Tagger, *docker.Reporter, docker.Registry) {
		if !*dockerEnabled {
			return nil, nil, nil
		}
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
			return nil, nil, nil
		}
		registry, err := docker.NewRegistry(*dockerInterval)
		if err != nil {
			log.Printf("Docker: failed to start registry: %v", err)
			return nil, nil, nil
		}
		return docker.NewTagger(registry, processCache), docker.NewReporter(registry, hostID), registry
	}()
	if dockerTagger != nil {
		taggers = append(taggers, dockerTagger)
	}
	if dockerReporter != nil {
		reporters = append(reporters, dockerReporter)
	}
	if dockerRegistry != nil {
		defer dockerRegistry.Stop()
	}

	if *weaveRouterAddr != "" {
		weave := overlay.NewWeave(hostID, *weaveRouterAddr)
		tickers = append(tickers, weave)
		taggers = append(taggers, weave)
		reporters = append(reporters, weave)
	}

	if *httpListen != "" {
		go func() {
			log.Printf("Profiling data being exported to %s", *httpListen)
			log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
			if *prometheusEndpoint != "" {
				log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
				http.Handle(*prometheusEndpoint, makePrometheusHandler())
			}
			log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
		}()
	}

	quit, done := make(chan struct{}), sync.WaitGroup{}
	done.Add(2)
	defer func() { done.Wait() }() // second, wait for the main loops to be killed
	defer close(quit)              // first, kill the main loops

	var rpt syncReport
	rpt.swap(report.MakeReport())

	go func() {
		defer done.Done()
		spyTick := time.Tick(*spyInterval)

		for {
			select {
			case <-spyTick:
				start := time.Now()
				for _, ticker := range tickers {
					if err := ticker.Tick(); err != nil {
						log.Printf("error doing ticker: %v", err)
					}
				}

				localReport := rpt.copy()
				localReport = localReport.Merge(doReport(reporters))
				localReport = Apply(localReport, taggers)
				rpt.swap(localReport)

				if took := time.Since(start); took > *spyInterval {
					log.Printf("report generation took too long (%s)", took)
				}

			case <-quit:
				return
			}
		}
	}()

	go func() {
		defer done.Done()
		var (
			pubTick = time.Tick(*publishInterval)
			p       = xfer.NewReportPublisher(publishers)
		)

		for {
			select {
			case <-pubTick:
				publishTicks.WithLabelValues().Add(1)
				localReport := rpt.swap(report.MakeReport())
				localReport.Window = *publishInterval
				if err := p.Publish(localReport); err != nil {
					log.Printf("publish: %v", err)
				}

			case <-quit:
				return
			}
		}
	}()

	log.Printf("%s", <-interrupt())
}
Esempio n. 10
0
func main() {
	var (
		targets            = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
		token              = flag.String("token", "default-token", "probe token")
		httpListen         = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval    = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval        = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
		spyProcs           = flag.Bool("processes", true, "report processes (needs root)")
		dockerEnabled      = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval     = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge       = flag.String("docker.bridge", "docker0", "the docker bridge name")
		weaveRouterAddr    = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
		captureEnabled     = flag.Bool("capture", false, "perform sampled packet capture")
		captureInterfaces  = flag.String("capture.interfaces", interfaces(), "packet capture on these interfaces")
		captureOn          = flag.Duration("capture.on", 1*time.Second, "packet capture duty cycle 'on'")
		captureOff         = flag.Duration("capture.off", 5*time.Second, "packet capture duty cycle 'off'")
		printVersion       = flag.Bool("version", false, "print version number and exit")
		useConntrack       = flag.Bool("conntrack", true, "also use conntrack to track connections")
	)
	flag.Parse()

	if *printVersion {
		fmt.Println(version)
		return
	}

	var (
		hostName = hostname()
		hostID   = hostName // TODO: we should sanitize the hostname
		probeID  = hostName // TODO: does this need to be a random string instead?
	)
	log.Printf("probe starting, version %s, ID %s", version, probeID)

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Printf("publishing to: %s", strings.Join(targets, ", "))

	procspy.SetProcRoot(*procRoot)

	if *httpListen != "" {
		log.Printf("profiling data being exported to %s", *httpListen)
		log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
		if *prometheusEndpoint != "" {
			log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
			http.Handle(*prometheusEndpoint, makePrometheusHandler())
		}
		go func() {
			err := http.ListenAndServe(*httpListen, nil)
			log.Print(err)
		}()
	}

	if *spyProcs && os.Getegid() != 0 {
		log.Printf("warning: process reporting enabled, but that requires root to find everything")
	}

	publisherFactory := func(target string) (xfer.Publisher, error) { return xfer.NewHTTPPublisher(target, *token, probeID) }
	publishers := xfer.NewMultiPublisher(publisherFactory)
	resolver := newStaticResolver(targets, publishers.Add)
	defer resolver.Stop()

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatal(err)
	}
	localNets := report.Networks{}
	for _, addr := range addrs {
		// Not all addrs are IPNets.
		if ipNet, ok := addr.(*net.IPNet); ok {
			localNets = append(localNets, ipNet)
		}
	}

	var (
		endpointReporter = endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
		processCache     = process.NewCachingWalker(process.NewWalker(*procRoot))
		reporters        = []Reporter{
			endpointReporter,
			host.NewReporter(hostID, hostName, localNets),
			process.NewReporter(processCache, hostID),
		}
		taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
	)
	defer endpointReporter.Stop()

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Fatalf("failed to get docker bridge address: %v", err)
		}

		dockerRegistry, err := docker.NewRegistry(*dockerInterval)
		if err != nil {
			log.Fatalf("failed to start docker registry: %v", err)
		}
		defer dockerRegistry.Stop()

		taggers = append(taggers, docker.NewTagger(dockerRegistry, processCache))
		reporters = append(reporters, docker.NewReporter(dockerRegistry, hostID))
	}

	if *weaveRouterAddr != "" {
		weave, err := overlay.NewWeave(hostID, *weaveRouterAddr)
		if err != nil {
			log.Fatalf("failed to start Weave tagger: %v", err)
		}
		taggers = append(taggers, weave)
		reporters = append(reporters, weave)
	}

	if *captureEnabled {
		var sniffers int
		for _, iface := range strings.Split(*captureInterfaces, ",") {
			source, err := sniff.NewSource(iface)
			if err != nil {
				log.Printf("warning: %v", err)
				continue
			}
			defer source.Close()
			log.Printf("capturing packets on %s", iface)
			reporters = append(reporters, sniff.New(hostID, localNets, source, *captureOn, *captureOff))
			sniffers++
		}
		// Packet capture can block OS threads on Linux, so we need to provide
		// sufficient overhead in GOMAXPROCS.
		if have, want := runtime.GOMAXPROCS(-1), (sniffers + 1); have < want {
			runtime.GOMAXPROCS(want)
		}
	}

	quit := make(chan struct{})
	defer close(quit)
	go func() {
		var (
			pubTick = time.Tick(*publishInterval)
			spyTick = time.Tick(*spyInterval)
			r       = report.MakeReport()
		)

		for {
			select {
			case <-pubTick:
				publishTicks.WithLabelValues().Add(1)
				r.Window = *publishInterval
				if err := publishers.Publish(r); err != nil {
					log.Printf("publish: %v", err)
				}
				r = report.MakeReport()

			case <-spyTick:
				if err := processCache.Update(); err != nil {
					log.Printf("error reading processes: %v", err)
				}
				for _, reporter := range reporters {
					newReport, err := reporter.Report()
					if err != nil {
						log.Printf("error generating report: %v", err)
					}
					r = r.Merge(newReport)
				}
				r = Apply(r, taggers)

			case <-quit:
				return
			}
		}
	}()
	log.Printf("%s", <-interrupt())
}
Esempio n. 11
0
File: main.go Progetto: neviim/scope
func main() {
	var (
		httpListen         = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval    = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval        = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		listen             = flag.String("listen", ":"+strconv.Itoa(xfer.ProbePort), "listen address")
		prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
		spyProcs           = flag.Bool("processes", true, "report processes (needs root)")
		dockerEnabled      = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval     = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge       = flag.String("docker.bridge", "docker0", "the docker bridge name")
		weaveRouterAddr    = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
	)
	flag.Parse()

	if len(flag.Args()) != 0 {
		flag.Usage()
		os.Exit(1)
	}

	log.Printf("probe version %s", version)

	procspy.SetProcRoot(*procRoot)

	if *httpListen != "" {
		log.Printf("profiling data being exported to %s", *httpListen)
		log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
		if *prometheusEndpoint != "" {
			log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
			http.Handle(*prometheusEndpoint, makePrometheusHandler())
		}
		go func() {
			err := http.ListenAndServe(*httpListen, nil)
			log.Print(err)
		}()
	}

	if *spyProcs && os.Getegid() != 0 {
		log.Printf("warning: process reporting enabled, but that requires root to find everything")
	}

	publisher, err := xfer.NewTCPPublisher(*listen)
	if err != nil {
		log.Fatal(err)
	}
	defer publisher.Close()

	var (
		hostName     = hostname()
		hostID       = hostName // TODO: we should sanitize the hostname
		taggers      = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
		reporters    = []Reporter{host.NewReporter(hostID, hostName), endpoint.NewReporter(hostID, hostName, *spyProcs)}
		processCache *process.CachingWalker
	)

	processCache = process.NewCachingWalker(process.NewWalker(*procRoot))
	reporters = append(reporters, process.NewReporter(processCache, hostID))

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Fatalf("failed to get docker bridge address: %v", err)
		}

		dockerRegistry, err := docker.NewRegistry(*dockerInterval)
		if err != nil {
			log.Fatalf("failed to start docker registry: %v", err)
		}
		defer dockerRegistry.Stop()

		taggers = append(taggers, docker.NewTagger(dockerRegistry, processCache))
		reporters = append(reporters, docker.NewReporter(dockerRegistry, hostID))
	}

	if *weaveRouterAddr != "" {
		weave, err := overlay.NewWeave(*weaveRouterAddr)
		if err != nil {
			log.Fatalf("failed to start Weave tagger: %v", err)
		}
		taggers = append(taggers, weave)
		reporters = append(reporters, weave)
	}

	log.Printf("listening on %s", *listen)

	quit := make(chan struct{})
	defer close(quit)
	go func() {
		var (
			pubTick = time.Tick(*publishInterval)
			spyTick = time.Tick(*spyInterval)
			r       = report.MakeReport()
		)

		for {
			select {
			case <-pubTick:
				publishTicks.WithLabelValues().Add(1)
				publisher.Publish(r)
				r = report.MakeReport()

			case <-spyTick:
				if err := processCache.Update(); err != nil {
					log.Printf("error reading processes: %v", err)
				}

				for _, reporter := range reporters {
					newReport, err := reporter.Report()
					if err != nil {
						log.Printf("error generating report: %v", err)
					}
					r.Merge(newReport)
				}

				r = Apply(r, taggers)

			case <-quit:
				return
			}
		}
	}()

	log.Printf("%s", <-interrupt())
}
Esempio n. 12
0
func TestReporter(t *testing.T) {
	var (
		release   = "release"
		version   = "version"
		network   = "192.168.0.0/16"
		hostID    = "hostid"
		hostname  = "hostname"
		timestamp = time.Now()
		metrics   = report.Metrics{
			host.Load1:       report.MakeMetric().Add(timestamp, 1.0),
			host.Load5:       report.MakeMetric().Add(timestamp, 5.0),
			host.Load15:      report.MakeMetric().Add(timestamp, 15.0),
			host.CPUUsage:    report.MakeMetric().Add(timestamp, 30.0).WithMax(100.0),
			host.MemoryUsage: report.MakeMetric().Add(timestamp, 60.0).WithMax(100.0),
		}
		uptime      = "278h55m43s"
		kernel      = "release version"
		_, ipnet, _ = net.ParseCIDR(network)
	)

	mtime.NowForce(timestamp)
	defer mtime.NowReset()

	var (
		oldGetKernelVersion    = host.GetKernelVersion
		oldGetLoad             = host.GetLoad
		oldGetUptime           = host.GetUptime
		oldGetCPUUsagePercent  = host.GetCPUUsagePercent
		oldGetMemoryUsageBytes = host.GetMemoryUsageBytes
		oldGetLocalNetworks    = host.GetLocalNetworks
	)
	defer func() {
		host.GetKernelVersion = oldGetKernelVersion
		host.GetLoad = oldGetLoad
		host.GetUptime = oldGetUptime
		host.GetCPUUsagePercent = oldGetCPUUsagePercent
		host.GetMemoryUsageBytes = oldGetMemoryUsageBytes
		host.GetLocalNetworks = oldGetLocalNetworks
	}()
	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
	host.GetLoad = func(time.Time) report.Metrics { return metrics }
	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
	host.GetCPUUsagePercent = func() (float64, float64) { return 30.0, 100.0 }
	host.GetMemoryUsageBytes = func() (float64, float64) { return 60.0, 100.0 }
	host.GetLocalNetworks = func() ([]*net.IPNet, error) { return []*net.IPNet{ipnet}, nil }

	rpt, err := host.NewReporter(hostID, hostname).Report()
	if err != nil {
		t.Fatal(err)
	}

	nodeID := report.MakeHostNodeID(hostID)
	node, ok := rpt.Host.Nodes[nodeID]
	if !ok {
		t.Errorf("Expected host node %q, but not found", nodeID)
	}

	// Should have a bunch of expected latest keys
	for _, tuple := range []struct {
		key, want string
	}{
		{host.Timestamp, timestamp.UTC().Format(time.RFC3339Nano)},
		{host.HostName, hostname},
		{host.OS, runtime.GOOS},
		{host.Uptime, uptime},
		{host.KernelVersion, kernel},
	} {
		if have, ok := node.Latest.Lookup(tuple.key); !ok || have != tuple.want {
			t.Errorf("Expected %s %q, got %q", tuple.key, tuple.want, have)
		}
	}

	// Should have the local network
	if have, ok := node.Sets.Lookup(host.LocalNetworks); !ok || !have.Contains(network) {
		t.Errorf("Expected host.LocalNetworks to include %q, got %q", network, have)
	}

	// Should have metrics
	for key, want := range metrics {
		wantSample := want.LastSample()
		if metric, ok := node.Metrics[key]; !ok {
			t.Errorf("Expected %s metric, but not found", key)
		} else if sample := metric.LastSample(); sample == nil {
			t.Errorf("Expected %s metric to have a sample, but there were none", key)
		} else if sample.Value != wantSample.Value {
			t.Errorf("Expected %s metric sample %f, got %f", key, wantSample, sample.Value)
		}
	}
}