func TestReporter(t *testing.T) {
var (
release = "release"
version = "version"
network = "192.168.0.0/16"
hostID = "hostid"
hostname = "hostname"
timestamp = time.Now()
load = report.Metrics{
host.Load1: report.MakeMetric().Add(timestamp, 1.0),
host.Load5: report.MakeMetric().Add(timestamp, 5.0),
host.Load15: report.MakeMetric().Add(timestamp, 15.0),
host.CPUUsage: report.MakeMetric().Add(timestamp, 30.0).WithMax(100.0),
host.MemUsage: report.MakeMetric().Add(timestamp, 60.0).WithMax(100.0),
}
uptime = "278h55m43s"
kernel = "release version"
_, ipnet, _ = net.ParseCIDR(network)
localNets = report.Networks([]*net.IPNet{ipnet})
)
mtime.NowForce(timestamp)
defer mtime.NowReset()
var (
oldGetKernelVersion = host.GetKernelVersion
oldGetLoad = host.GetLoad
oldGetUptime = host.GetUptime
oldGetCPUUsagePercent = host.GetCPUUsagePercent
oldGetMemoryUsageBytes = host.GetMemoryUsageBytes
)
defer func() {
host.GetKernelVersion = oldGetKernelVersion
host.GetLoad = oldGetLoad
host.GetUptime = oldGetUptime
host.GetCPUUsagePercent = oldGetCPUUsagePercent
host.GetMemoryUsageBytes = oldGetMemoryUsageBytes
}()
host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
host.GetLoad = func(time.Time) report.Metrics { return load }
host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
host.GetCPUUsagePercent = func() (float64, float64) { return 30.0, 100.0 }
host.GetMemoryUsageBytes = func() (float64, float64) { return 60.0, 100.0 }
want := report.MakeReport()
want.Host.AddNode(report.MakeHostNodeID(hostID), report.MakeNodeWith(map[string]string{
host.Timestamp: timestamp.UTC().Format(time.RFC3339Nano),
host.HostName: hostname,
host.OS: runtime.GOOS,
host.Uptime: uptime,
host.KernelVersion: kernel,
}).WithSets(report.Sets{
host.LocalNetworks: report.MakeStringSet(network),
}).WithMetrics(load))
have, _ := host.NewReporter(hostID, hostname, localNets).Report()
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
func TestReporter(t *testing.T) {
var (
release = "release"
version = "version"
network = "192.168.0.0/16"
hostID = "hostid"
now = "now"
hostname = "hostname"
timestamp = time.Now()
load = report.Metrics{
host.Load1: report.MakeMetric().Add(timestamp, 1.0),
host.Load5: report.MakeMetric().Add(timestamp, 5.0),
host.Load15: report.MakeMetric().Add(timestamp, 15.0),
}
uptime = "278h55m43s"
kernel = "release version"
_, ipnet, _ = net.ParseCIDR(network)
localNets = report.Networks([]*net.IPNet{ipnet})
)
var (
oldGetKernelVersion = host.GetKernelVersion
oldGetLoad = host.GetLoad
oldGetUptime = host.GetUptime
oldNow = host.Now
)
defer func() {
host.GetKernelVersion = oldGetKernelVersion
host.GetLoad = oldGetLoad
host.GetUptime = oldGetUptime
host.Now = oldNow
}()
host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
host.GetLoad = func() report.Metrics { return load }
host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
host.Now = func() string { return now }
want := report.MakeReport()
want.Host.AddNode(report.MakeHostNodeID(hostID), report.MakeNodeWith(map[string]string{
host.Timestamp: now,
host.HostName: hostname,
host.OS: runtime.GOOS,
host.Uptime: uptime,
host.KernelVersion: kernel,
}).WithSets(report.Sets{
host.LocalNetworks: report.MakeStringSet(network),
}).WithMetrics(load))
have, _ := host.NewReporter(hostID, hostname, localNets).Report()
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
func TestReporter(t *testing.T) {
oldInterfaceAddrs, oldNow, oldReadFile, oldUname := host.InterfaceAddrs, host.Now, host.ReadFile, host.Uname
defer func() {
host.InterfaceAddrs, host.Now, host.ReadFile, host.Uname = oldInterfaceAddrs, oldNow, oldReadFile, oldUname
}()
host.InterfaceAddrs = func() ([]net.Addr, error) {
_, ipnet, _ := net.ParseCIDR(network)
return []net.Addr{ipnet}, nil
}
host.Now = func() string { return now }
host.ReadFile = func(filename string) ([]byte, error) {
switch filename {
case host.ProcUptime:
return []byte(procUptime), nil
case host.ProcLoad:
return []byte(procLoad), nil
default:
panic(filename)
}
}
host.Uname = func(uts *syscall.Utsname) error {
uts.Release = string2c(release)
uts.Version = string2c(version)
return nil
}
r := host.NewReporter(hostid, hostname)
have, _ := r.Report()
want := report.MakeReport()
want.Host.NodeMetadatas[report.MakeHostNodeID(hostid)] = report.NodeMetadata{
host.Timestamp: now,
host.HostName: hostname,
host.LocalNetworks: network,
host.OS: runtime.GOOS,
host.Load: load,
host.Uptime: uptime,
host.KernelVersion: kernel,
}
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
func TestReporter(t *testing.T) {
var (
release = "release"
version = "version"
network = "192.168.0.0/16"
hostID = "hostid"
now = "now"
hostname = "hostname"
load = "0.59 0.36 0.29"
uptime = "278h55m43s"
kernel = "release version"
_, ipnet, _ = net.ParseCIDR(network)
localNets = report.Networks([]*net.IPNet{ipnet})
)
var (
oldGetKernelVersion = host.GetKernelVersion
oldGetLoad = host.GetLoad
oldGetUptime = host.GetUptime
oldNow = host.Now
)
defer func() {
host.GetKernelVersion = oldGetKernelVersion
host.GetLoad = oldGetLoad
host.GetUptime = oldGetUptime
host.Now = oldNow
}()
host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
host.GetLoad = func() string { return load }
host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
host.Now = func() string { return now }
want := report.MakeReport()
want.Host.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.MakeNodeMetadataWith(map[string]string{
host.Timestamp: now,
host.HostName: hostname,
host.LocalNetworks: network,
host.OS: runtime.GOOS,
host.Load: load,
host.Uptime: uptime,
host.KernelVersion: kernel,
})
have, _ := host.NewReporter(hostID, hostname, localNets).Report()
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
func TestReporter(t *testing.T) {
var (
oldGetKernelVersion = host.GetKernelVersion
oldGetLoad = host.GetLoad
oldGetUptime = host.GetUptime
oldInterfaceAddrs = host.InterfaceAddrs
oldNow = host.Now
)
defer func() {
host.GetKernelVersion = oldGetKernelVersion
host.GetLoad = oldGetLoad
host.GetUptime = oldGetUptime
host.InterfaceAddrs = oldInterfaceAddrs
host.Now = oldNow
}()
host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
host.GetLoad = func() string { return load }
host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
host.Now = func() string { return now }
host.InterfaceAddrs = func() ([]net.Addr, error) { _, ipnet, _ := net.ParseCIDR(network); return []net.Addr{ipnet}, nil }
want := report.MakeReport()
want.Host.NodeMetadatas[report.MakeHostNodeID(hostID)] = report.NewNodeMetadata(map[string]string{
host.Timestamp: now,
host.HostName: hostname,
host.LocalNetworks: network,
host.OS: runtime.GOOS,
host.Load: load,
host.Uptime: uptime,
host.KernelVersion: kernel,
})
r := host.NewReporter(hostID, hostname)
have, _ := r.Report()
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
// Main runs the probe
func probeMain() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
insecure = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
logLevel = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
kubernetesAPI = flag.String("kubernetes.api", "", "Address of kubernetes master api")
kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
weaveRouterAddr = flag.String("weave.router.addr", "127.0.0.1:6784", "IP address & port of the Weave router")
weaveDNSTarget = flag.String("weave.hostname", fmt.Sprintf("scope.weave.local:%d", xfer.AppPort), "Hostname to lookup in weaveDNS")
)
flag.Parse()
setLogLevel(*logLevel)
setLogFormatter(*logPrefix)
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
defer log.Info("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Warn("-process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
probeID := strconv.FormatInt(rand.Int63(), 16)
var (
hostName = hostname.Get()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Infof("probe starting, version %s, ID %s", version, probeID)
go check()
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Infof("publishing to: %s", strings.Join(targets, ", "))
probeConfig := appclient.ProbeConfig{
Token: *token,
ProbeID: probeID,
Insecure: *insecure,
}
clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
return appclient.NewAppClient(
probeConfig, hostname, endpoint,
xfer.ControlHandlerFunc(controls.HandleControlRequest),
)
})
defer clients.Stop()
resolver := appclient.NewResolver(targets, net.LookupIP, clients.Set)
defer resolver.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
scanner := procspy.NewConnectionScanner(processCache)
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack, scanner)
defer endpointReporter.Stop()
p := probe.New(*spyInterval, *publishInterval, clients)
p.AddTicker(processCache)
p.AddReporter(
endpointReporter,
host.NewReporter(hostID, hostName),
process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies),
)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Errorf("Docker: problem with bridge %s: %v", *dockerBridge, err)
}
if registry, err := docker.NewRegistry(*dockerInterval, clients); err == nil {
defer registry.Stop()
p.AddTagger(docker.NewTagger(registry, processCache))
p.AddReporter(docker.NewReporter(registry, hostID, p))
} else {
log.Errorf("Docker: failed to start registry: %v", err)
}
}
if *kubernetesEnabled {
if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
defer client.Stop()
p.AddReporter(kubernetes.NewReporter(client))
} else {
log.Errorf("Kubernetes: failed to start client: %v", err)
log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
}
}
if *weaveRouterAddr != "" {
client := weave.NewClient(sanitize.URL("http://", 6784, "")(*weaveRouterAddr))
weave := overlay.NewWeave(hostID, client)
defer weave.Stop()
p.AddTagger(weave)
p.AddReporter(weave)
dockerBridgeIP, err := getFirstAddressOf(*dockerBridge)
if err != nil {
log.Println("Error getting docker bridge ip:", err)
} else {
weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
weaveResolver := appclient.NewResolver([]string{*weaveDNSTarget}, weaveDNSLookup, clients.Set)
defer weaveResolver.Stop()
}
}
if *httpListen != "" {
go func() {
log.Infof("Profiling data being exported to %s", *httpListen)
log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
log.Infof("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
p.Start()
defer p.Stop()
common.SignalHandlerLoop()
}
func main() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
kubernetesAPI = flag.String("kubernetes.api", "http://localhost:8080", "Address of kubernetes master api")
kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
printVersion = flag.Bool("version", false, "print version number and exit")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
insecure = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
)
flag.Parse()
if *printVersion {
fmt.Println(version)
return
}
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
if !strings.HasSuffix(*logPrefix, " ") {
*logPrefix += " "
}
log.SetPrefix(*logPrefix)
defer log.Print("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: -process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
probeID := strconv.FormatInt(rand.Int63(), 16)
var (
hostName = probe.Hostname()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Printf("probe starting, version %s, ID %s", version, probeID)
addrs, err := net.InterfaceAddrs()
if err != nil {
log.Fatal(err)
}
localNets := report.Networks{}
for _, addr := range addrs {
// Not all addrs are IPNets.
if ipNet, ok := addr.(*net.IPNet); ok {
localNets = append(localNets, ipNet)
}
}
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Printf("publishing to: %s", strings.Join(targets, ", "))
factory := func(hostname, endpoint string) (string, xfer.Publisher, error) {
id, publisher, err := xfer.NewHTTPPublisher(hostname, endpoint, *token, probeID, *insecure)
if err != nil {
return "", nil, err
}
return id, xfer.NewBackgroundPublisher(publisher), nil
}
publishers := xfer.NewMultiPublisher(factory)
defer publishers.Stop()
clients := xfer.NewMultiAppClient(xfer.ProbeConfig{
Token: *token,
ProbeID: probeID,
Insecure: *insecure,
}, xfer.ControlHandlerFunc(controls.HandleControlRequest), xfer.NewAppClient)
defer clients.Stop()
resolver := xfer.NewStaticResolver(targets, publishers.Set, clients.Set)
defer resolver.Stop()
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
defer endpointReporter.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
p := probe.New(*spyInterval, *publishInterval, publishers)
p.AddTicker(processCache)
p.AddReporter(
endpointReporter,
host.NewReporter(hostID, hostName, localNets),
process.NewReporter(processCache, hostID),
)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
}
if registry, err := docker.NewRegistry(*dockerInterval); err == nil {
defer registry.Stop()
p.AddTagger(docker.NewTagger(registry, processCache))
p.AddReporter(docker.NewReporter(registry, hostID, p))
} else {
log.Printf("Docker: failed to start registry: %v", err)
}
}
if *kubernetesEnabled {
if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
defer client.Stop()
p.AddReporter(kubernetes.NewReporter(client))
} else {
log.Printf("Kubernetes: failed to start client: %v", err)
}
}
if *weaveRouterAddr != "" {
weave := overlay.NewWeave(hostID, *weaveRouterAddr)
defer weave.Stop()
p.AddTicker(weave)
p.AddTagger(weave)
p.AddReporter(weave)
}
if *httpListen != "" {
go func() {
log.Printf("Profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
p.Start()
defer p.Stop()
log.Printf("%s", <-interrupt())
}
// Main runs the probe
func probeMain(flags probeFlags) {
setLogLevel(flags.logLevel)
setLogFormatter(flags.logPrefix)
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
defer log.Info("probe exiting")
if flags.spyProcs && os.Getegid() != 0 {
log.Warn("--probe.process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
var (
probeID = strconv.FormatInt(rand.Int63(), 16)
hostName = hostname.Get()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Infof("probe starting, version %s, ID %s", version, probeID)
log.Infof("command line: %v", os.Args)
checkpointFlags := map[string]string{}
if flags.kubernetesEnabled {
checkpointFlags["kubernetes_enabled"] = "true"
}
go check(checkpointFlags)
var targets = []string{}
if flags.token != "" {
// service mode
if len(flag.Args()) == 0 {
targets = append(targets, defaultServiceHost)
}
} else if !flags.noApp {
targets = append(targets, fmt.Sprintf("localhost:%d", xfer.AppPort))
}
targets = append(targets, flag.Args()...)
log.Infof("publishing to: %s", strings.Join(targets, ", "))
probeConfig := appclient.ProbeConfig{
Token: flags.token,
ProbeVersion: version,
ProbeID: probeID,
Insecure: flags.insecure,
}
clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
return appclient.NewAppClient(
probeConfig, hostname, endpoint,
xfer.ControlHandlerFunc(controls.HandleControlRequest),
)
})
defer clients.Stop()
dnsLookupFn := net.LookupIP
if flags.resolver != "" {
dnsLookupFn = appclient.LookupUsing(flags.resolver)
}
resolver := appclient.NewResolver(targets, dnsLookupFn, clients.Set)
defer resolver.Stop()
p := probe.New(flags.spyInterval, flags.publishInterval, clients)
hostReporter := host.NewReporter(hostID, hostName, probeID, version, clients)
defer hostReporter.Stop()
p.AddReporter(hostReporter)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID))
var processCache *process.CachingWalker
var scanner procspy.ConnectionScanner
if flags.procEnabled {
processCache = process.NewCachingWalker(process.NewWalker(flags.procRoot))
scanner = procspy.NewConnectionScanner(processCache)
p.AddTicker(processCache)
p.AddReporter(process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies))
}
endpointReporter := endpoint.NewReporter(hostID, hostName, flags.spyProcs, flags.useConntrack, flags.procEnabled, scanner)
defer endpointReporter.Stop()
p.AddReporter(endpointReporter)
if flags.dockerEnabled {
// Don't add the bridge in Kubernetes since container IPs are global and
// shouldn't be scoped
if !flags.kubernetesEnabled {
if err := report.AddLocalBridge(flags.dockerBridge); err != nil {
log.Errorf("Docker: problem with bridge %s: %v", flags.dockerBridge, err)
}
}
if registry, err := docker.NewRegistry(flags.dockerInterval, clients, true, hostID); err == nil {
defer registry.Stop()
if flags.procEnabled {
p.AddTagger(docker.NewTagger(registry, processCache))
}
p.AddReporter(docker.NewReporter(registry, hostID, probeID, p))
} else {
log.Errorf("Docker: failed to start registry: %v", err)
}
}
if flags.kubernetesEnabled {
if client, err := kubernetes.NewClient(flags.kubernetesAPI, flags.kubernetesInterval); err == nil {
defer client.Stop()
reporter := kubernetes.NewReporter(client, clients, probeID, hostID, p)
defer reporter.Stop()
p.AddReporter(reporter)
p.AddTagger(reporter)
} else {
log.Errorf("Kubernetes: failed to start client: %v", err)
log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
}
}
if flags.weaveAddr != "" {
client := weave.NewClient(sanitize.URL("http://", 6784, "")(flags.weaveAddr))
weave := overlay.NewWeave(hostID, client)
defer weave.Stop()
p.AddTagger(weave)
p.AddReporter(weave)
dockerBridgeIP, err := network.GetFirstAddressOf(flags.dockerBridge)
if err != nil {
log.Println("Error getting docker bridge ip:", err)
} else {
weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
weaveResolver := appclient.NewResolver([]string{flags.weaveHostname}, weaveDNSLookup, clients.Set)
defer weaveResolver.Stop()
}
}
pluginRegistry, err := plugins.NewRegistry(
flags.pluginsRoot,
pluginAPIVersion,
map[string]string{
"probe_id": probeID,
"api_version": pluginAPIVersion,
},
)
if err != nil {
log.Errorf("plugins: problem loading: %v", err)
} else {
defer pluginRegistry.Close()
p.AddReporter(pluginRegistry)
}
if flags.httpListen != "" {
go func() {
log.Infof("Profiling data being exported to %s", flags.httpListen)
log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", flags.httpListen)
log.Infof("Profiling endpoint %s terminated: %v", flags.httpListen, http.ListenAndServe(flags.httpListen, nil))
}()
}
p.Start()
defer p.Stop()
common.SignalHandlerLoop()
}
func main() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
printVersion = flag.Bool("version", false, "print version number and exit")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
)
flag.Parse()
if *printVersion {
fmt.Println(version)
return
}
if !strings.HasSuffix(*logPrefix, " ") {
*logPrefix += " "
}
log.SetPrefix(*logPrefix)
defer log.Print("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: -process=true, but that requires root to find everything")
}
var (
hostName = hostname()
hostID = hostName // TODO(pb): we should sanitize the hostname
probeID = hostName // TODO(pb): does this need to be a random string instead?
)
log.Printf("probe starting, version %s, ID %s", version, probeID)
addrs, err := net.InterfaceAddrs()
if err != nil {
log.Fatal(err)
}
localNets := report.Networks{}
for _, addr := range addrs {
// Not all addrs are IPNets.
if ipNet, ok := addr.(*net.IPNet); ok {
localNets = append(localNets, ipNet)
}
}
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Printf("publishing to: %s", strings.Join(targets, ", "))
factory := func(endpoint string) (string, xfer.Publisher, error) {
id, publisher, err := xfer.NewHTTPPublisher(endpoint, *token, probeID)
if err != nil {
return "", nil, err
}
return id, xfer.NewBackgroundPublisher(publisher), nil
}
publishers := xfer.NewMultiPublisher(factory)
defer publishers.Stop()
resolver := newStaticResolver(targets, publishers.Set)
defer resolver.Stop()
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
defer endpointReporter.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
var (
tickers = []Ticker{processCache}
reporters = []Reporter{endpointReporter, host.NewReporter(hostID, hostName, localNets), process.NewReporter(processCache, hostID)}
taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
)
dockerTagger, dockerReporter, dockerRegistry := func() (*docker.Tagger, *docker.Reporter, docker.Registry) {
if !*dockerEnabled {
return nil, nil, nil
}
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
return nil, nil, nil
}
registry, err := docker.NewRegistry(*dockerInterval)
if err != nil {
log.Printf("Docker: failed to start registry: %v", err)
return nil, nil, nil
}
return docker.NewTagger(registry, processCache), docker.NewReporter(registry, hostID), registry
}()
if dockerTagger != nil {
taggers = append(taggers, dockerTagger)
}
if dockerReporter != nil {
reporters = append(reporters, dockerReporter)
}
if dockerRegistry != nil {
defer dockerRegistry.Stop()
}
if *weaveRouterAddr != "" {
weave := overlay.NewWeave(hostID, *weaveRouterAddr)
tickers = append(tickers, weave)
taggers = append(taggers, weave)
reporters = append(reporters, weave)
}
if *httpListen != "" {
go func() {
log.Printf("Profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
if *prometheusEndpoint != "" {
log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
http.Handle(*prometheusEndpoint, makePrometheusHandler())
}
log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
quit, done := make(chan struct{}), sync.WaitGroup{}
done.Add(2)
defer func() { done.Wait() }() // second, wait for the main loops to be killed
defer close(quit) // first, kill the main loops
var rpt syncReport
rpt.swap(report.MakeReport())
go func() {
defer done.Done()
spyTick := time.Tick(*spyInterval)
for {
select {
case <-spyTick:
start := time.Now()
for _, ticker := range tickers {
if err := ticker.Tick(); err != nil {
log.Printf("error doing ticker: %v", err)
}
}
localReport := rpt.copy()
localReport = localReport.Merge(doReport(reporters))
localReport = Apply(localReport, taggers)
rpt.swap(localReport)
if took := time.Since(start); took > *spyInterval {
log.Printf("report generation took too long (%s)", took)
}
case <-quit:
return
}
}
}()
go func() {
defer done.Done()
var (
pubTick = time.Tick(*publishInterval)
p = xfer.NewReportPublisher(publishers)
)
for {
select {
case <-pubTick:
publishTicks.WithLabelValues().Add(1)
localReport := rpt.swap(report.MakeReport())
localReport.Window = *publishInterval
if err := p.Publish(localReport); err != nil {
log.Printf("publish: %v", err)
}
case <-quit:
return
}
}
}()
log.Printf("%s", <-interrupt())
}
func main() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
captureEnabled = flag.Bool("capture", false, "perform sampled packet capture")
captureInterfaces = flag.String("capture.interfaces", interfaces(), "packet capture on these interfaces")
captureOn = flag.Duration("capture.on", 1*time.Second, "packet capture duty cycle 'on'")
captureOff = flag.Duration("capture.off", 5*time.Second, "packet capture duty cycle 'off'")
printVersion = flag.Bool("version", false, "print version number and exit")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
)
flag.Parse()
if *printVersion {
fmt.Println(version)
return
}
var (
hostName = hostname()
hostID = hostName // TODO: we should sanitize the hostname
probeID = hostName // TODO: does this need to be a random string instead?
)
log.Printf("probe starting, version %s, ID %s", version, probeID)
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Printf("publishing to: %s", strings.Join(targets, ", "))
procspy.SetProcRoot(*procRoot)
if *httpListen != "" {
log.Printf("profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
if *prometheusEndpoint != "" {
log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
http.Handle(*prometheusEndpoint, makePrometheusHandler())
}
go func() {
err := http.ListenAndServe(*httpListen, nil)
log.Print(err)
}()
}
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: process reporting enabled, but that requires root to find everything")
}
publisherFactory := func(target string) (xfer.Publisher, error) { return xfer.NewHTTPPublisher(target, *token, probeID) }
publishers := xfer.NewMultiPublisher(publisherFactory)
resolver := newStaticResolver(targets, publishers.Add)
defer resolver.Stop()
addrs, err := net.InterfaceAddrs()
if err != nil {
log.Fatal(err)
}
localNets := report.Networks{}
for _, addr := range addrs {
// Not all addrs are IPNets.
if ipNet, ok := addr.(*net.IPNet); ok {
localNets = append(localNets, ipNet)
}
}
var (
endpointReporter = endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
processCache = process.NewCachingWalker(process.NewWalker(*procRoot))
reporters = []Reporter{
endpointReporter,
host.NewReporter(hostID, hostName, localNets),
process.NewReporter(processCache, hostID),
}
taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
)
defer endpointReporter.Stop()
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Fatalf("failed to get docker bridge address: %v", err)
}
dockerRegistry, err := docker.NewRegistry(*dockerInterval)
if err != nil {
log.Fatalf("failed to start docker registry: %v", err)
}
defer dockerRegistry.Stop()
taggers = append(taggers, docker.NewTagger(dockerRegistry, processCache))
reporters = append(reporters, docker.NewReporter(dockerRegistry, hostID))
}
if *weaveRouterAddr != "" {
weave, err := overlay.NewWeave(hostID, *weaveRouterAddr)
if err != nil {
log.Fatalf("failed to start Weave tagger: %v", err)
}
taggers = append(taggers, weave)
reporters = append(reporters, weave)
}
if *captureEnabled {
var sniffers int
for _, iface := range strings.Split(*captureInterfaces, ",") {
source, err := sniff.NewSource(iface)
if err != nil {
log.Printf("warning: %v", err)
continue
}
defer source.Close()
log.Printf("capturing packets on %s", iface)
reporters = append(reporters, sniff.New(hostID, localNets, source, *captureOn, *captureOff))
sniffers++
}
// Packet capture can block OS threads on Linux, so we need to provide
// sufficient overhead in GOMAXPROCS.
if have, want := runtime.GOMAXPROCS(-1), (sniffers + 1); have < want {
runtime.GOMAXPROCS(want)
}
}
quit := make(chan struct{})
defer close(quit)
go func() {
var (
pubTick = time.Tick(*publishInterval)
spyTick = time.Tick(*spyInterval)
r = report.MakeReport()
)
for {
select {
case <-pubTick:
publishTicks.WithLabelValues().Add(1)
r.Window = *publishInterval
if err := publishers.Publish(r); err != nil {
log.Printf("publish: %v", err)
}
r = report.MakeReport()
case <-spyTick:
if err := processCache.Update(); err != nil {
log.Printf("error reading processes: %v", err)
}
for _, reporter := range reporters {
newReport, err := reporter.Report()
if err != nil {
log.Printf("error generating report: %v", err)
}
r = r.Merge(newReport)
}
r = Apply(r, taggers)
case <-quit:
return
}
}
}()
log.Printf("%s", <-interrupt())
}
func main() {
var (
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
listen = flag.String("listen", ":"+strconv.Itoa(xfer.ProbePort), "listen address")
prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
)
flag.Parse()
if len(flag.Args()) != 0 {
flag.Usage()
os.Exit(1)
}
log.Printf("probe version %s", version)
procspy.SetProcRoot(*procRoot)
if *httpListen != "" {
log.Printf("profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
if *prometheusEndpoint != "" {
log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
http.Handle(*prometheusEndpoint, makePrometheusHandler())
}
go func() {
err := http.ListenAndServe(*httpListen, nil)
log.Print(err)
}()
}
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: process reporting enabled, but that requires root to find everything")
}
publisher, err := xfer.NewTCPPublisher(*listen)
if err != nil {
log.Fatal(err)
}
defer publisher.Close()
var (
hostName = hostname()
hostID = hostName // TODO: we should sanitize the hostname
taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
reporters = []Reporter{host.NewReporter(hostID, hostName), endpoint.NewReporter(hostID, hostName, *spyProcs)}
processCache *process.CachingWalker
)
processCache = process.NewCachingWalker(process.NewWalker(*procRoot))
reporters = append(reporters, process.NewReporter(processCache, hostID))
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Fatalf("failed to get docker bridge address: %v", err)
}
dockerRegistry, err := docker.NewRegistry(*dockerInterval)
if err != nil {
log.Fatalf("failed to start docker registry: %v", err)
}
defer dockerRegistry.Stop()
taggers = append(taggers, docker.NewTagger(dockerRegistry, processCache))
reporters = append(reporters, docker.NewReporter(dockerRegistry, hostID))
}
if *weaveRouterAddr != "" {
weave, err := overlay.NewWeave(*weaveRouterAddr)
if err != nil {
log.Fatalf("failed to start Weave tagger: %v", err)
}
taggers = append(taggers, weave)
reporters = append(reporters, weave)
}
log.Printf("listening on %s", *listen)
quit := make(chan struct{})
defer close(quit)
go func() {
var (
pubTick = time.Tick(*publishInterval)
spyTick = time.Tick(*spyInterval)
r = report.MakeReport()
)
for {
select {
case <-pubTick:
publishTicks.WithLabelValues().Add(1)
publisher.Publish(r)
r = report.MakeReport()
case <-spyTick:
if err := processCache.Update(); err != nil {
log.Printf("error reading processes: %v", err)
}
for _, reporter := range reporters {
newReport, err := reporter.Report()
if err != nil {
log.Printf("error generating report: %v", err)
}
r.Merge(newReport)
}
r = Apply(r, taggers)
case <-quit:
return
}
}
}()
log.Printf("%s", <-interrupt())
}
func TestReporter(t *testing.T) {
var (
release = "release"
version = "version"
network = "192.168.0.0/16"
hostID = "hostid"
hostname = "hostname"
timestamp = time.Now()
metrics = report.Metrics{
host.Load1: report.MakeMetric().Add(timestamp, 1.0),
host.Load5: report.MakeMetric().Add(timestamp, 5.0),
host.Load15: report.MakeMetric().Add(timestamp, 15.0),
host.CPUUsage: report.MakeMetric().Add(timestamp, 30.0).WithMax(100.0),
host.MemoryUsage: report.MakeMetric().Add(timestamp, 60.0).WithMax(100.0),
}
uptime = "278h55m43s"
kernel = "release version"
_, ipnet, _ = net.ParseCIDR(network)
)
mtime.NowForce(timestamp)
defer mtime.NowReset()
var (
oldGetKernelVersion = host.GetKernelVersion
oldGetLoad = host.GetLoad
oldGetUptime = host.GetUptime
oldGetCPUUsagePercent = host.GetCPUUsagePercent
oldGetMemoryUsageBytes = host.GetMemoryUsageBytes
oldGetLocalNetworks = host.GetLocalNetworks
)
defer func() {
host.GetKernelVersion = oldGetKernelVersion
host.GetLoad = oldGetLoad
host.GetUptime = oldGetUptime
host.GetCPUUsagePercent = oldGetCPUUsagePercent
host.GetMemoryUsageBytes = oldGetMemoryUsageBytes
host.GetLocalNetworks = oldGetLocalNetworks
}()
host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
host.GetLoad = func(time.Time) report.Metrics { return metrics }
host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
host.GetCPUUsagePercent = func() (float64, float64) { return 30.0, 100.0 }
host.GetMemoryUsageBytes = func() (float64, float64) { return 60.0, 100.0 }
host.GetLocalNetworks = func() ([]*net.IPNet, error) { return []*net.IPNet{ipnet}, nil }
rpt, err := host.NewReporter(hostID, hostname).Report()
if err != nil {
t.Fatal(err)
}
nodeID := report.MakeHostNodeID(hostID)
node, ok := rpt.Host.Nodes[nodeID]
if !ok {
t.Errorf("Expected host node %q, but not found", nodeID)
}
// Should have a bunch of expected latest keys
for _, tuple := range []struct {
key, want string
}{
{host.Timestamp, timestamp.UTC().Format(time.RFC3339Nano)},
{host.HostName, hostname},
{host.OS, runtime.GOOS},
{host.Uptime, uptime},
{host.KernelVersion, kernel},
} {
if have, ok := node.Latest.Lookup(tuple.key); !ok || have != tuple.want {
t.Errorf("Expected %s %q, got %q", tuple.key, tuple.want, have)
}
}
// Should have the local network
if have, ok := node.Sets.Lookup(host.LocalNetworks); !ok || !have.Contains(network) {
t.Errorf("Expected host.LocalNetworks to include %q, got %q", network, have)
}
// Should have metrics
for key, want := range metrics {
wantSample := want.LastSample()
if metric, ok := node.Metrics[key]; !ok {
t.Errorf("Expected %s metric, but not found", key)
} else if sample := metric.LastSample(); sample == nil {
t.Errorf("Expected %s metric to have a sample, but there were none", key)
} else if sample.Value != wantSample.Value {
t.Errorf("Expected %s metric sample %f, got %f", key, wantSample, sample.Value)
}
}
}