func TestControl(t *testing.T) {
router := mux.NewRouter()
registerControlRoutes(router)
server := httptest.NewServer(router)
defer server.Close()
ip, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://"))
if err != nil {
t.Fatal(err)
}
probeConfig := xfer.ProbeConfig{
ProbeID: "foo",
}
client, err := xfer.NewAppClient(probeConfig, ip+":"+port, ip+":"+port)
if err != nil {
t.Fatal(err)
}
defer client.Stop()
client.ControlConnection(xfer.ControlHandlerFunc(func(req xfer.Request) xfer.Response {
if req.NodeID != "nodeid" {
t.Fatalf("'%s' != 'nodeid'", req.NodeID)
}
if req.Control != "control" {
t.Fatalf("'%s' != 'control'", req.Control)
}
return xfer.Response{
Value: "foo",
}
}))
time.Sleep(100 * time.Millisecond)
httpClient := http.Client{
Timeout: 1 * time.Second,
}
resp, err := httpClient.Post(server.URL+"/api/control/foo/nodeid/control", "", nil)
if err != nil {
t.Fatal(err)
}
var response string
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
t.Fatal(err)
}
if response != "foo" {
t.Fatalf("'%s' != 'foo'", response)
}
}
func TestMultiClient(t *testing.T) {
var (
a1 = &mockClient{id: "1"} // hostname a, app id 1
a2 = &mockClient{id: "2"} // hostname a, app id 2
b2 = &mockClient{id: "2"} // hostname b, app id 2 (duplicate)
b3 = &mockClient{id: "3"} // hostname b, app id 3
factory = func(_ xfer.ProbeConfig, hostname, target string) (xfer.AppClient, error) {
switch target {
case "a1":
return a1, nil
case "a2":
return a2, nil
case "b2":
return b2, nil
case "b3":
return b3, nil
}
t.Fatal(target)
return a1, nil
}
controlHandler = xfer.ControlHandlerFunc(func(_ xfer.Request) xfer.Response {
return xfer.Response{}
})
expect = func(i, j int) {
if i != j {
_, file, line, _ := runtime.Caller(1)
t.Fatalf("%s:%d: %d != %d", file, line, i, j)
}
}
)
mp := xfer.NewMultiAppClient(xfer.ProbeConfig{}, controlHandler, factory)
defer mp.Stop()
// Add two hostnames with overlapping apps, check we don't add the same app twice
mp.Set("a", []string{"a1", "a2"})
mp.Set("b", []string{"b2", "b3"})
expect(a1.count, 1)
expect(a2.count+b2.count, 1)
expect(b3.count, 1)
// Now drop the overlap, check we don't remove the app
mp.Set("b", []string{"b3"})
expect(a1.count, 1)
expect(a2.count+b2.count, 1)
expect(b3.count, 1)
// Now check we remove apps
mp.Set("b", []string{})
expect(b3.stopped, 1)
}
func main() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
kubernetesAPI = flag.String("kubernetes.api", "http://localhost:8080", "Address of kubernetes master api")
kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
printVersion = flag.Bool("version", false, "print version number and exit")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
insecure = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
)
flag.Parse()
if *printVersion {
fmt.Println(version)
return
}
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
if !strings.HasSuffix(*logPrefix, " ") {
*logPrefix += " "
}
log.SetPrefix(*logPrefix)
defer log.Print("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: -process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
probeID := strconv.FormatInt(rand.Int63(), 16)
var (
hostName = probe.Hostname()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Printf("probe starting, version %s, ID %s", version, probeID)
addrs, err := net.InterfaceAddrs()
if err != nil {
log.Fatal(err)
}
localNets := report.Networks{}
for _, addr := range addrs {
// Not all addrs are IPNets.
if ipNet, ok := addr.(*net.IPNet); ok {
localNets = append(localNets, ipNet)
}
}
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Printf("publishing to: %s", strings.Join(targets, ", "))
factory := func(hostname, endpoint string) (string, xfer.Publisher, error) {
id, publisher, err := xfer.NewHTTPPublisher(hostname, endpoint, *token, probeID, *insecure)
if err != nil {
return "", nil, err
}
return id, xfer.NewBackgroundPublisher(publisher), nil
}
publishers := xfer.NewMultiPublisher(factory)
defer publishers.Stop()
clients := xfer.NewMultiAppClient(xfer.ProbeConfig{
Token: *token,
ProbeID: probeID,
Insecure: *insecure,
}, xfer.ControlHandlerFunc(controls.HandleControlRequest), xfer.NewAppClient)
defer clients.Stop()
resolver := xfer.NewStaticResolver(targets, publishers.Set, clients.Set)
defer resolver.Stop()
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
defer endpointReporter.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
p := probe.New(*spyInterval, *publishInterval, publishers)
p.AddTicker(processCache)
p.AddReporter(
endpointReporter,
host.NewReporter(hostID, hostName, localNets),
process.NewReporter(processCache, hostID),
)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
}
if registry, err := docker.NewRegistry(*dockerInterval); err == nil {
defer registry.Stop()
p.AddTagger(docker.NewTagger(registry, processCache))
p.AddReporter(docker.NewReporter(registry, hostID, p))
} else {
log.Printf("Docker: failed to start registry: %v", err)
}
}
if *kubernetesEnabled {
if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
defer client.Stop()
p.AddReporter(kubernetes.NewReporter(client))
} else {
log.Printf("Kubernetes: failed to start client: %v", err)
}
}
if *weaveRouterAddr != "" {
weave := overlay.NewWeave(hostID, *weaveRouterAddr)
defer weave.Stop()
p.AddTicker(weave)
p.AddTagger(weave)
p.AddReporter(weave)
}
if *httpListen != "" {
go func() {
log.Printf("Profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
p.Start()
defer p.Stop()
log.Printf("%s", <-interrupt())
}
func main() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort), fmt.Sprintf("scope.weave.local:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
kubernetesAPI = flag.String("kubernetes.api", "http://localhost:8080", "Address of kubernetes master api")
kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
printVersion = flag.Bool("version", false, "print version number and exit")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
insecure = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
)
flag.Parse()
if *printVersion {
fmt.Println(version)
return
}
if !strings.HasSuffix(*logPrefix, " ") {
*logPrefix += " "
}
log.SetPrefix(*logPrefix)
defer log.Print("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Printf("warning: -process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
probeID := strconv.FormatInt(rand.Int63(), 16)
var (
hostName = hostname()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Printf("probe starting, version %s, ID %s", version, probeID)
addrs, err := net.InterfaceAddrs()
if err != nil {
log.Fatal(err)
}
localNets := report.Networks{}
for _, addr := range addrs {
// Not all addrs are IPNets.
if ipNet, ok := addr.(*net.IPNet); ok {
localNets = append(localNets, ipNet)
}
}
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Printf("publishing to: %s", strings.Join(targets, ", "))
factory := func(hostname, endpoint string) (string, xfer.Publisher, error) {
id, publisher, err := xfer.NewHTTPPublisher(hostname, endpoint, *token, probeID, *insecure)
if err != nil {
return "", nil, err
}
return id, xfer.NewBackgroundPublisher(publisher), nil
}
publishers := xfer.NewMultiPublisher(factory)
defer publishers.Stop()
clients := xfer.NewMultiAppClient(xfer.ProbeConfig{
Token: *token,
ProbeID: probeID,
Insecure: *insecure,
}, xfer.ControlHandlerFunc(controls.HandleControlRequest), xfer.NewAppClient)
defer clients.Stop()
resolver := newStaticResolver(targets, publishers.Set, clients.Set)
defer resolver.Stop()
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
defer endpointReporter.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
var (
tickers = []Ticker{processCache}
reporters = []Reporter{endpointReporter, host.NewReporter(hostID, hostName, localNets), process.NewReporter(processCache, hostID)}
taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID, probeID)}
)
dockerTagger, dockerReporter, dockerRegistry := func() (*docker.Tagger, *docker.Reporter, docker.Registry) {
if !*dockerEnabled {
return nil, nil, nil
}
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
return nil, nil, nil
}
registry, err := docker.NewRegistry(*dockerInterval)
if err != nil {
log.Printf("Docker: failed to start registry: %v", err)
return nil, nil, nil
}
return docker.NewTagger(registry, processCache), docker.NewReporter(registry, hostID), registry
}()
if dockerTagger != nil {
taggers = append(taggers, dockerTagger)
}
if dockerReporter != nil {
reporters = append(reporters, dockerReporter)
}
if dockerRegistry != nil {
defer dockerRegistry.Stop()
}
if *kubernetesEnabled {
if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
defer client.Stop()
reporters = append(reporters, kubernetes.NewReporter(client))
} else {
log.Printf("Kubernetes: failed to start client: %v", err)
}
}
if *weaveRouterAddr != "" {
weave := overlay.NewWeave(hostID, *weaveRouterAddr)
tickers = append(tickers, weave)
taggers = append(taggers, weave)
reporters = append(reporters, weave)
}
if *httpListen != "" {
go func() {
log.Printf("Profiling data being exported to %s", *httpListen)
log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
if *prometheusEndpoint != "" {
log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
http.Handle(*prometheusEndpoint, makePrometheusHandler())
}
log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
quit, done := make(chan struct{}), sync.WaitGroup{}
done.Add(2)
defer func() { done.Wait() }() // second, wait for the main loops to be killed
defer close(quit) // first, kill the main loops
var rpt syncReport
rpt.swap(report.MakeReport())
go func() {
defer done.Done()
spyTick := time.Tick(*spyInterval)
for {
select {
case <-spyTick:
start := time.Now()
for _, ticker := range tickers {
if err := ticker.Tick(); err != nil {
log.Printf("error doing ticker: %v", err)
}
}
localReport := rpt.copy()
localReport = localReport.Merge(doReport(reporters))
localReport = Apply(localReport, taggers)
rpt.swap(localReport)
if took := time.Since(start); took > *spyInterval {
log.Printf("report generation took too long (%s)", took)
}
case <-quit:
return
}
}
}()
go func() {
defer done.Done()
var (
pubTick = time.Tick(*publishInterval)
p = xfer.NewReportPublisher(publishers)
)
for {
select {
case <-pubTick:
publishTicks.WithLabelValues().Add(1)
localReport := rpt.swap(report.MakeReport())
localReport.Window = *publishInterval
if err := p.Publish(localReport); err != nil {
log.Printf("publish: %v", err)
}
case <-quit:
return
}
}
}()
log.Printf("%s", <-interrupt())
}