func TestSanitizeURL(t *testing.T) {
for _, input := range []struct {
scheme string
port int
path string
input string
want string
}{
{"", 0, "", "", ""},
{"", 0, "", "foo", "http://foo"},
{"", 80, "", "foo", "http://foo:80"},
{"", 0, "some/path", "foo", "http://foo/some/path"},
{"", 0, "/some/path", "foo", "http://foo/some/path"},
{"https://", 0, "", "foo", "https://foo"},
{"https://", 80, "", "foo", "https://foo:80"},
{"https://", 0, "some/path", "foo", "https://foo/some/path"},
{"https://", 0, "", "http://foo", "http://foo"}, // specified scheme beats default...
{"", 9999, "", "foo:80", "http://foo:80"}, // specified port beats default...
{"", 0, "/bar", "foo/baz", "http://foo/bar"}, // ...but default path beats specified!
} {
if want, have := input.want, sanitize.URL(input.scheme, input.port, input.path)(input.input); want != have {
t.Errorf("sanitize.URL(%q, %d, %q)(%q): want %q, have %q", input.scheme, input.port, input.path, input.input, want, have)
continue
}
}
}
func (c *appClient) controlConnection() (bool, error) {
headers := http.Header{}
c.ProbeConfig.authorizeHeaders(headers)
url := sanitize.URL("ws://", 0, "/api/control/ws")(c.target)
conn, _, err := c.wsDialer.Dial(url, headers)
if err != nil {
return false, err
}
defer func() {
conn.Close()
}()
codec := xfer.NewJSONWebsocketCodec(conn)
server := rpc.NewServer()
if err := server.RegisterName("control", c.control); err != nil {
return false, err
}
// Will return false if we are exiting
if !c.registerConn("control", conn) {
return true, nil
}
defer c.closeConn("control")
server.ServeCodec(codec)
return false, nil
}
// NewHTTPPublisher returns an HTTPPublisher ready for use.
func NewHTTPPublisher(target, token, probeID string) (string, *HTTPPublisher, error) {
targetAPI := sanitize.URL("http://", 0, "/api")(target)
resp, err := fastClient.Get(targetAPI)
if err != nil {
return "", nil, err
}
defer resp.Body.Close()
var apiResponse struct {
ID string `json:"id"`
}
if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {
return "", nil, err
}
return apiResponse.ID, &HTTPPublisher{
url: sanitize.URL("http://", 0, "/api/report")(target),
token: token,
probeID: probeID,
}, nil
}
// NewWeave returns a new Weave tagger based on the Weave router at
// address. The address should be an IP or FQDN, no port.
func NewWeave(hostID, weaveRouterAddress string) *Weave {
w := &Weave{
url: sanitize.URL("http://", 6784, "/report")(weaveRouterAddress),
hostID: hostID,
quit: make(chan struct{}),
ps: map[string]psEntry{},
}
w.done.Add(1)
go w.loop()
return w
}
// NewHTTPPublisher returns an HTTPPublisher ready for use.
func NewHTTPPublisher(hostname, target, token, probeID string, insecure bool) (string, *HTTPPublisher, error) {
pc := ProbeConfig{
Token: token,
ProbeID: probeID,
Insecure: insecure,
}
httpTransport, err := pc.getHTTPTransport(hostname)
if err != nil {
return "", nil, err
}
p := &HTTPPublisher{
ProbeConfig: pc,
url: sanitize.URL("", 0, "/api/report")(target),
client: &http.Client{
Transport: httpTransport,
},
}
client := &http.Client{
Timeout: 5 * time.Second,
Transport: httpTransport,
}
req, err := pc.authorizedRequest("GET", sanitize.URL("", 0, "/api")(target), nil)
if err != nil {
return "", nil, err
}
resp, err := client.Do(req)
if err != nil {
return "", nil, err
}
defer resp.Body.Close()
var apiResponse struct {
ID string `json:"id"`
}
if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {
return "", nil, err
}
return apiResponse.ID, p, nil
}
// PipeClose closes the given pipe id on the app.
func (c *appClient) PipeClose(id string) error {
url := sanitize.URL("", 0, fmt.Sprintf("/api/pipe/%s", id))(c.target)
req, err := c.ProbeConfig.authorizedRequest("DELETE", url, nil)
if err != nil {
return err
}
resp, err := c.client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
// Details fetches the details (version, id) of the app.
func (c *appClient) Details() (xfer.Details, error) {
result := xfer.Details{}
req, err := c.ProbeConfig.authorizedRequest("GET", sanitize.URL("", 0, "/api")(c.target), nil)
if err != nil {
return result, err
}
resp, err := c.client.Do(req)
if err != nil {
return result, err
}
defer resp.Body.Close()
return result, json.NewDecoder(resp.Body).Decode(&result)
}
func (c *appClient) publish(r io.Reader) error {
url := sanitize.URL("", 0, "/api/report")(c.target)
req, err := c.ProbeConfig.authorizedRequest("POST", url, r)
if err != nil {
return err
}
req.Header.Set("Content-Encoding", "gzip")
// req.Header.Set("Content-Type", "application/binary") // TODO: we should use http.DetectContentType(..) on the gob'ed
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf(resp.Status)
}
return nil
}
func (c *appClient) controlConnection(handler ControlHandler) error {
dialer := websocket.Dialer{}
headers := http.Header{}
c.ProbeConfig.authorizeHeaders(headers)
// TODO(twilkie) need to update sanitize to work with wss
url := sanitize.URL("ws://", 0, "/api/control/ws")(c.target)
conn, _, err := dialer.Dial(url, headers)
if err != nil {
return err
}
defer func() {
log.Printf("Closing control connection to %s", c.target)
conn.Close()
}()
codec := NewJSONWebsocketCodec(conn)
server := rpc.NewServer()
if err := server.RegisterName("control", handler); err != nil {
return err
}
c.controlServerCodecMtx.Lock()
c.controlServerCodec = codec
// At this point we may have tried to quit earlier, so check to see if the
// quit channel has been closed, non-blocking.
select {
default:
case <-c.quit:
codec.Close()
return nil
}
c.controlServerCodecMtx.Unlock()
server.ServeCodec(codec)
c.controlServerCodecMtx.Lock()
c.controlServerCodec = nil
c.controlServerCodecMtx.Unlock()
return nil
}
func (c *appClient) pipeConnection(id string, pipe xfer.Pipe) (bool, error) {
headers := http.Header{}
c.ProbeConfig.authorizeHeaders(headers)
url := sanitize.URL("ws://", 0, fmt.Sprintf("/api/pipe/%s/probe", id))(c.target)
conn, resp, err := c.wsDialer.Dial(url, headers)
if resp != nil && resp.StatusCode == http.StatusNotFound {
// Special handling - 404 means the app/user has closed the pipe
pipe.Close()
return true, nil
}
if err != nil {
return false, err
}
// Will return false if we are exiting
if !c.registerConn(id, conn) {
return true, nil
}
defer c.closeConn(id)
_, remote := pipe.Ends()
return false, pipe.CopyToWebsocket(remote, conn)
}
// Main runs the probe
func probeMain() {
var (
targets = []string{fmt.Sprintf("localhost:%d", xfer.AppPort)}
token = flag.String("token", "default-token", "probe token")
httpListen = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
spyInterval = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
spyProcs = flag.Bool("processes", true, "report processes (needs root)")
procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem")
useConntrack = flag.Bool("conntrack", true, "also use conntrack to track connections")
insecure = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
logPrefix = flag.String("log.prefix", "<probe>", "prefix for each log line")
logLevel = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
dockerEnabled = flag.Bool("docker", false, "collect Docker-related attributes for processes")
dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name")
kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
kubernetesAPI = flag.String("kubernetes.api", "", "Address of kubernetes master api")
kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")
weaveRouterAddr = flag.String("weave.router.addr", "127.0.0.1:6784", "IP address & port of the Weave router")
weaveDNSTarget = flag.String("weave.hostname", fmt.Sprintf("scope.weave.local:%d", xfer.AppPort), "Hostname to lookup in weaveDNS")
)
flag.Parse()
setLogLevel(*logLevel)
setLogFormatter(*logPrefix)
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
defer log.Info("probe exiting")
if *spyProcs && os.Getegid() != 0 {
log.Warn("-process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
probeID := strconv.FormatInt(rand.Int63(), 16)
var (
hostName = hostname.Get()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Infof("probe starting, version %s, ID %s", version, probeID)
go check()
if len(flag.Args()) > 0 {
targets = flag.Args()
}
log.Infof("publishing to: %s", strings.Join(targets, ", "))
probeConfig := appclient.ProbeConfig{
Token: *token,
ProbeID: probeID,
Insecure: *insecure,
}
clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
return appclient.NewAppClient(
probeConfig, hostname, endpoint,
xfer.ControlHandlerFunc(controls.HandleControlRequest),
)
})
defer clients.Stop()
resolver := appclient.NewResolver(targets, net.LookupIP, clients.Set)
defer resolver.Stop()
processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
scanner := procspy.NewConnectionScanner(processCache)
endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack, scanner)
defer endpointReporter.Stop()
p := probe.New(*spyInterval, *publishInterval, clients)
p.AddTicker(processCache)
p.AddReporter(
endpointReporter,
host.NewReporter(hostID, hostName),
process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies),
)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID, probeID))
if *dockerEnabled {
if err := report.AddLocalBridge(*dockerBridge); err != nil {
log.Errorf("Docker: problem with bridge %s: %v", *dockerBridge, err)
}
if registry, err := docker.NewRegistry(*dockerInterval, clients); err == nil {
defer registry.Stop()
p.AddTagger(docker.NewTagger(registry, processCache))
p.AddReporter(docker.NewReporter(registry, hostID, p))
} else {
log.Errorf("Docker: failed to start registry: %v", err)
}
}
if *kubernetesEnabled {
if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
defer client.Stop()
p.AddReporter(kubernetes.NewReporter(client))
} else {
log.Errorf("Kubernetes: failed to start client: %v", err)
log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
}
}
if *weaveRouterAddr != "" {
client := weave.NewClient(sanitize.URL("http://", 6784, "")(*weaveRouterAddr))
weave := overlay.NewWeave(hostID, client)
defer weave.Stop()
p.AddTagger(weave)
p.AddReporter(weave)
dockerBridgeIP, err := getFirstAddressOf(*dockerBridge)
if err != nil {
log.Println("Error getting docker bridge ip:", err)
} else {
weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
weaveResolver := appclient.NewResolver([]string{*weaveDNSTarget}, weaveDNSLookup, clients.Set)
defer weaveResolver.Stop()
}
}
if *httpListen != "" {
go func() {
log.Infof("Profiling data being exported to %s", *httpListen)
log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
log.Infof("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
}()
}
p.Start()
defer p.Stop()
common.SignalHandlerLoop()
}
// Main runs the probe
func probeMain(flags probeFlags) {
setLogLevel(flags.logLevel)
setLogFormatter(flags.logPrefix)
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
defer log.Info("probe exiting")
if flags.spyProcs && os.Getegid() != 0 {
log.Warn("--probe.process=true, but that requires root to find everything")
}
rand.Seed(time.Now().UnixNano())
var (
probeID = strconv.FormatInt(rand.Int63(), 16)
hostName = hostname.Get()
hostID = hostName // TODO(pb): we should sanitize the hostname
)
log.Infof("probe starting, version %s, ID %s", version, probeID)
log.Infof("command line: %v", os.Args)
checkpointFlags := map[string]string{}
if flags.kubernetesEnabled {
checkpointFlags["kubernetes_enabled"] = "true"
}
go check(checkpointFlags)
var targets = []string{}
if flags.token != "" {
// service mode
if len(flag.Args()) == 0 {
targets = append(targets, defaultServiceHost)
}
} else if !flags.noApp {
targets = append(targets, fmt.Sprintf("localhost:%d", xfer.AppPort))
}
targets = append(targets, flag.Args()...)
log.Infof("publishing to: %s", strings.Join(targets, ", "))
probeConfig := appclient.ProbeConfig{
Token: flags.token,
ProbeVersion: version,
ProbeID: probeID,
Insecure: flags.insecure,
}
clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
return appclient.NewAppClient(
probeConfig, hostname, endpoint,
xfer.ControlHandlerFunc(controls.HandleControlRequest),
)
})
defer clients.Stop()
dnsLookupFn := net.LookupIP
if flags.resolver != "" {
dnsLookupFn = appclient.LookupUsing(flags.resolver)
}
resolver := appclient.NewResolver(targets, dnsLookupFn, clients.Set)
defer resolver.Stop()
p := probe.New(flags.spyInterval, flags.publishInterval, clients)
hostReporter := host.NewReporter(hostID, hostName, probeID, version, clients)
defer hostReporter.Stop()
p.AddReporter(hostReporter)
p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID))
var processCache *process.CachingWalker
var scanner procspy.ConnectionScanner
if flags.procEnabled {
processCache = process.NewCachingWalker(process.NewWalker(flags.procRoot))
scanner = procspy.NewConnectionScanner(processCache)
p.AddTicker(processCache)
p.AddReporter(process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies))
}
endpointReporter := endpoint.NewReporter(hostID, hostName, flags.spyProcs, flags.useConntrack, flags.procEnabled, scanner)
defer endpointReporter.Stop()
p.AddReporter(endpointReporter)
if flags.dockerEnabled {
// Don't add the bridge in Kubernetes since container IPs are global and
// shouldn't be scoped
if !flags.kubernetesEnabled {
if err := report.AddLocalBridge(flags.dockerBridge); err != nil {
log.Errorf("Docker: problem with bridge %s: %v", flags.dockerBridge, err)
}
}
if registry, err := docker.NewRegistry(flags.dockerInterval, clients, true, hostID); err == nil {
defer registry.Stop()
if flags.procEnabled {
p.AddTagger(docker.NewTagger(registry, processCache))
}
p.AddReporter(docker.NewReporter(registry, hostID, probeID, p))
} else {
log.Errorf("Docker: failed to start registry: %v", err)
}
}
if flags.kubernetesEnabled {
if client, err := kubernetes.NewClient(flags.kubernetesAPI, flags.kubernetesInterval); err == nil {
defer client.Stop()
reporter := kubernetes.NewReporter(client, clients, probeID, hostID, p)
defer reporter.Stop()
p.AddReporter(reporter)
p.AddTagger(reporter)
} else {
log.Errorf("Kubernetes: failed to start client: %v", err)
log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
}
}
if flags.weaveAddr != "" {
client := weave.NewClient(sanitize.URL("http://", 6784, "")(flags.weaveAddr))
weave := overlay.NewWeave(hostID, client)
defer weave.Stop()
p.AddTagger(weave)
p.AddReporter(weave)
dockerBridgeIP, err := network.GetFirstAddressOf(flags.dockerBridge)
if err != nil {
log.Println("Error getting docker bridge ip:", err)
} else {
weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
weaveResolver := appclient.NewResolver([]string{flags.weaveHostname}, weaveDNSLookup, clients.Set)
defer weaveResolver.Stop()
}
}
pluginRegistry, err := plugins.NewRegistry(
flags.pluginsRoot,
pluginAPIVersion,
map[string]string{
"probe_id": probeID,
"api_version": pluginAPIVersion,
},
)
if err != nil {
log.Errorf("plugins: problem loading: %v", err)
} else {
defer pluginRegistry.Close()
p.AddReporter(pluginRegistry)
}
if flags.httpListen != "" {
go func() {
log.Infof("Profiling data being exported to %s", flags.httpListen)
log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", flags.httpListen)
log.Infof("Profiling endpoint %s terminated: %v", flags.httpListen, http.ListenAndServe(flags.httpListen, nil))
}()
}
p.Start()
defer p.Stop()
common.SignalHandlerLoop()
}
// NewWeave returns a new Weave tagger based on the Weave router at
// address. The address should be an IP or FQDN, no port.
func NewWeave(hostID, weaveRouterAddress string) *Weave {
return &Weave{
url: sanitize.URL("http://", 6784, "/report")(weaveRouterAddress),
hostID: hostID,
}
}