Пример #1
0
func TestSpyWithProcesses(t *testing.T) {
	const (
		nodeID   = "nikon"             // TODO rename to hostID
		nodeName = "fishermans-friend" // TODO rename to hostName
	)

	scanner := procspy.FixedScanner(fixConnectionsWithProcesses)
	reporter := endpoint.NewReporter(nodeID, nodeName, true, false, scanner)
	r, _ := reporter.Report()
	// buf, _ := json.MarshalIndent(r, "", "    ") ; t.Logf("\n%s\n", buf)

	var (
		scopedLocal  = report.MakeEndpointNodeID(nodeID, fixLocalAddress.String(), strconv.Itoa(int(fixLocalPort)))
		scopedRemote = report.MakeEndpointNodeID(nodeID, fixRemoteAddress.String(), strconv.Itoa(int(fixRemotePort)))
	)

	if want, have := 1, len(r.Endpoint.Nodes[scopedRemote].Adjacency); want != have {
		t.Fatalf("want %d, have %d", want, have)
	}

	if want, have := scopedLocal, r.Endpoint.Nodes[scopedRemote].Adjacency[0]; want != have {
		t.Fatalf("want %q, have %q", want, have)
	}

	for key, want := range map[string]string{
		"pid": strconv.FormatUint(uint64(fixProcessPID), 10),
	} {
		have, _ := r.Endpoint.Nodes[scopedLocal].Latest.Lookup(key)
		if want != have {
			t.Errorf("Process.Nodes[%q][%q]: want %q, have %q", scopedLocal, key, want, have)
		}
	}
}
Пример #2
0
func (r *Reporter) addConnection(rpt *report.Report, t fourTuple, extraFromNode, extraToNode map[string]string) {
	// Update endpoint topology
	if !r.includeProcesses {
		return
	}
	var (
		fromEndpointNodeID = report.MakeEndpointNodeID(r.hostID, t.fromAddr, strconv.Itoa(int(t.fromPort)))
		toEndpointNodeID   = report.MakeEndpointNodeID(r.hostID, t.toAddr, strconv.Itoa(int(t.toPort)))

		fromNode = report.MakeNodeWith(fromEndpointNodeID, map[string]string{
			Addr: t.fromAddr,
			Port: strconv.Itoa(int(t.fromPort)),
		}).WithEdge(toEndpointNodeID, report.EdgeMetadata{})
		toNode = report.MakeNodeWith(toEndpointNodeID, map[string]string{
			Addr: t.toAddr,
			Port: strconv.Itoa(int(t.toPort)),
		})
	)

	// In case we have a reverse resolution for the IP, we can use it for
	// the name...
	if toNames, err := r.reverseResolver.get(t.toAddr); err == nil {
		toNode = toNode.WithSet(ReverseDNSNames, report.MakeStringSet(toNames...))
	}

	if extraFromNode != nil {
		fromNode = fromNode.WithLatests(extraFromNode)
	}
	if extraToNode != nil {
		toNode = toNode.WithLatests(extraToNode)
	}
	rpt.Endpoint = rpt.Endpoint.AddNode(fromNode)
	rpt.Endpoint = rpt.Endpoint.AddNode(toNode)
}
Пример #3
0
func TestMerge(t *testing.T) {
	var (
		hostID = "xyz"
		src    = newMockSource([]byte{}, nil)
		on     = time.Millisecond
		off    = time.Millisecond
		rpt    = report.MakeReport()
		p      = sniff.Packet{
			SrcIP:     "1.0.0.0",
			SrcPort:   "1000",
			DstIP:     "2.0.0.0",
			DstPort:   "2000",
			Network:   512,
			Transport: 256,
		}

		_, ipnet, _ = net.ParseCIDR(p.SrcIP + "/24") // ;)
		localNets   = report.Networks([]*net.IPNet{ipnet})
	)
	sniff.New(hostID, localNets, src, on, off).Merge(p, &rpt)

	var (
		srcEndpointNodeID = report.MakeEndpointNodeID(hostID, p.SrcIP, p.SrcPort)
		dstEndpointNodeID = report.MakeEndpointNodeID(hostID, p.DstIP, p.DstPort)
	)
	if want, have := (report.Topology{
		Nodes: report.Nodes{
			srcEndpointNodeID: report.MakeNode(srcEndpointNodeID).WithEdge(dstEndpointNodeID, report.EdgeMetadata{
				EgressPacketCount: newu64(1),
				EgressByteCount:   newu64(256),
			}),
			dstEndpointNodeID: report.MakeNode(dstEndpointNodeID),
		},
	}), rpt.Endpoint; !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}

	var (
		srcAddressNodeID = report.MakeAddressNodeID(hostID, p.SrcIP)
		dstAddressNodeID = report.MakeAddressNodeID(hostID, p.DstIP)
	)
	if want, have := (report.Topology{
		Nodes: report.Nodes{
			srcAddressNodeID: report.MakeNode(srcAddressNodeID).WithEdge(dstAddressNodeID, report.EdgeMetadata{
				EgressPacketCount: newu64(1),
				EgressByteCount:   newu64(512),
			}),
			dstAddressNodeID: report.MakeNode(dstAddressNodeID),
		},
	}), rpt.Address; !reflect.DeepEqual(want, have) {
		t.Errorf("%s", test.Diff(want, have))
	}
}
Пример #4
0
func TestEndpointNodeID(t *testing.T) {
	for _, bad := range []string{
		clientAddressNodeID,
		serverAddressNodeID,
		unknownAddressNodeID,
		clientHostNodeID,
		serverHostNodeID,
		"host.com;1.2.3.4",
		"a;b",
		"a;",
		";b",
		";",
		"",
	} {
		if haveName, haveAddress, havePort, ok := report.ParseEndpointNodeID(bad); ok {
			t.Errorf("%q: expected failure, but got {%q, %q, %q}", bad, haveName, haveAddress, havePort)
		}
	}

	for input, want := range map[string]struct{ name, address, port string }{
		report.MakeEndpointNodeID("host.com", "1.2.3.4", "c"): {"", "1.2.3.4", "c"},
		"a;b;c": {"a", "b", "c"},
	} {
		haveName, haveAddress, havePort, ok := report.ParseEndpointNodeID(input)
		if !ok {
			t.Errorf("%q: not OK", input)
			continue
		}
		if want.name != haveName ||
			want.address != haveAddress ||
			want.port != havePort {
			t.Errorf("%q: want %q, have {%q, %q, %q}", input, want, haveName, haveAddress, havePort)
		}
	}
}
Пример #5
0
func TestTagger(t *testing.T) {
	var (
		hostID         = "foo"
		endpointNodeID = report.MakeEndpointNodeID(hostID, "1.2.3.4", "56789") // hostID ignored
		node           = report.MakeNodeWith(endpointNodeID, map[string]string{"foo": "bar"})
	)

	r := report.MakeReport()
	r.Process.AddNode(node)
	rpt, _ := host.NewTagger(hostID).Tag(r)
	have := rpt.Process.Nodes[endpointNodeID].Copy()

	// It should now have the host ID
	wantHostID := report.MakeHostNodeID(hostID)
	if hostID, ok := have.Latest.Lookup(report.HostNodeID); !ok || hostID != wantHostID {
		t.Errorf("Expected %q got %q", wantHostID, report.MakeHostNodeID(hostID))
	}

	// It should still have the other keys
	want := "bar"
	if have, ok := have.Latest.Lookup("foo"); !ok || have != want {
		t.Errorf("Expected %q got %q", want, have)
	}

	// It should have the host as a parent
	wantParent := report.MakeHostNodeID(hostID)
	if have, ok := have.Parents.Lookup(report.Host); !ok || len(have) != 1 || have[0] != wantParent {
		t.Errorf("Expected %q got %q", report.MakeStringSet(wantParent), have)
	}
}
Пример #6
0
func TestInterpolateCounts(t *testing.T) {
	var (
		hostID        = "macbook-air"
		srcNodeID     = report.MakeEndpointNodeID(hostID, "1.2.3.4", "5678")
		dstNodeID     = report.MakeEndpointNodeID(hostID, "5.6.7.8", "9012")
		samplingCount = uint64(200)
		samplingTotal = uint64(2345)
		packetCount   = uint64(123)
		byteCount     = uint64(4096)
	)

	r := report.MakeReport()
	r.Sampling.Count = samplingCount
	r.Sampling.Total = samplingTotal
	r.Endpoint.AddNode(report.MakeNode(srcNodeID).WithEdge(dstNodeID, report.EdgeMetadata{
		EgressPacketCount:  newu64(packetCount),
		IngressPacketCount: newu64(packetCount),
		EgressByteCount:    newu64(byteCount),
		IngressByteCount:   newu64(byteCount),
	}))

	interpolateCounts(r)

	var (
		rate   = float64(samplingCount) / float64(samplingTotal)
		factor = 1.0 / rate
		apply  = func(v uint64) uint64 { return uint64(factor * float64(v)) }
		emd    = r.Endpoint.Nodes[srcNodeID].Edges[dstNodeID]
	)
	if want, have := apply(packetCount), (*emd.EgressPacketCount); want != have {
		t.Errorf("want %d packets, have %d", want, have)
	}
	if want, have := apply(packetCount), (*emd.IngressPacketCount); want != have {
		t.Errorf("want %d packets, have %d", want, have)
	}
	if want, have := apply(byteCount), (*emd.EgressByteCount); want != have {
		t.Errorf("want %d bytes, have %d", want, have)
	}
	if want, have := apply(byteCount), (*emd.IngressByteCount); want != have {
		t.Errorf("want %d bytes, have %d", want, have)
	}
}
Пример #7
0
// applyNAT duplicates Nodes in the endpoint topology of a report, based on
// the NAT table.
func (n natMapper) applyNAT(rpt report.Report, scope string) {
	n.flowWalker.walkFlows(func(f flow) {
		var (
			mapping          = toMapping(f)
			realEndpointID   = report.MakeEndpointNodeID(scope, mapping.originalIP, strconv.Itoa(mapping.originalPort))
			copyEndpointPort = strconv.Itoa(mapping.rewrittenPort)
			copyEndpointID   = report.MakeEndpointNodeID(scope, mapping.rewrittenIP, copyEndpointPort)
			node, ok         = rpt.Endpoint.Nodes[realEndpointID]
		)
		if !ok {
			return
		}

		rpt.Endpoint.AddNode(node.WithID(copyEndpointID).WithLatests(map[string]string{
			Addr:      mapping.rewrittenIP,
			Port:      copyEndpointPort,
			"copy_of": realEndpointID,
		}))

	})
}
Пример #8
0
	Client1PID      = "10001"
	Client2PID      = "30020"
	ServerPID       = "215"
	NonContainerPID = "1234"

	Client1Name      = "/usr/bin/curl"
	Client2Name      = "/usr/bin/curl"
	ServerName       = "apache"
	NonContainerName = "bash"

	True = "true"

	ClientHostNodeID = report.MakeHostNodeID(ClientHostID)
	ServerHostNodeID = report.MakeHostNodeID(ServerHostID)

	Client54001NodeID    = report.MakeEndpointNodeID(ClientHostID, ClientIP, ClientPort54001)            // curl (1)
	Client54002NodeID    = report.MakeEndpointNodeID(ClientHostID, ClientIP, ClientPort54002)            // curl (2)
	Server80NodeID       = report.MakeEndpointNodeID(ServerHostID, ServerIP, ServerPort)                 // apache
	UnknownClient1NodeID = report.MakeEndpointNodeID(ServerHostID, UnknownClient1IP, UnknownClient1Port) // we want to ensure two unknown clients, connnected
	UnknownClient2NodeID = report.MakeEndpointNodeID(ServerHostID, UnknownClient2IP, UnknownClient2Port) // to the same server, are deduped.
	UnknownClient3NodeID = report.MakeEndpointNodeID(ServerHostID, UnknownClient3IP, UnknownClient3Port) // Check this one isn't deduped
	RandomClientNodeID   = report.MakeEndpointNodeID(ServerHostID, RandomClientIP, RandomClientPort)     // this should become an internet node
	NonContainerNodeID   = report.MakeEndpointNodeID(ServerHostID, ServerIP, NonContainerClientPort)
	GoogleEndpointNodeID = report.MakeEndpointNodeID(ServerHostID, GoogleIP, GooglePort)

	ClientProcess1NodeID      = report.MakeProcessNodeID(ClientHostID, Client1PID)
	ClientProcess2NodeID      = report.MakeProcessNodeID(ClientHostID, Client2PID)
	ServerProcessNodeID       = report.MakeProcessNodeID(ServerHostID, ServerPID)
	NonContainerProcessNodeID = report.MakeProcessNodeID(ServerHostID, NonContainerPID)

	ClientContainerID     = "a1b2c3d4e5"
Пример #9
0
func demoReport(nodeCount int) report.Report {
	r := report.MakeReport()

	// Make up some plausible IPv4 numbers.
	hosts := []string{}
	ip := [4]int{192, 168, 1, 1}
	for range make([]struct{}, nodeCount) {
		hosts = append(hosts, fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]))
		ip[3]++
		if ip[3] > 200 {
			ip[2]++
			ip[3] = 1
		}
	}
	hosts = append(hosts, []string{"1.2.3.4", "2.3.4.5"}...) // Some non-local ones, too.

	_, localNet, err := net.ParseCIDR("192.168.0.0/16")
	if err != nil {
		panic(err)
	}

	type conn struct {
		srcProc, dstProc string
		dstPort          int
	}
	procPool := []conn{
		{srcProc: "curl", dstPort: 80, dstProc: "apache"},
		{srcProc: "wget", dstPort: 80, dstProc: "apache"},
		{srcProc: "curl", dstPort: 80, dstProc: "nginx"},
		{srcProc: "curl", dstPort: 8080, dstProc: "app1"},
		{srcProc: "nginx", dstPort: 8080, dstProc: "app1"},
		{srcProc: "nginx", dstPort: 8080, dstProc: "app2"},
		{srcProc: "nginx", dstPort: 8080, dstProc: "app3"},
	}
	connectionCount := nodeCount * 2
	for i := 0; i < connectionCount; i++ {
		var (
			c         = procPool[rand.Intn(len(procPool))]
			src       = hosts[rand.Intn(len(hosts))]
			dst       = hosts[rand.Intn(len(hosts))]
			srcPort   = rand.Intn(50000) + 10000
			srcPortID = report.MakeEndpointNodeID("", src, strconv.Itoa(srcPort))
			dstPortID = report.MakeEndpointNodeID("", dst, strconv.Itoa(c.dstPort))
		)

		// Endpoint topology
		r.Endpoint = r.Endpoint.AddNode(report.MakeNodeWith(srcPortID, map[string]string{
			process.PID: "4000",
			"name":      c.srcProc,
			"domain":    "node-" + src,
		}).
			WithEdge(dstPortID, report.EdgeMetadata{}))

		r.Endpoint = r.Endpoint.AddNode(report.MakeNodeWith(dstPortID, map[string]string{
			process.PID: "4000",
			"name":      c.dstProc,
			"domain":    "node-" + dst,
		}).
			WithEdge(srcPortID, report.EdgeMetadata{}))

		// Host data
		r.Host = r.Host.AddNode(report.MakeNodeWith("hostX", map[string]string{
			"ts":             time.Now().UTC().Format(time.RFC3339Nano),
			"host_name":      "host-x",
			"local_networks": localNet.String(),
			"os":             "linux",
		}))

	}

	return r
}
Пример #10
0
// Merge puts the packet into the report.
//
// Note that, for the moment, we encode bidirectional traffic as ingress and
// egress traffic on a single edge whose src is local and dst is remote. That
// is, if we see a packet from the remote addr 9.8.7.6 to the local addr
// 1.2.3.4, we apply it as *ingress* on the edge (1.2.3.4 -> 9.8.7.6).
func (s *Sniffer) Merge(p Packet, rpt *report.Report) {
	if p.SrcIP == "" || p.DstIP == "" {
		return
	}

	// One end of the traffic has to be local. Otherwise, we don't know how to
	// construct the edge.
	//
	// If we need to get around this limitation, we may be able to change the
	// semantics of the report, and allow the src side of edges to be from
	// anywhere. But that will have ramifications throughout Scope (read: it
	// may violate implicit invariants) and needs to be thought through.
	var (
		srcLocal   = s.localNets.Contains(net.ParseIP(p.SrcIP))
		dstLocal   = s.localNets.Contains(net.ParseIP(p.DstIP))
		localIP    string
		remoteIP   string
		localPort  string
		remotePort string
		egress     bool
	)
	switch {
	case srcLocal && !dstLocal:
		localIP, localPort, remoteIP, remotePort, egress = p.SrcIP, p.SrcPort, p.DstIP, p.DstPort, true
	case !srcLocal && dstLocal:
		localIP, localPort, remoteIP, remotePort, egress = p.DstIP, p.DstPort, p.SrcIP, p.SrcPort, false
	case srcLocal && dstLocal:
		localIP, localPort, remoteIP, remotePort, egress = p.SrcIP, p.SrcPort, p.DstIP, p.DstPort, true // loopback
	case !srcLocal && !dstLocal:
		log.Printf("sniffer ignoring remote-to-remote (%s -> %s) traffic", p.SrcIP, p.DstIP)
		return
	}

	addAdjacency := func(t report.Topology, srcNodeID, dstNodeID string) report.Topology {
		result := t.AddNode(report.MakeNode(srcNodeID).WithAdjacent(dstNodeID))
		result = result.AddNode(report.MakeNode(dstNodeID))
		return result
	}

	// If we have ports, we can add to the endpoint topology, too.
	if p.SrcPort != "" && p.DstPort != "" {
		var (
			srcNodeID = report.MakeEndpointNodeID(s.hostID, localIP, localPort)
			dstNodeID = report.MakeEndpointNodeID(s.hostID, remoteIP, remotePort)
		)

		rpt.Endpoint = addAdjacency(rpt.Endpoint, srcNodeID, dstNodeID)

		node := rpt.Endpoint.Nodes[srcNodeID]
		emd, _ := node.Edges.Lookup(dstNodeID)
		if egress {
			if emd.EgressPacketCount == nil {
				emd.EgressPacketCount = new(uint64)
			}
			*emd.EgressPacketCount++
			if emd.EgressByteCount == nil {
				emd.EgressByteCount = new(uint64)
			}
			*emd.EgressByteCount += uint64(p.Transport)
		} else {
			if emd.IngressPacketCount == nil {
				emd.IngressPacketCount = new(uint64)
			}
			*emd.IngressPacketCount++
			if emd.IngressByteCount == nil {
				emd.IngressByteCount = new(uint64)
			}
			*emd.IngressByteCount += uint64(p.Transport)
		}
		rpt.Endpoint.Nodes[srcNodeID] = node.WithEdge(dstNodeID, emd)
	}
}
Пример #11
0
	"$GITHUB_URI/report"
)

var (
	clientHostID     = "client.host.com"
	clientHostName   = clientHostID
	clientHostNodeID = report.MakeHostNodeID(clientHostID)
	clientAddress    = "10.10.10.20"
	serverHostID     = "server.host.com"
	serverHostName   = serverHostID
	serverHostNodeID = report.MakeHostNodeID(serverHostID)
	serverAddress    = "10.10.10.1"
	unknownHostID    = ""              // by definition, we don't know it
	unknownAddress   = "172.16.93.112" // will be a pseudonode, no corresponding host

	client54001EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54001") // i.e. curl
	client54002EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54002") // also curl
	server80EndpointNodeID    = report.MakeEndpointNodeID(serverHostID, serverAddress, "80")    // i.e. apache
	unknown1EndpointNodeID    = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10001")
	unknown2EndpointNodeID    = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10002")
	unknown3EndpointNodeID    = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10003")

	clientAddressNodeID  = report.MakeAddressNodeID(clientHostID, clientAddress)
	serverAddressNodeID  = report.MakeAddressNodeID(serverHostID, serverAddress)
	unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, unknownAddress)
)

func TestEndpointNodeID(t *testing.T) {
	for _, bad := range []string{
		clientAddressNodeID,
		serverAddressNodeID,
Пример #12
0
func TestNat(t *testing.T) {
	mtime.NowForce(mtime.Now())
	defer mtime.NowReset()

	// test that two containers, on the docker network, get their connections mapped
	// correctly.
	// the setup is this:
	//
	// container2 (10.0.47.2:222222), host2 (2.3.4.5:22223) ->
	//     host1 (1.2.3.4:80), container1 (10.0.47.1:80)

	// from the PoV of host1
	{
		f := makeFlow(updateType)
		addIndependant(&f, 1, "")
		f.Original = addMeta(&f, "original", "2.3.4.5", "1.2.3.4", 222222, 80)
		f.Reply = addMeta(&f, "reply", "10.0.47.1", "2.3.4.5", 80, 222222)
		ct := &mockFlowWalker{
			flows: []flow{f},
		}

		have := report.MakeReport()
		originalID := report.MakeEndpointNodeID("host1", "10.0.47.1", "80")
		have.Endpoint.AddNode(report.MakeNodeWith(originalID, map[string]string{
			Addr:      "10.0.47.1",
			Port:      "80",
			"foo":     "bar",
			Procspied: "true",
		}))

		want := have.Copy()
		wantID := report.MakeEndpointNodeID("host1", "1.2.3.4", "80")
		want.Endpoint.AddNode(report.MakeNodeWith(wantID, map[string]string{
			Addr:      "1.2.3.4",
			Port:      "80",
			"copy_of": originalID,
			"foo":     "bar",
			Procspied: "true",
		}))

		makeNATMapper(ct).applyNAT(have, "host1")
		if !reflect.DeepEqual(want, have) {
			t.Fatal(test.Diff(want, have))
		}
	}

	// form the PoV of host2
	{
		f := makeFlow(updateType)
		addIndependant(&f, 2, "")
		f.Original = addMeta(&f, "original", "10.0.47.2", "1.2.3.4", 22222, 80)
		f.Reply = addMeta(&f, "reply", "1.2.3.4", "2.3.4.5", 80, 22223)
		ct := &mockFlowWalker{
			flows: []flow{f},
		}

		have := report.MakeReport()
		originalID := report.MakeEndpointNodeID("host2", "10.0.47.2", "22222")
		have.Endpoint.AddNode(report.MakeNodeWith(originalID, map[string]string{
			Addr:      "10.0.47.2",
			Port:      "22222",
			"foo":     "baz",
			Procspied: "true",
		}))

		want := have.Copy()
		want.Endpoint.AddNode(report.MakeNodeWith(report.MakeEndpointNodeID("host2", "2.3.4.5", "22223"), map[string]string{
			Addr:      "2.3.4.5",
			Port:      "22223",
			"copy_of": originalID,
			"foo":     "baz",
			Procspied: "true",
		}))

		makeNATMapper(ct).applyNAT(have, "host1")
		if !reflect.DeepEqual(want, have) {
			t.Fatal(test.Diff(want, have))
		}
	}
}
Пример #13
0
	"$GITHUB_URI/common/mtime"

	"$GITHUB_URI/probe/docker"
	"$GITHUB_URI/probe/endpoint"
	"$GITHUB_URI/probe/host"
	"$GITHUB_URI/render"
	"$GITHUB_URI/report"
)

var (
	serverHostID     = "host1"
	serverHostNodeID = report.MakeHostNodeID(serverHostID)

	randomIP             = "3.4.5.6"
	randomPort           = "56789"
	randomEndpointNodeID = report.MakeEndpointNodeID(serverHostID, randomIP, randomPort)

	serverIP             = "192.168.1.1"
	serverPort           = "80"
	serverEndpointNodeID = report.MakeEndpointNodeID(serverHostID, serverIP, serverPort)

	container1ID     = "11b2c3d4e5"
	container1IP     = "192.168.0.1"
	container1Name   = "foo"
	container1NodeID = report.MakeContainerNodeID(container1ID)

	container1Port           = "16782"
	container1EndpointNodeID = report.MakeEndpointNodeID(serverHostID, container1IP, container1Port)

	duplicatedIP             = "192.168.0.2"
	duplicatedPort           = "80"