Example #1
0
// 1. parse args
// 2. start the loghisto metric system
// 3. start the processing and printing goroutines
// 4. open the pcap handler
// 5. hand off packets from the handler to the decoder
func main() {
	portsArg := flag.String("ports", "2379", "etcd listening ports")
	iface := flag.String("iface", "eth0", "interface for sniffing traffic on")
	promisc := flag.Bool("promiscuous", true, "promiscuous mode")
	period := flag.Uint("period", 1, "seconds between submissions")
	topK := flag.Uint("topk", 10, "submit stats for the top <K> sniffed paths")
	flag.Parse()

	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	ms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false)
	ms.Start()
	metricStream := make(chan *loghisto.ProcessedMetricSet, 2)
	ms.SubscribeToProcessedMetrics(metricStream)
	defer ms.UnsubscribeFromProcessedMetrics(metricStream)

	go statPrinter(metricStream, *topK, *period)

	ports := []uint16{}
	for _, p := range strings.Split(*portsArg, ",") {
		port, err := strconv.Atoi(p)
		if err == nil {
			ports = append(ports, uint16(port))
		} else {
			fmt.Fprintf(os.Stderr, "Failed to parse port \"%s\": %v\n", p, err)
			os.Exit(1)
		}
	}

	if len(ports) == 0 {
		fmt.Fprint(os.Stderr, "No ports given!  Exiting.\n")
		os.Exit(1)
	}

	// We choose 1518 for the snaplen because it's the default
	// ethernet MTU at the link layer.  We choose 1000 for the
	// timeout based on a measurement for its impact on latency
	// impact, but it is less precise.
	h, err := pcap.Openlive(*iface, 1518, *promisc, 1000)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		os.Exit(1)
	}
	defer h.Close()

	portArray := strings.Split(*portsArg, ",")
	dst := strings.Join(portArray, " or dst port ")
	src := strings.Join(portArray, " or src port ")
	filter := fmt.Sprintf("tcp and (dst port %s or src port %s)", dst, src)
	fmt.Println("using bpf filter: ", filter)
	if err := h.Setfilter(filter); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		os.Exit(1)
	}

	unparsedPackets := make(chan *pcap.Packet, 16384)
	parsedPackets := make(chan *pcap.Packet, 16384)
	for i := 0; i < int(math.Max(2, float64(numCPU/4))); i++ {
		go packetDecoder(unparsedPackets, parsedPackets)
	}

	processors := []chan *pcap.Packet{}
	for i := 0; i < int(math.Max(2, float64(numCPU/4))); i++ {
		p := make(chan *pcap.Packet, 16384)
		processors = append(processors, p)
		go processor(ms, p)
	}

	go streamRouter(ports, parsedPackets, processors)

	for {
		pkt := h.Next()
		if pkt != nil {
			select {
			case unparsedPackets <- pkt:
			default:
				fmt.Fprint(os.Stderr, "SHEDDING IN MAIN")
			}
		}
	}
}
Example #2
0
// 1. parse args
// 2. start the prometheus listener if configured
// 3. start the loghisto metric system
// 4. start the processing and printing goroutines
// 5. open the pcap handler
// 6. hand off packets from the handler to the decoder
func main() {
	portsArg := flag.String("ports", "4001,2379", "etcd listening ports")
	iface := flag.String("iface", "eth0", "interface for sniffing traffic on")
	promisc := flag.Bool("promiscuous", false, "promiscuous mode")
	period := flag.Uint("period", 60, "seconds between submissions")
	topK := flag.Uint("topk", 10, "submit stats for the top <K> sniffed paths")
	prometheusPort := flag.Uint("prometheus-port", 0, "port for prometheus exporter to listen on")
	flag.Parse()

	if *prometheusPort != 0 {
		http.Handle("/metrics", prometheus.UninstrumentedHandler())
		go http.ListenAndServe(":"+strconv.Itoa(int(*prometheusPort)), nil)
	}

	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	ms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false)
	ms.Start()
	metricStream := make(chan *loghisto.ProcessedMetricSet, 2)
	ms.SubscribeToProcessedMetrics(metricStream)
	defer ms.UnsubscribeFromProcessedMetrics(metricStream)

	go statPrinter(metricStream, *topK, *period)

	ports := []uint16{}
	for _, p := range strings.Split(*portsArg, ",") {
		p, err := strconv.Atoi(p)
		if err == nil {
			ports = append(ports, uint16(p))
		}
	}

	h, err := pcap.Openlive(*iface, 1518, *promisc, 1000)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer h.Close()

	portArray := strings.Split(*portsArg, ",")
	dst := strings.Join(portArray, " or dst port ")
	src := strings.Join(portArray, " or src port ")
	filter := fmt.Sprintf("tcp and (dst port %s or src port %s)", dst, src)
	fmt.Println("using bpf filter: ", filter)
	if err := h.Setfilter(filter); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	unparsedPackets := make(chan *pcap.Packet, 10240)
	parsedPackets := make(chan *pcap.Packet, 10240)
	for i := 0; i < 5; i++ {
		go packetDecoder(unparsedPackets, parsedPackets)
	}

	processors := []chan *pcap.Packet{}
	for i := 0; i < 50; i++ {
		p := make(chan *pcap.Packet, 10240)
		processors = append(processors, p)
		go processor(ms, p)
	}

	go streamRouter(ports, parsedPackets, processors)

	for {
		pkt := h.Next()
		if pkt != nil {
			select {
			case unparsedPackets <- pkt:
			default:
				fmt.Println("SHEDDING IN MAIN")
			}
		}
	}
}
Example #3
0
func main() {
	hosts := flag.String("hosts", "http://localhost:2379", "comma separated etcd hosts to spew at")
	flag.Parse()

	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	ms := loghisto.NewMetricSystem(time.Second, false)
	ms.Start()
	metricStream := make(chan *loghisto.ProcessedMetricSet, 2)
	ms.SubscribeToProcessedMetrics(metricStream)
	defer ms.UnsubscribeFromProcessedMetrics(metricStream)

	machines := strings.Split(*hosts, ",")
	// use zipfian distribution
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	zipf := rand.NewZipf(r, 3.14, 2.72, 500000)
	go reporter(metricStream)
	for i := 0; i < 5; i++ {
		go func() {
			client := etcd.NewClient(machines)

			for i := 0; i < 3; i++ {
				go func() {
					for {
						rando := rand.Float64()
						valLen := int32(math.Max(float64(zipf.Uint64()), 1))
						if rando > 0.8 {
							t := ms.StartTimer("PutLat")
							if _, err := client.Set("/"+RandString(1), RandString(500), 0); err != nil {
								log.Fatal(err)
							}
							t.Stop()
							ms.Histogram("PutSz", float64(valLen))
							ms.Counter("Put", 1)
						} else if rando > 0.7 {
							t := ms.StartTimer("DeleteLat")
							client.Delete("/"+RandString(1), true)
							t.Stop()
							ms.Counter("Delete", 1)
						} else if rando > 0.65 {
							t := ms.StartTimer("AddChildLat")
							client.AddChild("/"+RandString(2), RandString(valLen), 0)
							t.Stop()
							ms.Counter("AddChild", 1)
						} else {
							t := ms.StartTimer("GetLat")
							r, err := client.Get("/"+RandString(1), false, false)
							if err == nil {
								ms.Histogram("GetSz", float64(len(r.Node.Value)))
							}
							t.Stop()
							ms.Counter("Get", 1)
						}
					}
				}()
			}
		}()
	}
	<-make(chan struct{})
}