コード例 #1
0
ファイル: syslog.go プロジェクト: JustinAzoff/flow-indexer
func (b SyslogBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
	br := bufio.NewReader(reader)

	lines := uint64(0)
	for {
		line, err := br.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Print(err)
			return lines, err
		}
		lines++
		ipsFound := IPRegex.FindAllStringSubmatch(line, -1)

		for _, ipMatches := range ipsFound {
			for _, idx := range IPIndexes {
				if ipMatches[idx] != "" {
					ips.AddString(ipMatches[idx])
				}
			}
		}
	}
	return lines, nil
}
コード例 #2
0
ファイル: leveldb.go プロジェクト: JustinAzoff/flow-indexer
func (ls *LevelDBStore) AddDocument(filename string, ips ipset.Set) error {
	exists, err := ls.HasDocument(filename)
	if err != nil {
		return err
	}
	if exists {
		return nil
	}

	nextID, err := ls.nextDocID()
	if err != nil {
		return err
	}
	ls.batch = new(leveldb.Batch)
	ls.setDocId(filename, nextID)
	for _, ip := range ips.SortedStrings() {
		//fmt.Printf("Add %#v to document\n", ip)
		err = ls.addIP(nextID, ip)
		if err != nil {
			return err
		}
	}
	err = ls.db.Write(ls.batch, nil)
	ls.batch = nil
	return err

}
コード例 #3
0
ファイル: nfdump.go プロジェクト: JustinAzoff/flow-indexer
func (b NFDUMPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
	cmd := exec.Command("nfdump", "-r", "-", "-o", "csv")
	cmd.Stdin = reader
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return 0, err
	}
	err = cmd.Start()
	if err != nil {
		return 0, err
	}
	br := bufio.NewReader(stdout)

	lines := uint64(0)
	for {
		line, err := br.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return lines, err
		}
		parts := strings.SplitN(line, ",", 6) //makes parts[4] the last full split
		if len(parts) == 6 {
			ips.AddString(parts[3])
			ips.AddString(parts[4])
			lines++
		}
	}
	err = cmd.Wait()
	return lines, err
}
コード例 #4
0
ファイル: boltdb.go プロジェクト: JustinAzoff/flow-indexer
func (bs *BoltStore) AddDocument(filename string, ips ipset.Set) error {
	exists, err := bs.HasDocument(filename)
	if err != nil {
		return err
	}
	if exists {
		return nil
	}
	err = bs.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("docs"))
		nextID, err := nextDocID(b)
		if err != nil {
			return err
		}
		fmt.Printf("NextDocID should be %d\n", nextID)
		setDocId(b, filename, nextID)
		ipBucket := tx.Bucket([]byte("ips"))
		for _, ip := range ips.SortedStrings() {
			//fmt.Printf("Add %#v to document\n", k)
			addIP(ipBucket, nextID, ip)
		}
		return nil
	})

	return nil
}
コード例 #5
0
ファイル: bro_json.go プロジェクト: JustinAzoff/flow-indexer
func (b BroJSONBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
	br := bufio.NewReader(reader)

	lines := uint64(0)
	for {
		var FoundIPS BroIPFields
		line, err := br.ReadSlice('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return lines, err
		}

		err = FoundIPS.UnmarshalJSON(line)
		if err != nil {
			return lines, err
		}
		if FoundIPS.ID_orig_h != "" {
			ips.AddString(FoundIPS.ID_orig_h)
		}
		if FoundIPS.ID_resp_h != "" {
			ips.AddString(FoundIPS.ID_resp_h)
		}
		if FoundIPS.Src != "" {
			ips.AddString(FoundIPS.Src)
		}
		if FoundIPS.Dst != "" {
			ips.AddString(FoundIPS.Dst)
		}
		lines++
	}
	return lines, nil
}
コード例 #6
0
ファイル: bro.go プロジェクト: JustinAzoff/flow-indexer
func (b BroBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
	br := bufio.NewReader(reader)

	lines := uint64(0)
	for {
		line, err := br.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return lines, err
		}
		if line[0] != '#' {
			parts := strings.SplitN(line, "\t", 6) //makes parts[4] the last full split
			ips.AddString(parts[2])
			ips.AddString(parts[4])
			lines++
		}
	}
	return lines, nil
}
コード例 #7
0
ファイル: pcap.go プロジェクト: JustinAzoff/flow-indexer
func (b PCAPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
	packets := uint64(0)

	pr, err := pcapgo.NewReader(reader)
	if err != nil {
		return 0, err
	}
	var eth layers.Ethernet
	var dot1q layers.Dot1Q
	var ip4 layers.IPv4
	var ip6 layers.IPv6
	var tcp layers.TCP
	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &dot1q, &ip4, &ip6, &tcp)
	decoded := []gopacket.LayerType{}
	for {
		packetData, _, err := pr.ReadPacketData()
		packets++
		if err == io.EOF {
			break
		}
		if err != nil {
			return packets, err
		}
		err = parser.DecodeLayers(packetData, &decoded)
		for _, layerType := range decoded {
			switch layerType {
			case layers.LayerTypeIPv6:
				ips.AddIP(ip6.SrcIP)
				ips.AddIP(ip6.DstIP)
			case layers.LayerTypeIPv4:
				ips.AddIP(ip4.SrcIP)
				ips.AddIP(ip4.DstIP)
			}
		}
	}

	return packets, nil
}