func (b SyslogBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) { br := bufio.NewReader(reader) lines := uint64(0) for { line, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { log.Print(err) return lines, err } lines++ ipsFound := IPRegex.FindAllStringSubmatch(line, -1) for _, ipMatches := range ipsFound { for _, idx := range IPIndexes { if ipMatches[idx] != "" { ips.AddString(ipMatches[idx]) } } } } return lines, nil }
func (ls *LevelDBStore) AddDocument(filename string, ips ipset.Set) error { exists, err := ls.HasDocument(filename) if err != nil { return err } if exists { return nil } nextID, err := ls.nextDocID() if err != nil { return err } ls.batch = new(leveldb.Batch) ls.setDocId(filename, nextID) for _, ip := range ips.SortedStrings() { //fmt.Printf("Add %#v to document\n", ip) err = ls.addIP(nextID, ip) if err != nil { return err } } err = ls.db.Write(ls.batch, nil) ls.batch = nil return err }
func (b NFDUMPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) { cmd := exec.Command("nfdump", "-r", "-", "-o", "csv") cmd.Stdin = reader stdout, err := cmd.StdoutPipe() if err != nil { return 0, err } err = cmd.Start() if err != nil { return 0, err } br := bufio.NewReader(stdout) lines := uint64(0) for { line, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { return lines, err } parts := strings.SplitN(line, ",", 6) //makes parts[4] the last full split if len(parts) == 6 { ips.AddString(parts[3]) ips.AddString(parts[4]) lines++ } } err = cmd.Wait() return lines, err }
func (bs *BoltStore) AddDocument(filename string, ips ipset.Set) error { exists, err := bs.HasDocument(filename) if err != nil { return err } if exists { return nil } err = bs.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("docs")) nextID, err := nextDocID(b) if err != nil { return err } fmt.Printf("NextDocID should be %d\n", nextID) setDocId(b, filename, nextID) ipBucket := tx.Bucket([]byte("ips")) for _, ip := range ips.SortedStrings() { //fmt.Printf("Add %#v to document\n", k) addIP(ipBucket, nextID, ip) } return nil }) return nil }
func (b BroJSONBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) { br := bufio.NewReader(reader) lines := uint64(0) for { var FoundIPS BroIPFields line, err := br.ReadSlice('\n') if err == io.EOF { break } if err != nil { return lines, err } err = FoundIPS.UnmarshalJSON(line) if err != nil { return lines, err } if FoundIPS.ID_orig_h != "" { ips.AddString(FoundIPS.ID_orig_h) } if FoundIPS.ID_resp_h != "" { ips.AddString(FoundIPS.ID_resp_h) } if FoundIPS.Src != "" { ips.AddString(FoundIPS.Src) } if FoundIPS.Dst != "" { ips.AddString(FoundIPS.Dst) } lines++ } return lines, nil }
func (b BroBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) { br := bufio.NewReader(reader) lines := uint64(0) for { line, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { return lines, err } if line[0] != '#' { parts := strings.SplitN(line, "\t", 6) //makes parts[4] the last full split ips.AddString(parts[2]) ips.AddString(parts[4]) lines++ } } return lines, nil }
func (b PCAPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) { packets := uint64(0) pr, err := pcapgo.NewReader(reader) if err != nil { return 0, err } var eth layers.Ethernet var dot1q layers.Dot1Q var ip4 layers.IPv4 var ip6 layers.IPv6 var tcp layers.TCP parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, ð, &dot1q, &ip4, &ip6, &tcp) decoded := []gopacket.LayerType{} for { packetData, _, err := pr.ReadPacketData() packets++ if err == io.EOF { break } if err != nil { return packets, err } err = parser.DecodeLayers(packetData, &decoded) for _, layerType := range decoded { switch layerType { case layers.LayerTypeIPv6: ips.AddIP(ip6.SrcIP) ips.AddIP(ip6.DstIP) case layers.LayerTypeIPv4: ips.AddIP(ip4.SrcIP) ips.AddIP(ip4.DstIP) } } } return packets, nil }