func (b SyslogBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
br := bufio.NewReader(reader)
lines := uint64(0)
for {
line, err := br.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
log.Print(err)
return lines, err
}
lines++
ipsFound := IPRegex.FindAllStringSubmatch(line, -1)
for _, ipMatches := range ipsFound {
for _, idx := range IPIndexes {
if ipMatches[idx] != "" {
ips.AddString(ipMatches[idx])
}
}
}
}
return lines, nil
}
func (ls *LevelDBStore) AddDocument(filename string, ips ipset.Set) error {
exists, err := ls.HasDocument(filename)
if err != nil {
return err
}
if exists {
return nil
}
nextID, err := ls.nextDocID()
if err != nil {
return err
}
ls.batch = new(leveldb.Batch)
ls.setDocId(filename, nextID)
for _, ip := range ips.SortedStrings() {
//fmt.Printf("Add %#v to document\n", ip)
err = ls.addIP(nextID, ip)
if err != nil {
return err
}
}
err = ls.db.Write(ls.batch, nil)
ls.batch = nil
return err
}
func (b NFDUMPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
cmd := exec.Command("nfdump", "-r", "-", "-o", "csv")
cmd.Stdin = reader
stdout, err := cmd.StdoutPipe()
if err != nil {
return 0, err
}
err = cmd.Start()
if err != nil {
return 0, err
}
br := bufio.NewReader(stdout)
lines := uint64(0)
for {
line, err := br.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
return lines, err
}
parts := strings.SplitN(line, ",", 6) //makes parts[4] the last full split
if len(parts) == 6 {
ips.AddString(parts[3])
ips.AddString(parts[4])
lines++
}
}
err = cmd.Wait()
return lines, err
}
func (bs *BoltStore) AddDocument(filename string, ips ipset.Set) error {
exists, err := bs.HasDocument(filename)
if err != nil {
return err
}
if exists {
return nil
}
err = bs.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("docs"))
nextID, err := nextDocID(b)
if err != nil {
return err
}
fmt.Printf("NextDocID should be %d\n", nextID)
setDocId(b, filename, nextID)
ipBucket := tx.Bucket([]byte("ips"))
for _, ip := range ips.SortedStrings() {
//fmt.Printf("Add %#v to document\n", k)
addIP(ipBucket, nextID, ip)
}
return nil
})
return nil
}
func (b BroJSONBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
br := bufio.NewReader(reader)
lines := uint64(0)
for {
var FoundIPS BroIPFields
line, err := br.ReadSlice('\n')
if err == io.EOF {
break
}
if err != nil {
return lines, err
}
err = FoundIPS.UnmarshalJSON(line)
if err != nil {
return lines, err
}
if FoundIPS.ID_orig_h != "" {
ips.AddString(FoundIPS.ID_orig_h)
}
if FoundIPS.ID_resp_h != "" {
ips.AddString(FoundIPS.ID_resp_h)
}
if FoundIPS.Src != "" {
ips.AddString(FoundIPS.Src)
}
if FoundIPS.Dst != "" {
ips.AddString(FoundIPS.Dst)
}
lines++
}
return lines, nil
}
func (b BroBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
br := bufio.NewReader(reader)
lines := uint64(0)
for {
line, err := br.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
return lines, err
}
if line[0] != '#' {
parts := strings.SplitN(line, "\t", 6) //makes parts[4] the last full split
ips.AddString(parts[2])
ips.AddString(parts[4])
lines++
}
}
return lines, nil
}
func (b PCAPBackend) ExtractIps(reader io.Reader, ips *ipset.Set) (uint64, error) {
packets := uint64(0)
pr, err := pcapgo.NewReader(reader)
if err != nil {
return 0, err
}
var eth layers.Ethernet
var dot1q layers.Dot1Q
var ip4 layers.IPv4
var ip6 layers.IPv6
var tcp layers.TCP
parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, ð, &dot1q, &ip4, &ip6, &tcp)
decoded := []gopacket.LayerType{}
for {
packetData, _, err := pr.ReadPacketData()
packets++
if err == io.EOF {
break
}
if err != nil {
return packets, err
}
err = parser.DecodeLayers(packetData, &decoded)
for _, layerType := range decoded {
switch layerType {
case layers.LayerTypeIPv6:
ips.AddIP(ip6.SrcIP)
ips.AddIP(ip6.DstIP)
case layers.LayerTypeIPv4:
ips.AddIP(ip4.SrcIP)
ips.AddIP(ip4.DstIP)
}
}
}
return packets, nil
}
