Example #1
0
func runTest(t *testing.T, s IpStore) {
	ips := ipset.New()
	ips.AddString("1.2.3.1")
	ips.AddString("1.2.3.2")
	ips.AddString("1.2.3.3")
	ips.AddString("1.2.3.4")
	ips.AddString("1.2.3.255")
	s.AddDocument("/log/1.txt", *ips)

	ips.AddString("2.0.0.2")
	ips.AddString("2.0.0.3")
	s.AddDocument("/log/2.txt", *ips)

	ips = ipset.New()
	ips.AddString("102:304::1")
	s.AddDocument("/log/3.txt", *ips)

	for _, tt := range basicSearchTable {
		matches, err := s.QueryString(tt.query)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(matches, tt.docs) {
			t.Errorf("store.QueryString(%s) => %#v, want %#v", tt.query, matches, tt.docs)
		}
	}

	for _, tt := range basicExpandCidrTable {
		matches, err := s.ExpandCIDR(tt.query)
		if err != nil {
			t.Fatal(err)
		}
		if fmt.Sprintf("%v", matches) != fmt.Sprintf("%v", tt.ips) {
			t.Errorf("store.ExpandCIDR(%s) => %v, want %v", tt.query, matches, tt.ips)
		}
	}

	ips = ipset.New()
	ips.AddString("100.111.99.58") //doc: in hex
	ips.AddString("646f:633a::1")  //doc: in hex
	s.AddDocument("/log/special.txt", *ips)
	for _, tt := range specialSearchTable {
		matches, err := s.QueryString(tt.query)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(matches, tt.docs) {
			t.Errorf("store.QueryString(%s) => %#v, want %#v", tt.query, matches, tt.docs)
		}
	}

}
Example #2
0
func Index(s store.IpStore, b backend.Backend, filename string) error {
	exists, err := s.HasDocument(filename)
	if err != nil {
		return err
	}
	if exists {
		//log.Printf("%s Already indexed\n", filename)
		return nil
	}
	ips := ipset.New()
	reader, err := backend.OpenDecompress(filename)
	if err != nil {
		return err
	}
	defer reader.Close()

	start := time.Now()
	lines, err := b.ExtractIps(reader, ips)
	duration := time.Since(start)
	log.Printf("%s: Read %d lines in %s\n", filename, lines, duration)

	if err != nil {
		log.Printf("%s: Non fatal read error: %s\n", filename, err)
	}
	start = time.Now()
	s.AddDocument(filename, *ips)
	duration = time.Since(start)
	log.Printf("%s: Wrote %d unique ips in %s\n", filename, ips.Count(), duration)
	return nil
}
Example #3
0
func (i *Indexer) ExpandCIDR(query string) ([]net.IP, error) {
	allips := ipset.New()

	for _, store := range i.stores {
		ips, err := store.ExpandCIDR(query)
		if err != nil {
			return []net.IP{}, err
		}
		for _, ip := range ips {
			allips.AddIP(ip)
		}
	}
	return allips.SortedIPs(), nil
}
Example #4
0
func runStoreBench(b *testing.B, storeType string, documents int) {
	mystore, err := NewStore(storeType, "test.db")
	if err != nil {
		b.Error(err)
		return
	}
	defer os.RemoveAll("test.db")
	for n := 0; n < b.N; n++ {
		for doc := 0; doc < documents; doc++ {
			b.StopTimer()
			ips := ipset.New()
			for i := 0; i < 30000; i++ {
				ips.AddString(loggen.PartiallyRandomIPv4(2))
			}
			b.StartTimer()
			mystore.AddDocument(fmt.Sprintf("test-%d-%d.txt", doc, n), *ips)
		}
	}
}
Example #5
0
func ExtractIpsReader(backend string, reader io.Reader) (*ipset.Set, error) {
	ips := ipset.New()
	b := NewBackend(backend)
	_, err := b.ExtractIps(reader, ips)
	return ips, err
}