func runTest(t *testing.T, s IpStore) {
ips := ipset.New()
ips.AddString("1.2.3.1")
ips.AddString("1.2.3.2")
ips.AddString("1.2.3.3")
ips.AddString("1.2.3.4")
ips.AddString("1.2.3.255")
s.AddDocument("/log/1.txt", *ips)
ips.AddString("2.0.0.2")
ips.AddString("2.0.0.3")
s.AddDocument("/log/2.txt", *ips)
ips = ipset.New()
ips.AddString("102:304::1")
s.AddDocument("/log/3.txt", *ips)
for _, tt := range basicSearchTable {
matches, err := s.QueryString(tt.query)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(matches, tt.docs) {
t.Errorf("store.QueryString(%s) => %#v, want %#v", tt.query, matches, tt.docs)
}
}
for _, tt := range basicExpandCidrTable {
matches, err := s.ExpandCIDR(tt.query)
if err != nil {
t.Fatal(err)
}
if fmt.Sprintf("%v", matches) != fmt.Sprintf("%v", tt.ips) {
t.Errorf("store.ExpandCIDR(%s) => %v, want %v", tt.query, matches, tt.ips)
}
}
ips = ipset.New()
ips.AddString("100.111.99.58") //doc: in hex
ips.AddString("646f:633a::1") //doc: in hex
s.AddDocument("/log/special.txt", *ips)
for _, tt := range specialSearchTable {
matches, err := s.QueryString(tt.query)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(matches, tt.docs) {
t.Errorf("store.QueryString(%s) => %#v, want %#v", tt.query, matches, tt.docs)
}
}
}
func Index(s store.IpStore, b backend.Backend, filename string) error {
exists, err := s.HasDocument(filename)
if err != nil {
return err
}
if exists {
//log.Printf("%s Already indexed\n", filename)
return nil
}
ips := ipset.New()
reader, err := backend.OpenDecompress(filename)
if err != nil {
return err
}
defer reader.Close()
start := time.Now()
lines, err := b.ExtractIps(reader, ips)
duration := time.Since(start)
log.Printf("%s: Read %d lines in %s\n", filename, lines, duration)
if err != nil {
log.Printf("%s: Non fatal read error: %s\n", filename, err)
}
start = time.Now()
s.AddDocument(filename, *ips)
duration = time.Since(start)
log.Printf("%s: Wrote %d unique ips in %s\n", filename, ips.Count(), duration)
return nil
}
func (i *Indexer) ExpandCIDR(query string) ([]net.IP, error) {
allips := ipset.New()
for _, store := range i.stores {
ips, err := store.ExpandCIDR(query)
if err != nil {
return []net.IP{}, err
}
for _, ip := range ips {
allips.AddIP(ip)
}
}
return allips.SortedIPs(), nil
}
func runStoreBench(b *testing.B, storeType string, documents int) {
mystore, err := NewStore(storeType, "test.db")
if err != nil {
b.Error(err)
return
}
defer os.RemoveAll("test.db")
for n := 0; n < b.N; n++ {
for doc := 0; doc < documents; doc++ {
b.StopTimer()
ips := ipset.New()
for i := 0; i < 30000; i++ {
ips.AddString(loggen.PartiallyRandomIPv4(2))
}
b.StartTimer()
mystore.AddDocument(fmt.Sprintf("test-%d-%d.txt", doc, n), *ips)
}
}
}
func ExtractIpsReader(backend string, reader io.Reader) (*ipset.Set, error) {
ips := ipset.New()
b := NewBackend(backend)
_, err := b.ExtractIps(reader, ips)
return ips, err
}