func ExampleImport() { r, err := os.Open("./testdata/testdata.txt.gz") if err != nil { panic("cannot open file") } // Create a database to write to mem := testdb.NewMemDBBulk() // The input is gzipped text file with // one input per line, so we choose a tokenizer // that matches. in, err := tokenizer.NewGzLine(r) if err != nil { panic(err) } // Import using the default sanitizer err = Import(in, mem, nil) if err != nil { panic(err) } // Data is now imported, let's do a check // Check a password that is in the sample data err = Check("tl1992rell", mem, nil) fmt.Println(err) // Output:password found in database }
func TestImportBz2(t *testing.T) { r, err := os.Open("rockyou.txt.bz2") if err != nil { t.Skip("Skipping bz2 file test. 'rockyou.txt.bz2' must be in current dir") } mem := testdb.NewMemDBBulk() in := tokenizer.NewBz2Line(r) err = Import(in, mem, nil) if err != nil { t.Fatal(err) } }
func TestImportBig(t *testing.T) { r, err := os.Open("crackstation-human-only.txt.gz") if err != nil { t.Skip("Skipping big file test. 'crackstation-human-only.txt.gz' must be in current dir") } mem := testdb.NewMemDBBulk() in, err := tokenizer.NewGzLine(r) if err != nil { t.Fatal(err) } err = Import(in, mem, nil) if err != nil { t.Fatal(err) } }
func TestInDBBulk(t *testing.T) { buf, err := testdata.Asset("testdata.txt.gz") if err != nil { t.Fatal(err) } mem := testdb.NewMemDBBulk() in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf)) if err != nil { t.Fatal(err) } err = Import(in, mem, nil) for p := range testdata.TestSet { if SanitizeOK(p, nil) != nil { continue } has, err := inDB(p, mem, nil) if err != nil { t.Fatal(err) } if !has { t.Fatalf("db should have: %s", p) } err = Check(p, mem, nil) if err != ErrPasswordInDB { t.Fatal("check failed on:", p, err) } } for p := range testdata.NotInSet { if SanitizeOK(p, nil) != nil { continue } has, err := inDB(p, mem, nil) if err != nil { t.Fatal(err) } if has { t.Fatalf("db should not have: %s", p) } err = Check(p, mem, nil) if err != nil { t.Fatal("check failed on:", p, err) } } }
func TestImportBulk(t *testing.T) { buf, err := testdata.Asset("testdata.txt.gz") if err != nil { t.Fatal(err) } mem := testdb.NewMemDBBulk() in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf)) if err != nil { t.Fatal(err) } err = Import(in, mem, nil) if err != nil { t.Fatal(err) } // Test everything is kept open. err = Import(in, mem, nil) if err != nil { t.Fatal(err) } }
// Open a xz compressed archive and import it. // Uses the "xi2.org/x/xz" package to read xz files. func ExampleImport_xz() { r, err := os.Open("rockyou.txt.xz") if err != nil { // Fake it fmt.Println("Imported", 9341543, "items") return } xzr, err := xz.NewReader(r, 0) if err != nil { panic(err) } mem := testdb.NewMemDBBulk() in := tokenizer.NewLine(xzr) err = Import(in, mem, nil) if err != nil { panic(err) } fmt.Println("Imported", len(*mem), "items") // Output: Imported 9341543 items }