func ExampleImport() {
r, err := os.Open("./testdata/testdata.txt.gz")
if err != nil {
panic("cannot open file")
}
// Create a database to write to
mem := testdb.NewMemDBBulk()
// The input is gzipped text file with
// one input per line, so we choose a tokenizer
// that matches.
in, err := tokenizer.NewGzLine(r)
if err != nil {
panic(err)
}
// Import using the default sanitizer
err = Import(in, mem, nil)
if err != nil {
panic(err)
}
// Data is now imported, let's do a check
// Check a password that is in the sample data
err = Check("tl1992rell", mem, nil)
fmt.Println(err)
// Output:password found in database
}
func TestImportBz2(t *testing.T) {
r, err := os.Open("rockyou.txt.bz2")
if err != nil {
t.Skip("Skipping bz2 file test. 'rockyou.txt.bz2' must be in current dir")
}
mem := testdb.NewMemDBBulk()
in := tokenizer.NewBz2Line(r)
err = Import(in, mem, nil)
if err != nil {
t.Fatal(err)
}
}
func TestImportBig(t *testing.T) {
r, err := os.Open("crackstation-human-only.txt.gz")
if err != nil {
t.Skip("Skipping big file test. 'crackstation-human-only.txt.gz' must be in current dir")
}
mem := testdb.NewMemDBBulk()
in, err := tokenizer.NewGzLine(r)
if err != nil {
t.Fatal(err)
}
err = Import(in, mem, nil)
if err != nil {
t.Fatal(err)
}
}
func TestInDBBulk(t *testing.T) {
buf, err := testdata.Asset("testdata.txt.gz")
if err != nil {
t.Fatal(err)
}
mem := testdb.NewMemDBBulk()
in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf))
if err != nil {
t.Fatal(err)
}
err = Import(in, mem, nil)
for p := range testdata.TestSet {
if SanitizeOK(p, nil) != nil {
continue
}
has, err := inDB(p, mem, nil)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatalf("db should have: %s", p)
}
err = Check(p, mem, nil)
if err != ErrPasswordInDB {
t.Fatal("check failed on:", p, err)
}
}
for p := range testdata.NotInSet {
if SanitizeOK(p, nil) != nil {
continue
}
has, err := inDB(p, mem, nil)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatalf("db should not have: %s", p)
}
err = Check(p, mem, nil)
if err != nil {
t.Fatal("check failed on:", p, err)
}
}
}
func TestImportBulk(t *testing.T) {
buf, err := testdata.Asset("testdata.txt.gz")
if err != nil {
t.Fatal(err)
}
mem := testdb.NewMemDBBulk()
in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf))
if err != nil {
t.Fatal(err)
}
err = Import(in, mem, nil)
if err != nil {
t.Fatal(err)
}
// Test everything is kept open.
err = Import(in, mem, nil)
if err != nil {
t.Fatal(err)
}
}
// Open a xz compressed archive and import it.
// Uses the "xi2.org/x/xz" package to read xz files.
func ExampleImport_xz() {
r, err := os.Open("rockyou.txt.xz")
if err != nil {
// Fake it
fmt.Println("Imported", 9341543, "items")
return
}
xzr, err := xz.NewReader(r, 0)
if err != nil {
panic(err)
}
mem := testdb.NewMemDBBulk()
in := tokenizer.NewLine(xzr)
err = Import(in, mem, nil)
if err != nil {
panic(err)
}
fmt.Println("Imported", len(*mem), "items")
// Output: Imported 9341543 items
}