Example #1
0
func ExampleImport() {
	r, err := os.Open("./testdata/testdata.txt.gz")
	if err != nil {
		panic("cannot open file")
	}
	// Create a database to write to
	mem := testdb.NewMemDBBulk()

	// The input is gzipped text file with
	// one input per line, so we choose a tokenizer
	// that matches.
	in, err := tokenizer.NewGzLine(r)
	if err != nil {
		panic(err)
	}
	// Import using the default sanitizer
	err = Import(in, mem, nil)
	if err != nil {
		panic(err)
	}
	// Data is now imported, let's do a check
	// Check a password that is in the sample data
	err = Check("tl1992rell", mem, nil)
	fmt.Println(err)
	// Output:password found in database
}
Example #2
0
func TestImportBz2(t *testing.T) {
	r, err := os.Open("rockyou.txt.bz2")
	if err != nil {
		t.Skip("Skipping bz2 file test. 'rockyou.txt.bz2' must be in current dir")
	}
	mem := testdb.NewMemDBBulk()
	in := tokenizer.NewBz2Line(r)
	err = Import(in, mem, nil)
	if err != nil {
		t.Fatal(err)
	}
}
Example #3
0
func TestImportBig(t *testing.T) {
	r, err := os.Open("crackstation-human-only.txt.gz")
	if err != nil {
		t.Skip("Skipping big file test. 'crackstation-human-only.txt.gz' must be in current dir")
	}
	mem := testdb.NewMemDBBulk()
	in, err := tokenizer.NewGzLine(r)
	if err != nil {
		t.Fatal(err)
	}
	err = Import(in, mem, nil)
	if err != nil {
		t.Fatal(err)
	}
}
Example #4
0
func TestInDBBulk(t *testing.T) {
	buf, err := testdata.Asset("testdata.txt.gz")
	if err != nil {
		t.Fatal(err)
	}
	mem := testdb.NewMemDBBulk()
	in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf))
	if err != nil {
		t.Fatal(err)
	}
	err = Import(in, mem, nil)
	for p := range testdata.TestSet {
		if SanitizeOK(p, nil) != nil {
			continue
		}
		has, err := inDB(p, mem, nil)
		if err != nil {
			t.Fatal(err)
		}
		if !has {
			t.Fatalf("db should have: %s", p)
		}
		err = Check(p, mem, nil)
		if err != ErrPasswordInDB {
			t.Fatal("check failed on:", p, err)
		}
	}
	for p := range testdata.NotInSet {
		if SanitizeOK(p, nil) != nil {
			continue
		}
		has, err := inDB(p, mem, nil)
		if err != nil {
			t.Fatal(err)
		}
		if has {
			t.Fatalf("db should not have: %s", p)
		}
		err = Check(p, mem, nil)
		if err != nil {
			t.Fatal("check failed on:", p, err)
		}
	}
}
Example #5
0
func TestImportBulk(t *testing.T) {
	buf, err := testdata.Asset("testdata.txt.gz")
	if err != nil {
		t.Fatal(err)
	}
	mem := testdb.NewMemDBBulk()
	in, err := tokenizer.NewGzLine(bytes.NewBuffer(buf))
	if err != nil {
		t.Fatal(err)
	}
	err = Import(in, mem, nil)
	if err != nil {
		t.Fatal(err)
	}
	// Test everything is kept open.
	err = Import(in, mem, nil)
	if err != nil {
		t.Fatal(err)
	}
}
Example #6
0
// Open a xz compressed archive and import it.
// Uses the "xi2.org/x/xz" package to read xz files.
func ExampleImport_xz() {
	r, err := os.Open("rockyou.txt.xz")
	if err != nil {
		// Fake it
		fmt.Println("Imported", 9341543, "items")
		return
	}
	xzr, err := xz.NewReader(r, 0)
	if err != nil {
		panic(err)
	}
	mem := testdb.NewMemDBBulk()
	in := tokenizer.NewLine(xzr)
	err = Import(in, mem, nil)
	if err != nil {
		panic(err)
	}
	fmt.Println("Imported", len(*mem), "items")
	// Output: Imported 9341543 items
}