func TestPrependedBlocks(t *testing.T) {
	const BLOCKSIZE = 100
	const BLOCK_COUNT = 20
	checksum := NewFileChecksumGenerator(BLOCKSIZE)

	file1 := io.LimitReader(
		readers.NewNonRepeatingSequence(0),
		BLOCKSIZE*BLOCK_COUNT,
	)

	file2 := io.LimitReader(
		io.MultiReader(
			readers.OneReader(BLOCKSIZE), // Off by one block
			readers.NewNonRepeatingSequence(0),
		),
		BLOCKSIZE*BLOCK_COUNT,
	)

	output1 := bytes.NewBuffer(nil)
	chksum1, _ := checksum.GenerateChecksums(file1, output1)

	output2 := bytes.NewBuffer(nil)
	chksum2, _ := checksum.GenerateChecksums(file2, output2)

	if bytes.Compare(chksum1, chksum2) == 0 {
		t.Fatal("Checksums should be different")
	}

	weaksize, strongSize := checksum.GetChecksumSizes()
	sums1, _ := chunks.LoadChecksumsFromReader(output1, weaksize, strongSize)
	sums2, _ := chunks.LoadChecksumsFromReader(output2, weaksize, strongSize)

	if len(sums1) != len(sums2) {
		t.Fatalf("Checksum lengths differ %v vs %v", len(sums1), len(sums2))
	}

	if sums1[0].Match(sums2[0]) {
		t.Error("Chunk sums1[0] should differ from sums2[0]")
	}

	for i, _ := range sums2 {
		if i == 0 {
			continue
		}

		if !sums1[i-1].Match(sums2[i]) {
			t.Errorf("Chunk sums1[%v] equal sums2[%v]", i-1, i)
		}

	}
}
func ExampleFileChecksumGenerator_LoadChecksumsFromReader() {
	const BLOCKSIZE = 8096
	checksum := NewFileChecksumGenerator(BLOCKSIZE)

	// This could be any source that conforms to io.Reader
	// sections of a file, or the body of an http response
	file1, err := os.Open("fileChecksums.chk")

	if err != nil {
		return
	}

	defer file1.Close()

	ws, ss := checksum.GetChecksumSizes()
	checksums, err := chunks.LoadChecksumsFromReader(file1, ws, ss)

	if err != nil {
		return
	}

	// Make an index that we can use against our local
	// checksums
	i := index.MakeChecksumIndex(checksums)

	// example checksum from a local file
	// look for the chunk in the index
	i.FindWeakChecksumInIndex([]byte("a"))

}
Beispiel #3
0
// Generates an index from a reader
// This is mostly a utility function to avoid being overly verbose in tests that need
// an index to work, but don't want to construct one by hand in order to avoid the dependencies
// obviously this means that those tests are likely to fail if there are issues with any of the other
// modules, which is not ideal.
// TODO: move to util?
func BuildChecksumIndex(check *filechecksum.FileChecksumGenerator, r io.Reader) (
	fcheck []byte,
	i *index.ChecksumIndex,
	lookup filechecksum.ChecksumLookup,
	err error,
) {
	b := bytes.NewBuffer(nil)
	fcheck, err = check.GenerateChecksums(r, b)

	if err != nil {
		return
	}

	weakSize := check.WeakRollingHash.Size()
	strongSize := check.GetStrongHash().Size()
	readChunks, err := chunks.LoadChecksumsFromReader(b, weakSize, strongSize)

	if err != nil {
		return
	}

	i = index.MakeChecksumIndex(readChunks)
	lookup = chunks.StrongChecksumGetter(readChunks)

	return
}
Beispiel #4
0
func readIndex(r io.Reader, blocksize uint) (
	i *index.ChecksumIndex,
	checksumLookup filechecksum.ChecksumLookup,
	blockCount uint,
	err error,
) {
	generator := filechecksum.NewFileChecksumGenerator(blocksize)

	readChunks, e := chunks.LoadChecksumsFromReader(
		r,
		generator.WeakRollingHash.Size(),
		generator.StrongHash.Size(),
	)

	err = e

	if err != nil {
		return
	}

	checksumLookup = chunks.StrongChecksumGetter(readChunks)
	i = index.MakeChecksumIndex(readChunks)
	blockCount = uint(len(readChunks))

	return
}
// Each of the data blocks is the same, so the checksums for the blocks should be the same
func TestChecksumBlocksTheSame(t *testing.T) {
	const BLOCKSIZE = 100
	const BLOCK_COUNT = 20

	checksum := NewFileChecksumGenerator(BLOCKSIZE)
	output := bytes.NewBuffer(nil)

	_, err := checksum.GenerateChecksums(
		readers.OneReader(BLOCKSIZE*BLOCK_COUNT),
		output,
	)

	if err != nil {
		t.Fatal(err)
	}

	weakSize, strongSize := checksum.GetChecksumSizes()

	if output.Len() != BLOCK_COUNT*(strongSize+weakSize) {
		t.Errorf(
			"Unexpected output length: %v, expected %v",
			output.Len(),
			BLOCK_COUNT*(strongSize+weakSize),
		)
	}

	results, err := chunks.LoadChecksumsFromReader(output, weakSize, strongSize)

	if err != nil {
		t.Fatal(err)
	}

	if len(results) != BLOCK_COUNT {
		t.Fatalf("Results too short! %v", len(results))
	}

	first := results[0]

	for i, chk := range results {
		if chk.ChunkOffset != uint(i) {
			t.Errorf("Unexpected offset %v on chunk %v", chk.ChunkOffset, i)
		}
		if !first.Match(chk) {
			t.Fatalf("Chunks have different checksums on %v", i)
		}
	}
}
// Test that partial content that ends in the middle of a strong
// hash is caught correctly
func TestInvalidReaderLength_Strong(t *testing.T) {
	const BLOCKSIZE = 100

	checksum := NewFileChecksumGenerator(BLOCKSIZE)

	file1 := io.LimitReader(
		readers.NewNonRepeatingSequence(0),
		int64(checksum.ChecksumSize())+int64(checksum.WeakRollingHash.Size())+2,
	)

	ws, ss := checksum.GetChecksumSizes()
	r, err := chunks.LoadChecksumsFromReader(file1, ws, ss)

	if r != nil || err != chunks.ErrPartialChecksum {
		t.Error("Expected partial checksum error")
	}
}