Пример #1
0
// RebuildSector takes a Ring and returns a Sector containing the original data.
// The encoding parameters are stored in params.
// k must be equal to the number of non-redundant segments when the file was originally built.
// Because recovery is just a bunch of matrix operations, there is no way to tell if the data has been corrupted
// or if an incorrect value of k has been chosen. This error checking must happen before calling RebuildSector.
// Each Segment's Data must have the correct Index from when it was encoded.
func RebuildSector(ring []common.Segment, params *common.EncodingParams) (sec *common.Sector, err error) {
	k, b, length := params.GetValues()
	if k == 0 && b == 0 {
		err = fmt.Errorf("could not rebuild using uninitialized encoding parameters")
		return
	}

	// check for legal size of k
	if k > common.QuorumSize || k < 1 {
		err = fmt.Errorf("k must be greater than 0 but smaller than %v", common.QuorumSize)
		return
	}

	// check for legal size of b
	if b < common.MinSegmentSize || b > common.MaxSegmentSize {
		err = fmt.Errorf("b must be greater than %v and smaller than %v", common.MinSegmentSize, common.MaxSegmentSize)
		return
	}

	// check for legal size of length
	if length > common.MaxSegmentSize*common.QuorumSize {
		err = fmt.Errorf("length must be smaller than %v", common.MaxSegmentSize*common.QuorumSize)
	}

	// check for correct number of segments
	if len(ring) < k {
		err = fmt.Errorf("insufficient segments: expected at least %v, got %v", k, len(ring))
		return
	}

	// move all data into a single slice
	var segmentData []byte
	var segmentIndices []uint8
	for i := 0; i < k; i++ {
		// verify that each segment is the correct length
		// TODO: skip bad segments and continue rebuilding if possible
		if len(ring[i].Data) != b {
			err = fmt.Errorf("at least 1 Segment's Data field is the wrong length")
			return
		}

		segmentData = append(segmentData, ring[i].Data...)
		segmentIndices = append(segmentIndices, ring[i].Index)

	}
	// call the recovery function
	C.recoverData(C.int(k), C.int(common.QuorumSize-k), C.int(b), (*C.uchar)(unsafe.Pointer(&segmentData[0])), (*C.uchar)(unsafe.Pointer(&segmentIndices[0])))

	// remove padding introduced by EncodeRing()
	sec, err = common.NewSector(segmentData[:length])
	return
}
Пример #2
0
// Basic test for reed-solomon coding, verifies that standard input
// will produce the correct results.
func TestCoding(t *testing.T) {
	// set encoding parameters
	k := common.QuorumSize / 2
	m := common.QuorumSize - k
	b := 1024

	// create sector data
	randomBytes, err := crypto.RandomByteSlice(b * k)
	if err != nil {
		t.Fatal(err)
	}

	// create sector
	sec, err := common.NewSector(randomBytes)
	if err != nil {
		t.Fatal(err)
	}

	// calculate encoding parameters
	params := sec.CalculateParams(k)

	// encode data into a Ring
	ring, err := EncodeRing(sec, params)
	if err != nil {
		t.Fatal(err)
	}

	// create Ring from subset of encoded segments
	var newRing []common.Segment
	for i := m; i < common.QuorumSize; i++ {
		newRing = append(newRing, ring[i])
	}

	// recover original data
	newSec, err := RebuildSector(newRing, params)
	if err != nil {
		t.Fatal(err)
	}

	// compare to hash of data when first generated
	recoveredDataHash, err := crypto.CalculateHash(newSec.Data)
	if err != nil {
		t.Fatal(err)
	} else if recoveredDataHash != sec.Hash {
		t.Fatal("recovered data is different from original data")
	}

	// In every test, we check that the hashes equal
	// every other hash that gets created. This makes
	// me uneasy.
}
Пример #3
0
func generateSector(q common.Quorum) (s *common.Sector, err error) {
	if q[0].Port == 0 {
		err = fmt.Errorf("you must connect to a quorum first")
		return
	}
	data, err := crypto.RandomByteSlice(70000)
	if err != nil {
		return
	}
	s, err = common.NewSector(data)
	if err != nil {
		return
	}
	SectorDB[s.Hash] = &common.RingHeader{
		Hosts:  q,
		Params: s.CalculateParams(common.QuorumSize / 2),
	}
	return
}
Пример #4
0
// TestRPCUploadSector tests the NewRPCServer and uploadFile functions.
// NewRPCServer must properly initialize a RPC server.
// uploadSector must succesfully distribute a Sector among a quorum.
// The uploaded Sector must be successfully reconstructed.
func TestRPCuploadSector(t *testing.T) {
	SectorDB = make(map[crypto.Hash]*common.RingHeader)

	// create RPCServer
	var err error
	router, err = network.NewRPCServer(9985)
	if err != nil {
		t.Fatal("Failed to initialize RPCServer:", err)
	}
	defer router.Close()

	// create quorum
	var q [common.QuorumSize]common.Address
	var shs [common.QuorumSize]Server
	for i := 0; i < common.QuorumSize; i++ {
		q[i] = common.Address{0, "localhost", 9000 + i}
		qrpc, err := network.NewRPCServer(9000 + i)
		defer qrpc.Close()
		if err != nil {
			t.Fatal("Failed to initialize RPCServer:", err)
		}
		q[i].ID = qrpc.RegisterHandler(&shs[i])
	}

	// create sector
	secData, err := crypto.RandomByteSlice(70000)
	if err != nil {
		t.Fatal("Could not generate test data:", err)
	}

	sec, err := common.NewSector(secData)
	if err != nil {
		t.Fatal("Failed to create sector:", err)
	}

	// add sector to database
	k := common.QuorumSize / 2
	SectorDB[sec.Hash] = &common.RingHeader{
		Hosts:  q,
		Params: sec.CalculateParams(k),
	}

	// upload sector to quorum
	err = uploadSector(sec)
	if err != nil {
		t.Fatal("Failed to upload file:", err)
	}

	// rebuild file from first k segments
	var newRing []common.Segment
	for i := 0; i < k; i++ {
		newRing = append(newRing, shs[i].seg)
	}

	sec, err = erasure.RebuildSector(newRing, SectorDB[sec.Hash].Params)
	if err != nil {
		t.Fatal("Failed to rebuild file:", err)
	}

	// check hash
	rebuiltHash, err := crypto.CalculateHash(sec.Data)
	if err != nil {
		t.Fatal("Failed to calculate hash:", err)
	}

	if sec.Hash != rebuiltHash {
		t.Fatal("Failed to recover file: hashes do not match")
	}
}
Пример #5
0
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions.
// NewRPCServer must properly initialize a RPC server.
// downloadSector must successfully retrieve a Sector from a quorum.
// The downloaded Sector must match the original Sector.
func TestRPCdownloadSector(t *testing.T) {
	SectorDB = make(map[crypto.Hash]*common.RingHeader)

	// create sector
	secData, err := crypto.RandomByteSlice(70000)
	if err != nil {
		t.Fatal("Could not generate test data:", err)
	}

	sec, err := common.NewSector(secData)
	if err != nil {
		t.Fatal("Failed to create sector:", err)
	}

	k := common.QuorumSize / 2
	params := sec.CalculateParams(k)

	// encode sector
	ring, err := erasure.EncodeRing(sec, params)
	if err != nil {
		t.Fatal("Failed to encode sector data:", err)
	}

	// create RPCServer
	router, err = network.NewRPCServer(9985)
	if err != nil {
		t.Fatal("Failed to initialize RPCServer:", err)
	}
	defer router.Close()

	// create quorum
	var q [common.QuorumSize]common.Address
	for i := 0; i < common.QuorumSize; i++ {
		q[i] = common.Address{0, "localhost", 9000 + i}
		qrpc, err := network.NewRPCServer(9000 + i)
		if err != nil {
			t.Fatal("Failed to initialize RPCServer:", err)
		}
		sh := new(Server)
		sh.seg = ring[i]
		q[i].ID = qrpc.RegisterHandler(sh)
	}

	// add sector to database
	SectorDB[sec.Hash] = &common.RingHeader{
		Hosts:  q,
		Params: params,
	}

	// download file from quorum
	sec, err = downloadSector(sec.Hash)
	if err != nil {
		t.Fatal("Failed to download file:", err)
	}

	// check hash
	rebuiltHash, err := crypto.CalculateHash(sec.Data)
	if err != nil {
		t.Fatal("Failed to calculate hash:", err)
	}

	if sec.Hash != rebuiltHash {
		t.Fatal("Failed to recover file: hashes do not match")
	}
}