// RebuildSector takes a Ring and returns a Sector containing the original data.
// The encoding parameters are stored in params.
// k must be equal to the number of non-redundant segments when the file was originally built.
// Because recovery is just a bunch of matrix operations, there is no way to tell if the data has been corrupted
// or if an incorrect value of k has been chosen. This error checking must happen before calling RebuildSector.
// Each Segment's Data must have the correct Index from when it was encoded.
func RebuildSector(ring []common.Segment, params *common.EncodingParams) (sec *common.Sector, err error) {
k, b, length := params.GetValues()
if k == 0 && b == 0 {
err = fmt.Errorf("could not rebuild using uninitialized encoding parameters")
return
}
// check for legal size of k
if k > common.QuorumSize || k < 1 {
err = fmt.Errorf("k must be greater than 0 but smaller than %v", common.QuorumSize)
return
}
// check for legal size of b
if b < common.MinSegmentSize || b > common.MaxSegmentSize {
err = fmt.Errorf("b must be greater than %v and smaller than %v", common.MinSegmentSize, common.MaxSegmentSize)
return
}
// check for legal size of length
if length > common.MaxSegmentSize*common.QuorumSize {
err = fmt.Errorf("length must be smaller than %v", common.MaxSegmentSize*common.QuorumSize)
}
// check for correct number of segments
if len(ring) < k {
err = fmt.Errorf("insufficient segments: expected at least %v, got %v", k, len(ring))
return
}
// move all data into a single slice
var segmentData []byte
var segmentIndices []uint8
for i := 0; i < k; i++ {
// verify that each segment is the correct length
// TODO: skip bad segments and continue rebuilding if possible
if len(ring[i].Data) != b {
err = fmt.Errorf("at least 1 Segment's Data field is the wrong length")
return
}
segmentData = append(segmentData, ring[i].Data...)
segmentIndices = append(segmentIndices, ring[i].Index)
}
// call the recovery function
C.recoverData(C.int(k), C.int(common.QuorumSize-k), C.int(b), (*C.uchar)(unsafe.Pointer(&segmentData[0])), (*C.uchar)(unsafe.Pointer(&segmentIndices[0])))
// remove padding introduced by EncodeRing()
sec, err = common.NewSector(segmentData[:length])
return
}
// Basic test for reed-solomon coding, verifies that standard input
// will produce the correct results.
func TestCoding(t *testing.T) {
// set encoding parameters
k := common.QuorumSize / 2
m := common.QuorumSize - k
b := 1024
// create sector data
randomBytes, err := crypto.RandomByteSlice(b * k)
if err != nil {
t.Fatal(err)
}
// create sector
sec, err := common.NewSector(randomBytes)
if err != nil {
t.Fatal(err)
}
// calculate encoding parameters
params := sec.CalculateParams(k)
// encode data into a Ring
ring, err := EncodeRing(sec, params)
if err != nil {
t.Fatal(err)
}
// create Ring from subset of encoded segments
var newRing []common.Segment
for i := m; i < common.QuorumSize; i++ {
newRing = append(newRing, ring[i])
}
// recover original data
newSec, err := RebuildSector(newRing, params)
if err != nil {
t.Fatal(err)
}
// compare to hash of data when first generated
recoveredDataHash, err := crypto.CalculateHash(newSec.Data)
if err != nil {
t.Fatal(err)
} else if recoveredDataHash != sec.Hash {
t.Fatal("recovered data is different from original data")
}
// In every test, we check that the hashes equal
// every other hash that gets created. This makes
// me uneasy.
}
func generateSector(q common.Quorum) (s *common.Sector, err error) {
if q[0].Port == 0 {
err = fmt.Errorf("you must connect to a quorum first")
return
}
data, err := crypto.RandomByteSlice(70000)
if err != nil {
return
}
s, err = common.NewSector(data)
if err != nil {
return
}
SectorDB[s.Hash] = &common.RingHeader{
Hosts: q,
Params: s.CalculateParams(common.QuorumSize / 2),
}
return
}
// TestRPCUploadSector tests the NewRPCServer and uploadFile functions.
// NewRPCServer must properly initialize a RPC server.
// uploadSector must succesfully distribute a Sector among a quorum.
// The uploaded Sector must be successfully reconstructed.
func TestRPCuploadSector(t *testing.T) {
SectorDB = make(map[crypto.Hash]*common.RingHeader)
// create RPCServer
var err error
router, err = network.NewRPCServer(9985)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
defer router.Close()
// create quorum
var q [common.QuorumSize]common.Address
var shs [common.QuorumSize]Server
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qrpc, err := network.NewRPCServer(9000 + i)
defer qrpc.Close()
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
q[i].ID = qrpc.RegisterHandler(&shs[i])
}
// create sector
secData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
sec, err := common.NewSector(secData)
if err != nil {
t.Fatal("Failed to create sector:", err)
}
// add sector to database
k := common.QuorumSize / 2
SectorDB[sec.Hash] = &common.RingHeader{
Hosts: q,
Params: sec.CalculateParams(k),
}
// upload sector to quorum
err = uploadSector(sec)
if err != nil {
t.Fatal("Failed to upload file:", err)
}
// rebuild file from first k segments
var newRing []common.Segment
for i := 0; i < k; i++ {
newRing = append(newRing, shs[i].seg)
}
sec, err = erasure.RebuildSector(newRing, SectorDB[sec.Hash].Params)
if err != nil {
t.Fatal("Failed to rebuild file:", err)
}
// check hash
rebuiltHash, err := crypto.CalculateHash(sec.Data)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if sec.Hash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions.
// NewRPCServer must properly initialize a RPC server.
// downloadSector must successfully retrieve a Sector from a quorum.
// The downloaded Sector must match the original Sector.
func TestRPCdownloadSector(t *testing.T) {
SectorDB = make(map[crypto.Hash]*common.RingHeader)
// create sector
secData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
sec, err := common.NewSector(secData)
if err != nil {
t.Fatal("Failed to create sector:", err)
}
k := common.QuorumSize / 2
params := sec.CalculateParams(k)
// encode sector
ring, err := erasure.EncodeRing(sec, params)
if err != nil {
t.Fatal("Failed to encode sector data:", err)
}
// create RPCServer
router, err = network.NewRPCServer(9985)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
defer router.Close()
// create quorum
var q [common.QuorumSize]common.Address
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qrpc, err := network.NewRPCServer(9000 + i)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
sh := new(Server)
sh.seg = ring[i]
q[i].ID = qrpc.RegisterHandler(sh)
}
// add sector to database
SectorDB[sec.Hash] = &common.RingHeader{
Hosts: q,
Params: params,
}
// download file from quorum
sec, err = downloadSector(sec.Hash)
if err != nil {
t.Fatal("Failed to download file:", err)
}
// check hash
rebuiltHash, err := crypto.CalculateHash(sec.Data)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if sec.Hash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}