// Basic test for reed-solomon coding, verifies that standard input
// will produce the correct results.
func TestCoding(t *testing.T) {
k := 100
m := common.QuorumSize - k
bytesPerSegment := 1024
randomBytes, err := crypto.RandomByteSlice(bytesPerSegment * k)
if err != nil {
t.Fatal(err)
}
// get hash of original file
randomBytesHash, err := crypto.CalculateHash(randomBytes)
if err != nil {
t.Fatal(err)
}
// encode original file into a data ring
ringSegments, err := EncodeRing(k, bytesPerSegment, randomBytes)
if err != nil {
t.Fatal(err)
}
// verify that first k segments are still original data
originalDataHash, err := crypto.CalculateHash(randomBytes)
if err != nil {
t.Fatal(err)
} else if originalDataHash != randomBytesHash {
t.Fatal("original data was modified after caling EncodeRing!")
}
// reduce file to a set of k segments and print those segments out
remainingSegments := make([]string, k)
segmentIndicies := make([]uint8, k)
for i := m; i < common.QuorumSize; i++ {
remainingSegments[i-m] = ringSegments[i]
segmentIndicies[i-m] = uint8(i)
}
// recover original data
recoveredData, err := RebuildSector(k, bytesPerSegment, remainingSegments, segmentIndicies)
if err != nil {
t.Fatal(err)
}
// compare to hash of data when first generated
recoveredDataHash, err := crypto.CalculateHash(recoveredData)
if err != nil {
t.Fatal(err)
} else if recoveredDataHash != randomBytesHash {
t.Fatal("recovered data is different from original data")
}
// In every test, we check that the hashes equal
// every other hash that gets created. This makes
// me uneasy.
}
// Basic test for reed-solomon coding, verifies that standard input
// will produce the correct results.
func TestCoding(t *testing.T) {
// set encoding parameters
k := common.QuorumSize / 2
m := common.QuorumSize - k
b := 1024
// create sector data
randomBytes, err := crypto.RandomByteSlice(b * k)
if err != nil {
t.Fatal(err)
}
// create sector
sec, err := common.NewSector(randomBytes)
if err != nil {
t.Fatal(err)
}
// calculate encoding parameters
params := sec.CalculateParams(k)
// encode data into a Ring
ring, err := EncodeRing(sec, params)
if err != nil {
t.Fatal(err)
}
// create Ring from subset of encoded segments
var newRing []common.Segment
for i := m; i < common.QuorumSize; i++ {
newRing = append(newRing, ring[i])
}
// recover original data
newSec, err := RebuildSector(newRing, params)
if err != nil {
t.Fatal(err)
}
// compare to hash of data when first generated
recoveredDataHash, err := crypto.CalculateHash(newSec.Data)
if err != nil {
t.Fatal(err)
} else if recoveredDataHash != sec.Hash {
t.Fatal("recovered data is different from original data")
}
// In every test, we check that the hashes equal
// every other hash that gets created. This makes
// me uneasy.
}
// uploadSector splits a Sector into a Ring and distributes it across a quorum.
// It hashes each of the Ring's segments and stores the hashes in the SectorDB.
func uploadSector(sec *common.Sector) (err error) {
// look up Sector in SectorDB
rh := SectorDB[sec.Hash]
if rh == nil {
return fmt.Errorf("Sector not found in database")
}
// create ring
ring, err := erasure.EncodeRing(sec, rh.Params)
if err != nil {
return
}
// calculate and store segment hashes
for i := range ring {
rh.SegHashes[i], err = crypto.CalculateHash(ring[i].Data)
if err != nil {
return
}
}
// for now we just send segment i to host i
// this may need to be randomized for security
for i := range rh.Hosts {
err = router.SendMessage(&common.Message{
Dest: rh.Hosts[i],
Proc: "Server.UploadSegment",
Args: ring[i],
Resp: nil,
})
if err != nil {
return
}
}
return
}
// TestRPCUploadSector tests the NewRPCServer and uploadFile functions.
// NewRPCServer must properly initialize a RPC server.
// uploadSector must succesfully distribute a Sector among a quorum.
// The uploaded Sector must be successfully reconstructed.
func TestRPCuploadSector(t *testing.T) {
SectorDB = make(map[crypto.Hash]*common.RingHeader)
// create RPCServer
var err error
router, err = network.NewRPCServer(9985)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
defer router.Close()
// create quorum
var q [common.QuorumSize]common.Address
var shs [common.QuorumSize]Server
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qrpc, err := network.NewRPCServer(9000 + i)
defer qrpc.Close()
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
q[i].ID = qrpc.RegisterHandler(&shs[i])
}
// create sector
secData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
sec, err := common.NewSector(secData)
if err != nil {
t.Fatal("Failed to create sector:", err)
}
// add sector to database
k := common.QuorumSize / 2
SectorDB[sec.Hash] = &common.RingHeader{
Hosts: q,
Params: sec.CalculateParams(k),
}
// upload sector to quorum
err = uploadSector(sec)
if err != nil {
t.Fatal("Failed to upload file:", err)
}
// rebuild file from first k segments
var newRing []common.Segment
for i := 0; i < k; i++ {
newRing = append(newRing, shs[i].seg)
}
sec, err = erasure.RebuildSector(newRing, SectorDB[sec.Hash].Params)
if err != nil {
t.Fatal("Failed to rebuild file:", err)
}
// check hash
rebuiltHash, err := crypto.CalculateHash(sec.Data)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if sec.Hash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions.
// NewRPCServer must properly initialize a RPC server.
// downloadSector must successfully retrieve a Sector from a quorum.
// The downloaded Sector must match the original Sector.
func TestRPCdownloadSector(t *testing.T) {
SectorDB = make(map[crypto.Hash]*common.RingHeader)
// create sector
secData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
sec, err := common.NewSector(secData)
if err != nil {
t.Fatal("Failed to create sector:", err)
}
k := common.QuorumSize / 2
params := sec.CalculateParams(k)
// encode sector
ring, err := erasure.EncodeRing(sec, params)
if err != nil {
t.Fatal("Failed to encode sector data:", err)
}
// create RPCServer
router, err = network.NewRPCServer(9985)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
defer router.Close()
// create quorum
var q [common.QuorumSize]common.Address
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qrpc, err := network.NewRPCServer(9000 + i)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
sh := new(Server)
sh.seg = ring[i]
q[i].ID = qrpc.RegisterHandler(sh)
}
// add sector to database
SectorDB[sec.Hash] = &common.RingHeader{
Hosts: q,
Params: params,
}
// download file from quorum
sec, err = downloadSector(sec.Hash)
if err != nil {
t.Fatal("Failed to download file:", err)
}
// check hash
rebuiltHash, err := crypto.CalculateHash(sec.Data)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if sec.Hash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}
func main() {
router, _ = network.NewRPCServer(9989)
defer router.Close()
SectorDB = make(map[crypto.Hash]*common.RingHeader)
var (
input string
q common.Quorum
s *common.Sector
h crypto.Hash
err error
)
for {
fmt.Print("Please enter a command: ")
fmt.Scanln(&input)
switch input {
default:
fmt.Println("unrecognized command")
case "j":
fmt.Println("joining quorum")
q = readQuorumAddresses()
fmt.Println("connected to quorum")
case "g":
fmt.Println("generating Sector")
s, err = generateSector(q)
if err != nil {
fmt.Println("error:", err)
fmt.Println("failed to generate Sector")
break
}
h = s.Hash
fmt.Println("created Sector with hash", h[:10])
case "u":
fmt.Println("uploading file")
err = uploadSector(s)
if err != nil {
fmt.Println("error:", err)
fmt.Println("upload failed")
break
}
fmt.Println("upload successful")
case "d":
fmt.Println("downloading file")
rs, err := downloadSector(h)
if err != nil {
fmt.Println("error:", err)
fmt.Println("download failed")
break
}
rh, err := crypto.CalculateHash(rs.Data)
if err != nil {
fmt.Println("error:", err)
break
}
fmt.Println("download successful")
fmt.Println("hash:", rh[:10])
case "q":
return
}
}
}
// TestTCPUploadFile tests the NewTCPServer and UploadFile functions.
// NewTCPServer must properly initialize a TCP server.
// UploadFile must succesfully distribute a file among a quorum.
// The uploaded file must be successfully reconstructed.
func TestTCPUploadFile(t *testing.T) {
// create TCPServer
tcp, err := network.NewTCPServer(9988)
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
defer tcp.Close()
// create quorum
var q [common.QuorumSize]common.Address
var uhs [common.QuorumSize]uploadHandler
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qtcp, err := network.NewTCPServer(9000 + i)
defer qtcp.Close()
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
uhs[i].done = make(chan bool, 1)
q[i].Id = qtcp.AddMessageHandler(&uhs[i]).Id
}
// create file
file, err := os.Create("InputFile")
if err != nil {
t.Fatal("Failed to create file \"InputFile\"")
}
defer file.Close()
defer os.Remove("InputFile")
fileData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
err = ioutil.WriteFile("InputFile", fileData, 0644)
if err != nil {
t.Fatal("Failed to write to file InputFile:", err)
}
// calculate hash
origHash, err := crypto.CalculateHash(fileData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
// upload file to quorum
k := 50
b, err := UploadFile(tcp, file, k, q)
if err != nil {
t.Fatal("Failed to upload file:", err)
}
// wait for all participants to complete
for i := range uhs {
<-uhs[i].done
}
// rebuild file from first k segments
segments := make([]string, k)
indices := make([]uint8, k)
for i := 0; i < k; i++ {
segments[i] = string(uhs[i].data)
indices[i] = uint8(uhs[i].index)
}
rebuiltData, err := erasure.RebuildSector(k, b, segments, indices)
if err != nil {
t.Fatal("Failed to rebuild file:", err)
}
// remove padding
rebuiltData = rebuiltData[:len(fileData)]
// check hash
rebuiltHash, err := crypto.CalculateHash(rebuiltData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if origHash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}
// NewTCPServer must properly initialize a TCP server.
// DownloadFile must successfully retrieve a file from a quorum.
// The downloaded file must match the original file.
func TestTCPDownloadFile(t *testing.T) {
t.Skip()
// create file
fileData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
// calculate hash
origHash, err := crypto.CalculateHash(fileData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
// encode file
k := 50
bytesPerSegment := len(fileData) / k
if bytesPerSegment%64 != 0 {
bytesPerSegment += 64 - (bytesPerSegment % 64)
padding := k*bytesPerSegment - len(fileData)
fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...)
}
segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData)
if err != nil {
t.Fatal("Failed to encode file data:", err)
}
// create TCPServer
tcp, err := network.NewTCPServer(9988)
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
defer tcp.Close()
tdh := new(downloadHandler)
tdh.segments = make([]string, k)
tdh.indices = make([]uint8, k)
tdh.k, tdh.b = k, bytesPerSegment
tcp.AddMessageHandler(tdh)
// create quorum
var q [common.QuorumSize]common.Address
var tfhs [common.QuorumSize]TestFileHandler
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qtcp, err := network.NewTCPServer(9000 + i)
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
tfhs[i].tcpServ = qtcp
tfhs[i].dest = tcp.Address()
tfhs[i].data = segments[i]
tfhs[i].done = make(chan bool, 1)
q[i].Id = qtcp.AddMessageHandler(&tfhs[i]).Id
}
// download file from quorum
downData, err := DownloadFile(tcp, origHash, len(fileData), k, q)
if err != nil {
t.Fatal("Failed to download file:", err)
}
// wait for download to complete
<-tdh.done
// check hash
rebuiltHash, err := crypto.CalculateHash(downData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if origHash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}