// TestTCPDownloadFile tests the NewTCPServer and DownloadFile functions.
// NewTCPServer must properly initialize a TCP server.
// UploadFile splits a file into erasure-coded segments and distributes them across a quorum.
// k is the number of non-redundant segments.
// The file is padded to satisfy the erasure-coding requirements that:
// len(fileData) = k*bytesPerSegment, and:
// bytesPerSegment % 64 = 0
func UploadFile(mr common.MessageRouter, file *os.File, k int, quorum [common.QuorumSize]common.Address) (bytesPerSegment int, err error) {
// read file
fileInfo, err := file.Stat()
if err != nil {
return
}
if fileInfo.Size() > int64(common.QuorumSize*common.MaxSegmentSize) {
err = fmt.Errorf("File exceeds maximum per-quorum size")
return
}
fileData := make([]byte, fileInfo.Size())
_, err = io.ReadFull(file, fileData)
if err != nil {
return
}
// calculate EncodeRing parameters, padding file if necessary
bytesPerSegment = len(fileData) / k
if bytesPerSegment%64 != 0 {
bytesPerSegment += 64 - (bytesPerSegment % 64)
padding := k*bytesPerSegment - len(fileData)
fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...)
}
// create erasure-coded segments
segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData)
if err != nil {
return
}
// for now we just send segment i to node i
// this may need to be randomized for security
for i := range quorum {
m := new(common.Message)
m.Destination = quorum[i]
m.Payload = append([]byte{byte(i)}, []byte(segments[i])...)
err = mr.SendMessage(m)
if err != nil {
return
}
}
return
}
// uploadSector splits a Sector into a Ring and distributes it across a quorum.
// It hashes each of the Ring's segments and stores the hashes in the SectorDB.
func uploadSector(sec *common.Sector) (err error) {
// look up Sector in SectorDB
rh := SectorDB[sec.Hash]
if rh == nil {
return fmt.Errorf("Sector not found in database")
}
// create ring
ring, err := erasure.EncodeRing(sec, rh.Params)
if err != nil {
return
}
// calculate and store segment hashes
for i := range ring {
rh.SegHashes[i], err = crypto.CalculateHash(ring[i].Data)
if err != nil {
return
}
}
// for now we just send segment i to host i
// this may need to be randomized for security
for i := range rh.Hosts {
err = router.SendMessage(&common.Message{
Dest: rh.Hosts[i],
Proc: "Server.UploadSegment",
Args: ring[i],
Resp: nil,
})
if err != nil {
return
}
}
return
}
func CreateFileAnnounce(file string) *Announce {
A := new(Announce)
f, err := os.Open(file)
if err != nil {
log.Fatal(err)
}
defer f.Close()
c, err := f.Stat()
if err != nil {
}
A.Size = int(c.Size())
//this should be enough.
A.Chunks = A.Size / (1 << 20)
//Chunk File
d := make([]byte, A.Size/(1<<20))
indexs := 0
//generate ID and dump the chunks into files
//A.ID=""
for _ = 0; err != io.EOF; _, err = f.Read(d) {
if err != nil {
}
redunt, _ := erasure.EncodeRing(A.Chunks, A.Size/A.Chunks, d)
A.Chunks = len(redunt)
indexs = indexs + 1
for index, val := range redunt {
k, e := os.Open(fmt.Sprintf("%s-%d%d", A.ID, indexs, index))
if e != nil {
log.Fatal(e)
}
if _, e = k.Write([]byte(val)); e != nil {
log.Fatal(e)
}
}
}
return A
}
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions.
// NewRPCServer must properly initialize a RPC server.
// downloadSector must successfully retrieve a Sector from a quorum.
// The downloaded Sector must match the original Sector.
func TestRPCdownloadSector(t *testing.T) {
SectorDB = make(map[crypto.Hash]*common.RingHeader)
// create sector
secData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
sec, err := common.NewSector(secData)
if err != nil {
t.Fatal("Failed to create sector:", err)
}
k := common.QuorumSize / 2
params := sec.CalculateParams(k)
// encode sector
ring, err := erasure.EncodeRing(sec, params)
if err != nil {
t.Fatal("Failed to encode sector data:", err)
}
// create RPCServer
router, err = network.NewRPCServer(9985)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
defer router.Close()
// create quorum
var q [common.QuorumSize]common.Address
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qrpc, err := network.NewRPCServer(9000 + i)
if err != nil {
t.Fatal("Failed to initialize RPCServer:", err)
}
sh := new(Server)
sh.seg = ring[i]
q[i].ID = qrpc.RegisterHandler(sh)
}
// add sector to database
SectorDB[sec.Hash] = &common.RingHeader{
Hosts: q,
Params: params,
}
// download file from quorum
sec, err = downloadSector(sec.Hash)
if err != nil {
t.Fatal("Failed to download file:", err)
}
// check hash
rebuiltHash, err := crypto.CalculateHash(sec.Data)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if sec.Hash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}
// NewTCPServer must properly initialize a TCP server.
// DownloadFile must successfully retrieve a file from a quorum.
// The downloaded file must match the original file.
func TestTCPDownloadFile(t *testing.T) {
t.Skip()
// create file
fileData, err := crypto.RandomByteSlice(70000)
if err != nil {
t.Fatal("Could not generate test data:", err)
}
// calculate hash
origHash, err := crypto.CalculateHash(fileData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
// encode file
k := 50
bytesPerSegment := len(fileData) / k
if bytesPerSegment%64 != 0 {
bytesPerSegment += 64 - (bytesPerSegment % 64)
padding := k*bytesPerSegment - len(fileData)
fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...)
}
segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData)
if err != nil {
t.Fatal("Failed to encode file data:", err)
}
// create TCPServer
tcp, err := network.NewTCPServer(9988)
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
defer tcp.Close()
tdh := new(downloadHandler)
tdh.segments = make([]string, k)
tdh.indices = make([]uint8, k)
tdh.k, tdh.b = k, bytesPerSegment
tcp.AddMessageHandler(tdh)
// create quorum
var q [common.QuorumSize]common.Address
var tfhs [common.QuorumSize]TestFileHandler
for i := 0; i < common.QuorumSize; i++ {
q[i] = common.Address{0, "localhost", 9000 + i}
qtcp, err := network.NewTCPServer(9000 + i)
if err != nil {
t.Fatal("Failed to initialize TCPServer:", err)
}
tfhs[i].tcpServ = qtcp
tfhs[i].dest = tcp.Address()
tfhs[i].data = segments[i]
tfhs[i].done = make(chan bool, 1)
q[i].Id = qtcp.AddMessageHandler(&tfhs[i]).Id
}
// download file from quorum
downData, err := DownloadFile(tcp, origHash, len(fileData), k, q)
if err != nil {
t.Fatal("Failed to download file:", err)
}
// wait for download to complete
<-tdh.done
// check hash
rebuiltHash, err := crypto.CalculateHash(downData)
if err != nil {
t.Fatal("Failed to calculate hash:", err)
}
if origHash != rebuiltHash {
t.Fatal("Failed to recover file: hashes do not match")
}
}