// TestTCPDownloadFile tests the NewTCPServer and DownloadFile functions. // NewTCPServer must properly initialize a TCP server. // UploadFile splits a file into erasure-coded segments and distributes them across a quorum. // k is the number of non-redundant segments. // The file is padded to satisfy the erasure-coding requirements that: // len(fileData) = k*bytesPerSegment, and: // bytesPerSegment % 64 = 0 func UploadFile(mr common.MessageRouter, file *os.File, k int, quorum [common.QuorumSize]common.Address) (bytesPerSegment int, err error) { // read file fileInfo, err := file.Stat() if err != nil { return } if fileInfo.Size() > int64(common.QuorumSize*common.MaxSegmentSize) { err = fmt.Errorf("File exceeds maximum per-quorum size") return } fileData := make([]byte, fileInfo.Size()) _, err = io.ReadFull(file, fileData) if err != nil { return } // calculate EncodeRing parameters, padding file if necessary bytesPerSegment = len(fileData) / k if bytesPerSegment%64 != 0 { bytesPerSegment += 64 - (bytesPerSegment % 64) padding := k*bytesPerSegment - len(fileData) fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...) } // create erasure-coded segments segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData) if err != nil { return } // for now we just send segment i to node i // this may need to be randomized for security for i := range quorum { m := new(common.Message) m.Destination = quorum[i] m.Payload = append([]byte{byte(i)}, []byte(segments[i])...) err = mr.SendMessage(m) if err != nil { return } } return }
// uploadSector splits a Sector into a Ring and distributes it across a quorum. // It hashes each of the Ring's segments and stores the hashes in the SectorDB. func uploadSector(sec *common.Sector) (err error) { // look up Sector in SectorDB rh := SectorDB[sec.Hash] if rh == nil { return fmt.Errorf("Sector not found in database") } // create ring ring, err := erasure.EncodeRing(sec, rh.Params) if err != nil { return } // calculate and store segment hashes for i := range ring { rh.SegHashes[i], err = crypto.CalculateHash(ring[i].Data) if err != nil { return } } // for now we just send segment i to host i // this may need to be randomized for security for i := range rh.Hosts { err = router.SendMessage(&common.Message{ Dest: rh.Hosts[i], Proc: "Server.UploadSegment", Args: ring[i], Resp: nil, }) if err != nil { return } } return }
func CreateFileAnnounce(file string) *Announce { A := new(Announce) f, err := os.Open(file) if err != nil { log.Fatal(err) } defer f.Close() c, err := f.Stat() if err != nil { } A.Size = int(c.Size()) //this should be enough. A.Chunks = A.Size / (1 << 20) //Chunk File d := make([]byte, A.Size/(1<<20)) indexs := 0 //generate ID and dump the chunks into files //A.ID="" for _ = 0; err != io.EOF; _, err = f.Read(d) { if err != nil { } redunt, _ := erasure.EncodeRing(A.Chunks, A.Size/A.Chunks, d) A.Chunks = len(redunt) indexs = indexs + 1 for index, val := range redunt { k, e := os.Open(fmt.Sprintf("%s-%d%d", A.ID, indexs, index)) if e != nil { log.Fatal(e) } if _, e = k.Write([]byte(val)); e != nil { log.Fatal(e) } } } return A }
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions. // NewRPCServer must properly initialize a RPC server. // downloadSector must successfully retrieve a Sector from a quorum. // The downloaded Sector must match the original Sector. func TestRPCdownloadSector(t *testing.T) { SectorDB = make(map[crypto.Hash]*common.RingHeader) // create sector secData, err := crypto.RandomByteSlice(70000) if err != nil { t.Fatal("Could not generate test data:", err) } sec, err := common.NewSector(secData) if err != nil { t.Fatal("Failed to create sector:", err) } k := common.QuorumSize / 2 params := sec.CalculateParams(k) // encode sector ring, err := erasure.EncodeRing(sec, params) if err != nil { t.Fatal("Failed to encode sector data:", err) } // create RPCServer router, err = network.NewRPCServer(9985) if err != nil { t.Fatal("Failed to initialize RPCServer:", err) } defer router.Close() // create quorum var q [common.QuorumSize]common.Address for i := 0; i < common.QuorumSize; i++ { q[i] = common.Address{0, "localhost", 9000 + i} qrpc, err := network.NewRPCServer(9000 + i) if err != nil { t.Fatal("Failed to initialize RPCServer:", err) } sh := new(Server) sh.seg = ring[i] q[i].ID = qrpc.RegisterHandler(sh) } // add sector to database SectorDB[sec.Hash] = &common.RingHeader{ Hosts: q, Params: params, } // download file from quorum sec, err = downloadSector(sec.Hash) if err != nil { t.Fatal("Failed to download file:", err) } // check hash rebuiltHash, err := crypto.CalculateHash(sec.Data) if err != nil { t.Fatal("Failed to calculate hash:", err) } if sec.Hash != rebuiltHash { t.Fatal("Failed to recover file: hashes do not match") } }
// NewTCPServer must properly initialize a TCP server. // DownloadFile must successfully retrieve a file from a quorum. // The downloaded file must match the original file. func TestTCPDownloadFile(t *testing.T) { t.Skip() // create file fileData, err := crypto.RandomByteSlice(70000) if err != nil { t.Fatal("Could not generate test data:", err) } // calculate hash origHash, err := crypto.CalculateHash(fileData) if err != nil { t.Fatal("Failed to calculate hash:", err) } // encode file k := 50 bytesPerSegment := len(fileData) / k if bytesPerSegment%64 != 0 { bytesPerSegment += 64 - (bytesPerSegment % 64) padding := k*bytesPerSegment - len(fileData) fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...) } segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData) if err != nil { t.Fatal("Failed to encode file data:", err) } // create TCPServer tcp, err := network.NewTCPServer(9988) if err != nil { t.Fatal("Failed to initialize TCPServer:", err) } defer tcp.Close() tdh := new(downloadHandler) tdh.segments = make([]string, k) tdh.indices = make([]uint8, k) tdh.k, tdh.b = k, bytesPerSegment tcp.AddMessageHandler(tdh) // create quorum var q [common.QuorumSize]common.Address var tfhs [common.QuorumSize]TestFileHandler for i := 0; i < common.QuorumSize; i++ { q[i] = common.Address{0, "localhost", 9000 + i} qtcp, err := network.NewTCPServer(9000 + i) if err != nil { t.Fatal("Failed to initialize TCPServer:", err) } tfhs[i].tcpServ = qtcp tfhs[i].dest = tcp.Address() tfhs[i].data = segments[i] tfhs[i].done = make(chan bool, 1) q[i].Id = qtcp.AddMessageHandler(&tfhs[i]).Id } // download file from quorum downData, err := DownloadFile(tcp, origHash, len(fileData), k, q) if err != nil { t.Fatal("Failed to download file:", err) } // wait for download to complete <-tdh.done // check hash rebuiltHash, err := crypto.CalculateHash(downData) if err != nil { t.Fatal("Failed to calculate hash:", err) } if origHash != rebuiltHash { t.Fatal("Failed to recover file: hashes do not match") } }