Beispiel #1
0
// TestTCPDownloadFile tests the NewTCPServer and DownloadFile functions.
// NewTCPServer must properly initialize a TCP server.
// UploadFile splits a file into erasure-coded segments and distributes them across a quorum.
// k is the number of non-redundant segments.
// The file is padded to satisfy the erasure-coding requirements that:
//     len(fileData) = k*bytesPerSegment, and:
//     bytesPerSegment % 64 = 0
func UploadFile(mr common.MessageRouter, file *os.File, k int, quorum [common.QuorumSize]common.Address) (bytesPerSegment int, err error) {
	// read file
	fileInfo, err := file.Stat()
	if err != nil {
		return
	}
	if fileInfo.Size() > int64(common.QuorumSize*common.MaxSegmentSize) {
		err = fmt.Errorf("File exceeds maximum per-quorum size")
		return
	}
	fileData := make([]byte, fileInfo.Size())
	_, err = io.ReadFull(file, fileData)
	if err != nil {
		return
	}

	// calculate EncodeRing parameters, padding file if necessary
	bytesPerSegment = len(fileData) / k
	if bytesPerSegment%64 != 0 {
		bytesPerSegment += 64 - (bytesPerSegment % 64)
		padding := k*bytesPerSegment - len(fileData)
		fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...)
	}

	// create erasure-coded segments
	segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData)
	if err != nil {
		return
	}

	// for now we just send segment i to node i
	// this may need to be randomized for security
	for i := range quorum {
		m := new(common.Message)
		m.Destination = quorum[i]
		m.Payload = append([]byte{byte(i)}, []byte(segments[i])...)
		err = mr.SendMessage(m)
		if err != nil {
			return
		}
	}

	return
}
Beispiel #2
0
// uploadSector splits a Sector into a Ring and distributes it across a quorum.
// It hashes each of the Ring's segments and stores the hashes in the SectorDB.
func uploadSector(sec *common.Sector) (err error) {
	// look up Sector in SectorDB
	rh := SectorDB[sec.Hash]
	if rh == nil {
		return fmt.Errorf("Sector not found in database")
	}

	// create ring
	ring, err := erasure.EncodeRing(sec, rh.Params)
	if err != nil {
		return
	}

	// calculate and store segment hashes
	for i := range ring {
		rh.SegHashes[i], err = crypto.CalculateHash(ring[i].Data)
		if err != nil {
			return
		}
	}

	// for now we just send segment i to host i
	// this may need to be randomized for security
	for i := range rh.Hosts {
		err = router.SendMessage(&common.Message{
			Dest: rh.Hosts[i],
			Proc: "Server.UploadSegment",
			Args: ring[i],
			Resp: nil,
		})
		if err != nil {
			return
		}
	}

	return
}
Beispiel #3
0
func CreateFileAnnounce(file string) *Announce {
	A := new(Announce)
	f, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	c, err := f.Stat()
	if err != nil {
	}
	A.Size = int(c.Size())
	//this should be enough.
	A.Chunks = A.Size / (1 << 20)
	//Chunk File
	d := make([]byte, A.Size/(1<<20))
	indexs := 0
	//generate ID and dump the chunks into files
	//A.ID=""
	for _ = 0; err != io.EOF; _, err = f.Read(d) {
		if err != nil {
		}
		redunt, _ := erasure.EncodeRing(A.Chunks, A.Size/A.Chunks, d)
		A.Chunks = len(redunt)
		indexs = indexs + 1
		for index, val := range redunt {
			k, e := os.Open(fmt.Sprintf("%s-%d%d", A.ID, indexs, index))
			if e != nil {
				log.Fatal(e)
			}
			if _, e = k.Write([]byte(val)); e != nil {
				log.Fatal(e)
			}
		}
	}

	return A
}
Beispiel #4
0
// TestRPCdownloadSector tests the NewRPCServer and downloadSector functions.
// NewRPCServer must properly initialize a RPC server.
// downloadSector must successfully retrieve a Sector from a quorum.
// The downloaded Sector must match the original Sector.
func TestRPCdownloadSector(t *testing.T) {
	SectorDB = make(map[crypto.Hash]*common.RingHeader)

	// create sector
	secData, err := crypto.RandomByteSlice(70000)
	if err != nil {
		t.Fatal("Could not generate test data:", err)
	}

	sec, err := common.NewSector(secData)
	if err != nil {
		t.Fatal("Failed to create sector:", err)
	}

	k := common.QuorumSize / 2
	params := sec.CalculateParams(k)

	// encode sector
	ring, err := erasure.EncodeRing(sec, params)
	if err != nil {
		t.Fatal("Failed to encode sector data:", err)
	}

	// create RPCServer
	router, err = network.NewRPCServer(9985)
	if err != nil {
		t.Fatal("Failed to initialize RPCServer:", err)
	}
	defer router.Close()

	// create quorum
	var q [common.QuorumSize]common.Address
	for i := 0; i < common.QuorumSize; i++ {
		q[i] = common.Address{0, "localhost", 9000 + i}
		qrpc, err := network.NewRPCServer(9000 + i)
		if err != nil {
			t.Fatal("Failed to initialize RPCServer:", err)
		}
		sh := new(Server)
		sh.seg = ring[i]
		q[i].ID = qrpc.RegisterHandler(sh)
	}

	// add sector to database
	SectorDB[sec.Hash] = &common.RingHeader{
		Hosts:  q,
		Params: params,
	}

	// download file from quorum
	sec, err = downloadSector(sec.Hash)
	if err != nil {
		t.Fatal("Failed to download file:", err)
	}

	// check hash
	rebuiltHash, err := crypto.CalculateHash(sec.Data)
	if err != nil {
		t.Fatal("Failed to calculate hash:", err)
	}

	if sec.Hash != rebuiltHash {
		t.Fatal("Failed to recover file: hashes do not match")
	}
}
Beispiel #5
0
// NewTCPServer must properly initialize a TCP server.
// DownloadFile must successfully retrieve a file from a quorum.
// The downloaded file must match the original file.
func TestTCPDownloadFile(t *testing.T) {
	t.Skip()
	// create file
	fileData, err := crypto.RandomByteSlice(70000)
	if err != nil {
		t.Fatal("Could not generate test data:", err)
	}

	// calculate hash
	origHash, err := crypto.CalculateHash(fileData)
	if err != nil {
		t.Fatal("Failed to calculate hash:", err)
	}

	// encode file
	k := 50
	bytesPerSegment := len(fileData) / k
	if bytesPerSegment%64 != 0 {
		bytesPerSegment += 64 - (bytesPerSegment % 64)
		padding := k*bytesPerSegment - len(fileData)
		fileData = append(fileData, bytes.Repeat([]byte{0x00}, padding)...)
	}
	segments, err := erasure.EncodeRing(k, bytesPerSegment, fileData)
	if err != nil {
		t.Fatal("Failed to encode file data:", err)
	}

	// create TCPServer
	tcp, err := network.NewTCPServer(9988)
	if err != nil {
		t.Fatal("Failed to initialize TCPServer:", err)
	}
	defer tcp.Close()
	tdh := new(downloadHandler)
	tdh.segments = make([]string, k)
	tdh.indices = make([]uint8, k)
	tdh.k, tdh.b = k, bytesPerSegment
	tcp.AddMessageHandler(tdh)

	// create quorum
	var q [common.QuorumSize]common.Address
	var tfhs [common.QuorumSize]TestFileHandler
	for i := 0; i < common.QuorumSize; i++ {
		q[i] = common.Address{0, "localhost", 9000 + i}
		qtcp, err := network.NewTCPServer(9000 + i)
		if err != nil {
			t.Fatal("Failed to initialize TCPServer:", err)
		}
		tfhs[i].tcpServ = qtcp
		tfhs[i].dest = tcp.Address()
		tfhs[i].data = segments[i]
		tfhs[i].done = make(chan bool, 1)
		q[i].Id = qtcp.AddMessageHandler(&tfhs[i]).Id
	}

	// download file from quorum
	downData, err := DownloadFile(tcp, origHash, len(fileData), k, q)
	if err != nil {
		t.Fatal("Failed to download file:", err)
	}

	// wait for download to complete
	<-tdh.done

	// check hash
	rebuiltHash, err := crypto.CalculateHash(downData)
	if err != nil {
		t.Fatal("Failed to calculate hash:", err)
	}

	if origHash != rebuiltHash {
		t.Fatal("Failed to recover file: hashes do not match")
	}
}