Example #1
0
func main() {
	// Parse flags
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "Error: No filenames given\n")
		flag.Usage()
		os.Exit(1)
	}
	fname := args[0]

	// Create matrix
	enc, err := reedsolomon.New(*dataShards, *parShards)
	checkErr(err)

	// Create shards and load the data.
	shards := make([][]byte, *dataShards+*parShards)
	for i := range shards {
		infn := fmt.Sprintf("%s.%d", fname, i)
		fmt.Println("Opening", infn)
		shards[i], err = ioutil.ReadFile(infn)
		if err != nil {
			fmt.Println("Error reading file", err)
			shards[i] = nil
		}
	}

	// Verify the shards
	ok, err := enc.Verify(shards)
	if ok {
		fmt.Println("No reconstruction needed")
	} else {
		fmt.Println("Verification failed. Reconstructing data")
		err = enc.Reconstruct(shards)
		if err != nil {
			fmt.Println("Reconstruct failed -", err)
			os.Exit(1)
		}
		ok, err = enc.Verify(shards)
		if !ok {
			fmt.Println("Verification failed after reconstruction, data likely corrupted.")
			os.Exit(1)
		}
		checkErr(err)
	}

	// Join the shards and write them
	outfn := *outFile
	if outfn == "" {
		outfn = fname
	}

	fmt.Println("Writing data to", outfn)
	f, err := os.Create(outfn)
	checkErr(err)

	// We don't know the exact filesize.
	err = enc.Join(f, shards, len(shards[0])**dataShards)
	checkErr(err)
}
Example #2
0
// Simple example of how to use all functions of the Encoder.
// Note that all error checks have been removed to keep it short.
func ExampleEncoder() {
	// Create some sample data
	var data = make([]byte, 250000)
	fillRandom(data)

	// Create an encoder with 17 data and 3 parity slices.
	enc, _ := reedsolomon.New(17, 3)

	// Split the data into shards
	shards, _ := enc.Split(data)

	// Encode the parity set
	_ = enc.Encode(shards)

	// Verify the parity set
	ok, _ := enc.Verify(shards)
	if ok {
		fmt.Println("ok")
	}

	// Delete two shards
	shards[10], shards[11] = nil, nil

	// Reconstruct the shards
	_ = enc.Reconstruct(shards)

	// Verify the data set
	ok, _ = enc.Verify(shards)
	if ok {
		fmt.Println("ok")
	}
	// Output: ok
	// ok
}
// decodeData - decode encoded blocks.
func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
	// Initialized reedsolomon.
	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
	if err != nil {
		return traceError(err)
	}

	// Reconstruct encoded blocks.
	err = rs.Reconstruct(enBlocks)
	if err != nil {
		return traceError(err)
	}

	// Verify reconstructed blocks (parity).
	ok, err := rs.Verify(enBlocks)
	if err != nil {
		return traceError(err)
	}
	if !ok {
		// Blocks cannot be reconstructed, corrupted data.
		err = errors.New("Verification failed after reconstruction, data likely corrupted")
		return traceError(err)
	}

	// Success.
	return nil
}
Example #4
0
// This demonstrates that shards can be arbitrary sliced and
// merged and still remain valid.
func ExampleEncoder_slicing() {
	// Create some sample data
	var data = make([]byte, 250000)
	fillRandom(data)

	// Create 5 data slices of 50000 elements each
	enc, _ := reedsolomon.New(5, 3)
	shards, _ := enc.Split(data)
	err := enc.Encode(shards)
	if err != nil {
		panic(err)
	}

	// Check that it verifies
	ok, err := enc.Verify(shards)
	if ok && err == nil {
		fmt.Println("encode ok")
	}

	// Split the data set of 50000 elements into two of 25000
	splitA := make([][]byte, 8)
	splitB := make([][]byte, 8)

	// Merge into a 100000 element set
	merged := make([][]byte, 8)

	// Split/merge the shards
	for i := range shards {
		splitA[i] = shards[i][:25000]
		splitB[i] = shards[i][25000:]

		// Concencate it to itself
		merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...)
		merged[i] = append(merged[i], shards[i]...)
	}

	// Each part should still verify as ok.
	ok, err = enc.Verify(shards)
	if ok && err == nil {
		fmt.Println("splitA ok")
	}

	ok, err = enc.Verify(splitB)
	if ok && err == nil {
		fmt.Println("splitB ok")
	}

	ok, err = enc.Verify(merged)
	if ok && err == nil {
		fmt.Println("merge ok")
	}
	// Output: encode ok
	// splitA ok
	// splitB ok
	// merge ok
}
Example #5
0
func (c *Conn) SetFec(DataShards, ParityShards int) (er error) {
	c.fecDataShards = DataShards
	c.fecParityShards = ParityShards
	var fec reedsolomon.Encoder
	fec, er = reedsolomon.New(DataShards, ParityShards)
	if er != nil {
		return
	}
	c.fecR = &fec
	fec, er = reedsolomon.New(DataShards, ParityShards)
	if er == nil {
		c.fecRCacheTbl = make(map[uint]*fecInfo)
		c.fecWCacheTbl = nil
		c.fecW = &fec
	} else {
		c.fecR = nil
	}
	return
}
Example #6
0
// NewRSCode creates a new Reed-Solomon encoder/decoder using the supplied
// parameters.
func NewRSCode(nData, nParity int) (modules.ErasureCoder, error) {
	enc, err := reedsolomon.New(nData, nParity)
	if err != nil {
		return nil, err
	}
	return &rsCode{
		enc:        enc,
		numPieces:  nData + nParity,
		dataPieces: nData,
	}, nil
}
Example #7
0
// Make takes a message as a byte slice, along with the expected loss rate and
// target reliability and produces the packets for that message. The packets are
// returned as a slice of byte-slices.
func (p *Packeter) Make(msg []byte, loss, reliability float64) [][]byte {
	l := uint32(len(msg) + 4)
	msk := uint32(255)
	lb := []byte{
		byte(l & msk),
		byte((l >> 8) & msk),
		byte((l >> 16) & msk),
		byte((l >> 24) & msk),
	}
	msg = append(lb, msg...)

	dataShards, parityShards := findRedundancy(len(msg), Packetlength, loss, reliability)
	if parityShards < 1 {
		// reedsolomon.Encoder cannot have 0 parity shards
		// at some point I want to change this so it doesn't use reedsolomon in this
		// case
		parityShards = 1
	}
	shards := dataShards + parityShards

	var data [][]byte
	if enc, e := reedsolomon.New(dataShards, parityShards); err.Warn(e) {
		if data, e = enc.Split(msg); err.Warn(e) {
			err.Warn(enc.Encode(data))
		}
	} else {
		return nil
	}

	idArr := make([]byte, 4)
	rand.Read(idArr)
	id := uint32(idArr[0]) + uint32(idArr[1])<<8 + uint32(idArr[2])<<16 + uint32(idArr[3])<<24

	pk := Packet{}
	pks := make([][]byte, shards)

	pk.MessageId = id
	pk.ParityShards = uint32(parityShards)
	for i := 0; i < shards; i++ {
		pk.PacketId = uint32(i<<16) + uint32(shards)
		pk.Data = data[i]
		if d, e := proto.Marshal(&pk); err.Log(e) {
			pks[i] = d
		} else {
			return nil
		}
	}
	return pks
}
Example #8
0
func main() {
	// Parse command line parameters.
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "Error: No input filename given\n")
		flag.Usage()
		os.Exit(1)
	}
	if *data > 257 {
		fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
		os.Exit(1)
	}
	fname := args[0]

	// Create encoding matrix.
	enc, err := reedsolomon.New(*dataShards, *parShards)
	checkErr(err)

	fmt.Println("Opening", fname)
	b, err := ioutil.ReadFile(fname)
	checkErr(err)

	// Split the file into equally sized shards.
	shards, err := enc.Split(b)
	checkErr(err)
	fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0]))

	// Encode parity
	err = enc.Encode(shards)
	checkErr(err)

	// Write out the resulting files.
	dir, file := filepath.Split(fname)
	if *outDir != "" {
		dir = *outDir
	}
	for i, shard := range shards {
		outfn := fmt.Sprintf("%s.%d", file, i)

		fmt.Println("Writing to", outfn)
		err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm)
		checkErr(err)
	}
}
Example #9
0
// encodeData - encodes incoming data buffer into
// dataBlocks+parityBlocks returns a 2 dimensional byte array.
func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
	if err != nil {
		return nil, traceError(err)
	}
	// Split the input buffer into data and parity blocks.
	var blocks [][]byte
	blocks, err = rs.Split(dataBuffer)
	if err != nil {
		return nil, traceError(err)
	}

	// Encode parity blocks using data blocks.
	err = rs.Encode(blocks)
	if err != nil {
		return nil, traceError(err)
	}

	// Return encoded blocks.
	return blocks, nil
}
Example #10
0
// This demonstrates that shards can xor'ed and
// still remain a valid set.
//
// The xor value must be the same for element 'n' in each shard,
// except if you xor with a similar sized encoded shard set.
func ExampleEncoder_xor() {
	// Create some sample data
	var data = make([]byte, 25000)
	fillRandom(data)

	// Create 5 data slices of 5000 elements each
	enc, _ := reedsolomon.New(5, 3)
	shards, _ := enc.Split(data)
	err := enc.Encode(shards)
	if err != nil {
		panic(err)
	}

	// Check that it verifies
	ok, err := enc.Verify(shards)
	if !ok || err != nil {
		fmt.Println("falied initial verify", err)
	}

	// Create an xor'ed set
	xored := make([][]byte, 8)

	// We xor by the index, so you can see that the xor can change,
	// It should however be constant vertically through your slices.
	for i := range shards {
		xored[i] = make([]byte, len(shards[i]))
		for j := range xored[i] {
			xored[i][j] = shards[i][j] ^ byte(j&0xff)
		}
	}

	// Each part should still verify as ok.
	ok, err = enc.Verify(xored)
	if ok && err == nil {
		fmt.Println("verified ok after xor")
	}
	// Output: verified ok after xor
}
Example #11
0
// Receive collects packets. When exactly enough packets have been recovered to
// reconstruct the message, the message is returned as a byte slice. Otherwise
// nil is returned. Receive can continue to collect packets after the message
// has been constructed for reliability statistics.
func (p Packeter) Receive(b []byte) []byte {
	var pk Packet
	if err.Log(proto.Unmarshal(b, &pk)) {
		return nil
	}
	mid, pid, dataShards, parityShards := pk.MetaData()
	clctr, ok := p[mid]
	if !ok {
		clctr = &collector{
			data:      make([][]byte, dataShards+parityShards),
			collected: make(map[uint32]bool),
			complete:  false,
		}
		p[mid] = clctr
	}
	clctr.collected[pid] = true
	if clctr.complete {
		return nil
	}
	clctr.data[pid] = pk.Data
	if uint32(len(clctr.collected)) >= dataShards {
		clctr.complete = true
		if enc, e := reedsolomon.New(int(dataShards), int(parityShards)); err.Warn(e) {
			if err.Log(enc.Reconstruct(clctr.data)) {

				//maybe should take out as an arg
				ln := int(clctr.data[0][0]) + int(clctr.data[0][1])<<8 + int(clctr.data[0][2])<<16 + int(clctr.data[0][2])<<24
				var out bytes.Buffer
				err.Warn(enc.Join(&out, clctr.data, ln))
				clctr.data = nil

				return out.Bytes()[4:]
			}
		}
	}
	return nil
}