Example #1
0
// Simple example of how to use all functions of the Encoder.
// Note that all error checks have been removed to keep it short.
func ExampleEncoder() {
	// Create some sample data
	var data = make([]byte, 250000)
	fillRandom(data)

	// Create an encoder with 17 data and 3 parity slices.
	enc, _ := reedsolomon.New(17, 3)

	// Split the data into shards
	shards, _ := enc.Split(data)

	// Encode the parity set
	_ = enc.Encode(shards)

	// Verify the parity set
	ok, _ := enc.Verify(shards)
	if ok {
		fmt.Println("ok")
	}

	// Delete two shards
	shards[10], shards[11] = nil, nil

	// Reconstruct the shards
	_ = enc.Reconstruct(shards)

	// Verify the data set
	ok, _ = enc.Verify(shards)
	if ok {
		fmt.Println("ok")
	}
	// Output: ok
	// ok
}
Example #2
0
// This demonstrates that shards can be arbitrary sliced and
// merged and still remain valid.
func ExampleEncoder_slicing() {
	// Create some sample data
	var data = make([]byte, 250000)
	fillRandom(data)

	// Create 5 data slices of 50000 elements each
	enc, _ := reedsolomon.New(5, 3)
	shards, _ := enc.Split(data)
	err := enc.Encode(shards)
	if err != nil {
		panic(err)
	}

	// Check that it verifies
	ok, err := enc.Verify(shards)
	if ok && err == nil {
		fmt.Println("encode ok")
	}

	// Split the data set of 50000 elements into two of 25000
	splitA := make([][]byte, 8)
	splitB := make([][]byte, 8)

	// Merge into a 100000 element set
	merged := make([][]byte, 8)

	// Split/merge the shards
	for i := range shards {
		splitA[i] = shards[i][:25000]
		splitB[i] = shards[i][25000:]

		// Concencate it to itself
		merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...)
		merged[i] = append(merged[i], shards[i]...)
	}

	// Each part should still verify as ok.
	ok, err = enc.Verify(shards)
	if ok && err == nil {
		fmt.Println("splitA ok")
	}

	ok, err = enc.Verify(splitB)
	if ok && err == nil {
		fmt.Println("splitB ok")
	}

	ok, err = enc.Verify(merged)
	if ok && err == nil {
		fmt.Println("merge ok")
	}
	// Output: encode ok
	// splitA ok
	// splitB ok
	// merge ok
}
Example #3
0
//Restore restores original file from m+n files.
func Restore(fnames []string, outfile string, fsize int64, n int, key []byte) error {
	m := len(fnames) - n
	enc, err := reedsolomon.New(n, m)
	if err != nil {
		log.Println(err)
		return err
	}
	data := make([][]byte, m+n)
	shardSize := blocks(int(fsize), n)
	rs := make([]*smap, m+n)
	for i := 0; i < len(fnames); i++ {
		if fnames[i] == "" {
			continue
		}
		if rs[i], err = rmmap(fnames[i]); err != nil {
			log.Println(err)
			return err
		}
		data[i] = rs[i].bytes()[:shardSize]
	}
	defer func() {
		for i := 0; i < m+n; i++ {
			if rs[i] != nil {
				rs[i].Close()
			}
		}
	}()

	if err := enc.Reconstruct(data); err != nil {
		log.Println(err)
		return err
	}
	w, err := os.Create(outfile)
	if err != nil {
		log.Println(err)
		return err
	}
	defer func() {
		if err := w.Close(); err != nil {
			log.Println(err)
		}
	}()
	if err := enc.Join(bufio.NewWriter(w), data, int(fsize)); err != nil {
		log.Println(err)
		return err
	}
	return nil
}
Example #4
0
// This demonstrates that shards can xor'ed and
// still remain a valid set.
//
// The xor value must be the same for element 'n' in each shard,
// except if you xor with a similar sized encoded shard set.
func ExampleEncoder_xor() {
	// Create some sample data
	var data = make([]byte, 25000)
	fillRandom(data)

	// Create 5 data slices of 5000 elements each
	enc, _ := reedsolomon.New(5, 3)
	shards, _ := enc.Split(data)
	err := enc.Encode(shards)
	if err != nil {
		panic(err)
	}

	// Check that it verifies
	ok, err := enc.Verify(shards)
	if !ok || err != nil {
		fmt.Println("falied initial verify", err)
	}

	// Create an xor'ed set
	xored := make([][]byte, 8)

	// We xor by the index, so you can see that the xor can change,
	// It should however be constant vertically through your slices.
	for i := range shards {
		xored[i] = make([]byte, len(shards[i]))
		for j := range xored[i] {
			xored[i][j] = shards[i][j] ^ byte(j&0xff)
		}
	}

	// Each part should still verify as ok.
	ok, err = enc.Verify(xored)
	if ok && err == nil {
		fmt.Println("verified ok after xor")
	}
	// Output: verified ok after xor
}
Example #5
0
//Build builds shards for servers with reed solomon coding
//and (255,1) redundant codes for each servers for challenge/response.
func Build(m, n int, fname string) ([]string, []byte, error) {
	key := make([]byte, 32)
	if _, err := rand.Read(key); err != nil {
		log.Fatal(err)
	}
	prf := &prf{key: key}

	enc, err := reedsolomon.New(n, m)
	if err != nil {
		log.Println(err)
		return nil, nil, err
	}
	data := make([][]byte, m+n)

	rs, err := rmmap(fname)
	if err != nil {
		log.Println(err)
		return nil, nil, err
	}
	defer rs.Close()

	shardSize, err := size(rs.f, n)
	if err != nil {
		return nil, nil, err
	}
	serverBlockSize := blocks(shardSize, nServerBlock-1)

	ss := make([]*smap, m+n)
	name := parseName(fname)
	outfiles := make([]string, m+n)
	for i := 0; i < m+n; i++ {
		outfiles[i] = name.rename(i)
		ss[i], err = wmmap(outfiles[i], int64(serverBlockSize)*nServerBlock)
		if err != nil {
			log.Println(err)
			return nil, nil, err
		}
		data[i] = ss[i].bytes()[:shardSize]
	}
	defer func() {
		for i := 0; i < m+n; i++ {
			ss[i].Close()
		}
	}()
	for i := 0; i < n; i++ {
		end := (i + 1) * shardSize
		if end > len(rs.bytes()) {
			end = len(rs.bytes())
		}
		copy(data[i], rs.bytes()[i*shardSize:end])
	}
	if err != nil {
		log.Println(err)
		return nil, nil, err
	}

	if err = enc.Encode(data); err != nil {
		log.Println(err)
		return nil, nil, err
	}
	encServer, err := reedsolomon.NewAN(nServerBlock-1, 1, prf.perm())
	if err != nil {
		log.Println(err)
		return nil, nil, err
	}
	for i := 0; i < m+n; i++ {
		prf.fnum = i
		if err := serverEncode(ss[i].bytes(), prf, serverBlockSize, encServer); err != nil {
			log.Println(err)
			return nil, nil, err
		}
	}
	return outfiles, key, err
}