// Simple example of how to use all functions of the Encoder. // Note that all error checks have been removed to keep it short. func ExampleEncoder() { // Create some sample data var data = make([]byte, 250000) fillRandom(data) // Create an encoder with 17 data and 3 parity slices. enc, _ := reedsolomon.New(17, 3) // Split the data into shards shards, _ := enc.Split(data) // Encode the parity set _ = enc.Encode(shards) // Verify the parity set ok, _ := enc.Verify(shards) if ok { fmt.Println("ok") } // Delete two shards shards[10], shards[11] = nil, nil // Reconstruct the shards _ = enc.Reconstruct(shards) // Verify the data set ok, _ = enc.Verify(shards) if ok { fmt.Println("ok") } // Output: ok // ok }
// This demonstrates that shards can be arbitrary sliced and // merged and still remain valid. func ExampleEncoder_slicing() { // Create some sample data var data = make([]byte, 250000) fillRandom(data) // Create 5 data slices of 50000 elements each enc, _ := reedsolomon.New(5, 3) shards, _ := enc.Split(data) err := enc.Encode(shards) if err != nil { panic(err) } // Check that it verifies ok, err := enc.Verify(shards) if ok && err == nil { fmt.Println("encode ok") } // Split the data set of 50000 elements into two of 25000 splitA := make([][]byte, 8) splitB := make([][]byte, 8) // Merge into a 100000 element set merged := make([][]byte, 8) // Split/merge the shards for i := range shards { splitA[i] = shards[i][:25000] splitB[i] = shards[i][25000:] // Concencate it to itself merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...) merged[i] = append(merged[i], shards[i]...) } // Each part should still verify as ok. ok, err = enc.Verify(shards) if ok && err == nil { fmt.Println("splitA ok") } ok, err = enc.Verify(splitB) if ok && err == nil { fmt.Println("splitB ok") } ok, err = enc.Verify(merged) if ok && err == nil { fmt.Println("merge ok") } // Output: encode ok // splitA ok // splitB ok // merge ok }
//Restore restores original file from m+n files. func Restore(fnames []string, outfile string, fsize int64, n int, key []byte) error { m := len(fnames) - n enc, err := reedsolomon.New(n, m) if err != nil { log.Println(err) return err } data := make([][]byte, m+n) shardSize := blocks(int(fsize), n) rs := make([]*smap, m+n) for i := 0; i < len(fnames); i++ { if fnames[i] == "" { continue } if rs[i], err = rmmap(fnames[i]); err != nil { log.Println(err) return err } data[i] = rs[i].bytes()[:shardSize] } defer func() { for i := 0; i < m+n; i++ { if rs[i] != nil { rs[i].Close() } } }() if err := enc.Reconstruct(data); err != nil { log.Println(err) return err } w, err := os.Create(outfile) if err != nil { log.Println(err) return err } defer func() { if err := w.Close(); err != nil { log.Println(err) } }() if err := enc.Join(bufio.NewWriter(w), data, int(fsize)); err != nil { log.Println(err) return err } return nil }
// This demonstrates that shards can xor'ed and // still remain a valid set. // // The xor value must be the same for element 'n' in each shard, // except if you xor with a similar sized encoded shard set. func ExampleEncoder_xor() { // Create some sample data var data = make([]byte, 25000) fillRandom(data) // Create 5 data slices of 5000 elements each enc, _ := reedsolomon.New(5, 3) shards, _ := enc.Split(data) err := enc.Encode(shards) if err != nil { panic(err) } // Check that it verifies ok, err := enc.Verify(shards) if !ok || err != nil { fmt.Println("falied initial verify", err) } // Create an xor'ed set xored := make([][]byte, 8) // We xor by the index, so you can see that the xor can change, // It should however be constant vertically through your slices. for i := range shards { xored[i] = make([]byte, len(shards[i])) for j := range xored[i] { xored[i][j] = shards[i][j] ^ byte(j&0xff) } } // Each part should still verify as ok. ok, err = enc.Verify(xored) if ok && err == nil { fmt.Println("verified ok after xor") } // Output: verified ok after xor }
//Build builds shards for servers with reed solomon coding //and (255,1) redundant codes for each servers for challenge/response. func Build(m, n int, fname string) ([]string, []byte, error) { key := make([]byte, 32) if _, err := rand.Read(key); err != nil { log.Fatal(err) } prf := &prf{key: key} enc, err := reedsolomon.New(n, m) if err != nil { log.Println(err) return nil, nil, err } data := make([][]byte, m+n) rs, err := rmmap(fname) if err != nil { log.Println(err) return nil, nil, err } defer rs.Close() shardSize, err := size(rs.f, n) if err != nil { return nil, nil, err } serverBlockSize := blocks(shardSize, nServerBlock-1) ss := make([]*smap, m+n) name := parseName(fname) outfiles := make([]string, m+n) for i := 0; i < m+n; i++ { outfiles[i] = name.rename(i) ss[i], err = wmmap(outfiles[i], int64(serverBlockSize)*nServerBlock) if err != nil { log.Println(err) return nil, nil, err } data[i] = ss[i].bytes()[:shardSize] } defer func() { for i := 0; i < m+n; i++ { ss[i].Close() } }() for i := 0; i < n; i++ { end := (i + 1) * shardSize if end > len(rs.bytes()) { end = len(rs.bytes()) } copy(data[i], rs.bytes()[i*shardSize:end]) } if err != nil { log.Println(err) return nil, nil, err } if err = enc.Encode(data); err != nil { log.Println(err) return nil, nil, err } encServer, err := reedsolomon.NewAN(nServerBlock-1, 1, prf.perm()) if err != nil { log.Println(err) return nil, nil, err } for i := 0; i < m+n; i++ { prf.fnum = i if err := serverEncode(ss[i].bytes(), prf, serverBlockSize, encServer); err != nil { log.Println(err) return nil, nil, err } } return outfiles, key, err }