// Simple example of how to use all functions of the Encoder.
// Note that all error checks have been removed to keep it short.
func ExampleEncoder() {
// Create some sample data
var data = make([]byte, 250000)
fillRandom(data)
// Create an encoder with 17 data and 3 parity slices.
enc, _ := reedsolomon.New(17, 3)
// Split the data into shards
shards, _ := enc.Split(data)
// Encode the parity set
_ = enc.Encode(shards)
// Verify the parity set
ok, _ := enc.Verify(shards)
if ok {
fmt.Println("ok")
}
// Delete two shards
shards[10], shards[11] = nil, nil
// Reconstruct the shards
_ = enc.Reconstruct(shards)
// Verify the data set
ok, _ = enc.Verify(shards)
if ok {
fmt.Println("ok")
}
// Output: ok
// ok
}
// This demonstrates that shards can be arbitrary sliced and
// merged and still remain valid.
func ExampleEncoder_slicing() {
// Create some sample data
var data = make([]byte, 250000)
fillRandom(data)
// Create 5 data slices of 50000 elements each
enc, _ := reedsolomon.New(5, 3)
shards, _ := enc.Split(data)
err := enc.Encode(shards)
if err != nil {
panic(err)
}
// Check that it verifies
ok, err := enc.Verify(shards)
if ok && err == nil {
fmt.Println("encode ok")
}
// Split the data set of 50000 elements into two of 25000
splitA := make([][]byte, 8)
splitB := make([][]byte, 8)
// Merge into a 100000 element set
merged := make([][]byte, 8)
// Split/merge the shards
for i := range shards {
splitA[i] = shards[i][:25000]
splitB[i] = shards[i][25000:]
// Concencate it to itself
merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...)
merged[i] = append(merged[i], shards[i]...)
}
// Each part should still verify as ok.
ok, err = enc.Verify(shards)
if ok && err == nil {
fmt.Println("splitA ok")
}
ok, err = enc.Verify(splitB)
if ok && err == nil {
fmt.Println("splitB ok")
}
ok, err = enc.Verify(merged)
if ok && err == nil {
fmt.Println("merge ok")
}
// Output: encode ok
// splitA ok
// splitB ok
// merge ok
}
//Restore restores original file from m+n files.
func Restore(fnames []string, outfile string, fsize int64, n int, key []byte) error {
m := len(fnames) - n
enc, err := reedsolomon.New(n, m)
if err != nil {
log.Println(err)
return err
}
data := make([][]byte, m+n)
shardSize := blocks(int(fsize), n)
rs := make([]*smap, m+n)
for i := 0; i < len(fnames); i++ {
if fnames[i] == "" {
continue
}
if rs[i], err = rmmap(fnames[i]); err != nil {
log.Println(err)
return err
}
data[i] = rs[i].bytes()[:shardSize]
}
defer func() {
for i := 0; i < m+n; i++ {
if rs[i] != nil {
rs[i].Close()
}
}
}()
if err := enc.Reconstruct(data); err != nil {
log.Println(err)
return err
}
w, err := os.Create(outfile)
if err != nil {
log.Println(err)
return err
}
defer func() {
if err := w.Close(); err != nil {
log.Println(err)
}
}()
if err := enc.Join(bufio.NewWriter(w), data, int(fsize)); err != nil {
log.Println(err)
return err
}
return nil
}
// This demonstrates that shards can xor'ed and
// still remain a valid set.
//
// The xor value must be the same for element 'n' in each shard,
// except if you xor with a similar sized encoded shard set.
func ExampleEncoder_xor() {
// Create some sample data
var data = make([]byte, 25000)
fillRandom(data)
// Create 5 data slices of 5000 elements each
enc, _ := reedsolomon.New(5, 3)
shards, _ := enc.Split(data)
err := enc.Encode(shards)
if err != nil {
panic(err)
}
// Check that it verifies
ok, err := enc.Verify(shards)
if !ok || err != nil {
fmt.Println("falied initial verify", err)
}
// Create an xor'ed set
xored := make([][]byte, 8)
// We xor by the index, so you can see that the xor can change,
// It should however be constant vertically through your slices.
for i := range shards {
xored[i] = make([]byte, len(shards[i]))
for j := range xored[i] {
xored[i][j] = shards[i][j] ^ byte(j&0xff)
}
}
// Each part should still verify as ok.
ok, err = enc.Verify(xored)
if ok && err == nil {
fmt.Println("verified ok after xor")
}
// Output: verified ok after xor
}
//Build builds shards for servers with reed solomon coding
//and (255,1) redundant codes for each servers for challenge/response.
func Build(m, n int, fname string) ([]string, []byte, error) {
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
log.Fatal(err)
}
prf := &prf{key: key}
enc, err := reedsolomon.New(n, m)
if err != nil {
log.Println(err)
return nil, nil, err
}
data := make([][]byte, m+n)
rs, err := rmmap(fname)
if err != nil {
log.Println(err)
return nil, nil, err
}
defer rs.Close()
shardSize, err := size(rs.f, n)
if err != nil {
return nil, nil, err
}
serverBlockSize := blocks(shardSize, nServerBlock-1)
ss := make([]*smap, m+n)
name := parseName(fname)
outfiles := make([]string, m+n)
for i := 0; i < m+n; i++ {
outfiles[i] = name.rename(i)
ss[i], err = wmmap(outfiles[i], int64(serverBlockSize)*nServerBlock)
if err != nil {
log.Println(err)
return nil, nil, err
}
data[i] = ss[i].bytes()[:shardSize]
}
defer func() {
for i := 0; i < m+n; i++ {
ss[i].Close()
}
}()
for i := 0; i < n; i++ {
end := (i + 1) * shardSize
if end > len(rs.bytes()) {
end = len(rs.bytes())
}
copy(data[i], rs.bytes()[i*shardSize:end])
}
if err != nil {
log.Println(err)
return nil, nil, err
}
if err = enc.Encode(data); err != nil {
log.Println(err)
return nil, nil, err
}
encServer, err := reedsolomon.NewAN(nServerBlock-1, 1, prf.perm())
if err != nil {
log.Println(err)
return nil, nil, err
}
for i := 0; i < m+n; i++ {
prf.fnum = i
if err := serverEncode(ss[i].bytes(), prf, serverBlockSize, encServer); err != nil {
log.Println(err)
return nil, nil, err
}
}
return outfiles, key, err
}