func main() {
// Parse flags
flag.Parse()
args := flag.Args()
if len(args) != 1 {
fmt.Fprintf(os.Stderr, "Error: No filenames given\n")
flag.Usage()
os.Exit(1)
}
fname := args[0]
// Create matrix
enc, err := reedsolomon.New(*dataShards, *parShards)
checkErr(err)
// Create shards and load the data.
shards := make([][]byte, *dataShards+*parShards)
for i := range shards {
infn := fmt.Sprintf("%s.%d", fname, i)
fmt.Println("Opening", infn)
shards[i], err = ioutil.ReadFile(infn)
if err != nil {
fmt.Println("Error reading file", err)
shards[i] = nil
}
}
// Verify the shards
ok, err := enc.Verify(shards)
if ok {
fmt.Println("No reconstruction needed")
} else {
fmt.Println("Verification failed. Reconstructing data")
err = enc.Reconstruct(shards)
if err != nil {
fmt.Println("Reconstruct failed -", err)
os.Exit(1)
}
ok, err = enc.Verify(shards)
if !ok {
fmt.Println("Verification failed after reconstruction, data likely corrupted.")
os.Exit(1)
}
checkErr(err)
}
// Join the shards and write them
outfn := *outFile
if outfn == "" {
outfn = fname
}
fmt.Println("Writing data to", outfn)
f, err := os.Create(outfn)
checkErr(err)
// We don't know the exact filesize.
err = enc.Join(f, shards, len(shards[0])**dataShards)
checkErr(err)
}
// Simple example of how to use all functions of the Encoder.
// Note that all error checks have been removed to keep it short.
func ExampleEncoder() {
// Create some sample data
var data = make([]byte, 250000)
fillRandom(data)
// Create an encoder with 17 data and 3 parity slices.
enc, _ := reedsolomon.New(17, 3)
// Split the data into shards
shards, _ := enc.Split(data)
// Encode the parity set
_ = enc.Encode(shards)
// Verify the parity set
ok, _ := enc.Verify(shards)
if ok {
fmt.Println("ok")
}
// Delete two shards
shards[10], shards[11] = nil, nil
// Reconstruct the shards
_ = enc.Reconstruct(shards)
// Verify the data set
ok, _ = enc.Verify(shards)
if ok {
fmt.Println("ok")
}
// Output: ok
// ok
}
// decodeData - decode encoded blocks.
func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
// Initialized reedsolomon.
rs, err := reedsolomon.New(dataBlocks, parityBlocks)
if err != nil {
return traceError(err)
}
// Reconstruct encoded blocks.
err = rs.Reconstruct(enBlocks)
if err != nil {
return traceError(err)
}
// Verify reconstructed blocks (parity).
ok, err := rs.Verify(enBlocks)
if err != nil {
return traceError(err)
}
if !ok {
// Blocks cannot be reconstructed, corrupted data.
err = errors.New("Verification failed after reconstruction, data likely corrupted")
return traceError(err)
}
// Success.
return nil
}
// This demonstrates that shards can be arbitrary sliced and
// merged and still remain valid.
func ExampleEncoder_slicing() {
// Create some sample data
var data = make([]byte, 250000)
fillRandom(data)
// Create 5 data slices of 50000 elements each
enc, _ := reedsolomon.New(5, 3)
shards, _ := enc.Split(data)
err := enc.Encode(shards)
if err != nil {
panic(err)
}
// Check that it verifies
ok, err := enc.Verify(shards)
if ok && err == nil {
fmt.Println("encode ok")
}
// Split the data set of 50000 elements into two of 25000
splitA := make([][]byte, 8)
splitB := make([][]byte, 8)
// Merge into a 100000 element set
merged := make([][]byte, 8)
// Split/merge the shards
for i := range shards {
splitA[i] = shards[i][:25000]
splitB[i] = shards[i][25000:]
// Concencate it to itself
merged[i] = append(make([]byte, 0, len(shards[i])*2), shards[i]...)
merged[i] = append(merged[i], shards[i]...)
}
// Each part should still verify as ok.
ok, err = enc.Verify(shards)
if ok && err == nil {
fmt.Println("splitA ok")
}
ok, err = enc.Verify(splitB)
if ok && err == nil {
fmt.Println("splitB ok")
}
ok, err = enc.Verify(merged)
if ok && err == nil {
fmt.Println("merge ok")
}
// Output: encode ok
// splitA ok
// splitB ok
// merge ok
}
func (c *Conn) SetFec(DataShards, ParityShards int) (er error) {
c.fecDataShards = DataShards
c.fecParityShards = ParityShards
var fec reedsolomon.Encoder
fec, er = reedsolomon.New(DataShards, ParityShards)
if er != nil {
return
}
c.fecR = &fec
fec, er = reedsolomon.New(DataShards, ParityShards)
if er == nil {
c.fecRCacheTbl = make(map[uint]*fecInfo)
c.fecWCacheTbl = nil
c.fecW = &fec
} else {
c.fecR = nil
}
return
}
// NewRSCode creates a new Reed-Solomon encoder/decoder using the supplied
// parameters.
func NewRSCode(nData, nParity int) (modules.ErasureCoder, error) {
enc, err := reedsolomon.New(nData, nParity)
if err != nil {
return nil, err
}
return &rsCode{
enc: enc,
numPieces: nData + nParity,
dataPieces: nData,
}, nil
}
// Make takes a message as a byte slice, along with the expected loss rate and
// target reliability and produces the packets for that message. The packets are
// returned as a slice of byte-slices.
func (p *Packeter) Make(msg []byte, loss, reliability float64) [][]byte {
l := uint32(len(msg) + 4)
msk := uint32(255)
lb := []byte{
byte(l & msk),
byte((l >> 8) & msk),
byte((l >> 16) & msk),
byte((l >> 24) & msk),
}
msg = append(lb, msg...)
dataShards, parityShards := findRedundancy(len(msg), Packetlength, loss, reliability)
if parityShards < 1 {
// reedsolomon.Encoder cannot have 0 parity shards
// at some point I want to change this so it doesn't use reedsolomon in this
// case
parityShards = 1
}
shards := dataShards + parityShards
var data [][]byte
if enc, e := reedsolomon.New(dataShards, parityShards); err.Warn(e) {
if data, e = enc.Split(msg); err.Warn(e) {
err.Warn(enc.Encode(data))
}
} else {
return nil
}
idArr := make([]byte, 4)
rand.Read(idArr)
id := uint32(idArr[0]) + uint32(idArr[1])<<8 + uint32(idArr[2])<<16 + uint32(idArr[3])<<24
pk := Packet{}
pks := make([][]byte, shards)
pk.MessageId = id
pk.ParityShards = uint32(parityShards)
for i := 0; i < shards; i++ {
pk.PacketId = uint32(i<<16) + uint32(shards)
pk.Data = data[i]
if d, e := proto.Marshal(&pk); err.Log(e) {
pks[i] = d
} else {
return nil
}
}
return pks
}
func main() {
// Parse command line parameters.
flag.Parse()
args := flag.Args()
if len(args) != 1 {
fmt.Fprintf(os.Stderr, "Error: No input filename given\n")
flag.Usage()
os.Exit(1)
}
if *data > 257 {
fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
os.Exit(1)
}
fname := args[0]
// Create encoding matrix.
enc, err := reedsolomon.New(*dataShards, *parShards)
checkErr(err)
fmt.Println("Opening", fname)
b, err := ioutil.ReadFile(fname)
checkErr(err)
// Split the file into equally sized shards.
shards, err := enc.Split(b)
checkErr(err)
fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0]))
// Encode parity
err = enc.Encode(shards)
checkErr(err)
// Write out the resulting files.
dir, file := filepath.Split(fname)
if *outDir != "" {
dir = *outDir
}
for i, shard := range shards {
outfn := fmt.Sprintf("%s.%d", file, i)
fmt.Println("Writing to", outfn)
err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm)
checkErr(err)
}
}
// encodeData - encodes incoming data buffer into
// dataBlocks+parityBlocks returns a 2 dimensional byte array.
func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
rs, err := reedsolomon.New(dataBlocks, parityBlocks)
if err != nil {
return nil, traceError(err)
}
// Split the input buffer into data and parity blocks.
var blocks [][]byte
blocks, err = rs.Split(dataBuffer)
if err != nil {
return nil, traceError(err)
}
// Encode parity blocks using data blocks.
err = rs.Encode(blocks)
if err != nil {
return nil, traceError(err)
}
// Return encoded blocks.
return blocks, nil
}
// This demonstrates that shards can xor'ed and
// still remain a valid set.
//
// The xor value must be the same for element 'n' in each shard,
// except if you xor with a similar sized encoded shard set.
func ExampleEncoder_xor() {
// Create some sample data
var data = make([]byte, 25000)
fillRandom(data)
// Create 5 data slices of 5000 elements each
enc, _ := reedsolomon.New(5, 3)
shards, _ := enc.Split(data)
err := enc.Encode(shards)
if err != nil {
panic(err)
}
// Check that it verifies
ok, err := enc.Verify(shards)
if !ok || err != nil {
fmt.Println("falied initial verify", err)
}
// Create an xor'ed set
xored := make([][]byte, 8)
// We xor by the index, so you can see that the xor can change,
// It should however be constant vertically through your slices.
for i := range shards {
xored[i] = make([]byte, len(shards[i]))
for j := range xored[i] {
xored[i][j] = shards[i][j] ^ byte(j&0xff)
}
}
// Each part should still verify as ok.
ok, err = enc.Verify(xored)
if ok && err == nil {
fmt.Println("verified ok after xor")
}
// Output: verified ok after xor
}
// Receive collects packets. When exactly enough packets have been recovered to
// reconstruct the message, the message is returned as a byte slice. Otherwise
// nil is returned. Receive can continue to collect packets after the message
// has been constructed for reliability statistics.
func (p Packeter) Receive(b []byte) []byte {
var pk Packet
if err.Log(proto.Unmarshal(b, &pk)) {
return nil
}
mid, pid, dataShards, parityShards := pk.MetaData()
clctr, ok := p[mid]
if !ok {
clctr = &collector{
data: make([][]byte, dataShards+parityShards),
collected: make(map[uint32]bool),
complete: false,
}
p[mid] = clctr
}
clctr.collected[pid] = true
if clctr.complete {
return nil
}
clctr.data[pid] = pk.Data
if uint32(len(clctr.collected)) >= dataShards {
clctr.complete = true
if enc, e := reedsolomon.New(int(dataShards), int(parityShards)); err.Warn(e) {
if err.Log(enc.Reconstruct(clctr.data)) {
//maybe should take out as an arg
ln := int(clctr.data[0][0]) + int(clctr.data[0][1])<<8 + int(clctr.data[0][2])<<16 + int(clctr.data[0][2])<<24
var out bytes.Buffer
err.Warn(enc.Join(&out, clctr.data, ln))
clctr.data = nil
return out.Bytes()[4:]
}
}
}
return nil
}