Ejemplo n.º 1
0
// Data buffer to decode must be of the (M/K)*Size given in the constructor.
// The error list must contain M-K values, corresponding to the shards
// with errors (eg. [0, 2, 4, 6]).
// Cache stores the decode matrices in a trie, enabling a faster decode
// with a memory tradeoff.
// The returned decoded data is the orignal data of length Size
func (c *Code) Decode(encoded []byte, errList []byte, cache bool) (recovered []byte) {
	if len(encoded) != c.M*c.ShardLength {
		panic("Data to decode is not the proper size")
	}
	if len(errList) > c.M-c.K {
		panic("Too many errors, cannot decode")
	}
	if len(errList) == 0 {
		recovered = append(recovered, encoded[:c.K*c.ShardLength]...)
	} else {
		node := c.getDecode(errList, cache)

		for i := 0; i < c.K; i++ {
			recovered = append(recovered, encoded[(int(node.decodeIndex[i])*c.ShardLength):int(node.decodeIndex[i]+1)*c.ShardLength]...)
		}

		decoded := make([]byte, c.M*c.ShardLength)
		C.ec_encode_data(C.int(c.ShardLength), C.int(c.K), C.int(c.M), (*C.uchar)(&node.galoisTables[0]), (*C.uchar)(&recovered[0]), (*C.uchar)(&decoded[0]))

		copy(recovered, encoded)

		for i, err := range errList {
			if int(err) < c.K {
				copy(recovered[int(err)*c.ShardLength:int(err+1)*c.ShardLength], decoded[i*c.ShardLength:(i+1)*c.ShardLength])
			}
		}
	}

	return recovered
}
Ejemplo n.º 2
0
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	k := int(e.params.K) // "k" data blocks
	m := int(e.params.M) // "m" parity blocks
	n := k + m           // "n" total encoded blocks

	// Length of a single encoded chunk.
	// Total number of encoded chunks = "k" data  + "m" parity blocks
	encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))

	// Length of total number of "k" data chunks
	encodedDataBlocksLen := encodedBlockLen * k

	// Length of extra padding required for the data blocks.
	encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)

	// Extend inputData buffer to accommodate coded data blocks if necesssary
	if encodedDataBlocksPadLen > 0 {
		padding := make([]byte, encodedDataBlocksPadLen)
		// Expand with new padded blocks to the byte array
		inputData = append(inputData, padding...)
	}

	// Extend inputData buffer to accommodate coded parity blocks
	{ // Local Scope
		encodedParityBlocksLen := encodedBlockLen * m
		parityBlocks := make([]byte, encodedParityBlocksLen)
		inputData = append(inputData, parityBlocks...)
	}

	// Allocate memory to the "encoded blocks" return buffer
	encodedBlocks = make([][]byte, n) // Return buffer

	// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
	// byte array. "encodedBlocks" is a 2D slice.
	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.

	// Copy data block slices to encoded block buffer
	for i := 0; i < k; i++ {
		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Copy erasure block slices to encoded block buffer
	for i := k; i < n; i++ {
		encodedBlocks[i] = make([]byte, encodedBlockLen)
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Erasure code the data into K data blocks and M parity
	// blocks. Only the parity blocks are filled. Data blocks remain
	// intact.
	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks

	return encodedBlocks, nil
}
Ejemplo n.º 3
0
// The data buffer to encode must be of the length Size given in the constructor.
// The returned encoded buffer is (M-K)*Shard length, since the first Size bytes
// of the encoded data is just the original data due to the identity matrix.
func (c *Code) Encode(data []byte) (encoded []byte) {
	if len(data) != c.K*c.ShardLength {
		panic("Data to encode is not the proper size")
	}
	// Since the first k rows of the encode matrix is actually the identity matrix
	// we only need to encode the last m-k shards of the data and append
	// them to the original data
	encoded = make([]byte, (c.M-c.K)*(c.ShardLength))
	C.ec_encode_data(C.int(c.ShardLength), C.int(c.K), C.int(c.M-c.K), (*C.uchar)(&c.galoisTables[0]), (*C.uchar)(&data[0]), (*C.uchar)(&encoded[0]))
	// return append(data, encoded...)
	return encoded
}
Ejemplo n.º 4
0
// EncodeStream erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Erasure) EncodeStream(data io.Reader, size int64) ([][]byte, []byte, error) {
	k := int(e.params.K) // "k" data blocks
	m := int(e.params.M) // "m" parity blocks
	n := k + m           // "n" total encoded blocks

	// Length of a single encoded chunk.
	// Total number of encoded chunks = "k" data  + "m" parity blocks
	encodedBlockLen := GetEncodedBlockLen(int(size), uint8(k))

	// Length of total number of "n" data chunks
	encodedDataBlocksLen := encodedBlockLen * n

	// allocate byte array for encodedBlock length
	inputData := make([]byte, size, encodedDataBlocksLen)

	_, err := io.ReadFull(data, inputData)
	if err != nil {
		// do not check for io.ErrUnexpectedEOF, we know the right amount of size
		// to be read if its a short read we need to throw error since reader could
		// have been prematurely closed.
		if err != io.EOF {
			return nil, nil, err
		}
	}

	// Allocate memory to the "encoded blocks" return buffer
	encodedBlocks := make([][]byte, n) // Return buffer

	// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
	// byte array. "encodedBlocks" is a 2D slice.
	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.

	// Copy data block slices to encoded block buffer
	for i := 0; i < n; i++ {
		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Erasure code the data into K data blocks and M parity
	// blocks. Only the parity blocks are filled. Data blocks remain
	// intact.
	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks

	return encodedBlocks, inputData[0:size], nil
}
Ejemplo n.º 5
0
// Decode decodes erasure coded blocks of data into its original
// form. Erasure coded data contains K data blocks and M parity
// blocks. Decode can withstand data loss up to any M number of blocks.
//
// "encodedDataBlocks" is an array of K data blocks and M parity
// blocks. Data blocks are position and order dependent. Missing blocks
// are set to "nil". There must be at least "K" number of data|parity
// blocks.
//
// "dataLen" is the length of original source data
func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	var source, target **C.uchar

	k := int(e.params.K)
	m := int(e.params.M)
	n := k + m
	// We need the data and parity blocks preserved in the same order. Missing blocks are set to nil.
	if len(encodedDataBlocks) != n {
		msg := fmt.Sprintf("Encoded data blocks slice must of length [%d]", n)
		return nil, errors.New(msg)
	}

	// Length of a single encoded block
	encodedBlockLen := GetEncodedBlockLen(dataLen, uint8(k))

	// Keep track of errors per block.
	missingEncodedBlocks := make([]int, n+1)
	var missingEncodedBlocksCount int

	// Check for the missing encoded blocks
	for i := range encodedDataBlocks {
		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
			missingEncodedBlocks[missingEncodedBlocksCount] = i
			missingEncodedBlocksCount++
		}
	}

	// Cannot reconstruct original data. Need at least M number of data or parity blocks.
	if missingEncodedBlocksCount > m {
		return nil, fmt.Errorf("Cannot reconstruct original data. Need at least [%d]  data or parity blocks", m)
	}

	// Convert from Go int slice to C int array
	missingEncodedBlocksC := intSlice2CIntArray(missingEncodedBlocks[:missingEncodedBlocksCount])

	// Allocate buffer for the missing blocks
	for i := range encodedDataBlocks {
		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
			encodedDataBlocks[i] = make([]byte, encodedBlockLen)
		}
	}

	// If not already initialized, recompute and cache
	if e.decodeMatrix == nil || e.decodeTbls == nil || e.decodeIndex == nil {
		var decodeMatrix, decodeTbls *C.uchar
		var decodeIndex *C.uint32_t

		C.minio_init_decoder(missingEncodedBlocksC, C.int(k), C.int(n), C.int(missingEncodedBlocksCount),
			e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)

		// cache this for future needs
		e.decodeMatrix = decodeMatrix
		e.decodeTbls = decodeTbls
		e.decodeIndex = decodeIndex
	}

	// Make a slice of pointers to encoded blocks. Necessary to bridge to the C world.
	pointers := make([]*byte, n)
	for i := range encodedDataBlocks {
		pointers[i] = &encodedDataBlocks[i][0]
	}

	// Get pointers to source "data" and target "parity" blocks from the output byte array.
	ret := C.minio_get_source_target(C.int(missingEncodedBlocksCount), C.int(k), C.int(m), missingEncodedBlocksC,
		e.decodeIndex, (**C.uchar)(unsafe.Pointer(&pointers[0])), &source, &target)
	if int(ret) == -1 {
		return nil, errors.New("Unable to decode data")
	}

	// Decode data
	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(missingEncodedBlocksCount), e.decodeTbls,
		source, target)

	// Allocate buffer to output buffer
	decodedData = make([]byte, 0, encodedBlockLen*int(k))
	for i := 0; i < int(k); i++ {
		decodedData = append(decodedData, encodedDataBlocks[i]...)
	}

	return decodedData[:dataLen], nil
}