// Get remote hashed intervals
func netDstReceiver(decoder *gob.Decoder, netInStream chan<- HashedInterval, netInStreamDone chan<- bool) {
status := true
for {
if verboseClient {
log.Debug("Client.netDstReceiver decoding...")
}
var r HashedInterval
err := decoder.Decode(&r)
if err != nil {
log.Fatal("Cient protocol error:", err)
status = false
break
}
// interval := r.Interval
if r.Kind == SparseIgnore {
if verboseClient {
log.Debug("Client.netDstReceiver got <eof>")
}
break
}
if verboseClient {
switch r.Kind {
case SparseData:
log.Debug("Client.netDstReceiver got data", r.FileInterval, "hash[", len(r.Hash), "]")
case SparseHole:
log.Debug("Client.netDstReceiver got hole", r.FileInterval)
}
}
netInStream <- r
}
close(netInStream)
netInStreamDone <- status
}
func netSender(netOutStream <-chan HashedInterval, encoder *gob.Encoder, netOutDoneStream chan<- bool) {
for r := range netOutStream {
if verboseServer {
log.Debug("Server.netSender: sending", r.FileInterval)
}
err := encoder.Encode(r)
if err != nil {
log.Fatal("Protocol encoder error:", err)
netOutDoneStream <- false
return
}
}
rEOF := HashedInterval{FileInterval{SparseIgnore, Interval{}}, make([]byte, 0)}
if rEOF.Len() != 0 {
log.Fatal("Server.netSender internal error")
}
// err := encoder.Encode(HashedInterval{FileInterval{}, make([]byte, 0)})
err := encoder.Encode(rEOF)
if err != nil {
log.Fatal("Protocol encoder error:", err)
netOutDoneStream <- false
return
}
if verboseServer {
log.Debug("Server.netSender: finished sending hashes")
}
netOutDoneStream <- true
}
// IntervalSplitter limits file intervals to predefined batch size
func IntervalSplitter(spltterStream <-chan FileInterval, fileStream chan<- FileInterval) {
const batch = 32 * Blocks
for r := range spltterStream {
if verboseServer {
log.Debug("Interval Splitter:", r)
}
switch r.Kind {
case SparseHole:
// Process hole
fileStream <- r
case SparseData:
// Process data in chunks
for offset := r.Begin; offset < r.End; {
size := batch
if offset+size > r.End {
size = r.End - offset
}
interval := Interval{offset, offset + size}
if size == batch && interval.End%batch != 0 {
interval.End = interval.End / batch * batch
}
log.Debug("Interval Splitter data:", interval)
fileStream <- FileInterval{SparseData, interval}
offset += interval.Len()
}
}
}
close(fileStream)
}
// OrderIntervals puts back "out of order" read results
func OrderIntervals(prefix string, unorderedStream <-chan HashedDataInterval, orderedStream chan<- HashedDataInterval) {
pos := int64(0)
m := make(map[int64]HashedDataInterval) // out of order completions
for r := range unorderedStream {
if pos == r.Begin {
// Handle "in order" range
log.Debug(prefix, r)
orderedStream <- r
pos = r.End
} else {
// push "out of order"" range
m[r.Begin] = r
}
// check the "out of order" stash for "in order"
for pop, existsNext := m[pos]; len(m) > 0 && existsNext; pop, existsNext = m[pos] {
// pop in order range
log.Debug(prefix, pop)
orderedStream <- pop
delete(m, pos)
pos = pop.End
}
}
close(orderedStream)
}
// FileWriter supports concurrent file reading
// add this writer to wgroup before invoking
func FileWriter(fileStream <-chan DataInterval, path string, wgroup *sync.WaitGroup) {
// open file
file, err := fileOpen(path, os.O_WRONLY, 0)
if err != nil {
log.Fatal("Failed to open file for wroting:", string(path), err)
}
defer file.Close()
for r := range fileStream {
switch r.Kind {
case SparseHole:
log.Debug("trimming...")
err := PunchHole(file, r.Interval)
if err != nil {
log.Fatal("Failed to trim file")
}
case SparseData:
log.Debug("writing data...")
_, err = fileWriteAt(file, r.Data, r.Begin)
if err != nil {
log.Fatal("Failed to write file")
}
}
}
wgroup.Done()
}
func logData(prefix string, data []byte) {
size := len(data)
if size > 0 {
log.Debug("\t", prefix, "of", size, "bytes", data[0], "...")
} else {
log.Debug("\t", prefix, "of", size, "bytes")
}
}
func netReceiver(decoder *gob.Decoder, file *os.File, netInStream chan<- DataInterval, fileStream chan<- DataInterval, deltaReceiverDone chan<- bool) {
// receive & process data diff
status := true
for status {
var delta FileInterval
err := decoder.Decode(&delta)
if err != nil {
log.Fatal("Protocol decoder error:", err)
status = false
break
}
log.Debug("receiving delta [", delta, "]")
if 0 == delta.Len() {
log.Debug("received end of transimission marker")
break // end of diff
}
switch delta.Kind {
case SparseData:
// Receive data
var data []byte
err = decoder.Decode(&data)
if err != nil {
log.Fatal("Protocol data decoder error:", err)
status = false
break
}
if int64(len(data)) != delta.Len() {
log.Fatal("Failed to receive data, expected=", delta.Len(), "received=", len(data))
status = false
break
}
// Push for writing and vaildator processing
fileStream <- DataInterval{delta, data}
netInStream <- DataInterval{delta, data}
case SparseHole:
// Push for writing and vaildator processing
fileStream <- DataInterval{delta, make([]byte, 0)}
netInStream <- DataInterval{delta, make([]byte, 0)}
case SparseIgnore:
// Push for vaildator processing
netInStream <- DataInterval{delta, make([]byte, 0)}
log.Debug("ignoring...")
}
}
log.Debug("Server.netReceiver done, sync")
close(netInStream)
close(fileStream)
deltaReceiverDone <- status
}
// RetrieveLayoutStream streams sparse file data/hole layout
// Based on fiemap
// To abort: abortStream <- error
// Check status: err := <- errStream
// Usage: go RetrieveLayoutStream(...)
func RetrieveLayoutStream(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error) {
const extents = 1024
const chunkSizeMax = 1 /*GB*/ << 30
chunkSize := r.Len()
if chunkSize > chunkSizeMax {
chunkSize = chunkSizeMax
}
chunk := Interval{r.Begin, r.Begin + chunkSize}
// Process file extents for each chunk
intervalLast := Interval{chunk.Begin, chunk.Begin}
for chunk.Begin < r.End {
if chunk.End > r.End {
chunk.End = r.End
}
for more := true; more && chunk.Len() > 0; {
ext, errno := fibmap.Fiemap(file.Fd(), uint64(chunk.Begin), uint64(chunk.Len()), 1024)
if errno != 0 {
close(layoutStream)
errStream <- &os.PathError{Op: "Fiemap", Path: file.Name(), Err: errno}
return
}
if len(ext) == 0 {
break
}
// Process each extent
for _, e := range ext {
interval := Interval{int64(e.Logical), int64(e.Logical + e.Length)}
log.Debug("Extent:", interval, e.Flags)
if e.Flags&fibmap.FIEMAP_EXTENT_LAST != 0 {
more = false
}
if intervalLast.End < interval.Begin {
if intervalLast.Len() > 0 {
// Pop last Data
layoutStream <- FileInterval{SparseData, intervalLast}
}
// report hole
intervalLast = Interval{intervalLast.End, interval.Begin}
layoutStream <- FileInterval{SparseHole, intervalLast}
// Start data
intervalLast = interval
} else {
// coalesce
intervalLast.End = interval.End
}
chunk.Begin = interval.End
}
}
chunk = Interval{chunk.End, chunk.End + chunkSize}
}
if intervalLast.Len() > 0 {
// Pop last Data
if intervalLast.End > r.End {
intervalLast.End = r.End
}
layoutStream <- FileInterval{SparseData, intervalLast}
}
if intervalLast.End < r.End {
// report hole
layoutStream <- FileInterval{SparseHole, Interval{intervalLast.End, r.End}}
}
close(layoutStream)
errStream <- nil
return
}
func networkSender(netStream <-chan diffChunk, encoder *gob.Encoder, netStatus chan<- netXferStatus) {
status := true
byteCount := int64(0)
for {
chunk := <-netStream
if 0 == chunk.header.Len() {
// eof: last 0 len header
if verboseClient {
log.Debug("Client.networkSender <eof>")
}
err := encoder.Encode(chunk.header.FileInterval)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
}
break
}
if !status {
// network error
continue // discard the chunk
}
if !chunk.status {
// read error
status = false
continue // discard the chunk
}
if traceChannelLoad {
fmt.Fprint(os.Stderr, len(netStream), "n")
}
// Encode and send data to the network
if verboseClient {
log.Debug("Client.networkSender sending:", chunk.header.FileInterval)
}
err := encoder.Encode(chunk.header.FileInterval)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
continue
}
if len(chunk.header.Data) == 0 {
continue
}
if verboseClient {
log.Debug("Client.networkSender sending data")
}
if int64(len(chunk.header.Data)) != chunk.header.FileInterval.Len() {
log.Fatal("Client.networkSender sending data internal error:", chunk.header.FileInterval.Len(), len(chunk.header.Data))
}
err = encoder.Encode(chunk.header.Data)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
continue
}
byteCount += int64(len(chunk.header.Data))
if traceChannelLoad {
fmt.Fprint(os.Stderr, "N\n")
}
}
netStatus <- netXferStatus{status, byteCount}
}
func processDiff(salt []byte, abortStream chan<- error, errStream <-chan error, encoder *gob.Encoder, decoder *gob.Decoder, local <-chan HashedDataInterval, remote <-chan HashedInterval, netInStreamDone <-chan bool, retry bool) (hashLocal []byte, err error) {
// Local: __ _*
// Remote: *_ **
hashLocal = make([]byte, 0) // empty hash for errors
const concurrentReaders = 4
netStream := make(chan diffChunk, 128)
netStatus := make(chan netXferStatus)
go networkSender(netStream, encoder, netStatus)
fileHasher := sha1.New()
fileHasher.Write(salt)
lrange := <-local
rrange := <-remote
for lrange.Len() != 0 {
if rrange.Len() == 0 {
// Copy local tail
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{SparseHole, lrange.Interval}, make([]byte, 0)}, netStream)
lrange = <-local
continue
}
// Diff
if verboseClient {
log.Debug("Diff:", lrange.HashedInterval, rrange)
}
if lrange.Begin == rrange.Begin {
if lrange.End > rrange.End {
data := lrange.Data
if len(data) > 0 {
data = lrange.Data[:rrange.Len()]
}
subrange := HashedDataInterval{HashedInterval{FileInterval{lrange.Kind, rrange.Interval}, lrange.Hash}, data}
if verboseClient {
logData("LHASH", subrange.Data)
}
hashFileData(fileHasher, subrange.Len(), subrange.Data)
processFileInterval(subrange, rrange, netStream)
if len(data) > 0 {
lrange.Data = lrange.Data[subrange.Len():]
}
lrange.Begin = rrange.End
rrange = <-remote
continue
} else if lrange.End < rrange.End {
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{rrange.Kind, lrange.Interval}, make([]byte, 0)}, netStream)
rrange.Begin = lrange.End
lrange = <-local
continue
}
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, rrange, netStream)
lrange = <-local
rrange = <-remote
} else {
// Should never happen
log.Fatal("processDiff internal error")
return
}
}
log.Info("Finished processing file diff")
status := true
err = <-errStream
if err != nil {
log.Error("Sync client file load aborted:", err)
status = false
}
// make sure we finished consuming dst hashes
status = <-netInStreamDone && status // netDstReceiver finished
log.Info("Finished consuming remote file hashes, status=", status)
// Send end of transmission
netStream <- diffChunk{true, DataInterval{FileInterval{SparseIgnore, Interval{0, 0}}, make([]byte, 0)}}
// get network sender status
net := <-netStatus
log.Info("Finished sending file diff of", net.byteCount, "(bytes), status=", net.status)
if !net.status {
err = errors.New("netwoek transfer failure")
return
}
var statusRemote bool
err = decoder.Decode(&statusRemote)
if err != nil {
log.Fatal("Cient protocol remote status error:", err)
return
}
if !statusRemote {
err = errors.New("failure on remote sync site")
return
}
var hashRemote []byte
err = decoder.Decode(&hashRemote)
if err != nil {
log.Fatal("Cient protocol remote hash error:", err)
return
}
// Compare file hashes
hashLocal = fileHasher.Sum(nil)
if isHashDifferent(hashLocal, hashRemote) || FailPointFileHashMatch() {
log.Warn("hashLocal =", hashLocal)
log.Warn("hashRemote=", hashRemote)
err = &HashCollsisionError{}
} else {
retry = false // success, don't retry anymore
}
// Final retry negotiation
{
err1 := encoder.Encode(retry)
if err1 != nil {
log.Fatal("Cient protocol remote retry error:", err)
}
err1 = decoder.Decode(&statusRemote)
if err1 != nil {
log.Fatal("Cient protocol remote retry status error:", err)
}
}
return
}
// Validator merges source and diff data; produces hash of the destination file
func Validator(salt []byte, checksumStream, netInStream <-chan DataInterval, resultStream chan<- []byte) {
fileHasher := sha1.New()
//TODO: error handling
fileHasher.Write(salt)
r := <-checksumStream // original dst file data
q := <-netInStream // diff data
for q.Len() != 0 || r.Len() != 0 {
if r.Len() == 0 /*end of dst file*/ {
// Hash diff data
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
q = <-netInStream
} else if q.Len() == 0 /*end of diff file*/ {
// Hash original data
if verboseServer {
logData("RHASH", r.Data)
}
hashFileData(fileHasher, r.Len(), r.Data)
r = <-checksumStream
} else {
qi := q.Interval
ri := r.Interval
if qi.Begin == ri.Begin {
if qi.End > ri.End {
log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
} else if qi.End < ri.End {
// Hash diff data
if verboseServer {
log.Debug("Server.Validator: hashing diff", q.FileInterval, r.FileInterval)
}
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
r.Begin = q.End
q = <-netInStream
} else {
if q.Kind == SparseIgnore {
// Hash original data
if verboseServer {
log.Debug("Server.Validator: hashing original", r.FileInterval)
}
if verboseServer {
logData("RHASH", r.Data)
}
hashFileData(fileHasher, r.Len(), r.Data)
} else {
// Hash diff data
if verboseServer {
log.Debug("Server.Validator: hashing diff", q.FileInterval)
}
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
}
q = <-netInStream
r = <-checksumStream
}
} else {
log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
}
}
}
if verboseServer {
log.Debug("Server.Validator: finished")
}
resultStream <- fileHasher.Sum(nil)
}