// FileReader supports concurrent file reading
// multiple readres are allowed
func FileReader(salt []byte, fileStream <-chan FileInterval, path string, wgroup *sync.WaitGroup, unorderedStream chan<- HashedDataInterval) {
// open file
file, err := fileOpen(path, os.O_RDONLY, 0)
if err != nil {
log.Fatal("Failed to open file for reading:", string(path), err)
}
defer file.Close()
for r := range fileStream {
switch r.Kind {
case SparseHole:
// Process hole
var hash, data []byte
unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}
case SparseData:
// Read file data
data := make([]byte, r.Len())
status := true
n, err := fileReadAt(file, data, r.Begin)
if err != nil {
status = false
log.Error("File read error", status)
} else if int64(n) != r.Len() {
status = false
log.Error("File read underrun")
}
hasher := sha1.New()
hasher.Write(salt)
hasher.Write(data)
hash := hasher.Sum(nil)
unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}
}
}
wgroup.Done() // indicate to other readers we are done
}
// SyncFile synchronizes local file to remote host
func SyncFile(localPath string, addr TCPEndPoint, remotePath string, timeout int) (hashLocal []byte, err error) {
for retries := 1; retries >= 0; retries-- {
hashLocal, err = syncFile(localPath, addr, remotePath, timeout, retries > 0)
if err != nil {
if _, ok := err.(*HashCollsisionError); ok {
// retry on HahsCollisionError
log.Warn("SSync: retrying on chunk hash collision...")
continue
} else {
log.Error("SSync error:", err)
}
}
break
}
return
}
// returns true if no retry is necessary
func serveConnection(conn net.Conn) bool {
defer conn.Close()
decoder := gob.NewDecoder(conn)
var request requestHeader
err := decoder.Decode(&request)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
if requestMagic != request.Magic {
log.Error("Bad request")
return true
}
switch request.Code {
case syncRequestCode:
var path string
err := decoder.Decode(&path)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
var size int64
err = decoder.Decode(&size)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
var salt []byte
err = decoder.Decode(&salt)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
encoder := gob.NewEncoder(conn)
return serveSyncRequest(encoder, decoder, path, size, salt)
}
return true
}
func syncFile(localPath string, addr TCPEndPoint, remotePath string, timeout int, retry bool) ([]byte, error) {
file, err := fio.OpenFile(localPath, os.O_RDONLY, 0)
if err != nil {
log.Error("Failed to open local source file:", localPath)
return nil, err
}
defer file.Close()
size, err := file.Seek(0, os.SEEK_END)
if err != nil {
log.Error("Failed to get size of local source file:", localPath, err)
return nil, err
}
SetupFileIO(size%Blocks == 0)
conn := connect(addr.Host, strconv.Itoa(int(addr.Port)), timeout)
if nil == conn {
err = fmt.Errorf("Failed to connect to %v", addr)
log.Error(err)
return nil, err
}
defer conn.Close()
encoder := gob.NewEncoder(conn)
decoder := gob.NewDecoder(conn)
// Use unix time as hash salt
salt := make([]byte, binary.MaxVarintLen64)
binary.PutVarint(salt, time.Now().UnixNano())
status := sendSyncRequest(encoder, decoder, remotePath, size, salt)
if !status {
err = fmt.Errorf("Sync request to %v failed", remotePath)
log.Error(err)
return nil, err
}
abortStream := make(chan error)
layoutStream := make(chan FileInterval, 128)
errStream := make(chan error)
// Initiate interval loading...
err = loadFileLayout(abortStream, file, layoutStream, errStream)
if err != nil {
log.Error("Failed to retrieve local file layout:", err)
return nil, err
}
fileStream := make(chan FileInterval, 128)
unorderedStream := make(chan HashedDataInterval, 128)
orderedStream := make(chan HashedDataInterval, 128)
go IntervalSplitter(layoutStream, fileStream)
FileReaderGroup(fileReaders, salt, fileStream, localPath, unorderedStream)
go OrderIntervals("src:", unorderedStream, orderedStream)
// Get remote file intervals and their hashes
netInStream := make(chan HashedInterval, 128)
netInStreamDone := make(chan bool)
go netDstReceiver(decoder, netInStream, netInStreamDone)
return processDiff(salt, abortStream, errStream, encoder, decoder, orderedStream, netInStream, netInStreamDone, retry)
}
func processDiff(salt []byte, abortStream chan<- error, errStream <-chan error, encoder *gob.Encoder, decoder *gob.Decoder, local <-chan HashedDataInterval, remote <-chan HashedInterval, netInStreamDone <-chan bool, retry bool) (hashLocal []byte, err error) {
// Local: __ _*
// Remote: *_ **
hashLocal = make([]byte, 0) // empty hash for errors
const concurrentReaders = 4
netStream := make(chan diffChunk, 128)
netStatus := make(chan netXferStatus)
go networkSender(netStream, encoder, netStatus)
fileHasher := sha1.New()
fileHasher.Write(salt)
lrange := <-local
rrange := <-remote
for lrange.Len() != 0 {
if rrange.Len() == 0 {
// Copy local tail
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{SparseHole, lrange.Interval}, make([]byte, 0)}, netStream)
lrange = <-local
continue
}
// Diff
if verboseClient {
log.Debug("Diff:", lrange.HashedInterval, rrange)
}
if lrange.Begin == rrange.Begin {
if lrange.End > rrange.End {
data := lrange.Data
if len(data) > 0 {
data = lrange.Data[:rrange.Len()]
}
subrange := HashedDataInterval{HashedInterval{FileInterval{lrange.Kind, rrange.Interval}, lrange.Hash}, data}
if verboseClient {
logData("LHASH", subrange.Data)
}
hashFileData(fileHasher, subrange.Len(), subrange.Data)
processFileInterval(subrange, rrange, netStream)
if len(data) > 0 {
lrange.Data = lrange.Data[subrange.Len():]
}
lrange.Begin = rrange.End
rrange = <-remote
continue
} else if lrange.End < rrange.End {
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{rrange.Kind, lrange.Interval}, make([]byte, 0)}, netStream)
rrange.Begin = lrange.End
lrange = <-local
continue
}
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, rrange, netStream)
lrange = <-local
rrange = <-remote
} else {
// Should never happen
log.Fatal("processDiff internal error")
return
}
}
log.Info("Finished processing file diff")
status := true
err = <-errStream
if err != nil {
log.Error("Sync client file load aborted:", err)
status = false
}
// make sure we finished consuming dst hashes
status = <-netInStreamDone && status // netDstReceiver finished
log.Info("Finished consuming remote file hashes, status=", status)
// Send end of transmission
netStream <- diffChunk{true, DataInterval{FileInterval{SparseIgnore, Interval{0, 0}}, make([]byte, 0)}}
// get network sender status
net := <-netStatus
log.Info("Finished sending file diff of", net.byteCount, "(bytes), status=", net.status)
if !net.status {
err = errors.New("netwoek transfer failure")
return
}
var statusRemote bool
err = decoder.Decode(&statusRemote)
if err != nil {
log.Fatal("Cient protocol remote status error:", err)
return
}
if !statusRemote {
err = errors.New("failure on remote sync site")
return
}
var hashRemote []byte
err = decoder.Decode(&hashRemote)
if err != nil {
log.Fatal("Cient protocol remote hash error:", err)
return
}
// Compare file hashes
hashLocal = fileHasher.Sum(nil)
if isHashDifferent(hashLocal, hashRemote) || FailPointFileHashMatch() {
log.Warn("hashLocal =", hashLocal)
log.Warn("hashRemote=", hashRemote)
err = &HashCollsisionError{}
} else {
retry = false // success, don't retry anymore
}
// Final retry negotiation
{
err1 := encoder.Encode(retry)
if err1 != nil {
log.Fatal("Cient protocol remote retry error:", err)
}
err1 = decoder.Decode(&statusRemote)
if err1 != nil {
log.Fatal("Cient protocol remote retry status error:", err)
}
}
return
}
// returns true if no retry is necessary
func serveSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
directFileIO := size%Blocks == 0
SetupFileIO(directFileIO)
// Open destination file
file, err := fileOpen(path, os.O_RDWR, 0666)
if err != nil {
file, err = os.Create(path)
if err != nil {
log.Error("Failed to create file:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
}
// Setup close sequence
if directFileIO {
defer file.Sync()
}
defer file.Close()
// Resize the file
if err = file.Truncate(size); err != nil {
log.Error("Failed to resize file:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
// open file
fileRO, err := fileOpen(path, os.O_RDONLY, 0)
if err != nil {
log.Error("Failed to open file for reading:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
defer fileRO.Close()
abortStream := make(chan error)
layoutStream := make(chan FileInterval, 128)
errStream := make(chan error)
fileStream := make(chan FileInterval, 128)
unorderedStream := make(chan HashedDataInterval, 128)
orderedStream := make(chan HashedDataInterval, 128)
netOutStream := make(chan HashedInterval, 128)
netOutDoneStream := make(chan bool)
netInStream := make(chan DataInterval, 128)
fileWriteStream := make(chan DataInterval, 128)
deltaReceiverDoneDone := make(chan bool)
checksumStream := make(chan DataInterval, 128)
resultStream := make(chan []byte)
// Initiate interval loading...
err = loadFileLayout(abortStream, fileRO, layoutStream, errStream)
if err != nil {
encoder.Encode(false) // NACK request
return true
}
encoder.Encode(true) // ACK request
go IntervalSplitter(layoutStream, fileStream)
FileReaderGroup(fileReaders, salt, fileStream, path, unorderedStream)
go OrderIntervals("dst:", unorderedStream, orderedStream)
go Tee(orderedStream, netOutStream, checksumStream)
// Send layout along with data hashes
go netSender(netOutStream, encoder, netOutDoneStream)
// Start receiving deltas, write those and compute file hash
fileWritten := FileWriterGroup(fileWriters, fileWriteStream, path)
go netReceiver(decoder, file, netInStream, fileWriteStream, deltaReceiverDoneDone) // receiver and checker
go Validator(salt, checksumStream, netInStream, resultStream)
// Block till completion
status := true
err = <-errStream // Done with file loadaing, possibly aborted on error
if err != nil {
log.Error("Sync server file load aborted:", err)
status = false
}
status = <-netOutDoneStream && status // done sending dst hashes
status = <-deltaReceiverDoneDone && status // done writing dst file
fileWritten.Wait() // wait for write stream completion
hash := <-resultStream // done processing diffs
// reply to client with status
log.Info("Sync sending server status=", status)
err = encoder.Encode(status)
if err != nil {
log.Fatal("Protocol encoder error:", err)
return true
}
// reply with local hash
err = encoder.Encode(hash)
if err != nil {
log.Fatal("Protocol encoder error:", err)
return true
}
var retry bool
err = decoder.Decode(&retry)
if err != nil {
log.Fatal("Protocol retry decoder error:", err)
return true
}
encoder.Encode(true) // ACK retry
if err != nil {
log.Fatal("Protocol retry encoder error:", err)
return true
}
return !retry // don't terminate server if retry expected
}
