func coalesce(parentFile *os.File, childFile *os.File) error {
blockSize, err := getFileSystemBlockSize(childFile)
if err != nil {
panic("can't get FS block size, error: " + err.Error())
}
var data, hole int64
for {
data, err = syscall.Seek(int(childFile.Fd()), hole, seekData)
if err != nil {
// reaches EOF
errno := err.(syscall.Errno)
if errno == syscall.ENXIO {
break
} else {
// unexpected errors
log.Fatal("Failed to syscall.Seek SEEK_DATA")
return err
}
}
hole, err = syscall.Seek(int(childFile.Fd()), data, seekHole)
if err != nil {
log.Fatal("Failed to syscall.Seek SEEK_HOLE")
return err
}
// now we have a data start offset and length(hole - data)
// let's read from child and write to parent file block by block
_, err = parentFile.Seek(data, os.SEEK_SET)
if err != nil {
log.Fatal("Failed to os.Seek os.SEEK_SET")
return err
}
offset := data
buffer := fio.AllocateAligned(blockSize)
for offset != hole {
// read a block from child, maybe use bufio or Reader stream
n, err := fio.ReadAt(childFile, buffer, offset)
if n != len(buffer) || err != nil {
log.Fatal("Failed to read from childFile")
return err
}
// write a block to parent
n, err = fio.WriteAt(parentFile, buffer, offset)
if n != len(buffer) || err != nil {
log.Fatal("Failed to write to parentFile")
return err
}
offset += int64(n)
}
}
return nil
}
func sendSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
err := encoder.Encode(requestHeader{requestMagic, syncRequestCode})
if err != nil {
log.Fatal("Client protocol encoder error:", err)
return false
}
err = encoder.Encode(path)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
return false
}
err = encoder.Encode(size)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
return false
}
err = encoder.Encode(salt)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
return false
}
var ack bool
err = decoder.Decode(&ack)
if err != nil {
log.Fatal("Client protocol decoder error:", err)
return false
}
return ack
}
func server(addr TCPEndPoint, serveOnce /*test flag*/ bool, timeout int) {
serverConnectionTimeout := time.Duration(timeout) * time.Second
// listen on all interfaces
EndPoint := addr.Host + ":" + strconv.Itoa(int(addr.Port))
laddr, err := net.ResolveTCPAddr("tcp", EndPoint)
if err != nil {
log.Fatal("Connection listener address resolution error:", err)
}
ln, err := net.ListenTCP("tcp", laddr)
if err != nil {
log.Fatal("Connection listener error:", err)
}
defer ln.Close()
ln.SetDeadline(time.Now().Add(serverConnectionTimeout))
log.Info("Sync server is up...")
for {
conn, err := ln.AcceptTCP()
if err != nil {
log.Fatal("Connection accept error:", err)
}
if serveOnce {
// This is to avoid server listening port conflicts while running tests
// exit after single connection request
if serveConnection(conn) {
break // no retries
}
log.Warn("Server: waiting for client sync retry...")
} else {
go serveConnection(conn)
}
}
log.Info("Sync server exit.")
}
func netSender(netOutStream <-chan HashedInterval, encoder *gob.Encoder, netOutDoneStream chan<- bool) {
for r := range netOutStream {
if verboseServer {
log.Debug("Server.netSender: sending", r.FileInterval)
}
err := encoder.Encode(r)
if err != nil {
log.Fatal("Protocol encoder error:", err)
netOutDoneStream <- false
return
}
}
rEOF := HashedInterval{FileInterval{SparseIgnore, Interval{}}, make([]byte, 0)}
if rEOF.Len() != 0 {
log.Fatal("Server.netSender internal error")
}
// err := encoder.Encode(HashedInterval{FileInterval{}, make([]byte, 0)})
err := encoder.Encode(rEOF)
if err != nil {
log.Fatal("Protocol encoder error:", err)
netOutDoneStream <- false
return
}
if verboseServer {
log.Debug("Server.netSender: finished sending hashes")
}
netOutDoneStream <- true
}
// FileWriter supports concurrent file reading
// add this writer to wgroup before invoking
func FileWriter(fileStream <-chan DataInterval, path string, wgroup *sync.WaitGroup) {
// open file
file, err := fileOpen(path, os.O_WRONLY, 0)
if err != nil {
log.Fatal("Failed to open file for wroting:", string(path), err)
}
defer file.Close()
for r := range fileStream {
switch r.Kind {
case SparseHole:
log.Debug("trimming...")
err := PunchHole(file, r.Interval)
if err != nil {
log.Fatal("Failed to trim file")
}
case SparseData:
log.Debug("writing data...")
_, err = fileWriteAt(file, r.Data, r.Begin)
if err != nil {
log.Fatal("Failed to write file")
}
}
}
wgroup.Done()
}
func netReceiver(decoder *gob.Decoder, file *os.File, netInStream chan<- DataInterval, fileStream chan<- DataInterval, deltaReceiverDone chan<- bool) {
// receive & process data diff
status := true
for status {
var delta FileInterval
err := decoder.Decode(&delta)
if err != nil {
log.Fatal("Protocol decoder error:", err)
status = false
break
}
log.Debug("receiving delta [", delta, "]")
if 0 == delta.Len() {
log.Debug("received end of transimission marker")
break // end of diff
}
switch delta.Kind {
case SparseData:
// Receive data
var data []byte
err = decoder.Decode(&data)
if err != nil {
log.Fatal("Protocol data decoder error:", err)
status = false
break
}
if int64(len(data)) != delta.Len() {
log.Fatal("Failed to receive data, expected=", delta.Len(), "received=", len(data))
status = false
break
}
// Push for writing and vaildator processing
fileStream <- DataInterval{delta, data}
netInStream <- DataInterval{delta, data}
case SparseHole:
// Push for writing and vaildator processing
fileStream <- DataInterval{delta, make([]byte, 0)}
netInStream <- DataInterval{delta, make([]byte, 0)}
case SparseIgnore:
// Push for vaildator processing
netInStream <- DataInterval{delta, make([]byte, 0)}
log.Debug("ignoring...")
}
}
log.Debug("Server.netReceiver done, sync")
close(netInStream)
close(fileStream)
deltaReceiverDone <- status
}
// Get remote hashed intervals
func netDstReceiver(decoder *gob.Decoder, netInStream chan<- HashedInterval, netInStreamDone chan<- bool) {
status := true
for {
if verboseClient {
log.Debug("Client.netDstReceiver decoding...")
}
var r HashedInterval
err := decoder.Decode(&r)
if err != nil {
log.Fatal("Cient protocol error:", err)
status = false
break
}
// interval := r.Interval
if r.Kind == SparseIgnore {
if verboseClient {
log.Debug("Client.netDstReceiver got <eof>")
}
break
}
if verboseClient {
switch r.Kind {
case SparseData:
log.Debug("Client.netDstReceiver got data", r.FileInterval, "hash[", len(r.Hash), "]")
case SparseHole:
log.Debug("Client.netDstReceiver got hole", r.FileInterval)
}
}
netInStream <- r
}
close(netInStream)
netInStreamDone <- status
}
// returns true if no retry is necessary
func serveConnection(conn net.Conn) bool {
defer conn.Close()
decoder := gob.NewDecoder(conn)
var request requestHeader
err := decoder.Decode(&request)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
if requestMagic != request.Magic {
log.Error("Bad request")
return true
}
switch request.Code {
case syncRequestCode:
var path string
err := decoder.Decode(&path)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
var size int64
err = decoder.Decode(&size)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
var salt []byte
err = decoder.Decode(&salt)
if err != nil {
log.Fatal("Protocol decoder error:", err)
return true
}
encoder := gob.NewEncoder(conn)
return serveSyncRequest(encoder, decoder, path, size, salt)
}
return true
}
func processFileInterval(local HashedDataInterval, remote HashedInterval, netStream chan<- diffChunk) {
if local.Interval != remote.Interval {
log.Fatal("Sync.processFileInterval range internal error:", local.FileInterval, remote.FileInterval)
}
if local.Kind != remote.Kind {
// Different intreval types, send the diff
if local.Kind == SparseData && int64(len(local.Data)) != local.FileInterval.Len() {
log.Fatal("Sync.processFileInterval data internal error:", local.FileInterval.Len(), len(local.Data))
}
netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
return
}
// The interval types are the same
if SparseHole == local.Kind {
// Process hole, no syncronization is required
local.Kind = SparseIgnore
netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
return
}
if local.Kind != SparseData {
log.Fatal("Sync.processFileInterval kind internal error:", local.FileInterval)
}
// Data file interval
if isHashDifferent(local.Hash, remote.Hash) {
if int64(len(local.Data)) != local.FileInterval.Len() {
log.Fatal("Sync.processFileInterval internal error:", local.FileInterval.Len(), len(local.Data))
}
netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
return
}
// No diff, just communicate we processed it
//TODO: this apparently can be avoided but requires revision of the protocol
local.Kind = SparseIgnore
netStream <- diffChunk{true, DataInterval{local.FileInterval, make([]byte, 0)}}
}
func connect(host, port string, timeout int) net.Conn {
// connect to this socket
endpoint := host + ":" + port
raddr, err := net.ResolveTCPAddr("tcp", endpoint)
if err != nil {
log.Fatal("Connection address resolution error:", err)
}
timeStart := time.Now()
timeStop := timeStart.Add(time.Duration(timeout) * time.Second)
for timeNow := timeStart; timeNow.Before(timeStop); timeNow = time.Now() {
conn, err := net.DialTCP("tcp", nil, raddr)
if err == nil {
return conn
}
log.Warn("Failed connection to", endpoint, "Retrying...")
if timeNow != timeStart {
// only sleep after the second attempt to speedup tests
time.Sleep(1 * time.Second)
}
}
return nil
}
// FileReader supports concurrent file reading
// multiple readres are allowed
func FileReader(salt []byte, fileStream <-chan FileInterval, path string, wgroup *sync.WaitGroup, unorderedStream chan<- HashedDataInterval) {
// open file
file, err := fileOpen(path, os.O_RDONLY, 0)
if err != nil {
log.Fatal("Failed to open file for reading:", string(path), err)
}
defer file.Close()
for r := range fileStream {
switch r.Kind {
case SparseHole:
// Process hole
var hash, data []byte
unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}
case SparseData:
// Read file data
data := make([]byte, r.Len())
status := true
n, err := fileReadAt(file, data, r.Begin)
if err != nil {
status = false
log.Error("File read error", status)
} else if int64(n) != r.Len() {
status = false
log.Error("File read underrun")
}
hasher := sha1.New()
hasher.Write(salt)
hasher.Write(data)
hash := hasher.Sum(nil)
unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}
}
}
wgroup.Done() // indicate to other readers we are done
}
func networkSender(netStream <-chan diffChunk, encoder *gob.Encoder, netStatus chan<- netXferStatus) {
status := true
byteCount := int64(0)
for {
chunk := <-netStream
if 0 == chunk.header.Len() {
// eof: last 0 len header
if verboseClient {
log.Debug("Client.networkSender <eof>")
}
err := encoder.Encode(chunk.header.FileInterval)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
}
break
}
if !status {
// network error
continue // discard the chunk
}
if !chunk.status {
// read error
status = false
continue // discard the chunk
}
if traceChannelLoad {
fmt.Fprint(os.Stderr, len(netStream), "n")
}
// Encode and send data to the network
if verboseClient {
log.Debug("Client.networkSender sending:", chunk.header.FileInterval)
}
err := encoder.Encode(chunk.header.FileInterval)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
continue
}
if len(chunk.header.Data) == 0 {
continue
}
if verboseClient {
log.Debug("Client.networkSender sending data")
}
if int64(len(chunk.header.Data)) != chunk.header.FileInterval.Len() {
log.Fatal("Client.networkSender sending data internal error:", chunk.header.FileInterval.Len(), len(chunk.header.Data))
}
err = encoder.Encode(chunk.header.Data)
if err != nil {
log.Fatal("Client protocol encoder error:", err)
status = false
continue
}
byteCount += int64(len(chunk.header.Data))
if traceChannelLoad {
fmt.Fprint(os.Stderr, "N\n")
}
}
netStatus <- netXferStatus{status, byteCount}
}
func processDiff(salt []byte, abortStream chan<- error, errStream <-chan error, encoder *gob.Encoder, decoder *gob.Decoder, local <-chan HashedDataInterval, remote <-chan HashedInterval, netInStreamDone <-chan bool, retry bool) (hashLocal []byte, err error) {
// Local: __ _*
// Remote: *_ **
hashLocal = make([]byte, 0) // empty hash for errors
const concurrentReaders = 4
netStream := make(chan diffChunk, 128)
netStatus := make(chan netXferStatus)
go networkSender(netStream, encoder, netStatus)
fileHasher := sha1.New()
fileHasher.Write(salt)
lrange := <-local
rrange := <-remote
for lrange.Len() != 0 {
if rrange.Len() == 0 {
// Copy local tail
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{SparseHole, lrange.Interval}, make([]byte, 0)}, netStream)
lrange = <-local
continue
}
// Diff
if verboseClient {
log.Debug("Diff:", lrange.HashedInterval, rrange)
}
if lrange.Begin == rrange.Begin {
if lrange.End > rrange.End {
data := lrange.Data
if len(data) > 0 {
data = lrange.Data[:rrange.Len()]
}
subrange := HashedDataInterval{HashedInterval{FileInterval{lrange.Kind, rrange.Interval}, lrange.Hash}, data}
if verboseClient {
logData("LHASH", subrange.Data)
}
hashFileData(fileHasher, subrange.Len(), subrange.Data)
processFileInterval(subrange, rrange, netStream)
if len(data) > 0 {
lrange.Data = lrange.Data[subrange.Len():]
}
lrange.Begin = rrange.End
rrange = <-remote
continue
} else if lrange.End < rrange.End {
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, HashedInterval{FileInterval{rrange.Kind, lrange.Interval}, make([]byte, 0)}, netStream)
rrange.Begin = lrange.End
lrange = <-local
continue
}
if verboseClient {
logData("LHASH", lrange.Data)
}
hashFileData(fileHasher, lrange.Len(), lrange.Data)
processFileInterval(lrange, rrange, netStream)
lrange = <-local
rrange = <-remote
} else {
// Should never happen
log.Fatal("processDiff internal error")
return
}
}
log.Info("Finished processing file diff")
status := true
err = <-errStream
if err != nil {
log.Error("Sync client file load aborted:", err)
status = false
}
// make sure we finished consuming dst hashes
status = <-netInStreamDone && status // netDstReceiver finished
log.Info("Finished consuming remote file hashes, status=", status)
// Send end of transmission
netStream <- diffChunk{true, DataInterval{FileInterval{SparseIgnore, Interval{0, 0}}, make([]byte, 0)}}
// get network sender status
net := <-netStatus
log.Info("Finished sending file diff of", net.byteCount, "(bytes), status=", net.status)
if !net.status {
err = errors.New("netwoek transfer failure")
return
}
var statusRemote bool
err = decoder.Decode(&statusRemote)
if err != nil {
log.Fatal("Cient protocol remote status error:", err)
return
}
if !statusRemote {
err = errors.New("failure on remote sync site")
return
}
var hashRemote []byte
err = decoder.Decode(&hashRemote)
if err != nil {
log.Fatal("Cient protocol remote hash error:", err)
return
}
// Compare file hashes
hashLocal = fileHasher.Sum(nil)
if isHashDifferent(hashLocal, hashRemote) || FailPointFileHashMatch() {
log.Warn("hashLocal =", hashLocal)
log.Warn("hashRemote=", hashRemote)
err = &HashCollsisionError{}
} else {
retry = false // success, don't retry anymore
}
// Final retry negotiation
{
err1 := encoder.Encode(retry)
if err1 != nil {
log.Fatal("Cient protocol remote retry error:", err)
}
err1 = decoder.Decode(&statusRemote)
if err1 != nil {
log.Fatal("Cient protocol remote retry status error:", err)
}
}
return
}
// Validator merges source and diff data; produces hash of the destination file
func Validator(salt []byte, checksumStream, netInStream <-chan DataInterval, resultStream chan<- []byte) {
fileHasher := sha1.New()
//TODO: error handling
fileHasher.Write(salt)
r := <-checksumStream // original dst file data
q := <-netInStream // diff data
for q.Len() != 0 || r.Len() != 0 {
if r.Len() == 0 /*end of dst file*/ {
// Hash diff data
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
q = <-netInStream
} else if q.Len() == 0 /*end of diff file*/ {
// Hash original data
if verboseServer {
logData("RHASH", r.Data)
}
hashFileData(fileHasher, r.Len(), r.Data)
r = <-checksumStream
} else {
qi := q.Interval
ri := r.Interval
if qi.Begin == ri.Begin {
if qi.End > ri.End {
log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
} else if qi.End < ri.End {
// Hash diff data
if verboseServer {
log.Debug("Server.Validator: hashing diff", q.FileInterval, r.FileInterval)
}
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
r.Begin = q.End
q = <-netInStream
} else {
if q.Kind == SparseIgnore {
// Hash original data
if verboseServer {
log.Debug("Server.Validator: hashing original", r.FileInterval)
}
if verboseServer {
logData("RHASH", r.Data)
}
hashFileData(fileHasher, r.Len(), r.Data)
} else {
// Hash diff data
if verboseServer {
log.Debug("Server.Validator: hashing diff", q.FileInterval)
}
if verboseServer {
logData("RHASH", q.Data)
}
hashFileData(fileHasher, q.Len(), q.Data)
}
q = <-netInStream
r = <-checksumStream
}
} else {
log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
}
}
}
if verboseServer {
log.Debug("Server.Validator: finished")
}
resultStream <- fileHasher.Sum(nil)
}
// returns true if no retry is necessary
func serveSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
directFileIO := size%Blocks == 0
SetupFileIO(directFileIO)
// Open destination file
file, err := fileOpen(path, os.O_RDWR, 0666)
if err != nil {
file, err = os.Create(path)
if err != nil {
log.Error("Failed to create file:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
}
// Setup close sequence
if directFileIO {
defer file.Sync()
}
defer file.Close()
// Resize the file
if err = file.Truncate(size); err != nil {
log.Error("Failed to resize file:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
// open file
fileRO, err := fileOpen(path, os.O_RDONLY, 0)
if err != nil {
log.Error("Failed to open file for reading:", string(path), err)
encoder.Encode(false) // NACK request
return true
}
defer fileRO.Close()
abortStream := make(chan error)
layoutStream := make(chan FileInterval, 128)
errStream := make(chan error)
fileStream := make(chan FileInterval, 128)
unorderedStream := make(chan HashedDataInterval, 128)
orderedStream := make(chan HashedDataInterval, 128)
netOutStream := make(chan HashedInterval, 128)
netOutDoneStream := make(chan bool)
netInStream := make(chan DataInterval, 128)
fileWriteStream := make(chan DataInterval, 128)
deltaReceiverDoneDone := make(chan bool)
checksumStream := make(chan DataInterval, 128)
resultStream := make(chan []byte)
// Initiate interval loading...
err = loadFileLayout(abortStream, fileRO, layoutStream, errStream)
if err != nil {
encoder.Encode(false) // NACK request
return true
}
encoder.Encode(true) // ACK request
go IntervalSplitter(layoutStream, fileStream)
FileReaderGroup(fileReaders, salt, fileStream, path, unorderedStream)
go OrderIntervals("dst:", unorderedStream, orderedStream)
go Tee(orderedStream, netOutStream, checksumStream)
// Send layout along with data hashes
go netSender(netOutStream, encoder, netOutDoneStream)
// Start receiving deltas, write those and compute file hash
fileWritten := FileWriterGroup(fileWriters, fileWriteStream, path)
go netReceiver(decoder, file, netInStream, fileWriteStream, deltaReceiverDoneDone) // receiver and checker
go Validator(salt, checksumStream, netInStream, resultStream)
// Block till completion
status := true
err = <-errStream // Done with file loadaing, possibly aborted on error
if err != nil {
log.Error("Sync server file load aborted:", err)
status = false
}
status = <-netOutDoneStream && status // done sending dst hashes
status = <-deltaReceiverDoneDone && status // done writing dst file
fileWritten.Wait() // wait for write stream completion
hash := <-resultStream // done processing diffs
// reply to client with status
log.Info("Sync sending server status=", status)
err = encoder.Encode(status)
if err != nil {
log.Fatal("Protocol encoder error:", err)
return true
}
// reply with local hash
err = encoder.Encode(hash)
if err != nil {
log.Fatal("Protocol encoder error:", err)
return true
}
var retry bool
err = decoder.Decode(&retry)
if err != nil {
log.Fatal("Protocol retry decoder error:", err)
return true
}
encoder.Encode(true) // ACK retry
if err != nil {
log.Fatal("Protocol retry encoder error:", err)
return true
}
return !retry // don't terminate server if retry expected
}
