Beispiel #1
2
func coalesce(parentFile *os.File, childFile *os.File) error {
	blockSize, err := getFileSystemBlockSize(childFile)
	if err != nil {
		panic("can't get FS block size, error: " + err.Error())
	}
	var data, hole int64
	for {
		data, err = syscall.Seek(int(childFile.Fd()), hole, seekData)
		if err != nil {
			// reaches EOF
			errno := err.(syscall.Errno)
			if errno == syscall.ENXIO {
				break
			} else {
				// unexpected errors
				log.Fatal("Failed to syscall.Seek SEEK_DATA")
				return err
			}
		}
		hole, err = syscall.Seek(int(childFile.Fd()), data, seekHole)
		if err != nil {
			log.Fatal("Failed to syscall.Seek SEEK_HOLE")
			return err
		}

		// now we have a data start offset and length(hole - data)
		// let's read from child and write to parent file block by block
		_, err = parentFile.Seek(data, os.SEEK_SET)
		if err != nil {
			log.Fatal("Failed to os.Seek os.SEEK_SET")
			return err
		}

		offset := data
		buffer := fio.AllocateAligned(blockSize)
		for offset != hole {
			// read a block from child, maybe use bufio or Reader stream
			n, err := fio.ReadAt(childFile, buffer, offset)
			if n != len(buffer) || err != nil {
				log.Fatal("Failed to read from childFile")
				return err
			}
			// write a block to parent
			n, err = fio.WriteAt(parentFile, buffer, offset)
			if n != len(buffer) || err != nil {
				log.Fatal("Failed to write to parentFile")
				return err
			}
			offset += int64(n)
		}
	}

	return nil
}
Beispiel #2
0
func sendSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
	err := encoder.Encode(requestHeader{requestMagic, syncRequestCode})
	if err != nil {
		log.Fatal("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(path)
	if err != nil {
		log.Fatal("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(size)
	if err != nil {
		log.Fatal("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(salt)
	if err != nil {
		log.Fatal("Client protocol encoder error:", err)
		return false
	}

	var ack bool
	err = decoder.Decode(&ack)
	if err != nil {
		log.Fatal("Client protocol decoder error:", err)
		return false
	}

	return ack
}
Beispiel #3
0
func server(addr TCPEndPoint, serveOnce /*test flag*/ bool, timeout int) {
	serverConnectionTimeout := time.Duration(timeout) * time.Second
	// listen on all interfaces
	EndPoint := addr.Host + ":" + strconv.Itoa(int(addr.Port))
	laddr, err := net.ResolveTCPAddr("tcp", EndPoint)
	if err != nil {
		log.Fatal("Connection listener address resolution error:", err)
	}
	ln, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatal("Connection listener error:", err)
	}
	defer ln.Close()
	ln.SetDeadline(time.Now().Add(serverConnectionTimeout))
	log.Info("Sync server is up...")

	for {
		conn, err := ln.AcceptTCP()
		if err != nil {
			log.Fatal("Connection accept error:", err)
		}

		if serveOnce {
			// This is to avoid server listening port conflicts while running tests
			// exit after single connection request
			if serveConnection(conn) {
				break // no retries
			}
			log.Warn("Server: waiting for client sync retry...")
		} else {
			go serveConnection(conn)
		}
	}
	log.Info("Sync server exit.")
}
Beispiel #4
0
func netSender(netOutStream <-chan HashedInterval, encoder *gob.Encoder, netOutDoneStream chan<- bool) {
	for r := range netOutStream {
		if verboseServer {
			log.Debug("Server.netSender: sending", r.FileInterval)
		}
		err := encoder.Encode(r)
		if err != nil {
			log.Fatal("Protocol encoder error:", err)
			netOutDoneStream <- false
			return
		}
	}

	rEOF := HashedInterval{FileInterval{SparseIgnore, Interval{}}, make([]byte, 0)}
	if rEOF.Len() != 0 {
		log.Fatal("Server.netSender internal error")
	}
	// err := encoder.Encode(HashedInterval{FileInterval{}, make([]byte, 0)})
	err := encoder.Encode(rEOF)
	if err != nil {
		log.Fatal("Protocol encoder error:", err)
		netOutDoneStream <- false
		return
	}
	if verboseServer {
		log.Debug("Server.netSender: finished sending hashes")
	}
	netOutDoneStream <- true
}
Beispiel #5
0
// FileWriter supports concurrent file reading
// add this writer to wgroup before invoking
func FileWriter(fileStream <-chan DataInterval, path string, wgroup *sync.WaitGroup) {
	// open file
	file, err := fileOpen(path, os.O_WRONLY, 0)
	if err != nil {
		log.Fatal("Failed to open file for wroting:", string(path), err)
	}
	defer file.Close()

	for r := range fileStream {
		switch r.Kind {
		case SparseHole:
			log.Debug("trimming...")
			err := PunchHole(file, r.Interval)
			if err != nil {
				log.Fatal("Failed to trim file")
			}

		case SparseData:
			log.Debug("writing data...")
			_, err = fileWriteAt(file, r.Data, r.Begin)
			if err != nil {
				log.Fatal("Failed to write file")
			}
		}
	}
	wgroup.Done()
}
Beispiel #6
0
func netReceiver(decoder *gob.Decoder, file *os.File, netInStream chan<- DataInterval, fileStream chan<- DataInterval, deltaReceiverDone chan<- bool) {
	// receive & process data diff
	status := true
	for status {
		var delta FileInterval
		err := decoder.Decode(&delta)
		if err != nil {
			log.Fatal("Protocol decoder error:", err)
			status = false
			break
		}
		log.Debug("receiving delta [", delta, "]")
		if 0 == delta.Len() {
			log.Debug("received end of transimission marker")
			break // end of diff
		}
		switch delta.Kind {
		case SparseData:
			// Receive data
			var data []byte
			err = decoder.Decode(&data)
			if err != nil {
				log.Fatal("Protocol data decoder error:", err)
				status = false
				break
			}
			if int64(len(data)) != delta.Len() {
				log.Fatal("Failed to receive data, expected=", delta.Len(), "received=", len(data))
				status = false
				break
			}
			// Push for writing and vaildator processing
			fileStream <- DataInterval{delta, data}
			netInStream <- DataInterval{delta, data}

		case SparseHole:
			// Push for writing and vaildator processing
			fileStream <- DataInterval{delta, make([]byte, 0)}
			netInStream <- DataInterval{delta, make([]byte, 0)}

		case SparseIgnore:
			// Push for vaildator processing
			netInStream <- DataInterval{delta, make([]byte, 0)}
			log.Debug("ignoring...")
		}
	}

	log.Debug("Server.netReceiver done, sync")
	close(netInStream)
	close(fileStream)
	deltaReceiverDone <- status
}
Beispiel #7
0
// Get remote hashed intervals
func netDstReceiver(decoder *gob.Decoder, netInStream chan<- HashedInterval, netInStreamDone chan<- bool) {
	status := true
	for {
		if verboseClient {
			log.Debug("Client.netDstReceiver decoding...")
		}
		var r HashedInterval
		err := decoder.Decode(&r)
		if err != nil {
			log.Fatal("Cient protocol error:", err)
			status = false
			break
		}
		// interval := r.Interval
		if r.Kind == SparseIgnore {
			if verboseClient {
				log.Debug("Client.netDstReceiver got <eof>")
			}
			break
		}
		if verboseClient {
			switch r.Kind {
			case SparseData:
				log.Debug("Client.netDstReceiver got data", r.FileInterval, "hash[", len(r.Hash), "]")
			case SparseHole:
				log.Debug("Client.netDstReceiver got hole", r.FileInterval)
			}
		}
		netInStream <- r
	}
	close(netInStream)
	netInStreamDone <- status
}
Beispiel #8
0
// returns true if no retry is necessary
func serveConnection(conn net.Conn) bool {
	defer conn.Close()

	decoder := gob.NewDecoder(conn)
	var request requestHeader
	err := decoder.Decode(&request)
	if err != nil {
		log.Fatal("Protocol decoder error:", err)
		return true
	}
	if requestMagic != request.Magic {
		log.Error("Bad request")
		return true
	}

	switch request.Code {
	case syncRequestCode:
		var path string
		err := decoder.Decode(&path)
		if err != nil {
			log.Fatal("Protocol decoder error:", err)
			return true
		}
		var size int64
		err = decoder.Decode(&size)
		if err != nil {
			log.Fatal("Protocol decoder error:", err)
			return true
		}
		var salt []byte
		err = decoder.Decode(&salt)
		if err != nil {
			log.Fatal("Protocol decoder error:", err)
			return true
		}
		encoder := gob.NewEncoder(conn)
		return serveSyncRequest(encoder, decoder, path, size, salt)
	}
	return true
}
Beispiel #9
0
func processFileInterval(local HashedDataInterval, remote HashedInterval, netStream chan<- diffChunk) {
	if local.Interval != remote.Interval {
		log.Fatal("Sync.processFileInterval range internal error:", local.FileInterval, remote.FileInterval)
	}
	if local.Kind != remote.Kind {
		// Different intreval types, send the diff
		if local.Kind == SparseData && int64(len(local.Data)) != local.FileInterval.Len() {
			log.Fatal("Sync.processFileInterval data internal error:", local.FileInterval.Len(), len(local.Data))
		}
		netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
		return
	}

	// The interval types are the same
	if SparseHole == local.Kind {
		// Process hole, no syncronization is required
		local.Kind = SparseIgnore
		netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
		return
	}

	if local.Kind != SparseData {
		log.Fatal("Sync.processFileInterval kind internal error:", local.FileInterval)
	}
	// Data file interval
	if isHashDifferent(local.Hash, remote.Hash) {
		if int64(len(local.Data)) != local.FileInterval.Len() {
			log.Fatal("Sync.processFileInterval internal error:", local.FileInterval.Len(), len(local.Data))
		}
		netStream <- diffChunk{true, DataInterval{local.FileInterval, local.Data}}
		return
	}

	// No diff, just communicate we processed it
	//TODO: this apparently can be avoided but requires revision of the protocol
	local.Kind = SparseIgnore
	netStream <- diffChunk{true, DataInterval{local.FileInterval, make([]byte, 0)}}
}
Beispiel #10
0
func connect(host, port string, timeout int) net.Conn {
	// connect to this socket
	endpoint := host + ":" + port
	raddr, err := net.ResolveTCPAddr("tcp", endpoint)
	if err != nil {
		log.Fatal("Connection address resolution error:", err)
	}
	timeStart := time.Now()
	timeStop := timeStart.Add(time.Duration(timeout) * time.Second)
	for timeNow := timeStart; timeNow.Before(timeStop); timeNow = time.Now() {
		conn, err := net.DialTCP("tcp", nil, raddr)
		if err == nil {
			return conn
		}
		log.Warn("Failed connection to", endpoint, "Retrying...")
		if timeNow != timeStart {
			// only sleep after the second attempt to speedup tests
			time.Sleep(1 * time.Second)
		}
	}
	return nil
}
Beispiel #11
0
// FileReader supports concurrent file reading
// multiple readres are allowed
func FileReader(salt []byte, fileStream <-chan FileInterval, path string, wgroup *sync.WaitGroup, unorderedStream chan<- HashedDataInterval) {
	// open file
	file, err := fileOpen(path, os.O_RDONLY, 0)
	if err != nil {
		log.Fatal("Failed to open file for reading:", string(path), err)
	}
	defer file.Close()

	for r := range fileStream {
		switch r.Kind {
		case SparseHole:
			// Process hole
			var hash, data []byte
			unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}

		case SparseData:
			// Read file data
			data := make([]byte, r.Len())
			status := true
			n, err := fileReadAt(file, data, r.Begin)
			if err != nil {
				status = false
				log.Error("File read error", status)
			} else if int64(n) != r.Len() {
				status = false
				log.Error("File read underrun")
			}
			hasher := sha1.New()
			hasher.Write(salt)
			hasher.Write(data)
			hash := hasher.Sum(nil)
			unorderedStream <- HashedDataInterval{HashedInterval{r, hash}, data}
		}
	}
	wgroup.Done() // indicate to other readers we are done
}
Beispiel #12
0
func networkSender(netStream <-chan diffChunk, encoder *gob.Encoder, netStatus chan<- netXferStatus) {
	status := true
	byteCount := int64(0)
	for {
		chunk := <-netStream
		if 0 == chunk.header.Len() {
			// eof: last 0 len header
			if verboseClient {
				log.Debug("Client.networkSender <eof>")
			}
			err := encoder.Encode(chunk.header.FileInterval)
			if err != nil {
				log.Fatal("Client protocol encoder error:", err)
				status = false
			}
			break
		}

		if !status {
			// network error
			continue // discard the chunk
		}
		if !chunk.status {
			// read error
			status = false
			continue // discard the chunk
		}

		if traceChannelLoad {
			fmt.Fprint(os.Stderr, len(netStream), "n")
		}

		// Encode and send data to the network
		if verboseClient {
			log.Debug("Client.networkSender sending:", chunk.header.FileInterval)
		}
		err := encoder.Encode(chunk.header.FileInterval)
		if err != nil {
			log.Fatal("Client protocol encoder error:", err)
			status = false
			continue
		}
		if len(chunk.header.Data) == 0 {
			continue
		}
		if verboseClient {
			log.Debug("Client.networkSender sending data")
		}
		if int64(len(chunk.header.Data)) != chunk.header.FileInterval.Len() {
			log.Fatal("Client.networkSender sending data internal error:", chunk.header.FileInterval.Len(), len(chunk.header.Data))
		}
		err = encoder.Encode(chunk.header.Data)
		if err != nil {
			log.Fatal("Client protocol encoder error:", err)
			status = false
			continue
		}
		byteCount += int64(len(chunk.header.Data))
		if traceChannelLoad {
			fmt.Fprint(os.Stderr, "N\n")
		}
	}
	netStatus <- netXferStatus{status, byteCount}
}
Beispiel #13
0
func processDiff(salt []byte, abortStream chan<- error, errStream <-chan error, encoder *gob.Encoder, decoder *gob.Decoder, local <-chan HashedDataInterval, remote <-chan HashedInterval, netInStreamDone <-chan bool, retry bool) (hashLocal []byte, err error) {
	// Local:   __ _*
	// Remote:  *_ **
	hashLocal = make([]byte, 0) // empty hash for errors
	const concurrentReaders = 4
	netStream := make(chan diffChunk, 128)
	netStatus := make(chan netXferStatus)
	go networkSender(netStream, encoder, netStatus)
	fileHasher := sha1.New()
	fileHasher.Write(salt)

	lrange := <-local
	rrange := <-remote
	for lrange.Len() != 0 {
		if rrange.Len() == 0 {
			// Copy local tail
			if verboseClient {
				logData("LHASH", lrange.Data)
			}
			hashFileData(fileHasher, lrange.Len(), lrange.Data)
			processFileInterval(lrange, HashedInterval{FileInterval{SparseHole, lrange.Interval}, make([]byte, 0)}, netStream)
			lrange = <-local
			continue
		}
		// Diff
		if verboseClient {
			log.Debug("Diff:", lrange.HashedInterval, rrange)
		}
		if lrange.Begin == rrange.Begin {
			if lrange.End > rrange.End {
				data := lrange.Data
				if len(data) > 0 {
					data = lrange.Data[:rrange.Len()]
				}
				subrange := HashedDataInterval{HashedInterval{FileInterval{lrange.Kind, rrange.Interval}, lrange.Hash}, data}
				if verboseClient {
					logData("LHASH", subrange.Data)
				}

				hashFileData(fileHasher, subrange.Len(), subrange.Data)
				processFileInterval(subrange, rrange, netStream)
				if len(data) > 0 {
					lrange.Data = lrange.Data[subrange.Len():]
				}
				lrange.Begin = rrange.End
				rrange = <-remote
				continue
			} else if lrange.End < rrange.End {
				if verboseClient {
					logData("LHASH", lrange.Data)
				}

				hashFileData(fileHasher, lrange.Len(), lrange.Data)
				processFileInterval(lrange, HashedInterval{FileInterval{rrange.Kind, lrange.Interval}, make([]byte, 0)}, netStream)
				rrange.Begin = lrange.End
				lrange = <-local
				continue
			}
			if verboseClient {
				logData("LHASH", lrange.Data)
			}
			hashFileData(fileHasher, lrange.Len(), lrange.Data)
			processFileInterval(lrange, rrange, netStream)
			lrange = <-local
			rrange = <-remote
		} else {
			// Should never happen
			log.Fatal("processDiff internal error")
			return
		}
	}
	log.Info("Finished processing file diff")

	status := true
	err = <-errStream
	if err != nil {
		log.Error("Sync client file load aborted:", err)
		status = false
	}
	// make sure we finished consuming dst hashes
	status = <-netInStreamDone && status // netDstReceiver finished
	log.Info("Finished consuming remote file hashes, status=", status)

	// Send end of transmission
	netStream <- diffChunk{true, DataInterval{FileInterval{SparseIgnore, Interval{0, 0}}, make([]byte, 0)}}

	// get network sender status
	net := <-netStatus
	log.Info("Finished sending file diff of", net.byteCount, "(bytes), status=", net.status)
	if !net.status {
		err = errors.New("netwoek transfer failure")
		return
	}

	var statusRemote bool
	err = decoder.Decode(&statusRemote)
	if err != nil {
		log.Fatal("Cient protocol remote status error:", err)
		return
	}
	if !statusRemote {
		err = errors.New("failure on remote sync site")
		return
	}
	var hashRemote []byte
	err = decoder.Decode(&hashRemote)
	if err != nil {
		log.Fatal("Cient protocol remote hash error:", err)
		return
	}

	// Compare file hashes
	hashLocal = fileHasher.Sum(nil)
	if isHashDifferent(hashLocal, hashRemote) || FailPointFileHashMatch() {
		log.Warn("hashLocal =", hashLocal)
		log.Warn("hashRemote=", hashRemote)
		err = &HashCollsisionError{}
	} else {
		retry = false // success, don't retry anymore
	}

	// Final retry negotiation
	{
		err1 := encoder.Encode(retry)
		if err1 != nil {
			log.Fatal("Cient protocol remote retry error:", err)
		}
		err1 = decoder.Decode(&statusRemote)
		if err1 != nil {
			log.Fatal("Cient protocol remote retry status error:", err)
		}
	}
	return
}
Beispiel #14
0
// Validator merges source and diff data; produces hash of the destination file
func Validator(salt []byte, checksumStream, netInStream <-chan DataInterval, resultStream chan<- []byte) {
	fileHasher := sha1.New()
	//TODO: error handling
	fileHasher.Write(salt)
	r := <-checksumStream // original dst file data
	q := <-netInStream    // diff data
	for q.Len() != 0 || r.Len() != 0 {
		if r.Len() == 0 /*end of dst file*/ {
			// Hash diff data
			if verboseServer {
				logData("RHASH", q.Data)
			}
			hashFileData(fileHasher, q.Len(), q.Data)
			q = <-netInStream
		} else if q.Len() == 0 /*end of diff file*/ {
			// Hash original data
			if verboseServer {
				logData("RHASH", r.Data)
			}
			hashFileData(fileHasher, r.Len(), r.Data)
			r = <-checksumStream
		} else {
			qi := q.Interval
			ri := r.Interval
			if qi.Begin == ri.Begin {
				if qi.End > ri.End {
					log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
				} else if qi.End < ri.End {
					// Hash diff data
					if verboseServer {
						log.Debug("Server.Validator: hashing diff", q.FileInterval, r.FileInterval)
					}
					if verboseServer {
						logData("RHASH", q.Data)
					}
					hashFileData(fileHasher, q.Len(), q.Data)
					r.Begin = q.End
					q = <-netInStream
				} else {
					if q.Kind == SparseIgnore {
						// Hash original data
						if verboseServer {
							log.Debug("Server.Validator: hashing original", r.FileInterval)
						}
						if verboseServer {
							logData("RHASH", r.Data)
						}
						hashFileData(fileHasher, r.Len(), r.Data)
					} else {
						// Hash diff data
						if verboseServer {
							log.Debug("Server.Validator: hashing diff", q.FileInterval)
						}
						if verboseServer {
							logData("RHASH", q.Data)
						}
						hashFileData(fileHasher, q.Len(), q.Data)
					}
					q = <-netInStream
					r = <-checksumStream
				}
			} else {
				log.Fatal("Server.Validator internal error, diff=", q.FileInterval, "local=", r.FileInterval)
			}
		}
	}

	if verboseServer {
		log.Debug("Server.Validator: finished")
	}
	resultStream <- fileHasher.Sum(nil)
}
Beispiel #15
0
// returns true if no retry is necessary
func serveSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
	directFileIO := size%Blocks == 0
	SetupFileIO(directFileIO)

	// Open destination file
	file, err := fileOpen(path, os.O_RDWR, 0666)
	if err != nil {
		file, err = os.Create(path)
		if err != nil {
			log.Error("Failed to create file:", string(path), err)
			encoder.Encode(false) // NACK request
			return true
		}
	}
	// Setup close sequence
	if directFileIO {
		defer file.Sync()
	}
	defer file.Close()

	// Resize the file
	if err = file.Truncate(size); err != nil {
		log.Error("Failed to resize file:", string(path), err)
		encoder.Encode(false) // NACK request
		return true
	}

	// open file
	fileRO, err := fileOpen(path, os.O_RDONLY, 0)
	if err != nil {
		log.Error("Failed to open file for reading:", string(path), err)
		encoder.Encode(false) // NACK request
		return true
	}
	defer fileRO.Close()

	abortStream := make(chan error)
	layoutStream := make(chan FileInterval, 128)
	errStream := make(chan error)

	fileStream := make(chan FileInterval, 128)
	unorderedStream := make(chan HashedDataInterval, 128)
	orderedStream := make(chan HashedDataInterval, 128)
	netOutStream := make(chan HashedInterval, 128)
	netOutDoneStream := make(chan bool)

	netInStream := make(chan DataInterval, 128)
	fileWriteStream := make(chan DataInterval, 128)
	deltaReceiverDoneDone := make(chan bool)
	checksumStream := make(chan DataInterval, 128)
	resultStream := make(chan []byte)

	// Initiate interval loading...
	err = loadFileLayout(abortStream, fileRO, layoutStream, errStream)
	if err != nil {
		encoder.Encode(false) // NACK request
		return true
	}
	encoder.Encode(true) // ACK request

	go IntervalSplitter(layoutStream, fileStream)
	FileReaderGroup(fileReaders, salt, fileStream, path, unorderedStream)
	go OrderIntervals("dst:", unorderedStream, orderedStream)
	go Tee(orderedStream, netOutStream, checksumStream)

	// Send layout along with data hashes
	go netSender(netOutStream, encoder, netOutDoneStream)

	// Start receiving deltas, write those and compute file hash
	fileWritten := FileWriterGroup(fileWriters, fileWriteStream, path)
	go netReceiver(decoder, file, netInStream, fileWriteStream, deltaReceiverDoneDone) // receiver and checker
	go Validator(salt, checksumStream, netInStream, resultStream)

	// Block till completion
	status := true
	err = <-errStream // Done with file loadaing, possibly aborted on error
	if err != nil {
		log.Error("Sync server file load aborted:", err)
		status = false
	}
	status = <-netOutDoneStream && status      // done sending dst hashes
	status = <-deltaReceiverDoneDone && status // done writing dst file
	fileWritten.Wait()                         // wait for write stream completion
	hash := <-resultStream                     // done processing diffs

	// reply to client with status
	log.Info("Sync sending server status=", status)
	err = encoder.Encode(status)
	if err != nil {
		log.Fatal("Protocol encoder error:", err)
		return true
	}
	// reply with local hash
	err = encoder.Encode(hash)
	if err != nil {
		log.Fatal("Protocol encoder error:", err)
		return true
	}

	var retry bool
	err = decoder.Decode(&retry)
	if err != nil {
		log.Fatal("Protocol retry decoder error:", err)
		return true
	}
	encoder.Encode(true) // ACK retry
	if err != nil {
		log.Fatal("Protocol retry encoder error:", err)
		return true
	}
	return !retry // don't terminate server if retry expected
}