Exemple #1
0
func server(addr TCPEndPoint, serveOnce /*test flag*/ bool, timeout int) {
	serverConnectionTimeout := time.Duration(timeout) * time.Second
	// listen on all interfaces
	EndPoint := addr.Host + ":" + strconv.Itoa(int(addr.Port))
	laddr, err := net.ResolveTCPAddr("tcp", EndPoint)
	if err != nil {
		log.Fatal("Connection listener address resolution error:", err)
	}
	ln, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatal("Connection listener error:", err)
	}
	defer ln.Close()
	ln.SetDeadline(time.Now().Add(serverConnectionTimeout))
	log.Info("Sync server is up...")

	for {
		conn, err := ln.AcceptTCP()
		if err != nil {
			log.Fatal("Connection accept error:", err)
		}

		if serveOnce {
			// This is to avoid server listening port conflicts while running tests
			// exit after single connection request
			if serveConnection(conn) {
				break // no retries
			}
			log.Warn("Server: waiting for client sync retry...")
		} else {
			go serveConnection(conn)
		}
	}
	log.Info("Sync server exit.")
}
Exemple #2
0
// SetupFileIO Sets up direct file I/O or buffered for small unaligned files
func SetupFileIO(direct bool) {
	if direct {
		fileOpen = fio.OpenFile
		fileReadAt = fio.ReadAt
		fileWriteAt = fio.WriteAt
		log.Info("Mode: directfio")
	} else {
		fileOpen = fileBufferedOpen
		fileReadAt = fileBufferedReadAt
		fileWriteAt = fileBufferedWriteAt
		log.Info("Mode: buffered")
	}
}
Exemple #3
0
func cmdError(msg string) {
	fmt.Fprintln(os.Stderr, "Error:", msg)
	flag.Usage()
	log.Info("ssync: exit code 2")
	os.Exit(2)
}
Exemple #4
0
func Main() {
	defaultVerboseLogLevel := log.LevelInfo    // set if -verbose
	defaultNonVerboseLogLevel := log.LevelWarn // set if -verbose=false
	// Command line parsing
	verbose := flag.Bool("verbose", false, "verbose mode")
	daemon := flag.Bool("daemon", false, "daemon mode (run on remote host)")
	port := flag.Int("port", 5000, "optional daemon port")
	timeout := flag.Int("timeout", 120, "optional daemon/client timeout (seconds)")
	host := flag.String("host", "", "remote host of <DstFile> (requires running daemon)")
	flag.Usage = func() {
		const usage = "sync <Options> <SrcFile> [<DstFile>]"
		const examples = `
Examples:
  sync -daemon
  sync -host remote.net file.data`
		fmt.Fprintf(os.Stderr, "\nUsage of %s:\n", os.Args[0])
		fmt.Fprintln(os.Stderr, usage)
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, examples)
	}
	flag.Parse()

	args := flag.Args()
	if *daemon {
		// Daemon mode
		endpoint := sparse.TCPEndPoint{Host: "" /*bind to all*/, Port: int16(*port)}
		if *verbose {
			log.LevelPush(defaultVerboseLogLevel)
			defer log.LevelPop()
			fmt.Fprintln(os.Stderr, "Listening on", endpoint, "...")
		} else {
			log.LevelPush(defaultNonVerboseLogLevel)
			defer log.LevelPop()
		}

		sparse.Server(endpoint, *timeout)
	} else {
		// "local to remote"" file sync mode
		if len(args) < 1 {
			cmdError("missing file path")
		}
		srcPath := args[0]
		dstPath := srcPath
		if len(args) == 2 {
			dstPath = args[1]
		} else if len(args) > 2 {
			cmdError("too many arguments")
		}

		endpoint := sparse.TCPEndPoint{Host: *host, Port: int16(*port)}
		if *verbose {
			log.LevelPush(defaultVerboseLogLevel)
			defer log.LevelPop()
			fmt.Fprintf(os.Stderr, "Syncing %s to %s@%s:%d...\n", srcPath, dstPath, endpoint.Host, endpoint.Port)
		} else {
			log.LevelPush(defaultNonVerboseLogLevel)
			defer log.LevelPop()
		}

		_, err := sparse.SyncFile(srcPath, endpoint, dstPath, *timeout)
		if err != nil {
			log.Info("ssync: error:", err, "exit code 1")
			os.Exit(1)
		}
		log.Info("ssync: exit code 0")
	}
}
Exemple #5
0
func processDiff(salt []byte, abortStream chan<- error, errStream <-chan error, encoder *gob.Encoder, decoder *gob.Decoder, local <-chan HashedDataInterval, remote <-chan HashedInterval, netInStreamDone <-chan bool, retry bool) (hashLocal []byte, err error) {
	// Local:   __ _*
	// Remote:  *_ **
	hashLocal = make([]byte, 0) // empty hash for errors
	const concurrentReaders = 4
	netStream := make(chan diffChunk, 128)
	netStatus := make(chan netXferStatus)
	go networkSender(netStream, encoder, netStatus)
	fileHasher := sha1.New()
	fileHasher.Write(salt)

	lrange := <-local
	rrange := <-remote
	for lrange.Len() != 0 {
		if rrange.Len() == 0 {
			// Copy local tail
			if verboseClient {
				logData("LHASH", lrange.Data)
			}
			hashFileData(fileHasher, lrange.Len(), lrange.Data)
			processFileInterval(lrange, HashedInterval{FileInterval{SparseHole, lrange.Interval}, make([]byte, 0)}, netStream)
			lrange = <-local
			continue
		}
		// Diff
		if verboseClient {
			log.Debug("Diff:", lrange.HashedInterval, rrange)
		}
		if lrange.Begin == rrange.Begin {
			if lrange.End > rrange.End {
				data := lrange.Data
				if len(data) > 0 {
					data = lrange.Data[:rrange.Len()]
				}
				subrange := HashedDataInterval{HashedInterval{FileInterval{lrange.Kind, rrange.Interval}, lrange.Hash}, data}
				if verboseClient {
					logData("LHASH", subrange.Data)
				}

				hashFileData(fileHasher, subrange.Len(), subrange.Data)
				processFileInterval(subrange, rrange, netStream)
				if len(data) > 0 {
					lrange.Data = lrange.Data[subrange.Len():]
				}
				lrange.Begin = rrange.End
				rrange = <-remote
				continue
			} else if lrange.End < rrange.End {
				if verboseClient {
					logData("LHASH", lrange.Data)
				}

				hashFileData(fileHasher, lrange.Len(), lrange.Data)
				processFileInterval(lrange, HashedInterval{FileInterval{rrange.Kind, lrange.Interval}, make([]byte, 0)}, netStream)
				rrange.Begin = lrange.End
				lrange = <-local
				continue
			}
			if verboseClient {
				logData("LHASH", lrange.Data)
			}
			hashFileData(fileHasher, lrange.Len(), lrange.Data)
			processFileInterval(lrange, rrange, netStream)
			lrange = <-local
			rrange = <-remote
		} else {
			// Should never happen
			log.Fatal("processDiff internal error")
			return
		}
	}
	log.Info("Finished processing file diff")

	status := true
	err = <-errStream
	if err != nil {
		log.Error("Sync client file load aborted:", err)
		status = false
	}
	// make sure we finished consuming dst hashes
	status = <-netInStreamDone && status // netDstReceiver finished
	log.Info("Finished consuming remote file hashes, status=", status)

	// Send end of transmission
	netStream <- diffChunk{true, DataInterval{FileInterval{SparseIgnore, Interval{0, 0}}, make([]byte, 0)}}

	// get network sender status
	net := <-netStatus
	log.Info("Finished sending file diff of", net.byteCount, "(bytes), status=", net.status)
	if !net.status {
		err = errors.New("netwoek transfer failure")
		return
	}

	var statusRemote bool
	err = decoder.Decode(&statusRemote)
	if err != nil {
		log.Fatal("Cient protocol remote status error:", err)
		return
	}
	if !statusRemote {
		err = errors.New("failure on remote sync site")
		return
	}
	var hashRemote []byte
	err = decoder.Decode(&hashRemote)
	if err != nil {
		log.Fatal("Cient protocol remote hash error:", err)
		return
	}

	// Compare file hashes
	hashLocal = fileHasher.Sum(nil)
	if isHashDifferent(hashLocal, hashRemote) || FailPointFileHashMatch() {
		log.Warn("hashLocal =", hashLocal)
		log.Warn("hashRemote=", hashRemote)
		err = &HashCollsisionError{}
	} else {
		retry = false // success, don't retry anymore
	}

	// Final retry negotiation
	{
		err1 := encoder.Encode(retry)
		if err1 != nil {
			log.Fatal("Cient protocol remote retry error:", err)
		}
		err1 = decoder.Decode(&statusRemote)
		if err1 != nil {
			log.Fatal("Cient protocol remote retry status error:", err)
		}
	}
	return
}
Exemple #6
0
// returns true if no retry is necessary
func serveSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64, salt []byte) bool {
	directFileIO := size%Blocks == 0
	SetupFileIO(directFileIO)

	// Open destination file
	file, err := fileOpen(path, os.O_RDWR, 0666)
	if err != nil {
		file, err = os.Create(path)
		if err != nil {
			log.Error("Failed to create file:", string(path), err)
			encoder.Encode(false) // NACK request
			return true
		}
	}
	// Setup close sequence
	if directFileIO {
		defer file.Sync()
	}
	defer file.Close()

	// Resize the file
	if err = file.Truncate(size); err != nil {
		log.Error("Failed to resize file:", string(path), err)
		encoder.Encode(false) // NACK request
		return true
	}

	// open file
	fileRO, err := fileOpen(path, os.O_RDONLY, 0)
	if err != nil {
		log.Error("Failed to open file for reading:", string(path), err)
		encoder.Encode(false) // NACK request
		return true
	}
	defer fileRO.Close()

	abortStream := make(chan error)
	layoutStream := make(chan FileInterval, 128)
	errStream := make(chan error)

	fileStream := make(chan FileInterval, 128)
	unorderedStream := make(chan HashedDataInterval, 128)
	orderedStream := make(chan HashedDataInterval, 128)
	netOutStream := make(chan HashedInterval, 128)
	netOutDoneStream := make(chan bool)

	netInStream := make(chan DataInterval, 128)
	fileWriteStream := make(chan DataInterval, 128)
	deltaReceiverDoneDone := make(chan bool)
	checksumStream := make(chan DataInterval, 128)
	resultStream := make(chan []byte)

	// Initiate interval loading...
	err = loadFileLayout(abortStream, fileRO, layoutStream, errStream)
	if err != nil {
		encoder.Encode(false) // NACK request
		return true
	}
	encoder.Encode(true) // ACK request

	go IntervalSplitter(layoutStream, fileStream)
	FileReaderGroup(fileReaders, salt, fileStream, path, unorderedStream)
	go OrderIntervals("dst:", unorderedStream, orderedStream)
	go Tee(orderedStream, netOutStream, checksumStream)

	// Send layout along with data hashes
	go netSender(netOutStream, encoder, netOutDoneStream)

	// Start receiving deltas, write those and compute file hash
	fileWritten := FileWriterGroup(fileWriters, fileWriteStream, path)
	go netReceiver(decoder, file, netInStream, fileWriteStream, deltaReceiverDoneDone) // receiver and checker
	go Validator(salt, checksumStream, netInStream, resultStream)

	// Block till completion
	status := true
	err = <-errStream // Done with file loadaing, possibly aborted on error
	if err != nil {
		log.Error("Sync server file load aborted:", err)
		status = false
	}
	status = <-netOutDoneStream && status      // done sending dst hashes
	status = <-deltaReceiverDoneDone && status // done writing dst file
	fileWritten.Wait()                         // wait for write stream completion
	hash := <-resultStream                     // done processing diffs

	// reply to client with status
	log.Info("Sync sending server status=", status)
	err = encoder.Encode(status)
	if err != nil {
		log.Fatal("Protocol encoder error:", err)
		return true
	}
	// reply with local hash
	err = encoder.Encode(hash)
	if err != nil {
		log.Fatal("Protocol encoder error:", err)
		return true
	}

	var retry bool
	err = decoder.Decode(&retry)
	if err != nil {
		log.Fatal("Protocol retry decoder error:", err)
		return true
	}
	encoder.Encode(true) // ACK retry
	if err != nil {
		log.Fatal("Protocol retry encoder error:", err)
		return true
	}
	return !retry // don't terminate server if retry expected
}