Пример #1
0
func sendIndexes(conn protocol.Connection, repo string, fs *files.Set, ignores ignore.Patterns) {
	nodeID := conn.ID()
	name := conn.Name()
	var err error

	if debug {
		l.Debugf("sendIndexes for %s-%s@/%q starting", nodeID, name, repo)
	}

	defer func() {
		if debug {
			l.Debugf("sendIndexes for %s-%s@/%q exiting: %v", nodeID, name, repo, err)
		}
	}()

	minLocalVer, err := sendIndexTo(true, 0, conn, repo, fs, ignores)

	for err == nil {
		time.Sleep(5 * time.Second)
		if fs.LocalVersion(protocol.LocalNodeID) <= minLocalVer {
			continue
		}

		minLocalVer, err = sendIndexTo(false, minLocalVer, conn, repo, fs, ignores)
	}
}
Пример #2
0
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
	nodeID := protoConn.ID()

	m.pmut.Lock()
	if _, ok := m.protoConn[nodeID]; ok {
		panic("add existing node")
	}
	m.protoConn[nodeID] = protoConn
	if _, ok := m.rawConn[nodeID]; ok {
		panic("add existing node")
	}
	m.rawConn[nodeID] = rawConn

	cm := m.clusterConfig(nodeID)
	protoConn.ClusterConfig(cm)

	m.rmut.RLock()
	for _, repo := range m.nodeRepos[nodeID] {
		fs := m.repoFiles[repo]
		go sendIndexes(protoConn, repo, fs, m.repoIgnores[repo])
	}
	if statRef, ok := m.nodeStatRefs[nodeID]; ok {
		statRef.WasSeen()
	} else {
		l.Warnf("AddConnection for unconfigured node %v?", nodeID)
	}
	m.rmut.RUnlock()
	m.pmut.Unlock()
}
Пример #3
0
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
	nodeID := protoConn.ID()

	m.pmut.Lock()
	if _, ok := m.protoConn[nodeID]; ok {
		panic("add existing node")
	}
	m.protoConn[nodeID] = protoConn
	if _, ok := m.rawConn[nodeID]; ok {
		panic("add existing node")
	}
	m.rawConn[nodeID] = rawConn

	cm := m.clusterConfig(nodeID)
	protoConn.ClusterConfig(cm)

	m.rmut.RLock()
	for _, repo := range m.nodeRepos[nodeID] {
		fs := m.repoFiles[repo]
		go sendIndexes(protoConn, repo, fs)
	}
	m.rmut.RUnlock()
	m.pmut.Unlock()
}
Пример #4
0
func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, repo string, fs *files.Set, ignores ignore.Patterns) (uint64, error) {
	nodeID := conn.ID()
	name := conn.Name()
	batch := make([]protocol.FileInfo, 0, indexBatchSize)
	currentBatchSize := 0
	maxLocalVer := uint64(0)
	var err error

	fs.WithHave(protocol.LocalNodeID, func(fi protocol.FileIntf) bool {
		f := fi.(protocol.FileInfo)
		if f.LocalVersion <= minLocalVer {
			return true
		}

		if f.LocalVersion > maxLocalVer {
			maxLocalVer = f.LocalVersion
		}

		if ignores.Match(f.Name) {
			return true
		}

		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
			if initial {
				if err = conn.Index(repo, batch); err != nil {
					return false
				}
				if debug {
					l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", nodeID, name, repo, len(batch), currentBatchSize)
				}
				initial = false
			} else {
				if err = conn.IndexUpdate(repo, batch); err != nil {
					return false
				}
				if debug {
					l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", nodeID, name, repo, len(batch), currentBatchSize)
				}
			}

			batch = make([]protocol.FileInfo, 0, indexBatchSize)
			currentBatchSize = 0
		}

		batch = append(batch, f)
		currentBatchSize += indexPerFileSize + len(f.Blocks)*IndexPerBlockSize
		return true
	})

	if initial && err == nil {
		err = conn.Index(repo, batch)
		if debug && err == nil {
			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", nodeID, name, repo, len(batch))
		}
	} else if len(batch) > 0 && err == nil {
		err = conn.IndexUpdate(repo, batch)
		if debug && err == nil {
			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", nodeID, name, repo, len(batch))
		}
	}

	return maxLocalVer, err
}
Пример #5
0
func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, repo string, fs *files.Set) (uint64, error) {
	nodeID := conn.ID()
	name := conn.Name()
	batch := make([]protocol.FileInfo, 0, indexBatchSize)
	maxLocalVer := uint64(0)
	var err error

	fs.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
		if f.LocalVersion <= minLocalVer {
			return true
		}

		if f.LocalVersion > maxLocalVer {
			maxLocalVer = f.LocalVersion
		}

		if len(batch) == indexBatchSize {
			if initial {
				if err = conn.Index(repo, batch); err != nil {
					return false
				}
				if debug {
					l.Debugf("sendIndexes for %s-%s/%q: %d files (initial index)", nodeID, name, repo, len(batch))
				}
				initial = false
			} else {
				if err = conn.IndexUpdate(repo, batch); err != nil {
					return false
				}
				if debug {
					l.Debugf("sendIndexes for %s-%s/%q: %d files (batched update)", nodeID, name, repo, len(batch))
				}
			}

			batch = make([]protocol.FileInfo, 0, indexBatchSize)
		}

		batch = append(batch, f)
		return true
	})

	if initial && err == nil {
		err = conn.Index(repo, batch)
		if debug && err == nil {
			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", nodeID, name, repo, len(batch))
		}
	} else if len(batch) > 0 && err == nil {
		err = conn.IndexUpdate(repo, batch)
		if debug && err == nil {
			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", nodeID, name, repo, len(batch))
		}
	}

	return maxLocalVer, err
}