Example #1
0
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
	nodeID := protoConn.ID()
	m.pmut.Lock()
	if _, ok := m.protoConn[nodeID]; ok {
		panic("add existing node")
	}
	m.protoConn[nodeID] = protoConn
	if _, ok := m.rawConn[nodeID]; ok {
		panic("add existing node")
	}
	m.rawConn[nodeID] = rawConn
	m.pmut.Unlock()

	cm := m.clusterConfig(nodeID)
	protoConn.ClusterConfig(cm)

	var idxToSend = make(map[string][]protocol.FileInfo)

	m.rmut.RLock()
	for _, repo := range m.nodeRepos[nodeID] {
		idxToSend[repo] = m.protocolIndex(repo)
	}
	m.rmut.RUnlock()

	go func() {
		for repo, idx := range idxToSend {
			if debugNet {
				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
			}
			protoConn.Index(repo, idx)
		}
	}()
}
Example #2
0
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
	nodeID := protoConn.ID()

	m.pmut.Lock()
	if _, ok := m.protoConn[nodeID]; ok {
		panic("add existing node")
	}
	m.protoConn[nodeID] = protoConn
	if _, ok := m.rawConn[nodeID]; ok {
		panic("add existing node")
	}
	m.rawConn[nodeID] = rawConn

	cm := m.clusterConfig(nodeID)
	protoConn.ClusterConfig(cm)

	m.rmut.RLock()
	for _, repo := range m.nodeRepos[nodeID] {
		fs := m.repoFiles[repo]
		go sendIndexes(protoConn, repo, fs)
	}
	m.rmut.RUnlock()
	m.pmut.Unlock()
}
Example #3
0
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
	nodeID := protoConn.ID()
	m.pmut.Lock()
	if _, ok := m.protoConn[nodeID]; ok {
		panic("add existing node")
	}
	m.protoConn[nodeID] = protoConn
	if _, ok := m.rawConn[nodeID]; ok {
		panic("add existing node")
	}
	m.rawConn[nodeID] = rawConn
	m.pmut.Unlock()

	cm := m.clusterConfig(nodeID)
	protoConn.ClusterConfig(cm)

	var idxToSend = make(map[string][]protocol.FileInfo)

	m.rmut.RLock()
	for _, repo := range m.nodeRepos[nodeID] {
		idxToSend[repo] = m.protocolIndex(repo)
	}
	m.rmut.RUnlock()

	go func() {
		for repo, idx := range idxToSend {
			if debug {
				l.Debugf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
			}
			const batchSize = 1000
			for i := 0; i < len(idx); i += batchSize {
				if len(idx[i:]) < batchSize {
					protoConn.Index(repo, idx[i:])
				} else {
					protoConn.Index(repo, idx[i:i+batchSize])
				}
			}
		}
	}()
}
Example #4
0
func sendIndexes(conn protocol.Connection, repo string, fs *files.Set) {
	nodeID := conn.ID()
	name := conn.Name()

	if debug {
		l.Debugf("sendIndexes for %s-%s@/%q starting", nodeID, name, repo)
	}

	initial := true
	minLocalVer := uint64(0)
	var err error

	defer func() {
		if debug {
			l.Debugf("sendIndexes for %s-%s@/%q exiting: %v", nodeID, name, repo, err)
		}
	}()

	for err == nil {
		if !initial && fs.LocalVersion(protocol.LocalNodeID) <= minLocalVer {
			time.Sleep(1 * time.Second)
			continue
		}

		batch := make([]protocol.FileInfo, 0, indexBatchSize)
		maxLocalVer := uint64(0)

		fs.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
			if f.LocalVersion <= minLocalVer {
				return true
			}

			if f.LocalVersion > maxLocalVer {
				maxLocalVer = f.LocalVersion
			}

			if len(batch) == indexBatchSize {
				if initial {
					if err = conn.Index(repo, batch); err != nil {
						return false
					}
					if debug {
						l.Debugf("sendIndexes for %s-%s/%q: %d files (initial index)", nodeID, name, repo, len(batch))
					}
					initial = false
				} else {
					if err = conn.IndexUpdate(repo, batch); err != nil {
						return false
					}
					if debug {
						l.Debugf("sendIndexes for %s-%s/%q: %d files (batched update)", nodeID, name, repo, len(batch))
					}
				}

				batch = make([]protocol.FileInfo, 0, indexBatchSize)
			}

			batch = append(batch, f)
			return true
		})

		if initial {
			err = conn.Index(repo, batch)
			if debug && err == nil {
				l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", nodeID, name, repo, len(batch))
			}
			initial = false
		} else if len(batch) > 0 {
			err = conn.IndexUpdate(repo, batch)
			if debug && err == nil {
				l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", nodeID, name, repo, len(batch))
			}
		}

		minLocalVer = maxLocalVer
	}
}