func newAPIService(id protocol.DeviceID, cfg *config.Wrapper, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.CachingMux, relayService *relay.Service, errors, systemLog *logger.Recorder) (*apiService, error) { service := &apiService{ id: id, cfg: cfg, assetDir: assetDir, model: m, eventSub: eventSub, discoverer: discoverer, relayService: relayService, systemConfigMut: sync.NewMutex(), stop: make(chan struct{}), configChanged: make(chan struct{}), listenerMut: sync.NewMutex(), guiErrors: errors, systemLog: systemLog, } seen := make(map[string]struct{}) for file := range auto.Assets() { theme := strings.Split(file, "/")[0] if _, ok := seen[theme]; !ok { seen[theme] = struct{}{} service.themes = append(service.themes, theme) } } var err error service.listener, err = service.getListener(cfg.GUI()) return service, err }
func NewBroadcast(port int) *Broadcast { b := &Broadcast{ Supervisor: suture.New("broadcastBeacon", suture.Spec{ // Don't retry too frenetically: an error to open a socket or // whatever is usually something that is either permanent or takes // a while to get solved... FailureThreshold: 2, FailureBackoff: 60 * time.Second, // Only log restarts in debug mode. Log: func(line string) { l.Debugln(line) }, }), port: port, inbox: make(chan []byte), outbox: make(chan recv, 16), } b.br = &broadcastReader{ port: port, outbox: b.outbox, connMut: sync.NewMutex(), } b.Add(b.br) b.bw = &broadcastWriter{ port: port, inbox: b.inbox, connMut: sync.NewMutex(), } b.Add(b.bw) return b }
func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFolder { return &rwFolder{ stateTracker: stateTracker{ folder: cfg.ID, mut: sync.NewMutex(), }, model: m, progressEmitter: m.progressEmitter, virtualMtimeRepo: db.NewVirtualMtimeRepo(m.db, cfg.ID), folder: cfg.ID, dir: cfg.Path(), scanIntv: time.Duration(cfg.RescanIntervalS) * time.Second, ignorePerms: cfg.IgnorePerms, copiers: cfg.Copiers, pullers: cfg.Pullers, shortID: shortID, order: cfg.Order, stop: make(chan struct{}), queue: newJobQueue(), pullTimer: time.NewTimer(shortPullIntv), scanTimer: time.NewTimer(time.Millisecond), // The first scan should be done immediately. delayScan: make(chan time.Duration), scanNow: make(chan rescanRequest), remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), } }
// Wrap wraps an existing Configuration structure and ties it to a file on // disk. func Wrap(path string, cfg Configuration) *Wrapper { w := &Wrapper{ cfg: cfg, path: path, mut: sync.NewMutex(), sMut: sync.NewMutex(), } w.replaces = make(chan Configuration) return w }
func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFolder { p := &rwFolder{ stateTracker: stateTracker{ folder: cfg.ID, mut: sync.NewMutex(), }, model: m, progressEmitter: m.progressEmitter, virtualMtimeRepo: db.NewVirtualMtimeRepo(m.db, cfg.ID), folder: cfg.ID, dir: cfg.Path(), scanIntv: time.Duration(cfg.RescanIntervalS) * time.Second, ignorePerms: cfg.IgnorePerms, copiers: cfg.Copiers, pullers: cfg.Pullers, shortID: shortID, order: cfg.Order, maxConflicts: cfg.MaxConflicts, allowSparse: !cfg.DisableSparseFiles, checkFreeSpace: cfg.MinDiskFreePct != 0, stop: make(chan struct{}), queue: newJobQueue(), pullTimer: time.NewTimer(time.Second), scanTimer: time.NewTimer(time.Millisecond), // The first scan should be done immediately. delayScan: make(chan time.Duration), scanNow: make(chan rescanRequest), remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), } if p.copiers == 0 { p.copiers = defaultCopiers } if p.pullers == 0 { p.pullers = defaultPullers } if cfg.PullerPauseS == 0 { p.pause = defaultPullerPause } else { p.pause = time.Duration(cfg.PullerPauseS) * time.Second } if cfg.PullerSleepS == 0 { p.sleep = defaultPullerSleep } else { p.sleep = time.Duration(cfg.PullerSleepS) * time.Second } return p }
func NewFileSet(folder string, db *leveldb.DB) *FileSet { var s = FileSet{ localVersion: make(map[protocol.DeviceID]int64), folder: folder, db: db, blockmap: NewBlockMap(db, folder), mutex: sync.NewMutex(), } ldbCheckGlobals(db, []byte(folder)) var deviceID protocol.DeviceID ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool { copy(deviceID[:], device) if f.LocalVersion > s.localVersion[deviceID] { s.localVersion[deviceID] = f.LocalVersion } return true }) if debug { l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion) } clock(s.localVersion[protocol.LocalDeviceID]) return &s }
func NewFileSet(folder string, db *Instance) *FileSet { var s = FileSet{ remoteSequence: make(map[protocol.DeviceID]int64), folder: folder, db: db, blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))), updateMutex: sync.NewMutex(), } s.db.checkGlobals([]byte(folder), &s.globalSize) var deviceID protocol.DeviceID s.db.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool { copy(deviceID[:], device) if deviceID == protocol.LocalDeviceID { if f.Sequence > s.sequence { s.sequence = f.Sequence } s.localSize.addFile(f) } else if f.Sequence > s.remoteSequence[deviceID] { s.remoteSequence[deviceID] = f.Sequence } return true }) l.Debugf("loaded sequence for %q: %#v", folder, s.sequence) return &s }
func NewFileSet(folder string, db *Instance) *FileSet { var s = FileSet{ localVersion: make(map[protocol.DeviceID]int64), folder: folder, db: db, blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))), mutex: sync.NewMutex(), } s.db.checkGlobals([]byte(folder), &s.globalSize) var deviceID protocol.DeviceID s.db.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool { copy(deviceID[:], device) if f.LocalVersion > s.localVersion[deviceID] { s.localVersion[deviceID] = f.LocalVersion } if deviceID == protocol.LocalDeviceID { s.localSize.addFile(f) } return true }) l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion) clock(s.localVersion[protocol.LocalDeviceID]) return &s }
func newUPnPSvc(cfg *config.Wrapper, localPort int) *upnpSvc { return &upnpSvc{ cfg: cfg, localPort: localPort, extPortMut: sync.NewMutex(), } }
func TestSourceFileOK(t *testing.T) { s := sharedPullerState{ realName: "testdata/foo", mut: sync.NewMutex(), } fd, err := s.sourceFile() if err != nil { t.Fatal(err) } if fd == nil { t.Fatal("Unexpected nil fd") } bs := make([]byte, 6) n, err := fd.Read(bs) if n != len(bs) { t.Fatalf("Wrong read length %d != %d", n, len(bs)) } if string(bs) != "foobar" { t.Fatalf("Wrong contents %s != foobar", string(bs)) } if err := s.failed(); err != nil { t.Fatal(err) } }
func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service { f := &rwFolder{ folder: folder{ stateTracker: newStateTracker(cfg.ID), scan: newFolderScanner(cfg), stop: make(chan struct{}), model: model, }, mtimeFS: mtimeFS, dir: cfg.Path(), versioner: ver, ignorePerms: cfg.IgnorePerms, copiers: cfg.Copiers, pullers: cfg.Pullers, order: cfg.Order, maxConflicts: cfg.MaxConflicts, allowSparse: !cfg.DisableSparseFiles, checkFreeSpace: cfg.MinDiskFreePct != 0, ignoreDelete: cfg.IgnoreDelete, fsync: cfg.Fsync, queue: newJobQueue(), pullTimer: time.NewTimer(time.Second), remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), initialScanCompleted: make(chan struct{}), } f.configureCopiersAndPullers(cfg) return f }
func NewUPnPService(cfg *config.Wrapper, localPort int) *Service { return &Service{ cfg: cfg, localPort: localPort, extPortMut: sync.NewMutex(), } }
func newFolderSummaryService(cfg *config.Wrapper, m *model.Model) *folderSummaryService { service := &folderSummaryService{ Supervisor: suture.NewSimple("folderSummaryService"), cfg: cfg, model: m, stop: make(chan struct{}), immediate: make(chan string), folders: make(map[string]struct{}), foldersMut: sync.NewMutex(), lastEventReqMut: sync.NewMutex(), } service.Add(serviceFunc(service.listenForUpdates)) service.Add(serviceFunc(service.calculateSummaries)) return service }
// Wrap wraps an existing Configuration structure and ties it to a file on // disk. func Wrap(path string, cfg Configuration) *Wrapper { w := &Wrapper{ cfg: cfg, path: path, mut: sync.NewMutex(), } return w }
func NewService(cfg *config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, lans []*net.IPNet) *Service { service := &Service{ Supervisor: suture.New("connections.Service", suture.Spec{ Log: func(line string) { l.Infoln(line) }, }), cfg: cfg, myID: myID, model: mdl, tlsCfg: tlsCfg, discoverer: discoverer, conns: make(chan internalConn), bepProtocolName: bepProtocolName, tlsDefaultCommonName: tlsDefaultCommonName, lans: lans, limiter: newLimiter(cfg), natService: nat.NewService(myID, cfg), listenersMut: sync.NewRWMutex(), listeners: make(map[string]genericListener), listenerTokens: make(map[string]suture.ServiceToken), // A listener can fail twice, rapidly. Any more than that and it // will be put on suspension for ten minutes. Restarts and changes // due to config are done by removing and adding services, so are // not subject to these limitations. listenerSupervisor: suture.New("c.S.listenerSupervisor", suture.Spec{ Log: func(line string) { l.Infoln(line) }, FailureThreshold: 2, FailureBackoff: 600 * time.Second, }), curConMut: sync.NewMutex(), currentConnection: make(map[protocol.DeviceID]completeConn), } cfg.Subscribe(service) // There are several moving parts here; one routine per listening address // (handled in configuration changing) to handle incoming connections, // one routine to periodically attempt outgoing connections, one routine to // the the common handling regardless of whether the connection was // incoming or outgoing. service.Add(serviceFunc(service.connect)) service.Add(serviceFunc(service.handle)) service.Add(service.listenerSupervisor) raw := cfg.RawCopy() // Actually starts the listeners and NAT service service.CommitConfiguration(raw, raw) return service }
func setUpRwFolder(model *Model) rwFolder { return rwFolder{ folder: "default", dir: "testdata", model: model, errors: make(map[string]string), errorsMut: sync.NewMutex(), } }
func NewBufferedSubscription(s *Subscription, size int) *BufferedSubscription { bs := &BufferedSubscription{ sub: s, buf: make([]Event, size), mut: sync.NewMutex(), } bs.cond = stdsync.NewCond(bs.mut) go bs.pollingLoop() return bs }
// NewProcess returns a new Process talking to Syncthing at the specified address. // Example: NewProcess("127.0.0.1:8082") func NewProcess(addr string) *Process { p := &Process{ addr: addr, localVersion: make(map[string]map[string]int64), done: make(map[string]bool), eventMut: sync.NewMutex(), } p.startCompleteCond = stdsync.NewCond(p.eventMut) return p }
func TestHandleFileWithTemp(t *testing.T) { // After diff between required and existing we should: // Copy: 2, 5, 8 // Pull: 1, 3, 4, 6, 7 // After dropping out blocks already on the temp file we should: // Copy: 5, 8 // Pull: 1, 6 // Create existing file existingFile := protocol.FileInfo{ Name: "file", Flags: 0, Modified: 0, Blocks: []protocol.BlockInfo{ blocks[0], blocks[2], blocks[0], blocks[0], blocks[5], blocks[0], blocks[0], blocks[8], }, } // Create target file requiredFile := existingFile requiredFile.Blocks = blocks[1:] db, _ := leveldb.Open(storage.NewMemStorage(), nil) m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db) m.AddFolder(defaultFolderConfig) // Update index m.updateLocals("default", []protocol.FileInfo{existingFile}) p := rwFolder{ folder: "default", dir: "testdata", model: m, errors: make(map[string]string), errorsMut: sync.NewMutex(), } copyChan := make(chan copyBlocksState, 1) p.handleFile(requiredFile, copyChan, nil) // Receive the results toCopy := <-copyChan if len(toCopy.blocks) != 4 { t.Errorf("Unexpected count of copy blocks: %d != 4", len(toCopy.blocks)) } for i, eq := range []int{1, 5, 6, 8} { if string(toCopy.blocks[i].Hash) != string(blocks[eq].Hash) { t.Errorf("Block mismatch: %s != %s", toCopy.blocks[i].String(), blocks[eq].String()) } } }
func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, discoverer discover.CachingMux, relayService relay.Service, errors, systemLog logger.Recorder) (*apiService, error) { service := &apiService{ id: id, cfg: cfg, httpsCertFile: httpsCertFile, httpsKeyFile: httpsKeyFile, assetDir: assetDir, model: m, eventSub: eventSub, discoverer: discoverer, relayService: relayService, systemConfigMut: sync.NewMutex(), stop: make(chan struct{}), configChanged: make(chan struct{}), listenerMut: sync.NewMutex(), guiErrors: errors, systemLog: systemLog, } seen := make(map[string]struct{}) // Load themes from compiled in assets. for file := range auto.Assets() { theme := strings.Split(file, "/")[0] if _, ok := seen[theme]; !ok { seen[theme] = struct{}{} service.themes = append(service.themes, theme) } } if assetDir != "" { // Load any extra themes from the asset override dir. for _, dir := range dirNames(assetDir) { if _, ok := seen[dir]; !ok { seen[dir] = struct{}{} service.themes = append(service.themes, dir) } } } var err error service.listener, err = service.getListener(cfg.GUI()) return service, err }
func newSmallIndex(db *Instance, prefix []byte) *smallIndex { idx := &smallIndex{ db: db, prefix: prefix, id2val: make(map[uint32]string), val2id: make(map[string]uint32), mut: sync.NewMutex(), } idx.load() return idx }
func New(withCache bool) *Matcher { m := &Matcher{ withCache: withCache, stop: make(chan struct{}), mut: sync.NewMutex(), } if withCache { go m.clean(2 * time.Hour) } return m }
func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner) service { f := &rwFolder{ folder: folder{ stateTracker: stateTracker{ folderID: cfg.ID, mut: sync.NewMutex(), }, scan: folderscan{ interval: time.Duration(cfg.RescanIntervalS) * time.Second, timer: time.NewTimer(time.Millisecond), // The first scan should be done immediately. now: make(chan rescanRequest), delay: make(chan time.Duration), }, stop: make(chan struct{}), model: model, }, virtualMtimeRepo: db.NewVirtualMtimeRepo(model.db, cfg.ID), dir: cfg.Path(), ignorePerms: cfg.IgnorePerms, copiers: cfg.Copiers, pullers: cfg.Pullers, order: cfg.Order, maxConflicts: cfg.MaxConflicts, allowSparse: !cfg.DisableSparseFiles, checkFreeSpace: cfg.MinDiskFreePct != 0, versioner: ver, queue: newJobQueue(), pullTimer: time.NewTimer(time.Second), remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), initialScanCompleted: make(chan struct{}), } f.configureCopiersAndPullers(cfg) return f }
func setUpRwFolder(model *Model) rwFolder { return rwFolder{ folder: folder{ stateTracker: newStateTracker("default"), model: model, }, dir: "testdata", queue: newJobQueue(), errors: make(map[string]string), errorsMut: sync.NewMutex(), } }
func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) *autoclosedFile { f := &autoclosedFile{ name: name, closeDelay: closeDelay, maxOpenTime: maxOpenTime, mut: sync.NewMutex(), closed: make(chan struct{}), closeTimer: time.NewTimer(time.Minute), } go f.closerLoop() return f }
func NewService(cfg *config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, lans []*net.IPNet) *Service { service := &Service{ Supervisor: suture.NewSimple("connections.Service"), cfg: cfg, myID: myID, model: mdl, tlsCfg: tlsCfg, discoverer: discoverer, conns: make(chan IntermediateConnection), bepProtocolName: bepProtocolName, tlsDefaultCommonName: tlsDefaultCommonName, lans: lans, natService: nat.NewService(myID, cfg), listenersMut: sync.NewRWMutex(), listeners: make(map[string]genericListener), listenerTokens: make(map[string]suture.ServiceToken), curConMut: sync.NewMutex(), currentConnection: make(map[protocol.DeviceID]Connection), } cfg.Subscribe(service) // The rate variables are in KiB/s in the UI (despite the camel casing // of the name). We multiply by 1024 here to get B/s. options := service.cfg.Options() if options.MaxSendKbps > 0 { service.writeRateLimit = ratelimit.NewBucketWithRate(float64(1024*options.MaxSendKbps), int64(5*1024*options.MaxSendKbps)) } if options.MaxRecvKbps > 0 { service.readRateLimit = ratelimit.NewBucketWithRate(float64(1024*options.MaxRecvKbps), int64(5*1024*options.MaxRecvKbps)) } // There are several moving parts here; one routine per listening address // (handled in configuration changing) to handle incoming connections, // one routine to periodically attempt outgoing connections, one routine to // the the common handling regardless of whether the connection was // incoming or outgoing. service.Add(serviceFunc(service.connect)) service.Add(serviceFunc(service.handle)) raw := cfg.Raw() // Actually starts the listeners and NAT service service.CommitConfiguration(raw, raw) return service }
func TestHandleFile(t *testing.T) { // After the diff between required and existing we should: // Copy: 2, 5, 8 // Pull: 1, 3, 4, 6, 7 // Create existing file existingFile := protocol.FileInfo{ Name: "filex", Flags: 0, Modified: 0, Blocks: []protocol.BlockInfo{ blocks[0], blocks[2], blocks[0], blocks[0], blocks[5], blocks[0], blocks[0], blocks[8], }, } // Create target file requiredFile := existingFile requiredFile.Blocks = blocks[1:] db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) // Update index m.updateLocals("default", []protocol.FileInfo{existingFile}) p := rwFolder{ folder: "default", dir: "testdata", model: m, errors: make(map[string]string), errorsMut: sync.NewMutex(), } copyChan := make(chan copyBlocksState, 1) p.handleFile(requiredFile, copyChan, nil) // Receive the results toCopy := <-copyChan if len(toCopy.blocks) != 8 { t.Errorf("Unexpected count of copy blocks: %d != 8", len(toCopy.blocks)) } for i, block := range toCopy.blocks { if string(block.Hash) != string(blocks[i+1].Hash) { t.Errorf("Block mismatch: %s != %s", block.String(), blocks[i+1].String()) } } }
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner { maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0) if err != nil { maxAge = 31536000 // Default: ~1 year } cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0) if err != nil { cleanInterval = 3600 // Default: clean once per hour } // Use custom path if set, otherwise .stversions in folderPath var versionsDir string if params["versionsPath"] == "" { if debug { l.Debugln("using default dir .stversions") } versionsDir = filepath.Join(folderPath, ".stversions") } else { if debug { l.Debugln("using dir", params["versionsPath"]) } versionsDir = params["versionsPath"] } s := Staggered{ versionsPath: versionsDir, cleanInterval: cleanInterval, folderPath: folderPath, interval: [4]Interval{ {30, 3600}, // first hour -> 30 sec between versions {3600, 86400}, // next day -> 1 h between versions {86400, 592000}, // next 30 days -> 1 day between versions {604800, maxAge}, // next year -> 1 week between versions }, mutex: sync.NewMutex(), } if debug { l.Debugf("instantiated %#v", s) } go func() { s.clean() for _ = range time.Tick(time.Duration(cleanInterval) * time.Second) { s.clean() } }() return s }
func setUpSendReceiveFolder(model *Model) sendReceiveFolder { return sendReceiveFolder{ folder: folder{ stateTracker: newStateTracker("default"), model: model, }, mtimeFS: fs.NewMtimeFS(db.NewNamespacedKV(model.db, "mtime")), dir: "testdata", queue: newJobQueue(), errors: make(map[string]string), errorsMut: sync.NewMutex(), } }
func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription) (*apiSvc, error) { svc := &apiSvc{ id: id, cfg: cfg, assetDir: assetDir, model: m, systemConfigMut: sync.NewMutex(), eventSub: eventSub, } var err error svc.listener, err = svc.getListener(cfg) return svc, err }