func withDetailsMiddleware(id protocol.DeviceID, h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Syncthing-Version", Version)
w.Header().Set("X-Syncthing-ID", id.String())
h.ServeHTTP(w, r)
})
}
// requires write lock on model.fmut before entry
// requires file does not exist in local model
func (m *Model) addToLocalModel(deviceID protocol.DeviceID, folder string, file protocol.FileInfo) {
if file.IsDeleted() {
if debug {
l.Debugln("peer", deviceID.String(), "has deleted file, doing nothing", file.Name)
}
return
}
if file.IsInvalid() {
if debug {
l.Debugln("peer", deviceID.String(), "has invalid file, doing nothing", file.Name)
}
return
}
if file.IsSymlink() {
if debug {
l.Debugln("peer", deviceID.String(), "has symlink, doing nothing", file.Name)
}
return
}
if debug && file.IsDirectory() {
l.Debugln("peer", deviceID.String(), "has directory, adding", file.Name)
} else if debug {
l.Debugln("peer", deviceID.String(), "has file, adding", file.Name)
}
m.treeCaches[folder].AddEntry(file)
m.addPeerForEntry(deviceID, folder, file)
}
// Lookup returns the list of addresses where the given device is available;
// direct, and via relays.
func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
qURL, err := url.Parse(c.server)
if err != nil {
return nil, nil, err
}
q := qURL.Query()
q.Set("device", device.String())
qURL.RawQuery = q.Encode()
resp, err := c.queryClient.Get(qURL.String())
if err != nil {
if debug {
l.Debugln("globalClient.Lookup", qURL.String(), err)
}
return nil, nil, err
}
if resp.StatusCode != 200 {
resp.Body.Close()
if debug {
l.Debugln("globalClient.Lookup", qURL.String(), resp.Status)
}
return nil, nil, errors.New(resp.Status)
}
// TODO: Handle 429 and Retry-After?
var ann announcement
err = json.NewDecoder(resp.Body).Decode(&ann)
resp.Body.Close()
return ann.Direct, ann.Relays, err
}
// Lookup returns the list of addresses where the given device is available;
// direct, and via relays.
func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
qURL, err := url.Parse(c.server)
if err != nil {
return nil, nil, err
}
q := qURL.Query()
q.Set("device", device.String())
qURL.RawQuery = q.Encode()
resp, err := c.queryClient.Get(qURL.String())
if err != nil {
l.Debugln("globalClient.Lookup", qURL, err)
return nil, nil, err
}
if resp.StatusCode != 200 {
resp.Body.Close()
l.Debugln("globalClient.Lookup", qURL, resp.Status)
err := errors.New(resp.Status)
if secs, atoiErr := strconv.Atoi(resp.Header.Get("Retry-After")); atoiErr == nil && secs > 0 {
err = lookupError{
error: err,
cacheFor: time.Duration(secs) * time.Second,
}
}
return nil, nil, err
}
var ann announcement
err = json.NewDecoder(resp.Body).Decode(&ann)
resp.Body.Close()
return ann.Direct, ann.Relays, err
}
func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
row := s.prep["selectDevice"].QueryRow(device.String())
var seen time.Time
if err := row.Scan(&seen); err != nil {
return time.Time{}, err
}
return seen.In(time.UTC), nil
}
func (c *localClient) registerDevice(src net.Addr, device Device) bool {
var id protocol.DeviceID
copy(id[:], device.ID)
// Remember whether we already had a valid cache entry for this device.
ce, existsAlready := c.Get(id)
isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
// Any empty or unspecified addresses should be set to the source address
// of the announcement. We also skip any addresses we can't parse.
l.Debugln("discover: Registering addresses for", id)
var validAddresses []string
for _, addr := range device.Addresses {
u, err := url.Parse(addr.URL)
if err != nil {
continue
}
tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
if err != nil {
continue
}
if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
host, _, err := net.SplitHostPort(src.String())
if err != nil {
continue
}
u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
l.Debugf("discover: Reconstructed URL is %#v", u)
validAddresses = append(validAddresses, u.String())
l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr.URL, u.String())
} else {
validAddresses = append(validAddresses, addr.URL)
l.Debugf("discover: Accepted address %s verbatim", addr.URL)
}
}
c.Set(id, CacheEntry{
Direct: validAddresses,
Relays: device.Relays,
when: time.Now(),
found: true,
})
if isNewDevice {
events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
"device": id.String(),
"addrs": validAddresses,
"relays": device.Relays,
})
}
return isNewDevice
}
func (d *FileTreeCache) AddEntry(entry protocol.FileInfo, peer protocol.DeviceID) {
d.db.Update(func(tx *bolt.Tx) error {
eb := tx.Bucket(d.folderBucketKey).Bucket(entriesBucket)
/* save entry */
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
enc.Encode(entry)
eb.Put([]byte(entry.Name), buf.Bytes()) // TODO handle error?
/* add peer */
edb := tx.Bucket(d.folderBucketKey).Bucket(entryDevicesBucket)
v := edb.Get([]byte(entry.Name))
var devices map[string]bool
if v == nil {
devices = make(map[string]bool)
} else {
rbuf := bytes.NewBuffer(v)
dec := gob.NewDecoder(rbuf)
dec.Decode(&devices)
}
devices[peer.String()] = true
var dbuf bytes.Buffer
enc = gob.NewEncoder(&dbuf)
enc.Encode(devices)
edb.Put([]byte(entry.Name), dbuf.Bytes())
/* add child lookup */
dir := path.Dir(entry.Name)
clb := tx.Bucket(d.folderBucketKey).Bucket(childLookupBucket)
v = clb.Get([]byte(dir))
if debug {
l.Debugln("Adding child", entry.Name, "for dir", dir)
}
var children map[string]bool
if v == nil {
children = make(map[string]bool)
} else {
rbuf := bytes.NewBuffer(v)
dec := gob.NewDecoder(rbuf)
dec.Decode(&children)
}
children[entry.Name] = true
var cbuf bytes.Buffer
enc = gob.NewEncoder(&cbuf)
enc.Encode(children)
clb.Put([]byte(dir), cbuf.Bytes())
return nil
})
}
func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
var cfg Configuration
cfg.MyID = myID.String()
setDefaults(&cfg)
setDefaults(&cfg.GUI)
setDefaults(&cfg.Options)
err := xml.NewDecoder(r).Decode(&cfg)
cfg.prepare()
return cfg, err
}
func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows == 0 {
_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
}
return nil
}
// An index update was received from the peer device
func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
if debug {
l.Debugln("model: receiving index update from device", deviceID.String()[:5], "for folder", folder)
}
m.fmut.Lock()
defer m.fmut.Unlock()
m.lmut.L.Lock()
defer m.lmut.L.Unlock()
if false == m.isFolderSharedWithDevice(folder, deviceID) {
if debug {
l.Debugln("model:", deviceID.String()[:5], "not shared with folder", folder, "so ignoring")
}
return
}
m.updateIndex(deviceID, folder, files)
}
func setup(deviceID protocol.DeviceID, dir string, peers ...protocol.DeviceID) (*config.Wrapper, *bolt.DB, string) {
configFile, _ := ioutil.TempFile(dir, "config")
realCfg := config.New(deviceID, deviceID.String()[:5])
cfg := config.Wrap(configFile.Name(), realCfg)
databasePath := path.Join(path.Dir(cfg.ConfigPath()), "boltdb")
database, _ := bolt.Open(databasePath, 0600, nil)
folder := "syncthingfusetest"
folderCfg := config.FolderConfiguration{
ID: folder,
CacheSize: "1MiB",
Devices: make([]stconfig.FolderDeviceConfiguration, len(peers)),
}
for i, peer := range peers {
folderCfg.Devices[i] = stconfig.FolderDeviceConfiguration{DeviceID: peer}
}
cfg.SetFolder(folderCfg)
return cfg, database, folder
}
func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
rows, err := s.prep["selectAddress"].Query(device.String())
if err != nil {
return nil, err
}
defer rows.Close()
var res []string
for rows.Next() {
var addr string
err := rows.Scan(&addr)
if err != nil {
log.Println("Scan:", err)
continue
}
res = append(res, addr)
}
return res, nil
}
func New(myID protocol.DeviceID, myName string) Configuration {
var cfg Configuration
cfg.Version = CurrentVersion
cfg.MyID = myID.String()
setDefaults(&cfg)
setDefaults(&cfg.GUI)
setDefaults(&cfg.Options)
thisDevice, _ := protocol.DeviceIDFromString(cfg.MyID)
thisDeviceCfg := config.NewDeviceConfiguration(thisDevice, myName)
thisDeviceCfg.Addresses = []string{"dynamic"}
cfg.Folders = []FolderConfiguration{}
cfg.Devices = []config.DeviceConfiguration{thisDeviceCfg}
cfg.prepare()
usr, _ := user.Current()
cfg.MountPoint = path.Join(usr.HomeDir, "SyncthingFUSE")
return cfg
}
func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
reqID := ctx.Value("id").(requestID)
t0 := time.Now()
res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "updateDevice in", time.Since(t0))
}
if rows, _ := res.RowsAffected(); rows == 0 {
t0 = time.Now()
_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "insertDevice in", time.Since(t0))
}
}
return nil
}
func (m *Model) updateIndex(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
m.fmut.Lock()
treeCache, ok := m.treeCaches[folder]
if !ok {
if debug {
l.Debugln("folder", folder, "from", deviceID.String(), "not configured, skipping")
}
m.fmut.Unlock()
return
}
for _, file := range files {
entry, existsInLocalModel := treeCache.GetEntry(file.Name)
if !existsInLocalModel {
if debug {
l.Debugln("file", file.Name, "from", deviceID.String(), "does not exist in local model, trying to add")
}
m.addToLocalModel(deviceID, folder, file)
continue
}
localToGlobal := entry.Version.Compare(file.Version)
if localToGlobal == protocol.Equal {
if debug {
l.Debugln("peer", deviceID.String(), "has same version for file", file.Name, ", adding as peer.")
}
m.addPeerForEntry(deviceID, folder, file)
continue
}
if localToGlobal == protocol.Lesser || localToGlobal == protocol.ConcurrentLesser {
if debug {
l.Debugln("peer", deviceID.String(), "has new version for file", file.Name, ", replacing current data.")
}
m.removeEntryFromLocalModel(folder, file.Name)
m.addToLocalModel(deviceID, folder, file)
// TODO probably want to re-fill disk cache if file small enough (10MB?) or within certain size (5%?)
continue
}
}
m.fmut.Unlock()
}
func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
var id protocol.DeviceID
copy(id[:], device.ID)
// Remember whether we already had a valid cache entry for this device.
// If the instance ID has changed the remote device has restarted since
// we last heard from it, so we should treat it as a new device.
ce, existsAlready := c.Get(id)
isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime || ce.instanceID != device.InstanceID
// Any empty or unspecified addresses should be set to the source address
// of the announcement. We also skip any addresses we can't parse.
l.Debugln("discover: Registering addresses for", id)
var validAddresses []string
for _, addr := range device.Addresses {
u, err := url.Parse(addr)
if err != nil {
continue
}
tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
if err != nil {
continue
}
if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
srcAddr, err := net.ResolveTCPAddr("tcp", src.String())
if err != nil {
continue
}
// Do not use IPv6 source address if requested scheme is tcp4
if u.Scheme == "tcp4" && srcAddr.IP.To4() == nil {
continue
}
// Do not use IPv4 source address if requested scheme is tcp6
if u.Scheme == "tcp6" && srcAddr.IP.To4() != nil {
continue
}
host, _, err := net.SplitHostPort(src.String())
if err != nil {
continue
}
u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
l.Debugf("discover: Reconstructed URL is %#v", u)
validAddresses = append(validAddresses, u.String())
l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr, u.String())
} else {
validAddresses = append(validAddresses, addr)
l.Debugf("discover: Accepted address %s verbatim", addr)
}
}
c.Set(id, CacheEntry{
Addresses: validAddresses,
when: time.Now(),
found: true,
instanceID: device.InstanceID,
})
if isNewDevice {
events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
"device": id.String(),
"addrs": validAddresses,
})
}
return isNewDevice
}
// requires write locks on fmut and lmut before entry
func (m *Model) updateIndex(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
treeCache, ok := m.treeCaches[folder]
if !ok {
if debug {
l.Debugln("folder", folder, "from", deviceID.String()[:5], "tree not configured, skipping")
}
return
}
fbc, ok := m.blockCaches[folder]
if !ok {
if debug {
l.Debugln("folder", folder, "from", deviceID.String()[:5], "block not configured, skipping")
}
return
}
for _, file := range files {
entry, existsInLocalModel := treeCache.GetEntry(file.Name)
var globalToLocal protocol.Ordering
if existsInLocalModel {
globalToLocal = file.Version.Compare(entry.Version)
}
if debug {
l.Debugln("updating entry for", file.Name, "from", deviceID.String()[:5], existsInLocalModel, globalToLocal)
}
// remove if necessary
if existsInLocalModel && (globalToLocal == protocol.Greater || (file.Version.Concurrent(entry.Version) && file.WinsConflict(entry))) {
if debug {
l.Debugln("remove entry for", file.Name, "from", deviceID.String()[:5])
}
treeCache.RemoveEntry(file.Name)
if m.isFilePinned(folder, file.Name) {
for _, block := range entry.Blocks {
fbc.UnpinBlock(block.Hash)
}
}
}
// add if necessary
if !existsInLocalModel || (globalToLocal == protocol.Greater || (file.Version.Concurrent(entry.Version) && file.WinsConflict(entry))) || (globalToLocal == protocol.Equal) {
if file.IsDeleted() {
if debug {
l.Debugln("peer", deviceID.String()[:5], "has deleted file, doing nothing", file.Name)
}
continue
}
if file.IsInvalid() {
if debug {
l.Debugln("peer", deviceID.String()[:5], "has invalid file, doing nothing", file.Name)
}
continue
}
if file.IsSymlink() {
if debug {
l.Debugln("peer", deviceID.String()[:5], "has symlink, doing nothing", file.Name)
}
continue
}
if debug && file.IsDirectory() {
l.Debugln("add directory", file.Name, "from", deviceID.String()[:5])
} else if debug {
l.Debugln("add file", file.Name, "from", deviceID.String()[:5])
}
treeCache.AddEntry(file, deviceID)
// trigger pull on unsatisfied blocks for pinned files
if m.isFilePinned(folder, file.Name) {
for i, block := range file.Blocks {
if false == fbc.HasPinnedBlock(block.Hash) {
blockStart := int64(i * protocol.BlockSize)
status := m.getOrCreatePullStatus("Pin fetch", folder, file.Name, block, blockStart, queued)
m.pinnedList.PushBack(status)
}
}
}
}
}
m.lmut.Broadcast()
}
// A cluster configuration message was received
func (m *Model) ClusterConfig(deviceID protocol.DeviceID, config protocol.ClusterConfig) {
if debug {
l.Debugln("model: receiving cluster config from device", deviceID.String()[:5])
}
}
func (p *Process) ResumeDevice(dev protocol.DeviceID) error {
_, err := p.Post("/rest/system/resume?device="+dev.String(), nil)
return err
}