func (s *scan) ScannerAddFile(f filedata) {
s.count++
// Add all the files to a temporary key
s.conn.Send("SADD", s.filesTmpKey, f.path)
// Mark the file as being supported by this mirror
rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
s.conn.Send("SADD", rk, s.identifier)
// Save the size of the current file found on this mirror
ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
s.conn.Send("HSET", ik, "size", f.size)
// Publish update
database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}
func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
s := &scan{
redis: r,
identifier: identifier,
}
var scanner Scanner
switch typ {
case RSYNC:
scanner = &RsyncScanner{
scan: s,
}
case FTP:
scanner = &FTPScanner{
scan: s,
}
default:
panic(fmt.Sprintf("Unknown scanner"))
}
// Connect to the database
conn := s.redis.Get()
defer conn.Close()
s.conn = conn
lockKey := fmt.Sprintf("SCANNING_%s", identifier)
// Try to aquire a lock so we don't have a scanning race
// from different nodes.
lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
if err != nil {
return err
}
if lock {
// Lock aquired.
defer conn.Do("DEL", lockKey)
// Make the key expire automatically in case our process gets killed
conn.Do("EXPIRE", lockKey, 600)
} else {
return ScanInProgress
}
s.setLastSync(conn, identifier, false)
conn.Send("MULTI")
s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)
// Remove any left over
conn.Send("DEL", s.filesTmpKey)
err = scanner.Scan(url, identifier, conn, stop)
if err != nil {
// Discard MULTI
s.ScannerDiscard()
// Remove the temporary key
conn.Do("DEL", s.filesTmpKey)
log.Errorf("[%s] %s", identifier, err.Error())
return err
}
// Exec multi
s.ScannerCommit()
// Get the list of files no more present on this mirror
toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
if err != nil {
return err
}
// Remove this mirror from the given file SET
if len(toremove) > 0 {
conn.Send("MULTI")
for _, e := range toremove {
log.Debugf("[%s] Removing %s from mirror", identifier, e)
conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
// Publish update
database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))
}
_, err = conn.Do("EXEC")
if err != nil {
return err
}
}
// Finally rename the temporary sets containing the list
// of files for this mirror to the production key
_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
if err != nil {
return err
}
sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)
// Count the number of files known on the remote end
common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))
if err != nil {
return err
}
s.setLastSync(conn, identifier, true)
log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
return nil
}
func ScanSource(r *database.Redis, stop chan bool) (err error) {
s := &scan{
redis: r,
}
s.walkRedisConn = s.redis.Get()
defer s.walkRedisConn.Close()
if s.walkRedisConn.Err() != nil {
return s.walkRedisConn.Err()
}
s.walkSourceFiles = make([]*filedata, 0, 1000)
defer func() {
// Reset the slice so it can be garbage collected
s.walkSourceFiles = nil
}()
//TODO lock atomically inside redis to avoid two simultanous scan
if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
}
log.Info("[source] Scanning the filesystem...")
err = filepath.Walk(GetConfig().Repository, s.walkSource)
if utils.IsStopped(stop) {
return ScanAborted
}
if err != nil {
return err
}
log.Info("[source] Indexing the files...")
s.walkRedisConn.Send("MULTI")
// Remove any left over
s.walkRedisConn.Send("DEL", "FILES_TMP")
// Add all the files to a temporary key
count := 0
for _, e := range s.walkSourceFiles {
s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
count++
}
_, err = s.walkRedisConn.Do("EXEC")
if err != nil {
return err
}
// Do a diff between the sets to get the removed files
toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))
// Create/Update the files' hash keys with the fresh infos
s.walkRedisConn.Send("MULTI")
for _, e := range s.walkSourceFiles {
s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
"size", e.size,
"modTime", e.modTime,
"sha1", e.sha1,
"sha256", e.sha256,
"md5", e.md5)
// Publish update
database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
}
// Remove old keys
if len(toremove) > 0 {
for _, e := range toremove {
s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))
// Publish update
database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
}
}
// Finally rename the temporary sets containing the list
// of files to the production key
s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")
_, err = s.walkRedisConn.Do("EXEC")
if err != nil {
return err
}
log.Infof("[source] Scanned %d files", count)
return nil
}