示例#1
0
func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}
示例#2
0
func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return ScanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}
示例#3
0
func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.Get()
	defer s.walkRedisConn.Close()

	if s.walkRedisConn.Err() != nil {
		return s.walkRedisConn.Err()
	}

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}