// Main health check loop
// TODO merge with the monitorLoop?
func (m *Monitor) healthCheckLoop() {
m.wg.Add(1)
defer m.wg.Done()
for {
select {
case <-m.stop:
return
case k := <-m.healthCheckChan:
if utils.IsStopped(m.stop) {
return
}
m.mapLock.Lock()
mirror := m.mirrors[k]
m.mapLock.Unlock()
err := m.healthCheck(mirror.Mirror)
if err == mirrorNotScanned {
// Not removing the 'checking' lock is intended here so the mirror won't
// be checked again until the rsync/ftp scan is finished.
continue
}
m.mapLock.Lock()
if _, ok := m.mirrors[k]; ok {
if !database.RedisIsLoading(err) {
m.mirrors[k].lastCheck = time.Now().UTC().Unix()
}
m.mirrors[k].checking = false
}
m.mapLock.Unlock()
}
}
}
// Walk inside an FTP repository
func (f *FTPScanner) walkFtp(c *ftp.ServerConn, files []*filedata, path string, stop chan bool) ([]*filedata, error) {
if utils.IsStopped(stop) {
return nil, ScanAborted
}
flist, err := c.List(path)
if err != nil {
return nil, err
}
for _, e := range flist {
if e.Type == ftp.EntryTypeFile {
newf := &filedata{}
newf.path = path + e.Name
newf.size = int64(e.Size)
files = append(files, newf)
} else if e.Type == ftp.EntryTypeFolder {
files, err = f.walkFtp(c, files, path+e.Name+"/", stop)
if err != nil {
return files, err
}
}
}
return files, err
}
func ScanSource(r *database.Redis, stop chan bool) (err error) {
s := &scan{
redis: r,
}
s.walkRedisConn = s.redis.Get()
defer s.walkRedisConn.Close()
if s.walkRedisConn.Err() != nil {
return s.walkRedisConn.Err()
}
s.walkSourceFiles = make([]*filedata, 0, 1000)
defer func() {
// Reset the slice so it can be garbage collected
s.walkSourceFiles = nil
}()
//TODO lock atomically inside redis to avoid two simultanous scan
if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
}
log.Info("[source] Scanning the filesystem...")
err = filepath.Walk(GetConfig().Repository, s.walkSource)
if utils.IsStopped(stop) {
return ScanAborted
}
if err != nil {
return err
}
log.Info("[source] Indexing the files...")
s.walkRedisConn.Send("MULTI")
// Remove any left over
s.walkRedisConn.Send("DEL", "FILES_TMP")
// Add all the files to a temporary key
count := 0
for _, e := range s.walkSourceFiles {
s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
count++
}
_, err = s.walkRedisConn.Do("EXEC")
if err != nil {
return err
}
// Do a diff between the sets to get the removed files
toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))
// Create/Update the files' hash keys with the fresh infos
s.walkRedisConn.Send("MULTI")
for _, e := range s.walkSourceFiles {
s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
"size", e.size,
"modTime", e.modTime,
"sha1", e.sha1,
"sha256", e.sha256,
"md5", e.md5)
// Publish update
database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
}
// Remove old keys
if len(toremove) > 0 {
for _, e := range toremove {
s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))
// Publish update
database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
}
}
// Finally rename the temporary sets containing the list
// of files to the production key
s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")
_, err = s.walkRedisConn.Do("EXEC")
if err != nil {
return err
}
log.Infof("[source] Scanned %d files", count)
return nil
}
// Do an actual health check against a given mirror
func (m *Monitor) healthCheck(mirror mirrors.Mirror) error {
// Format log output
format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4)
// Copy the stop channel to make it nilable locally
stopflag := m.stop
// Get the URL to a random file available on this mirror
file, size, err := m.getRandomFile(mirror.ID)
if err != nil {
if err == redis.ErrNil {
return mirrorNotScanned
} else if !database.RedisIsLoading(err) {
log.Warningf(format+"Error: Cannot obtain a random file: %s", mirror.ID, err)
}
return err
}
// Prepare the HTTP request
req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil)
req.Header.Set("User-Agent", userAgent)
req.Close = true
done := make(chan bool)
var resp *http.Response
var elapsed time.Duration
// Execute the request inside a goroutine to allow aborting the request
go func() {
start := time.Now()
resp, err = m.httpClient.Do(req)
elapsed = time.Since(start)
if err == nil {
resp.Body.Close()
}
done <- true
}()
x:
for {
select {
case <-stopflag:
log.Debugf("Aborting health-check for %s", mirror.HttpURL)
m.httpTransport.CancelRequest(req)
stopflag = nil
case <-done:
if utils.IsStopped(m.stop) {
return nil
}
break x
}
}
if err != nil {
if opErr, ok := err.(*net.OpError); ok {
log.Debugf("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary())
}
mirrors.MarkMirrorDown(m.redis, mirror.ID, "Unreachable")
log.Errorf(format+"Error: %s (%dms)", mirror.ID, err.Error(), elapsed/time.Millisecond)
return err
}
contentLength := resp.Header.Get("Content-Length")
if resp.StatusCode == 404 {
mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file))
if GetConfig().DisableOnMissingFile {
mirrors.DisableMirror(m.redis, mirror.ID)
}
log.Errorf(format+"Error: File %s not found (error 404)", mirror.ID, file)
} else if resp.StatusCode != 200 {
mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", resp.StatusCode))
log.Warningf(format+"Down! Status: %d", mirror.ID, resp.StatusCode)
} else {
mirrors.MarkMirrorUp(m.redis, mirror.ID)
rsize, err := strconv.ParseInt(contentLength, 10, 64)
if err == nil && rsize != size {
log.Warningf(format+"File size mismatch! [%s] (%dms)", mirror.ID, file, elapsed/time.Millisecond)
} else {
log.Noticef(format+"Up! (%dms)", mirror.ID, elapsed/time.Millisecond)
}
}
return nil
}
// Main monitor loop
func (m *Monitor) MonitorLoop() {
m.wg.Add(1)
defer m.wg.Done()
mirrorUpdateEvent := make(chan string, 10)
m.redis.Pubsub.SubscribeEvent(database.MIRROR_UPDATE, mirrorUpdateEvent)
// Scan the local repository
m.retry(func() error {
return m.scanRepository()
}, 1*time.Second)
// Synchronize the list of all known mirrors
m.retry(func() error {
ids, err := m.mirrorsID()
if err != nil {
return err
}
m.syncMirrorList(ids...)
return nil
}, 500*time.Millisecond)
if utils.IsStopped(m.stop) {
return
}
// Start the cluster manager
m.cluster.Start()
// Start the health check routines
for i := 0; i < healthCheckThreads; i++ {
go m.healthCheckLoop()
}
// Start the mirror sync routines
for i := 0; i < GetConfig().ConcurrentSync; i++ {
go m.syncLoop()
}
// Setup recurrent tasks
var repositoryScanTicker <-chan time.Time
repositoryScanInterval := -1
mirrorCheckTicker := time.NewTicker(1 * time.Second)
// Disable the mirror check while stopping to avoid spurious events
go func() {
select {
case <-m.stop:
mirrorCheckTicker.Stop()
}
}()
// Force a first configuration reload to setup the timers
select {
case m.configNotifier <- true:
default:
}
for {
select {
case <-m.stop:
return
case id := <-mirrorUpdateEvent:
m.syncMirrorList(id)
case <-m.configNotifier:
if repositoryScanInterval != GetConfig().RepositoryScanInterval {
repositoryScanInterval = GetConfig().RepositoryScanInterval
if repositoryScanInterval == 0 {
repositoryScanTicker = nil
} else {
repositoryScanTicker = time.Tick(time.Duration(repositoryScanInterval) * time.Minute)
}
}
case <-repositoryScanTicker:
m.scanRepository()
case <-mirrorCheckTicker.C:
if m.redis.Failure() {
continue
}
m.mapLock.Lock()
for k, v := range m.mirrors {
if !v.Enabled {
// Ignore disabled mirrors
continue
}
if v.NeedHealthCheck() && !v.IsChecking() && m.cluster.IsHandled(k) {
select {
case m.healthCheckChan <- k:
m.mirrors[k].checking = true
default:
}
}
if v.NeedSync() && !v.IsScanning() && m.cluster.IsHandled(k) {
select {
case m.syncChan <- k:
m.mirrors[k].scanning = true
default:
}
}
}
m.mapLock.Unlock()
}
}
}
func (r *RsyncScanner) Scan(url, identifier string, conn redis.Conn, stop chan bool) error {
if !strings.HasPrefix(url, "rsync://") {
return fmt.Errorf("%s does not start with rsync://", url)
}
// Always ensures there's a trailing slash
if url[len(url)-1] != '/' {
url = url + "/"
}
cmd := exec.Command("rsync", "-r", "--no-motd", "--timeout=30", "--contimeout=30", url)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
// Pipe stdout
reader := bufio.NewReader(stdout)
if utils.IsStopped(stop) {
return ScanAborted
}
// Start the process
if err := cmd.Start(); err != nil {
return err
}
log.Infof("[%s] Requesting file list via rsync...", identifier)
scanfinished := make(chan bool)
go func() {
select {
case <-stop:
cmd.Process.Kill()
return
case <-scanfinished:
return
}
}()
defer close(scanfinished)
// Get the list of all source files (we do not want to
// index files than are not provided by the source)
//sourceFiles, err := redis.Values(conn.Do("SMEMBERS", "FILES"))
//if err != nil {
// log.Errorf("[%s] Cannot get the list of source files", identifier)
// return err
//}
count := 0
line, err := readln(reader)
for err == nil {
var size int64
var f filedata
if utils.IsStopped(stop) {
return ScanAborted
}
// Parse one line returned by rsync
ret := rsyncOutputLine.FindStringSubmatch(line)
if ret[0][0] == 'd' || ret[0][0] == 'l' {
// Skip directories and links
goto cont
}
// Add the leading slash
if ret[4][0] != '/' {
ret[4] = "/" + ret[4]
}
// Remove the commas in the file size
ret[1] = strings.Replace(ret[1], ",", "", -1)
// Convert the size to int
size, err = strconv.ParseInt(ret[1], 10, 64)
if err != nil {
log.Errorf("[%s] ScanRsync: Invalid size: %s", identifier, ret[1])
goto cont
}
// Fill the struct
f.size = size
f.path = ret[4]
if os.Getenv("DEBUG") != "" {
//fmt.Printf("[%s] %s", identifier, f.path)
}
r.scan.ScannerAddFile(f)
count++
cont:
line, err = readln(reader)
}
if err1 := cmd.Wait(); err1 != nil {
switch err1.Error() {
case "exit status 5":
err1 = errors.New("rsync: Error starting client-server protocol")
break
case "exit status 10":
err1 = errors.New("rsync: Error in socket I/O")
break
case "exit status 11":
err1 = errors.New("rsync: Error in file I/O")
break
case "exit status 30":
err1 = errors.New("rsync: Timeout in data send/receive")
break
default:
if utils.IsStopped(stop) {
err1 = ScanAborted
} else {
err1 = errors.New("rsync: " + err1.Error())
}
}
return err1
}
if err != io.EOF {
return err
}
return nil
}
func (f *FTPScanner) Scan(scanurl, identifier string, conn redis.Conn, stop chan bool) error {
if !strings.HasPrefix(scanurl, "ftp://") {
return fmt.Errorf("%s does not start with ftp://", scanurl)
}
ftpurl, err := url.Parse(scanurl)
if err != nil {
return err
}
host := ftpurl.Host
if !strings.Contains(host, ":") {
host += ":21"
}
if utils.IsStopped(stop) {
return ScanAborted
}
c, err := ftp.DialTimeout(host, 5*time.Second)
if err != nil {
return err
}
defer c.Quit()
username, password := "******", "anonymous"
if ftpurl.User != nil {
username = ftpurl.User.Username()
pass, hasPassword := ftpurl.User.Password()
if hasPassword {
password = pass
}
}
err = c.Login(username, password)
if err != nil {
return err
}
log.Infof("[%s] Requesting file list via ftp...", identifier)
var files []*filedata = make([]*filedata, 0, 1000)
err = c.ChangeDir(ftpurl.Path)
if err != nil {
return fmt.Errorf("ftp error %s", err.Error())
}
prefixDir, err := c.CurrentDir()
if err != nil {
return fmt.Errorf("ftp error %s", err.Error())
}
if os.Getenv("DEBUG") != "" {
_ = prefixDir
//fmt.Printf("[%s] Current dir: %s\n", identifier, prefixDir)
}
prefix := ftpurl.Path
// Remove the trailing slash
prefix = strings.TrimRight(prefix, "/")
files, err = f.walkFtp(c, files, prefix+"/", stop)
if err != nil {
return fmt.Errorf("ftp error %s", err.Error())
}
count := 0
for _, fd := range files {
fd.path = strings.TrimPrefix(fd.path, prefix)
if os.Getenv("DEBUG") != "" {
fmt.Printf("%s\n", fd.path)
}
f.scan.ScannerAddFile(*fd)
count++
}
return nil
}
