// Main health check loop // TODO merge with the monitorLoop? func (m *Monitor) healthCheckLoop() { m.wg.Add(1) defer m.wg.Done() for { select { case <-m.stop: return case k := <-m.healthCheckChan: if utils.IsStopped(m.stop) { return } m.mapLock.Lock() mirror := m.mirrors[k] m.mapLock.Unlock() err := m.healthCheck(mirror.Mirror) if err == mirrorNotScanned { // Not removing the 'checking' lock is intended here so the mirror won't // be checked again until the rsync/ftp scan is finished. continue } m.mapLock.Lock() if _, ok := m.mirrors[k]; ok { if !database.RedisIsLoading(err) { m.mirrors[k].lastCheck = time.Now().UTC().Unix() } m.mirrors[k].checking = false } m.mapLock.Unlock() } } }
// Walk inside an FTP repository func (f *FTPScanner) walkFtp(c *ftp.ServerConn, files []*filedata, path string, stop chan bool) ([]*filedata, error) { if utils.IsStopped(stop) { return nil, ScanAborted } flist, err := c.List(path) if err != nil { return nil, err } for _, e := range flist { if e.Type == ftp.EntryTypeFile { newf := &filedata{} newf.path = path + e.Name newf.size = int64(e.Size) files = append(files, newf) } else if e.Type == ftp.EntryTypeFolder { files, err = f.walkFtp(c, files, path+e.Name+"/", stop) if err != nil { return files, err } } } return files, err }
func ScanSource(r *database.Redis, stop chan bool) (err error) { s := &scan{ redis: r, } s.walkRedisConn = s.redis.Get() defer s.walkRedisConn.Close() if s.walkRedisConn.Err() != nil { return s.walkRedisConn.Err() } s.walkSourceFiles = make([]*filedata, 0, 1000) defer func() { // Reset the slice so it can be garbage collected s.walkSourceFiles = nil }() //TODO lock atomically inside redis to avoid two simultanous scan if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) { return fmt.Errorf("%s: No such file or directory", GetConfig().Repository) } log.Info("[source] Scanning the filesystem...") err = filepath.Walk(GetConfig().Repository, s.walkSource) if utils.IsStopped(stop) { return ScanAborted } if err != nil { return err } log.Info("[source] Indexing the files...") s.walkRedisConn.Send("MULTI") // Remove any left over s.walkRedisConn.Send("DEL", "FILES_TMP") // Add all the files to a temporary key count := 0 for _, e := range s.walkSourceFiles { s.walkRedisConn.Send("SADD", "FILES_TMP", e.path) count++ } _, err = s.walkRedisConn.Do("EXEC") if err != nil { return err } // Do a diff between the sets to get the removed files toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP")) // Create/Update the files' hash keys with the fresh infos s.walkRedisConn.Send("MULTI") for _, e := range s.walkSourceFiles { s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path), "size", e.size, "modTime", e.modTime, "sha1", e.sha1, "sha256", e.sha256, "md5", e.md5) // Publish update database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path) } // Remove old keys if len(toremove) > 0 { for _, e := range toremove { s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e)) // Publish update database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e)) } } // Finally rename the temporary sets containing the list // of files to the production key s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES") _, err = s.walkRedisConn.Do("EXEC") if err != nil { return err } log.Infof("[source] Scanned %d files", count) return nil }
// Do an actual health check against a given mirror func (m *Monitor) healthCheck(mirror mirrors.Mirror) error { // Format log output format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4) // Copy the stop channel to make it nilable locally stopflag := m.stop // Get the URL to a random file available on this mirror file, size, err := m.getRandomFile(mirror.ID) if err != nil { if err == redis.ErrNil { return mirrorNotScanned } else if !database.RedisIsLoading(err) { log.Warningf(format+"Error: Cannot obtain a random file: %s", mirror.ID, err) } return err } // Prepare the HTTP request req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil) req.Header.Set("User-Agent", userAgent) req.Close = true done := make(chan bool) var resp *http.Response var elapsed time.Duration // Execute the request inside a goroutine to allow aborting the request go func() { start := time.Now() resp, err = m.httpClient.Do(req) elapsed = time.Since(start) if err == nil { resp.Body.Close() } done <- true }() x: for { select { case <-stopflag: log.Debugf("Aborting health-check for %s", mirror.HttpURL) m.httpTransport.CancelRequest(req) stopflag = nil case <-done: if utils.IsStopped(m.stop) { return nil } break x } } if err != nil { if opErr, ok := err.(*net.OpError); ok { log.Debugf("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary()) } mirrors.MarkMirrorDown(m.redis, mirror.ID, "Unreachable") log.Errorf(format+"Error: %s (%dms)", mirror.ID, err.Error(), elapsed/time.Millisecond) return err } contentLength := resp.Header.Get("Content-Length") if resp.StatusCode == 404 { mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file)) if GetConfig().DisableOnMissingFile { mirrors.DisableMirror(m.redis, mirror.ID) } log.Errorf(format+"Error: File %s not found (error 404)", mirror.ID, file) } else if resp.StatusCode != 200 { mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", resp.StatusCode)) log.Warningf(format+"Down! Status: %d", mirror.ID, resp.StatusCode) } else { mirrors.MarkMirrorUp(m.redis, mirror.ID) rsize, err := strconv.ParseInt(contentLength, 10, 64) if err == nil && rsize != size { log.Warningf(format+"File size mismatch! [%s] (%dms)", mirror.ID, file, elapsed/time.Millisecond) } else { log.Noticef(format+"Up! (%dms)", mirror.ID, elapsed/time.Millisecond) } } return nil }
// Main monitor loop func (m *Monitor) MonitorLoop() { m.wg.Add(1) defer m.wg.Done() mirrorUpdateEvent := make(chan string, 10) m.redis.Pubsub.SubscribeEvent(database.MIRROR_UPDATE, mirrorUpdateEvent) // Scan the local repository m.retry(func() error { return m.scanRepository() }, 1*time.Second) // Synchronize the list of all known mirrors m.retry(func() error { ids, err := m.mirrorsID() if err != nil { return err } m.syncMirrorList(ids...) return nil }, 500*time.Millisecond) if utils.IsStopped(m.stop) { return } // Start the cluster manager m.cluster.Start() // Start the health check routines for i := 0; i < healthCheckThreads; i++ { go m.healthCheckLoop() } // Start the mirror sync routines for i := 0; i < GetConfig().ConcurrentSync; i++ { go m.syncLoop() } // Setup recurrent tasks var repositoryScanTicker <-chan time.Time repositoryScanInterval := -1 mirrorCheckTicker := time.NewTicker(1 * time.Second) // Disable the mirror check while stopping to avoid spurious events go func() { select { case <-m.stop: mirrorCheckTicker.Stop() } }() // Force a first configuration reload to setup the timers select { case m.configNotifier <- true: default: } for { select { case <-m.stop: return case id := <-mirrorUpdateEvent: m.syncMirrorList(id) case <-m.configNotifier: if repositoryScanInterval != GetConfig().RepositoryScanInterval { repositoryScanInterval = GetConfig().RepositoryScanInterval if repositoryScanInterval == 0 { repositoryScanTicker = nil } else { repositoryScanTicker = time.Tick(time.Duration(repositoryScanInterval) * time.Minute) } } case <-repositoryScanTicker: m.scanRepository() case <-mirrorCheckTicker.C: if m.redis.Failure() { continue } m.mapLock.Lock() for k, v := range m.mirrors { if !v.Enabled { // Ignore disabled mirrors continue } if v.NeedHealthCheck() && !v.IsChecking() && m.cluster.IsHandled(k) { select { case m.healthCheckChan <- k: m.mirrors[k].checking = true default: } } if v.NeedSync() && !v.IsScanning() && m.cluster.IsHandled(k) { select { case m.syncChan <- k: m.mirrors[k].scanning = true default: } } } m.mapLock.Unlock() } } }
func (r *RsyncScanner) Scan(url, identifier string, conn redis.Conn, stop chan bool) error { if !strings.HasPrefix(url, "rsync://") { return fmt.Errorf("%s does not start with rsync://", url) } // Always ensures there's a trailing slash if url[len(url)-1] != '/' { url = url + "/" } cmd := exec.Command("rsync", "-r", "--no-motd", "--timeout=30", "--contimeout=30", url) stdout, err := cmd.StdoutPipe() if err != nil { return err } // Pipe stdout reader := bufio.NewReader(stdout) if utils.IsStopped(stop) { return ScanAborted } // Start the process if err := cmd.Start(); err != nil { return err } log.Infof("[%s] Requesting file list via rsync...", identifier) scanfinished := make(chan bool) go func() { select { case <-stop: cmd.Process.Kill() return case <-scanfinished: return } }() defer close(scanfinished) // Get the list of all source files (we do not want to // index files than are not provided by the source) //sourceFiles, err := redis.Values(conn.Do("SMEMBERS", "FILES")) //if err != nil { // log.Errorf("[%s] Cannot get the list of source files", identifier) // return err //} count := 0 line, err := readln(reader) for err == nil { var size int64 var f filedata if utils.IsStopped(stop) { return ScanAborted } // Parse one line returned by rsync ret := rsyncOutputLine.FindStringSubmatch(line) if ret[0][0] == 'd' || ret[0][0] == 'l' { // Skip directories and links goto cont } // Add the leading slash if ret[4][0] != '/' { ret[4] = "/" + ret[4] } // Remove the commas in the file size ret[1] = strings.Replace(ret[1], ",", "", -1) // Convert the size to int size, err = strconv.ParseInt(ret[1], 10, 64) if err != nil { log.Errorf("[%s] ScanRsync: Invalid size: %s", identifier, ret[1]) goto cont } // Fill the struct f.size = size f.path = ret[4] if os.Getenv("DEBUG") != "" { //fmt.Printf("[%s] %s", identifier, f.path) } r.scan.ScannerAddFile(f) count++ cont: line, err = readln(reader) } if err1 := cmd.Wait(); err1 != nil { switch err1.Error() { case "exit status 5": err1 = errors.New("rsync: Error starting client-server protocol") break case "exit status 10": err1 = errors.New("rsync: Error in socket I/O") break case "exit status 11": err1 = errors.New("rsync: Error in file I/O") break case "exit status 30": err1 = errors.New("rsync: Timeout in data send/receive") break default: if utils.IsStopped(stop) { err1 = ScanAborted } else { err1 = errors.New("rsync: " + err1.Error()) } } return err1 } if err != io.EOF { return err } return nil }
func (f *FTPScanner) Scan(scanurl, identifier string, conn redis.Conn, stop chan bool) error { if !strings.HasPrefix(scanurl, "ftp://") { return fmt.Errorf("%s does not start with ftp://", scanurl) } ftpurl, err := url.Parse(scanurl) if err != nil { return err } host := ftpurl.Host if !strings.Contains(host, ":") { host += ":21" } if utils.IsStopped(stop) { return ScanAborted } c, err := ftp.DialTimeout(host, 5*time.Second) if err != nil { return err } defer c.Quit() username, password := "******", "anonymous" if ftpurl.User != nil { username = ftpurl.User.Username() pass, hasPassword := ftpurl.User.Password() if hasPassword { password = pass } } err = c.Login(username, password) if err != nil { return err } log.Infof("[%s] Requesting file list via ftp...", identifier) var files []*filedata = make([]*filedata, 0, 1000) err = c.ChangeDir(ftpurl.Path) if err != nil { return fmt.Errorf("ftp error %s", err.Error()) } prefixDir, err := c.CurrentDir() if err != nil { return fmt.Errorf("ftp error %s", err.Error()) } if os.Getenv("DEBUG") != "" { _ = prefixDir //fmt.Printf("[%s] Current dir: %s\n", identifier, prefixDir) } prefix := ftpurl.Path // Remove the trailing slash prefix = strings.TrimRight(prefix, "/") files, err = f.walkFtp(c, files, prefix+"/", stop) if err != nil { return fmt.Errorf("ftp error %s", err.Error()) } count := 0 for _, fd := range files { fd.path = strings.TrimPrefix(fd.path, prefix) if os.Getenv("DEBUG") != "" { fmt.Printf("%s\n", fd.path) } f.scan.ScannerAddFile(*fd) count++ } return nil }