// Main health check loop
// TODO merge with the monitorLoop?
func (m *Monitor) healthCheckLoop() {
m.wg.Add(1)
defer m.wg.Done()
for {
select {
case <-m.stop:
return
case k := <-m.healthCheckChan:
if utils.IsStopped(m.stop) {
return
}
m.mapLock.Lock()
mirror := m.mirrors[k]
m.mapLock.Unlock()
err := m.healthCheck(mirror.Mirror)
if err == mirrorNotScanned {
// Not removing the 'checking' lock is intended here so the mirror won't
// be checked again until the rsync/ftp scan is finished.
continue
}
m.mapLock.Lock()
if _, ok := m.mirrors[k]; ok {
if !database.RedisIsLoading(err) {
m.mirrors[k].lastCheck = time.Now().UTC().Unix()
}
m.mirrors[k].checking = false
}
m.mapLock.Unlock()
}
}
}
// Do an actual health check against a given mirror
func (m *Monitor) healthCheck(mirror mirrors.Mirror) error {
// Format log output
format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4)
// Copy the stop channel to make it nilable locally
stopflag := m.stop
// Get the URL to a random file available on this mirror
file, size, err := m.getRandomFile(mirror.ID)
if err != nil {
if err == redis.ErrNil {
return mirrorNotScanned
} else if !database.RedisIsLoading(err) {
log.Warningf(format+"Error: Cannot obtain a random file: %s", mirror.ID, err)
}
return err
}
// Prepare the HTTP request
req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil)
req.Header.Set("User-Agent", userAgent)
req.Close = true
done := make(chan bool)
var resp *http.Response
var elapsed time.Duration
// Execute the request inside a goroutine to allow aborting the request
go func() {
start := time.Now()
resp, err = m.httpClient.Do(req)
elapsed = time.Since(start)
if err == nil {
resp.Body.Close()
}
done <- true
}()
x:
for {
select {
case <-stopflag:
log.Debugf("Aborting health-check for %s", mirror.HttpURL)
m.httpTransport.CancelRequest(req)
stopflag = nil
case <-done:
if utils.IsStopped(m.stop) {
return nil
}
break x
}
}
if err != nil {
if opErr, ok := err.(*net.OpError); ok {
log.Debugf("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary())
}
mirrors.MarkMirrorDown(m.redis, mirror.ID, "Unreachable")
log.Errorf(format+"Error: %s (%dms)", mirror.ID, err.Error(), elapsed/time.Millisecond)
return err
}
contentLength := resp.Header.Get("Content-Length")
if resp.StatusCode == 404 {
mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file))
if GetConfig().DisableOnMissingFile {
mirrors.DisableMirror(m.redis, mirror.ID)
}
log.Errorf(format+"Error: File %s not found (error 404)", mirror.ID, file)
} else if resp.StatusCode != 200 {
mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", resp.StatusCode))
log.Warningf(format+"Down! Status: %d", mirror.ID, resp.StatusCode)
} else {
mirrors.MarkMirrorUp(m.redis, mirror.ID)
rsize, err := strconv.ParseInt(contentLength, 10, 64)
if err == nil && rsize != size {
log.Warningf(format+"File size mismatch! [%s] (%dms)", mirror.ID, file, elapsed/time.Millisecond)
} else {
log.Noticef(format+"Up! (%dms)", mirror.ID, elapsed/time.Millisecond)
}
}
return nil
}
// Main sync loop
// TODO merge with the monitorLoop?
func (m *Monitor) syncLoop() {
m.wg.Add(1)
defer m.wg.Done()
for {
select {
case <-m.stop:
return
case k := <-m.syncChan:
m.mapLock.Lock()
mirror := m.mirrors[k]
m.mapLock.Unlock()
conn := m.redis.Get()
scanning, err := scan.IsScanning(conn, k)
if err != nil {
conn.Close()
if !database.RedisIsLoading(err) {
log.Warningf("syncloop: %s", err.Error())
}
goto unlock
} else if scanning {
// A scan is already in progress on another node
conn.Close()
goto unlock
}
conn.Close()
log.Debugf("Scanning %s", k)
err = cli.NoSyncMethod
// First try to scan with rsync
if mirror.RsyncURL != "" {
err = scan.Scan(scan.RSYNC, m.redis, mirror.RsyncURL, k, m.stop)
}
// If it failed or rsync wasn't supported
// fallback to FTP
if err != nil && err != scan.ScanAborted && mirror.FtpURL != "" {
err = scan.Scan(scan.FTP, m.redis, mirror.FtpURL, k, m.stop)
}
if err == scan.ScanInProgress {
log.Warningf("%-30.30s Scan already in progress", k)
goto unlock
}
if mirror.Up == false {
select {
case m.healthCheckChan <- k:
default:
}
}
unlock:
m.mapLock.Lock()
if _, ok := m.mirrors[k]; ok {
m.mirrors[k].scanning = false
}
m.mapLock.Unlock()
}
}
}
