// checkHealth check the store health.
func (p *Pitchfork) checkHealth(store *meta.Store, stop chan struct{}) (err error) {
var (
status, i int
volume *meta.Volume
volumes []*meta.Volume
)
log.Infof("check_health job start")
for {
select {
case <-stop:
log.Infof("check_health job stop")
return
case <-time.After(p.config.GetInterval):
break
}
status = store.Status
store.Status = meta.StoreStatusHealth
for i = 0; i < retryCount; i++ {
if volumes, err = store.Info(); err == nil {
break
}
time.Sleep(retrySleep)
}
if err == nil {
for _, volume = range volumes {
if volume.Block.LastErr != nil {
log.Infof("get store block.lastErr:%s host:%s", volume.Block.LastErr, store.Stat)
store.Status = meta.StoreStatusFail
break
} else if volume.Block.Full() {
log.Infof("block: %s, offset: %d", volume.Block.File, volume.Block.Offset)
store.Status = meta.StoreStatusRead
}
if err = p.zk.SetVolumeState(volume); err != nil {
log.Errorf("zk.SetVolumeState() error(%v)", err)
}
}
} else {
log.Errorf("get store info failed, retry host:%s", store.Stat)
store.Status = meta.StoreStatusFail
}
if status != store.Status {
if err = p.zk.SetStore(store); err != nil {
log.Errorf("update store zk status failed, retry")
continue
}
}
}
return
}
// checkNeedles check the store health.
func (p *Pitchfork) checkNeedles(store *meta.Store, stop chan struct{}) (err error) {
var (
status, i int
needle *meta.Needle
volume *meta.Volume
volumes []*meta.Volume
)
log.Infof("checkNeedles job start")
for {
select {
case <-stop:
log.Infof("checkNeedles job stop")
return
case <-time.After(p.config.HeadInterval):
break
}
if volumes, err = store.Info(); err != nil {
log.Errorf("get store info failed, retry host:%s", store.Stat)
continue
}
status = store.Status
for _, volume = range volumes {
if volume.Block.LastErr != nil {
break
} else {
for _, needle = range volume.CheckNeedles {
for i = 0; i < retryCount; i++ {
if err = store.Head(needle, volume.Id); err == nil {
break
}
}
if err != nil {
log.Errorf("head store failed, needle:%d host:%s", needle.Key, store.Stat)
store.Status = meta.StoreStatusFail
goto feedback
}
}
}
}
feedback:
if status != store.Status {
if err = p.zk.SetStore(store); err != nil {
log.Errorf("update store zk status failed, retry")
continue
}
if err = p.zk.setRoot(); err != nil {
log.Errorf("setRoot zk failed")
}
}
}
return
}