func (self *DetailStatsInfo) SaveHistory(fileName string) error {
nsqLog.LogDebugf("persisting history stats to %s", fileName)
data, err := json.Marshal(self.historyStatsInfo)
if err != nil {
nsqLog.LogWarningf("failed to save history stats: %v", err)
return err
}
tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
nsqLog.LogWarningf("failed to save history stats: %v", err)
return err
}
_, err = f.Write(data)
if err != nil {
f.Close()
nsqLog.LogWarningf("failed to save history stats: %v", err)
return err
}
f.Sync()
f.Close()
err = util.AtomicRename(tmpFileName, fileName)
if err != nil {
nsqLog.LogWarningf("failed to save history stats: %v", err)
}
return err
}
func (n *NSQD) PersistMetadata(currentTopicMap map[string]map[int]*Topic) error {
if !atomic.CompareAndSwapInt32(&n.persisting, 0, 1) {
nsqLog.Logf("NSQ: persisting is already running")
return nil
}
defer atomic.StoreInt32(&n.persisting, 0)
// persist metadata about what topics/channels we have
// so that upon restart we can get back to the same state
fileName := fmt.Sprintf(path.Join(n.GetOpts().DataPath, "nsqd.%d.dat"), n.GetOpts().ID)
nsqLog.Logf("NSQ: persisting topic/channel metadata to %s", fileName)
js := make(map[string]interface{})
topics := []interface{}{}
for _, topicParts := range currentTopicMap {
for _, topic := range topicParts {
if topic.ephemeral {
continue
}
topicData := make(map[string]interface{})
topicData["name"] = topic.GetTopicName()
topicData["partition"] = topic.GetTopicPart()
// we save the channels to topic, but for compatible we need save empty channels to json
channels := []interface{}{}
err := topic.SaveChannelMeta()
if err != nil {
nsqLog.Warningf("save topic %v channel meta failed: %v", topic.GetFullName(), err)
}
topicData["channels"] = channels
topics = append(topics, topicData)
}
}
js["version"] = version.Binary
js["topics"] = topics
data, err := json.Marshal(&js)
if err != nil {
return err
}
tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
_, err = f.Write(data)
if err != nil {
f.Close()
return err
}
f.Sync()
f.Close()
err = util.AtomicRename(tmpFileName, fileName)
if err != nil {
return err
}
return nil
}
// persistMetaData atomically writes state to the filesystem
func (d *diskQueueReader) persistMetaData() error {
var f *os.File
var err error
fileName := d.metaDataFileName(true)
tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
// write to tmp file
f, err = os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return err
}
_, err = fmt.Fprintf(f, "%d\n%d\n%d,%d,%d\n%d,%d,%d\n",
d.confirmedQueueInfo.TotalMsgCnt(),
d.queueEndInfo.totalMsgCnt,
d.confirmedQueueInfo.EndOffset.FileNum, d.confirmedQueueInfo.EndOffset.Pos, d.confirmedQueueInfo.Offset(),
d.queueEndInfo.EndOffset.FileNum, d.queueEndInfo.EndOffset.Pos, d.queueEndInfo.Offset())
if err != nil {
f.Close()
return err
}
f.Sync()
f.Close()
// atomically rename
return util.AtomicRename(tmpFileName, fileName)
}
func (t *Topic) MarkAsRemoved() (string, error) {
t.Lock()
defer t.Unlock()
atomic.CompareAndSwapInt32(&t.exitFlag, 0, 1)
nsqLog.Logf("TOPIC(%s): deleting", t.GetFullName())
// since we are explicitly deleting a topic (not just at system exit time)
// de-register this from the lookupd
t.notifyCall(t)
t.channelLock.Lock()
for _, channel := range t.channelMap {
delete(t.channelMap, channel.name)
channel.Delete()
}
t.channelLock.Unlock()
// we should move our partition only
renamePath := t.dataPath + "-removed-" + strconv.Itoa(int(time.Now().Unix()))
nsqLog.Warningf("mark the topic %v as removed: %v", t.GetFullName(), renamePath)
os.MkdirAll(renamePath, 0755)
err := t.backend.RemoveTo(renamePath)
if err != nil {
nsqLog.Errorf("failed to mark the topic %v as removed %v failed: %v", t.GetFullName(), renamePath, err)
}
util.AtomicRename(t.getMagicCodeFileName(), path.Join(renamePath, "magic"+strconv.Itoa(t.partition)))
t.removeHistoryStat()
t.RemoveChannelMeta()
t.removeMagicCode()
return renamePath, err
}
func (d *diskQueueWriter) RemoveTo(destPath string) error {
d.Lock()
defer d.Unlock()
d.exitFlag = 1
nsqLog.Logf("DISKQUEUE(%s): removing to %v", d.name, destPath)
d.sync()
d.closeCurrentFile()
d.saveFileOffsetMeta()
for i := int64(0); i <= d.diskWriteEnd.EndOffset.FileNum; i++ {
fn := d.fileName(i)
destFile := fmt.Sprintf(path.Join(destPath, "%s.diskqueue.%06d.dat"), d.name, i)
innerErr := util.AtomicRename(fn, destFile)
nsqLog.Logf("DISKQUEUE(%s): renamed data file %v to %v", d.name, fn, destFile)
if innerErr != nil && !os.IsNotExist(innerErr) {
nsqLog.LogErrorf("diskqueue(%s) failed to remove data file - %s", d.name, innerErr)
}
fName := d.fileName(i) + ".offsetmeta.dat"
innerErr = util.AtomicRename(fName, destFile+".offsetmeta.dat")
nsqLog.Logf("DISKQUEUE(%s): rename offset meta file %v ", d.name, fName)
if innerErr != nil && !os.IsNotExist(innerErr) {
nsqLog.LogErrorf("diskqueue(%s) failed to remove offset meta file %v - %s", d.name, fName, innerErr)
}
}
d.diskWriteEnd.EndOffset.FileNum++
d.diskWriteEnd.EndOffset.Pos = 0
d.diskReadEnd = d.diskWriteEnd
destFile := fmt.Sprintf(path.Join(destPath, "%s.diskqueue.meta.writer.dat"), d.name)
nsqLog.Logf("DISKQUEUE(%s): rename meta file to %v", d.name, destFile)
innerErr := util.AtomicRename(d.metaDataFileName(), destFile)
if innerErr != nil && !os.IsNotExist(innerErr) {
nsqLog.LogErrorf("diskqueue(%s) failed to remove metadata file - %s", d.name, innerErr)
return innerErr
}
destFile = fmt.Sprintf(path.Join(destPath, "%s.diskqueue.meta.extra.dat"), d.name)
util.AtomicRename(d.extraMetaFileName(), destFile)
return nil
}
func (d *diskQueueWriter) saveExtraMeta() error {
var err error
fileName := d.extraMetaFileName()
tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
var tmp extraMeta
tmp.SegOffset = d.diskQueueStart.EndOffset
tmp.VirtualOffset = d.diskQueueStart.Offset()
tmp.TotalMsgCnt = d.diskQueueStart.TotalMsgCnt()
data, _ := json.Marshal(tmp)
err = ioutil.WriteFile(tmpFileName, data, 0644)
if err != nil {
return err
}
// atomically rename
return util.AtomicRename(tmpFileName, fileName)
}
func (t *Topic) SaveChannelMeta() error {
fileName := t.getChannelMetaFileName()
channels := make([]*ChannelMetaInfo, 0)
t.channelLock.RLock()
for _, channel := range t.channelMap {
channel.RLock()
if !channel.ephemeral {
meta := &ChannelMetaInfo{
Name: channel.name,
Paused: channel.IsPaused(),
}
channels = append(channels, meta)
}
channel.RUnlock()
}
t.channelLock.RUnlock()
d, err := json.Marshal(channels)
if err != nil {
return err
}
tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
_, err = f.Write(d)
if err != nil {
f.Close()
return err
}
f.Sync()
f.Close()
err = util.AtomicRename(tmpFileName, fileName)
if err != nil {
return err
}
return nil
}