Example #1
0
func (self *DetailStatsInfo) SaveHistory(fileName string) error {
	nsqLog.LogDebugf("persisting history stats to %s", fileName)
	data, err := json.Marshal(self.historyStatsInfo)
	if err != nil {
		nsqLog.LogWarningf("failed to save history stats: %v", err)
		return err
	}
	tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
	f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		nsqLog.LogWarningf("failed to save history stats: %v", err)
		return err
	}
	_, err = f.Write(data)
	if err != nil {
		f.Close()
		nsqLog.LogWarningf("failed to save history stats: %v", err)
		return err
	}
	f.Sync()
	f.Close()

	err = util.AtomicRename(tmpFileName, fileName)
	if err != nil {
		nsqLog.LogWarningf("failed to save history stats: %v", err)
	}
	return err
}
Example #2
0
func (n *NSQD) PersistMetadata(currentTopicMap map[string]map[int]*Topic) error {
	if !atomic.CompareAndSwapInt32(&n.persisting, 0, 1) {
		nsqLog.Logf("NSQ: persisting is already running")
		return nil
	}
	defer atomic.StoreInt32(&n.persisting, 0)
	// persist metadata about what topics/channels we have
	// so that upon restart we can get back to the same state
	fileName := fmt.Sprintf(path.Join(n.GetOpts().DataPath, "nsqd.%d.dat"), n.GetOpts().ID)
	nsqLog.Logf("NSQ: persisting topic/channel metadata to %s", fileName)

	js := make(map[string]interface{})
	topics := []interface{}{}
	for _, topicParts := range currentTopicMap {
		for _, topic := range topicParts {
			if topic.ephemeral {
				continue
			}
			topicData := make(map[string]interface{})
			topicData["name"] = topic.GetTopicName()
			topicData["partition"] = topic.GetTopicPart()
			// we save the channels to topic, but for compatible we need save empty channels to json
			channels := []interface{}{}
			err := topic.SaveChannelMeta()
			if err != nil {
				nsqLog.Warningf("save topic %v channel meta failed: %v", topic.GetFullName(), err)
			}
			topicData["channels"] = channels
			topics = append(topics, topicData)
		}
	}
	js["version"] = version.Binary
	js["topics"] = topics

	data, err := json.Marshal(&js)
	if err != nil {
		return err
	}

	tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
	f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}

	_, err = f.Write(data)
	if err != nil {
		f.Close()
		return err
	}
	f.Sync()
	f.Close()

	err = util.AtomicRename(tmpFileName, fileName)
	if err != nil {
		return err
	}

	return nil
}
Example #3
0
// persistMetaData atomically writes state to the filesystem
func (d *diskQueueReader) persistMetaData() error {
	var f *os.File
	var err error

	fileName := d.metaDataFileName(true)
	tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())

	// write to tmp file
	f, err = os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, "%d\n%d\n%d,%d,%d\n%d,%d,%d\n",
		d.confirmedQueueInfo.TotalMsgCnt(),
		d.queueEndInfo.totalMsgCnt,
		d.confirmedQueueInfo.EndOffset.FileNum, d.confirmedQueueInfo.EndOffset.Pos, d.confirmedQueueInfo.Offset(),
		d.queueEndInfo.EndOffset.FileNum, d.queueEndInfo.EndOffset.Pos, d.queueEndInfo.Offset())
	if err != nil {
		f.Close()
		return err
	}
	f.Sync()
	f.Close()

	// atomically rename
	return util.AtomicRename(tmpFileName, fileName)
}
Example #4
0
func (t *Topic) MarkAsRemoved() (string, error) {
	t.Lock()
	defer t.Unlock()
	atomic.CompareAndSwapInt32(&t.exitFlag, 0, 1)
	nsqLog.Logf("TOPIC(%s): deleting", t.GetFullName())
	// since we are explicitly deleting a topic (not just at system exit time)
	// de-register this from the lookupd
	t.notifyCall(t)

	t.channelLock.Lock()
	for _, channel := range t.channelMap {
		delete(t.channelMap, channel.name)
		channel.Delete()
	}
	t.channelLock.Unlock()
	// we should move our partition only
	renamePath := t.dataPath + "-removed-" + strconv.Itoa(int(time.Now().Unix()))
	nsqLog.Warningf("mark the topic %v as removed: %v", t.GetFullName(), renamePath)
	os.MkdirAll(renamePath, 0755)
	err := t.backend.RemoveTo(renamePath)
	if err != nil {
		nsqLog.Errorf("failed to mark the topic %v as removed %v failed: %v", t.GetFullName(), renamePath, err)
	}
	util.AtomicRename(t.getMagicCodeFileName(), path.Join(renamePath, "magic"+strconv.Itoa(t.partition)))
	t.removeHistoryStat()
	t.RemoveChannelMeta()
	t.removeMagicCode()
	return renamePath, err
}
Example #5
0
func (d *diskQueueWriter) RemoveTo(destPath string) error {
	d.Lock()
	defer d.Unlock()
	d.exitFlag = 1
	nsqLog.Logf("DISKQUEUE(%s): removing to %v", d.name, destPath)
	d.sync()
	d.closeCurrentFile()
	d.saveFileOffsetMeta()
	for i := int64(0); i <= d.diskWriteEnd.EndOffset.FileNum; i++ {
		fn := d.fileName(i)
		destFile := fmt.Sprintf(path.Join(destPath, "%s.diskqueue.%06d.dat"), d.name, i)
		innerErr := util.AtomicRename(fn, destFile)
		nsqLog.Logf("DISKQUEUE(%s): renamed data file %v to %v", d.name, fn, destFile)
		if innerErr != nil && !os.IsNotExist(innerErr) {
			nsqLog.LogErrorf("diskqueue(%s) failed to remove data file - %s", d.name, innerErr)
		}
		fName := d.fileName(i) + ".offsetmeta.dat"
		innerErr = util.AtomicRename(fName, destFile+".offsetmeta.dat")
		nsqLog.Logf("DISKQUEUE(%s): rename offset meta file %v ", d.name, fName)
		if innerErr != nil && !os.IsNotExist(innerErr) {
			nsqLog.LogErrorf("diskqueue(%s) failed to remove offset meta file %v - %s", d.name, fName, innerErr)
		}
	}
	d.diskWriteEnd.EndOffset.FileNum++
	d.diskWriteEnd.EndOffset.Pos = 0
	d.diskReadEnd = d.diskWriteEnd
	destFile := fmt.Sprintf(path.Join(destPath, "%s.diskqueue.meta.writer.dat"), d.name)
	nsqLog.Logf("DISKQUEUE(%s): rename meta file to %v", d.name, destFile)
	innerErr := util.AtomicRename(d.metaDataFileName(), destFile)
	if innerErr != nil && !os.IsNotExist(innerErr) {
		nsqLog.LogErrorf("diskqueue(%s) failed to remove metadata file - %s", d.name, innerErr)
		return innerErr
	}
	destFile = fmt.Sprintf(path.Join(destPath, "%s.diskqueue.meta.extra.dat"), d.name)
	util.AtomicRename(d.extraMetaFileName(), destFile)
	return nil
}
Example #6
0
func (d *diskQueueWriter) saveExtraMeta() error {
	var err error

	fileName := d.extraMetaFileName()
	tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())

	var tmp extraMeta
	tmp.SegOffset = d.diskQueueStart.EndOffset
	tmp.VirtualOffset = d.diskQueueStart.Offset()
	tmp.TotalMsgCnt = d.diskQueueStart.TotalMsgCnt()
	data, _ := json.Marshal(tmp)
	err = ioutil.WriteFile(tmpFileName, data, 0644)
	if err != nil {
		return err
	}
	// atomically rename
	return util.AtomicRename(tmpFileName, fileName)
}
Example #7
0
func (t *Topic) SaveChannelMeta() error {
	fileName := t.getChannelMetaFileName()
	channels := make([]*ChannelMetaInfo, 0)
	t.channelLock.RLock()
	for _, channel := range t.channelMap {
		channel.RLock()
		if !channel.ephemeral {
			meta := &ChannelMetaInfo{
				Name:   channel.name,
				Paused: channel.IsPaused(),
			}
			channels = append(channels, meta)
		}
		channel.RUnlock()
	}
	t.channelLock.RUnlock()
	d, err := json.Marshal(channels)
	if err != nil {
		return err
	}
	tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int())
	f, err := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = f.Write(d)
	if err != nil {
		f.Close()
		return err
	}
	f.Sync()
	f.Close()
	err = util.AtomicRename(tmpFileName, fileName)
	if err != nil {
		return err
	}
	return nil
}