Example #1
0
func (cmd *Command) writeWALFiles(w io.WriteCloser, files []string, key string) error {
	fmt.Fprintln(w, "# writing wal data")

	// we need to make sure we write the same order that the wal received the data
	sort.Strings(files)

	var once sync.Once
	warn := func() {
		msg := fmt.Sprintf(`WARNING: detected deletes in wal file.
		Some series for %q may be brought back by replaying this data.
		To resolve, you can either let the shard snapshot prior to exporting the data
		or manually editing the exported file.
		`, key)
		fmt.Fprintln(cmd.Stderr, msg)
	}

	// use a function here to close the files in the defers and not let them accumulate in the loop
	write := func(f string) error {
		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
		defer file.Close()

		reader := tsm1.NewWALSegmentReader(file)
		defer reader.Close()
		for reader.Next() {
			entry, err := reader.Read()
			if err != nil {
				n := reader.Count()
				fmt.Fprintf(os.Stderr, "file %s corrupt at position %d", file.Name(), n)
				break
			}

			switch t := entry.(type) {
			case *tsm1.DeleteWALEntry:
				once.Do(warn)
				continue
			case *tsm1.DeleteRangeWALEntry:
				once.Do(warn)
				continue
			case *tsm1.WriteWALEntry:
				var pairs string

				for key, values := range t.Values {
					measurement, field := tsm1.SeriesAndFieldFromCompositeKey([]byte(key))

					for _, value := range values {
						if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
							continue
						}

						switch value.Value().(type) {
						case float64:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case int64:
							pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
						case bool:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case string:
							pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
						default:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						}
						fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
					}
				}
			}
		}
		return nil
	}

	for _, f := range files {
		if err := write(f); err != nil {
			return err
		}
	}

	return nil
}
Example #2
0
func (c *cmdExport) writeFiles() error {
	// open our output file and create an output buffer
	var w io.WriteCloser
	w, err := os.Create(c.out)
	if err != nil {
		return err
	}
	defer w.Close()
	if c.compress {
		w = gzip.NewWriter(w)
	}

	// Write out all the DDL
	fmt.Fprintln(w, "# DDL")
	for key := range c.files {
		keys := strings.Split(key, string(byte(os.PathSeparator)))
		fmt.Fprintf(w, "CREATE DATABASE %s\n", keys[0])
		fmt.Fprintf(w, "CREATE RETENTION POLICY %s ON %s DURATION inf REPLICATION 1\n", keys[1], keys[0])
	}

	fmt.Fprintln(w, "# DML")
	for key, files := range c.files {
		keys := strings.Split(key, string(byte(os.PathSeparator)))
		fmt.Fprintf(w, "# CONTEXT-DATABASE:%s\n", keys[0])
		fmt.Fprintf(w, "# CONTEXT-RETENTION-POLICY:%s\n", keys[1])
		for _, f := range files {
			// use an anonymous function here to close the files in the defers and not let them
			// accumulate in the loop
			if err := func(f string) error {
				file, err := os.OpenFile(f, os.O_RDONLY, 0600)
				if err != nil {
					return fmt.Errorf("%v", err)
				}
				defer file.Close()
				reader, err := tsm1.NewTSMReader(file)
				if err != nil {
					log.Printf("unable to read %s, skipping\n", f)
					return nil
				}
				defer reader.Close()

				for i := 0; i < reader.KeyCount(); i++ {
					var pairs string
					key, typ := reader.KeyAt(i)
					values, _ := reader.ReadAll(key)
					measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key)

					for _, value := range values {
						switch typ {
						case tsm1.BlockFloat64:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case tsm1.BlockInteger:
							pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
						case tsm1.BlockBoolean:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case tsm1.BlockString:
							pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
						default:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						}

						fmt.Fprintln(w, measurement, pairs, value.UnixNano())
					}
				}
				return nil
			}(f); err != nil {
				return err
			}
		}
		_ = key
	}
	return nil
}
Example #3
0
func (cmd *Command) writeTsmFiles(w io.WriteCloser, files []string) error {
	fmt.Fprintln(w, "# writing tsm data")

	// we need to make sure we write the same order that the files were written
	sort.Strings(files)

	// use a function here to close the files in the defers and not let them accumulate in the loop
	write := func(f string) error {
		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
		defer file.Close()
		reader, err := tsm1.NewTSMReader(file)
		if err != nil {
			log.Printf("unable to read %s, skipping\n", f)
			return nil
		}
		defer reader.Close()

		if sgStart, sgEnd := reader.TimeRange(); sgStart > cmd.endTime || sgEnd < cmd.startTime {
			return nil
		}

		for i := 0; i < reader.KeyCount(); i++ {
			var pairs string
			key, typ := reader.KeyAt(i)
			values, _ := reader.ReadAll(string(key))
			measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key)

			for _, value := range values {
				if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
					continue
				}

				switch typ {
				case tsm1.BlockFloat64:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				case tsm1.BlockInteger:
					pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
				case tsm1.BlockBoolean:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				case tsm1.BlockString:
					pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
				default:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				}

				fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
			}
		}
		return nil
	}

	for _, f := range files {
		if err := write(f); err != nil {
			return err
		}
	}

	return nil
}