Example #1
0
func Test001ArchiverFileMgr(t *testing.T) {

	cv.Convey("given an archiver, messages that roll to different dates should be stored in distinct files", t, func() {
		tmp, err := ioutil.TempDir("", "test-archiver-filemgr")
		panicOn(err)
		defer os.RemoveAll(tmp)
		fm := NewFileMgr(&ArchiverConfig{WriteDir: tmp})

		tm1, err := time.Parse(time.RFC3339, "2016-01-01T00:00:00Z")
		panicOn(err)
		tm2, err := time.Parse(time.RFC3339, "2016-01-02T00:00:00Z")
		panicOn(err)

		streamName := "test"
		data1 := []byte("data1")
		data2 := []byte("data2")

		frame1, err := ts.NewFrame(tm1, ts.EvUtf8, 0, 0, data1)
		panicOn(err)
		by1, err := frame1.Marshal(nil)
		file1, err := fm.Store(tm1, streamName, by1)
		panicOn(err)

		frame2, err := ts.NewFrame(tm2, ts.EvUtf8, 0, 0, data2)
		panicOn(err)
		by2, err := frame2.Marshal(nil)
		file2, err := fm.Store(tm2, streamName, by2)
		panicOn(err)

		p("For inspection, we stored into file1 = '%v'\n", file1.Path)
		q("file2 = '%#v'\n", file2)
		cv.So(strings.Contains(file1.Path, "2016/01/01"), cv.ShouldBeTrue)
		cv.So(strings.Contains(file2.Path, "2016/01/02"), cv.ShouldBeTrue)

		cv.So(FileExists(file1.Path), cv.ShouldBeTrue)
		cv.So(FileExists(file2.Path), cv.ShouldBeTrue)

		by, err := ioutil.ReadFile(file1.Path)
		panicOn(err)

		var fr ts.Frame
		_, err = fr.Unmarshal(by, true)
		p("fr = %#v", fr)
		panicOn(err)

		cv.So(fr.GetEvtnum(), cv.ShouldEqual, ts.EvUtf8)
		cv.So(fr.GetUlen(), cv.ShouldEqual, len(frame1.Data)+1)

		p("Given that we've written an event to its file, we should be able to recover what we've written")
		cv.So(string(fr.Data), cv.ShouldResemble, string(data1))
	})
}
Example #2
0
func main() {
	myflags := flag.NewFlagSet("tfcat", flag.ExitOnError)
	cfg := &TfcatConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//Q("leftover = %v", leftover)
	if len(leftover) == 0 {
		fmt.Fprintf(os.Stderr, "no input files given\n")
		showUse(myflags)
		os.Exit(1)
	}
	GlobalPrettyPrint = cfg.PrettyPrint

	i := int64(0)
nextfile:
	for _, inputFile := range leftover {
		//P("starting on inputFile '%s'", inputFile)
		if !FileExists(inputFile) {
			fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
			os.Exit(1)
		}

		f, err := os.Open(inputFile)
		panicOn(err)
		fr := tf.NewFrameReader(f, 1024*1024)

		var frame *tf.Frame

		for ; err == nil; i++ {
			frame, _, err = fr.NextFrame()
			if err != nil {
				if err == io.EOF {
					continue nextfile
				}
				fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame() at i=%v: '%v'\n", i, err)
				os.Exit(1)
			}
			fmt.Printf("%v", frame)
			evtnum := frame.GetEvtnum()
			if evtnum == tf.EvJson {
				pp := prettyPrintJson(cfg.PrettyPrint, frame.Data)
				fmt.Printf("  %s", string(pp))
			}
			if evtnum == tf.EvMsgpKafka || evtnum == tf.EvMsgpack {
				// decode msgpack to json with ugorji/go/codec

				var iface interface{}
				dec := codec.NewDecoderBytes(frame.Data, &msgpHelper.mh)
				err := dec.Decode(&iface)
				panicOn(err)

				//Q("iface = '%#v'", iface)

				var w bytes.Buffer
				enc := codec.NewEncoder(&w, &msgpHelper.jh)
				err = enc.Encode(&iface)
				panicOn(err)
				pp := prettyPrintJson(cfg.PrettyPrint, w.Bytes())
				fmt.Printf(" %s", string(pp))
			}
			fmt.Printf("\n")
		}
	}
}