Example #1
0
func BenchmarkUnmarshalByAvro(b *testing.B) {
	codec, err := goavro.NewCodec(avroSchema)
	if err != nil {
		panic(err)
	}
	someRecord, err := goavro.NewRecord(goavro.RecordSchema(avroSchema))
	someRecord.Set("id", int32(1))
	someRecord.Set("name", "Reds")
	colors := []string{"Crimson", "Red", "Ruby", "Maroon"}
	s := make([]interface{}, len(colors))
	for i, v := range colors {
		s[i] = v
	}
	someRecord.Set("colors", s)

	buf := new(bytes.Buffer)
	err = codec.Encode(buf, someRecord)
	if err != nil {
		panic(err)
	}
	objectBytes := buf.Bytes()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err = codec.Decode(bytes.NewReader(objectBytes))
		if err != nil {
			panic(err)
		}
	}
}
func init() {
	outerSchema = fmt.Sprintf(`
{
  "type": "record",
  "name": "comments",
  "namespace": "com.example",
  "doc:": "List of users",
  "fields": [
    {
      "type": {
        "items": %s,
        "type": "array"
      },
      "name": "users"
    }
  ]
}
`, innerSchema)

	var err error
	// If you want speed, create the codec one time for each
	// schema and reuse it to create multiple Writer instances.
	codec, err = goavro.NewCodec(outerSchema)
	if err != nil {
		log.Fatal(err)
	}
}
func init() {
	outerSchema = fmt.Sprintf(`
{
  "type": "record",
  "name": "comments",
  "doc:": "A basic schema for storing blog comments",
  "namespace": "com.example",
  "fields": [
    {
      "name": "user",
      "type": %s
    },
    {
      "doc": "The content of the user's message",
      "type": "string",
      "name": "comment"
    },
    {
      "doc": "Unix epoch time in milliseconds",
      "type": "long",
      "name": "timestamp"
    }
  ]
}
`, innerSchema)

	var err error
	// If you want speed, create the codec one time for each
	// schema and reuse it to create multiple Writer instances.
	codec, err = goavro.NewCodec(outerSchema)
	if err != nil {
		log.Fatal(err)
	}
}
Example #4
0
func init() {
	schema = `
{
  "type" : "record",
  "name" : "Weather",
  "namespace" : "test",
  "doc" : "A weather reading.",
  "fields" : [ {
    "name" : "station",
    "type" : "string"
  }, {
    "name" : "time",
    "type" : "long"
  }, {
    "name" : "temp",
    "type" : "int"
  } ]
}
`

	var err error
	// If you want speed, create the codec one time for each
	// schema and reuse it to create multiple Writer instances.
	codec, err = goavro.NewCodec(schema)
	if err != nil {
		log.Fatal(err)
	}
}
Example #5
0
func init() {
	var err error
	// If you want speed, create the codec one time for each
	// schema and reuse it to create multiple Writer instances.
	codec, err = goavro.NewCodec(recordSchema)
	if err != nil {
		log.Fatal(err)
	}
}
func main() {
	recordSchemaJSON := `
{
  "type": "record",
  "name": "comments",
  "doc:": "A basic schema for storing blog comments",
  "namespace": "com.example",
  "fields": [
    {
      "doc": "Name of user",
      "type": "string",
      "name": "username"
    },
    {
      "doc": "The content of the user's message",
      "type": "string",
      "name": "comment"
    },
    {
      "doc": "Unix epoch time in milliseconds",
      "type": "long",
      "name": "timestamp"
    }
  ]
}
`
	someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchemaJSON))
	if err != nil {
		log.Fatal(err)
	}
	// identify field name to set datum for
	someRecord.Set("username", "Aquaman")
	someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
	// you can fully qualify the field name
	someRecord.Set("com.example.timestamp", int64(1082196484))

	codec, err := goavro.NewCodec(recordSchemaJSON)
	if err != nil {
		log.Fatal(err)
	}

	bb := new(bytes.Buffer)
	if err = codec.Encode(bb, someRecord); err != nil {
		log.Fatal(err)
	}

	actual := bb.Bytes()
	expected := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
	if bytes.Compare(actual, expected) != 0 {
		log.Printf("Actual: %#v; Expected: %#v", actual, expected)
	}
}
func bufferedEncoder(someSchemaJSON string, datum interface{}) (bits []byte, err error) {
	bb := new(bytes.Buffer)
	defer func() {
		bits = bb.Bytes()
	}()

	var c goavro.Codec
	c, err = goavro.NewCodec(someSchemaJSON)
	if err != nil {
		return
	}
	err = encodeWithBufferedWriter(c, bb, datum)
	return
}
Example #8
0
func LoadAvroSchema(outerFile string, innerFile string) (goavro.RecordSetter, goavro.RecordSetter, goavro.Codec) {
	dat, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", config.Path, innerFile))
	check(err)
	innerSchemaStr := string(dat)

	dat2, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", config.Path, outerFile))
	check(err)
	outerSchemaStr := fmt.Sprintf(string(dat2), innerSchemaStr)

	outerSchema := goavro.RecordSchema(outerSchemaStr)
	innerSchema := goavro.RecordSchema(innerSchemaStr)
	codec, err := goavro.NewCodec(outerSchemaStr)
	check(err)
	return outerSchema, innerSchema, codec
}
func main() {
	recordSchemaJSON := `
{
  "type": "record",
  "name": "comments",
  "doc:": "A basic schema for storing blog comments",
  "namespace": "com.example",
  "fields": [
    {
      "doc": "Name of user",
      "type": "string",
      "name": "username"
    },
    {
      "doc": "The content of the user's message",
      "type": "string",
      "name": "comment"
    },
    {
      "doc": "Unix epoch time in milliseconds",
      "type": "long",
      "name": "timestamp"
    }
  ]
}
`
	codec, err := goavro.NewCodec(recordSchemaJSON)
	if err != nil {
		log.Fatal(err)
	}
	encoded := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
	bb := bytes.NewBuffer(encoded)
	decoded, err := codec.Decode(bb)

	fmt.Println(decoded) // default String() representation is JSON

	// but direct access to data is provided
	record := decoded.(*goavro.Record)
	fmt.Println("Record Name:", record.Name)
	fmt.Println("Record Fields:")
	for i, field := range record.Fields {
		fmt.Println(" field", i, field.Name, ":", field.Datum)
	}
}
Example #10
0
// Init initializes a avro package
func NewAvroInst(w *logrus.Logger, conf avro_config) *Avro {
	data, err := ioutil.ReadFile(conf.Schema)
	if err != nil {
		w.WithFields(logrus.Fields{
			"module": "avro",
		}).Fatalln("Failed to open schema file:", err)
	}
	recordSchemaJSON := string(data)
	codec, err := goavro.NewCodec(recordSchemaJSON)
	if err != nil {
		w.WithFields(logrus.Fields{
			"module": "avro",
		}).Fatalln("Failed to init codec from schema:", err)
	}
	w.WithFields(logrus.Fields{
		"module": "avro",
	}).Println("Init completed.")
	return &Avro{codec: codec, recordSchemaJSON: recordSchemaJSON, logger: w}
}
Example #11
0
// use Avro
func makeSomeData(w io.Writer) error {
	var err error
	codec, err := goavro.NewCodec(schema)
	if err != nil {
		log.Fatal(err)
	}

	fw, err := codec.NewWriter(
		//		goavro.BlockSize(13),
		goavro.Compression(goavro.CompressionSnappy),
		goavro.ToWriter(w))

	if err != nil {
		log.Fatal("avro: cannot create Writer: ", err)
	}
	defer fw.Close()

	// GENERATE A Record based on the type.
	rec := map[string]interface{}{"long_field": int64(1), "integer_field": int32(2), "decimal_field": string(3),
		"float_field": float32(4), "double_field": float64(5), "boolean_field": true,
		"string_field": string("7"), "date_field": int64(8), "timestamp_field": int64(9),
	}

	for i := 0; i < 100; i++ {

		record, err := goavro.NewRecord(goavro.RecordSchema(schema))
		if err != nil {
			log.Fatal(err)
		}

		for k, v := range rec {
			record.Set(k, v)
		}

		fw.Write(record)
	}

	return nil
}
func main() {
	innerRecords := make([]interface{}, 0)
	// make first inner record
	innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatalf("cannot create innerRecord: %v", err)
	}
	if err = innerRecord.Set("stringValue", "Hello"); err != nil {
		log.Fatal(err)
	}
	if err = innerRecord.Set("intValue", int32(1)); err != nil {
		log.Fatal(err)
	}
	innerRecords = append(innerRecords, innerRecord)
	// make another inner record
	innerRecord, _ = goavro.NewRecord(goavro.RecordSchema(innerSchema))
	innerRecord.Set("stringValue", "World")
	innerRecord.Set("intValue", int32(2))
	innerRecords = append(innerRecords, innerRecord)
	// make outer record
	outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
	if err != nil {
		log.Fatalf("cannot create outerRecord: %v", err)
	}
	outerRecord.Set("value", int32(3))
	outerRecord.Set("rec", innerRecords)
	// make a codec
	c, err := goavro.NewCodec(outerSchema)
	if err != nil {
		log.Fatal(err)
	}
	// encode outerRecord to io.Writer (here, a bytes.Buffer)
	bb := new(bytes.Buffer)
	err = c.Encode(bb, outerRecord)
	if err != nil {
		log.Fatal(err)
	}
	// decode bytes
	decoded, err := c.Decode(bytes.NewReader(bb.Bytes()))
	if err != nil {
		log.Fatal(err)
	}
	decodedRecord, ok := decoded.(*goavro.Record)
	if !ok {
		log.Fatalf("expected *goavro.Record; received: %T", decoded)
	}
	decodedValue, err := decodedRecord.Get("value")
	if err != nil {
		log.Fatal(err)
	}
	if decodedValue != int32(3) {
		log.Printf("Actual: %#v; Expected: %#v\n", decodedValue, int32(3))
	}
	fmt.Printf("Read a value: %d\n", decodedValue)
	rec, err := decodedRecord.Get("rec")
	if err != nil {
		log.Fatal(err)
	}
	decodedArray := rec.([]interface{})
	if len(decodedArray) != 2 {
		log.Fatalf("Actual: %#v; Expected: %#v\n", len(decodedArray), 2)
	}
	for index, decodedSubRecord := range decodedArray {
		r := decodedSubRecord.(*goavro.Record)
		sv, err := r.Get("stringValue")
		if err != nil {
			log.Fatal(err)
		}
		iv, err := r.Get("intValue")
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Read a subrecord %d string value: %s\n", index, sv)
		fmt.Printf("Read a subrecord %d int value: %d\n", index, iv)
	}
}