func BenchmarkUnmarshalByAvro(b *testing.B) {
codec, err := goavro.NewCodec(avroSchema)
if err != nil {
panic(err)
}
someRecord, err := goavro.NewRecord(goavro.RecordSchema(avroSchema))
someRecord.Set("id", int32(1))
someRecord.Set("name", "Reds")
colors := []string{"Crimson", "Red", "Ruby", "Maroon"}
s := make([]interface{}, len(colors))
for i, v := range colors {
s[i] = v
}
someRecord.Set("colors", s)
buf := new(bytes.Buffer)
err = codec.Encode(buf, someRecord)
if err != nil {
panic(err)
}
objectBytes := buf.Bytes()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = codec.Decode(bytes.NewReader(objectBytes))
if err != nil {
panic(err)
}
}
}
func init() {
outerSchema = fmt.Sprintf(`
{
"type": "record",
"name": "comments",
"namespace": "com.example",
"doc:": "List of users",
"fields": [
{
"type": {
"items": %s,
"type": "array"
},
"name": "users"
}
]
}
`, innerSchema)
var err error
// If you want speed, create the codec one time for each
// schema and reuse it to create multiple Writer instances.
codec, err = goavro.NewCodec(outerSchema)
if err != nil {
log.Fatal(err)
}
}
func init() {
outerSchema = fmt.Sprintf(`
{
"type": "record",
"name": "comments",
"doc:": "A basic schema for storing blog comments",
"namespace": "com.example",
"fields": [
{
"name": "user",
"type": %s
},
{
"doc": "The content of the user's message",
"type": "string",
"name": "comment"
},
{
"doc": "Unix epoch time in milliseconds",
"type": "long",
"name": "timestamp"
}
]
}
`, innerSchema)
var err error
// If you want speed, create the codec one time for each
// schema and reuse it to create multiple Writer instances.
codec, err = goavro.NewCodec(outerSchema)
if err != nil {
log.Fatal(err)
}
}
func init() {
schema = `
{
"type" : "record",
"name" : "Weather",
"namespace" : "test",
"doc" : "A weather reading.",
"fields" : [ {
"name" : "station",
"type" : "string"
}, {
"name" : "time",
"type" : "long"
}, {
"name" : "temp",
"type" : "int"
} ]
}
`
var err error
// If you want speed, create the codec one time for each
// schema and reuse it to create multiple Writer instances.
codec, err = goavro.NewCodec(schema)
if err != nil {
log.Fatal(err)
}
}
func init() {
var err error
// If you want speed, create the codec one time for each
// schema and reuse it to create multiple Writer instances.
codec, err = goavro.NewCodec(recordSchema)
if err != nil {
log.Fatal(err)
}
}
func main() {
recordSchemaJSON := `
{
"type": "record",
"name": "comments",
"doc:": "A basic schema for storing blog comments",
"namespace": "com.example",
"fields": [
{
"doc": "Name of user",
"type": "string",
"name": "username"
},
{
"doc": "The content of the user's message",
"type": "string",
"name": "comment"
},
{
"doc": "Unix epoch time in milliseconds",
"type": "long",
"name": "timestamp"
}
]
}
`
someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchemaJSON))
if err != nil {
log.Fatal(err)
}
// identify field name to set datum for
someRecord.Set("username", "Aquaman")
someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
// you can fully qualify the field name
someRecord.Set("com.example.timestamp", int64(1082196484))
codec, err := goavro.NewCodec(recordSchemaJSON)
if err != nil {
log.Fatal(err)
}
bb := new(bytes.Buffer)
if err = codec.Encode(bb, someRecord); err != nil {
log.Fatal(err)
}
actual := bb.Bytes()
expected := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
if bytes.Compare(actual, expected) != 0 {
log.Printf("Actual: %#v; Expected: %#v", actual, expected)
}
}
func bufferedEncoder(someSchemaJSON string, datum interface{}) (bits []byte, err error) {
bb := new(bytes.Buffer)
defer func() {
bits = bb.Bytes()
}()
var c goavro.Codec
c, err = goavro.NewCodec(someSchemaJSON)
if err != nil {
return
}
err = encodeWithBufferedWriter(c, bb, datum)
return
}
func LoadAvroSchema(outerFile string, innerFile string) (goavro.RecordSetter, goavro.RecordSetter, goavro.Codec) {
dat, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", config.Path, innerFile))
check(err)
innerSchemaStr := string(dat)
dat2, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", config.Path, outerFile))
check(err)
outerSchemaStr := fmt.Sprintf(string(dat2), innerSchemaStr)
outerSchema := goavro.RecordSchema(outerSchemaStr)
innerSchema := goavro.RecordSchema(innerSchemaStr)
codec, err := goavro.NewCodec(outerSchemaStr)
check(err)
return outerSchema, innerSchema, codec
}
func main() {
recordSchemaJSON := `
{
"type": "record",
"name": "comments",
"doc:": "A basic schema for storing blog comments",
"namespace": "com.example",
"fields": [
{
"doc": "Name of user",
"type": "string",
"name": "username"
},
{
"doc": "The content of the user's message",
"type": "string",
"name": "comment"
},
{
"doc": "Unix epoch time in milliseconds",
"type": "long",
"name": "timestamp"
}
]
}
`
codec, err := goavro.NewCodec(recordSchemaJSON)
if err != nil {
log.Fatal(err)
}
encoded := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
bb := bytes.NewBuffer(encoded)
decoded, err := codec.Decode(bb)
fmt.Println(decoded) // default String() representation is JSON
// but direct access to data is provided
record := decoded.(*goavro.Record)
fmt.Println("Record Name:", record.Name)
fmt.Println("Record Fields:")
for i, field := range record.Fields {
fmt.Println(" field", i, field.Name, ":", field.Datum)
}
}
// Init initializes a avro package
func NewAvroInst(w *logrus.Logger, conf avro_config) *Avro {
data, err := ioutil.ReadFile(conf.Schema)
if err != nil {
w.WithFields(logrus.Fields{
"module": "avro",
}).Fatalln("Failed to open schema file:", err)
}
recordSchemaJSON := string(data)
codec, err := goavro.NewCodec(recordSchemaJSON)
if err != nil {
w.WithFields(logrus.Fields{
"module": "avro",
}).Fatalln("Failed to init codec from schema:", err)
}
w.WithFields(logrus.Fields{
"module": "avro",
}).Println("Init completed.")
return &Avro{codec: codec, recordSchemaJSON: recordSchemaJSON, logger: w}
}
// use Avro
func makeSomeData(w io.Writer) error {
var err error
codec, err := goavro.NewCodec(schema)
if err != nil {
log.Fatal(err)
}
fw, err := codec.NewWriter(
// goavro.BlockSize(13),
goavro.Compression(goavro.CompressionSnappy),
goavro.ToWriter(w))
if err != nil {
log.Fatal("avro: cannot create Writer: ", err)
}
defer fw.Close()
// GENERATE A Record based on the type.
rec := map[string]interface{}{"long_field": int64(1), "integer_field": int32(2), "decimal_field": string(3),
"float_field": float32(4), "double_field": float64(5), "boolean_field": true,
"string_field": string("7"), "date_field": int64(8), "timestamp_field": int64(9),
}
for i := 0; i < 100; i++ {
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Fatal(err)
}
for k, v := range rec {
record.Set(k, v)
}
fw.Write(record)
}
return nil
}
func main() {
innerRecords := make([]interface{}, 0)
// make first inner record
innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatalf("cannot create innerRecord: %v", err)
}
if err = innerRecord.Set("stringValue", "Hello"); err != nil {
log.Fatal(err)
}
if err = innerRecord.Set("intValue", int32(1)); err != nil {
log.Fatal(err)
}
innerRecords = append(innerRecords, innerRecord)
// make another inner record
innerRecord, _ = goavro.NewRecord(goavro.RecordSchema(innerSchema))
innerRecord.Set("stringValue", "World")
innerRecord.Set("intValue", int32(2))
innerRecords = append(innerRecords, innerRecord)
// make outer record
outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
if err != nil {
log.Fatalf("cannot create outerRecord: %v", err)
}
outerRecord.Set("value", int32(3))
outerRecord.Set("rec", innerRecords)
// make a codec
c, err := goavro.NewCodec(outerSchema)
if err != nil {
log.Fatal(err)
}
// encode outerRecord to io.Writer (here, a bytes.Buffer)
bb := new(bytes.Buffer)
err = c.Encode(bb, outerRecord)
if err != nil {
log.Fatal(err)
}
// decode bytes
decoded, err := c.Decode(bytes.NewReader(bb.Bytes()))
if err != nil {
log.Fatal(err)
}
decodedRecord, ok := decoded.(*goavro.Record)
if !ok {
log.Fatalf("expected *goavro.Record; received: %T", decoded)
}
decodedValue, err := decodedRecord.Get("value")
if err != nil {
log.Fatal(err)
}
if decodedValue != int32(3) {
log.Printf("Actual: %#v; Expected: %#v\n", decodedValue, int32(3))
}
fmt.Printf("Read a value: %d\n", decodedValue)
rec, err := decodedRecord.Get("rec")
if err != nil {
log.Fatal(err)
}
decodedArray := rec.([]interface{})
if len(decodedArray) != 2 {
log.Fatalf("Actual: %#v; Expected: %#v\n", len(decodedArray), 2)
}
for index, decodedSubRecord := range decodedArray {
r := decodedSubRecord.(*goavro.Record)
sv, err := r.Get("stringValue")
if err != nil {
log.Fatal(err)
}
iv, err := r.Get("intValue")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Read a subrecord %d string value: %s\n", index, sv)
fmt.Printf("Read a subrecord %d int value: %d\n", index, iv)
}
}