func serveClient(conn net.Conn, codec goavro.Codec) {
fw, err := codec.NewWriter(
goavro.Compression(goavro.CompressionDeflate),
goavro.ToWriter(conn))
if err != nil {
log.Fatal(err)
}
defer fw.Close()
// create a record that matches the schema we want to encode
someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema))
if err != nil {
log.Fatal(err)
}
// identify field name to set datum for
someRecord.Set("username", "Aquaman")
someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
// you can fully qualify the field name
someRecord.Set("com.example.timestamp", int64(1082196484))
fw.Write(someRecord)
// create another record
if someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)); err != nil {
log.Fatal(err)
}
someRecord.Set("username", "Batman")
someRecord.Set("comment", "Who are all of these crazies?")
someRecord.Set("com.example.timestamp", int64(1427383430))
fw.Write(someRecord)
}
func dumpWriter(w io.Writer, codec goavro.Codec) {
fw, err := codec.NewWriter(
// goavro.Compression(goavro.CompressionDeflate),
goavro.Compression(goavro.CompressionSnappy),
goavro.ToWriter(w))
if err != nil {
log.Fatal(err)
}
defer func() {
err := fw.Close()
if err != nil {
log.Fatal(err)
}
}()
raw := []map[string]interface{}{
{"station": "011990-99999", "time": int64(-619524000000), "temp": int32(0)},
{"station": "011990-99999", "time": int64(-619506000000), "temp": int32(22)},
{"station": "011990-99999", "time": int64(-619484400000), "temp": int32(-11)},
{"station": "012650-99999", "time": int64(-655531200000), "temp": int32(111)},
{"station": "012650-99999", "time": int64(-655509600000), "temp": int32(78)},
}
for _, rec := range raw {
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Fatal(err)
}
for k, v := range rec {
record.Set(k, v)
}
fw.Write(record)
}
}
func dumpWriter(w io.Writer, codec goavro.Codec) {
fw, err := codec.NewWriter(
goavro.BlockSize(5), // queue up no more than 5 items
goavro.BlockTick(3*time.Second), // but flush at least every 3 seconds
goavro.Compression(goavro.CompressionDeflate),
goavro.ToWriter(w))
if err != nil {
log.Fatal(err)
}
defer fw.Close()
sigs := make(chan os.Signal)
signal.Notify(sigs)
defer func() {
signal.Stop(sigs)
}()
writeLoop:
for {
select {
case <-time.After(time.Duration(rand.Intn(500)) * time.Millisecond):
sendRecord(fw)
case <-sigs:
break writeLoop
}
}
}
// use Avro
func makeSomeData(w io.Writer) error {
var err error
codec, err := goavro.NewCodec(schema)
if err != nil {
log.Fatal(err)
}
fw, err := codec.NewWriter(
// goavro.BlockSize(13),
goavro.Compression(goavro.CompressionSnappy),
goavro.ToWriter(w))
if err != nil {
log.Fatal("avro: cannot create Writer: ", err)
}
defer fw.Close()
// GENERATE A Record based on the type.
rec := map[string]interface{}{"long_field": int64(1), "integer_field": int32(2), "decimal_field": string(3),
"float_field": float32(4), "double_field": float64(5), "boolean_field": true,
"string_field": string("7"), "date_field": int64(8), "timestamp_field": int64(9),
}
for i := 0; i < 100; i++ {
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Fatal(err)
}
for k, v := range rec {
record.Set(k, v)
}
fw.Write(record)
}
return nil
}