func serveClient(conn net.Conn, codec goavro.Codec) {
fw, err := codec.NewWriter(
goavro.Compression(goavro.CompressionDeflate),
goavro.ToWriter(conn))
if err != nil {
log.Fatal(err)
}
defer fw.Close()
// create a record that matches the schema we want to encode
someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema))
if err != nil {
log.Fatal(err)
}
// identify field name to set datum for
someRecord.Set("username", "Aquaman")
someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
// you can fully qualify the field name
someRecord.Set("com.example.timestamp", int64(1082196484))
fw.Write(someRecord)
// create another record
if someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)); err != nil {
log.Fatal(err)
}
someRecord.Set("username", "Batman")
someRecord.Set("comment", "Who are all of these crazies?")
someRecord.Set("com.example.timestamp", int64(1427383430))
fw.Write(someRecord)
}
func sendRecord(fw *goavro.Writer) {
// If we want to encode data, we need to put it in an actual
// goavro.Record instance corresponding to the schema we wish
// to encode against.
//
// NewRecord will create a goavro.Record instance
// corresponding to the specified schema.
innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatal(err)
}
innerRecord.Set("account", "Aquaman")
innerRecord.Set("creationDate", int64(1082196484))
// We create both an innerRecord and an outerRecord.
outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
if err != nil {
log.Fatal(err)
}
// innerRecord is a completely seperate record instance from
// outerRecord. Once we have an innerRecord instance it can be
// assigned to the appropriate Datum item of the outerRecord.
outerRecord.Set("user", innerRecord)
// Other fields are set on the outerRecord.
outerRecord.Set("comment", "The Atlantic is oddly cold this morning!")
outerRecord.Set("timestamp", int64(time.Now().UnixNano()))
fw.Write(outerRecord)
}
func main() {
// If we want to encode data, we need to put it in an actual
// goavro.Record instance corresponding to the schema we wish
// to encode against.
//
// NewRecord will create a goavro.Record instance
// corresponding to the specified schema.
innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatal(err)
}
innerRecord.Set("account", "Aquaman")
innerRecord.Set("creationDate", int64(1082196484))
// We create both an innerRecord and an outerRecord.
outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
if err != nil {
log.Fatal(err)
}
// innerRecord is a completely seperate record instance from
// outerRecord. Once we have an innerRecord instance it can be
// assigned to the appropriate Datum item of the outerRecord.
outerRecord.Set("user", innerRecord)
// Other fields are set on the outerRecord.
outerRecord.Set("comment", "The Atlantic is oddly cold this morning!")
outerRecord.Set("timestamp", int64(1427255074))
// Encode the outerRecord into a bytes.Buffer
bb := new(bytes.Buffer)
if err = codec.Encode(bb, outerRecord); err != nil {
log.Fatal(err)
}
// Compare encoded bytes against the expected bytes.
actual := bb.Bytes()
expected := []byte(
"\x0eAquaman" + // account
"\x88\x88\x88\x88\x08" + // creationDate
"\x50" + // 50 hex == 80 dec variable length integer encoded == 40 -> string is 40 characters long
"The Atlantic is oddly cold this morning!" + // comment
"\xc4\xbc\x91\xd1\x0a") // timestamp
if bytes.Compare(actual, expected) != 0 {
log.Printf("Actual: %#v; Expected: %#v", actual, expected)
}
// Let's decode the blob and print the output in JSON format
// using goavro.Record's String() method.
decoded, err := codec.Decode(bytes.NewReader(actual))
fmt.Println(decoded)
// we only need to perform type assertion if we want to access inside
record := decoded.(*goavro.Record)
fmt.Println("Record Name:", record.Name)
fmt.Println("Record Fields:")
for i, field := range record.Fields {
fmt.Println(" field", i, field.Name, ":", field.Datum)
}
}
func BenchmarkUnmarshalByAvro(b *testing.B) {
codec, err := goavro.NewCodec(avroSchema)
if err != nil {
panic(err)
}
someRecord, err := goavro.NewRecord(goavro.RecordSchema(avroSchema))
someRecord.Set("id", int32(1))
someRecord.Set("name", "Reds")
colors := []string{"Crimson", "Red", "Ruby", "Maroon"}
s := make([]interface{}, len(colors))
for i, v := range colors {
s[i] = v
}
someRecord.Set("colors", s)
buf := new(bytes.Buffer)
err = codec.Encode(buf, someRecord)
if err != nil {
panic(err)
}
objectBytes := buf.Bytes()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = codec.Decode(bytes.NewReader(objectBytes))
if err != nil {
panic(err)
}
}
}
func dumpWriter(w io.Writer, codec goavro.Codec) {
fw, err := codec.NewWriter(
// goavro.Compression(goavro.CompressionDeflate),
goavro.Compression(goavro.CompressionSnappy),
goavro.ToWriter(w))
if err != nil {
log.Fatal(err)
}
defer func() {
err := fw.Close()
if err != nil {
log.Fatal(err)
}
}()
raw := []map[string]interface{}{
{"station": "011990-99999", "time": int64(-619524000000), "temp": int32(0)},
{"station": "011990-99999", "time": int64(-619506000000), "temp": int32(22)},
{"station": "011990-99999", "time": int64(-619484400000), "temp": int32(-11)},
{"station": "012650-99999", "time": int64(-655531200000), "temp": int32(111)},
{"station": "012650-99999", "time": int64(-655509600000), "temp": int32(78)},
}
for _, rec := range raw {
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Fatal(err)
}
for k, v := range rec {
record.Set(k, v)
}
fw.Write(record)
}
}
func main() {
recordSchemaJSON := `
{
"type": "record",
"name": "comments",
"doc:": "A basic schema for storing blog comments",
"namespace": "com.example",
"fields": [
{
"doc": "Name of user",
"type": "string",
"name": "username"
},
{
"doc": "The content of the user's message",
"type": "string",
"name": "comment"
},
{
"doc": "Unix epoch time in milliseconds",
"type": "long",
"name": "timestamp"
}
]
}
`
someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchemaJSON))
if err != nil {
log.Fatal(err)
}
// identify field name to set datum for
someRecord.Set("username", "Aquaman")
someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
// you can fully qualify the field name
someRecord.Set("com.example.timestamp", int64(1082196484))
codec, err := goavro.NewCodec(recordSchemaJSON)
if err != nil {
log.Fatal(err)
}
bb := new(bytes.Buffer)
if err = codec.Encode(bb, someRecord); err != nil {
log.Fatal(err)
}
actual := bb.Bytes()
expected := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
if bytes.Compare(actual, expected) != 0 {
log.Printf("Actual: %#v; Expected: %#v", actual, expected)
}
}
// use Avro
func makeSomeData(w io.Writer) error {
var err error
codec, err := goavro.NewCodec(schema)
if err != nil {
log.Fatal(err)
}
fw, err := codec.NewWriter(
// goavro.BlockSize(13),
goavro.Compression(goavro.CompressionSnappy),
goavro.ToWriter(w))
if err != nil {
log.Fatal("avro: cannot create Writer: ", err)
}
defer fw.Close()
// GENERATE A Record based on the type.
rec := map[string]interface{}{"long_field": int64(1), "integer_field": int32(2), "decimal_field": string(3),
"float_field": float32(4), "double_field": float64(5), "boolean_field": true,
"string_field": string("7"), "date_field": int64(8), "timestamp_field": int64(9),
}
for i := 0; i < 100; i++ {
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Fatal(err)
}
for k, v := range rec {
record.Set(k, v)
}
fw.Write(record)
}
return nil
}
// NewRecord inits a new record
func (self *Avro) NewRecord() (*goavro.Record, error) {
record, err := goavro.NewRecord(goavro.RecordSchema(self.recordSchemaJSON))
return record, err
}
func main() {
var innerRecords []interface{}
// If we want to encode data, we need to put it in an actual
// goavro.Record instance corresponding to the schema we wish
// to encode against.
//
// NewRecord will create a goavro.Record instance
// corresponding to the specified schema.
innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatal(err)
}
innerRecord.Set("account", "Aquaman")
innerRecord.Set("creationDate", int64(1082196484))
innerRecords = append(innerRecords, innerRecord)
innerRecord, err = goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatal(err)
}
innerRecord.Set("account", "Super Girl")
innerRecord.Set("creationDate", int64(1442240878))
innerRecords = append(innerRecords, innerRecord)
// We create both an innerRecord and an outerRecord.
outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
if err != nil {
log.Fatal(err)
}
// innerRecord is a completely seperate record instance from
// outerRecord. Once we have an innerRecord instance it can be
// assigned to the appropriate Datum item of the outerRecord.
outerRecord.Set("users", innerRecords)
// Encode the outerRecord into a bytes.Buffer
bb := new(bytes.Buffer)
if err = codec.Encode(bb, outerRecord); err != nil {
log.Fatal(err)
}
// Compare encoded bytes against the expected bytes.
actual := bb.Bytes()
expected := []byte(
"\x04" + // array of two elements
"\x0eAquaman" + // first account
"\x88\x88\x88\x88\x08" + // creationDate
"\x14Super Girl" + // second account
"\xdc\xe5\xb6\xdf\x0a\x00") // creationDate
if bytes.Compare(actual, expected) != 0 {
log.Printf("Actual: %#v; Expected: %#v", actual, expected)
}
// Let's decode the blob and print the output in JSON format
// using goavro.Record's String() method.
decoded, err := codec.Decode(bytes.NewReader(actual))
fmt.Println(decoded)
// we only need to perform type assertion if we want to access inside
record := decoded.(*goavro.Record)
fmt.Println("Record Name:", record.Name)
fmt.Println("Record Fields:")
for i, field := range record.Fields {
fmt.Println(" field", i, field.Name, ":", field.Datum)
}
}
func SelectDBData(db *sql.DB, query string) {
defer timeTrack(time.Now(), "SelectDBData")
rows, err := db.Query(query)
check(err)
defer rows.Close()
columns, err := rows.Columns()
check(err)
values := make([]interface{}, len(columns))
scanArgs := make([]interface{}, len(values))
for i := range values {
scanArgs[i] = &values[i]
}
c := 0
var claims []interface{}
outerSchema, innerSchema, codec := LoadAvroSchema(conf.Avro.OuterSchema, conf.Avro.InnerSchema)
for rows.Next() {
c += 1
err = rows.Scan(scanArgs...)
check(err)
m := make(map[string]interface{})
claim, err := goavro.NewRecord(innerSchema)
check(err)
for i, colName := range columns {
if val, ok := values[i].(time.Time); ok {
claim.Set(colName, val.Unix())
m[colName] = val.Unix()
} else {
claim.Set(colName, values[i])
m[colName] = values[i]
}
}
claims = append(claims, claim)
if len(claims) == 3000 {
claims_avro, err := goavro.NewRecord(outerSchema)
check(err)
claims_avro.Set("claims", claims)
buf := new(bytes.Buffer)
err = codec.Encode(buf, claims_avro)
go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
claims = make([]interface{}, 0)
}
}
if rows.Err() != nil {
pialog.Error(rows.Err())
}
claims_avro, err := goavro.NewRecord(outerSchema)
check(err)
claims_avro.Set("claims", claims)
buf := new(bytes.Buffer)
err = codec.Encode(buf, claims_avro)
check(err)
go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
pialog.Info("Total input records:", c)
}
func main() {
fd, err := os.Create("temp.avro")
if err != nil {
log.Println("error", err)
}
makeSomeData(fd)
fd.Close()
fd, err = os.Open("temp.avro")
if err != nil {
log.Println("error", err)
}
//dumpReader(fd)
fd.Close()
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Println("error", err)
}
{
fd, err = os.Create("temp.parquet")
if err != nil {
log.Println("error", err)
}
pschema := parquet.NewSchema()
for _, f := range record.Fields {
fschema, err := record.GetFieldSchema(f.Name)
if err != nil {
panic(err)
}
err = pschema.AddColumnFromThriftSchema(fschema.(map[string]interface{}))
if err != nil {
panic(err)
}
}
enc := parquet.NewEncoder(pschema, fd)
// Read the file
fd, err = os.Open("temp.avro")
if err != nil {
log.Println("error", err)
}
fr, err := goavro.NewReader(goavro.BufferFromReader(fd))
if err != nil {
log.Fatal("cannot create Reader: ", err)
}
defer func() {
if err := fr.Close(); err != nil {
log.Fatal(err)
}
}()
for fr.Scan() {
datum, err := fr.Read()
if err != nil {
log.Println("cannot read datum: ", err)
continue
}
r, ok := datum.(*goavro.Record)
if !ok {
panic("expected goavro.Record")
}
rec := make(map[string]interface{})
for _, field := range r.Fields {
v, err := r.Get(field.Name)
if err != nil {
panic(err)
}
rec[field.Name] = v
}
err = enc.WriteRecords([]map[string]interface{}{rec})
if err != nil {
panic(err)
}
}
if err := enc.Close(); err != nil {
panic(err)
}
fd.Close()
}
log.Println("finished")
}
func main() {
innerRecords := make([]interface{}, 0)
// make first inner record
innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
if err != nil {
log.Fatalf("cannot create innerRecord: %v", err)
}
if err = innerRecord.Set("stringValue", "Hello"); err != nil {
log.Fatal(err)
}
if err = innerRecord.Set("intValue", int32(1)); err != nil {
log.Fatal(err)
}
innerRecords = append(innerRecords, innerRecord)
// make another inner record
innerRecord, _ = goavro.NewRecord(goavro.RecordSchema(innerSchema))
innerRecord.Set("stringValue", "World")
innerRecord.Set("intValue", int32(2))
innerRecords = append(innerRecords, innerRecord)
// make outer record
outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
if err != nil {
log.Fatalf("cannot create outerRecord: %v", err)
}
outerRecord.Set("value", int32(3))
outerRecord.Set("rec", innerRecords)
// make a codec
c, err := goavro.NewCodec(outerSchema)
if err != nil {
log.Fatal(err)
}
// encode outerRecord to io.Writer (here, a bytes.Buffer)
bb := new(bytes.Buffer)
err = c.Encode(bb, outerRecord)
if err != nil {
log.Fatal(err)
}
// decode bytes
decoded, err := c.Decode(bytes.NewReader(bb.Bytes()))
if err != nil {
log.Fatal(err)
}
decodedRecord, ok := decoded.(*goavro.Record)
if !ok {
log.Fatalf("expected *goavro.Record; received: %T", decoded)
}
decodedValue, err := decodedRecord.Get("value")
if err != nil {
log.Fatal(err)
}
if decodedValue != int32(3) {
log.Printf("Actual: %#v; Expected: %#v\n", decodedValue, int32(3))
}
fmt.Printf("Read a value: %d\n", decodedValue)
rec, err := decodedRecord.Get("rec")
if err != nil {
log.Fatal(err)
}
decodedArray := rec.([]interface{})
if len(decodedArray) != 2 {
log.Fatalf("Actual: %#v; Expected: %#v\n", len(decodedArray), 2)
}
for index, decodedSubRecord := range decodedArray {
r := decodedSubRecord.(*goavro.Record)
sv, err := r.Get("stringValue")
if err != nil {
log.Fatal(err)
}
iv, err := r.Get("intValue")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Read a subrecord %d string value: %s\n", index, sv)
fmt.Printf("Read a subrecord %d int value: %d\n", index, iv)
}
}