Beispiel #1
0
func serveClient(conn net.Conn, codec goavro.Codec) {
	fw, err := codec.NewWriter(
		goavro.Compression(goavro.CompressionDeflate),
		goavro.ToWriter(conn))
	if err != nil {
		log.Fatal(err)
	}
	defer fw.Close()

	// create a record that matches the schema we want to encode
	someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema))
	if err != nil {
		log.Fatal(err)
	}
	// identify field name to set datum for
	someRecord.Set("username", "Aquaman")
	someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
	// you can fully qualify the field name
	someRecord.Set("com.example.timestamp", int64(1082196484))
	fw.Write(someRecord)

	// create another record
	if someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)); err != nil {
		log.Fatal(err)
	}
	someRecord.Set("username", "Batman")
	someRecord.Set("comment", "Who are all of these crazies?")
	someRecord.Set("com.example.timestamp", int64(1427383430))
	fw.Write(someRecord)
}
Beispiel #2
0
func sendRecord(fw *goavro.Writer) {
	// If we want to encode data, we need to put it in an actual
	// goavro.Record instance corresponding to the schema we wish
	// to encode against.
	//
	// NewRecord will create a goavro.Record instance
	// corresponding to the specified schema.
	innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatal(err)
	}
	innerRecord.Set("account", "Aquaman")
	innerRecord.Set("creationDate", int64(1082196484))

	// We create both an innerRecord and an outerRecord.
	outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
	if err != nil {
		log.Fatal(err)
	}
	// innerRecord is a completely seperate record instance from
	// outerRecord. Once we have an innerRecord instance it can be
	// assigned to the appropriate Datum item of the outerRecord.
	outerRecord.Set("user", innerRecord)
	// Other fields are set on the outerRecord.
	outerRecord.Set("comment", "The Atlantic is oddly cold this morning!")
	outerRecord.Set("timestamp", int64(time.Now().UnixNano()))
	fw.Write(outerRecord)
}
func main() {
	// If we want to encode data, we need to put it in an actual
	// goavro.Record instance corresponding to the schema we wish
	// to encode against.
	//
	// NewRecord will create a goavro.Record instance
	// corresponding to the specified schema.
	innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatal(err)
	}
	innerRecord.Set("account", "Aquaman")
	innerRecord.Set("creationDate", int64(1082196484))

	// We create both an innerRecord and an outerRecord.
	outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
	if err != nil {
		log.Fatal(err)
	}
	// innerRecord is a completely seperate record instance from
	// outerRecord. Once we have an innerRecord instance it can be
	// assigned to the appropriate Datum item of the outerRecord.
	outerRecord.Set("user", innerRecord)
	// Other fields are set on the outerRecord.
	outerRecord.Set("comment", "The Atlantic is oddly cold this morning!")
	outerRecord.Set("timestamp", int64(1427255074))

	// Encode the outerRecord into a bytes.Buffer
	bb := new(bytes.Buffer)
	if err = codec.Encode(bb, outerRecord); err != nil {
		log.Fatal(err)
	}
	// Compare encoded bytes against the expected bytes.
	actual := bb.Bytes()
	expected := []byte(
		"\x0eAquaman" + // account
			"\x88\x88\x88\x88\x08" + // creationDate
			"\x50" + // 50 hex == 80 dec variable length integer encoded == 40 -> string is 40 characters long
			"The Atlantic is oddly cold this morning!" + // comment
			"\xc4\xbc\x91\xd1\x0a") // timestamp
	if bytes.Compare(actual, expected) != 0 {
		log.Printf("Actual: %#v; Expected: %#v", actual, expected)
	}
	// Let's decode the blob and print the output in JSON format
	// using goavro.Record's String() method.
	decoded, err := codec.Decode(bytes.NewReader(actual))
	fmt.Println(decoded)
	// we only need to perform type assertion if we want to access inside
	record := decoded.(*goavro.Record)
	fmt.Println("Record Name:", record.Name)
	fmt.Println("Record Fields:")
	for i, field := range record.Fields {
		fmt.Println(" field", i, field.Name, ":", field.Datum)
	}
}
Beispiel #4
0
func BenchmarkUnmarshalByAvro(b *testing.B) {
	codec, err := goavro.NewCodec(avroSchema)
	if err != nil {
		panic(err)
	}
	someRecord, err := goavro.NewRecord(goavro.RecordSchema(avroSchema))
	someRecord.Set("id", int32(1))
	someRecord.Set("name", "Reds")
	colors := []string{"Crimson", "Red", "Ruby", "Maroon"}
	s := make([]interface{}, len(colors))
	for i, v := range colors {
		s[i] = v
	}
	someRecord.Set("colors", s)

	buf := new(bytes.Buffer)
	err = codec.Encode(buf, someRecord)
	if err != nil {
		panic(err)
	}
	objectBytes := buf.Bytes()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err = codec.Decode(bytes.NewReader(objectBytes))
		if err != nil {
			panic(err)
		}
	}
}
Beispiel #5
0
func dumpWriter(w io.Writer, codec goavro.Codec) {
	fw, err := codec.NewWriter(
		// goavro.Compression(goavro.CompressionDeflate),
		goavro.Compression(goavro.CompressionSnappy),
		goavro.ToWriter(w))
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		err := fw.Close()
		if err != nil {
			log.Fatal(err)
		}
	}()

	raw := []map[string]interface{}{
		{"station": "011990-99999", "time": int64(-619524000000), "temp": int32(0)},
		{"station": "011990-99999", "time": int64(-619506000000), "temp": int32(22)},
		{"station": "011990-99999", "time": int64(-619484400000), "temp": int32(-11)},
		{"station": "012650-99999", "time": int64(-655531200000), "temp": int32(111)},
		{"station": "012650-99999", "time": int64(-655509600000), "temp": int32(78)},
	}
	for _, rec := range raw {
		record, err := goavro.NewRecord(goavro.RecordSchema(schema))
		if err != nil {
			log.Fatal(err)
		}
		for k, v := range rec {
			record.Set(k, v)
		}
		fw.Write(record)
	}
}
func main() {
	recordSchemaJSON := `
{
  "type": "record",
  "name": "comments",
  "doc:": "A basic schema for storing blog comments",
  "namespace": "com.example",
  "fields": [
    {
      "doc": "Name of user",
      "type": "string",
      "name": "username"
    },
    {
      "doc": "The content of the user's message",
      "type": "string",
      "name": "comment"
    },
    {
      "doc": "Unix epoch time in milliseconds",
      "type": "long",
      "name": "timestamp"
    }
  ]
}
`
	someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchemaJSON))
	if err != nil {
		log.Fatal(err)
	}
	// identify field name to set datum for
	someRecord.Set("username", "Aquaman")
	someRecord.Set("comment", "The Atlantic is oddly cold this morning!")
	// you can fully qualify the field name
	someRecord.Set("com.example.timestamp", int64(1082196484))

	codec, err := goavro.NewCodec(recordSchemaJSON)
	if err != nil {
		log.Fatal(err)
	}

	bb := new(bytes.Buffer)
	if err = codec.Encode(bb, someRecord); err != nil {
		log.Fatal(err)
	}

	actual := bb.Bytes()
	expected := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08")
	if bytes.Compare(actual, expected) != 0 {
		log.Printf("Actual: %#v; Expected: %#v", actual, expected)
	}
}
Beispiel #7
0
// use Avro
func makeSomeData(w io.Writer) error {
	var err error
	codec, err := goavro.NewCodec(schema)
	if err != nil {
		log.Fatal(err)
	}

	fw, err := codec.NewWriter(
		//		goavro.BlockSize(13),
		goavro.Compression(goavro.CompressionSnappy),
		goavro.ToWriter(w))

	if err != nil {
		log.Fatal("avro: cannot create Writer: ", err)
	}
	defer fw.Close()

	// GENERATE A Record based on the type.
	rec := map[string]interface{}{"long_field": int64(1), "integer_field": int32(2), "decimal_field": string(3),
		"float_field": float32(4), "double_field": float64(5), "boolean_field": true,
		"string_field": string("7"), "date_field": int64(8), "timestamp_field": int64(9),
	}

	for i := 0; i < 100; i++ {

		record, err := goavro.NewRecord(goavro.RecordSchema(schema))
		if err != nil {
			log.Fatal(err)
		}

		for k, v := range rec {
			record.Set(k, v)
		}

		fw.Write(record)
	}

	return nil
}
Beispiel #8
0
// NewRecord inits a new record
func (self *Avro) NewRecord() (*goavro.Record, error) {
	record, err := goavro.NewRecord(goavro.RecordSchema(self.recordSchemaJSON))
	return record, err
}
func main() {
	var innerRecords []interface{}

	// If we want to encode data, we need to put it in an actual
	// goavro.Record instance corresponding to the schema we wish
	// to encode against.
	//
	// NewRecord will create a goavro.Record instance
	// corresponding to the specified schema.

	innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatal(err)
	}
	innerRecord.Set("account", "Aquaman")
	innerRecord.Set("creationDate", int64(1082196484))
	innerRecords = append(innerRecords, innerRecord)

	innerRecord, err = goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatal(err)
	}
	innerRecord.Set("account", "Super Girl")
	innerRecord.Set("creationDate", int64(1442240878))
	innerRecords = append(innerRecords, innerRecord)

	// We create both an innerRecord and an outerRecord.
	outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
	if err != nil {
		log.Fatal(err)
	}
	// innerRecord is a completely seperate record instance from
	// outerRecord. Once we have an innerRecord instance it can be
	// assigned to the appropriate Datum item of the outerRecord.
	outerRecord.Set("users", innerRecords)

	// Encode the outerRecord into a bytes.Buffer
	bb := new(bytes.Buffer)
	if err = codec.Encode(bb, outerRecord); err != nil {
		log.Fatal(err)
	}
	// Compare encoded bytes against the expected bytes.
	actual := bb.Bytes()
	expected := []byte(
		"\x04" + // array of two elements
			"\x0eAquaman" + // first account
			"\x88\x88\x88\x88\x08" + // creationDate
			"\x14Super Girl" + // second account
			"\xdc\xe5\xb6\xdf\x0a\x00") // creationDate
	if bytes.Compare(actual, expected) != 0 {
		log.Printf("Actual: %#v; Expected: %#v", actual, expected)
	}
	// Let's decode the blob and print the output in JSON format
	// using goavro.Record's String() method.
	decoded, err := codec.Decode(bytes.NewReader(actual))
	fmt.Println(decoded)
	// we only need to perform type assertion if we want to access inside
	record := decoded.(*goavro.Record)
	fmt.Println("Record Name:", record.Name)
	fmt.Println("Record Fields:")
	for i, field := range record.Fields {
		fmt.Println(" field", i, field.Name, ":", field.Datum)
	}
}
Beispiel #10
0
func SelectDBData(db *sql.DB, query string) {
	defer timeTrack(time.Now(), "SelectDBData")

	rows, err := db.Query(query)
	check(err)
	defer rows.Close()

	columns, err := rows.Columns()
	check(err)

	values := make([]interface{}, len(columns))
	scanArgs := make([]interface{}, len(values))
	for i := range values {
		scanArgs[i] = &values[i]
	}

	c := 0
	var claims []interface{}
	outerSchema, innerSchema, codec := LoadAvroSchema(conf.Avro.OuterSchema, conf.Avro.InnerSchema)

	for rows.Next() {
		c += 1
		err = rows.Scan(scanArgs...)
		check(err)

		m := make(map[string]interface{})
		claim, err := goavro.NewRecord(innerSchema)
		check(err)
		for i, colName := range columns {
			if val, ok := values[i].(time.Time); ok {
				claim.Set(colName, val.Unix())
				m[colName] = val.Unix()
			} else {
				claim.Set(colName, values[i])
				m[colName] = values[i]
			}
		}

		claims = append(claims, claim)

		if len(claims) == 3000 {

			claims_avro, err := goavro.NewRecord(outerSchema)
			check(err)
			claims_avro.Set("claims", claims)
			buf := new(bytes.Buffer)
			err = codec.Encode(buf, claims_avro)
			go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
			claims = make([]interface{}, 0)
		}
	}

	if rows.Err() != nil {
		pialog.Error(rows.Err())
	}

	claims_avro, err := goavro.NewRecord(outerSchema)
	check(err)
	claims_avro.Set("claims", claims)

	buf := new(bytes.Buffer)
	err = codec.Encode(buf, claims_avro)
	check(err)

	go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)

	pialog.Info("Total input records:", c)
}
Beispiel #11
0
func main() {
	fd, err := os.Create("temp.avro")
	if err != nil {
		log.Println("error", err)
	}

	makeSomeData(fd)

	fd.Close()

	fd, err = os.Open("temp.avro")
	if err != nil {
		log.Println("error", err)
	}

	//dumpReader(fd)

	fd.Close()

	record, err := goavro.NewRecord(goavro.RecordSchema(schema))
	if err != nil {
		log.Println("error", err)
	}
	{
		fd, err = os.Create("temp.parquet")
		if err != nil {
			log.Println("error", err)
		}

		pschema := parquet.NewSchema()

		for _, f := range record.Fields {

			fschema, err := record.GetFieldSchema(f.Name)
			if err != nil {
				panic(err)
			}

			err = pschema.AddColumnFromThriftSchema(fschema.(map[string]interface{}))
			if err != nil {
				panic(err)
			}
		}

		enc := parquet.NewEncoder(pschema, fd)

		// Read the file
		fd, err = os.Open("temp.avro")
		if err != nil {
			log.Println("error", err)
		}

		fr, err := goavro.NewReader(goavro.BufferFromReader(fd))
		if err != nil {
			log.Fatal("cannot create Reader: ", err)
		}

		defer func() {
			if err := fr.Close(); err != nil {
				log.Fatal(err)
			}
		}()

		for fr.Scan() {
			datum, err := fr.Read()
			if err != nil {
				log.Println("cannot read datum: ", err)
				continue
			}
			r, ok := datum.(*goavro.Record)
			if !ok {
				panic("expected goavro.Record")
			}

			rec := make(map[string]interface{})
			for _, field := range r.Fields {
				v, err := r.Get(field.Name)
				if err != nil {
					panic(err)
				}
				rec[field.Name] = v
			}

			err = enc.WriteRecords([]map[string]interface{}{rec})
			if err != nil {
				panic(err)
			}
		}

		if err := enc.Close(); err != nil {
			panic(err)
		}

		fd.Close()
	}

	log.Println("finished")
}
func main() {
	innerRecords := make([]interface{}, 0)
	// make first inner record
	innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema))
	if err != nil {
		log.Fatalf("cannot create innerRecord: %v", err)
	}
	if err = innerRecord.Set("stringValue", "Hello"); err != nil {
		log.Fatal(err)
	}
	if err = innerRecord.Set("intValue", int32(1)); err != nil {
		log.Fatal(err)
	}
	innerRecords = append(innerRecords, innerRecord)
	// make another inner record
	innerRecord, _ = goavro.NewRecord(goavro.RecordSchema(innerSchema))
	innerRecord.Set("stringValue", "World")
	innerRecord.Set("intValue", int32(2))
	innerRecords = append(innerRecords, innerRecord)
	// make outer record
	outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema))
	if err != nil {
		log.Fatalf("cannot create outerRecord: %v", err)
	}
	outerRecord.Set("value", int32(3))
	outerRecord.Set("rec", innerRecords)
	// make a codec
	c, err := goavro.NewCodec(outerSchema)
	if err != nil {
		log.Fatal(err)
	}
	// encode outerRecord to io.Writer (here, a bytes.Buffer)
	bb := new(bytes.Buffer)
	err = c.Encode(bb, outerRecord)
	if err != nil {
		log.Fatal(err)
	}
	// decode bytes
	decoded, err := c.Decode(bytes.NewReader(bb.Bytes()))
	if err != nil {
		log.Fatal(err)
	}
	decodedRecord, ok := decoded.(*goavro.Record)
	if !ok {
		log.Fatalf("expected *goavro.Record; received: %T", decoded)
	}
	decodedValue, err := decodedRecord.Get("value")
	if err != nil {
		log.Fatal(err)
	}
	if decodedValue != int32(3) {
		log.Printf("Actual: %#v; Expected: %#v\n", decodedValue, int32(3))
	}
	fmt.Printf("Read a value: %d\n", decodedValue)
	rec, err := decodedRecord.Get("rec")
	if err != nil {
		log.Fatal(err)
	}
	decodedArray := rec.([]interface{})
	if len(decodedArray) != 2 {
		log.Fatalf("Actual: %#v; Expected: %#v\n", len(decodedArray), 2)
	}
	for index, decodedSubRecord := range decodedArray {
		r := decodedSubRecord.(*goavro.Record)
		sv, err := r.Get("stringValue")
		if err != nil {
			log.Fatal(err)
		}
		iv, err := r.Get("intValue")
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Read a subrecord %d string value: %s\n", index, sv)
		fmt.Printf("Read a subrecord %d int value: %d\n", index, iv)
	}
}