func dumpReader(r io.Reader) {
fr, err := goavro.NewReader(goavro.BufferFromReader(r))
if err != nil {
log.Fatal(err)
}
defer func() {
if err := fr.Close(); err != nil {
log.Fatal(err)
}
}()
for fr.Scan() {
datum, err := fr.Read()
if err != nil {
log.Println(err)
continue
}
fmt.Println(datum)
}
}
func main() {
fd, err := os.Create("temp.avro")
if err != nil {
log.Println("error", err)
}
makeSomeData(fd)
fd.Close()
fd, err = os.Open("temp.avro")
if err != nil {
log.Println("error", err)
}
//dumpReader(fd)
fd.Close()
record, err := goavro.NewRecord(goavro.RecordSchema(schema))
if err != nil {
log.Println("error", err)
}
{
fd, err = os.Create("temp.parquet")
if err != nil {
log.Println("error", err)
}
pschema := parquet.NewSchema()
for _, f := range record.Fields {
fschema, err := record.GetFieldSchema(f.Name)
if err != nil {
panic(err)
}
err = pschema.AddColumnFromThriftSchema(fschema.(map[string]interface{}))
if err != nil {
panic(err)
}
}
enc := parquet.NewEncoder(pschema, fd)
// Read the file
fd, err = os.Open("temp.avro")
if err != nil {
log.Println("error", err)
}
fr, err := goavro.NewReader(goavro.BufferFromReader(fd))
if err != nil {
log.Fatal("cannot create Reader: ", err)
}
defer func() {
if err := fr.Close(); err != nil {
log.Fatal(err)
}
}()
for fr.Scan() {
datum, err := fr.Read()
if err != nil {
log.Println("cannot read datum: ", err)
continue
}
r, ok := datum.(*goavro.Record)
if !ok {
panic("expected goavro.Record")
}
rec := make(map[string]interface{})
for _, field := range r.Fields {
v, err := r.Get(field.Name)
if err != nil {
panic(err)
}
rec[field.Name] = v
}
err = enc.WriteRecords([]map[string]interface{}{rec})
if err != nil {
panic(err)
}
}
if err := enc.Close(); err != nil {
panic(err)
}
fd.Close()
}
log.Println("finished")
}