func ExampleObjectsFromJSONIsNull() {
d := []byte("null")
objects, _ := data.ObjectsFromJSON(d)
fmt.Println(fmt.Sprintf("%+v", objects))
// Output: []
}
func ExampleObjectsFromJSON() {
d := []byte(`[{"One":1},
{"Two":2}]`)
objects, _ := data.ObjectsFromJSON(d)
fmt.Println(fmt.Sprintf("%+v", objects))
// Output: [map[One:1] map[Two:2]]
}
// ProcessData defers to WriterBatch
func (w *BigQueryWriter) ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error) {
queuedRows, err := data.ObjectsFromJSON(d)
util.KillPipelineIfErr(err, killChan)
logger.Info("BigQueryWriter: Writing -", len(queuedRows))
err = w.WriteBatch(queuedRows)
if err != nil {
util.KillPipelineIfErr(err, killChan)
}
logger.Info("BigQueryWriter: Write complete")
}
// CSVProcess writes the contents to the file and optionally sends the written bytes
// upstream on outputChan
func CSVProcess(params *CSVParameters, d data.JSON, outputChan chan data.JSON, killChan chan error) {
objects, err := data.ObjectsFromJSON(d)
KillPipelineIfErr(err, killChan)
if params.Header == nil {
for k := range objects[0] {
params.Header = append(params.Header, k)
}
sort.Strings(params.Header)
}
rows := [][]string{}
if params.WriteHeader && !params.HeaderWritten {
headerRow := []string{}
for _, k := range params.Header {
headerRow = append(headerRow, CSVString(k))
}
rows = append(rows, headerRow)
params.HeaderWritten = true
}
for _, object := range objects {
row := []string{}
for i := range params.Header {
v := object[params.Header[i]]
row = append(row, CSVString(v))
}
rows = append(rows, row)
}
if params.SendUpstream {
var b bytes.Buffer
params.Writer.SetWriter(bufio.NewWriter(&b))
err = params.Writer.WriteAll(rows)
KillPipelineIfErr(err, killChan)
outputChan <- []byte(b.String())
} else {
err = params.Writer.WriteAll(rows)
KillPipelineIfErr(err, killChan)
}
}
// SQLInsertData abstracts building and executing a SQL INSERT
// statement for the given Data object.
//
// Note that the Data must be a valid JSON object
// (or an array of valid objects all with the same keys),
// where the keys are column names and the
// the values are SQL values to be inserted into those columns.
func SQLInsertData(db *sql.DB, d data.JSON, tableName string, onDupKeyUpdate bool, onDupKeyFields []string, batchSize int) error {
objects, err := data.ObjectsFromJSON(d)
if err != nil {
return err
}
if batchSize > 0 {
for i := 0; i < len(objects); i += batchSize {
maxIndex := i + batchSize
if maxIndex > len(objects) {
maxIndex = len(objects)
}
err = insertObjects(db, objects[i:maxIndex], tableName, onDupKeyUpdate, onDupKeyFields)
if err != nil {
return err
}
}
return nil
}
return insertObjects(db, objects, tableName, onDupKeyUpdate, onDupKeyFields)
}