func ExampleObjectsFromJSONIsNull() { d := []byte("null") objects, _ := data.ObjectsFromJSON(d) fmt.Println(fmt.Sprintf("%+v", objects)) // Output: [] }
func ExampleObjectsFromJSON() { d := []byte(`[{"One":1}, {"Two":2}]`) objects, _ := data.ObjectsFromJSON(d) fmt.Println(fmt.Sprintf("%+v", objects)) // Output: [map[One:1] map[Two:2]] }
// ProcessData defers to WriterBatch func (w *BigQueryWriter) ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error) { queuedRows, err := data.ObjectsFromJSON(d) util.KillPipelineIfErr(err, killChan) logger.Info("BigQueryWriter: Writing -", len(queuedRows)) err = w.WriteBatch(queuedRows) if err != nil { util.KillPipelineIfErr(err, killChan) } logger.Info("BigQueryWriter: Write complete") }
// CSVProcess writes the contents to the file and optionally sends the written bytes // upstream on outputChan func CSVProcess(params *CSVParameters, d data.JSON, outputChan chan data.JSON, killChan chan error) { objects, err := data.ObjectsFromJSON(d) KillPipelineIfErr(err, killChan) if params.Header == nil { for k := range objects[0] { params.Header = append(params.Header, k) } sort.Strings(params.Header) } rows := [][]string{} if params.WriteHeader && !params.HeaderWritten { headerRow := []string{} for _, k := range params.Header { headerRow = append(headerRow, CSVString(k)) } rows = append(rows, headerRow) params.HeaderWritten = true } for _, object := range objects { row := []string{} for i := range params.Header { v := object[params.Header[i]] row = append(row, CSVString(v)) } rows = append(rows, row) } if params.SendUpstream { var b bytes.Buffer params.Writer.SetWriter(bufio.NewWriter(&b)) err = params.Writer.WriteAll(rows) KillPipelineIfErr(err, killChan) outputChan <- []byte(b.String()) } else { err = params.Writer.WriteAll(rows) KillPipelineIfErr(err, killChan) } }
// SQLInsertData abstracts building and executing a SQL INSERT // statement for the given Data object. // // Note that the Data must be a valid JSON object // (or an array of valid objects all with the same keys), // where the keys are column names and the // the values are SQL values to be inserted into those columns. func SQLInsertData(db *sql.DB, d data.JSON, tableName string, onDupKeyUpdate bool, onDupKeyFields []string, batchSize int) error { objects, err := data.ObjectsFromJSON(d) if err != nil { return err } if batchSize > 0 { for i := 0; i < len(objects); i += batchSize { maxIndex := i + batchSize if maxIndex > len(objects) { maxIndex = len(objects) } err = insertObjects(db, objects[i:maxIndex], tableName, onDupKeyUpdate, onDupKeyFields) if err != nil { return err } } return nil } return insertObjects(db, objects, tableName, onDupKeyUpdate, onDupKeyFields) }