Golang Elasticsearch.Bulk Examples

Example #1

File: test_data.go Project: sk163/bonito

// Inserts into Elasticsearch the transactions from the channel
// Uses batches and the bulk API.
func InsertInEsFromChan(es *elasticsearch.Elasticsearch, index string,
	transactions chan TestTransaction) (int, error) {

	var buf bytes.Buffer

	enc := json.NewEncoder(&buf)

	var insOp struct {
		Index struct {
			Type string `json:"_type"`
		} `json:"index"`
	}
	insOp.Index.Type = "trans"

	flush := func() error {
		_, err := es.Bulk(index, &buf)
		if err != nil {
			return err
		}

		buf.Reset()
		enc = json.NewEncoder(&buf)
		return nil
	}

	i := 0
	for trans := range transactions {
		enc.Encode(insOp)
		enc.Encode(trans)

		if i%1000 == 0 {
			if err := flush(); err != nil {
				return i, err
			}
		}
		i++
	}

	if err := flush(); err != nil {
		return i, err
	}

	_, err := es.Refresh(index)
	if err != nil {
		return i, err
	}

	return i, nil
}