func (pb *Bar) formatCounts() (string, string) {
currentCount, maxCount := pb.Watching.Progress()
if pb.IsBytes {
return text.FormatByteAmount(maxCount), text.FormatByteAmount(currentCount)
}
return fmt.Sprintf("%v", maxCount), fmt.Sprintf("%v", currentCount)
}
// runInsertionWorker is a helper to InsertDocuments - it reads document off
// the read channel and prepares then in batches for insertion into the databas
func (imp *MongoImport) runInsertionWorker(readDocs chan bson.D) (err error) {
session, err := imp.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error connecting to mongod: %v", err)
}
defer session.Close()
if err = imp.configureSession(session); err != nil {
return fmt.Errorf("error configuring session: %v", err)
}
collection := session.DB(imp.ToolOptions.DB).C(imp.ToolOptions.Collection)
ignoreBlanks := imp.IngestOptions.IgnoreBlanks && imp.InputOptions.Type != JSON
var documentBytes []byte
var documents []bson.Raw
numMessageBytes := 0
readLoop:
for {
select {
case document, alive := <-readDocs:
if !alive {
break readLoop
}
// the mgo driver doesn't currently respect the maxBatchSize
// limit so we self impose a limit by using maxMessageSizeBytes
// and send documents over the wire when we hit the batch size
// or when we're at/over the maximum message size threshold
if len(documents) == imp.ToolOptions.BulkBufferSize || numMessageBytes >= maxMessageSizeBytes {
if err = imp.insert(documents, collection); err != nil {
return err
}
documents = documents[:0]
numMessageBytes = 0
}
// ignore blank fields if specified
if ignoreBlanks {
document = removeBlankFields(document)
}
if documentBytes, err = bson.Marshal(document); err != nil {
return err
}
if len(documentBytes) > maxBSONSize {
log.Logf(log.Always, "warning: attempting to insert document with size %v (exceeds %v limit)",
text.FormatByteAmount(int64(len(documentBytes))), text.FormatByteAmount(maxBSONSize))
}
numMessageBytes += len(documentBytes)
documents = append(documents, bson.Raw{3, documentBytes})
case <-imp.Dying():
return nil
}
}
// ingest any documents left in slice
if len(documents) != 0 {
return imp.insert(documents, collection)
}
return nil
}