func (pb *Bar) formatCounts() (string, string) { currentCount, maxCount := pb.Watching.Progress() if pb.IsBytes { return text.FormatByteAmount(maxCount), text.FormatByteAmount(currentCount) } return fmt.Sprintf("%v", maxCount), fmt.Sprintf("%v", currentCount) }
// runInsertionWorker is a helper to InsertDocuments - it reads document off // the read channel and prepares then in batches for insertion into the databas func (imp *MongoImport) runInsertionWorker(readDocs chan bson.D) (err error) { session, err := imp.SessionProvider.GetSession() if err != nil { return fmt.Errorf("error connecting to mongod: %v", err) } defer session.Close() if err = imp.configureSession(session); err != nil { return fmt.Errorf("error configuring session: %v", err) } collection := session.DB(imp.ToolOptions.DB).C(imp.ToolOptions.Collection) ignoreBlanks := imp.IngestOptions.IgnoreBlanks && imp.InputOptions.Type != JSON var documentBytes []byte var documents []bson.Raw numMessageBytes := 0 readLoop: for { select { case document, alive := <-readDocs: if !alive { break readLoop } // the mgo driver doesn't currently respect the maxBatchSize // limit so we self impose a limit by using maxMessageSizeBytes // and send documents over the wire when we hit the batch size // or when we're at/over the maximum message size threshold if len(documents) == imp.ToolOptions.BulkBufferSize || numMessageBytes >= maxMessageSizeBytes { if err = imp.insert(documents, collection); err != nil { return err } documents = documents[:0] numMessageBytes = 0 } // ignore blank fields if specified if ignoreBlanks { document = removeBlankFields(document) } if documentBytes, err = bson.Marshal(document); err != nil { return err } if len(documentBytes) > maxBSONSize { log.Logf(log.Always, "warning: attempting to insert document with size %v (exceeds %v limit)", text.FormatByteAmount(int64(len(documentBytes))), text.FormatByteAmount(maxBSONSize)) } numMessageBytes += len(documentBytes) documents = append(documents, bson.Raw{3, documentBytes}) case <-imp.Dying(): return nil } } // ingest any documents left in slice if len(documents) != 0 { return imp.insert(documents, collection) } return nil }