func processImporter(importer censysdata.Importer, db *sqldb.EntriesDatabase, wg *sync.WaitGroup) error { entryChan := make(chan censysdata.CensysEntry, 100) defer close(entryChan) statusChan := make(chan OperationStatus, 1) defer close(statusChan) wg.Add(1) defer wg.Done() displayProgress(statusChan, wg) numWorkers := *config.NumThreads * runtime.NumCPU() for i := 0; i < numWorkers; i++ { go insertCensysWorker(entryChan, db, wg) } startOffset := *config.OffsetByte // // Fast forward // if *config.Offset > 0 && *config.OffsetByte > 0 { return fmt.Errorf("You may not set both offset and offsetByte") } if *config.Offset > 0 { err := importer.SeekLine(*config.Offset) if err != nil { return err } } if *config.OffsetByte > 0 { err := importer.SeekByte(*config.OffsetByte) if err != nil { return err } } var maxOffset uint64 log.Printf("Starting import from offset=%d, line_limit=%d", importer.ByteOffset(), *config.Limit) // We've already fast-forwarded, so start at 0. for count := uint64(0); ; count++ { if *config.Limit > uint64(0) && count >= *config.Limit { return nil } ent, err := importer.NextEntry() if err != nil || ent == nil { return err } if count%128 == 0 { // Lazily obtain the max offset if maxOffset < 1 { maxOffset, err = importer.Size() if err != nil { return err } } statusChan <- OperationStatus{int64(startOffset), int64(ent.Offset), int64(maxOffset)} } entryChan <- *ent } return nil }