func mergeDB(newDB, db *gumshoe.DB, parallelism, flushSegments int) error {
resp := db.MakeRequest()
defer resp.Done()
allSegments := findSegments(resp.StaticTable)
progress := NewProgress("segments processed", len(allSegments))
progress.Print()
segments := make(chan *timestampSegment)
var wg wait.Group
for i := 0; i < parallelism; i++ {
wg.Go(func(quit <-chan struct{}) error {
for {
select {
case <-quit:
return nil
case segment, ok := <-segments:
if !ok {
return nil
}
if err := mergeSegment(newDB, db, segment); err != nil {
return err
}
progress.Add(1)
}
}
})
}
wg.Go(func(quit <-chan struct{}) error {
flushSegmentCount := 0
for _, segment := range allSegments {
select {
case <-quit:
return nil
default:
select {
case <-quit:
return nil
case segments <- segment:
flushSegmentCount++
if flushSegmentCount == flushSegments {
flushSegmentCount = 0
if err := newDB.Flush(); err != nil {
return err
}
}
}
}
}
close(segments)
return nil
})
err := wg.Wait()
if err != nil {
return err
}
return newDB.Flush()
}
func migrateDBs(newDB, oldDB *gumshoe.DB, parallelism, flushSegments int) error {
resp := oldDB.MakeRequest()
defer resp.Done()
allSegments := findSegments(resp.StaticTable)
progress := NewProgress("segments processed", len(allSegments))
progress.Print()
segments := make(chan *timestampSegment)
shutdown := make(chan struct{})
var workerErr error
errc := make(chan error)
var wg sync.WaitGroup
wg.Add(parallelism)
conversionFunc, err := makeConversionFunc(newDB, oldDB)
if err != nil {
return err
}
for i := 0; i < parallelism; i++ {
go func() {
defer wg.Done()
for {
select {
case segment := <-segments:
if err := migrateSegment(newDB, oldDB, segment, conversionFunc); err != nil {
select {
case errc <- err:
case <-shutdown:
}
return
}
progress.Add(1)
case <-shutdown:
return
}
}
}()
}
flushSegmentCount := 0
outer:
for _, segment := range allSegments {
select {
case err := <-errc:
workerErr = err
break outer
default:
select {
case segments <- segment:
flushSegmentCount++
if flushSegmentCount == flushSegments {
flushSegmentCount = 0
if err := newDB.Flush(); err != nil {
workerErr = err
break outer
}
}
case err := <-errc:
workerErr = err
break outer
}
}
}
close(shutdown)
wg.Wait()
if workerErr != nil {
return workerErr
}
return newDB.Flush()
}