func ReduceFunc(c context.Context, mr MapReducePipeline, writer SingleOutputWriter, shardNames []string,
separateReduceItems bool, statusFunc StatusUpdateFunc) error {
merger := newMerger(mr)
toClose := make([]io.Closer, 0, len(shardNames))
defer func() {
for _, c := range toClose {
c.Close()
}
}()
type result struct {
iterator IntermediateStorageIterator
err error
}
results := make([]result, len(shardNames))
wg := sync.WaitGroup{}
for i, shardName := range shardNames {
i, shardName := i, shardName
wg.Add(1)
go func() {
defer wg.Done()
iterator, err := mr.Iterator(c, shardName, mr)
results[i] = result{iterator, err}
}()
}
wg.Wait()
for i, result := range results {
if result.err != nil {
return tryAgainError{fmt.Errorf("cannot open intermediate file %s: %s", shardNames[i], result.err)}
}
merger.addSource(result.iterator)
toClose = append(toClose, result.iterator)
}
values := make([]interface{}, 1)
var key interface{}
if first, err := merger.next(); err != nil {
return err
} else if first == nil {
logInfo(c, "No results to process from map")
writer.Close(c)
for _, shardName := range shardNames {
if err := mr.RemoveIntermediate(c, shardName); err != nil {
logError(c, "failed to remove intermediate file: %s", err.Error())
}
}
return nil
} else {
key = first.Key
values[0] = first.Value
}
for !merger.empty() {
item, err := merger.next()
if err != nil {
return tryAgainError{err}
}
if !separateReduceItems && mr.Equal(key, item.Key) {
values = append(values, item.Value)
continue
}
if result, err := mr.Reduce(key, values, statusFunc); err != nil {
if _, ok := err.(FatalError); ok {
err = err.(FatalError).Err
} else {
err = tryAgainError{err}
}
return err
} else if result != nil {
if err := writer.Write(result); err != nil {
return err
}
}
key = item.Key
values = values[0:1]
values[0] = item.Value
}
if result, err := mr.Reduce(key, values, statusFunc); err != nil {
if _, ok := err.(FatalError); ok {
err = err.(FatalError).Err
} else {
err = tryAgainError{err}
}
return err
} else if result != nil {
if err := writer.Write(result); err != nil {
return tryAgainError{err}
}
}
if results, err := mr.ReduceComplete(statusFunc); err != nil {
return err
} else {
for _, result := range results {
if err := writer.Write(result); err != nil {
return tryAgainError{err}
}
}
}
for _, shardName := range shardNames {
if err := mr.RemoveIntermediate(c, shardName); err != nil {
logError(c, "failed to remove intermediate file: %s", err.Error())
}
}
return nil
}