コード例 #1
0
ファイル: reduce.go プロジェクト: pendo-io/mapreduce
func ReduceFunc(c context.Context, mr MapReducePipeline, writer SingleOutputWriter, shardNames []string,
	separateReduceItems bool, statusFunc StatusUpdateFunc) error {

	merger := newMerger(mr)

	toClose := make([]io.Closer, 0, len(shardNames))
	defer func() {
		for _, c := range toClose {
			c.Close()
		}
	}()

	type result struct {
		iterator IntermediateStorageIterator
		err      error
	}

	results := make([]result, len(shardNames))
	wg := sync.WaitGroup{}

	for i, shardName := range shardNames {
		i, shardName := i, shardName
		wg.Add(1)

		go func() {
			defer wg.Done()
			iterator, err := mr.Iterator(c, shardName, mr)
			results[i] = result{iterator, err}
		}()
	}

	wg.Wait()

	for i, result := range results {
		if result.err != nil {
			return tryAgainError{fmt.Errorf("cannot open intermediate file %s: %s", shardNames[i], result.err)}
		}

		merger.addSource(result.iterator)
		toClose = append(toClose, result.iterator)
	}

	values := make([]interface{}, 1)
	var key interface{}

	if first, err := merger.next(); err != nil {
		return err
	} else if first == nil {
		logInfo(c, "No results to process from map")
		writer.Close(c)
		for _, shardName := range shardNames {
			if err := mr.RemoveIntermediate(c, shardName); err != nil {
				logError(c, "failed to remove intermediate file: %s", err.Error())
			}
		}

		return nil
	} else {
		key = first.Key
		values[0] = first.Value
	}

	for !merger.empty() {
		item, err := merger.next()
		if err != nil {
			return tryAgainError{err}
		}

		if !separateReduceItems && mr.Equal(key, item.Key) {
			values = append(values, item.Value)
			continue
		}

		if result, err := mr.Reduce(key, values, statusFunc); err != nil {
			if _, ok := err.(FatalError); ok {
				err = err.(FatalError).Err
			} else {
				err = tryAgainError{err}
			}
			return err
		} else if result != nil {
			if err := writer.Write(result); err != nil {
				return err
			}
		}

		key = item.Key
		values = values[0:1]
		values[0] = item.Value
	}

	if result, err := mr.Reduce(key, values, statusFunc); err != nil {
		if _, ok := err.(FatalError); ok {
			err = err.(FatalError).Err
		} else {
			err = tryAgainError{err}
		}
		return err
	} else if result != nil {
		if err := writer.Write(result); err != nil {
			return tryAgainError{err}
		}
	}

	if results, err := mr.ReduceComplete(statusFunc); err != nil {
		return err
	} else {
		for _, result := range results {
			if err := writer.Write(result); err != nil {
				return tryAgainError{err}
			}
		}
	}

	for _, shardName := range shardNames {
		if err := mr.RemoveIntermediate(c, shardName); err != nil {
			logError(c, "failed to remove intermediate file: %s", err.Error())
		}
	}

	return nil
}