func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) { for _, shard := range shards { ds := shard.Parent if len(ds.ExternalOutputChans) == 0 { continue } // connect remote raw chan to local typed chan readChanName := shard.Name() location := s.datasetShard2Location[readChanName] rawChan, err := io.GetDirectReadChannel(readChanName, location.URL()) if err != nil { log.Panic(err) } for _, out := range ds.ExternalOutputChans { ch := make(chan reflect.Value) io.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup) waitGroup.Add(1) go func() { defer waitGroup.Done() for v := range ch { v = io.CleanObject(v, ds.Type, out.Type().Elem()) out.Send(v) } }() } } }
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) { task := tr.Tasks[0] for i, shard := range task.Inputs { d := shard.Parent readChanName := shard.Name() // println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans)) rawChan, err := io.GetDirectReadChannel(readChanName, name2Location[readChanName]) if err != nil { log.Panic(err) } io.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg) } }