func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
for _, shard := range shards {
ds := shard.Parent
if len(ds.ExternalOutputChans) == 0 {
continue
}
// connect remote raw chan to local typed chan
readChanName := shard.Name()
location := s.datasetShard2Location[readChanName]
rawChan, err := io.GetDirectReadChannel(readChanName, location.URL())
if err != nil {
log.Panic(err)
}
for _, out := range ds.ExternalOutputChans {
ch := make(chan reflect.Value)
io.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
for v := range ch {
v = io.CleanObject(v, ds.Type, out.Type().Elem())
out.Send(v)
}
}()
}
}
}
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
task := tr.Tasks[0]
for i, shard := range task.Inputs {
d := shard.Parent
readChanName := shard.Name()
// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
rawChan, err := io.GetDirectReadChannel(readChanName, name2Location[readChanName])
if err != nil {
log.Panic(err)
}
io.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
}
}
func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) {
// this is only for Channel dataset
firstTask := tr.Tasks[0]
if firstTask.Inputs != nil {
return
}
ds := firstTask.Outputs[0].Parent
for i, _ := range ds.ExternalInputChans {
inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", tr.option.ContextId, ds.Id, i)
rawChan, err := io.GetLocalReadChannel(inputChanName)
if err != nil {
log.Panic(err)
}
typedInputChan := make(chan reflect.Value)
io.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg)
firstTask.InputChans = append(firstTask.InputChans, typedInputChan)
}
}