func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
for _, shard := range shards {
ds := shard.Parent
if len(ds.ExternalOutputChans) == 0 {
continue
}
// connect remote raw chan to local typed chan
readChanName := s.option.ExecutableFileHash + "-" + shard.Name()
location, _ := s.shardLocator.GetShardLocation(readChanName)
rawChan, err := netchan.GetDirectReadChannel(readChanName, location.URL(), 1024)
if err != nil {
log.Panic(err)
}
for _, out := range ds.ExternalOutputChans {
ch := make(chan reflect.Value)
netchan.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
for v := range ch {
v = netchan.CleanObject(v, ds.Type, out.Type().Elem())
out.Send(v)
}
}()
}
}
}
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
task := tr.Tasks[0]
for i, shard := range task.Inputs {
d := shard.Parent
readChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
rawChan, err := netchan.GetDirectReadChannel(readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize)
if err != nil {
log.Panic(err)
}
netchan.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
}
}
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
firstTask := tr.Tasks[0]
for i, shard := range firstTask.Inputs {
d := shard.Parent
readChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
// println("taskGroup", tr.option.TaskGroupId, "firstTask", firstTask.Name(), "trying to read from:", readChanName, len(firstTask.InputChans))
rawChan, err := netchan.GetDirectReadChannel(tr.option.TaskTlsConfig, readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize)
if err != nil {
log.Panic(err)
}
inChanStatus := netchan.ConnectRawReadChannelToTyped(rawChan, firstTask.InputChans[i], d.Type, wg)
inChanStatus.Name = shard.DisplayName()
tr.executorStatus.InputChannelStatuses = append(tr.executorStatus.InputChannelStatuses, inChanStatus)
}
}
func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) {
// this is only for Channel dataset
firstTask := tr.Tasks[0]
if firstTask.Inputs != nil {
return
}
ds := firstTask.Outputs[0].Parent
for i, _ := range ds.ExternalInputChans {
inputChanName := fmt.Sprintf("%s-ct-%d-input-%d-p-%d", tr.option.ExecutableFileHash, tr.option.ContextId, ds.Id, i)
rawChan, err := netchan.GetLocalReadChannel(inputChanName, tr.FlowContext.ChannelBufferSize)
if err != nil {
log.Panic(err)
}
typedInputChan := make(chan reflect.Value)
netchan.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg)
firstTask.InputChans = append(firstTask.InputChans, typedInputChan)
}
}