func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) { for _, shard := range shards { ds := shard.Parent if len(ds.ExternalOutputChans) == 0 { continue } // connect remote raw chan to local typed chan readChanName := s.option.ExecutableFileHash + "-" + shard.Name() location, _ := s.shardLocator.GetShardLocation(readChanName) rawChan, err := netchan.GetDirectReadChannel(readChanName, location.URL(), 1024) if err != nil { log.Panic(err) } for _, out := range ds.ExternalOutputChans { ch := make(chan reflect.Value) netchan.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup) waitGroup.Add(1) go func() { defer waitGroup.Done() for v := range ch { v = netchan.CleanObject(v, ds.Type, out.Type().Elem()) out.Send(v) } }() } } }
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) { task := tr.Tasks[0] for i, shard := range task.Inputs { d := shard.Parent readChanName := tr.option.ExecutableFileHash + "-" + shard.Name() // println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans)) rawChan, err := netchan.GetDirectReadChannel(readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize) if err != nil { log.Panic(err) } netchan.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg) } }
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) { firstTask := tr.Tasks[0] for i, shard := range firstTask.Inputs { d := shard.Parent readChanName := tr.option.ExecutableFileHash + "-" + shard.Name() // println("taskGroup", tr.option.TaskGroupId, "firstTask", firstTask.Name(), "trying to read from:", readChanName, len(firstTask.InputChans)) rawChan, err := netchan.GetDirectReadChannel(tr.option.TaskTlsConfig, readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize) if err != nil { log.Panic(err) } inChanStatus := netchan.ConnectRawReadChannelToTyped(rawChan, firstTask.InputChans[i], d.Type, wg) inChanStatus.Name = shard.DisplayName() tr.executorStatus.InputChannelStatuses = append(tr.executorStatus.InputChannelStatuses, inChanStatus) } }
func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) { // this is only for Channel dataset firstTask := tr.Tasks[0] if firstTask.Inputs != nil { return } ds := firstTask.Outputs[0].Parent for i, _ := range ds.ExternalInputChans { inputChanName := fmt.Sprintf("%s-ct-%d-input-%d-p-%d", tr.option.ExecutableFileHash, tr.option.ContextId, ds.Id, i) rawChan, err := netchan.GetLocalReadChannel(inputChanName, tr.FlowContext.ChannelBufferSize) if err != nil { log.Panic(err) } typedInputChan := make(chan reflect.Value) netchan.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg) firstTask.InputChans = append(firstTask.InputChans, typedInputChan) } }