func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) { task := tr.Tasks[len(tr.Tasks)-1] for _, shard := range task.Outputs { writeChanName := shard.Name() // println("taskGroup", tr.option.TaskGroupId, "step", task.Step.Id, "task", task.Id, "writing to:", writeChanName) rawChan, err := io.GetLocalSendChannel(writeChanName, wg) if err != nil { log.Panic(err) } io.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg) } }
func (s *Scheduler) setupInputChannels(fc *flow.FlowContext, task *flow.Task, location resource.Location, waitGroup *sync.WaitGroup) { if len(task.Inputs) > 0 { return } ds := task.Outputs[0].Parent if len(ds.ExternalInputChans) == 0 { return } // connect local typed chan to remote raw chan // write to the dataset location in the cluster so that the task can be retried if needed. for i, inChan := range ds.ExternalInputChans { inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", fc.Id, ds.Id, i) // println("setup input channel for", task.Name(), "on", location.URL()) s.registerDatasetShardLocation(inputChanName, location) rawChan, err := io.GetDirectSendChannel(inputChanName, location.URL(), waitGroup) if err != nil { log.Panic(err) } // println("writing", inputChanName, "to", location.URL()) io.ConnectTypedWriteChannelToRaw(inChan, rawChan, waitGroup) } }