func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) {
task := tr.Tasks[len(tr.Tasks)-1]
for _, shard := range task.Outputs {
writeChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
// println("taskGroup", tr.option.TaskGroupId, "step", task.Step.Id, "task", task.Id, "writing to:", writeChanName)
rawChan, err := netchan.GetLocalSendChannel(writeChanName, wg)
if err != nil {
log.Panic(err)
}
netchan.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg)
}
}
func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) {
lastTask := tr.Tasks[len(tr.Tasks)-1]
for _, shard := range lastTask.Outputs {
writeChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
// println("taskGroup", tr.option.TaskGroupId, "step", lastTask.Step.Id, "lastTask", lastTask.Id, "writing to:", writeChanName, "on", tr.option.AgentAddress)
rawChan, err := netchan.GetDirectSendChannel(tr.option.TaskTlsConfig, writeChanName, tr.option.AgentAddress, wg)
if err != nil {
log.Panic(err)
}
outChanStatus := netchan.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg)
outChanStatus.Name = shard.DisplayName()
tr.executorStatus.OutputChannelStatuses = append(tr.executorStatus.OutputChannelStatuses, outChanStatus)
}
}
func (s *Scheduler) setupInputChannels(fc *flow.FlowContext, task *flow.Task, location resource.Location, waitGroup *sync.WaitGroup) {
if len(task.Inputs) > 0 {
return
}
ds := task.Outputs[0].Parent
if len(ds.ExternalInputChans) == 0 {
return
}
// connect local typed chan to remote raw chan
// write to the dataset location in the cluster so that the task can be retried if needed.
for i, inChan := range ds.ExternalInputChans {
inputChanName := fmt.Sprintf("%s-ct-%d-input-%d-p-%d", s.option.ExecutableFileHash, fc.Id, ds.Id, i)
// println("setup input channel for", task.Name(), "on", location.URL())
s.shardLocator.SetShardLocation(inputChanName, location)
rawChan, err := netchan.GetDirectSendChannel(inputChanName, location.URL(), waitGroup)
if err != nil {
log.Panic(err)
}
// println("writing", inputChanName, "to", location.URL())
netchan.ConnectTypedWriteChannelToRaw(inChan, rawChan, waitGroup)
}
}