Example #1
0
func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) {
	task := tr.Tasks[len(tr.Tasks)-1]
	for _, shard := range task.Outputs {
		writeChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "step", task.Step.Id, "task", task.Id, "writing to:", writeChanName)
		rawChan, err := io.GetLocalSendChannel(writeChanName, wg)
		if err != nil {
			log.Panic(err)
		}
		io.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg)
	}
}
Example #2
0
func (s *Scheduler) setupInputChannels(fc *flow.FlowContext, task *flow.Task, location resource.Location, waitGroup *sync.WaitGroup) {
	if len(task.Inputs) > 0 {
		return
	}
	ds := task.Outputs[0].Parent
	if len(ds.ExternalInputChans) == 0 {
		return
	}
	// connect local typed chan to remote raw chan
	// write to the dataset location in the cluster so that the task can be retried if needed.
	for i, inChan := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", fc.Id, ds.Id, i)
		// println("setup input channel for", task.Name(), "on", location.URL())
		s.registerDatasetShardLocation(inputChanName, location)
		rawChan, err := io.GetDirectSendChannel(inputChanName, location.URL(), waitGroup)
		if err != nil {
			log.Panic(err)
		}
		// println("writing", inputChanName, "to", location.URL())
		io.ConnectTypedWriteChannelToRaw(inChan, rawChan, waitGroup)
	}
}