Example #1
0
func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
	for _, shard := range shards {
		ds := shard.Parent
		if len(ds.ExternalOutputChans) == 0 {
			continue
		}
		// connect remote raw chan to local typed chan
		readChanName := shard.Name()
		location := s.datasetShard2Location[readChanName]
		rawChan, err := io.GetDirectReadChannel(readChanName, location.URL())
		if err != nil {
			log.Panic(err)
		}
		for _, out := range ds.ExternalOutputChans {
			ch := make(chan reflect.Value)
			io.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
			waitGroup.Add(1)
			go func() {
				defer waitGroup.Done()
				for v := range ch {
					v = io.CleanObject(v, ds.Type, out.Type().Elem())
					out.Send(v)
				}
			}()
		}
	}
}
Example #2
0
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
	task := tr.Tasks[0]
	for i, shard := range task.Inputs {
		d := shard.Parent
		readChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
		rawChan, err := io.GetDirectReadChannel(readChanName, name2Location[readChanName])
		if err != nil {
			log.Panic(err)
		}
		io.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
	}
}