Exemple #1
0
func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
	for _, shard := range shards {
		ds := shard.Parent
		if len(ds.ExternalOutputChans) == 0 {
			continue
		}
		// connect remote raw chan to local typed chan
		readChanName := s.option.ExecutableFileHash + "-" + shard.Name()
		location, _ := s.shardLocator.GetShardLocation(readChanName)
		rawChan, err := netchan.GetDirectReadChannel(readChanName, location.URL(), 1024)
		if err != nil {
			log.Panic(err)
		}
		for _, out := range ds.ExternalOutputChans {
			ch := make(chan reflect.Value)
			netchan.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
			waitGroup.Add(1)
			go func() {
				defer waitGroup.Done()
				for v := range ch {
					v = netchan.CleanObject(v, ds.Type, out.Type().Elem())
					out.Send(v)
				}
			}()
		}
	}
}
Exemple #2
0
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
	task := tr.Tasks[0]
	for i, shard := range task.Inputs {
		d := shard.Parent
		readChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
		rawChan, err := netchan.GetDirectReadChannel(readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize)
		if err != nil {
			log.Panic(err)
		}
		netchan.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
	}
}
Exemple #3
0
func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
	firstTask := tr.Tasks[0]
	for i, shard := range firstTask.Inputs {
		d := shard.Parent
		readChanName := tr.option.ExecutableFileHash + "-" + shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "firstTask", firstTask.Name(), "trying to read from:", readChanName, len(firstTask.InputChans))
		rawChan, err := netchan.GetDirectReadChannel(tr.option.TaskTlsConfig, readChanName, name2Location[readChanName], tr.FlowContext.ChannelBufferSize)
		if err != nil {
			log.Panic(err)
		}
		inChanStatus := netchan.ConnectRawReadChannelToTyped(rawChan, firstTask.InputChans[i], d.Type, wg)
		inChanStatus.Name = shard.DisplayName()
		tr.executorStatus.InputChannelStatuses = append(tr.executorStatus.InputChannelStatuses, inChanStatus)
	}
}
Exemple #4
0
func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) {
	// this is only for Channel dataset
	firstTask := tr.Tasks[0]
	if firstTask.Inputs != nil {
		return
	}
	ds := firstTask.Outputs[0].Parent
	for i, _ := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("%s-ct-%d-input-%d-p-%d", tr.option.ExecutableFileHash, tr.option.ContextId, ds.Id, i)
		rawChan, err := netchan.GetLocalReadChannel(inputChanName, tr.FlowContext.ChannelBufferSize)
		if err != nil {
			log.Panic(err)
		}
		typedInputChan := make(chan reflect.Value)
		netchan.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg)
		firstTask.InputChans = append(firstTask.InputChans, typedInputChan)
	}
}