Beispiel #1
0
func (m *Upsert) Run(ctx *expr.Context) error {
	defer ctx.Recover()
	defer close(m.msgOutCh)

	var err error
	var affectedCt int64
	switch {
	case m.insert != nil:
		//u.Debugf("Insert.Run():  %v   %#v", len(m.insert.Rows), m.insert)
		affectedCt, err = m.insertRows(ctx, m.insert.Rows)
	case m.upsert != nil && len(m.upsert.Rows) > 0:
		u.Debugf("Upsert.Run():  %v   %#v", len(m.upsert.Rows), m.upsert)
		affectedCt, err = m.insertRows(ctx, m.upsert.Rows)
	case m.update != nil:
		u.Debugf("Update.Run() %s", m.update.String())
		affectedCt, err = m.updateValues(ctx)
	default:
		u.Warnf("unknown mutation op?  %v", m)
	}
	if err != nil {
		return err
	}
	vals := make([]driver.Value, 2)
	vals[0] = int64(0) // status?
	vals[1] = affectedCt
	m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}
	return nil
}
Beispiel #2
0
func (m *Source) Run(context *expr.Context) error {
	defer context.Recover()
	defer close(m.msgOutCh)

	scanner, ok := m.source.(datasource.Scanner)
	if !ok {
		return fmt.Errorf("Does not implement Scanner: %T", m.source)
	}
	//u.Debugf("scanner: %T %v", scanner, scanner)
	iter := scanner.CreateIterator(nil)
	//u.Debugf("iter in source: %T  %#v", iter, iter)
	sigChan := m.SigChan()

	for item := iter.Next(); item != nil; item = iter.Next() {

		//u.Infof("In source Scanner iter %#v", item)
		select {
		case <-sigChan:
			return nil
		case m.msgOutCh <- item:
			// continue
		}

	}
	//u.Debugf("leaving source scanner")
	return nil
}
Beispiel #3
0
func (m *DeletionScanner) Run(context *expr.Context) error {
	defer context.Recover()
	defer close(m.msgOutCh)

	u.Infof("In Delete Scanner expr %#v", m.sql.Where)
	select {
	case <-m.SigChan():
		return nil
	default:
		if m.sql.Where != nil {
			// Hm, how do i evaluate here?  Do i need a special Vm?
			//return fmt.Errorf("Not implemented delete vm")
			deletedCt, err := m.db.DeleteExpression(m.sql.Where)
			if err != nil {
				u.Errorf("Could not put values: %v", err)
				return err
			}
			m.deleted = deletedCt
			vals := make([]driver.Value, 2)
			vals[0] = int64(0)
			vals[1] = int64(deletedCt)
			m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}
		}
	}
	return nil
}
Beispiel #4
0
func (m *TaskStepper) Run(ctx *expr.Context) error {
	defer ctx.Recover()     // Our context can recover panics, save error msg
	defer close(m.msgOutCh) // closing output channels is the signal to stop

	//u.Infof("runner: %T inchan", m)
	for {
		select {
		case <-m.sigCh:
			break
		}
	}
	//u.Warnf("end of Runner")
	return nil
}
Beispiel #5
0
// For ResultWriter, since we are are not paging through messages
//  using this mesage channel, instead using Next() as defined by sql/driver
//  we don't read the input channel, just watch stop channels
func (m *ResultWriter) Run(ctx *expr.Context) error {
	defer ctx.Recover() // Our context can recover panics, save error msg
	defer func() {
		close(m.msgOutCh) // closing output channels is the signal to stop
		//u.Warnf("close taskbase: %v", m.Type())
	}()
	//u.Debugf("start Run() for ResultWriter")
	select {
	case err := <-m.errCh:
		u.Errorf("got error:  %v", err)
		return err
	case <-m.sigCh:
		return nil
	}
	return nil
}
Beispiel #6
0
func (m *TaskBase) Run(ctx *expr.Context) error {
	defer ctx.Recover() // Our context can recover panics, save error msg
	defer func() {
		close(m.msgOutCh) // closing output channels is the signal to stop
		//u.Debugf("close taskbase: ch:%p    %v", m.msgOutCh, m.Type())
	}()

	//u.Debugf("TaskBase: %T inchan", m)
	if m.Handler == nil {
		u.Warnf("returning, no handler %T", m)
		return fmt.Errorf("Must have a handler to run base runner")
	}
	ok := true
	var err error
	var msg datasource.Message
msgLoop:
	for ok {

		// Either of the SigQuit, or error channel will
		//  cause breaking out of message channels below
		select {
		case err = <-m.errCh:
			//m.errors = append(m.errors, err)
			break msgLoop
		case <-m.sigCh: // Signal, ie quit etc
			u.Debugf("got taskbase sig")
			break msgLoop
		default:
		}

		//
		select {
		case msg, ok = <-m.msgInCh:
			if ok {
				//u.Debugf("sending to handler: %v %T  %+v", m.Type(), msg, msg)
				m.Handler(ctx, msg)
			} else {
				//u.Debugf("msg in closed shutting down: %s", m.TaskType)
				break msgLoop
			}
		case <-m.sigCh:
			break msgLoop
		}
	}

	return err
}
Beispiel #7
0
func (m *DeletionTask) Run(context *expr.Context) error {
	defer context.Recover()
	defer close(m.msgOutCh)
	u.Infof("In Delete Task expr:: %s", m.sql.Where)

	deletedCt, err := m.db.DeleteExpression(m.sql.Where)
	if err != nil {
		u.Errorf("Could not put values: %v", err)
		return err
	}
	m.deleted = deletedCt
	vals := make([]driver.Value, 2)
	vals[0] = int64(0)
	vals[1] = int64(deletedCt)
	m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}

	return nil
}
Beispiel #8
0
func (m *JoinKey) Run(context *expr.Context) error {
	defer context.Recover()
	defer close(m.msgOutCh)

	outCh := m.MessageOut()
	inCh := m.MessageIn()
	joinNodes := m.from.JoinNodes()

	for {

		select {
		case <-m.SigChan():
			//u.Debugf("got signal quit")
			return nil
		case msg, ok := <-inCh:
			if !ok {
				//u.Debugf("NICE, got msg shutdown")
				return nil
			} else {
				//u.Infof("In joinkey msg %#v", msg)
			msgTypeSwitch:
				switch mt := msg.(type) {
				case *datasource.SqlDriverMessageMap:
					vals := make([]string, len(joinNodes))
					for i, node := range joinNodes {
						joinVal, ok := vm.Eval(mt, node)
						//u.Debugf("evaluating: ok?%v T:%T result=%v node '%v'", ok, joinVal, joinVal.ToString(), node.String())
						if !ok {
							u.Errorf("could not evaluate: %T %#v   %v", joinVal, joinVal, msg)
							break msgTypeSwitch
						}
						vals[i] = joinVal.ToString()
					}
					key := strings.Join(vals, string(byte(0)))
					mt.SetKeyHashed(key)
					outCh <- mt
				default:
					return fmt.Errorf("To use JoinKey must use SqlDriverMessageMap but got %T", msg)
				}
			}
		}
	}
	return nil
}
Beispiel #9
0
func (m *TaskSequential) Run(ctx *expr.Context) error {
	defer ctx.Recover() // Our context can recover panics, save error msg
	defer func() {
		//close(m.msgOutCh) // closing output channels is the signal to stop
		//u.Debugf("close TaskSequential: %v", m.Type())
	}()

	// Either of the SigQuit, or error channel will
	//  cause breaking out of message channels below
	select {
	case err := <-m.errCh:
		u.Errorf("%v", err)
	case <-m.sigCh:
		u.Warnf("got quit channel?")
	default:
	}

	var wg sync.WaitGroup

	// start tasks in reverse order, so that by time
	// source starts up all downstreams have started
	for i := len(m.tasks) - 1; i >= 0; i-- {
		wg.Add(1)
		go func(taskId int) {
			task := m.tasks[taskId]
			//u.Infof("starting task %d-%d %T in:%p  out:%p", m.depth, taskId, task, task.MessageIn(), task.MessageOut())
			if err := task.Run(ctx); err != nil {
				u.Errorf("%T.Run() errored %v", task, err)
				// TODO:  what do we do with this error?   send to error channel?
			}
			//u.Warnf("exiting taskId: %v %T", taskId, m.tasks[taskId])
			wg.Done()
		}(i)
	}

	wg.Wait() // block until all tasks have finished
	//u.Debugf("exit TaskSequential Run()")
	return nil
}
Beispiel #10
0
func (m *TaskParallel) Run(ctx *expr.Context) error {
	defer ctx.Recover() // Our context can recover panics, save error msg
	defer func() {
		close(m.msgOutCh) // closing output channels is the signal to stop
		//u.Warnf("close TaskParallel: %v", m.Type())
	}()

	// Either of the SigQuit, or error channel will
	//  cause breaking out of message channels below
	select {
	case err := <-m.errCh:
		//m.errors = append(m.errors, err)
		u.Errorf("%v", err)
	case <-m.sigCh:

	default:
	}

	var wg sync.WaitGroup

	// start tasks in reverse order, so that by time
	// source starts up all downstreams have started
	for i := len(m.tasks) - 1; i >= 0; i-- {
		wg.Add(1)
		go func(taskId int) {
			if err := m.tasks[taskId].Run(ctx); err != nil {
				u.Errorf("%T.Run() errored %v", m.tasks[taskId], err)
				// TODO:  what do we do with this error?   send to error channel?
			}
			//u.Warnf("exiting taskId: %v %T", taskId, tasks[taskId])
			wg.Done()
		}(i)
	}

	wg.Wait()

	return nil
}
Beispiel #11
0
func (m *JoinMerge) Run(context *expr.Context) error {
	defer context.Recover()
	defer close(m.msgOutCh)

	outCh := m.MessageOut()

	leftIn := m.ltask.MessageOut()
	rightIn := m.rtask.MessageOut()

	//u.Infof("left? %s", m.leftStmt)
	// lhNodes := m.leftStmt.JoinNodes()
	// rhNodes := m.rightStmt.JoinNodes()

	// Build an index of source to destination column indexing
	for _, col := range m.leftStmt.Source.Columns {
		//u.Debugf("left col:  idx=%d  key=%q as=%q col=%v parentidx=%v", len(m.colIndex), col.Key(), col.As, col.String(), col.ParentIndex)
		m.colIndex[m.leftStmt.Alias+"."+col.Key()] = col.ParentIndex
		//u.Debugf("colIndex:  %15q : %d", m.leftStmt.Alias+"."+col.Key(), col.SourceIndex)
	}
	for _, col := range m.rightStmt.Source.Columns {
		//u.Debugf("right col:  idx=%d  key=%q as=%q col=%v", len(m.colIndex), col.Key(), col.As, col.String())
		m.colIndex[m.rightStmt.Alias+"."+col.Key()] = col.ParentIndex
		//u.Debugf("colIndex:  %15q : %d", m.rightStmt.Alias+"."+col.Key(), col.SourceIndex)
	}

	// lcols := m.leftStmt.Source.AliasedColumns()
	// rcols := m.rightStmt.Source.AliasedColumns()

	//u.Infof("lcols:  %#v for sql %s", lcols, m.leftStmt.Source.String())
	//u.Infof("rcols:  %#v for sql %v", rcols, m.rightStmt.Source.String())
	lh := make(map[string][]*datasource.SqlDriverMessageMap)
	rh := make(map[string][]*datasource.SqlDriverMessageMap)

	wg := new(sync.WaitGroup)
	wg.Add(1)
	var fatalErr error
	go func() {
		for {
			//u.Infof("In source Scanner msg %#v", msg)
			select {
			case <-m.SigChan():
				u.Warnf("got signal quit")
				return
			case msg, ok := <-leftIn:
				if !ok {
					//u.Warnf("NICE, got left shutdown")
					wg.Done()
					return
				} else {
					switch mt := msg.(type) {
					case *datasource.SqlDriverMessageMap:
						key := mt.Key()
						if key == "" {
							fatalErr = fmt.Errorf(`To use Join msgs must have keys but got "" for %+v`, mt.Row())
							close(m.TaskBase.sigCh)
							return
						}
						lh[key] = append(lh[key], mt)
					default:
						fatalErr = fmt.Errorf("To use Join must use SqlDriverMessageMap but got %T", msg)
						close(m.TaskBase.sigCh)
						return
					}
				}
			}

		}
	}()
	wg.Add(1)
	go func() {
		for {

			//u.Infof("In source Scanner iter %#v", item)
			select {
			case <-m.SigChan():
				u.Warnf("got quit signal join source 1")
				return
			case msg, ok := <-rightIn:
				if !ok {
					//u.Warnf("NICE, got right shutdown")
					wg.Done()
					return
				} else {
					switch mt := msg.(type) {
					case *datasource.SqlDriverMessageMap:
						key := mt.Key()
						if key == "" {
							fatalErr = fmt.Errorf(`To use Join msgs must have keys but got "" for %+v`, mt.Row())
							close(m.TaskBase.sigCh)
							return
						}
						rh[key] = append(rh[key], mt)
					default:
						fatalErr = fmt.Errorf("To use Join must use SqlDriverMessageMap but got %T", msg)
						close(m.TaskBase.sigCh)
						return
					}
				}
			}

		}
	}()
	wg.Wait()
	//u.Info("leaving source scanner")
	i := uint64(0)
	for keyLeft, valLeft := range lh {
		//u.Debugf("compare:  key:%v  left:%#v  right:%#v  rh: %#v", keyLeft, valLeft, rh[keyLeft], rh)
		if valRight, ok := rh[keyLeft]; ok {
			//u.Debugf("found match?\n\t%d left=%#v\n\t%d right=%#v", len(valLeft), valLeft, len(valRight), valRight)
			msgs := m.mergeValueMessages(valLeft, valRight)
			//u.Debugf("msgsct: %v   msgs:%#v", len(msgs), msgs)
			for _, msg := range msgs {
				//outCh <- datasource.NewUrlValuesMsg(i, msg)
				//u.Debugf("i:%d   msg:%#v", i, msg.Row())
				msg.IdVal = i
				i++
				outCh <- msg
			}
		}
	}
	return nil
}