func (m *Upsert) Run(ctx *expr.Context) error {
defer ctx.Recover()
defer close(m.msgOutCh)
var err error
var affectedCt int64
switch {
case m.insert != nil:
//u.Debugf("Insert.Run(): %v %#v", len(m.insert.Rows), m.insert)
affectedCt, err = m.insertRows(ctx, m.insert.Rows)
case m.upsert != nil && len(m.upsert.Rows) > 0:
u.Debugf("Upsert.Run(): %v %#v", len(m.upsert.Rows), m.upsert)
affectedCt, err = m.insertRows(ctx, m.upsert.Rows)
case m.update != nil:
u.Debugf("Update.Run() %s", m.update.String())
affectedCt, err = m.updateValues(ctx)
default:
u.Warnf("unknown mutation op? %v", m)
}
if err != nil {
return err
}
vals := make([]driver.Value, 2)
vals[0] = int64(0) // status?
vals[1] = affectedCt
m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}
return nil
}
func (m *Source) Run(context *expr.Context) error {
defer context.Recover()
defer close(m.msgOutCh)
scanner, ok := m.source.(datasource.Scanner)
if !ok {
return fmt.Errorf("Does not implement Scanner: %T", m.source)
}
//u.Debugf("scanner: %T %v", scanner, scanner)
iter := scanner.CreateIterator(nil)
//u.Debugf("iter in source: %T %#v", iter, iter)
sigChan := m.SigChan()
for item := iter.Next(); item != nil; item = iter.Next() {
//u.Infof("In source Scanner iter %#v", item)
select {
case <-sigChan:
return nil
case m.msgOutCh <- item:
// continue
}
}
//u.Debugf("leaving source scanner")
return nil
}
func (m *DeletionScanner) Run(context *expr.Context) error {
defer context.Recover()
defer close(m.msgOutCh)
u.Infof("In Delete Scanner expr %#v", m.sql.Where)
select {
case <-m.SigChan():
return nil
default:
if m.sql.Where != nil {
// Hm, how do i evaluate here? Do i need a special Vm?
//return fmt.Errorf("Not implemented delete vm")
deletedCt, err := m.db.DeleteExpression(m.sql.Where)
if err != nil {
u.Errorf("Could not put values: %v", err)
return err
}
m.deleted = deletedCt
vals := make([]driver.Value, 2)
vals[0] = int64(0)
vals[1] = int64(deletedCt)
m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}
}
}
return nil
}
func (m *TaskStepper) Run(ctx *expr.Context) error {
defer ctx.Recover() // Our context can recover panics, save error msg
defer close(m.msgOutCh) // closing output channels is the signal to stop
//u.Infof("runner: %T inchan", m)
for {
select {
case <-m.sigCh:
break
}
}
//u.Warnf("end of Runner")
return nil
}
// For ResultWriter, since we are are not paging through messages
// using this mesage channel, instead using Next() as defined by sql/driver
// we don't read the input channel, just watch stop channels
func (m *ResultWriter) Run(ctx *expr.Context) error {
defer ctx.Recover() // Our context can recover panics, save error msg
defer func() {
close(m.msgOutCh) // closing output channels is the signal to stop
//u.Warnf("close taskbase: %v", m.Type())
}()
//u.Debugf("start Run() for ResultWriter")
select {
case err := <-m.errCh:
u.Errorf("got error: %v", err)
return err
case <-m.sigCh:
return nil
}
return nil
}
func (m *TaskBase) Run(ctx *expr.Context) error {
defer ctx.Recover() // Our context can recover panics, save error msg
defer func() {
close(m.msgOutCh) // closing output channels is the signal to stop
//u.Debugf("close taskbase: ch:%p %v", m.msgOutCh, m.Type())
}()
//u.Debugf("TaskBase: %T inchan", m)
if m.Handler == nil {
u.Warnf("returning, no handler %T", m)
return fmt.Errorf("Must have a handler to run base runner")
}
ok := true
var err error
var msg datasource.Message
msgLoop:
for ok {
// Either of the SigQuit, or error channel will
// cause breaking out of message channels below
select {
case err = <-m.errCh:
//m.errors = append(m.errors, err)
break msgLoop
case <-m.sigCh: // Signal, ie quit etc
u.Debugf("got taskbase sig")
break msgLoop
default:
}
//
select {
case msg, ok = <-m.msgInCh:
if ok {
//u.Debugf("sending to handler: %v %T %+v", m.Type(), msg, msg)
m.Handler(ctx, msg)
} else {
//u.Debugf("msg in closed shutting down: %s", m.TaskType)
break msgLoop
}
case <-m.sigCh:
break msgLoop
}
}
return err
}
func (m *DeletionTask) Run(context *expr.Context) error {
defer context.Recover()
defer close(m.msgOutCh)
u.Infof("In Delete Task expr:: %s", m.sql.Where)
deletedCt, err := m.db.DeleteExpression(m.sql.Where)
if err != nil {
u.Errorf("Could not put values: %v", err)
return err
}
m.deleted = deletedCt
vals := make([]driver.Value, 2)
vals[0] = int64(0)
vals[1] = int64(deletedCt)
m.msgOutCh <- &datasource.SqlDriverMessage{vals, 1}
return nil
}
func (m *JoinKey) Run(context *expr.Context) error {
defer context.Recover()
defer close(m.msgOutCh)
outCh := m.MessageOut()
inCh := m.MessageIn()
joinNodes := m.from.JoinNodes()
for {
select {
case <-m.SigChan():
//u.Debugf("got signal quit")
return nil
case msg, ok := <-inCh:
if !ok {
//u.Debugf("NICE, got msg shutdown")
return nil
} else {
//u.Infof("In joinkey msg %#v", msg)
msgTypeSwitch:
switch mt := msg.(type) {
case *datasource.SqlDriverMessageMap:
vals := make([]string, len(joinNodes))
for i, node := range joinNodes {
joinVal, ok := vm.Eval(mt, node)
//u.Debugf("evaluating: ok?%v T:%T result=%v node '%v'", ok, joinVal, joinVal.ToString(), node.String())
if !ok {
u.Errorf("could not evaluate: %T %#v %v", joinVal, joinVal, msg)
break msgTypeSwitch
}
vals[i] = joinVal.ToString()
}
key := strings.Join(vals, string(byte(0)))
mt.SetKeyHashed(key)
outCh <- mt
default:
return fmt.Errorf("To use JoinKey must use SqlDriverMessageMap but got %T", msg)
}
}
}
}
return nil
}
func (m *TaskSequential) Run(ctx *expr.Context) error {
defer ctx.Recover() // Our context can recover panics, save error msg
defer func() {
//close(m.msgOutCh) // closing output channels is the signal to stop
//u.Debugf("close TaskSequential: %v", m.Type())
}()
// Either of the SigQuit, or error channel will
// cause breaking out of message channels below
select {
case err := <-m.errCh:
u.Errorf("%v", err)
case <-m.sigCh:
u.Warnf("got quit channel?")
default:
}
var wg sync.WaitGroup
// start tasks in reverse order, so that by time
// source starts up all downstreams have started
for i := len(m.tasks) - 1; i >= 0; i-- {
wg.Add(1)
go func(taskId int) {
task := m.tasks[taskId]
//u.Infof("starting task %d-%d %T in:%p out:%p", m.depth, taskId, task, task.MessageIn(), task.MessageOut())
if err := task.Run(ctx); err != nil {
u.Errorf("%T.Run() errored %v", task, err)
// TODO: what do we do with this error? send to error channel?
}
//u.Warnf("exiting taskId: %v %T", taskId, m.tasks[taskId])
wg.Done()
}(i)
}
wg.Wait() // block until all tasks have finished
//u.Debugf("exit TaskSequential Run()")
return nil
}
func (m *TaskParallel) Run(ctx *expr.Context) error {
defer ctx.Recover() // Our context can recover panics, save error msg
defer func() {
close(m.msgOutCh) // closing output channels is the signal to stop
//u.Warnf("close TaskParallel: %v", m.Type())
}()
// Either of the SigQuit, or error channel will
// cause breaking out of message channels below
select {
case err := <-m.errCh:
//m.errors = append(m.errors, err)
u.Errorf("%v", err)
case <-m.sigCh:
default:
}
var wg sync.WaitGroup
// start tasks in reverse order, so that by time
// source starts up all downstreams have started
for i := len(m.tasks) - 1; i >= 0; i-- {
wg.Add(1)
go func(taskId int) {
if err := m.tasks[taskId].Run(ctx); err != nil {
u.Errorf("%T.Run() errored %v", m.tasks[taskId], err)
// TODO: what do we do with this error? send to error channel?
}
//u.Warnf("exiting taskId: %v %T", taskId, tasks[taskId])
wg.Done()
}(i)
}
wg.Wait()
return nil
}
func (m *JoinMerge) Run(context *expr.Context) error {
defer context.Recover()
defer close(m.msgOutCh)
outCh := m.MessageOut()
leftIn := m.ltask.MessageOut()
rightIn := m.rtask.MessageOut()
//u.Infof("left? %s", m.leftStmt)
// lhNodes := m.leftStmt.JoinNodes()
// rhNodes := m.rightStmt.JoinNodes()
// Build an index of source to destination column indexing
for _, col := range m.leftStmt.Source.Columns {
//u.Debugf("left col: idx=%d key=%q as=%q col=%v parentidx=%v", len(m.colIndex), col.Key(), col.As, col.String(), col.ParentIndex)
m.colIndex[m.leftStmt.Alias+"."+col.Key()] = col.ParentIndex
//u.Debugf("colIndex: %15q : %d", m.leftStmt.Alias+"."+col.Key(), col.SourceIndex)
}
for _, col := range m.rightStmt.Source.Columns {
//u.Debugf("right col: idx=%d key=%q as=%q col=%v", len(m.colIndex), col.Key(), col.As, col.String())
m.colIndex[m.rightStmt.Alias+"."+col.Key()] = col.ParentIndex
//u.Debugf("colIndex: %15q : %d", m.rightStmt.Alias+"."+col.Key(), col.SourceIndex)
}
// lcols := m.leftStmt.Source.AliasedColumns()
// rcols := m.rightStmt.Source.AliasedColumns()
//u.Infof("lcols: %#v for sql %s", lcols, m.leftStmt.Source.String())
//u.Infof("rcols: %#v for sql %v", rcols, m.rightStmt.Source.String())
lh := make(map[string][]*datasource.SqlDriverMessageMap)
rh := make(map[string][]*datasource.SqlDriverMessageMap)
wg := new(sync.WaitGroup)
wg.Add(1)
var fatalErr error
go func() {
for {
//u.Infof("In source Scanner msg %#v", msg)
select {
case <-m.SigChan():
u.Warnf("got signal quit")
return
case msg, ok := <-leftIn:
if !ok {
//u.Warnf("NICE, got left shutdown")
wg.Done()
return
} else {
switch mt := msg.(type) {
case *datasource.SqlDriverMessageMap:
key := mt.Key()
if key == "" {
fatalErr = fmt.Errorf(`To use Join msgs must have keys but got "" for %+v`, mt.Row())
close(m.TaskBase.sigCh)
return
}
lh[key] = append(lh[key], mt)
default:
fatalErr = fmt.Errorf("To use Join must use SqlDriverMessageMap but got %T", msg)
close(m.TaskBase.sigCh)
return
}
}
}
}
}()
wg.Add(1)
go func() {
for {
//u.Infof("In source Scanner iter %#v", item)
select {
case <-m.SigChan():
u.Warnf("got quit signal join source 1")
return
case msg, ok := <-rightIn:
if !ok {
//u.Warnf("NICE, got right shutdown")
wg.Done()
return
} else {
switch mt := msg.(type) {
case *datasource.SqlDriverMessageMap:
key := mt.Key()
if key == "" {
fatalErr = fmt.Errorf(`To use Join msgs must have keys but got "" for %+v`, mt.Row())
close(m.TaskBase.sigCh)
return
}
rh[key] = append(rh[key], mt)
default:
fatalErr = fmt.Errorf("To use Join must use SqlDriverMessageMap but got %T", msg)
close(m.TaskBase.sigCh)
return
}
}
}
}
}()
wg.Wait()
//u.Info("leaving source scanner")
i := uint64(0)
for keyLeft, valLeft := range lh {
//u.Debugf("compare: key:%v left:%#v right:%#v rh: %#v", keyLeft, valLeft, rh[keyLeft], rh)
if valRight, ok := rh[keyLeft]; ok {
//u.Debugf("found match?\n\t%d left=%#v\n\t%d right=%#v", len(valLeft), valLeft, len(valRight), valRight)
msgs := m.mergeValueMessages(valLeft, valRight)
//u.Debugf("msgsct: %v msgs:%#v", len(msgs), msgs)
for _, msg := range msgs {
//outCh <- datasource.NewUrlValuesMsg(i, msg)
//u.Debugf("i:%d msg:%#v", i, msg.Row())
msg.IdVal = i
i++
outCh <- msg
}
}
}
return nil
}