func getresultobj(strsplits []string, tval interface{}, val toolkit.M) interface{} {
var xval interface{}
if val == nil {
val = toolkit.M{}
}
switch {
case len(strsplits) == 1:
xval = tval
case len(strsplits) > 1:
tm := toolkit.M{}
if val.Has(strsplits[1]) {
tm, _ = toolkit.ToM(val[strsplits[1]])
}
xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
}
return xval
// if {
// val = tval
// } else {
// val = val.Set(k, v)
// if val == nil {
// } else {
// }
// val = getresultobj(strsplits[1:], tval)
// }
// return val
}
func CheckValue(v toolkit.M, f *Filter) (bool, interface{}) {
resbool := false
filedtemp := []interface{}{}
resbool = v.Has(f.Field)
if resbool {
return resbool, v.Get(f.Field)
} else if strings.Contains(f.Field, ".") {
ar := strings.Split(f.Field, ".")
for i, dt := range ar {
if i == 0 {
resbool = v.Has(dt)
if !resbool {
break
} else {
filedtemp = append(filedtemp, v.Get(dt))
}
} else {
temp := toolkit.M(filedtemp[i-1].(map[string]interface{}))
resbool = temp.Has(dt)
if !resbool {
break
} else {
filedtemp = append(filedtemp, temp.Get(dt))
}
}
if i == len(ar)-1 {
return true, filedtemp[i]
}
}
}
return false, nil
}
func (q *Query) execQueryPartUpdate(dt toolkit.M, Cond QueryCondition) error {
if len(dt) == 0 {
return errorlib.Error(packageName, "Query", modQuery, "data to update is not found")
}
writer := q.Connection().(*Connection).writer
reader := q.Connection().(*Connection).reader
tempHeader := []string{}
for _, val := range q.Connection().(*Connection).headerColumn {
tempHeader = append(tempHeader, val.name)
}
for {
foundChange := false
recData := toolkit.M{}
dataTemp, e := reader.Read()
for i, val := range dataTemp {
recData.Set(tempHeader[i], val)
}
if len(Cond.Find) > 0 || (len(Cond.Find) == 0 && toolkit.IdField(dt) == "") {
foundChange = Cond.getCondition(recData)
}
// Check ID IF Condition Not Found
if nameid := toolkit.IdField(dt); nameid != "" && !foundChange {
if recData.Has(nameid) && dt[nameid] == recData[nameid] {
foundChange = true
}
}
if foundChange && len(dataTemp) > 0 {
for n, v := range tempHeader {
if dt.Has(v) {
dataTemp[n] = cast.ToString(dt[v])
}
}
}
if e == io.EOF {
if dataTemp != nil {
writer.Write(dataTemp)
writer.Flush()
}
break
} else if e != nil {
return errorlib.Error(packageName, modQuery, "Update", e.Error())
}
if dataTemp != nil {
writer.Write(dataTemp)
writer.Flush()
}
}
return nil
}
func populateParmValue(inputM *toolkit.M, parms toolkit.M) {
in := *inputM
for k, _ := range in {
if parms.Has(k) {
in[k] = parms[k]
} else {
in[k] = ""
}
}
*inputM = in
}
func (q *Query) execQueryPartUpdate(dt toolkit.M) error {
var e error
e = q.startWriteMode()
if e != nil {
return err.Error(packageName, modQuery, "Exec-Update: ", e.Error())
}
writer := q.writer
reader := q.reader
tempHeader := []string{}
for _, val := range q.headerColumn {
tempHeader = append(tempHeader, val.name)
}
var i int = 0
for {
i += 1
dataTemp, e := reader.Read()
if toolkit.HasMember(q.indexes, i) && len(dataTemp) > 0 {
for n, v := range tempHeader {
if dt.Has(v) {
dataTemp[n] = cast.ToString(dt[v])
}
}
}
if e == io.EOF {
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
break
} else if e != nil {
_ = q.endWriteMode()
return err.Error(packageName, modQuery, "Exec-Update:", e.Error())
}
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
}
q.execOpr = true
e = q.endWriteMode()
if e != nil {
return err.Error(packageName, modQuery, "Exec-Update: ", e.Error())
}
return nil
}
func (l *LoginController) ProcessLogin(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := toolkit.M{}
err := r.GetPayload(&payload)
switch {
case err != nil:
return helper.CreateResult(false, nil, err.Error())
case !payload.Has("username") || !payload.Has("password"):
return helper.CreateResult(false, nil, "username or password not found")
case payload.Has("username") && len(toolkit.ToString(payload["username"])) == 0:
return helper.CreateResult(false, nil, "username cannot empty")
case payload.Has("password") && len(toolkit.ToString(payload["password"])) == 0:
return helper.CreateResult(false, nil, "password cannot empty")
}
sessid, err := acl.Login(toolkit.ToString(payload["username"]), toolkit.ToString(payload["password"]))
if err != nil {
return helper.CreateResult(true, "", err.Error())
}
r.SetSession("sessionid", sessid)
return helper.CreateResult(true, toolkit.M{}.Set("status", true), "Login Success")
}
func (q *Query) compileUpdateFrom(queryString string, ins toolkit.M) string {
updateString := func() string {
if !ins.Has("data") {
return ""
}
var updates []string
for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
updates = append(updates, fmt.Sprintf("%s = %s", k, q.getAsString(v)))
}
return strings.Join(updates, ", ")
}()
queryString = fmt.Sprintf("%sSET %s ", queryString, updateString)
return queryString
}
func (a *Controller) KendoGridSettings(ins toolkit.M) toolkit.M {
if ins == nil {
ins = toolkit.M{}
}
s := toolkit.M{}
q_skip := a.Ctx.Input.Query("skip")
q_page := a.Ctx.Input.Query("page")
q_size := a.Ctx.Input.Query("pageSize")
if q_skip != "" {
s.Set("skip", toolkit.ToInt(q_skip))
}
if q_page != "" {
s.Set("page", toolkit.ToInt(q_page))
}
if q_size != "" {
s.Set("limit", toolkit.ToInt(q_size))
}
sortField := strings.ToLower(a.Ctx.Input.Query("sort[0][field]"))
sortDir := a.Ctx.Input.Query("sort[0][dir]")
if sortField != "" {
if sortField == "id" {
sortField = "_id"
}
if sortDir == "" || sortDir == "asc" {
s.Set("order", []string{sortField})
} else {
s.Set("order", []string{"-" + sortField})
}
}
if fqe := a.KendoGridFilter("filter"); fqe != nil {
if ins.Has("where") {
fqe = dbs.And(fqe, ins.Get("where").(*dbs.QE))
}
s.Set("where", fqe)
}
return s
}
func MatchM(v toolkit.M, filters []*Filter) bool {
var match bool
for _, f := range filters {
if f.Field != "" {
//--- if has field: $eq, $ne, $gt, $lt, $gte, $lte, $contains
//toolkit.Printf("Filter:%s V:%s Has:%s Match:%s", f.Field, toolkit.JsonString(v), v.Has("random"), match)
if v.Has(f.Field) {
match = MatchV(v.Get(f.Field), f)
//toolkit.Printf("Filter:%s Value: %v Match:%s \n", toolkit.JsonString(f), v.Get(f.Field), match)
if !match {
return false
}
} else {
if f.Op != FilterOpNoEqual && f.Op != FilterOpNin {
return false
}
}
} else {
//-- no field: $and, $or
//toolkit.Printf("Filter: %s\n", toolkit.JsonString(f))
if f.Op == FilterOpAnd || f.Op == FilterOpOr {
filters2 := f.Value.([]*Filter)
for k, f2 := range filters2 {
if f.Op == FilterOpAnd {
if k == 0 {
match = MatchM(v, []*Filter{f2})
} else {
match = match && MatchM(v, []*Filter{f2})
}
} else {
if k == 0 {
match = MatchM(v, []*Filter{f2})
} else {
match = match || MatchM(v, []*Filter{f2})
}
}
}
}
//toolkit.Printf("\n")
}
}
return match
}
func (q *Query) compileInsertFrom(queryString string, ins toolkit.M) string {
keyString, valString := func() (string, string) {
if !ins.Has("data") {
return "", ""
}
keys := []string{}
vals := []string{}
for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
keys = append(keys, k)
vals = append(vals, q.getAsString(v))
}
return strings.Join(keys, ", "), strings.Join(vals, ", ")
}()
queryString = fmt.Sprintf("%s(%s) VALUES (%s) ", queryString, keyString, valString)
return queryString
}
func (l *LoginController) SavePassword(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := toolkit.M{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if !payload.Has("newpassword") || !payload.Has("userid") {
return helper.CreateResult(false, nil, "Data is not complete")
}
switch {
case payload.Has("tokenid"):
err = acl.ChangePasswordToken(toolkit.ToString(payload["userid"]), toolkit.ToString(payload["newpassword"]), toolkit.ToString(payload["tokenid"]))
default:
// check sessionid first
savedsessionid := "" //change with get session
//=======================
userid, err := acl.FindUserBySessionID(savedsessionid)
if err == nil && userid == toolkit.ToString(payload["userid"]) {
err = acl.ChangePassword(toolkit.ToString(payload["userid"]), toolkit.ToString(payload["newpassword"]))
} else if err == nil {
err = errors.New("Userid is not match")
}
}
return helper.CreateResult(true, nil, "save password success")
}
func (d *DataContext) Get(m IModel, config tk.M) error {
var e error
q := d.Connection.NewQuery().SetConfig("pooling", d.Pooling()).From(m.(IModel).TableName())
if config.Has(ConfigWhere) {
q = q.Where(config.Get(ConfigWhere).(*dbox.Filter))
}
if config.Has(ConfigOrder) {
q = q.Order(config.Get(ConfigOrder).([]string)...)
}
q = q.Take(1)
//q := d.Connection.NewQuery().From(m.(IModel).TableName()).Where(dbox.Eq("_id", id))
c, e := q.Cursor(nil)
if e != nil {
return err.Error(packageName, modCtx, "Get", "Cursor fail. "+e.Error())
}
defer c.Close()
e = c.Fetch(m, 1, false)
if e != nil {
return err.Error(packageName, modCtx, "Get", e.Error())
}
return nil
}
func (a *Client) Call(methodName string, in toolkit.M) *toolkit.Result {
if a.client == nil {
return toolkit.NewResult().SetErrorTxt("Unable to call, no connection handshake")
}
if in == nil {
in = toolkit.M{}
}
out := toolkit.NewResult()
in["method"] = methodName
if in.GetString("auth_referenceid") == "" {
in["auth_referenceid"] = a.sessionID
}
//fmt.Println("SessionID: " + a.sessionID)
if in.Has("auth_secret") == false {
in.Set("auth_secret", a.secret)
}
e := a.client.Call("Rpc.Do", in, out)
//_ = "breakpoint"
if e != nil {
return out.SetErrorTxt(a.address + "." + methodName + " Fail: " + e.Error())
}
return out
}
func (q *Query) execQueryPartInsert(dt toolkit.M) error {
var e error
e = q.startWriteMode()
if e != nil {
return err.Error(packageName, modQuery, "Exec-Insert: ", e.Error())
}
if q.newheader {
q.setNewHeader(dt)
}
writer := q.writer
// reader := q.reader
dataTemp := []string{}
for _, v := range q.headerColumn {
if dt.Has(v.name) {
dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
} else {
dataTemp = append(dataTemp, "")
}
}
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
q.execOpr = true
e = q.endWriteMode()
if e != nil {
return err.Error(packageName, modQuery, "Exec-Insert: ", e.Error())
}
return nil
}
func (l *LoginController) ResetPassword(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := toolkit.M{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
fmt.Println(payload.Has("email"))
fmt.Println(payload.Has("baseurl"))
if !payload.Has("email") || !payload.Has("baseurl") {
return helper.CreateResult(false, nil, "Data is not complete")
}
uname, tokenid, err := acl.ResetPassword(toolkit.ToString(payload["email"]))
fmt.Printf("%v, %v, %v \n\n", uname, tokenid, err)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
linkstr := fmt.Sprintf("<a href='%v/web/confirmreset?1=%v&2=%v'>Click</a>", toolkit.ToString(payload["baseurl"]), uname, tokenid)
mailmsg := fmt.Sprintf("Hi, <br/><br/> We received a request to reset your password, <br/><br/>")
mailmsg = fmt.Sprintf("%vFollow the link below to set a new password : <br/><br/> %v <br/><br/>", mailmsg, linkstr)
mailmsg = fmt.Sprintf("%vIf you don't want to change your password, you can ignore this email <br/><br/> Thanks,</body></html>", mailmsg)
m := gomail.NewMessage()
m.SetHeader("From", "*****@*****.**")
m.SetHeader("To", toolkit.ToString(payload["email"]))
m.SetHeader("Subject", "[no-reply] Self password reset")
m.SetBody("text/html", mailmsg)
d := gomail.NewPlainDialer("smtp.office365.com", 587, "*****@*****.**", "******")
err = d.DialAndSend(m)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, nil, "reset password success")
}
func (a *AclController) Logout(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := toolkit.M{}
err := r.GetPayload(&payload)
sessionid := ""
switch {
case err != nil:
return helper.CreateResult(false, nil, err.Error())
case !payload.Has("username") && !payload.Has("sessionid"):
return helper.CreateResult(false, nil, "username or session not found")
case payload.Has("sessionid"):
sessionid = toolkit.ToString(payload["sessionid"])
case payload.Has("username"):
tUser := new(acl.User)
err = acl.FindUserByLoginID(tUser, toolkit.ToString(payload["username"]))
if err != nil {
return helper.CreateResult(false, nil, "fail to get userid")
}
tSession := new(acl.Session)
err = acl.FindActiveSessionByUser(tSession, tUser.ID)
if err != nil {
return helper.CreateResult(false, nil, "fail to get sessionid")
}
sessionid = tSession.ID
}
if sessionid == "" {
return helper.CreateResult(true, nil, "Active sessionid not found")
}
err = acl.Logout(sessionid)
if err != nil && (err.Error() == "Session id not found" || err.Error() == "Session id is expired") {
return helper.CreateResult(true, nil, "Active sessionid not found")
} else if err != nil {
return helper.CreateResult(true, nil, toolkit.Sprintf("Error found : %v", err.Error()))
}
return helper.CreateResult(true, nil, "Logout success")
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
// var e error
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// b, ok := val.([]byte)
// // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val))
// var out interface{}
// e = toolkit.Unjson(b, &out)
// // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e)
// if e != nil {
// ok = false
// }
// if ok {
// v = out
// toolkit.Println("error OK :: ", ok, " :v :", v)
// } else {
// toolkit.Println("error OK :: ", ok, " :b :", b)
// v = string(b)
// }
v = val
entry.Set(strings.ToLower(col), v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
// toolkit.Println("... ::: ", tableData)
maxIndex := toolkit.SliceLen(tableData)
var e2 error
if e2 != nil {
return e2
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e2 = errors.New("No more data to fetched!")
} else {
e2 = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e2
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if commandType == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
datam := toolkit.M{}
if in.Has("data") {
datam, e = toolkit.ToM(in["data"])
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
}
if commandType != dbox.QueryPartDelete && len(datam) == 0 {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working, need data to param input")
}
// fmt.Println("DEBUG-113")
e = q.openFile()
if e != nil {
return err.Error(packageName, modQuery, "Exec ", e.Error())
}
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if nameid := toolkit.IdField(datam); nameid != "" {
if commandType == dbox.QueryPartInsert || commandType == dbox.QueryPartSave {
where = make([]*dbox.Filter, 0, 0)
}
where = append(where, dbox.Eq(strings.ToLower(nameid), datam[nameid]))
}
q.indexes = make([]int, 0, 0)
if len(where) > 0 {
q.indexes, e = q.generateIndex(where)
if e != nil {
return err.Error(packageName, modQuery, "Exec: ", e.Error())
}
}
if q.newheader && commandType != dbox.QueryPartSave && commandType != dbox.QueryPartInsert {
return err.Error(packageName, modQuery, "Exec: ", "Only save and insert permited")
}
q.newFileWrite = false
q.execOpr = false
switch commandType {
case dbox.QueryPartSave:
if len(q.indexes) > 0 {
q.newFileWrite = true
e = q.execQueryPartUpdate(datam)
} else {
e = q.execQueryPartInsert(datam)
}
case dbox.QueryPartInsert:
if len(q.indexes) > 0 {
e = fmt.Errorf("Id found, cannot insert with same Id")
} else {
e = q.execQueryPartInsert(datam)
}
case dbox.QueryPartDelete:
q.newFileWrite = true
if len(q.indexes) > 0 {
e = q.execQueryPartDelete()
}
case dbox.QueryPartUpdate:
q.newFileWrite = true
if len(q.indexes) > 0 {
e = q.execQueryPartUpdate(datam)
}
}
if e != nil {
return err.Error(packageName, modQuery, "Exec: ", e.Error())
}
return nil
}
func (q *Query) Exec(parm toolkit.M) error {
var e error
q.save = false
// useHeader := q.Connection().Info().Settings.Get("useheader", false).(bool)
if parm == nil {
parm = toolkit.M{}
}
data := toolkit.M{}
if parm.Has("data") {
data, _ = toolkit.ToM(parm["data"])
}
parts := crowd.From(q.Parts()).Group(func(x interface{}) interface{} {
qp := x.(*dbox.QueryPart)
return qp.PartType
}, nil).Data
// fromParts, hasFrom := parts[dbox.QueryPartFrom]
// if !hasFrom {
// return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
// }
// tablename = fromParts.([]interface{})[0].(*dbox.QueryPart).Value.(string)
// var where interface{}
commandType := ""
// multi := false
_, hasDelete := parts[dbox.QueryPartDelete]
_, hasInsert := parts[dbox.QueryPartInsert]
_, hasUpdate := parts[dbox.QueryPartUpdate]
_, hasSave := parts[dbox.QueryPartSave]
if hasDelete {
commandType = dbox.QueryPartDelete
} else if hasInsert {
commandType = dbox.QueryPartInsert
} else if hasUpdate {
commandType = dbox.QueryPartUpdate
} else if hasSave {
commandType = dbox.QueryPartSave
q.save = true
}
var where interface{}
whereParts, hasWhere := parts[dbox.QueryPartWhere]
if hasWhere {
fb := q.Connection().Fb()
for _, p := range whereParts.([]interface{}) {
fs := p.(*dbox.QueryPart).Value.([]*dbox.Filter)
for _, f := range fs {
if len(parm) > 0 {
f = ReadVariable(f, parm)
}
fb.AddFilter(f)
}
}
where, e = fb.Build()
if e != nil {
return errorlib.Error(packageName, modQuery, "Cursor", e.Error())
}
}
//Check setNewHeader First
if q.Connection().(*Connection).setNewHeader && commandType != dbox.QueryPartInsert {
q.Connection().(*Connection).Close()
filename := q.Connection().(*Connection).Info().Host
os.Remove(filename)
return errorlib.Error(packageName, "Query", modQuery, "Only Insert Permited")
}
q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Append
if hasDelete || hasUpdate {
q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Create
}
q.Connection().(*Connection).ExecOpr = false
if !q.Connection().(*Connection).setNewHeader && (commandType != dbox.QueryPartSave || (commandType == dbox.QueryPartSave && q.Connection().(*Connection).writer == nil)) {
e = q.Connection().(*Connection).StartSessionWrite()
}
if e != nil {
return errorlib.Error(packageName, "Query", modQuery, e.Error())
}
var execCond QueryCondition
execCond.Find, _ = toolkit.ToM(where)
switch commandType {
case dbox.QueryPartSave:
q.updatessave = false
e = q.execQueryPartSave(data)
case dbox.QueryPartInsert:
e = q.execQueryPartInsert(data)
case dbox.QueryPartDelete:
e = q.execQueryPartDelete(execCond)
case dbox.QueryPartUpdate:
e = q.execQueryPartUpdate(data, execCond)
}
if e != nil {
return e
}
q.Connection().(*Connection).ExecOpr = true
if commandType != dbox.QueryPartSave || q.updatessave {
e = q.Connection().(*Connection).EndSessionWrite()
q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Append
}
if e != nil {
return errorlib.Error(packageName, "Query", modQuery, e.Error())
}
return nil
}
func checkconfig(cm toolkit.M) (err error) {
err = nil
if !cm.Has("_id") {
err = errors.New(fmt.Sprintf("_id is not found"))
return
}
if !cm.Has("sourcetype") {
err = errors.New(fmt.Sprintf("sourcetype is not found"))
return
}
if cm.Has("grabconf") {
_, err = toolkit.ToM(cm["grabconf"])
} else {
err = errors.New(fmt.Sprintf("grab config is not found"))
}
if err != nil {
return
}
if cm.Has("intervalconf") {
_, err = toolkit.ToM(cm["intervalconf"])
} else {
err = errors.New(fmt.Sprintf("interval configuration is not found"))
}
if err != nil {
return
}
if cm.Has("logconf") {
_, err = toolkit.ToM(cm["logconf"])
} else {
err = errors.New(fmt.Sprintf("log configuration is not found"))
}
if err != nil {
return
}
if cm.Has("datasettings") {
if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" {
err = errors.New(fmt.Sprintf("data settings must be []interface {}"))
}
} else {
err = errors.New(fmt.Sprintf("data settings is not found"))
}
if err != nil {
return
}
if cm.Has("histconf") {
_, err = toolkit.ToM(cm["histconf"])
} else {
err = errors.New(fmt.Sprintf("history configuration is not found"))
}
if err != nil {
return
}
return
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
// log.Printf("start parse:\n")
if !toolkit.IsPointer(m) {
// log.Printf("not pointer\n")
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
// log.Printf("pointer\n")
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
// log.Printf("outputType: %v\n", outputType)
if outputType == CSV {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
}
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for _, val := range header {
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
// log.Printf("v: %v\n", v)
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
// log.Printf("ivs: %v\n", ivs)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
/*log.Printf("data: %v\n", data)
log.Printf("iv: %v\n", iv)*/
splitted := strings.Split(data, "\t")
/*log.Printf("appendData: %v\n", appendData)
log.Printf("kind: %v\n", v.Kind())
log.Printf("test: %v", fmt.Sprintf("%v", v))
//log.Printf("v.Name: %T\n", v)
if fmt.Sprintf("%v", v) == "reflect.Value" {
log.Printf("else: %v\n", "reflect.Value")
for _, val := range header {
log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
log.Printf("appendData: %v\n", appendData)
} else */
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
// log.Printf("struct: %v\n", v.Kind())
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
// log.Printf("i: %v\n", i)
// log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name")))
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
// log.Printf("valthis: %v\n", valthis)
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
// log.Printf("else: %v\n", v.Kind())
for _, val := range header {
// log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
// log.Printf("iv result: %v\n", iv)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
// log.Printf("ivs result: %v\n", ivs)
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
// log.Printf("result: %v\n", m)
}
return nil
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_HttpHtml":
SourceType = SourceType_HttpHtml
case "SourceType_HttpJson":
SourceType = SourceType_HttpJson
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
grabconfig := sedotan.Config{}
if tconfgrab.Has("formvalues") {
tfromvalues := toolkit.M{}
tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
return
}
grabconfig.SetFormValues(tfromvalues)
}
if tconfgrab.Has("loginvalues") {
grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
}
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
return
}
grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))
grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))
grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))
Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")
wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tDataSetting := sedotan.DataSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))
// Fetch columnsettings
if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.GrabColumn{}
tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))
tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tDataSetting.Column(tindex, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tDataSetting.SetFilterCond(tfiltercond)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
wGrabber.DataSettings[tnameid] = &tDataSetting
}
err = nil
if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
return
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
if len(c.indexes) == 0 {
return nil
}
var lower, upper int
lower = c.currentIndex
if lower < c.skip {
lower = c.skip
}
upper = lower + n
if upper > c.limit && c.limit > 0 {
upper = c.limit
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
datatemp, ef := c.q.reader.Read()
c.realIndex += 1
if c.indexes[c.currentIndex] != c.realIndex {
continue
}
c.currentIndex += 1
for i, val := range datatemp {
if len(c.fields) == 0 || c.fields.Has("*") || c.fields.Has(c.q.headerColumn[i].name) {
appendData[c.q.headerColumn[i].name] = val
}
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if appendData.Has(v.Field(i).Name) {
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
}
}
}
}
isAppend := lower < c.currentIndex && upper >= c.currentIndex
if (ef == io.EOF || ef == nil) && isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if ef != nil && ef != io.EOF {
return errorlib.Error(packageName, modCursor, "Fetch", ef.Error())
} else if ef == io.EOF || (ivs.Len() >= n && n > 0) {
break
}
}
// if e != nil {
// return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
// }
if n == 1 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
//toolkit.Printf("Command type: %s\n", commandType)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if setting.GetString("commandtype") == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
var dataM toolkit.M
var dataMs []toolkit.M
hasData := in.Has("data")
dataIsSlice := false
data := in.Get("data")
if toolkit.IsSlice(data) {
dataIsSlice = true
e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
}
hasWhere := setting.Has("where")
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if hasWhere && len(where) == 0 {
inWhere := in.Get("where")
if inWhere == nil {
hasWhere = false
where = nil
} else {
if !toolkit.IsSlice(inWhere) {
where = append(where, inWhere.(*dbox.Filter))
} else {
where = inWhere.([]*dbox.Filter)
}
}
}
if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartDelete,
dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
hasWhere = true
//toolkit.Println("check where")
if toolkit.IsSlice(data) {
ids := []interface{}{}
idField := ""
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data send is a slice, but its element has no ID")
}
dataCount := toolkit.SliceLen(data)
for i := 0; i < dataCount; i++ {
dataI := toolkit.SliceItem(data, i)
if i == 0 {
idField = toolkit.IdField(dataI)
}
ids = append(ids, toolkit.Id(dataI))
}
where = []*dbox.Filter{dbox.In(idField, ids)}
} else {
idfield := "_id"
id := toolkit.Id(data)
if !toolkit.IsNilOrEmpty(id) {
where = []*dbox.Filter{dbox.Eq(idfield, id)}
} else {
where = nil
hasWhere = false
}
}
}
/*
toolkit.Printf("CommandType: %s HasData: %v HasWhere: %v Where: %s\n",
commandType, hasData, hasWhere, toolkit.JsonString(where))
*/
e = q.openFile(commandType)
//toolkit.Printf(commandType+" Open File, found record: %d\nData:%s\n", len(q.data), toolkit.JsonString(q.data))
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
var indexes []interface{}
if hasWhere && commandType != dbox.QueryPartInsert {
whereIndex := dbox.Find(q.data, where)
indexes = toolkit.ToInterfaceArray(&whereIndex)
//toolkit.Printf("Where Index: %s Index:%s\n", toolkit.JsonString(whereIndex), toolkit.JsonString(indexes))
}
if commandType == dbox.QueryPartInsert {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
if !dataIsSlice {
dataMs = []toolkit.M{dataM}
}
//-- validate
for _, datam := range dataMs {
idField, idValue := toolkit.IdInfo(datam)
toolkit.Serde(dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}), &indexes, "")
if len(indexes) > 0 {
return err.Error(packageName, modQuery, "Exec: "+commandType, toolkit.Sprintf("Data %v already exist", idValue))
}
}
//-- insert the data
q.data = append(q.data, dataMs...)
} else if commandType == dbox.QueryPartUpdate {
//-- valida
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataUpdate toolkit.M
var updateDataIndex int
// if it is a slice then we need to update each data passed on its slice
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
}
var idField string
//toolkit.Printf("Indexes: %s\n", toolkit.JsonString(indexes))
for i, v := range q.data {
// update only data that match given inde
if toolkit.HasMember(indexes, i) || !hasWhere {
if idField == "" {
idField = toolkit.IdField(v)
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "No ID")
}
}
// If dataslice is sent, iterate f
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail "+e.Error())
}
updateDataIndex++
}
dataOrigin := q.data[i]
toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"})
toolkit.Serde(dataOrigin, &v, "")
q.data[i] = v
}
}
} else if commandType == dbox.QueryPartDelete {
if hasWhere && len(where) > 0 {
indexes := dbox.Find(q.data, where)
if len(indexes) > 0 {
newdata := []toolkit.M{}
for index, v := range q.data {
partOfIndex := toolkit.HasMember(indexes, index)
if partOfIndex == false {
newdata = append(newdata, v)
}
//toolkit.Println("i:", indexes, ", index:", index, ", p.ofIndex: ", partOfIndex, ", data: ", toolkit.JsonString(newdata))
}
q.data = newdata
}
} else {
q.data = []toolkit.M{}
}
//toolkit.Printf("Data now: %s\n", toolkit.JsonString(q.data))
} else if commandType == dbox.QueryPartSave {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataMs []toolkit.M
var dataM toolkit.M
if !toolkit.IsSlice(data) {
e = toolkit.Serde(&data, &dataM, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
dataMs = append(dataMs, dataM)
} else {
e = toolkit.Serde(&data, &dataMs, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
}
//toolkit.Printf("Saving: %s\n", toolkit.JsonString(dataMs))
for _, v := range dataMs {
idField, idValue := toolkit.IdInfo(v)
indexes := dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)})
if len(indexes) == 0 {
q.data = append(q.data, v)
} else {
dataOrigin := q.data[indexes[0]]
//toolkit.Printf("Copy data %s to %s\n", toolkit.JsonString(v), toolkit.JsonString(dataOrigin))
toolkit.CopyM(&v, &dataOrigin, false, []string{idField})
q.data[indexes[0]] = dataOrigin
}
}
}
e = q.writeFile()
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Write fail", e.Error())
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
fmt.Println(c.QueryString)
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
tableData := []toolkit.M{}
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
// valueint := values
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
// rows.Scan(valuePtrs...)
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// fmt.Println("Nilai val : ", val)
b, ok := val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
entry.Set(strings.ToLower(col), v)
// entry.Set(col, values[i])
// e = toolkit.DecodeByte(val.([]byte), v)
// toolkit.FromBytes(toolkit.ToBytes(val, ""), "", v)
// entry.Set(col, v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
fmt.Println("isi entry : ", entry[namaField], dataType)
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
fmt.Println("Nilai table data : ", tableData)
if e != nil {
return e
}
if n == 0 {
// *m.(*[]map[string]interface{}) = tableData
// toolkit.Unjson(toolkit.Jsonify(tableData), m)
e = toolkit.Serde(tableData, m, "json")
fmt.Println("Nilai Model : ", m)
} else {
end := c.start + n
if end > len(tableData) {
e = errors.New("index out of range")
} else {
// *m.(*[]map[string]interface{}) = tableData[0:n]
//toolkit.Unjson(toolkit.Jsonify(tableData[0:n]), m)
e = toolkit.Serde(tableData[0:n], m, "json")
}
}
return e
}
func (q *Query) execQueryPartSave(dt toolkit.M) error {
if len(dt) == 0 {
return errorlib.Error(packageName, modQuery, "save", "data to insert is not found")
}
writer := q.Connection().(*Connection).writer
reader := q.Connection().(*Connection).reader
tempHeader := []string{}
for _, val := range q.Connection().(*Connection).headerColumn {
tempHeader = append(tempHeader, val.name)
}
// Check ID Before Insert
checkidfound := false
if nameid := toolkit.IdField(dt); nameid != "" {
q.updatessave = true
var colsid int
for i, val := range q.Connection().(*Connection).headerColumn {
if val.name == nameid {
colsid = i
}
}
for {
dataTempSearch, e := reader.Read()
for i, val := range dataTempSearch {
if i == colsid && val == dt[nameid] {
checkidfound = true
break
}
}
if e == io.EOF {
break
} else if e != nil {
return errorlib.Error(packageName, modQuery, "Save", e.Error())
}
}
}
if checkidfound {
e := q.Connection().(*Connection).EndSessionWrite()
if e != nil {
return errorlib.Error(packageName, modQuery, "Save", e.Error())
}
q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Create
e = q.Connection().(*Connection).StartSessionWrite()
if e != nil {
return errorlib.Error(packageName, modQuery, "Save", e.Error())
}
e = q.execQueryPartUpdate(dt, QueryCondition{})
if e != nil {
return errorlib.Error(packageName, modQuery, "Save", e.Error())
}
// time.Sleep(1000 * time.Millisecond)
} else {
//Change to Do Insert
dataTemp := []string{}
for _, v := range q.Connection().(*Connection).headerColumn {
if dt.Has(v.name) {
dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
} else {
dataTemp = append(dataTemp, "")
}
}
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
}
return nil
}
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
dsOrigin := new(colonycore.DataSource)
err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dsDestination := new(colonycore.DataSource)
err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dataDS, _, conn, query, metaSave, err := new(DataSourceController).
ConnectToDataSource(dataGrabber.DataSourceOrigin)
if len(dataDS.QueryInfo) == 0 {
message := "Data source origin has invalid query"
logConf.AddLog(message, "ERROR")
return false, nil, message
}
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer conn.Close()
if metaSave.keyword != "" {
message := `Data source origin query is not "Select"`
logConf.AddLog(message, "ERROR")
return false, nil, message
}
cursor, err := query.Cursor(nil)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer cursor.Close()
data := []toolkit.M{}
err = cursor.Fetch(&data, 0, false)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
connDesc := new(colonycore.Connection)
err = colonycore.Get(connDesc, dsDestination.ConnectionID)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
const FLAG_ARG_DATA string = `%1`
transformedData := []toolkit.M{}
for _, each := range data {
eachTransformedData := toolkit.M{}
for _, eachMap := range dataGrabber.Maps {
var valueEachSourceField interface{}
// ============================================ SOURCE
if !strings.Contains(eachMap.Source, "|") {
// source could be: field, object, array
valueEachSourceField = each.Get(eachMap.Source)
} else {
// source could be: field of object, field of array-objects
prev := strings.Split(eachMap.Source, "|")[0]
next := strings.Split(eachMap.Source, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsOrigin.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
// source is field of array-objects
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil {
valueObjects = make([]interface{}, len(temp))
for i, each := range temp {
if tempSub, _ := toolkit.ToM(each); tempSub != nil {
valueObjects[i] = tempSub.Get(next)
}
}
}
valueEachSourceField = valueObjects
} else {
// source is field of object
valueObject := toolkit.M{}
if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil {
valueEachSourceField = valueObject.Get(next)
}
}
}
}
// ============================================ DESTINATION
if !strings.Contains(eachMap.Destination, "|") {
if eachMap.SourceType == "object" {
sourceObject, _ := toolkit.ToM(valueEachSourceField)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
eachTransformedData.Set(eachMap.Destination, valueObject)
} else if eachMap.SourceType == "array-objects" {
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
valueObjects = append(valueObjects, valueObject)
}
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
if strings.Contains(eachMap.DestinationType, "array") {
valueObjects := each.Get(eachMap.Source)
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
// eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each))
eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType))
}
}
} else {
prev := strings.Split(eachMap.Destination, "|")[0]
next := strings.Split(eachMap.Destination, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsDestination.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp := eachTransformedData.Get(prev, nil); temp == nil {
valueObjects = []interface{}{}
} else {
valueObjects, _ = temp.([]interface{})
if valueObjects == nil {
valueObjects = []interface{}{}
}
}
if temp, _ := valueEachSourceField.([]interface{}); temp != nil {
for i, eachVal := range temp {
valueObject := toolkit.M{}
if len(valueObjects) > i {
if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil {
valueObject = temp2
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
}
valueObjects[i] = valueObject
} else {
if fieldInfoDes.Sub != nil {
for _, subMeta := range fieldInfoDes.Sub {
valueObject.Set(subMeta.ID, nil)
}
}
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
valueObjects = append(valueObjects, valueObject)
}
}
}
eachTransformedData.Set(prev, valueObjects)
} else {
valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev))
if valueObject == nil {
valueObject = toolkit.M{}
}
//tambahan
prevSource := strings.Split(eachMap.Source, "|")[0]
nextSource := strings.Split(eachMap.Source, "|")[1]
mval, _ := toolkit.ToM(each.Get(prevSource, nil))
//=========
valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType))
// valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval))
eachTransformedData.Set(prev, valueObject)
}
}
}
}
transformedData = append(transformedData, eachTransformedData)
dataToSave := eachTransformedData
// ================ pre transfer command
if dataGrabber.PreTransferCommand != "" {
// jsonTranformedDataBytes, err := json.Marshal(each)
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var preCommand = dataGrabber.PreTransferCommand
if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) {
preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1))
}
dataToSave = toolkit.M{}
output, err := toolkit.RunCommand(preCommand, jsonTranformedData)
fmt.Printf("===> Pre Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", preCommand, jsonTranformedData, output)
if err == nil {
postData := toolkit.M{}
if err := json.Unmarshal([]byte(output), &postData); err == nil {
dataToSave = postData
}
}
}
// ================
if len(dataToSave) == 0 {
continue
}
nilFieldDest := eachTransformedData
for _, metadataDest := range dsDestination.MetaData {
if temp := eachTransformedData.Get(metadataDest.ID); temp == nil {
if metadataDest.ID != "_id" {
if metadataDest.Type == "object" {
valueObject := toolkit.M{}
for _, eachMetaSub := range metadataDest.Sub {
valueObject.Set(eachMetaSub.ID, nil)
}
nilFieldDest.Set(metadataDest.ID, valueObject)
} else if metadataDest.Type == "array-objects" {
valueEachSourceField := each.Get(metadataDest.ID)
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for keyss, _ := range sourceObject {
valueObject.Set(keyss, nil)
}
valueObjects = append(valueObjects, valueObject)
}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
if strings.Contains(metadataDest.Type, "array") {
valueObjects := []interface{}{}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
nilFieldDest.Set(metadataDest.ID, nil)
}
}
}
}
}
tableName := dsDestination.QueryInfo.GetString("from")
queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
if dataGrabber.InsertMode == "fresh" {
queryWrapper.Delete(tableName, dbox.Or())
}
if eachTransformedData.Has("_id") {
err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id")))
}
if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") {
queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings)
} else {
queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
}
if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" {
nilFieldDest.Set("_id", helper.RandomIDWithPrefix(""))
}
err = queryWrapper.Save(tableName, nilFieldDest)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
// ================ post transfer command
if dataGrabber.PostTransferCommand != "" {
eachTransformedData = dataToSave
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var postCommand = dataGrabber.PostTransferCommand
if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) {
postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1))
}
output, err := toolkit.RunCommand(postCommand, jsonTranformedData)
fmt.Printf("===> Post Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", postCommand, jsonTranformedData, output)
}
}
message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
return true, transformedData, ""
}
func (q *Query) execQueryPartInsert(dt toolkit.M) error {
if len(dt) == 0 {
return errorlib.Error(packageName, "Query", modQuery, "data to insert is not found")
}
writer := q.Connection().(*Connection).writer
reader := q.Connection().(*Connection).reader
dataTemp := []string{}
if q.Connection().(*Connection).setNewHeader {
q.Connection().(*Connection).SetHeaderToolkitM(dt)
q.Connection().(*Connection).setNewHeader = false
for _, v := range q.Connection().(*Connection).headerColumn {
dataTemp = append(dataTemp, v.name)
}
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
dataTemp = []string{}
}
// Check ID Before Insert
if nameid := toolkit.IdField(dt); nameid != "" {
var colsid int
for i, val := range q.Connection().(*Connection).headerColumn {
if val.name == nameid {
colsid = i
}
}
for {
dataTempSearch, e := reader.Read()
for i, val := range dataTempSearch {
if i == colsid && val == dt[nameid] {
return errorlib.Error(packageName, modQuery, "Insert", "Unique id is found")
}
}
if e == io.EOF {
break
} else if e != nil {
return errorlib.Error(packageName, modQuery, "Insert", e.Error())
}
}
}
for _, v := range q.Connection().(*Connection).headerColumn {
if dt.Has(v.name) {
dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
} else {
dataTemp = append(dataTemp, "")
}
}
if len(dataTemp) > 0 {
writer.Write(dataTemp)
writer.Flush()
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
e := c.prepIter()
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
lineCount := 0
//=============================
// fmt.Println("Qursor 133 : ", c.ConditionVal.Find)
for {
iv := reflect.New(v).Interface()
isAppend := true
c.count += 1
recData := toolkit.M{}
appendData := toolkit.M{}
dataTemp, e := c.reader.Read()
for i, val := range dataTemp {
orgname := c.headerColumn[i].name
lowername := strings.ToLower(c.headerColumn[i].name)
switch c.headerColumn[i].dataType {
case "int":
recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
case "float":
recData[lowername] = cast.ToF64(val, 2, cast.RoundingAuto)
default:
recData[lowername] = val
}
if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
appendData[orgname] = recData[lowername]
} else {
if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
appendData[orgname] = recData[lowername]
}
}
}
isAppend = c.ConditionVal.getCondition(recData)
if c.count < c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
isAppend = false
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if appendData.Has(v.Field(i).Name) {
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
}
}
}
}
if e == io.EOF {
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
break
} else if e != nil {
return errorlib.Error(packageName, modCursor,
"Fetch", e.Error())
}
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
if n > 0 {
if lineCount >= n {
break
}
}
}
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if n == 1 {
if ivs.Len() > 0 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
var hasLookup bool
toolkit.Println("payload : ", payload)
if payload.Has("haslookup") {
hasLookup = payload.Get("haslookup").(bool)
}
_id := toolkit.ToString(payload.Get("browserid", ""))
sort := payload.Get("sort")
search := payload.Get("search")
_ = search
take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)
TblName := toolkit.M{}
payload.Unset("browserid")
//sorter = ""
if sort != nil {
tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
fmt.Printf("====== sort %#v\n", tmsort["dir"])
if tmsort["dir"] == "asc" {
sorter = tmsort["field"].(string)
} else if tmsort["dir"] == "desc" {
sorter = "-" + tmsort["field"].(string)
} else if tmsort["dir"] == nil {
sorter = " "
}
} else {
sorter = " "
}
dataDS := new(colonycore.DataBrowser)
err := colonycore.Get(dataDS, _id)
if err != nil {
return 0, nil, nil, err
}
dataConn := new(colonycore.Connection)
err = colonycore.Get(dataConn, dataDS.ConnectionID)
if err != nil {
return 0, nil, nil, err
}
if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
return 0, nil, nil, err
}
connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
if err != nil {
return 0, nil, nil, err
}
if dataDS.QueryType == "" {
TblName.Set("from", dataDS.TableNames)
payload.Set("from", dataDS.TableNames)
} else if dataDS.QueryType == "Dbox" {
getTableName := toolkit.M{}
toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
payload.Set("from", getTableName.Get("from").(string))
if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
payload.Set("select", getTableName.Get("select").(string))
}
} else if dataDS.QueryType == "SQL" {
var QueryString string
if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
} else if dataConn.Driver == "mssql" {
QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
toolkit.ToString(take) + " ROWS ONLY "
} else if dataConn.Driver == "postgres" {
QueryString = " LIMIT " + toolkit.ToString(take) +
" OFFSET " + toolkit.ToString(skip)
}
stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
payload.Set("freetext", stringQuery)
// toolkit.Println(stringQuery)
}
qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)
var selectfield string
for _, metadata := range dataDS.MetaData {
tField := metadata.Field
if payload.Has(tField) {
selectfield = toolkit.ToString(tField)
if toolkit.IsSlice(payload[tField]) {
query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
} else if !toolkit.IsNilOrEmpty(payload[tField]) {
var hasPattern bool
for _, val := range querypattern {
if strings.Contains(toolkit.ToString(payload[tField]), val) {
hasPattern = true
}
}
if hasPattern {
query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
} else {
switch toolkit.ToString(metadata.DataType) {
case "int":
query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
case "float32":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
case "float64":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
default:
query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
}
}
}
}
}
if hasLookup && selectfield != "" {
if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
query = query.Select(selectfield)
qcount = qcount.Select(selectfield)
} else {
query = query.Select(selectfield).Group(selectfield)
qcount = qcount.Select(selectfield).Group(selectfield)
}
}
ccount, err := qcount.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer ccount.Close()
dcount := ccount.Count()
cursor, err := query.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer cursor.Close()
data := []toolkit.M{}
cursor.Fetch(&data, 0, false)
if err != nil {
return 0, nil, nil, err
}
if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
dataMongo := []toolkit.M{}
for _, val := range data {
mVal, _ := toolkit.ToM(val.Get("_id"))
dataMongo = append(dataMongo, mVal)
}
data = dataMongo
} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
/*distinct value for flat file*/
dataFlat := []toolkit.M{}
var existingVal = []string{""}
for _, val := range data {
valString := toolkit.ToString(val.Get(selectfield))
if !toolkit.HasMember(existingVal, valString) {
dataFlat = append(dataFlat, val)
existingVal = append(existingVal, valString)
}
}
data = dataFlat
}
return dcount, data, dataDS, nil
}