Example #1
0
func getresultobj(strsplits []string, tval interface{}, val toolkit.M) interface{} {
	var xval interface{}
	if val == nil {
		val = toolkit.M{}
	}

	switch {
	case len(strsplits) == 1:
		xval = tval
	case len(strsplits) > 1:
		tm := toolkit.M{}
		if val.Has(strsplits[1]) {
			tm, _ = toolkit.ToM(val[strsplits[1]])
		}
		xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
	}

	return xval
	// if  {
	// 	val = tval
	// } else {
	// 	val = val.Set(k, v)
	// 	if val == nil {

	// 	} else {

	// 	}

	// 	val = getresultobj(strsplits[1:], tval)
	// }

	// return val
}
Example #2
0
func CheckValue(v toolkit.M, f *Filter) (bool, interface{}) {
	resbool := false
	filedtemp := []interface{}{}

	resbool = v.Has(f.Field)
	if resbool {
		return resbool, v.Get(f.Field)
	} else if strings.Contains(f.Field, ".") {
		ar := strings.Split(f.Field, ".")
		for i, dt := range ar {
			if i == 0 {
				resbool = v.Has(dt)
				if !resbool {
					break
				} else {
					filedtemp = append(filedtemp, v.Get(dt))
				}
			} else {
				temp := toolkit.M(filedtemp[i-1].(map[string]interface{}))
				resbool = temp.Has(dt)
				if !resbool {
					break
				} else {
					filedtemp = append(filedtemp, temp.Get(dt))
				}
			}

			if i == len(ar)-1 {
				return true, filedtemp[i]
			}
		}
	}
	return false, nil
}
Example #3
0
func (q *Query) execQueryPartUpdate(dt toolkit.M, Cond QueryCondition) error {

	if len(dt) == 0 {
		return errorlib.Error(packageName, "Query", modQuery, "data to update is not found")
	}

	writer := q.Connection().(*Connection).writer
	reader := q.Connection().(*Connection).reader
	tempHeader := []string{}

	for _, val := range q.Connection().(*Connection).headerColumn {
		tempHeader = append(tempHeader, val.name)
	}

	for {
		foundChange := false

		recData := toolkit.M{}
		dataTemp, e := reader.Read()
		for i, val := range dataTemp {
			recData.Set(tempHeader[i], val)
		}

		if len(Cond.Find) > 0 || (len(Cond.Find) == 0 && toolkit.IdField(dt) == "") {
			foundChange = Cond.getCondition(recData)
		}

		// Check ID IF Condition Not Found
		if nameid := toolkit.IdField(dt); nameid != "" && !foundChange {
			if recData.Has(nameid) && dt[nameid] == recData[nameid] {
				foundChange = true
			}
		}

		if foundChange && len(dataTemp) > 0 {
			for n, v := range tempHeader {
				if dt.Has(v) {
					dataTemp[n] = cast.ToString(dt[v])
				}
			}
		}

		if e == io.EOF {
			if dataTemp != nil {
				writer.Write(dataTemp)
				writer.Flush()
			}
			break
		} else if e != nil {
			return errorlib.Error(packageName, modQuery, "Update", e.Error())
		}
		if dataTemp != nil {
			writer.Write(dataTemp)
			writer.Flush()
		}
	}

	return nil
}
Example #4
0
func populateParmValue(inputM *toolkit.M, parms toolkit.M) {
	in := *inputM
	for k, _ := range in {
		if parms.Has(k) {
			in[k] = parms[k]
		} else {
			in[k] = ""
		}
	}
	*inputM = in
}
Example #5
0
func (q *Query) execQueryPartUpdate(dt toolkit.M) error {
	var e error
	e = q.startWriteMode()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec-Update: ", e.Error())
	}

	writer := q.writer
	reader := q.reader
	tempHeader := []string{}

	for _, val := range q.headerColumn {
		tempHeader = append(tempHeader, val.name)
	}

	var i int = 0

	for {
		i += 1

		dataTemp, e := reader.Read()
		if toolkit.HasMember(q.indexes, i) && len(dataTemp) > 0 {
			for n, v := range tempHeader {
				if dt.Has(v) {
					dataTemp[n] = cast.ToString(dt[v])
				}
			}
		}

		if e == io.EOF {
			if len(dataTemp) > 0 {
				writer.Write(dataTemp)
				writer.Flush()
			}
			break
		} else if e != nil {
			_ = q.endWriteMode()
			return err.Error(packageName, modQuery, "Exec-Update:", e.Error())
		}
		if len(dataTemp) > 0 {
			writer.Write(dataTemp)
			writer.Flush()
		}
	}

	q.execOpr = true
	e = q.endWriteMode()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec-Update: ", e.Error())
	}
	return nil
}
Example #6
0
func (l *LoginController) ProcessLogin(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := toolkit.M{}
	err := r.GetPayload(&payload)

	switch {
	case err != nil:
		return helper.CreateResult(false, nil, err.Error())
	case !payload.Has("username") || !payload.Has("password"):
		return helper.CreateResult(false, nil, "username or password not found")
	case payload.Has("username") && len(toolkit.ToString(payload["username"])) == 0:
		return helper.CreateResult(false, nil, "username cannot empty")
	case payload.Has("password") && len(toolkit.ToString(payload["password"])) == 0:
		return helper.CreateResult(false, nil, "password cannot empty")
	}

	sessid, err := acl.Login(toolkit.ToString(payload["username"]), toolkit.ToString(payload["password"]))
	if err != nil {
		return helper.CreateResult(true, "", err.Error())
	}
	r.SetSession("sessionid", sessid)
	return helper.CreateResult(true, toolkit.M{}.Set("status", true), "Login Success")

}
Example #7
0
func (q *Query) compileUpdateFrom(queryString string, ins toolkit.M) string {
	updateString := func() string {
		if !ins.Has("data") {
			return ""
		}

		var updates []string

		for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
			updates = append(updates, fmt.Sprintf("%s = %s", k, q.getAsString(v)))
		}

		return strings.Join(updates, ", ")
	}()

	queryString = fmt.Sprintf("%sSET %s ", queryString, updateString)
	return queryString
}
Example #8
0
func (a *Controller) KendoGridSettings(ins toolkit.M) toolkit.M {
	if ins == nil {
		ins = toolkit.M{}
	}
	s := toolkit.M{}
	q_skip := a.Ctx.Input.Query("skip")
	q_page := a.Ctx.Input.Query("page")
	q_size := a.Ctx.Input.Query("pageSize")

	if q_skip != "" {
		s.Set("skip", toolkit.ToInt(q_skip))
	}

	if q_page != "" {
		s.Set("page", toolkit.ToInt(q_page))
	}

	if q_size != "" {
		s.Set("limit", toolkit.ToInt(q_size))
	}

	sortField := strings.ToLower(a.Ctx.Input.Query("sort[0][field]"))
	sortDir := a.Ctx.Input.Query("sort[0][dir]")

	if sortField != "" {
		if sortField == "id" {
			sortField = "_id"
		}
		if sortDir == "" || sortDir == "asc" {
			s.Set("order", []string{sortField})
		} else {
			s.Set("order", []string{"-" + sortField})
		}
	}

	if fqe := a.KendoGridFilter("filter"); fqe != nil {
		if ins.Has("where") {
			fqe = dbs.And(fqe, ins.Get("where").(*dbs.QE))
		}
		s.Set("where", fqe)
	}

	return s
}
Example #9
0
File: dbox.go Project: haibudi/dbox
func MatchM(v toolkit.M, filters []*Filter) bool {
	var match bool

	for _, f := range filters {
		if f.Field != "" {
			//--- if has field: $eq, $ne, $gt, $lt, $gte, $lte, $contains
			//toolkit.Printf("Filter:%s V:%s Has:%s Match:%s", f.Field, toolkit.JsonString(v), v.Has("random"), match)
			if v.Has(f.Field) {
				match = MatchV(v.Get(f.Field), f)
				//toolkit.Printf("Filter:%s Value: %v Match:%s \n", toolkit.JsonString(f), v.Get(f.Field), match)
				if !match {
					return false
				}
			} else {
				if f.Op != FilterOpNoEqual && f.Op != FilterOpNin {
					return false
				}
			}
		} else {
			//-- no field: $and, $or
			//toolkit.Printf("Filter: %s\n", toolkit.JsonString(f))
			if f.Op == FilterOpAnd || f.Op == FilterOpOr {
				filters2 := f.Value.([]*Filter)
				for k, f2 := range filters2 {
					if f.Op == FilterOpAnd {
						if k == 0 {
							match = MatchM(v, []*Filter{f2})
						} else {
							match = match && MatchM(v, []*Filter{f2})
						}
					} else {
						if k == 0 {
							match = MatchM(v, []*Filter{f2})
						} else {
							match = match || MatchM(v, []*Filter{f2})
						}
					}
				}
			}
			//toolkit.Printf("\n")
		}
	}
	return match
}
Example #10
0
func (q *Query) compileInsertFrom(queryString string, ins toolkit.M) string {
	keyString, valString := func() (string, string) {
		if !ins.Has("data") {
			return "", ""
		}

		keys := []string{}
		vals := []string{}

		for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
			keys = append(keys, k)
			vals = append(vals, q.getAsString(v))
		}

		return strings.Join(keys, ", "), strings.Join(vals, ", ")
	}()

	queryString = fmt.Sprintf("%s(%s) VALUES (%s) ", queryString, keyString, valString)
	return queryString
}
Example #11
0
func (l *LoginController) SavePassword(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := toolkit.M{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	if !payload.Has("newpassword") || !payload.Has("userid") {
		return helper.CreateResult(false, nil, "Data is not complete")
	}

	switch {
	case payload.Has("tokenid"):
		err = acl.ChangePasswordToken(toolkit.ToString(payload["userid"]), toolkit.ToString(payload["newpassword"]), toolkit.ToString(payload["tokenid"]))
	default:
		// check sessionid first
		savedsessionid := "" //change with get session
		//=======================
		userid, err := acl.FindUserBySessionID(savedsessionid)
		if err == nil && userid == toolkit.ToString(payload["userid"]) {
			err = acl.ChangePassword(toolkit.ToString(payload["userid"]), toolkit.ToString(payload["newpassword"]))
		} else if err == nil {
			err = errors.New("Userid is not match")
		}
	}

	return helper.CreateResult(true, nil, "save password success")
}
Example #12
0
func (d *DataContext) Get(m IModel, config tk.M) error {
	var e error
	q := d.Connection.NewQuery().SetConfig("pooling", d.Pooling()).From(m.(IModel).TableName())
	if config.Has(ConfigWhere) {
		q = q.Where(config.Get(ConfigWhere).(*dbox.Filter))
	}
	if config.Has(ConfigOrder) {
		q = q.Order(config.Get(ConfigOrder).([]string)...)
	}
	q = q.Take(1)
	//q := d.Connection.NewQuery().From(m.(IModel).TableName()).Where(dbox.Eq("_id", id))
	c, e := q.Cursor(nil)
	if e != nil {
		return err.Error(packageName, modCtx, "Get", "Cursor fail. "+e.Error())
	}
	defer c.Close()
	e = c.Fetch(m, 1, false)
	if e != nil {
		return err.Error(packageName, modCtx, "Get", e.Error())
	}
	return nil
}
Example #13
0
func (a *Client) Call(methodName string, in toolkit.M) *toolkit.Result {
	if a.client == nil {
		return toolkit.NewResult().SetErrorTxt("Unable to call, no connection handshake")
	}
	if in == nil {
		in = toolkit.M{}
	}
	out := toolkit.NewResult()
	in["method"] = methodName
	if in.GetString("auth_referenceid") == "" {
		in["auth_referenceid"] = a.sessionID
	}
	//fmt.Println("SessionID: " + a.sessionID)
	if in.Has("auth_secret") == false {
		in.Set("auth_secret", a.secret)
	}
	e := a.client.Call("Rpc.Do", in, out)
	//_ = "breakpoint"
	if e != nil {
		return out.SetErrorTxt(a.address + "." + methodName + " Fail: " + e.Error())
	}
	return out
}
Example #14
0
func (q *Query) execQueryPartInsert(dt toolkit.M) error {
	var e error
	e = q.startWriteMode()

	if e != nil {
		return err.Error(packageName, modQuery, "Exec-Insert: ", e.Error())
	}

	if q.newheader {
		q.setNewHeader(dt)
	}

	writer := q.writer
	// reader := q.reader
	dataTemp := []string{}

	for _, v := range q.headerColumn {
		if dt.Has(v.name) {
			dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
		} else {
			dataTemp = append(dataTemp, "")
		}
	}

	if len(dataTemp) > 0 {
		writer.Write(dataTemp)
		writer.Flush()
	}

	q.execOpr = true
	e = q.endWriteMode()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec-Insert: ", e.Error())
	}

	return nil
}
Example #15
0
func (l *LoginController) ResetPassword(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := toolkit.M{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	fmt.Println(payload.Has("email"))
	fmt.Println(payload.Has("baseurl"))
	if !payload.Has("email") || !payload.Has("baseurl") {
		return helper.CreateResult(false, nil, "Data is not complete")
	}

	uname, tokenid, err := acl.ResetPassword(toolkit.ToString(payload["email"]))
	fmt.Printf("%v, %v, %v \n\n", uname, tokenid, err)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	linkstr := fmt.Sprintf("<a href='%v/web/confirmreset?1=%v&2=%v'>Click</a>", toolkit.ToString(payload["baseurl"]), uname, tokenid)

	mailmsg := fmt.Sprintf("Hi, <br/><br/> We received a request to reset your password, <br/><br/>")
	mailmsg = fmt.Sprintf("%vFollow the link below to set a new password : <br/><br/> %v <br/><br/>", mailmsg, linkstr)
	mailmsg = fmt.Sprintf("%vIf you don't want to change your password, you can ignore this email <br/><br/> Thanks,</body></html>", mailmsg)

	m := gomail.NewMessage()

	m.SetHeader("From", "*****@*****.**")
	m.SetHeader("To", toolkit.ToString(payload["email"]))

	m.SetHeader("Subject", "[no-reply] Self password reset")
	m.SetBody("text/html", mailmsg)

	d := gomail.NewPlainDialer("smtp.office365.com", 587, "*****@*****.**", "******")
	err = d.DialAndSend(m)

	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, nil, "reset password success")
}
Example #16
0
func (a *AclController) Logout(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := toolkit.M{}
	err := r.GetPayload(&payload)
	sessionid := ""
	switch {
	case err != nil:
		return helper.CreateResult(false, nil, err.Error())
	case !payload.Has("username") && !payload.Has("sessionid"):
		return helper.CreateResult(false, nil, "username or session not found")
	case payload.Has("sessionid"):
		sessionid = toolkit.ToString(payload["sessionid"])
	case payload.Has("username"):
		tUser := new(acl.User)
		err = acl.FindUserByLoginID(tUser, toolkit.ToString(payload["username"]))
		if err != nil {
			return helper.CreateResult(false, nil, "fail to get userid")
		}

		tSession := new(acl.Session)
		err = acl.FindActiveSessionByUser(tSession, tUser.ID)
		if err != nil {
			return helper.CreateResult(false, nil, "fail to get sessionid")
		}
		sessionid = tSession.ID
	}

	if sessionid == "" {
		return helper.CreateResult(true, nil, "Active sessionid not found")
	}

	err = acl.Logout(sessionid)
	if err != nil && (err.Error() == "Session id not found" || err.Error() == "Session id is expired") {
		return helper.CreateResult(true, nil, "Active sessionid not found")
	} else if err != nil {
		return helper.CreateResult(true, nil, toolkit.Sprintf("Error found : %v", err.Error()))
	}

	return helper.CreateResult(true, nil, "Logout success")
}
Example #17
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
	tableData := []toolkit.M{}
	// var e error

	rows, e := c.session.Query(c.QueryString)
	var valueType reflect.Type

	if n == 1 {
		valueType = reflect.TypeOf(m).Elem()
	} else {
		valueType = reflect.TypeOf(m).Elem().Elem()
	}

	if e != nil {
		return e
	}
	defer rows.Close()
	columns, e := rows.Columns()

	if e != nil {
		return e
	}

	count := len(columns)

	values := make([]interface{}, count)
	valuePtrs := make([]interface{}, count)

	for rows.Next() {
		for i := 0; i < count; i++ {
			valuePtrs[i] = &values[i]
		}

		rows.Scan(valuePtrs...)
		entry := toolkit.M{}

		for i, col := range columns {
			var v interface{}
			val := values[i]
			// b, ok := val.([]byte)

			// // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val))

			// var out interface{}
			// e = toolkit.Unjson(b, &out)
			// // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e)

			// if e != nil {
			// 	ok = false
			// }
			// if ok {
			// 	v = out
			// 	toolkit.Println("error OK :: ", ok, " :v :", v)
			// } else {
			// 	toolkit.Println("error OK :: ", ok, " :b :", b)
			// 	v = string(b)
			// }
			v = val
			entry.Set(strings.ToLower(col), v)
		}

		if valueType.Kind() == reflect.Struct {
			for i := 0; i < valueType.NumField(); i++ {
				namaField := strings.ToLower(valueType.Field(i).Name)
				dataType := strings.ToLower(valueType.Field(i).Type.String())

				if entry.Has(namaField) {
					if strings.Contains(dataType, "int") {
						entry.Set(namaField,
							cast.ToInt(entry[namaField], cast.RoundingAuto))
					} else if strings.Contains(dataType, "time.time") {
						entry.Set(namaField,
							cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
					}
				}
			}
		}

		tableData = append(tableData, entry)
	}
	// toolkit.Println("... ::: ", tableData)

	maxIndex := toolkit.SliceLen(tableData)

	var e2 error
	if e2 != nil {
		return e2
	}
	end := c.start + n

	if end > maxIndex || n == 0 {
		end = maxIndex
	}

	if c.start >= maxIndex {
		e2 = errors.New("No more data to fetched!")
	} else {
		e2 = toolkit.Serde(tableData[c.start:end], m, "json")
	}
	c.start = end

	return e2
}
Example #18
0
func (q *Query) Exec(in toolkit.M) error {
	setting, e := q.prepare(in)
	commandType := setting["commandtype"].(string)

	if e != nil {
		return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
	}

	if commandType == dbox.QueryPartSelect {
		return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
	}

	q.Lock()
	defer q.Unlock()

	datam := toolkit.M{}

	if in.Has("data") {
		datam, e = toolkit.ToM(in["data"])
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
		}
	}

	if commandType != dbox.QueryPartDelete && len(datam) == 0 {
		return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working, need data to param input")
	}
	// fmt.Println("DEBUG-113")
	e = q.openFile()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec ", e.Error())
	}

	where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
	if nameid := toolkit.IdField(datam); nameid != "" {
		if commandType == dbox.QueryPartInsert || commandType == dbox.QueryPartSave {
			where = make([]*dbox.Filter, 0, 0)
		}
		where = append(where, dbox.Eq(strings.ToLower(nameid), datam[nameid]))
	}

	q.indexes = make([]int, 0, 0)
	if len(where) > 0 {
		q.indexes, e = q.generateIndex(where)
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: ", e.Error())
		}
	}

	if q.newheader && commandType != dbox.QueryPartSave && commandType != dbox.QueryPartInsert {
		return err.Error(packageName, modQuery, "Exec: ", "Only save and insert permited")
	}

	q.newFileWrite = false
	q.execOpr = false
	switch commandType {
	case dbox.QueryPartSave:
		if len(q.indexes) > 0 {
			q.newFileWrite = true
			e = q.execQueryPartUpdate(datam)
		} else {
			e = q.execQueryPartInsert(datam)
		}
	case dbox.QueryPartInsert:
		if len(q.indexes) > 0 {
			e = fmt.Errorf("Id found, cannot insert with same Id")
		} else {
			e = q.execQueryPartInsert(datam)
		}
	case dbox.QueryPartDelete:
		q.newFileWrite = true
		if len(q.indexes) > 0 {
			e = q.execQueryPartDelete()
		}
	case dbox.QueryPartUpdate:
		q.newFileWrite = true
		if len(q.indexes) > 0 {
			e = q.execQueryPartUpdate(datam)
		}
	}

	if e != nil {
		return err.Error(packageName, modQuery, "Exec: ", e.Error())
	}

	return nil
}
Example #19
0
func (q *Query) Exec(parm toolkit.M) error {
	var e error
	q.save = false

	// useHeader := q.Connection().Info().Settings.Get("useheader", false).(bool)
	if parm == nil {
		parm = toolkit.M{}
	}

	data := toolkit.M{}
	if parm.Has("data") {
		data, _ = toolkit.ToM(parm["data"])
	}

	parts := crowd.From(q.Parts()).Group(func(x interface{}) interface{} {
		qp := x.(*dbox.QueryPart)
		return qp.PartType
	}, nil).Data

	// fromParts, hasFrom := parts[dbox.QueryPartFrom]
	// if !hasFrom {
	// 	return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
	// }
	// tablename = fromParts.([]interface{})[0].(*dbox.QueryPart).Value.(string)

	// var where interface{}
	commandType := ""
	//	multi := false

	_, hasDelete := parts[dbox.QueryPartDelete]
	_, hasInsert := parts[dbox.QueryPartInsert]
	_, hasUpdate := parts[dbox.QueryPartUpdate]
	_, hasSave := parts[dbox.QueryPartSave]

	if hasDelete {
		commandType = dbox.QueryPartDelete
	} else if hasInsert {
		commandType = dbox.QueryPartInsert
	} else if hasUpdate {
		commandType = dbox.QueryPartUpdate
	} else if hasSave {
		commandType = dbox.QueryPartSave
		q.save = true
	}

	var where interface{}
	whereParts, hasWhere := parts[dbox.QueryPartWhere]
	if hasWhere {
		fb := q.Connection().Fb()
		for _, p := range whereParts.([]interface{}) {
			fs := p.(*dbox.QueryPart).Value.([]*dbox.Filter)
			for _, f := range fs {
				if len(parm) > 0 {
					f = ReadVariable(f, parm)
				}
				fb.AddFilter(f)
			}
		}
		where, e = fb.Build()
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Cursor", e.Error())
		}
	}

	//Check setNewHeader First
	if q.Connection().(*Connection).setNewHeader && commandType != dbox.QueryPartInsert {
		q.Connection().(*Connection).Close()
		filename := q.Connection().(*Connection).Info().Host
		os.Remove(filename)
		return errorlib.Error(packageName, "Query", modQuery, "Only Insert Permited")
	}

	q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Append
	if hasDelete || hasUpdate {
		q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Create
	}

	q.Connection().(*Connection).ExecOpr = false
	if !q.Connection().(*Connection).setNewHeader && (commandType != dbox.QueryPartSave || (commandType == dbox.QueryPartSave && q.Connection().(*Connection).writer == nil)) {
		e = q.Connection().(*Connection).StartSessionWrite()
	}

	if e != nil {
		return errorlib.Error(packageName, "Query", modQuery, e.Error())
	}

	var execCond QueryCondition
	execCond.Find, _ = toolkit.ToM(where)

	switch commandType {
	case dbox.QueryPartSave:
		q.updatessave = false
		e = q.execQueryPartSave(data)
	case dbox.QueryPartInsert:
		e = q.execQueryPartInsert(data)
	case dbox.QueryPartDelete:
		e = q.execQueryPartDelete(execCond)
	case dbox.QueryPartUpdate:
		e = q.execQueryPartUpdate(data, execCond)
	}

	if e != nil {
		return e
	}

	q.Connection().(*Connection).ExecOpr = true
	if commandType != dbox.QueryPartSave || q.updatessave {
		e = q.Connection().(*Connection).EndSessionWrite()
		q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Append
	}

	if e != nil {
		return errorlib.Error(packageName, "Query", modQuery, e.Error())
	}

	return nil
}
Example #20
0
func checkconfig(cm toolkit.M) (err error) {
	err = nil

	if !cm.Has("_id") {
		err = errors.New(fmt.Sprintf("_id is not found"))
		return
	}

	if !cm.Has("sourcetype") {
		err = errors.New(fmt.Sprintf("sourcetype is not found"))
		return
	}

	if cm.Has("grabconf") {
		_, err = toolkit.ToM(cm["grabconf"])
	} else {
		err = errors.New(fmt.Sprintf("grab config is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("intervalconf") {
		_, err = toolkit.ToM(cm["intervalconf"])
	} else {
		err = errors.New(fmt.Sprintf("interval configuration is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("logconf") {
		_, err = toolkit.ToM(cm["logconf"])
	} else {
		err = errors.New(fmt.Sprintf("log configuration is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("datasettings") {
		if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" {
			err = errors.New(fmt.Sprintf("data settings must be []interface {}"))
		}
	} else {
		err = errors.New(fmt.Sprintf("data settings is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("histconf") {
		_, err = toolkit.ToM(cm["histconf"])
	} else {
		err = errors.New(fmt.Sprintf("history configuration is not found"))
	}

	if err != nil {
		return
	}
	return
}
Example #21
0
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
	// log.Printf("start parse:\n")
	if !toolkit.IsPointer(m) {
		// log.Printf("not pointer\n")
		return errorlib.Error("", "", "Fetch", "Model object should be pointer")
	}
	// log.Printf("pointer\n")
	slice := false
	var ins []string
	if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
		slice = true
		ins = in.([]string)
	} else {
		ins = append(ins, in.(string))
	}

	// log.Printf("outputType: %v\n", outputType)

	if outputType == CSV {
		var v reflect.Type

		if slice {
			v = reflect.TypeOf(m).Elem().Elem()
		} else {
			v = reflect.TypeOf(m).Elem()
		}

		ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
		for _, data := range ins {
			appendData := toolkit.M{}
			iv := reflect.New(v).Interface()
			reader := csv.NewReader(strings.NewReader(""))
			if strings.Contains(data, "','") {
				reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
			} else {
				reader = csv.NewReader(strings.NewReader(data))
			}
			record, e := reader.Read()

			if e != nil {
				return e
			}

			if v.Kind() == reflect.Struct {
				for i := 0; i < v.NumField(); i++ {
					appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
				}

				for i := 0; i < v.NumField(); i++ {
					tag := v.Field(i).Tag

					if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
						valthis := appendData[v.Field(i).Name]
						if valthis == nil {
							valthis = appendData[tag.Get("tag_name")]
						}

						switch v.Field(i).Type.Kind() {
						case reflect.Int:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int16:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int32:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int64:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Float32:
							valf, _ := strconv.ParseFloat(valthis.(string), 32)
							appendData.Set(v.Field(i).Name, valf)
						case reflect.Float64:
							valf, _ := strconv.ParseFloat(valthis.(string), 64)
							appendData.Set(v.Field(i).Name, valf)
						}

						dtype := DetectFormat(valthis.(string), dateFormat)
						if dtype == "date" {
							valf := cast.String2Date(valthis.(string), dateFormat)
							appendData.Set(v.Field(i).Name, valf)
						} else if dtype == "bool" {
							valf, _ := strconv.ParseBool(valthis.(string))
							appendData.Set(v.Field(i).Name, valf)
						}
					}
				}
			} else {
				for i, val := range header {
					appendData[val] = strings.TrimSpace(record[i])
				}

				if len(header) == 0 {
					e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
					return e
				}

				for _, val := range header {
					valthis := appendData[val]
					dtype := DetectFormat(valthis.(string), dateFormat)
					if dtype == "int" {
						appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
					} else if dtype == "float" {
						valf, _ := strconv.ParseFloat(valthis.(string), 64)
						appendData.Set(val, valf)
					} else if dtype == "date" {
						valf := cast.String2Date(valthis.(string), dateFormat)
						appendData.Set(val, valf)
					} else if dtype == "bool" {
						valf, _ := strconv.ParseBool(valthis.(string))
						appendData.Set(val, valf)
					}
				}
			}

			toolkit.Serde(appendData, iv, JSON)
			ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
		}
		if slice {
			reflect.ValueOf(m).Elem().Set(ivs)
		} else {
			reflect.ValueOf(m).Elem().Set(ivs.Index(0))
		}
	} else if outputType == JSON {
		var temp interface{}
		ins = InspectJson(ins)

		//for catch multi json in one line
		if JsonPart != "" && slice {
			for {
				tempjsonpart := JsonPart
				JsonPart = ""
				tempIn := InspectJson([]string{tempjsonpart})
				if len(tempIn) == 0 {
					break
				} else {
					for _, tin := range tempIn {
						ins = append(ins, tin)
					}
				}
			}
		}

		forSerde := strings.Join(ins, ",")
		if slice {
			forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
		}
		if len(ins) > 0 {
			e := json.Unmarshal([]byte(forSerde), &temp)
			if e != nil {
				return e
			}
			e = toolkit.Serde(temp, m, "json")
			if e != nil {
				return e
			}
		}
	} else {
		var v reflect.Type

		if slice {
			v = reflect.TypeOf(m).Elem().Elem()
		} else {
			v = reflect.TypeOf(m).Elem()
		}

		// log.Printf("v: %v\n", v)

		ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)

		// log.Printf("ivs: %v\n", ivs)

		for _, data := range ins {
			appendData := toolkit.M{}
			iv := reflect.New(v).Interface()

			/*log.Printf("data: %v\n", data)
			log.Printf("iv: %v\n", iv)*/

			splitted := strings.Split(data, "\t")

			/*log.Printf("appendData: %v\n", appendData)
			log.Printf("kind: %v\n", v.Kind())
			log.Printf("test: %v", fmt.Sprintf("%v", v))
			//log.Printf("v.Name: %T\n", v)

			if fmt.Sprintf("%v", v) == "reflect.Value" {
				log.Printf("else: %v\n", "reflect.Value")
				for _, val := range header {
					log.Printf("val: %v\n", val)
					valthis := appendData[val]
					dtype := DetectFormat(valthis.(string), dateFormat)
					if dtype == "int" {
						appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
					} else if dtype == "float" {
						valf, _ := strconv.ParseFloat(valthis.(string), 64)
						appendData.Set(val, valf)
					} else if dtype == "date" {
						valf := cast.String2Date(valthis.(string), dateFormat)
						appendData.Set(val, valf)
					} else if dtype == "bool" {
						valf, _ := strconv.ParseBool(valthis.(string))
						appendData.Set(val, valf)
					}
				}
				log.Printf("appendData: %v\n", appendData)
			} else */
			if v.Kind() == reflect.Struct {
				for i := 0; i < v.NumField(); i++ {
					appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
				}

				// log.Printf("struct: %v\n", v.Kind())
				for i := 0; i < v.NumField(); i++ {
					tag := v.Field(i).Tag
					// log.Printf("i: %v\n", i)

					// log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name")))

					if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
						valthis := appendData[v.Field(i).Name]
						if valthis == nil {
							valthis = appendData[tag.Get("tag_name")]
						}
						// log.Printf("valthis: %v\n", valthis)
						switch v.Field(i).Type.Kind() {
						case reflect.Int:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int16:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int32:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Int64:
							appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
						case reflect.Float32:
							valf, _ := strconv.ParseFloat(valthis.(string), 32)
							appendData.Set(v.Field(i).Name, valf)
						case reflect.Float64:
							valf, _ := strconv.ParseFloat(valthis.(string), 64)
							appendData.Set(v.Field(i).Name, valf)
						}
						dtype := DetectFormat(valthis.(string), dateFormat)
						if dtype == "date" {
							valf := cast.String2Date(valthis.(string), dateFormat)
							appendData.Set(v.Field(i).Name, valf)
						} else if dtype == "bool" {
							valf, _ := strconv.ParseBool(valthis.(string))
							appendData.Set(v.Field(i).Name, valf)
						}
					}
				}

			} else {
				for i, val := range header {
					appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
				}

				if len(header) == 0 {
					e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
					return e
				}

				// log.Printf("else: %v\n", v.Kind())
				for _, val := range header {
					// log.Printf("val: %v\n", val)
					valthis := appendData[val]
					dtype := DetectFormat(valthis.(string), dateFormat)
					if dtype == "int" {
						appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
					} else if dtype == "float" {
						valf, _ := strconv.ParseFloat(valthis.(string), 64)
						appendData.Set(val, valf)
					} else if dtype == "date" {
						valf := cast.String2Date(valthis.(string), dateFormat)
						appendData.Set(val, valf)
					} else if dtype == "bool" {
						valf, _ := strconv.ParseBool(valthis.(string))
						appendData.Set(val, valf)
					}
				}
			}

			toolkit.Serde(appendData, iv, JSON)
			// log.Printf("iv result: %v\n", iv)
			ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
			// log.Printf("ivs result: %v\n", ivs)
		}

		if slice {
			reflect.ValueOf(m).Elem().Set(ivs)
		} else {
			reflect.ValueOf(m).Elem().Set(ivs.Index(0))
		}
		// log.Printf("result: %v\n", m)
	}
	return nil
}
Example #22
0
func fetchConfig() (err error) {

	switch toolkit.ToString(config.Get("sourcetype", "")) {
	case "SourceType_HttpHtml":
		SourceType = SourceType_HttpHtml
	case "SourceType_HttpJson":
		SourceType = SourceType_HttpJson
	default:
		err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
		return
	}

	Log.AddLog("Start fetch grabconf", "INFO")
	if !config.Has("grabconf") {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
		return
	}

	tconfgrab := toolkit.M{}
	tconfgrab, err = toolkit.ToM(config["grabconf"])
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
		return
	}

	grabconfig := sedotan.Config{}
	if tconfgrab.Has("formvalues") {
		tfromvalues := toolkit.M{}
		tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
		if err != nil {
			err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
			return
		}
		grabconfig.SetFormValues(tfromvalues)
	}

	if tconfgrab.Has("loginvalues") {
		grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
	}

	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
		return
	}

	grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
	grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))

	grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
	grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
	grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))

	grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
	grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))

	Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")

	wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)

	Log.AddLog("Start fetch datasettings", "INFO")
	if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
	destDboxs = make(map[string]*DestInfo)

	for i, xVal := range config["datasettings"].([]interface{}) {
		err = nil
		tDataSetting := sedotan.DataSetting{}
		tDestDbox := DestInfo{}

		mVal := toolkit.M{}
		mVal, err = toolkit.ToM(xVal)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
			continue
		}

		tnameid := toolkit.ToString(mVal.Get("nameid", ""))
		if tnameid == "" {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
			continue
		}
		tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))

		// Fetch columnsettings
		if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
			continue
		}

		tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
		for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
			mValcs := toolkit.M{}
			mValcs, err = toolkit.ToM(Valcs)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
				continue
			}

			tgrabcolumn := sedotan.GrabColumn{}
			tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
			tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
			tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
			tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))

			tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
			tDataSetting.Column(tindex, &tgrabcolumn)
		}

		//Fetch Filter Condition
		if mVal.Has("filtercond") {
			tfiltercond := toolkit.M{}
			tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
			} else {
				tDataSetting.SetFilterCond(tfiltercond)
			}
		}

		//Fetch Connection Info
		tConnInfo := toolkit.M{}
		tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
		tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))

		tHost := toolkit.ToString(tConnInfo.Get("host", ""))
		tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
		tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
		tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
		tSettings := toolkit.M{}
		tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}

		tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.IConnection.Close()

		destDboxs[tnameid] = &tDestDbox
		wGrabber.DataSettings[tnameid] = &tDataSetting

	}
	err = nil

	if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	if !config.Has("histconf") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	Log.AddLog("Start fetch histconf", "INFO")
	histConf, err = toolkit.ToM(config.Get("histconf", nil))
	if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	return
}
Example #23
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
	if closeWhenDone {
		defer c.Close()
	}

	if len(c.indexes) == 0 {
		return nil
	}

	var lower, upper int

	lower = c.currentIndex
	if lower < c.skip {
		lower = c.skip
	}

	upper = lower + n
	if upper > c.limit && c.limit > 0 {
		upper = c.limit
	}

	if !toolkit.IsPointer(m) {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
	}

	if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
	}

	var v reflect.Type

	if n == 1 {
		v = reflect.TypeOf(m).Elem()
	} else {
		v = reflect.TypeOf(m).Elem().Elem()
	}

	ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)

	for {
		appendData := toolkit.M{}
		iv := reflect.New(v).Interface()

		datatemp, ef := c.q.reader.Read()
		c.realIndex += 1
		if c.indexes[c.currentIndex] != c.realIndex {
			continue
		}

		c.currentIndex += 1
		for i, val := range datatemp {
			if len(c.fields) == 0 || c.fields.Has("*") || c.fields.Has(c.q.headerColumn[i].name) {
				appendData[c.q.headerColumn[i].name] = val
			}
		}

		if v.Kind() == reflect.Struct {
			for i := 0; i < v.NumField(); i++ {
				if appendData.Has(v.Field(i).Name) {
					switch v.Field(i).Type.Kind() {
					case reflect.Int:
						appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
					}
				}
			}
		}

		isAppend := lower < c.currentIndex && upper >= c.currentIndex

		if (ef == io.EOF || ef == nil) && isAppend && len(appendData) > 0 {
			toolkit.Serde(appendData, iv, "json")
			ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
		}

		if ef != nil && ef != io.EOF {
			return errorlib.Error(packageName, modCursor, "Fetch", ef.Error())
		} else if ef == io.EOF || (ivs.Len() >= n && n > 0) {
			break
		}
	}

	// if e != nil {
	// 	return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	// }

	if n == 1 {
		reflect.ValueOf(m).Elem().Set(ivs.Index(0))
	} else {
		reflect.ValueOf(m).Elem().Set(ivs)
	}

	return nil
}
Example #24
0
func (q *Query) Exec(in toolkit.M) error {
	setting, e := q.prepare(in)
	commandType := setting["commandtype"].(string)
	//toolkit.Printf("Command type: %s\n", commandType)
	if e != nil {
		return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
	}

	if setting.GetString("commandtype") == dbox.QueryPartSelect {
		return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
	}

	q.Lock()
	defer q.Unlock()

	var dataM toolkit.M
	var dataMs []toolkit.M

	hasData := in.Has("data")
	dataIsSlice := false
	data := in.Get("data")
	if toolkit.IsSlice(data) {
		dataIsSlice = true
		e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
		}
	} else {
		dataM, e = toolkit.ToM(data)
		dataMs = append(dataMs, dataM)
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
		}
	}

	hasWhere := setting.Has("where")
	where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
	if hasWhere && len(where) == 0 {
		inWhere := in.Get("where")
		if inWhere == nil {
			hasWhere = false
			where = nil
		} else {
			if !toolkit.IsSlice(inWhere) {
				where = append(where, inWhere.(*dbox.Filter))
			} else {
				where = inWhere.([]*dbox.Filter)
			}
		}
	}

	if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartDelete,
		dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
		hasWhere = true
		//toolkit.Println("check where")
		if toolkit.IsSlice(data) {
			ids := []interface{}{}
			idField := ""
			if idField == "" {
				return err.Error(packageName, modQuery, "Exec: "+commandType, "Data send is a slice, but its element has no ID")
			}
			dataCount := toolkit.SliceLen(data)
			for i := 0; i < dataCount; i++ {
				dataI := toolkit.SliceItem(data, i)
				if i == 0 {
					idField = toolkit.IdField(dataI)
				}
				ids = append(ids, toolkit.Id(dataI))
			}
			where = []*dbox.Filter{dbox.In(idField, ids)}
		} else {
			idfield := "_id"
			id := toolkit.Id(data)
			if !toolkit.IsNilOrEmpty(id) {
				where = []*dbox.Filter{dbox.Eq(idfield, id)}
			} else {
				where = nil
				hasWhere = false
			}
		}
	}
	/*
		toolkit.Printf("CommandType: %s HasData: %v HasWhere: %v Where: %s\n",
			commandType, hasData, hasWhere, toolkit.JsonString(where))
	*/
	e = q.openFile(commandType)
	//toolkit.Printf(commandType+" Open File, found record: %d\nData:%s\n", len(q.data), toolkit.JsonString(q.data))
	if e != nil {
		return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
	}

	var indexes []interface{}
	if hasWhere && commandType != dbox.QueryPartInsert {
		whereIndex := dbox.Find(q.data, where)
		indexes = toolkit.ToInterfaceArray(&whereIndex)
		//toolkit.Printf("Where Index: %s Index:%s\n", toolkit.JsonString(whereIndex), toolkit.JsonString(indexes))
	}
	if commandType == dbox.QueryPartInsert {
		if !hasData {
			return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
		}
		if !dataIsSlice {
			dataMs = []toolkit.M{dataM}
		}

		//-- validate
		for _, datam := range dataMs {
			idField, idValue := toolkit.IdInfo(datam)
			toolkit.Serde(dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}), &indexes, "")
			if len(indexes) > 0 {
				return err.Error(packageName, modQuery, "Exec: "+commandType, toolkit.Sprintf("Data %v already exist", idValue))
			}
		}

		//-- insert the data
		q.data = append(q.data, dataMs...)
	} else if commandType == dbox.QueryPartUpdate {

		//-- valida
		if !hasData {
			return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
		}

		var dataUpdate toolkit.M
		var updateDataIndex int

		// if it is a slice then we need to update each data passed on its slice
		isDataSlice := toolkit.IsSlice(data)
		if isDataSlice == false {
			isDataSlice = false
			e = toolkit.Serde(data, &dataUpdate, "")
			if e != nil {
				return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
			}
		}

		var idField string
		//toolkit.Printf("Indexes: %s\n", toolkit.JsonString(indexes))

		for i, v := range q.data {
			// update only data that match given inde
			if toolkit.HasMember(indexes, i) || !hasWhere {
				if idField == "" {
					idField = toolkit.IdField(v)
					if idField == "" {
						return err.Error(packageName, modQuery, "Exec: "+commandType, "No ID")
					}
				}

				// If dataslice is sent, iterate f
				if isDataSlice {
					e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
					if e != nil {
						return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail "+e.Error())
					}
					updateDataIndex++
				}
				dataOrigin := q.data[i]
				toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"})
				toolkit.Serde(dataOrigin, &v, "")
				q.data[i] = v
			}
		}
	} else if commandType == dbox.QueryPartDelete {
		if hasWhere && len(where) > 0 {
			indexes := dbox.Find(q.data, where)
			if len(indexes) > 0 {
				newdata := []toolkit.M{}
				for index, v := range q.data {
					partOfIndex := toolkit.HasMember(indexes, index)
					if partOfIndex == false {
						newdata = append(newdata, v)
					}
					//toolkit.Println("i:", indexes, ", index:", index, ", p.ofIndex: ", partOfIndex, ", data: ", toolkit.JsonString(newdata))
				}
				q.data = newdata
			}
		} else {
			q.data = []toolkit.M{}
		}
		//toolkit.Printf("Data now: %s\n", toolkit.JsonString(q.data))
	} else if commandType == dbox.QueryPartSave {
		if !hasData {
			return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
		}

		var dataMs []toolkit.M
		var dataM toolkit.M
		if !toolkit.IsSlice(data) {
			e = toolkit.Serde(&data, &dataM, "json")
			if e != nil {
				return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
			}
			dataMs = append(dataMs, dataM)
		} else {
			e = toolkit.Serde(&data, &dataMs, "json")
			if e != nil {
				return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
			}
		}
		//toolkit.Printf("Saving: %s\n", toolkit.JsonString(dataMs))

		for _, v := range dataMs {
			idField, idValue := toolkit.IdInfo(v)
			indexes := dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)})
			if len(indexes) == 0 {
				q.data = append(q.data, v)
			} else {
				dataOrigin := q.data[indexes[0]]
				//toolkit.Printf("Copy data %s to %s\n", toolkit.JsonString(v), toolkit.JsonString(dataOrigin))
				toolkit.CopyM(&v, &dataOrigin, false, []string{idField})
				q.data[indexes[0]] = dataOrigin
			}
		}
	}
	e = q.writeFile()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec: "+commandType+" Write fail", e.Error())
	}
	return nil
}
Example #25
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
	fmt.Println(c.QueryString)

	rows, e := c.session.Query(c.QueryString)

	var valueType reflect.Type

	if n == 1 {
		valueType = reflect.TypeOf(m).Elem()
	} else {
		valueType = reflect.TypeOf(m).Elem().Elem()
	}

	if e != nil {
		return e
	}
	defer rows.Close()
	columns, e := rows.Columns()
	if e != nil {
		return e
	}

	count := len(columns)

	tableData := []toolkit.M{}
	values := make([]interface{}, count)
	valuePtrs := make([]interface{}, count)

	// valueint := values
	for rows.Next() {
		for i := 0; i < count; i++ {
			valuePtrs[i] = &values[i]
		}
		// rows.Scan(valuePtrs...)

		rows.Scan(valuePtrs...)
		entry := toolkit.M{}

		for i, col := range columns {
			var v interface{}
			val := values[i]

			// fmt.Println("Nilai val : ", val)
			b, ok := val.([]byte)
			if ok {
				v = string(b)
			} else {
				v = val
			}
			entry.Set(strings.ToLower(col), v)
			// entry.Set(col, values[i])
			// e = toolkit.DecodeByte(val.([]byte), v)
			// toolkit.FromBytes(toolkit.ToBytes(val, ""), "", v)

			// entry.Set(col, v)
		}

		if valueType.Kind() == reflect.Struct {
			for i := 0; i < valueType.NumField(); i++ {
				namaField := strings.ToLower(valueType.Field(i).Name)
				dataType := strings.ToLower(valueType.Field(i).Type.String())

				if entry.Has(namaField) {
					fmt.Println("isi entry : ", entry[namaField], dataType)
					if strings.Contains(dataType, "int") {
						entry.Set(namaField,
							cast.ToInt(entry[namaField], cast.RoundingAuto))
					} else if strings.Contains(dataType, "time.time") {
						entry.Set(namaField,
							cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
					}
				}
			}
		}

		tableData = append(tableData, entry)
	}

	fmt.Println("Nilai table data : ", tableData)
	if e != nil {
		return e
	}
	if n == 0 {
		// *m.(*[]map[string]interface{}) = tableData
		// toolkit.Unjson(toolkit.Jsonify(tableData), m)
		e = toolkit.Serde(tableData, m, "json")
		fmt.Println("Nilai Model : ", m)
	} else {
		end := c.start + n
		if end > len(tableData) {
			e = errors.New("index out of range")
		} else {
			// *m.(*[]map[string]interface{}) = tableData[0:n]
			//toolkit.Unjson(toolkit.Jsonify(tableData[0:n]), m)
			e = toolkit.Serde(tableData[0:n], m, "json")
		}
	}
	return e
}
Example #26
0
func (q *Query) execQueryPartSave(dt toolkit.M) error {
	if len(dt) == 0 {
		return errorlib.Error(packageName, modQuery, "save", "data to insert is not found")
	}

	writer := q.Connection().(*Connection).writer
	reader := q.Connection().(*Connection).reader
	tempHeader := []string{}

	for _, val := range q.Connection().(*Connection).headerColumn {
		tempHeader = append(tempHeader, val.name)
	}

	// Check ID Before Insert
	checkidfound := false
	if nameid := toolkit.IdField(dt); nameid != "" {
		q.updatessave = true

		var colsid int
		for i, val := range q.Connection().(*Connection).headerColumn {
			if val.name == nameid {
				colsid = i
			}
		}

		for {
			dataTempSearch, e := reader.Read()
			for i, val := range dataTempSearch {
				if i == colsid && val == dt[nameid] {
					checkidfound = true
					break
				}
			}
			if e == io.EOF {
				break
			} else if e != nil {
				return errorlib.Error(packageName, modQuery, "Save", e.Error())
			}
		}
	}

	if checkidfound {
		e := q.Connection().(*Connection).EndSessionWrite()
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Save", e.Error())
		}

		q.Connection().(*Connection).TypeOpenFile = TypeOpenFile_Create

		e = q.Connection().(*Connection).StartSessionWrite()
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Save", e.Error())
		}

		e = q.execQueryPartUpdate(dt, QueryCondition{})
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Save", e.Error())
		}
		// time.Sleep(1000 * time.Millisecond)
	} else {
		//Change to Do Insert
		dataTemp := []string{}

		for _, v := range q.Connection().(*Connection).headerColumn {
			if dt.Has(v.name) {
				dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
			} else {
				dataTemp = append(dataTemp, "")
			}
		}

		if len(dataTemp) > 0 {
			writer.Write(dataTemp)
			writer.Flush()
		}
	}

	return nil
}
Example #27
0
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {

	logConf, err := d.getLogger(dataGrabber)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
		defer logConf.Close()
	}

	message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
	logConf.AddLog(message, "SUCCESS")
	fmt.Println(message)

	dsOrigin := new(colonycore.DataSource)
	err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	dsDestination := new(colonycore.DataSource)
	err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	dataDS, _, conn, query, metaSave, err := new(DataSourceController).
		ConnectToDataSource(dataGrabber.DataSourceOrigin)
	if len(dataDS.QueryInfo) == 0 {
		message := "Data source origin has invalid query"
		logConf.AddLog(message, "ERROR")

		return false, nil, message
	}
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}
	defer conn.Close()

	if metaSave.keyword != "" {
		message := `Data source origin query is not "Select"`
		logConf.AddLog(message, "ERROR")

		return false, nil, message
	}

	cursor, err := query.Cursor(nil)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}
	defer cursor.Close()

	data := []toolkit.M{}
	err = cursor.Fetch(&data, 0, false)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	connDesc := new(colonycore.Connection)
	err = colonycore.Get(connDesc, dsDestination.ConnectionID)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	const FLAG_ARG_DATA string = `%1`
	transformedData := []toolkit.M{}

	for _, each := range data {
		eachTransformedData := toolkit.M{}

		for _, eachMap := range dataGrabber.Maps {
			var valueEachSourceField interface{}

			// ============================================ SOURCE
			if !strings.Contains(eachMap.Source, "|") {
				// source could be: field, object, array
				valueEachSourceField = each.Get(eachMap.Source)
			} else {
				// source could be: field of object, field of array-objects
				prev := strings.Split(eachMap.Source, "|")[0]
				next := strings.Split(eachMap.Source, "|")[1]

				var fieldInfoDes *colonycore.FieldInfo = nil
				for _, eds := range dsOrigin.MetaData {
					if eds.ID == prev {
						fieldInfoDes = eds
						break
					}
				}

				if fieldInfoDes != nil {
					// source is field of array-objects
					if fieldInfoDes.Type == "array-objects" {
						valueObjects := []interface{}{}
						if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil {
							valueObjects = make([]interface{}, len(temp))
							for i, each := range temp {
								if tempSub, _ := toolkit.ToM(each); tempSub != nil {
									valueObjects[i] = tempSub.Get(next)
								}
							}
						}
						valueEachSourceField = valueObjects
					} else {
						// source is field of object
						valueObject := toolkit.M{}
						if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil {
							valueEachSourceField = valueObject.Get(next)
						}
					}
				}
			}

			// ============================================ DESTINATION
			if !strings.Contains(eachMap.Destination, "|") {
				if eachMap.SourceType == "object" {
					sourceObject, _ := toolkit.ToM(valueEachSourceField)
					if sourceObject == nil {
						sourceObject = toolkit.M{}
					}

					valueObject := toolkit.M{}
					for _, desMeta := range dsDestination.MetaData {
						if desMeta.ID == eachMap.Destination {
							for _, eachMetaSub := range desMeta.Sub {
								// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
								valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
							}
							break
						}
					}

					eachTransformedData.Set(eachMap.Destination, valueObject)
				} else if eachMap.SourceType == "array-objects" {
					sourceObjects, _ := valueEachSourceField.([]interface{})
					if sourceObjects == nil {
						sourceObjects = []interface{}{}
					}

					valueObjects := []interface{}{}
					for _, sourceObjectRaw := range sourceObjects {
						sourceObject, _ := toolkit.ToM(sourceObjectRaw)
						if sourceObject == nil {
							sourceObject = toolkit.M{}
						}

						valueObject := toolkit.M{}
						for _, desMeta := range dsDestination.MetaData {
							if desMeta.ID == eachMap.Destination {
								for _, eachMetaSub := range desMeta.Sub {
									// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
									valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
								}
								break
							}
						}

						valueObjects = append(valueObjects, valueObject)
					}

					eachTransformedData.Set(eachMap.Destination, valueObjects)
				} else {
					if strings.Contains(eachMap.DestinationType, "array") {
						valueObjects := each.Get(eachMap.Source)

						eachTransformedData.Set(eachMap.Destination, valueObjects)
					} else {
						// eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each))
						eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType))
					}
				}
			} else {
				prev := strings.Split(eachMap.Destination, "|")[0]
				next := strings.Split(eachMap.Destination, "|")[1]

				var fieldInfoDes *colonycore.FieldInfo = nil
				for _, eds := range dsDestination.MetaData {
					if eds.ID == prev {
						fieldInfoDes = eds
						break
					}
				}

				if fieldInfoDes != nil {
					if fieldInfoDes.Type == "array-objects" {
						valueObjects := []interface{}{}
						if temp := eachTransformedData.Get(prev, nil); temp == nil {
							valueObjects = []interface{}{}
						} else {
							valueObjects, _ = temp.([]interface{})
							if valueObjects == nil {
								valueObjects = []interface{}{}
							}
						}

						if temp, _ := valueEachSourceField.([]interface{}); temp != nil {
							for i, eachVal := range temp {
								valueObject := toolkit.M{}
								if len(valueObjects) > i {
									if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil {
										valueObject = temp2
										// valueObject.Set(next, eachVal)
										valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
									}

									valueObjects[i] = valueObject
								} else {
									if fieldInfoDes.Sub != nil {
										for _, subMeta := range fieldInfoDes.Sub {
											valueObject.Set(subMeta.ID, nil)
										}
									}

									// valueObject.Set(next, eachVal)
									valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
									valueObjects = append(valueObjects, valueObject)
								}
							}
						}

						eachTransformedData.Set(prev, valueObjects)
					} else {
						valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev))
						if valueObject == nil {
							valueObject = toolkit.M{}
						}

						//tambahan
						prevSource := strings.Split(eachMap.Source, "|")[0]
						nextSource := strings.Split(eachMap.Source, "|")[1]
						mval, _ := toolkit.ToM(each.Get(prevSource, nil))

						//=========
						valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType))
						// valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval))
						eachTransformedData.Set(prev, valueObject)
					}
				}
			}
		}

		transformedData = append(transformedData, eachTransformedData)
		dataToSave := eachTransformedData

		// ================ pre transfer command
		if dataGrabber.PreTransferCommand != "" {
			// jsonTranformedDataBytes, err := json.Marshal(each)
			jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
			if err != nil {
				return false, nil, err.Error()
			}
			jsonTranformedData := string(jsonTranformedDataBytes)

			var preCommand = dataGrabber.PreTransferCommand
			if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) {
				preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1))
			}

			dataToSave = toolkit.M{}

			output, err := toolkit.RunCommand(preCommand, jsonTranformedData)
			fmt.Printf("===> Pre Transfer Command Result\n  COMMAND -> %s %s\n  OUTPUT  -> %s\n", preCommand, jsonTranformedData, output)
			if err == nil {
				postData := toolkit.M{}
				if err := json.Unmarshal([]byte(output), &postData); err == nil {
					dataToSave = postData
				}
			}
		}
		// ================

		if len(dataToSave) == 0 {
			continue
		}

		nilFieldDest := eachTransformedData
		for _, metadataDest := range dsDestination.MetaData {
			if temp := eachTransformedData.Get(metadataDest.ID); temp == nil {
				if metadataDest.ID != "_id" {
					if metadataDest.Type == "object" {
						valueObject := toolkit.M{}

						for _, eachMetaSub := range metadataDest.Sub {
							valueObject.Set(eachMetaSub.ID, nil)
						}
						nilFieldDest.Set(metadataDest.ID, valueObject)
					} else if metadataDest.Type == "array-objects" {
						valueEachSourceField := each.Get(metadataDest.ID)

						sourceObjects, _ := valueEachSourceField.([]interface{})
						if sourceObjects == nil {
							sourceObjects = []interface{}{}
						}

						valueObjects := []interface{}{}
						for _, sourceObjectRaw := range sourceObjects {
							sourceObject, _ := toolkit.ToM(sourceObjectRaw)

							if sourceObject == nil {
								sourceObject = toolkit.M{}
							}
							valueObject := toolkit.M{}
							for keyss, _ := range sourceObject {
								valueObject.Set(keyss, nil)
							}

							valueObjects = append(valueObjects, valueObject)
						}

						nilFieldDest.Set(metadataDest.ID, valueObjects)
					} else {
						if strings.Contains(metadataDest.Type, "array") {
							valueObjects := []interface{}{}

							nilFieldDest.Set(metadataDest.ID, valueObjects)
						} else {
							nilFieldDest.Set(metadataDest.ID, nil)
						}
					}
				}
			}
		}

		tableName := dsDestination.QueryInfo.GetString("from")
		queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
		if dataGrabber.InsertMode == "fresh" {
			queryWrapper.Delete(tableName, dbox.Or())
		}
		if eachTransformedData.Has("_id") {
			err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id")))
		}

		if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") {
			queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings)
		} else {
			queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
		}

		if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" {
			nilFieldDest.Set("_id", helper.RandomIDWithPrefix(""))
		}

		err = queryWrapper.Save(tableName, nilFieldDest)
		if err != nil {
			logConf.AddLog(err.Error(), "ERROR")

			return false, nil, err.Error()
		}

		// ================ post transfer command
		if dataGrabber.PostTransferCommand != "" {
			eachTransformedData = dataToSave
			jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
			if err != nil {
				return false, nil, err.Error()
			}
			jsonTranformedData := string(jsonTranformedDataBytes)

			var postCommand = dataGrabber.PostTransferCommand
			if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) {
				postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1))
			}

			output, err := toolkit.RunCommand(postCommand, jsonTranformedData)
			fmt.Printf("===> Post Transfer Command Result\n  COMMAND -> %s %s\n  OUTPUT  -> %s\n", postCommand, jsonTranformedData, output)
		}
	}

	message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
	logConf.AddLog(message, "SUCCESS")
	fmt.Println(message)

	return true, transformedData, ""
}
Example #28
0
func (q *Query) execQueryPartInsert(dt toolkit.M) error {

	if len(dt) == 0 {
		return errorlib.Error(packageName, "Query", modQuery, "data to insert is not found")
	}

	writer := q.Connection().(*Connection).writer
	reader := q.Connection().(*Connection).reader
	dataTemp := []string{}

	if q.Connection().(*Connection).setNewHeader {
		q.Connection().(*Connection).SetHeaderToolkitM(dt)
		q.Connection().(*Connection).setNewHeader = false

		for _, v := range q.Connection().(*Connection).headerColumn {
			dataTemp = append(dataTemp, v.name)
		}

		if len(dataTemp) > 0 {
			writer.Write(dataTemp)
			writer.Flush()
		}
		dataTemp = []string{}
	}

	// Check ID Before Insert
	if nameid := toolkit.IdField(dt); nameid != "" {
		var colsid int
		for i, val := range q.Connection().(*Connection).headerColumn {
			if val.name == nameid {
				colsid = i
			}
		}

		for {
			dataTempSearch, e := reader.Read()
			for i, val := range dataTempSearch {
				if i == colsid && val == dt[nameid] {
					return errorlib.Error(packageName, modQuery, "Insert", "Unique id is found")
				}
			}
			if e == io.EOF {
				break
			} else if e != nil {
				return errorlib.Error(packageName, modQuery, "Insert", e.Error())
			}
		}
	}

	for _, v := range q.Connection().(*Connection).headerColumn {
		if dt.Has(v.name) {
			dataTemp = append(dataTemp, cast.ToString(dt[v.name]))
		} else {
			dataTemp = append(dataTemp, "")
		}
	}

	if len(dataTemp) > 0 {
		writer.Write(dataTemp)
		writer.Flush()
	}

	return nil
}
Example #29
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {

	if closeWhenDone {
		defer c.Close()
	}

	e := c.prepIter()
	if e != nil {
		return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	}

	if !toolkit.IsPointer(m) {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
	}

	if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
	}

	var v reflect.Type

	if n == 1 {
		v = reflect.TypeOf(m).Elem()
	} else {
		v = reflect.TypeOf(m).Elem().Elem()
	}

	ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)

	lineCount := 0

	//=============================
	// fmt.Println("Qursor 133 : ", c.ConditionVal.Find)
	for {
		iv := reflect.New(v).Interface()

		isAppend := true
		c.count += 1
		recData := toolkit.M{}
		appendData := toolkit.M{}

		dataTemp, e := c.reader.Read()

		for i, val := range dataTemp {
			orgname := c.headerColumn[i].name
			lowername := strings.ToLower(c.headerColumn[i].name)

			switch c.headerColumn[i].dataType {
			case "int":
				recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
			case "float":
				recData[lowername] = cast.ToF64(val, 2, cast.RoundingAuto)
			default:
				recData[lowername] = val
			}

			if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
				appendData[orgname] = recData[lowername]
			} else {
				if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
					appendData[orgname] = recData[lowername]
				}
			}
		}

		isAppend = c.ConditionVal.getCondition(recData)

		if c.count < c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
			isAppend = false
		}

		if v.Kind() == reflect.Struct {
			for i := 0; i < v.NumField(); i++ {
				if appendData.Has(v.Field(i).Name) {
					switch v.Field(i).Type.Kind() {
					case reflect.Int:
						appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
					}
				}
			}
		}

		if e == io.EOF {
			if isAppend && len(appendData) > 0 {
				toolkit.Serde(appendData, iv, "json")
				ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
				lineCount += 1
			}
			break
		} else if e != nil {
			return errorlib.Error(packageName, modCursor,
				"Fetch", e.Error())
		}

		if isAppend && len(appendData) > 0 {
			toolkit.Serde(appendData, iv, "json")
			ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
			lineCount += 1
		}

		if n > 0 {
			if lineCount >= n {
				break
			}
		}
	}

	if e != nil {
		return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	}

	if n == 1 {
		if ivs.Len() > 0 {
			reflect.ValueOf(m).Elem().Set(ivs.Index(0))
		}
	} else {
		reflect.ValueOf(m).Elem().Set(ivs)
	}

	return nil
}
Example #30
0
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
	var hasLookup bool
	toolkit.Println("payload : ", payload)
	if payload.Has("haslookup") {
		hasLookup = payload.Get("haslookup").(bool)
	}
	_id := toolkit.ToString(payload.Get("browserid", ""))
	sort := payload.Get("sort")
	search := payload.Get("search")
	_ = search
	take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)

	TblName := toolkit.M{}
	payload.Unset("browserid")
	//sorter = ""
	if sort != nil {
		tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
		fmt.Printf("====== sort %#v\n", tmsort["dir"])
		if tmsort["dir"] == "asc" {
			sorter = tmsort["field"].(string)
		} else if tmsort["dir"] == "desc" {
			sorter = "-" + tmsort["field"].(string)
		} else if tmsort["dir"] == nil {
			sorter = " "
		}
	} else {
		sorter = " "
	}

	dataDS := new(colonycore.DataBrowser)
	err := colonycore.Get(dataDS, _id)
	if err != nil {
		return 0, nil, nil, err
	}

	dataConn := new(colonycore.Connection)
	err = colonycore.Get(dataConn, dataDS.ConnectionID)
	if err != nil {
		return 0, nil, nil, err
	}

	if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
		return 0, nil, nil, err
	}

	connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
	if err != nil {
		return 0, nil, nil, err
	}

	if dataDS.QueryType == "" {
		TblName.Set("from", dataDS.TableNames)
		payload.Set("from", dataDS.TableNames)
	} else if dataDS.QueryType == "Dbox" {
		getTableName := toolkit.M{}
		toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
		payload.Set("from", getTableName.Get("from").(string))

		if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
			payload.Set("select", getTableName.Get("select").(string))
		}
	} else if dataDS.QueryType == "SQL" {
		var QueryString string
		if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
			QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
		} else if dataConn.Driver == "mssql" {
			QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
				toolkit.ToString(take) + " ROWS ONLY "

		} else if dataConn.Driver == "postgres" {
			QueryString = " LIMIT " + toolkit.ToString(take) +
				" OFFSET " + toolkit.ToString(skip)
		}
		stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
		payload.Set("freetext", stringQuery)
		// toolkit.Println(stringQuery)
	}

	qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
	query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)

	var selectfield string
	for _, metadata := range dataDS.MetaData {
		tField := metadata.Field
		if payload.Has(tField) {
			selectfield = toolkit.ToString(tField)
			if toolkit.IsSlice(payload[tField]) {
				query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
				qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
			} else if !toolkit.IsNilOrEmpty(payload[tField]) {
				var hasPattern bool
				for _, val := range querypattern {
					if strings.Contains(toolkit.ToString(payload[tField]), val) {
						hasPattern = true
					}
				}
				if hasPattern {
					query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
					qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
				} else {
					switch toolkit.ToString(metadata.DataType) {
					case "int":
						query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
					case "float32":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
					case "float64":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
					default:
						query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
						qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
					}
				}
			}
		}
	}

	if hasLookup && selectfield != "" {
		if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
			query = query.Select(selectfield)
			qcount = qcount.Select(selectfield)
		} else {
			query = query.Select(selectfield).Group(selectfield)
			qcount = qcount.Select(selectfield).Group(selectfield)
		}

	}

	ccount, err := qcount.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer ccount.Close()

	dcount := ccount.Count()

	cursor, err := query.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer cursor.Close()

	data := []toolkit.M{}
	cursor.Fetch(&data, 0, false)

	if err != nil {
		return 0, nil, nil, err
	}

	if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
		!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		dataMongo := []toolkit.M{}
		for _, val := range data {
			mVal, _ := toolkit.ToM(val.Get("_id"))
			dataMongo = append(dataMongo, mVal)
		}
		data = dataMongo
	} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		/*distinct value for flat file*/
		dataFlat := []toolkit.M{}
		var existingVal = []string{""}
		for _, val := range data {
			valString := toolkit.ToString(val.Get(selectfield))
			if !toolkit.HasMember(existingVal, valString) {
				dataFlat = append(dataFlat, val)
				existingVal = append(existingVal, valString)
			}
		}
		data = dataFlat
	}

	return dcount, data, dataDS, nil
}