func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
var source []toolkit.M
var lower, upper int
lower = c.currentIndex
upper = lower + n
if n == 0 {
if c.where == nil {
upper = len(c.q.data)
} else {
upper = len(c.indexes)
}
} else if n == 1 {
upper = lower + 1
} else {
upper = lower + n
if c.where == nil {
if upper > len(c.q.data) {
upper = len(c.q.data)
}
} else {
if upper > len(c.indexes) {
upper = len(c.indexes)
}
}
}
if c.where == nil {
source = c.q.data[lower:upper]
} else {
for _, v := range c.indexes[lower:upper] {
/*
toolkit.Printf("Add index: %d. Source info now: %s \n", v, func() string {
var ret []string
for _, id := range source {
ret = append(ret, id.Get("_id").(string))
}
return strings.Join(ret, ",")
}())
*/
if v < len(c.q.data) {
source = append(source, c.q.data[v])
}
}
}
var e error
if n == 1 {
e = toolkit.Serde(&source[0], m, "json")
} else {
e = toolkit.Serde(&source, m, "json")
}
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
//toolkit.Printf("Data: %s\nLower, Upper = %d, %d\nSource: %s\nResult:%s\n\n", toolkit.JsonString(c.q.data), lower, upper, toolkit.JsonString(source), toolkit.JsonString(m))
return nil
}
func TestHiveExec(t *testing.T) {
q := "select * from sample_07 limit 1;"
// x := "select * from sample_07 limit 3;"
DoSomething := func(res HiveResult) (e error) {
toolkit.Serde(res, &res.ResultObj, "json")
log.Printf("result: \n%v\n", res.ResultObj)
return
}
/*DoElse := func(res HiveResult) (e error) {
tmp := toolkit.M{}
toolkit.Serde(res, &res.ResultObj, "json")
log.Printf("limit 3: \n%v\n", tmp)
return
}*/
h.Conn.FnReceive = DoSomething
h.Conn.Open()
h.Exec(q)
/*h.Conn.Wait()
h.Conn.FnReceive = DoElse
h.Exec(x)
h.Conn.Wait()
h.Conn.Open()
h.Exec(q)
h.Conn.FnReceive = DoElse
h.Exec(x)
h.Conn.Close()
/*h.Conn.Exec = true
h.Conn.Open()
h.Conn.FnReceive = DoSomething
h.Exec(q)
h.Conn.FnReceive = DoElse
h.Exec(x)
h.Conn.Exec = false
var res []toolkit.M
e := h.Populate(q, &res)
log.Printf("res: %v\n", res)
log.Printf("e: %v\n", e)
h.Conn.Close()
}*/
/*e := h.Populate(q, &res)
log.Printf("populate res: \n%v\n", res)
log.Printf("populate e: \n%v\n", e)*/
h.Conn.Close()
}
func (a *GroupController) SaveGroupConfigLdap(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o := new(colonycore.Ldap)
o.ID = payload["Address"].(string)
o.Address = payload["Address"].(string)
o.BaseDN = payload["BaseDN"].(string)
o.FilterGroup = payload["Filter"].(string)
o.Username = payload["Username"].(string)
//o.Password = payload["Password"].(string)
err = toolkit.Serde(payload["Attribute"], &o.AttributesGroup, "json")
if err != nil {
return helper.CreateResult(false, err.Error(), "error")
}
err = colonycore.Save(o)
if err != nil {
return helper.CreateResult(false, o, err.Error())
}
return helper.CreateResult(true, o, "")
}
func (a *GroupController) FindUserLdap(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
addr := payload["Address"].(string)
basedn := payload["BaseDN"].(string)
filter := payload["Filter"].(string)
username := payload["Username"].(string)
password := payload["Password"].(string)
var attr []string
err = toolkit.Serde(payload["Attribute"], &attr, "json")
if err != nil {
return helper.CreateResult(false, err, "error")
}
param := toolkit.M{}
param.Set("username", username)
param.Set("password", password)
param.Set("attributes", attr)
arrm, err := acl.FindDataLdap(addr, basedn, filter, param)
if err != nil {
return helper.CreateResult(false, err, "error")
}
return helper.CreateResult(true, arrm, "success")
}
func (a *UserController) TestFindUserLdap(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
addr := toolkit.ToString(payload["Address"]) //192.168.0.200:389
basedn := toolkit.ToString(payload["BaseDN"]) //DC=eaciit,DC=local
filter := toolkit.ToString(payload["Filter"]) //(&(objectclass=person)(objectclass=organizationalPerson)(cn=*))
var attr []string
err = toolkit.Serde(payload["Attribute"], &attr, "json")
if err != nil {
return helper.CreateResult(true, err, "error")
}
param := toolkit.M{}
param.Set("username", toolkit.ToString(payload["Username"])) //Alip Sidik
param.Set("password", toolkit.ToString(payload["Password"])) //Password.1
// param.Set("attributes", []string{"cn", "givenName"})
param.Set("attributes", attr)
arrtkm, err := acl.FindDataLdap(addr, basedn, filter, param)
if err != nil {
return helper.CreateResult(true, err, "error")
}
return helper.CreateResult(true, arrtkm, "sukses")
}
func (b *BaseController) InsertBulk(result []tk.M, m orm.IModel, wg *sync.WaitGroup) {
var datas []orm.IModel
for _, i := range result {
valueType := reflect.TypeOf(m).Elem()
for f := 0; f < valueType.NumField(); f++ {
field := valueType.Field(f)
bsonField := field.Tag.Get("bson")
jsonField := field.Tag.Get("json")
if jsonField != bsonField && field.Name != "RWMutex" && field.Name != "ModelBase" {
i.Set(field.Name, GetMgoValue(i, bsonField))
}
switch field.Type.Name() {
case "string":
if GetMgoValue(i, bsonField) == nil {
i.Set(field.Name, "")
}
break
case "Time":
if GetMgoValue(i, bsonField) == nil {
i.Set(field.Name, time.Time{})
} else {
i.Set(field.Name, GetMgoValue(i, bsonField).(time.Time).UTC())
}
break
default:
break
}
}
newPointer := getNewPointer(m)
e := tk.Serde(i, newPointer, "json")
datas = append(datas, newPointer)
if e != nil {
tk.Printf("\n----------- ERROR -------------- \n %v \n\n %#v \n\n %#v \n------------------------- \n", e.Error(), i, newPointer)
wg.Done()
}
}
if nil != datas {
muinsert.Lock()
for {
e := b.Ctx.InsertBulk(datas)
if e == nil {
ctn := len(result)
idx += ctn
tk.Printf("saved: %v data(s)\n", idx)
break
} else {
b.Ctx.Connection.Connect()
}
}
muinsert.Unlock()
}
wg.Done()
}
func (b *BaseController) Insert(result []tk.M, m orm.IModel, wg *sync.WaitGroup) {
// muinsert := &sync.Mutex{}
for _, i := range result {
valueType := reflect.TypeOf(m).Elem()
for f := 0; f < valueType.NumField(); f++ {
field := valueType.Field(f)
bsonField := field.Tag.Get("bson")
jsonField := field.Tag.Get("json")
if jsonField != bsonField && field.Name != "RWMutex" && field.Name != "ModelBase" {
i.Set(field.Name, GetMgoValue(i, bsonField))
}
switch field.Type.Name() {
case "string":
if GetMgoValue(i, bsonField) == nil {
i.Set(field.Name, "")
}
break
case "Time":
if GetMgoValue(i, bsonField) == nil {
i.Set(field.Name, time.Time{})
} else {
i.Set(field.Name, GetMgoValue(i, bsonField).(time.Time).UTC())
}
break
default:
break
}
}
newPointer := getNewPointer(m)
e := tk.Serde(i, newPointer, "json")
var newId int64
for index := 0; index < retry; index++ {
muinsert.Lock()
newId, e = b.Ctx.InsertOut(newPointer)
_ = newId
muinsert.Unlock()
if e == nil {
wg.Done()
break
} else {
b.Ctx.Connection.Connect()
}
}
if e != nil {
tk.Printf("\n----------- ERROR -------------- \n %v \n\n %#v \n\n %#v \n------------------------- \n", e.Error(), i, newPointer)
wg.Done()
}
}
wg.Done()
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
end := c.start + n
if end > c.count || n == 0 {
end = c.count
}
if c.start >= c.count {
return errors.New("No more data to fetched!")
}
e := toolkit.Serde(c.data[c.start:end], m, "json")
if e != nil {
return err.Error(packageName, modCursor, "Fetch", e.Error())
}
return nil
}
func (m *MigrateData) InsertOut(in tk.M, mod orm.IModel) (out int64, e error) {
muinsert := &sync.Mutex{}
valueType := reflect.TypeOf(mod).Elem()
for f := 0; f < valueType.NumField(); f++ {
field := valueType.Field(f)
bsonField := field.Tag.Get("bson")
jsonField := field.Tag.Get("json")
if jsonField != bsonField && field.Name != "RWMutex" && field.Name != "ModelBase" {
in.Set(field.Name, GetMgoValue(in, bsonField))
}
switch field.Type.Name() {
case "string":
if GetMgoValue(in, bsonField) == nil {
in.Set(field.Name, "")
}
break
case "Time":
if GetMgoValue(in, bsonField) == nil {
in.Set(field.Name, time.Time{})
} else {
in.Set(field.Name, GetMgoValue(in, bsonField).(time.Time).UTC())
}
break
default:
break
}
}
e = tk.Serde(in, mod, "json")
if e != nil {
return
}
muinsert.Lock()
out, e = m.BaseController.SqlCtx.InsertOut(mod)
muinsert.Unlock()
return out, e
}
//func Find(ms []toolkit.M, filters []*Filter) (output []int) {
func Find(ms interface{}, filters []*Filter) (output []int) {
//-- is not a slice
if !toolkit.IsSlice(ms) {
toolkit.Println("Data is not slice")
return []int{}
}
//toolkit.Printf("Find:%s Filter:%s\n", toolkit.JsonString(ms), toolkit.JsonString(filters))
sliceLen := toolkit.SliceLen(ms)
for i := 0; i < sliceLen; i++ {
var v toolkit.M
item := toolkit.SliceItem(ms, i)
e := toolkit.Serde(item, &v, "json")
if e == nil {
match := MatchM(v, filters)
if match {
output = append(output, i)
}
} else {
//toolkit.Println("Serde Fail: ", e.Error(), " Data: ", item)
}
}
return
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
c.Close()
}
e := c.prepIter()
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
/*if c.jsonSelect == nil {
return errorlib.Error(packageName, modCursor, "Fetch", "Iter object is not yet initialized")
}*/
// var mData []interface{}
datas := []toolkit.M{}
dataJson := []toolkit.M{}
dec := json.NewDecoder(strings.NewReader(string(c.readFile)))
dec.Decode(&datas)
if n == 0 {
whereFieldsToMap, e := toolkit.ToM(c.whereFields)
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
b := c.getCondition(whereFieldsToMap)
var foundSelected = toolkit.M{}
var foundData = []toolkit.M{}
var getRemField = toolkit.M{}
if c.isWhere {
if b {
for _, v := range datas {
for i, subData := range v {
getRemField[i] = i //append(getRemField, i)
for _, vWhere := range whereFieldsToMap {
for _, subWhere := range vWhere.([]interface{}) {
for _, subsubWhere := range subWhere.(map[string]interface{}) {
if len(c.jsonSelect) == 0 {
if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
dataJson = append(dataJson, v)
}
} else {
if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
foundData = append(foundData, v)
}
}
}
}
}
}
}
itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect)
if len(foundData) > 0 {
var found toolkit.M
for _, found = range foundData {
for _, remitem := range itemToRemove {
found.Unset(remitem)
}
dataJson = append(dataJson, found)
}
}
} else {
for _, v := range datas {
for _, v2 := range v {
for _, vWhere := range c.whereFields.(toolkit.M) {
if reflect.ValueOf(v2).Kind() == reflect.String {
if strings.ToLower(v2.(string)) == strings.ToLower(vWhere.(string)) {
if len(c.jsonSelect) == 0 {
dataJson = append(dataJson, v)
} else {
foundData = append(foundData, v)
}
}
}
}
}
}
if len(foundData) > 0 {
for _, found := range foundData {
for i, subData := range found {
for _, selected := range c.jsonSelect {
if strings.ToLower(selected) == strings.ToLower(i) {
foundSelected[i] = subData
} else if selected == "*" {
foundSelected[i] = subData
}
}
}
}
dataJson = append(dataJson, foundSelected)
}
}
// toolkit.Unjson(toolkit.Jsonify(dataJson), m)
toolkit.Serde(dataJson, m, "json")
} else {
if c.jsonSelect == nil {
toolkit.Unjson(toolkit.Jsonify(datas), m)
} else {
isSelectedFields := false
for _, selectField := range c.jsonSelect {
if selectField == "*" {
// toolkit.Unjson(toolkit.Jsonify(datas), m)
toolkit.Serde(datas, m, "json")
} else {
isSelectedFields = true
}
}
if isSelectedFields {
for _, v := range datas {
for i, _ := range v {
getRemField[i] = i
}
}
itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect)
for _, found := range datas {
toMap := toolkit.M(found)
for _, remitem := range itemToRemove {
toMap.Unset(remitem)
}
dataJson = append(dataJson, toMap)
}
// toolkit.Unjson(toolkit.Jsonify(dataJson), m)
toolkit.Serde(dataJson, m, "json")
}
}
}
} else if n > 0 {
fetched := 0
fetching := true
c.Connection().(*Connection).FetchSession()
c.tempPathFile = c.Connection().(*Connection).tempPathFile
///read line
fetchFile, e := os.OpenFile(c.Connection().(*Connection).tempPathFile, os.O_RDWR, 0)
defer fetchFile.Close()
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", "Fetch file", e.Error())
}
c.fetchSession = fetchFile
scanner := bufio.NewScanner(fetchFile)
lines := 0
for scanner.Scan() {
lines++
}
if lines > 0 {
fetched = lines
n = n + lines
}
for fetching {
var dataM = toolkit.M{}
if c.jsonSelect == nil {
dataJson = append(dataJson, datas[fetched])
} else {
for _, selectField := range c.jsonSelect {
if selectField == "*" {
dataJson = append(dataJson, datas[fetched])
} else {
dataM.Set(selectField, datas[fetched][selectField])
if len(dataM) == len(c.jsonSelect) {
dataJson = append(dataJson, dataM)
}
}
}
}
// toolkit.Unjson(toolkit.Jsonify(dataJson), m)
toolkit.Serde(dataJson, m, "json")
io.WriteString(fetchFile, toolkit.JsonString(dataM)+"\n")
fetched++
if fetched == n {
fetching = false
}
}
}
fmt.Sprintln("")
return nil
}
func (s *ServerController) SaveServers(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
r.Request.ParseMultipartForm(32 << 20)
r.Request.ParseForm()
path := filepath.Join(EC_DATA_PATH, "server", "log")
log, _ := toolkit.NewLog(false, true, path, "log-%s", "20060102-1504")
data := new(colonycore.Server)
if r.Request.FormValue("serviceSSH[type]") == "File" {
log.AddLog("Get forms", "INFO")
dataRaw := map[string]interface{}{}
err := r.GetForms(&dataRaw)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
log.AddLog("Serding data", "INFO")
err = toolkit.Serde(dataRaw, &data, "json")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
} else {
log.AddLog("Get payload", "INFO")
err := r.GetPayload(&data)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
}
if data.ServiceSSH.Type == "File" {
log.AddLog("Fetching public key", "INFO")
reqFileName := "privatekey"
file, _, err := r.Request.FormFile(reqFileName)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
if file != nil {
log.AddLog("Saving public key", "INFO")
data.ServiceSSH.File = filepath.Join(EC_DATA_PATH, "server", "privatekeys", data.ID)
_, _, err = helper.FetchThenSaveFile(r.Request, reqFileName, data.ServiceSSH.File)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
}
}
if data.IsAccessValid("node") && data.IsColonyServer {
if data.OS == "linux" {
if err := data.InstallColonyOnLinux(log); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
} else {
if err := data.InstallColonyOnWindows(log); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
data.DetectInstalledLang()
data.DetectService()
data.UpdateInternalAppDeploymentStatus("add")
log.AddLog("Restart sedotand", "INFO")
if _, err := data.ToggleSedotanService("start stop", data.ID); err != nil {
log.AddLog(err.Error(), "ERROR")
}
}
if data.IsAccessValid("hdfs") {
log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
hdfsConfig := hdfs.NewHdfsConfig(data.ServiceHDFS.Host, data.ServiceHDFS.User)
hdfsConfig.Password = data.ServiceHDFS.Pass
hadeepes, err := hdfs.NewWebHdfs(hdfsConfig)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
_, err = hadeepes.List("/")
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
hadeepes.Config.TimeOut = 5 * time.Millisecond
hadeepes.Config.PoolSize = 100
}
runsrv, e := new(colonycore.ServerService).GetAll()
if e != nil {
return helper.CreateResult(false, nil, e.Error())
}
svr, _, a := data.Connect()
if a != nil {
return helper.CreateResult(false, nil, a.Error())
}
for _, sr := range runsrv {
b, e := svr.RunCommandSshAsMap(fmt.Sprintf("which %s", sr.ID))
if e != nil {
return helper.CreateResult(false, nil, e.Error())
}
res := strings.TrimSpace(b[0].Output)
i := strings.Index(res, "/")
if i > -1 {
sr.IsInstalled = true
}
hh := colonycore.ServerService{ID: sr.ID, Name: sr.Name, IsInstalled: sr.IsInstalled}
data.OtherServices = append(data.OtherServices, &hh)
}
log.AddLog(fmt.Sprintf("Saving data ID: %s", data.ID), "INFO")
err := colonycore.Save(data)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, nil, "")
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
fmt.Println(c.QueryString)
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
tableData := []toolkit.M{}
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
// valueint := values
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
// rows.Scan(valuePtrs...)
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// fmt.Println("Nilai val : ", val)
b, ok := val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
entry.Set(strings.ToLower(col), v)
// entry.Set(col, values[i])
// e = toolkit.DecodeByte(val.([]byte), v)
// toolkit.FromBytes(toolkit.ToBytes(val, ""), "", v)
// entry.Set(col, v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
fmt.Println("isi entry : ", entry[namaField], dataType)
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
fmt.Println("Nilai table data : ", tableData)
if e != nil {
return e
}
if n == 0 {
// *m.(*[]map[string]interface{}) = tableData
// toolkit.Unjson(toolkit.Jsonify(tableData), m)
e = toolkit.Serde(tableData, m, "json")
fmt.Println("Nilai Model : ", m)
} else {
end := c.start + n
if end > len(tableData) {
e = errors.New("index out of range")
} else {
// *m.(*[]map[string]interface{}) = tableData[0:n]
//toolkit.Unjson(toolkit.Jsonify(tableData[0:n]), m)
e = toolkit.Serde(tableData[0:n], m, "json")
}
}
return e
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
e := c.prepIter()
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
lineCount := 0
//=============================
// fmt.Println("Qursor 191 : ", c.headerColumn)
for {
iv := reflect.New(v).Interface()
isAppend := true
// c.count += 1
recData := toolkit.M{}
appendData := toolkit.M{}
dataTemp, e := c.reader.Read()
for i, val := range dataTemp {
orgname := c.headerColumn[i].name
lowername := strings.ToLower(c.headerColumn[i].name)
switch c.headerColumn[i].dataType {
case "int":
recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
case "float":
decimalPoint := len(val) - (strings.Index(val, ".") + 1)
recData[lowername] = cast.ToF64(val, decimalPoint, cast.RoundingAuto)
case "date":
recData[lowername] = toolkit.String2Date(val, c.headerColumn[i].format) // Just for test
// fmt.Printf("FOR DEBUG : %v \n", c.headerColumn[i].format)
default:
recData[lowername] = val
}
if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
appendData[orgname] = recData[lowername]
} else {
if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
appendData[orgname] = recData[lowername]
}
}
}
isAppend = c.ConditionVal.getCondition(recData)
if isAppend {
c.count += 1
}
if c.count <= c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
isAppend = false
}
// fmt.Printf("%v - %v \n", v.Elem().Kind(), toolkit.TypeName(v))
if v.Kind() == reflect.Struct || v.Elem().Kind() == reflect.Struct {
tv := v
if v.Elem().Kind() == reflect.Struct {
tv = v.Elem()
}
for i := 0; i < tv.NumField(); i++ {
str := tv.Field(i).Name
fcond := false
if appendData.Has(str) {
fcond = true
} else if appendData.Has(strings.ToLower(str)) {
fcond = true
str = strings.ToLower(str)
} else if strings.ToLower(str) == "id" && appendData.Has("_id") {
str = "_id"
fcond = true
}
if fcond {
switch tv.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(str, cast.ToInt(appendData[str], cast.RoundingAuto))
case reflect.String:
appendData.Set(str, toolkit.ToString(appendData[str]))
case reflect.Float64:
tstr := toolkit.ToString(appendData[str])
decimalPoint := len(tstr) - (strings.Index(tstr, ".") + 1)
appendData.Set(str, toolkit.ToFloat64(tstr, decimalPoint, toolkit.RoundingAuto))
}
}
}
}
if e == io.EOF {
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
break
} else if e != nil {
return errorlib.Error(packageName, modCursor,
"Fetch", e.Error())
}
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
if n > 0 {
if lineCount >= n {
break
}
}
}
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
if ivs.Len() > 0 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
if !toolkit.IsPointer(m) {
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
if outputType == CSV {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
splitted := strings.Split(data, "\t")
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
c.Close()
}
var source []toolkit.M
var lower, upper, lenData, lenIndex int
if !c.isWhere {
lenData = len(c.datas)
if c.lastFetched == 0 {
c.maxIndex = lenData
}
} else {
lenIndex = len(c.indexes)
if c.lastFetched == 0 {
c.maxIndex = lenIndex
}
}
if c.lastFetched == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/
c.maxIndex = c.skip + c.take
}
lower = c.lastFetched
upper = lower + n
if c.skip > 0 && c.lastFetched < 1 {
lower += c.skip
}
if n == 0 {
if !c.isWhere {
upper = lenData
} else {
upper = lenIndex
}
if c.take > 0 {
upper = lower + c.take
}
} else if n == 1 {
upper = lower + 1
} else {
upper = lower + n
if c.take > 0 && n > c.take {
upper = lower + c.take
}
}
if !c.isWhere {
if toolkit.SliceLen(c.datas) > 0 {
if lower >= lenData {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenData {
upper = lenData
}
}
} else {
if toolkit.SliceLen(c.indexes) > 0 {
if lower >= lenIndex {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenIndex {
upper = lenIndex
}
}
}
if upper >= c.maxIndex {
upper = c.maxIndex
}
if !c.isWhere {
source = c.datas[lower:upper]
} else {
for _, v := range c.indexes[lower:upper] {
if v < len(c.datas) {
source = append(source, c.datas[v])
}
}
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
source = c.GetSelected(source, c.jsonSelect)
}
var e error
e = toolkit.Serde(&source, m, "json")
c.lastFetched = upper
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
// var first, last int
// dataJson := []toolkit.M{}
// c.count = len(c.datas)
// c.lastFetched = c.count
// if c.skip > 0 {
// if c.take > 0 {
// c.count = c.skip + c.take
// } else {
// first = c.skip
// }
// } else {
// c.count = c.take
// }
// if c.take > 0 {
// if c.take == c.skip {
// first = c.skip
// } else {
// first = c.count - c.take
// }
// }
// // toolkit.Printf("first = skip>%v last = take>%v lastfetched>%v count>%v\n", first, last, c.lastFetched, c.count)
// if n == 0 {
// last = c.count
// if c.lastFetched <= c.count || c.count == 0 {
// last = c.lastFetched
// }
// } else if n > 0 {
// switch {
// case c.lastFetched == 0:
// last = n
// c.lastFetched = n
// case n > c.lastFetched || n < c.lastFetched || n == c.lastFetched:
// first = c.lastFetched
// last = c.lastFetched + n
// c.lastFetched = last
// if c.lastFetched > c.count {
// if first > c.count {
// return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
// }
// last = c.count
// c.lastFetched = last
// }
// }
// if first > last {
// return errorlib.Error(packageName, modCursor, "Fetch", "Wrong fetched data!")
// }
// }
// if c.isWhere {
// i := dbox.Find(c.datas, c.whereFields)
// c.lastFetched = len(i)
// if c.lastFetched < c.count || c.count == 0 {
// last = c.lastFetched
// }
// for _, index := range i[first:last] {
// dataJson = append(dataJson, c.datas[index])
// }
// } else {
// dataJson = c.datas[first:last]
// }
// if toolkit.SliceLen(c.jsonSelect) > 0 {
// dataJson = c.GetSelected(dataJson, c.jsonSelect)
// }
// e := toolkit.Serde(dataJson, m, "json")
// if e != nil {
// return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
// }
return nil
}
func (q *Query) Exec(parm toolkit.M) error {
var (
e error
updatedValue, dataMs []toolkit.M
dataM toolkit.M
)
filters, e := q.Filters(parm)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec", e.Error())
}
if parm == nil {
parm = toolkit.M{}
}
data := parm.Get("data", nil)
filePath := q.Connection().(*Connection).filePath
commandType := filters.Get("cmdType").(string)
hasWhere := filters.Has("where")
hasCmdType := toolkit.M{}
hasData := parm.Has("data")
getWhere := filters.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
dataIsSlice := toolkit.IsSlice(data)
if dataIsSlice {
e = toolkit.Unjson(toolkit.Jsonify(data), &dataMs)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
for _, v := range dataMs {
id := toolkit.Id(v)
idF := toolkit.IdField(v)
if toolkit.IsNilOrEmpty(id) {
return errorlib.Error(packageName, modCursor+".Exec", commandType, "Unable to find ID in slice data")
} else {
getWhere = []*dbox.Filter{dbox.Eq(idF, id)}
}
}
} else {
dataM, e = toolkit.ToM(data)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: "+commandType, "Unable to Map, error: "+e.Error())
}
id := toolkit.Id(dataM)
if !toolkit.IsNilOrEmpty(id) {
getWhere = []*dbox.Filter{dbox.Eq(toolkit.IdField(dataM), id)}
}
}
var dataMaps []toolkit.M
q.ReadFile(&dataMaps, filePath)
if commandType == dbox.QueryPartInsert {
hasCmdType.Set("hasInsert", true)
if !hasData {
return errorlib.Error(packageName, modCursor+".Exec", commandType, "Sorry data not found!, unable to insert data")
}
result := dbox.Find(dataMaps, getWhere)
if len(result) > 0 {
return errorlib.Error(packageName, modCursor+".Exec", commandType, "ID already exist, unable insert data ")
}
if dataIsSlice {
var sliceData []toolkit.M
for _, v := range dataMs {
sliceData = finUpdateObj(dataMaps, v, "insert")
}
updatedValue = sliceData
} else {
updatedValue = finUpdateObj(dataMaps, dataM, "insert")
}
} else if commandType == dbox.QueryPartUpdate {
hasCmdType.Set("hasUpdate", true)
if !hasData {
return errorlib.Error(packageName, modCursor+".Exec", commandType, "Sorry data not found!, unable to update data")
}
if hasWhere {
var indexes []interface{}
whereIndex := dbox.Find(dataMaps, getWhere)
indexes = toolkit.ToInterfaceArray(&whereIndex)
// toolkit.Printf("whereIndex>%v indexes%v\n", whereIndex, indexes)
var dataUpdate toolkit.M
var updateDataIndex int
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
data, e = toolkit.ToM(data)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
}
for i, v := range dataMaps {
if toolkit.HasMember(indexes, i) || !hasWhere {
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
updateDataIndex++
}
dataOrigin := dataMaps[i]
toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"})
toolkit.Serde(dataOrigin, &v, "")
dataMaps[i] = v
}
}
updatedValue = dataMaps
} else {
updatedValue = finUpdateObj(dataMaps, dataM, "update")
}
} else if commandType == dbox.QueryPartDelete {
hasCmdType.Set("hasDelete", true)
// if multi {
if hasWhere {
result := dbox.Find(dataMaps, getWhere)
if len(result) > 0 {
for i, v := range dataMaps {
if toolkit.HasMember(result, i) == false {
updatedValue = append(updatedValue, v)
}
}
}
} else {
updatedValue = []toolkit.M{}
}
} else if commandType == dbox.QueryPartSave {
hasCmdType.Set("hasSave", true)
if !hasData {
return errorlib.Error(packageName, modCursor+".Exec", commandType, "Sorry data not found!, unable to update data")
}
q.dataType = "save"
q.whereData = append(q.whereData, getWhere...)
q.sliceData = append(q.sliceData, dataM)
}
if hasCmdType.Has("hasInsert") || hasCmdType.Has("hasUpdate") || hasCmdType.Has("hasDelete") {
e = q.WriteFile(updatedValue)
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
}
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
// var e error
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// b, ok := val.([]byte)
// // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val))
// var out interface{}
// e = toolkit.Unjson(b, &out)
// // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e)
// if e != nil {
// ok = false
// }
// if ok {
// v = out
// toolkit.Println("error OK :: ", ok, " :v :", v)
// } else {
// toolkit.Println("error OK :: ", ok, " :b :", b)
// v = string(b)
// }
v = val
entry.Set(strings.ToLower(col), v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
// toolkit.Println("... ::: ", tableData)
maxIndex := toolkit.SliceLen(tableData)
var e2 error
if e2 != nil {
return e2
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e2 = errors.New("No more data to fetched!")
} else {
e2 = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e2
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
if len(c.indexes) == 0 {
return nil
}
var lower, upper int
lower = c.currentIndex
if lower < c.skip {
lower = c.skip
}
upper = lower + n
if upper > c.limit && c.limit > 0 {
upper = c.limit
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
datatemp, ef := c.q.reader.Read()
c.realIndex += 1
if c.indexes[c.currentIndex] != c.realIndex {
continue
}
c.currentIndex += 1
for i, val := range datatemp {
if len(c.fields) == 0 || c.fields.Has("*") || c.fields.Has(c.q.headerColumn[i].name) {
appendData[c.q.headerColumn[i].name] = val
}
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if appendData.Has(v.Field(i).Name) {
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
}
}
}
}
isAppend := lower < c.currentIndex && upper >= c.currentIndex
if (ef == io.EOF || ef == nil) && isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if ef != nil && ef != io.EOF {
return errorlib.Error(packageName, modCursor, "Fetch", ef.Error())
} else if ef == io.EOF || (ivs.Len() >= n && n > 0) {
break
}
}
// if e != nil {
// return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
// }
if n == 1 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
// log.Printf("start parse:\n")
if !toolkit.IsPointer(m) {
// log.Printf("not pointer\n")
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
// log.Printf("pointer\n")
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
// log.Printf("outputType: %v\n", outputType)
if outputType == CSV {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
}
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for _, val := range header {
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
// log.Printf("v: %v\n", v)
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
// log.Printf("ivs: %v\n", ivs)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
/*log.Printf("data: %v\n", data)
log.Printf("iv: %v\n", iv)*/
splitted := strings.Split(data, "\t")
/*log.Printf("appendData: %v\n", appendData)
log.Printf("kind: %v\n", v.Kind())
log.Printf("test: %v", fmt.Sprintf("%v", v))
//log.Printf("v.Name: %T\n", v)
if fmt.Sprintf("%v", v) == "reflect.Value" {
log.Printf("else: %v\n", "reflect.Value")
for _, val := range header {
log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
log.Printf("appendData: %v\n", appendData)
} else */
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
// log.Printf("struct: %v\n", v.Kind())
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
// log.Printf("i: %v\n", i)
// log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name")))
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
// log.Printf("valthis: %v\n", valthis)
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
// log.Printf("else: %v\n", v.Kind())
for _, val := range header {
// log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
// log.Printf("iv result: %v\n", iv)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
// log.Printf("ivs result: %v\n", ivs)
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
// log.Printf("result: %v\n", m)
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
c.Close()
}
var first, last int
dataJson := []toolkit.M{}
var datas []toolkit.M
toolkit.Unjson(c.readFile, &datas)
c.count = len(datas)
if n == 0 {
last = c.count
} else if n > 0 {
switch {
case c.lastFeteched == 0:
last = n
c.lastFeteched = n
case n > c.lastFeteched || n < c.lastFeteched || n == c.lastFeteched:
first = c.lastFeteched
last = c.lastFeteched + n
c.lastFeteched = last
if c.lastFeteched > c.count {
if first > c.count {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
last = c.count
}
// toolkit.Printf("first>%v last>%v lastfetched>%v count>%v\n", first, last, c.lastFeteched, c.count)
}
}
if c.isWhere {
i := dbox.Find(datas, c.whereFields)
last = len(i)
for _, index := range i[first:last] {
dataJson = append(dataJson, datas[index])
}
} else {
dataJson = datas[first:last]
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
var getRemField = toolkit.M{}
for _, v := range dataJson {
for i, _ := range v {
getRemField.Set(i, i)
}
if c.jsonSelect[0] != "*" {
fields := c.removeDuplicatesUnordered(getRemField, c.jsonSelect)
for _, field := range fields {
v.Unset(field)
}
}
}
}
e := toolkit.Serde(dataJson, m, "json")
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
return nil
}
func (s *ServerController) SaveServers(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
r.Request.ParseMultipartForm(32 << 20)
r.Request.ParseForm()
path := filepath.Join(EC_DATA_PATH, "server", "log")
log, _ := toolkit.NewLog(false, true, path, "log-%s", "20060102-1504")
data := new(colonycore.Server)
if r.Request.FormValue("sshtype") == "File" {
log.AddLog("Get forms", "INFO")
dataRaw := map[string]interface{}{}
err := r.GetForms(&dataRaw)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
log.AddLog("Serding data", "INFO")
err = toolkit.Serde(dataRaw, &data, "json")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
} else {
log.AddLog("Get payload", "INFO")
err := r.GetPayload(&data)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
}
if data.SSHType == "File" {
log.AddLog("Fetching public key", "INFO")
reqFileName := "privatekey"
file, _, err := r.Request.FormFile(reqFileName)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
if file != nil {
log.AddLog("Saving public key", "INFO")
data.SSHFile = filepath.Join(EC_DATA_PATH, "server", "privatekeys", data.ID)
_, _, err = helper.FetchThenSaveFile(r.Request, reqFileName, data.SSHFile)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
}
}
oldData := new(colonycore.Server)
log.AddLog(fmt.Sprintf("Find server ID: %s", data.ID), "INFO")
cursor, err := colonycore.Find(new(colonycore.Server), dbox.Eq("_id", data.ID))
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
oldDataAll := []colonycore.Server{}
err = cursor.Fetch(&oldDataAll, 0, false)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
defer cursor.Close()
if len(oldDataAll) > 0 {
oldData = &oldDataAll[0]
}
if data.ServerType == "hdfs" {
log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
hadeepes, err := hdfs.NewWebHdfs(hdfs.NewHdfsConfig(data.Host, data.SSHUser))
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
_, err = hadeepes.List("/")
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
hadeepes.Config.TimeOut = 5 * time.Millisecond
hadeepes.Config.PoolSize = 100
return helper.CreateResult(true, nil, "")
}
log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
sshSetting, client, err := s.SSHConnect(data)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
defer client.Close()
if data.OS == "linux" {
setEnvPath := func() error {
cmd1 := `sed -i '/export EC_APP_PATH/d' ~/.bashrc`
log.AddLog(cmd1, "INFO")
sshSetting.GetOutputCommandSsh(cmd1)
cmd2 := `sed -i '/export EC_DATA_PATH/d' ~/.bashrc`
log.AddLog(cmd2, "INFO")
sshSetting.GetOutputCommandSsh(cmd2)
cmd3 := "echo 'export EC_APP_PATH=" + data.AppPath + "' >> ~/.bashrc"
log.AddLog(cmd3, "INFO")
sshSetting.GetOutputCommandSsh(cmd3)
cmd4 := "echo 'export EC_DATA_PATH=" + data.DataPath + "' >> ~/.bashrc"
log.AddLog(cmd4, "INFO")
sshSetting.GetOutputCommandSsh(cmd4)
return nil
}
if oldData.AppPath == "" || oldData.DataPath == "" {
cmdRmAppPath := fmt.Sprintf("rm -rf %s", data.AppPath)
log.AddLog(cmdRmAppPath, "INFO")
sshSetting.GetOutputCommandSsh(cmdRmAppPath)
cmdMkdirAppPath := fmt.Sprintf(`mkdir -p "%s"`, data.AppPath)
log.AddLog(cmdMkdirAppPath, "INFO")
_, err := sshSetting.GetOutputCommandSsh(cmdMkdirAppPath)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
cmdRmDataPath := fmt.Sprintf("rm -rf %s", data.DataPath)
log.AddLog(cmdRmDataPath, "INFO")
sshSetting.GetOutputCommandSsh(cmdRmDataPath)
cmdMkdirDataPath := fmt.Sprintf(`mkdir -p "%s"`, data.DataPath)
log.AddLog(cmdMkdirDataPath, "INFO")
_, err = sshSetting.GetOutputCommandSsh(cmdMkdirDataPath)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
appDistSrc := filepath.Join(EC_DATA_PATH, "dist", "app-root.zip")
err = sshSetting.SshCopyByPath(appDistSrc, data.AppPath)
log.AddLog(fmt.Sprintf("scp from %s to %s", appDistSrc, data.AppPath), "INFO")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
appDistSrcDest := filepath.Join(data.AppPath, "app-root.zip")
unzipAppCmd := fmt.Sprintf("unzip %s -d %s", appDistSrcDest, data.AppPath)
log.AddLog(unzipAppCmd, "INFO")
_, err = sshSetting.GetOutputCommandSsh(unzipAppCmd)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
rmTempAppPath := fmt.Sprintf("rm -rf %s", appDistSrcDest)
_, err = sshSetting.GetOutputCommandSsh(rmTempAppPath)
log.AddLog(rmTempAppPath, "INFO")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
dataDistSrc := filepath.Join(EC_DATA_PATH, "dist", "data-root.zip")
err = sshSetting.SshCopyByPath(dataDistSrc, data.DataPath)
log.AddLog(fmt.Sprintf("scp from %s to %s", dataDistSrc, data.DataPath), "INFO")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
dataDistSrcDest := filepath.Join(data.DataPath, "data-root.zip")
unzipDataCmd := fmt.Sprintf("unzip %s -d %s", dataDistSrcDest, data.DataPath)
log.AddLog(unzipDataCmd, "INFO")
_, err = sshSetting.GetOutputCommandSsh(unzipDataCmd)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
rmTempDataPath := fmt.Sprintf("rm -rf %s", dataDistSrcDest)
_, err = sshSetting.GetOutputCommandSsh(rmTempDataPath)
log.AddLog(rmTempDataPath, "INFO")
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
osArchCmd := "uname -m"
log.AddLog(osArchCmd, "INFO")
osArchRes, err := sshSetting.GetOutputCommandSsh(osArchCmd)
osArchRes = strings.TrimSpace(osArchRes)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
for _, each := range []string{"sedotand", "sedotans", "sedotanw"} {
src := filepath.Join(EC_APP_PATH, "cli", "dist", fmt.Sprintf("linux_%s", osArchRes), each)
dst := filepath.Join(data.AppPath, "cli", each)
rmSedotanCmd := fmt.Sprintf("rm -rf %s", dst)
log.AddLog(rmSedotanCmd, "INFO")
_, err := sshSetting.GetOutputCommandSsh(rmSedotanCmd)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
log.AddLog(fmt.Sprintf("scp %s to %s", src, dst), "INFO")
err = sshSetting.SshCopyByPath(src, dst)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
chmodCliCmd := fmt.Sprintf("chmod 755 %s", dst)
log.AddLog(chmodCliCmd, "INFO")
_, err = sshSetting.GetOutputCommandSsh(chmodCliCmd)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
}
checkPathCmd := fmt.Sprintf("ls %s", data.AppPath)
isPathCreated, err := sshSetting.GetOutputCommandSsh(checkPathCmd)
log.AddLog(checkPathCmd, "INFO")
if err != nil || strings.TrimSpace(isPathCreated) == "" {
errString := fmt.Sprintf("Invalid path. %s", err.Error())
log.AddLog(errString, "ERROR")
return helper.CreateResult(false, nil, errString)
}
if err := setEnvPath(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
} else if oldData.AppPath != data.AppPath {
moveDir := fmt.Sprintf(`mv %s %s`, oldData.AppPath, data.AppPath)
log.AddLog(moveDir, "INFO")
_, err := sshSetting.GetOutputCommandSsh(moveDir)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
if err := setEnvPath(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
} else if oldData.DataPath != data.DataPath {
moveDir := fmt.Sprintf(`mv %s %s`, oldData.DataPath, data.DataPath)
log.AddLog(moveDir, "INFO")
_, err := sshSetting.GetOutputCommandSsh(moveDir)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
if err := setEnvPath(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
} else {
// windows
}
log.AddLog(fmt.Sprintf("Saving data ID: %s", data.ID), "INFO")
err = colonycore.Save(data)
if err != nil {
log.AddLog(err.Error(), "ERROR")
return helper.CreateResult(false, nil, err.Error())
}
log.AddLog("Restart sedotand", "INFO")
_, err = s.ToggleSedotanService("start stop", data.ID)
if err != nil {
log.AddLog(err.Error(), "ERROR")
}
return helper.CreateResult(true, nil, "")
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
//toolkit.Printf("Command type: %s\n", commandType)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if setting.GetString("commandtype") == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
var dataM toolkit.M
var dataMs []toolkit.M
hasData := in.Has("data")
dataIsSlice := false
data := in.Get("data")
if toolkit.IsSlice(data) {
dataIsSlice = true
e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
}
hasWhere := setting.Has("where")
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if hasWhere && len(where) == 0 {
inWhere := in.Get("where")
if inWhere == nil {
hasWhere = false
where = nil
} else {
if !toolkit.IsSlice(inWhere) {
where = append(where, inWhere.(*dbox.Filter))
} else {
where = inWhere.([]*dbox.Filter)
}
}
}
if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartDelete,
dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
hasWhere = true
//toolkit.Println("check where")
if toolkit.IsSlice(data) {
ids := []interface{}{}
idField := ""
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data send is a slice, but its element has no ID")
}
dataCount := toolkit.SliceLen(data)
for i := 0; i < dataCount; i++ {
dataI := toolkit.SliceItem(data, i)
if i == 0 {
idField = toolkit.IdField(dataI)
}
ids = append(ids, toolkit.Id(dataI))
}
where = []*dbox.Filter{dbox.In(idField, ids)}
} else {
idfield := "_id"
id := toolkit.Id(data)
if !toolkit.IsNilOrEmpty(id) {
where = []*dbox.Filter{dbox.Eq(idfield, id)}
} else {
where = nil
hasWhere = false
}
}
}
/*
toolkit.Printf("CommandType: %s HasData: %v HasWhere: %v Where: %s\n",
commandType, hasData, hasWhere, toolkit.JsonString(where))
*/
e = q.openFile(commandType)
//toolkit.Printf(commandType+" Open File, found record: %d\nData:%s\n", len(q.data), toolkit.JsonString(q.data))
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
var indexes []interface{}
if hasWhere && commandType != dbox.QueryPartInsert {
whereIndex := dbox.Find(q.data, where)
indexes = toolkit.ToInterfaceArray(&whereIndex)
//toolkit.Printf("Where Index: %s Index:%s\n", toolkit.JsonString(whereIndex), toolkit.JsonString(indexes))
}
if commandType == dbox.QueryPartInsert {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
if !dataIsSlice {
dataMs = []toolkit.M{dataM}
}
//-- validate
for _, datam := range dataMs {
idField, idValue := toolkit.IdInfo(datam)
toolkit.Serde(dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}), &indexes, "")
if len(indexes) > 0 {
return err.Error(packageName, modQuery, "Exec: "+commandType, toolkit.Sprintf("Data %v already exist", idValue))
}
}
//-- insert the data
q.data = append(q.data, dataMs...)
} else if commandType == dbox.QueryPartUpdate {
//-- valida
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataUpdate toolkit.M
var updateDataIndex int
// if it is a slice then we need to update each data passed on its slice
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
}
var idField string
//toolkit.Printf("Indexes: %s\n", toolkit.JsonString(indexes))
for i, v := range q.data {
// update only data that match given inde
if toolkit.HasMember(indexes, i) || !hasWhere {
if idField == "" {
idField = toolkit.IdField(v)
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "No ID")
}
}
// If dataslice is sent, iterate f
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail "+e.Error())
}
updateDataIndex++
}
dataOrigin := q.data[i]
toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"})
toolkit.Serde(dataOrigin, &v, "")
q.data[i] = v
}
}
} else if commandType == dbox.QueryPartDelete {
if hasWhere && len(where) > 0 {
indexes := dbox.Find(q.data, where)
if len(indexes) > 0 {
newdata := []toolkit.M{}
for index, v := range q.data {
partOfIndex := toolkit.HasMember(indexes, index)
if partOfIndex == false {
newdata = append(newdata, v)
}
//toolkit.Println("i:", indexes, ", index:", index, ", p.ofIndex: ", partOfIndex, ", data: ", toolkit.JsonString(newdata))
}
q.data = newdata
}
} else {
q.data = []toolkit.M{}
}
//toolkit.Printf("Data now: %s\n", toolkit.JsonString(q.data))
} else if commandType == dbox.QueryPartSave {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataMs []toolkit.M
var dataM toolkit.M
if !toolkit.IsSlice(data) {
e = toolkit.Serde(&data, &dataM, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
dataMs = append(dataMs, dataM)
} else {
e = toolkit.Serde(&data, &dataMs, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
}
//toolkit.Printf("Saving: %s\n", toolkit.JsonString(dataMs))
for _, v := range dataMs {
idField, idValue := toolkit.IdInfo(v)
indexes := dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)})
if len(indexes) == 0 {
q.data = append(q.data, v)
} else {
dataOrigin := q.data[indexes[0]]
//toolkit.Printf("Copy data %s to %s\n", toolkit.JsonString(v), toolkit.JsonString(dataOrigin))
toolkit.CopyM(&v, &dataOrigin, false, []string{idField})
q.data[indexes[0]] = dataOrigin
}
}
}
e = q.writeFile()
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Write fail", e.Error())
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
var e error
h := c.sessionHive
if h != nil {
e = h.Exec(c.QueryString, func(x hive.HiveResult) error {
tableData = append(tableData, x.ResultObj.(map[string]interface{}))
return nil
})
} else {
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
dataTypeList := toolkit.M{}
var isStruct bool
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := valueType.Field(i).Type.String()
dataTypeList.Set(namaField, dataType)
}
isStruct = true
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
var ok bool
var b []byte
if val == nil {
v = nil
} else {
b, ok = val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
}
/*mysql always byte, postgres only string that byte, oracle always string, mssql agree with field datatype*/
if (ok && (c.driver == "mysql" || c.driver == "postgres")) || c.driver == "oci8" {
if isStruct {
v = c.structValue(dataTypeList, col, v)
} else {
intVal, e := strconv.Atoi(toolkit.ToString(v))
if e != nil {
e = nil
floatVal, e := strconv.ParseFloat(toolkit.ToString(v), 64)
if e != nil {
e = nil
boolVal, e := strconv.ParseBool(toolkit.ToString(v))
if e != nil {
e = nil
dateVal, e := time.Parse(c.DateFormat, toolkit.ToString(v))
if e != nil {
v = v
} else { /*if string is date*/
v = dateVal
}
} else { /*if string is bool*/
v = boolVal
}
} else { /*if string is float*/
v = floatVal
}
} else { /*if string is int*/
v = intVal
}
}
}
toolkit.Println(col, toolkit.TypeName(v), v)
entry.Set(strings.ToLower(col), v)
}
tableData = append(tableData, entry)
}
}
maxIndex := toolkit.SliceLen(tableData)
if e != nil {
return e
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e = errors.New("No more data to fetched!")
} else {
e = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
var source []toolkit.M
var lower, upper, lenData, lenIndex int
if c.where == nil {
lenData = len(c.q.data)
if c.currentIndex == 0 {
c.maxIndex = lenData
}
} else {
lenIndex = len(c.indexes)
if c.currentIndex == 0 {
c.maxIndex = lenIndex
}
}
if c.currentIndex == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/
c.maxIndex = c.skip + c.take
}
lower = c.currentIndex
upper = lower + n
if c.skip > 0 && c.currentIndex < 1 {
lower += c.skip
}
if n == 0 {
if c.where == nil {
upper = lenData
} else {
upper = lenIndex
}
if c.take > 0 {
upper = lower + c.take
}
} else if n == 1 {
upper = lower + 1
} else {
upper = lower + n
if c.take > 0 && n > c.take {
upper = lower + c.take
}
}
if c.where == nil {
if toolkit.SliceLen(c.q.data) > 0 {
if lower >= lenData {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenData {
upper = lenData
}
}
} else {
if toolkit.SliceLen(c.indexes) > 0 {
if lower >= lenIndex {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenIndex {
upper = lenIndex
}
}
}
if upper >= c.maxIndex {
upper = c.maxIndex
}
if c.where == nil {
source = c.q.data[lower:upper]
} else if len(c.indexes) > 0 {
for _, v := range c.indexes[lower:upper] {
/*
toolkit.Printf("Add index: %d. Source info now: %s \n", v, func() string {
var ret []string
for _, id := range source {
ret = append(ret, id.Get("_id").(string))
}
return strings.Join(ret, ",")
}())
*/
if v < len(c.q.data) {
source = append(source, c.q.data[v])
}
}
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
source = getSelected(source, c.jsonSelect)
}
var e error
if n == 1 && !toolkit.IsSlice(m) {
if len(source) > 0 {
e = toolkit.Serde(&source[0], m, "json")
}
} else {
e = toolkit.Serde(&source, m, "json")
}
c.currentIndex = upper
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
//toolkit.Printf("Data: %s\nLower, Upper = %d, %d\nSource: %s\nResult:%s\n\n", toolkit.JsonString(c.q.data), lower, upper, toolkit.JsonString(source), toolkit.JsonString(m))
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
e := c.prepIter()
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
lineCount := 0
//=============================
// fmt.Println("Qursor 133 : ", c.ConditionVal.Find)
for {
iv := reflect.New(v).Interface()
isAppend := true
c.count += 1
recData := toolkit.M{}
appendData := toolkit.M{}
dataTemp, e := c.reader.Read()
for i, val := range dataTemp {
orgname := c.headerColumn[i].name
lowername := strings.ToLower(c.headerColumn[i].name)
switch c.headerColumn[i].dataType {
case "int":
recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
case "float":
recData[lowername] = cast.ToF64(val, 2, cast.RoundingAuto)
default:
recData[lowername] = val
}
if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
appendData[orgname] = recData[lowername]
} else {
if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
appendData[orgname] = recData[lowername]
}
}
}
isAppend = c.ConditionVal.getCondition(recData)
if c.count < c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
isAppend = false
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if appendData.Has(v.Field(i).Name) {
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(appendData[v.Field(i).Name], cast.RoundingAuto))
}
}
}
}
if e == io.EOF {
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
break
} else if e != nil {
return errorlib.Error(packageName, modCursor,
"Fetch", e.Error())
}
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
if n > 0 {
if lineCount >= n {
break
}
}
}
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if n == 1 {
if ivs.Len() > 0 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if setting.GetString("commandtype") == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
var dataM toolkit.M
var dataMs []toolkit.M
hasData := in.Has("data")
dataIsSlice := false
data := in.Get("data")
if toolkit.IsSlice(data) {
dataIsSlice = true
e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
}
hasWhere := in.Has("where")
where := in.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
hasWhere = true
if toolkit.IsSlice(data) {
ids := []interface{}{}
idField := ""
if idField == "" {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data send is a slice, but its element has no ID")
}
dataCount := toolkit.SliceLen(data)
for i := 0; i < dataCount; i++ {
dataI := toolkit.SliceItem(data, i)
if i == 0 {
idField = toolkit.IdField(dataI)
}
ids = append(ids, toolkit.Id(dataI))
}
where = []*dbox.Filter{dbox.In(idField, ids)}
} else {
id := toolkit.Id(data)
if toolkit.IsNilOrEmpty(id) {
where = []*dbox.Filter{dbox.Eq(toolkit.IdField(id), id)}
} else {
where = nil
hasWhere = false
}
}
}
q.openFile()
if commandType == dbox.QueryPartInsert {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
if dataIsSlice {
q.data = append(q.data, dataMs...)
} else {
q.data = append(q.data, dataM)
}
} else if commandType == dbox.QueryPartUpdate {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
var indexes []interface{}
if hasWhere {
toolkit.Serde(dbox.Find(q.data, where), &indexes, "")
}
var dataUpdate toolkit.M
var updateDataIndex int
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error())
}
}
var idField string
for i, v := range q.data {
if toolkit.HasMember(indexes, i) || len(indexes) == 0 {
if idField == "" {
idField = toolkit.IdField(v)
if idField == "" {
return err.Error(packageName, modQuery, "Exec:"+commandType, "No ID")
}
}
var dataOrigin toolkit.M
e = toolkit.Serde(v, &dataOrigin, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data origin. "+e.Error())
}
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error())
}
updateDataIndex++
}
for fieldName, fieldValue := range dataUpdate {
if fieldName != idField {
if dataOrigin.Has(fieldName) {
dataOrigin.Set(fieldName, fieldValue)
}
}
}
toolkit.Serde(dataOrigin, &v, "")
q.data[i] = v
}
}
} else if commandType == dbox.QueryPartDelete {
if hasWhere {
var indexes []interface{}
toolkit.Serde(dbox.Find(q.data, where), &indexes, "")
if len(indexes) > 0 {
newdata := []toolkit.M{}
for index, v := range q.data {
if toolkit.HasMember(indexes, index) == false {
newdata = append(newdata, v)
}
}
q.data = newdata
}
} else {
q.data = []toolkit.M{}
}
} else if commandType == dbox.QueryPartSave {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
}
q.writeFile()
return nil
}