func (q *Query) HasPartExec() error {
var e error
var lastJson []toolkit.M
q.ReadFile(&lastJson, q.Connection().(*Connection).filePath)
if toolkit.SliceLen(lastJson) > 0 {
getWhere := []*dbox.Filter{}
for _, v := range q.whereData {
getWhere = []*dbox.Filter{v}
i := dbox.Find(q.sliceData, getWhere)
for idSlice, _ := range q.sliceData {
if toolkit.HasMember(i, idSlice) {
idata := dbox.Find(lastJson, getWhere)
for idx, _ := range lastJson {
if toolkit.HasMember(idata, idx) {
lastJson[idx] = q.sliceData[idSlice]
}
}
if toolkit.SliceLen(idata) == 0 {
lastJson = append(lastJson, q.sliceData[idSlice])
// toolkit.Printf("idata>%v\n", q.sliceData[idSlice])
}
}
}
}
q.sliceData = lastJson
}
e = q.WriteFile(q.sliceData)
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", "HasPartExec", e.Error())
}
return nil
}
func (q *Query) Cursor(in toolkit.M) (dbox.ICursor, error) {
var cursor *Cursor
setting, e := q.prepare(in)
if e != nil {
return nil, err.Error(packageName, modQuery, "Cursor", e.Error())
}
commandtype := setting.GetString("commandtype")
if commandtype != dbox.QueryPartSelect {
return nil, err.Error(packageName, modQuery, "Cursor", "Cursor is only working with select command, for "+commandtype+" please use .Exec instead")
}
e = q.openFile(commandtype)
if e != nil {
return nil, err.Error(packageName, modQuery, "Cursor", e.Error())
}
cursor = newCursor(q)
skip := 0
if skip = setting.Get("skip").(int); skip > 0 {
cursor.skip = skip
}
take := 0
if take = setting.Get("take").(int); take > 0 {
cursor.take = take
}
if sort := setting.Get("sort").([]string); toolkit.SliceLen(sort) > 0 {
fb := new(json.FilterBuilder)
sorter := fb.SortFetch(sort, q.data)
q.data = sorter
}
cursor.jsonSelect = setting.Get("fields").([]string)
var count int
count = toolkit.SliceLen(q.data)
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if len(where) > 0 {
cursor.where = where
cursor.indexes = dbox.Find(q.data, where)
count = toolkit.SliceLen(cursor.indexes)
}
if count <= skip {
count = 0
} else {
count -= skip
}
if count >= take && take > 0 {
count = take
}
cursor.count = count
return cursor, nil
}
func (q *Query) HasPartExec() error {
var e error
var lastJson []toolkit.M
q.ReadFile(&lastJson, q.Connection().(*Connection).filePath)
if toolkit.SliceLen(lastJson) > 0 {
getWhere := []*dbox.Filter{}
for _, v := range q.whereData {
getWhere = []*dbox.Filter{v}
i := dbox.Find(q.sliceData, getWhere)
for idSlice := range q.sliceData {
if toolkit.HasMember(i, idSlice) {
idata := dbox.Find(lastJson, getWhere)
for idx := range lastJson {
if toolkit.HasMember(idata, idx) {
lastJson[idx] = q.sliceData[idSlice]
}
}
if toolkit.SliceLen(idata) == 0 {
lastJson = append(lastJson, q.sliceData[idSlice])
}
}
}
}
q.sliceData = lastJson
} else {
idx := []int{}
for _, v := range q.whereData {
getWhere := []*dbox.Filter{v}
idx = dbox.Find(q.sliceData, getWhere)
}
// toolkit.Printf("newdata>%v\n", idx)
if toolkit.SliceLen(idx) > 1 {
newdata := toolkit.M{}
for idslice, dataslice := range q.sliceData {
if toolkit.HasMember(idx, idslice) {
idf, _ := toolkit.IdInfo(dataslice)
newdata = q.sliceData[idslice]
toolkit.CopyM(&dataslice, &newdata, false, []string{idf})
}
}
q.sliceData = []toolkit.M{}
q.sliceData = append(q.sliceData, newdata)
}
}
e = q.WriteFile(q.sliceData)
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", "HasPartExec", e.Error())
}
return nil
}
func TestSelect(t *testing.T) {
t.Skip()
c, e := prepareConnection()
if e != nil {
t.Errorf("Unable to connect %s \n", e.Error())
}
defer c.Close()
// csr, e := c.NewQuery().Select().From("tes").Where(dbox.Eq("id", "3")).Cursor(nil)
csr, e := c.NewQuery().
// Select("empno", "ename", "hiredate").
From(tableCustomers).Cursor(nil)
if e != nil {
t.Errorf("Cursor pre error: %s \n", e.Error())
return
}
if csr == nil {
t.Errorf("Cursor not initialized")
return
}
defer csr.Close()
rets := []toolkit.M{}
e = csr.Fetch(&rets, 0, false)
if e != nil {
t.Errorf("Unable to fetch N: %s \n", e.Error())
} else {
toolkit.Printf("Fetch N OK. Result: %v \n", toolkit.JsonString(rets))
toolkit.Printf("Total Fetch OK : %v \n", toolkit.SliceLen(rets))
}
}
func TestTakeSkip(t *testing.T) {
t.Skip()
c, e := prepareConnection()
if e != nil {
t.Errorf("Unable to connect %s \n", e.Error())
}
defer c.Close()
csr, e := c.NewQuery().
Select("id", "productname").
From(tableProducts).
Take(5).
Skip(10).
Cursor(nil)
if e != nil {
t.Errorf("Cursor pre error: %s \n", e.Error())
return
}
if csr == nil {
t.Errorf("Cursor not initialized")
return
}
defer csr.Close()
rets := []toolkit.M{}
e = csr.Fetch(&rets, 0, false)
if e != nil {
t.Errorf("Unable to fetch: %s \n", e.Error())
} else {
toolkit.Printf("Fetch OK. Result: %v \n", toolkit.JsonString(rets))
toolkit.Printf("Total Record OK. Result: %v \n", toolkit.SliceLen(rets))
}
}
func checkDir(basepath string, scanDir string, dirName string) error {
dirList, err := ioutil.ReadDir(scanDir)
if err != nil {
return err
}
if toolkit.SliceLen(dirList) == 1 {
for _, f := range dirList {
if f.IsDir() {
oldpath := filepath.Join(scanDir, f.Name())
temp_oldpath := filepath.Join(scanDir, dirName)
if err := os.Rename(oldpath, temp_oldpath); err != nil {
return err
}
if err := toolkit.ZipCompress(temp_oldpath, scanDir+".zip"); err != nil {
return err
}
if err := os.RemoveAll(scanDir); err != nil {
return err
}
if err := toolkit.ZipExtract(scanDir+".zip", basepath); err != nil {
return err
}
if err := os.Remove(scanDir + ".zip"); err != nil {
return err
}
}
}
}
return nil
}
//func Find(ms []toolkit.M, filters []*Filter) (output []int) {
func Find(ms interface{}, filters []*Filter) (output []int) {
//-- is not a slice
if !toolkit.IsSlice(ms) {
toolkit.Println("Data is not slice")
return []int{}
}
//toolkit.Printf("Find:%s Filter:%s\n", toolkit.JsonString(ms), toolkit.JsonString(filters))
sliceLen := toolkit.SliceLen(ms)
for i := 0; i < sliceLen; i++ {
var v toolkit.M
item := toolkit.SliceItem(ms, i)
e := toolkit.Serde(item, &v, "json")
if e == nil {
match := MatchM(v, filters)
if match {
output = append(output, i)
}
} else {
//toolkit.Println("Serde Fail: ", e.Error(), " Data: ", item)
}
}
return
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
var source []toolkit.M
var lower, upper, lenData, lenIndex int
if c.where == nil {
lenData = len(c.q.data)
if c.currentIndex == 0 {
c.maxIndex = lenData
}
} else {
lenIndex = len(c.indexes)
if c.currentIndex == 0 {
c.maxIndex = lenIndex
}
}
if c.currentIndex == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/
c.maxIndex = c.skip + c.take
}
lower = c.currentIndex
upper = lower + n
if c.skip > 0 && c.currentIndex < 1 {
lower += c.skip
}
if n == 0 {
if c.where == nil {
upper = lenData
} else {
upper = lenIndex
}
if c.take > 0 {
upper = lower + c.take
}
} else if n == 1 {
upper = lower + 1
} else {
upper = lower + n
if c.take > 0 && n > c.take {
upper = lower + c.take
}
}
if c.where == nil {
if toolkit.SliceLen(c.q.data) > 0 {
if lower >= lenData {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenData {
upper = lenData
}
}
} else {
if toolkit.SliceLen(c.indexes) > 0 {
if lower >= lenIndex {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenIndex {
upper = lenIndex
}
}
}
if upper >= c.maxIndex {
upper = c.maxIndex
}
if c.where == nil {
source = c.q.data[lower:upper]
} else if len(c.indexes) > 0 {
for _, v := range c.indexes[lower:upper] {
/*
toolkit.Printf("Add index: %d. Source info now: %s \n", v, func() string {
var ret []string
for _, id := range source {
ret = append(ret, id.Get("_id").(string))
}
return strings.Join(ret, ",")
}())
*/
if v < len(c.q.data) {
source = append(source, c.q.data[v])
}
}
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
source = getSelected(source, c.jsonSelect)
}
var e error
if n == 1 && !toolkit.IsSlice(m) {
if len(source) > 0 {
e = toolkit.Serde(&source[0], m, "json")
}
} else {
e = toolkit.Serde(&source, m, "json")
}
c.currentIndex = upper
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
//toolkit.Printf("Data: %s\nLower, Upper = %d, %d\nSource: %s\nResult:%s\n\n", toolkit.JsonString(c.q.data), lower, upper, toolkit.JsonString(source), toolkit.JsonString(m))
return nil
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
//toolkit.Printf("Command type: %s\n", commandType)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if setting.GetString("commandtype") == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
var dataM toolkit.M
var dataMs []toolkit.M
hasData := in.Has("data")
dataIsSlice := false
data := in.Get("data")
if toolkit.IsSlice(data) {
dataIsSlice = true
e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
}
hasWhere := setting.Has("where")
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if hasWhere && len(where) == 0 {
inWhere := in.Get("where")
if inWhere == nil {
hasWhere = false
where = nil
} else {
if !toolkit.IsSlice(inWhere) {
where = append(where, inWhere.(*dbox.Filter))
} else {
where = inWhere.([]*dbox.Filter)
}
}
}
if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartDelete,
dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
hasWhere = true
//toolkit.Println("check where")
if toolkit.IsSlice(data) {
ids := []interface{}{}
idField := ""
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data send is a slice, but its element has no ID")
}
dataCount := toolkit.SliceLen(data)
for i := 0; i < dataCount; i++ {
dataI := toolkit.SliceItem(data, i)
if i == 0 {
idField = toolkit.IdField(dataI)
}
ids = append(ids, toolkit.Id(dataI))
}
where = []*dbox.Filter{dbox.In(idField, ids)}
} else {
idfield := "_id"
id := toolkit.Id(data)
if !toolkit.IsNilOrEmpty(id) {
where = []*dbox.Filter{dbox.Eq(idfield, id)}
} else {
where = nil
hasWhere = false
}
}
}
/*
toolkit.Printf("CommandType: %s HasData: %v HasWhere: %v Where: %s\n",
commandType, hasData, hasWhere, toolkit.JsonString(where))
*/
e = q.openFile(commandType)
//toolkit.Printf(commandType+" Open File, found record: %d\nData:%s\n", len(q.data), toolkit.JsonString(q.data))
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
var indexes []interface{}
if hasWhere && commandType != dbox.QueryPartInsert {
whereIndex := dbox.Find(q.data, where)
indexes = toolkit.ToInterfaceArray(&whereIndex)
//toolkit.Printf("Where Index: %s Index:%s\n", toolkit.JsonString(whereIndex), toolkit.JsonString(indexes))
}
if commandType == dbox.QueryPartInsert {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
if !dataIsSlice {
dataMs = []toolkit.M{dataM}
}
//-- validate
for _, datam := range dataMs {
idField, idValue := toolkit.IdInfo(datam)
toolkit.Serde(dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}), &indexes, "")
if len(indexes) > 0 {
return err.Error(packageName, modQuery, "Exec: "+commandType, toolkit.Sprintf("Data %v already exist", idValue))
}
}
//-- insert the data
q.data = append(q.data, dataMs...)
} else if commandType == dbox.QueryPartUpdate {
//-- valida
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataUpdate toolkit.M
var updateDataIndex int
// if it is a slice then we need to update each data passed on its slice
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error())
}
}
var idField string
//toolkit.Printf("Indexes: %s\n", toolkit.JsonString(indexes))
for i, v := range q.data {
// update only data that match given inde
if toolkit.HasMember(indexes, i) || !hasWhere {
if idField == "" {
idField = toolkit.IdField(v)
if idField == "" {
return err.Error(packageName, modQuery, "Exec: "+commandType, "No ID")
}
}
// If dataslice is sent, iterate f
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail "+e.Error())
}
updateDataIndex++
}
dataOrigin := q.data[i]
toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"})
toolkit.Serde(dataOrigin, &v, "")
q.data[i] = v
}
}
} else if commandType == dbox.QueryPartDelete {
if hasWhere && len(where) > 0 {
indexes := dbox.Find(q.data, where)
if len(indexes) > 0 {
newdata := []toolkit.M{}
for index, v := range q.data {
partOfIndex := toolkit.HasMember(indexes, index)
if partOfIndex == false {
newdata = append(newdata, v)
}
//toolkit.Println("i:", indexes, ", index:", index, ", p.ofIndex: ", partOfIndex, ", data: ", toolkit.JsonString(newdata))
}
q.data = newdata
}
} else {
q.data = []toolkit.M{}
}
//toolkit.Printf("Data now: %s\n", toolkit.JsonString(q.data))
} else if commandType == dbox.QueryPartSave {
if !hasData {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty")
}
var dataMs []toolkit.M
var dataM toolkit.M
if !toolkit.IsSlice(data) {
e = toolkit.Serde(&data, &dataM, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
dataMs = append(dataMs, dataM)
} else {
e = toolkit.Serde(&data, &dataMs, "json")
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error())
}
}
//toolkit.Printf("Saving: %s\n", toolkit.JsonString(dataMs))
for _, v := range dataMs {
idField, idValue := toolkit.IdInfo(v)
indexes := dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)})
if len(indexes) == 0 {
q.data = append(q.data, v)
} else {
dataOrigin := q.data[indexes[0]]
//toolkit.Printf("Copy data %s to %s\n", toolkit.JsonString(v), toolkit.JsonString(dataOrigin))
toolkit.CopyM(&v, &dataOrigin, false, []string{idField})
q.data[indexes[0]] = dataOrigin
}
}
}
e = q.writeFile()
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType+" Write fail", e.Error())
}
return nil
}
func (s *SliceBase) Len() int {
return toolkit.SliceLen(s.data)
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
c.Close()
}
var source []toolkit.M
var lower, upper, lenData, lenIndex int
if !c.isWhere {
lenData = len(c.datas)
if c.lastFetched == 0 {
c.maxIndex = lenData
}
} else {
lenIndex = len(c.indexes)
if c.lastFetched == 0 {
c.maxIndex = lenIndex
}
}
if c.lastFetched == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/
c.maxIndex = c.skip + c.take
}
lower = c.lastFetched
upper = lower + n
if c.skip > 0 && c.lastFetched < 1 {
lower += c.skip
}
if n == 0 {
if !c.isWhere {
upper = lenData
} else {
upper = lenIndex
}
if c.take > 0 {
upper = lower + c.take
}
} else if n == 1 {
upper = lower + 1
} else {
upper = lower + n
if c.take > 0 && n > c.take {
upper = lower + c.take
}
}
if !c.isWhere {
if toolkit.SliceLen(c.datas) > 0 {
if lower >= lenData {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenData {
upper = lenData
}
}
} else {
if toolkit.SliceLen(c.indexes) > 0 {
if lower >= lenIndex {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
if upper >= lenIndex {
upper = lenIndex
}
}
}
if upper >= c.maxIndex {
upper = c.maxIndex
}
if !c.isWhere {
source = c.datas[lower:upper]
} else {
for _, v := range c.indexes[lower:upper] {
if v < len(c.datas) {
source = append(source, c.datas[v])
}
}
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
source = c.GetSelected(source, c.jsonSelect)
}
var e error
e = toolkit.Serde(&source, m, "json")
c.lastFetched = upper
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
// var first, last int
// dataJson := []toolkit.M{}
// c.count = len(c.datas)
// c.lastFetched = c.count
// if c.skip > 0 {
// if c.take > 0 {
// c.count = c.skip + c.take
// } else {
// first = c.skip
// }
// } else {
// c.count = c.take
// }
// if c.take > 0 {
// if c.take == c.skip {
// first = c.skip
// } else {
// first = c.count - c.take
// }
// }
// // toolkit.Printf("first = skip>%v last = take>%v lastfetched>%v count>%v\n", first, last, c.lastFetched, c.count)
// if n == 0 {
// last = c.count
// if c.lastFetched <= c.count || c.count == 0 {
// last = c.lastFetched
// }
// } else if n > 0 {
// switch {
// case c.lastFetched == 0:
// last = n
// c.lastFetched = n
// case n > c.lastFetched || n < c.lastFetched || n == c.lastFetched:
// first = c.lastFetched
// last = c.lastFetched + n
// c.lastFetched = last
// if c.lastFetched > c.count {
// if first > c.count {
// return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
// }
// last = c.count
// c.lastFetched = last
// }
// }
// if first > last {
// return errorlib.Error(packageName, modCursor, "Fetch", "Wrong fetched data!")
// }
// }
// if c.isWhere {
// i := dbox.Find(c.datas, c.whereFields)
// c.lastFetched = len(i)
// if c.lastFetched < c.count || c.count == 0 {
// last = c.lastFetched
// }
// for _, index := range i[first:last] {
// dataJson = append(dataJson, c.datas[index])
// }
// } else {
// dataJson = c.datas[first:last]
// }
// if toolkit.SliceLen(c.jsonSelect) > 0 {
// dataJson = c.GetSelected(dataJson, c.jsonSelect)
// }
// e := toolkit.Serde(dataJson, m, "json")
// if e != nil {
// return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
// }
return nil
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if setting.GetString("commandtype") == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
var dataM toolkit.M
var dataMs []toolkit.M
hasData := in.Has("data")
dataIsSlice := false
data := in.Get("data")
if toolkit.IsSlice(data) {
dataIsSlice = true
e = toolkit.Unjson(toolkit.Jsonify(data), dataMs)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error())
}
}
hasWhere := in.Has("where")
where := in.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) {
hasWhere = true
if toolkit.IsSlice(data) {
ids := []interface{}{}
idField := ""
if idField == "" {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data send is a slice, but its element has no ID")
}
dataCount := toolkit.SliceLen(data)
for i := 0; i < dataCount; i++ {
dataI := toolkit.SliceItem(data, i)
if i == 0 {
idField = toolkit.IdField(dataI)
}
ids = append(ids, toolkit.Id(dataI))
}
where = []*dbox.Filter{dbox.In(idField, ids)}
} else {
id := toolkit.Id(data)
if toolkit.IsNilOrEmpty(id) {
where = []*dbox.Filter{dbox.Eq(toolkit.IdField(id), id)}
} else {
where = nil
hasWhere = false
}
}
}
q.openFile()
if commandType == dbox.QueryPartInsert {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
if dataIsSlice {
q.data = append(q.data, dataMs...)
} else {
q.data = append(q.data, dataM)
}
} else if commandType == dbox.QueryPartUpdate {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
var indexes []interface{}
if hasWhere {
toolkit.Serde(dbox.Find(q.data, where), &indexes, "")
}
var dataUpdate toolkit.M
var updateDataIndex int
isDataSlice := toolkit.IsSlice(data)
if isDataSlice == false {
isDataSlice = false
e = toolkit.Serde(data, &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error())
}
}
var idField string
for i, v := range q.data {
if toolkit.HasMember(indexes, i) || len(indexes) == 0 {
if idField == "" {
idField = toolkit.IdField(v)
if idField == "" {
return err.Error(packageName, modQuery, "Exec:"+commandType, "No ID")
}
}
var dataOrigin toolkit.M
e = toolkit.Serde(v, &dataOrigin, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data origin. "+e.Error())
}
if isDataSlice {
e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "")
if e != nil {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error())
}
updateDataIndex++
}
for fieldName, fieldValue := range dataUpdate {
if fieldName != idField {
if dataOrigin.Has(fieldName) {
dataOrigin.Set(fieldName, fieldValue)
}
}
}
toolkit.Serde(dataOrigin, &v, "")
q.data[i] = v
}
}
} else if commandType == dbox.QueryPartDelete {
if hasWhere {
var indexes []interface{}
toolkit.Serde(dbox.Find(q.data, where), &indexes, "")
if len(indexes) > 0 {
newdata := []toolkit.M{}
for index, v := range q.data {
if toolkit.HasMember(indexes, index) == false {
newdata = append(newdata, v)
}
}
q.data = newdata
}
} else {
q.data = []toolkit.M{}
}
} else if commandType == dbox.QueryPartSave {
if !hasData {
return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty")
}
}
q.writeFile()
return nil
}
func (c *Crowd) Len() int {
if c.data == nil {
return 0
}
return toolkit.SliceLen(c.data)
}
func (q *Query) Cursor(in toolkit.M) (dbox.ICursor, error) {
var (
e error
dataMaps []toolkit.M
)
q.ReadFile(&dataMaps, q.Connection().(*Connection).filePath)
cursor := dbox.NewCursor(new(Cursor))
cursor = cursor.SetConnection(q.Connection())
filters, e := q.Filters(in)
if e != nil {
return nil, errorlib.Error(packageName, modQuery, "Cursor", e.Error())
}
commandType := filters.GetString("cmdType")
if commandType != dbox.QueryPartSelect {
return nil, errorlib.Error(packageName, modQuery, "Cursor", "Cursor is only working with select command, for "+commandType+" please use .Exec instead")
}
aggregate := false
hasWhere := filters.Has("where")
hasAggregate := filters.Get("aggregate").(bool)
if hasAggregate {
aggregate = true
}
if !aggregate {
if hasWhere {
// toolkit.Println("where:", toolkit.JsonString(filters.Get("where")))
cursor.(*Cursor).whereFields = filters.Get("where").([]*dbox.Filter)
cursor.(*Cursor).isWhere = true
cursor.(*Cursor).indexes = dbox.Find(dataMaps, filters.Get("where", []*dbox.Filter{}).([]*dbox.Filter))
}
// toolkit.Println("skip:", toolkit.JsonString(filters.Get("skip")))
skip := 0
if skip = filters.Get("skip").(int); skip > 0 {
cursor.(*Cursor).skip = skip
}
// toolkit.Println("take:", toolkit.JsonString(filters.Get("take")))
take := 0
if take = filters.Get("take").(int); take > 0 {
cursor.(*Cursor).take = take
}
if sort := filters.Get("sort").([]string); toolkit.SliceLen(sort) > 0 {
fb := new(FilterBuilder)
// toolkit.Printf("sorter:%v\n", sort)
sorter := fb.SortFetch(sort, dataMaps)
cursor.(*Cursor).datas = sorter
} else {
cursor.(*Cursor).datas = dataMaps
}
var count int
if hasWhere {
count = toolkit.SliceLen(cursor.(*Cursor).indexes)
} else {
count = toolkit.SliceLen(cursor.(*Cursor).datas)
}
if count <= skip {
count = 0
} else {
count -= skip
}
if count >= take && take > 0 {
count = take
}
cursor.(*Cursor).count = count
cursor.(*Cursor).jsonSelect = filters.Get("select").([]string)
} else {
return nil, errorlib.Error(packageName, modQuery, "Cursor", "No Aggregate function")
}
return cursor, nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
c.Close()
}
var first, last int
dataJson := []toolkit.M{}
var datas []toolkit.M
toolkit.Unjson(c.readFile, &datas)
c.count = len(datas)
if n == 0 {
last = c.count
} else if n > 0 {
switch {
case c.lastFeteched == 0:
last = n
c.lastFeteched = n
case n > c.lastFeteched || n < c.lastFeteched || n == c.lastFeteched:
first = c.lastFeteched
last = c.lastFeteched + n
c.lastFeteched = last
if c.lastFeteched > c.count {
if first > c.count {
return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!")
}
last = c.count
}
// toolkit.Printf("first>%v last>%v lastfetched>%v count>%v\n", first, last, c.lastFeteched, c.count)
}
}
if c.isWhere {
i := dbox.Find(datas, c.whereFields)
last = len(i)
for _, index := range i[first:last] {
dataJson = append(dataJson, datas[index])
}
} else {
dataJson = datas[first:last]
}
if toolkit.SliceLen(c.jsonSelect) > 0 {
var getRemField = toolkit.M{}
for _, v := range dataJson {
for i, _ := range v {
getRemField.Set(i, i)
}
if c.jsonSelect[0] != "*" {
fields := c.removeDuplicatesUnordered(getRemField, c.jsonSelect)
for _, field := range fields {
v.Unset(field)
}
}
}
}
e := toolkit.Serde(dataJson, m, "json")
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
// var e error
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// b, ok := val.([]byte)
// // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val))
// var out interface{}
// e = toolkit.Unjson(b, &out)
// // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e)
// if e != nil {
// ok = false
// }
// if ok {
// v = out
// toolkit.Println("error OK :: ", ok, " :v :", v)
// } else {
// toolkit.Println("error OK :: ", ok, " :b :", b)
// v = string(b)
// }
v = val
entry.Set(strings.ToLower(col), v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
// toolkit.Println("... ::: ", tableData)
maxIndex := toolkit.SliceLen(tableData)
var e2 error
if e2 != nil {
return e2
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e2 = errors.New("No more data to fetched!")
} else {
e2 = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e2
}
func (d *DataBrowserController) hasAggr(ctx dbox.IConnection, data *colonycore.DataBrowser, conn *colonycore.Connection) (*colonycore.DataBrowser, error) {
var fieldArr, aggrArr []string
var indexAggr []map[int]string
var query dbox.IQuery
fieldAggr := toolkit.M{}
for i, v := range data.MetaData {
if v.Aggregate != "" {
result := toolkit.M{}
toolkit.UnjsonFromString(v.Aggregate, &result)
cursor := []toolkit.M{}
if data.QueryType == "" {
aggregate, e := d.dboxAggr(data.TableNames, v.Field, ctx, query, result, fieldAggr, cursor, conn)
if e != nil {
return nil, e
}
v.Aggregate = toolkit.JsonString(aggregate)
} else if data.QueryType == "SQL" {
names := map[int]string{}
fieldArr = append(fieldArr, v.Field)
if _, sumOK := result["SUM"]; sumOK {
aggrArr = append(aggrArr, "SUM("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "sum"})
} else {
names[i] = "sum"
}
}
if _, avgOK := result["AVG"]; avgOK {
aggrArr = append(aggrArr, "AVG("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "avg"})
} else {
names[i] = "avg"
}
}
if _, maxOK := result["MAX"]; maxOK {
aggrArr = append(aggrArr, "MAX("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "max"})
} else {
names[i] = "max"
}
}
if _, minOK := result["MIN"]; minOK {
aggrArr = append(aggrArr, "MIN("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "min"})
} else {
names[i] = "min"
}
}
if _, minOK := result["COUNT"]; minOK {
aggrArr = append(aggrArr, "COUNT("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "count"})
} else {
names[i] = "count"
}
}
if len(result) > 1 {
fieldAggr.Set(v.Field, indexAggr)
} else {
fieldAggr.Set(v.Field, names)
}
} else if data.QueryType == "Dbox" {
getQuery := toolkit.M{}
toolkit.UnjsonFromString(data.QueryText, &getQuery)
aggregate, e := d.dboxAggr(getQuery.Get("from").(string), v.Field, ctx, query, result, fieldAggr, cursor, conn)
if e != nil {
return nil, e
}
v.Aggregate = toolkit.JsonString(aggregate)
}
}
}
if data.QueryType == "SQL" {
// fieldString := strings.Join(fieldArr, ", ")
aggrString := strings.Join(aggrArr, ", ")
var queryText string
r := regexp.MustCompile(`(([Ff][Rr][Oo][Mm])) (?P<from>([a-zA-Z][_a-zA-Z]+[_a-zA-Z0-1].*))`)
temparray := r.FindStringSubmatch(data.QueryText)
sqlpart := toolkit.M{}
for i, val := range r.SubexpNames() {
if val != "" {
sqlpart.Set(val, temparray[i])
}
}
if fromOK := sqlpart.Get("from", "").(string); fromOK != "" {
queryText = toolkit.Sprintf("select %s FROM %s", aggrString, sqlpart.Get("from", "").(string))
// toolkit.Printf("queryString:%v\n", queryString)
}
query = ctx.NewQuery().Command("freequery", toolkit.M{}.
Set("syntax", queryText))
csr, e := query.Cursor(nil)
if e != nil {
return nil, e
}
defer csr.Close()
cursor := []toolkit.M{}
e = csr.Fetch(&cursor, 0, false)
if e != nil {
return nil, e
}
for f, m := range fieldAggr {
aggrData := toolkit.M{}
for _, aggs := range cursor {
for k, agg := range aggs {
if toolkit.SliceLen(m) > 0 {
for _, vals := range m.([]map[int]string) {
for key, val := range vals {
if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) {
aggrData.Set(val, agg)
data.MetaData[key].Aggregate = toolkit.JsonString(aggrData)
}
}
}
} else {
for key, val := range m.(map[int]string) {
if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) {
aggrData.Set(val, agg)
data.MetaData[key].Aggregate = toolkit.JsonString(aggrData)
// toolkit.Printf("k:%v f:%v key:%v val:%v agg:%v\n", k, f, key, val, data.MetaData[key].Aggregate)
}
}
}
}
}
}
}
return data, nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
var e error
h := c.sessionHive
if h != nil {
e = h.Exec(c.QueryString, func(x hive.HiveResult) error {
tableData = append(tableData, x.ResultObj.(map[string]interface{}))
return nil
})
} else {
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
dataTypeList := toolkit.M{}
var isStruct bool
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := valueType.Field(i).Type.String()
dataTypeList.Set(namaField, dataType)
}
isStruct = true
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
var ok bool
var b []byte
if val == nil {
v = nil
} else {
b, ok = val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
}
/*mysql always byte, postgres only string that byte, oracle always string, mssql agree with field datatype*/
if (ok && (c.driver == "mysql" || c.driver == "postgres")) || c.driver == "oci8" {
if isStruct {
v = c.structValue(dataTypeList, col, v)
} else {
intVal, e := strconv.Atoi(toolkit.ToString(v))
if e != nil {
e = nil
floatVal, e := strconv.ParseFloat(toolkit.ToString(v), 64)
if e != nil {
e = nil
boolVal, e := strconv.ParseBool(toolkit.ToString(v))
if e != nil {
e = nil
dateVal, e := time.Parse(c.DateFormat, toolkit.ToString(v))
if e != nil {
v = v
} else { /*if string is date*/
v = dateVal
}
} else { /*if string is bool*/
v = boolVal
}
} else { /*if string is float*/
v = floatVal
}
} else { /*if string is int*/
v = intVal
}
}
}
toolkit.Println(col, toolkit.TypeName(v), v)
entry.Set(strings.ToLower(col), v)
}
tableData = append(tableData, entry)
}
}
maxIndex := toolkit.SliceLen(tableData)
if e != nil {
return e
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e = errors.New("No more data to fetched!")
} else {
e = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e
}
func (cmd *Command) Exec(c *Crowd) error {
if c.data == nil {
return errors.New("Exec: Data is empty")
}
l := c.Len()
if cmd.CommandType == CommandSum {
fn := cmd.Fns[0]
sum := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
sum += item
}
c.Result.Sum = sum
} else if cmd.CommandType == CommandMin {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$gt") {
ret = item
}
}
c.Result.Min = ret
} else if cmd.CommandType == CommandMax {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$lt") {
ret = item
}
}
c.Result.Max = ret
} else if cmd.CommandType == CommandAvg {
fn := cmd.Fns[0]
ret := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
ret += item
}
c.Result.Avg = toolkit.Div(ret, toolkit.ToFloat64(l, 0, toolkit.RoundingAuto))
} else if cmd.CommandType == CommandWhere {
fn := cmd.Fns[0]
el, _ := toolkit.GetEmptySliceElement(c.data)
tel := reflect.TypeOf(el)
array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0)
for i := 0; i < l; i++ {
item := c.Item(i)
if fn(item).(bool) {
//toolkit.Printfn("Where data %d: %v", i, item)
array = reflect.Append(array, reflect.ValueOf(item))
}
}
c.Result.data = array.Interface()
} else if cmd.CommandType == CommandApply {
fn := cmd.Fns[0]
var array reflect.Value
for i := 0; i < l; i++ {
item := fn(c.Item(i))
//toolkit.Printfn("Applying data %d of %d: %v", i, l, item)
if i == 0 {
//toolkit.Println(reflect.ValueOf(item).Type().String())
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(item))
}
c.Result.data = array.Interface()
c.data = c.Result.data
} else if cmd.CommandType == CommandGroup {
fng := cmd.Fns[0]
fnc := cmd.Fns[1]
mvs := map[interface{}]reflect.Value{}
//mvo := map[interface{}]interface{}{}
var mvo []KV
for i := 0; i < l; i++ {
item := c.Item(i)
//toolkit.Printfn("Processing data %d of %d: %v", i, l, item)
g := fng(item)
gi := fnc(item)
array, exist := mvs[g]
if !exist {
//array = []interface{}{}
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(gi))
//toolkit.Println("Data:",g,array)
mvs[g] = array
}
for k, v := range mvs {
mvo = append(mvo, KV{k, v.Interface()})
}
c.Result.data = mvo
c.data = mvo
} else if cmd.CommandType == CommandSort {
sorter, e := NewSorter(c.data, cmd.Fns[0])
if e != nil {
return e
}
var direction SortDirection
direction = cmd.Parms.Get("direction", SortAscending).(SortDirection)
c.Result.data = sorter.Sort(direction)
c.data = c.Result.data
} else if cmd.CommandType == CommandJoin {
joinData := cmd.Parms.Get("joindata")
if joinData == nil {
return errors.New("crowd.join: Right side join data is nil")
}
if cmd.FnJoinKey == nil {
cmd.FnJoinKey = func(x, y interface{}) bool {
return x == y
}
}
if cmd.FnJoinSelect == nil {
cmd.FnJoinSelect = func(x, y interface{}) interface{} {
return toolkit.M{}.Set("data1", x).Set("data2", y)
}
}
l1 := toolkit.SliceLen(joinData)
var array reflect.Value
arrayBuilt := false
for i := 0; i < l; i++ {
item1 := c.Item(i)
for i1 := 0; i1 < l1; i1++ {
item2 := toolkit.SliceItem(joinData, i1)
joinOK := cmd.FnJoinKey(item1, item2)
if joinOK {
outObj := cmd.FnJoinSelect(item1, item2)
if !arrayBuilt {
arrayBuilt = true
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(outObj)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(outObj))
}
}
}
if !arrayBuilt {
return errors.New("crowd.join: No match")
}
c.Result.data = array.Interface()
c.data = array.Interface()
} else {
return errors.New(string(cmd.CommandType) + ": not yet applicable")
}
return nil
}