Esempio n. 1
0
func extractDataBulk(attrList []string, data toolkit.M, driverName string) string {
	var values string
	if data != nil {
		for i, attr := range attrList {
			val := data.Get(attr)
			var datatypelist = []string{"map", "invalid", "struct", "slice"}
			var value reflect.Value
			if val != nil {
				value = reflect.Zero(reflect.TypeOf(val))
			}
			if toolkit.HasMember(datatypelist, value.Kind().String()) {
				continue
			}
			stringValues := StringValue(val, driverName)
			if i == 0 {
				values = "(" + stringValues

			} else {
				values += ", " + stringValues

			}
		}
		values += ")"
	}
	return values
}
Esempio n. 2
0
func CheckValue(v toolkit.M, f *Filter) (bool, interface{}) {
	resbool := false
	filedtemp := []interface{}{}

	resbool = v.Has(f.Field)
	if resbool {
		return resbool, v.Get(f.Field)
	} else if strings.Contains(f.Field, ".") {
		ar := strings.Split(f.Field, ".")
		for i, dt := range ar {
			if i == 0 {
				resbool = v.Has(dt)
				if !resbool {
					break
				} else {
					filedtemp = append(filedtemp, v.Get(dt))
				}
			} else {
				temp := toolkit.M(filedtemp[i-1].(map[string]interface{}))
				resbool = temp.Has(dt)
				if !resbool {
					break
				} else {
					filedtemp = append(filedtemp, temp.Get(dt))
				}
			}

			if i == len(ar)-1 {
				return true, filedtemp[i]
			}
		}
	}
	return false, nil
}
Esempio n. 3
0
/*
Write Write bytes of data into sebar storage.
- Data need to be defined as []byte on in["data"]
- To use memory or disk should be defined on in["storage"] as: MEM, DSK (sebar.StorageTypeMemory, sebar.StorageTypeMemory)
- If no in["storage"] or the value is not eq to either disk or memory, it will be defaulted to memory
*/
func (s *Storage) Write(in toolkit.M) *toolkit.Result {
	r := toolkit.NewResult()
	key := in.Get("key").(string)
	storage := StorageTypeEnum(in.GetString("storage"))
	if storage != StorageTypeMemory && storage != StorageTypeDisk {
		storage = StorageTypeMemory
	}
	dataToWrite := in.Get("data").([]byte)
	dataLen := len(dataToWrite)

	// Validation
	nodeCoordinator := s.NodeByID(s.Coordinator)
	if nodeCoordinator == nil {
		return r.SetErrorTxt(s.Address + " no Coordinator has been setup")
	}

	// Since all is ok commit the change
	var ms *StorageMedia
	if storage == StorageTypeMemory {
		ms = s.MemoryStorage
	} else {
		ms = s.DiskStorage
	}
	ms.write(key, dataToWrite, nodeCoordinator)
	s.Log.Info(toolkit.Sprintf("Writing %s (%s) to node %s", key, ParseSize(float64(dataLen)), s.Address))
	return r
}
Esempio n. 4
0
func (c *Coordinator) UpdateMetadata(in toolkit.M) *toolkit.Result {
	result := toolkit.NewResult()
	keys := []string{}
	bs := in.Get("keys", []byte{}).([]byte)
	toolkit.FromBytes(bs, "gob", &keys)
	return result
}
Esempio n. 5
0
func (d *DataBrowserController) parseQuery(conn dbox.IConnection, dbrowser colonycore.DataBrowser, datacon *colonycore.Connection) (dbox.IQuery, error) {
	var dataQuery dbox.IQuery

	if dbrowser.QueryType == "nonQueryText" {
		dataQuery = conn.NewQuery().From(dbrowser.TableNames)
	} else if dbrowser.QueryType == "SQL" {
		if toolkit.HasMember(rdbms, datacon.Driver) {
			dataQuery = conn.NewQuery().Command("freequery", toolkit.M{}.
				Set("syntax", dbrowser.QueryText))
		} else {
			return nil, errors.New("Free Text Query with SQL only for RDBMS, please use Dbox")
		}
	} else if dbrowser.QueryType == "Dbox" {
		queryInfo := toolkit.M{}
		toolkit.UnjsonFromString(dbrowser.QueryText, &queryInfo)
		toolkit.Println("queryinfo", queryInfo)

		if qFrom := queryInfo.Get("from", "").(string); qFrom != "" {
			dataQuery = conn.NewQuery()
			dataQuery = dataQuery.From(qFrom)
		}
		if qSelect := queryInfo.Get("select", "").(string); qSelect != "" {
			if qSelect != "*" {
				dataQuery = dataQuery.Select(strings.Split(qSelect, ",")...)
			}
		}
	}
	return dataQuery, nil
}
Esempio n. 6
0
func ReadVariable(f *dbox.Filter, in toolkit.M) *dbox.Filter {
	f.Field = strings.ToLower(f.Field)
	if (f.Op == "$and" || f.Op == "$or") && strings.Contains(reflect.TypeOf(f.Value).String(), "dbox.Filter") {
		fs := f.Value.([]*dbox.Filter)
		for i, ff := range fs {
			bf := ReadVariable(ff, in)
			fs[i] = bf
		}
		f.Value = fs
	} else {
		if reflect.TypeOf(f.Value).Kind() == reflect.Slice {
			fSlice := f.Value.([]interface{})
			// nilai fSlice : [@name1 @name2]
			for i := 0; i < len(fSlice); i++ {
				// nilai fSlice [i] : @name1
				if len(cast.ToString(f.Value)) > 0 && string(cast.ToString(fSlice[i])[0]) == "@" {
					fSlice[i] = in.Get(cast.ToString(fSlice[i]), "")
				}
			}
			f.Value = fSlice
		} else if len(cast.ToString(f.Value)) > 0 && string(cast.ToString(f.Value)[0]) == "@" {
			f.Value = in.Get(cast.ToString(f.Value), "")
		}
	}
	return f
}
Esempio n. 7
0
func (d *DataContext) Find(m IModel, parms tk.M) (dbox.ICursor, error) {
	////_ = "breakpoint"
	q := d.Connection.NewQuery().From(m.TableName())
	if qe := parms.Get(ConfigSelect); qe != nil {
		fields := qe.(string)
		selectFields := strings.Split(fields, ",")
		q = q.Select(selectFields...)
	}
	if qe := parms.Get(ConfigWhere, nil); qe != nil {
		//q = q.Where(qe.(*dbox.Filter))
		filters := qe.([]*dbox.Filter)
		if len(filters) > 0 {
			q = q.Where(dbox.And(filters...))
		}
	}
	if qe := parms.Get(ConfigOrder, nil); qe != nil {
		q = q.Order(qe.([]string)...)
	}
	if qe := parms.Get(ConfigSkip, nil); qe != nil {
		q = q.Skip(qe.(int))
	}
	if qe := parms.Get(ConfigLimit, nil); qe != nil {
		q = q.Take(qe.(int))
	}
	//fmt.Printf("Debug Q: %s\n", tk.JsonString(q))
	return q.Cursor(nil)
	//return c
}
Esempio n. 8
0
func (p *Page) SaveNewWidget(payload toolkit.M, widgetPath string) (*WidgetPage, error) {
	wp := new(WidgetPage)
	wp.ID = payload.Get("widgetPageId", "").(string)
	wp.WidgetID = payload.Get("widgetId", "").(string)
	widget := new(Widget)
	widget.ID = wp.WidgetID
	widget.GetById()
	wp.ConfigDefault = widget.Config

	extractDest := filepath.Join(widgetPath, widget.ID)
	path, err := GetWidgetPath(extractDest)
	if path == "" {
		return nil, errors.New("directory doesn't contains index.html")
	}
	if err != nil {
		return nil, err
	}
	getConfigFile := filepath.Join(path, "config-widget.json")
	result, err := GetJsonFile(getConfigFile)
	if err != nil {
		return nil, err
	}
	if result != nil { /*going to select dataSources field only from config-widget.json*/
		for _, value := range result[0].Get("dataSources").([]interface{}) {
			wp.DataSources = append(wp.DataSources, value.(map[string]interface{}))
		}
	}
	return wp, nil
}
Esempio n. 9
0
func (q *Query) Exec(parm toolkit.M) (e error) {
	data := parm.Get("data")
	driverName := q.GetDriverDB()

	if toolkit.IsSlice(data) && driverName == "mssql" {
		e = q.insertBulk(parm)
	} else {
		_, e = q.ExecOut(parm)
	}
	return e
}
Esempio n. 10
0
func GetMgoValue(d tk.M, fieldName string) interface{} {
	index := strings.Index(fieldName, ".")
	if index < 0 {
		return d.Get(fieldName)
	} else {
		data := d.Get(fieldName[0:index])
		if data != nil {
			return GetMgoValue(data.(tk.M), fieldName[(index+1):len(fieldName)])
		} else {
			return nil
		}
	}
}
Esempio n. 11
0
func Finds(o orm.IModel, param toolkit.M) (dbox.ICursor, error) {
	var filters []*dbox.Filter

	params := toolkit.M{}
	params.Set("where", filters)

	if qe := param.Get("order", nil); qe != nil {
		params.Set("order", qe.([]string))
	}
	if qe := param.Get("skip", nil); qe != nil {
		params.Set("skip", qe.(int))
	}
	if qe := param.Get("take", nil); qe != nil {
		params.Set("limit", qe.(int))
	}
	if qe := param.Get("where", nil); qe != nil {
		filters = append(filters, qe.(*dbox.Filter))
		params.Set("where", filters)
	}

	c, e := ctx().Find(o, params)
	if e != nil {
		return nil, errors.New("Core.Find: " + e.Error())
	}
	return c, nil
}
Esempio n. 12
0
func (c *Cursor) getCondition(condition toolkit.M) bool {
	var flag bool
	var dataCheck toolkit.M

	for i, v := range condition {
		if i == "$and" || i == "$or" {
			flag = true
		} else if v != dataCheck.Get(i, "").(string) {
			flag = false
		}

	}
	return flag
}
Esempio n. 13
0
func (this *DataBrowserController) getExcelValue(data tk.M, field string) (result interface{}) {
	field = strings.ToLower(field)
	numberOfDot := strings.Count(field, ".")
	if numberOfDot > 0 {
		d := data.Get(field[0:strings.Index(field, ".")]).(tk.M)
		newField := field[strings.Index(field, ".")+1 : len(field)]
		result = this.getExcelValue(d, newField)
	} else {
		result = data.Get(field)
	}
	if result == nil {
		result = ""
	}
	return
}
Esempio n. 14
0
func (q *Query) compileUpdateFrom(queryString string, ins toolkit.M) string {
	updateString := func() string {
		if !ins.Has("data") {
			return ""
		}

		var updates []string

		for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
			updates = append(updates, fmt.Sprintf("%s = %s", k, q.getAsString(v)))
		}

		return strings.Join(updates, ", ")
	}()

	queryString = fmt.Sprintf("%sSET %s ", queryString, updateString)
	return queryString
}
Esempio n. 15
0
func (a *Controller) KendoGridSettings(ins toolkit.M) toolkit.M {
	if ins == nil {
		ins = toolkit.M{}
	}
	s := toolkit.M{}
	q_skip := a.Ctx.Input.Query("skip")
	q_page := a.Ctx.Input.Query("page")
	q_size := a.Ctx.Input.Query("pageSize")

	if q_skip != "" {
		s.Set("skip", toolkit.ToInt(q_skip))
	}

	if q_page != "" {
		s.Set("page", toolkit.ToInt(q_page))
	}

	if q_size != "" {
		s.Set("limit", toolkit.ToInt(q_size))
	}

	sortField := strings.ToLower(a.Ctx.Input.Query("sort[0][field]"))
	sortDir := a.Ctx.Input.Query("sort[0][dir]")

	if sortField != "" {
		if sortField == "id" {
			sortField = "_id"
		}
		if sortDir == "" || sortDir == "asc" {
			s.Set("order", []string{sortField})
		} else {
			s.Set("order", []string{"-" + sortField})
		}
	}

	if fqe := a.KendoGridFilter("filter"); fqe != nil {
		if ins.Has("where") {
			fqe = dbs.And(fqe, ins.Get("where").(*dbs.QE))
		}
		s.Set("where", fqe)
	}

	return s
}
Esempio n. 16
0
File: dbox.go Progetto: haibudi/dbox
func MatchM(v toolkit.M, filters []*Filter) bool {
	var match bool

	for _, f := range filters {
		if f.Field != "" {
			//--- if has field: $eq, $ne, $gt, $lt, $gte, $lte, $contains
			//toolkit.Printf("Filter:%s V:%s Has:%s Match:%s", f.Field, toolkit.JsonString(v), v.Has("random"), match)
			if v.Has(f.Field) {
				match = MatchV(v.Get(f.Field), f)
				//toolkit.Printf("Filter:%s Value: %v Match:%s \n", toolkit.JsonString(f), v.Get(f.Field), match)
				if !match {
					return false
				}
			} else {
				if f.Op != FilterOpNoEqual && f.Op != FilterOpNin {
					return false
				}
			}
		} else {
			//-- no field: $and, $or
			//toolkit.Printf("Filter: %s\n", toolkit.JsonString(f))
			if f.Op == FilterOpAnd || f.Op == FilterOpOr {
				filters2 := f.Value.([]*Filter)
				for k, f2 := range filters2 {
					if f.Op == FilterOpAnd {
						if k == 0 {
							match = MatchM(v, []*Filter{f2})
						} else {
							match = match && MatchM(v, []*Filter{f2})
						}
					} else {
						if k == 0 {
							match = MatchM(v, []*Filter{f2})
						} else {
							match = match || MatchM(v, []*Filter{f2})
						}
					}
				}
			}
			//toolkit.Printf("\n")
		}
	}
	return match
}
Esempio n. 17
0
func (q *Query) compileInsertFrom(queryString string, ins toolkit.M) string {
	keyString, valString := func() (string, string) {
		if !ins.Has("data") {
			return "", ""
		}

		keys := []string{}
		vals := []string{}

		for k, v := range ins.Get("data", toolkit.M{}).(toolkit.M) {
			keys = append(keys, k)
			vals = append(vals, q.getAsString(v))
		}

		return strings.Join(keys, ", "), strings.Join(vals, ", ")
	}()

	queryString = fmt.Sprintf("%s(%s) VALUES (%s) ", queryString, keyString, valString)
	return queryString
}
Esempio n. 18
0
func convertDataType(typedt string, dtget string, toolmap toolkit.M) interface{} {
	var resValueEachSF interface{}
	switch typedt {
	case "string":
		resValueEachSF = fmt.Sprintf("%v", toolmap.Get(dtget))
	case "int":
		var resDefault int
		resDefault = toolmap.GetInt(dtget)
		resValueEachSF = resDefault
	case "double":
		var resDefault float32
		resDefault = toolmap.GetFloat32(dtget)
		resValueEachSF = resDefault
	case "bool":
		var resDefault bool
		resDefault, _ = strconv.ParseBool(toolmap.GetString(dtget))
		resValueEachSF = resDefault
	}

	return resValueEachSF
}
Esempio n. 19
0
func (d *DataContext) Get(m IModel, config tk.M) error {
	var e error
	q := d.Connection.NewQuery().SetConfig("pooling", d.Pooling()).From(m.(IModel).TableName())
	if config.Has(ConfigWhere) {
		q = q.Where(config.Get(ConfigWhere).(*dbox.Filter))
	}
	if config.Has(ConfigOrder) {
		q = q.Order(config.Get(ConfigOrder).([]string)...)
	}
	q = q.Take(1)
	//q := d.Connection.NewQuery().From(m.(IModel).TableName()).Where(dbox.Eq("_id", id))
	c, e := q.Cursor(nil)
	if e != nil {
		return err.Error(packageName, modCtx, "Get", "Cursor fail. "+e.Error())
	}
	defer c.Close()
	e = c.Fetch(m, 1, false)
	if e != nil {
		return err.Error(packageName, modCtx, "Get", e.Error())
	}
	return nil
}
Esempio n. 20
0
func (c *Coordinator) Set(in toolkit.M) *toolkit.Result {
	var e error
	if in == nil {
		in = toolkit.M{}
	}
	result := toolkit.NewResult()

	key := in.GetString("key")

	// validation
	if key == "" {
		return result.SetErrorTxt("Key is empty")
	}

	data := in.Get("data", []byte{}).([]byte)
	if len(data) == 0 {
		return result.SetErrorTxt("Data is not valid")
	}

	// get available node
	nodeIdx, e := c.getAvailableNode(data)
	if e != nil {
		return result.SetErrorTxt("Coordinator.Set: " + e.Error())
	}
	node := c.Node(RoleStorage, nodeIdx)

	// write the data
	delete(in, "auth_referenceid")
	delete(in, "auth_secret")
	in.Set("data", data)
	rw := node.Call("write", in)

	result.Data = rw.Data
	result.Status = rw.Status
	if result.Status != toolkit.Status_OK {
		result.SetErrorTxt("Coordinator.Set: " + rw.Message)
	}
	return result
}
Esempio n. 21
0
func checkisonprocess(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	if _, f := mapsnapshot[id]; !f {
		return
	}

	mtkdata := mapsnapshot[id]
	mapcron, _ := toolkit.ToM(intervalconf["cronconf"])
	if mtkdata.Grabstatus == "running" && len(mapcron) <= 0 {
		cond = true
	}

	mtkstarttime := sedotan.StringToDate(mtkdata.Starttime)

	timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
	timeoutsec := time.Second * time.Duration(timeoutint)

	if cond && (thistime.After(mtkstarttime.Add(timeoutsec)) && timeoutint > 0) {
		cond = false
	}

	return
}
Esempio n. 22
0
func TestLoad(t *testing.T) {
	t.Skip()
	if econnect := connect(); econnect != nil {
		t.Error("Unable to connect to database")
		return
	}
	defer close()

	q := ctx.Query().From("tmpOutages").Select("Start Date", "Start Time", "Total")
	c := q.Cursor(nil)
	defer c.Close()
	var m tk.M
	for b, _ := c.Fetch(&m); b; b, _ = c.Fetch(&m) {
		//go func(m tk.M) {
		dt := tk.MakeDate("1/2/06", m.Get("Start Date", "1/1/1980").(string)).UTC()
		tm := tk.MakeDate("03:04� ", m.Get("Start Time", "00:00").(string)).UTC()
		//tm := tk.MakeDate("03:04� ", m.Get("Start Time", "00:00").(string)).UTC().Sub(tk.MakeDate("03:04", "00:00").UTC())
		dt = tk.AddTime(dt, tm)
		fmt.Printf("Data %v has: %v\n",
			dt.Format("2-Jan-2006 03:04"),
			m.GetFloat64("Total"))
		//}(m)
	}
}
Esempio n. 23
0
func (d *DataContext) Find(m IModel, parms tk.M) (dbox.ICursor, error) {
	////_ = "breakpoint"
	q := d.Connection.NewQuery().From(m.TableName())
	if qe := parms.Get("where", nil); qe != nil {
		q = q.Where(qe.([]*dbox.Filter)...)
	}
	if qe := parms.Get("order", nil); qe != nil {
		q = q.Order(qe.([]string)...)
	}
	if qe := parms.Get("skip", nil); qe != nil {
		q = q.Skip(qe.(int))
	}
	if qe := parms.Get("limit", nil); qe != nil {
		q = q.Take(qe.(int))
	}
	//fmt.Printf("Debug Q: %s\n", tk.JsonString(q))
	return q.Cursor(nil)
	//return c
}
Esempio n. 24
0
func (mp *MapPage) Save(payload toolkit.M) error {
	mp.ID = payload.Get("_id", "").(string)
	mp.Title = payload.Get("title", "").(string)
	mp.URL = payload.Get("url", "").(string)
	mp.FileName = mp.ID + ".json"
	page := new(Page)
	page.ID = mp.ID
	page.Title = mp.Title
	page.URL = mp.URL

	// page.ParentMenu = payload.Get("parentMenu", "").(string)
	// page.URL = payload.Get("url", "").(string)
	if err := Save(mp); err != nil {
		return err
	}
	if err := page.Save(payload, false, ""); err != nil {
		return err
	}
	return nil
}
Esempio n. 25
0
func (b *Broadcaster) initRoute() {
	//-- add node
	b.Route("/nodeadd", func(kr *knot.WebContext) interface{} {
		url := kr.Query("node")
		b.Subscibers = append(b.Subscibers, url)
		kr.Server.Log().Info(fmt.Sprintf("Add node %s to %s", url, b.Address))
		return "OK"
	})

	//-- add channel subscribtion
	b.Route("/channelregister", func(k *knot.WebContext) interface{} {
		k.Config.OutputType = knot.OutputJson
		result := toolkit.NewResult()
		cr := &ChannelRegister{}
		k.GetPayload(cr)
		b.addChannelSubcriber(cr.Channel, cr.Subscriber)
		return result
	})

	//-- get the message (used for DstributeAsQue type)
	b.Route("/getmsg", func(k *knot.WebContext) interface{} {
		k.Config.OutputType = knot.OutputJson
		result := toolkit.NewResult()
		tm := toolkit.M{}
		e := k.GetPayload(&tm)
		if e != nil {
			fmt.Println(e.Error())
			result.SetErrorTxt(fmt.Sprintf("Broadcaster GetMsg Payload Error: %s", e.Error()))
		} else {
			key := tm.Get("Key", "").(string)
			if key == "" {
				result.SetErrorTxt("Broadcaste GetMsg Error: No key is provided")
			} else {
				m, exist := b.messages[key]
				//fmt.Println(m.Key + " : " + m.Data.(string))
				if exist == false {
					result.SetErrorTxt("Message " + key + " is not exist")
				} else {
					result.Data = m.Data
					//fmt.Printf("Sent data: %v \n", m.Data)
				}
			}
		}
		return result
	})

	//-- invoked to identify if a msg has been received (used for DistAsQue)
	b.Route("/msgreceived", func(k *knot.WebContext) interface{} {
		k.Config.OutputType = knot.OutputJson
		result := toolkit.NewResult()

		tm := toolkit.M{}
		e := k.GetPayload(&tm)
		if e != nil {
			result.SetErrorTxt("Broadcaster MsgReceived Payload Error: " + e.Error())
		} else {
			key := tm.Get("key", "").(string)
			subscriber := tm.Get("subscriber", "").(string)

			mm, exist := b.messages[key]
			if exist == false {
				result.SetErrorTxt("Broadcaster MsgReceived Error: " + key + " is not exist")
			} else {
				for tIndex, t := range mm.Targets {
					if t == subscriber {
						mm.Status[tIndex] = "OK"
						break
					}
				}
			}
		}

		return result
	})

	//-- gracefully stop the server
	b.Route("/stop", func(k *knot.WebContext) interface{} {
		result := toolkit.NewResult()
		defer func() {
			if result.Status == toolkit.Status_OK {
				k.Server.Stop()
			}
		}()
		for _, s := range b.Subscibers {
			url := "http://" + s + "/stop"
			bs, e := webcall(url, "GET", nil)
			if e != nil {
				result.SetErrorTxt("Unable to stop " + s + ": " + e.Error())
			}
			sresult := toolkit.NewResult()
			toolkit.Unjson(bs, sresult)
			if sresult.Status == toolkit.Status_NOK {
				result.SetErrorTxt("Unable to stop " + s + ": " + sresult.Message)
			}
		}

		k.Config.OutputType = knot.OutputJson
		return result
	})
}
Esempio n. 26
0
//Check Time run and record to snapshot
func checkistimerun(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	var mtklaststarttime, mtklastupdate time.Time
	tempss := Snapshot{Id: id,
		Starttime:      "",
		Laststartgrab:  "",
		Lastupdate:     "",
		Grabcount:      1,
		Rowgrabbed:     0,
		Errorfound:     0,
		Lastgrabstatus: "",
		Grabstatus:     "running",
		Cgtotal:        0,
		Cgprocess:      0,
		Note:           "",
		Pid:            0}
	// 	Lastgrabstatus string //[success|failed]
	// 	Grabstatus     string //[running|done]

	strintervalconf := intervalconf.Get("starttime", "").(string)
	intervalstart := sedotan.StringToDate(strintervalconf)
	strintervaltype := intervalconf.Get("intervaltype", "").(string)
	grabinterval := toolkit.ToInt(intervalconf.Get("grabinterval", 0), toolkit.RoundingAuto)

	mtkdata := Snapshot{}

	if _, f := mapsnapshot[id]; f {
		mtkdata = mapsnapshot[id]
		tempss.Starttime = mtkdata.Starttime
		//for data timeout
		mtklaststarttime = sedotan.StringToDate(mtkdata.Laststartgrab)
		mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)

		timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
		timeoutsec := time.Second * time.Duration(timeoutint)
		if mtkdata.Grabstatus == "running" && thistime.After(mtklaststarttime.Add(timeoutsec)) && timeoutint > 0 {
			mtkdata.Lastupdate = sedotan.DateToString(mtklaststarttime.Add(timeoutsec))
			mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
			mtkdata.Lastgrabstatus = "failed"
			mtkdata.Grabstatus = "done"
		}

		// set initial - found a long time ago snapshot
		if intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Starttime = sedotan.DateToString(thistime)

			mtkdata.Grabcount = 0
			mtkdata.Rowgrabbed = 0
			mtkdata.Errorfound = 0
			mtkdata.Lastgrabstatus = ""
		}
	}

	if mtkdata.Lastgrabstatus == "failed" {
		grabinterval = toolkit.ToInt(intervalconf.Get("timeoutinterval", 0), toolkit.RoundingAuto)
	}

	if strintervalconf != "" && intervalstart.Before(thistime) {
		_, fcond := mapsnapshot[id]

		switch {
		case !fcond || intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)):
			cond = true
		case intervalconf.Get("grabinterval", 0).(float64) > 0:
			var durationgrab time.Duration

			switch strintervaltype {
			case "seconds":
				durationgrab = time.Second * time.Duration(grabinterval)
			case "minutes":
				durationgrab = time.Minute * time.Duration(grabinterval)
			case "hours":
				durationgrab = time.Hour * time.Duration(grabinterval)
			}
			nextgrab := mtklastupdate.Add(durationgrab)

			if nextgrab.Before(thistime) && !mtklastupdate.IsZero() {
				cond = true
				tempss.Grabcount = mtkdata.Grabcount + 1
				tempss.Rowgrabbed = mtkdata.Rowgrabbed
				tempss.Errorfound = mtkdata.Errorfound
				tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
			}
		}
	}

	mapcronconf, _ := toolkit.ToM(intervalconf.Get("cronconf", nil))
	minutetime := sedotan.DateMinutePress(thistime) //review this and the usage and parsing in cron
	if len(mapcronconf) > 0 {
		//min hour dayofmonth month dayofweek
		cond = true
		arrstr := [6]string{"min", "hour", "dayofmonth", "month", "dayofweek"}
		// arrstr := [6]string{"month", "dayofmonth", "dayofweek", "hour", "min", "second"}
		for _, str := range arrstr {
			sval := toolkit.ToString(mapcronconf.Get(str, ""))
			ival := toolkit.ToInt(sval, toolkit.RoundingAuto)

			var valcom int
			switch str {
			// case "second":
			// 	valcom = thistime.Second()
			case "min":
				valcom = thistime.Minute()
			case "hour":
				valcom = thistime.Hour()
			case "dayofmonth":
				valcom = thistime.Day()
			case "month":
				valcom = toolkit.ToInt(thistime.Month(), toolkit.RoundingAuto)
			case "dayofweek":
				valcom = toolkit.ToInt(thistime.Weekday(), toolkit.RoundingAuto)
			}

			if sval != "*" && valcom != ival {
				cond = false
			} else {
				cond = cond && true
			}
		}

		if mtkdata.Laststartgrab != "" {
			cond = cond && minutetime.After(sedotan.StringToDate(mtkdata.Laststartgrab))
		}

		if cond && minutetime.Before(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Grabcount = mtkdata.Grabcount + 1
			tempss.Rowgrabbed = mtkdata.Rowgrabbed
			tempss.Errorfound = mtkdata.Errorfound
			tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
		}
	}

	if cond {
		mapsnapshot[id] = tempss
	}

	return
}
func (d *DataSourceController) parseQuery(query dbox.IQuery, queryInfo toolkit.M) (dbox.IQuery, MetaSave) {
	metaSave := MetaSave{}

	if qFrom := queryInfo.Get("from", "").(string); qFrom != "" {
		query = query.From(qFrom)
	}
	if qSelect := queryInfo.Get("select", "").(string); qSelect != "" {
		if qSelect != "*" {
			query = query.Select(strings.Split(qSelect, ",")...)
		}
	}
	if qTakeRaw, qTakeOK := queryInfo["take"]; qTakeOK {
		if qTake, ok := qTakeRaw.(float64); ok {
			query = query.Take(int(qTake))
		}
		if qTake, ok := qTakeRaw.(int); ok {
			query = query.Take(qTake)
		}
	}
	if qSkipRaw, qSkipOK := queryInfo["skip"]; qSkipOK {
		if qSkip, ok := qSkipRaw.(float64); ok {
			query = query.Take(int(qSkip))
		}
		if qSkip, ok := qSkipRaw.(int); ok {
			query = query.Take(qSkip)
		}
	}
	if qOrder := queryInfo.Get("order", "").(string); qOrder != "" {
		orderAll := map[string]string{}
		err := json.Unmarshal([]byte(qOrder), &orderAll)
		if err == nil {
			orderString := []string{}
			for key, val := range orderAll {
				orderString = append(orderString, key)
				orderString = append(orderString, val)
			}
			query = query.Order(orderString...)
		}
	}

	if qInsert := queryInfo.Get("insert", "").(string); qInsert != "" {
		if qInsert != "" {
			metaSave.keyword = "insert"
			metaSave.data = qInsert
			query = query.Insert()
		}
	}
	if qUpdate := queryInfo.Get("update", "").(string); qUpdate != "" {
		if qUpdate != "" {
			metaSave.keyword = "update"
			metaSave.data = qUpdate
			query = query.Update()
		}
	}
	if _, qDeleteOK := queryInfo["delete"]; qDeleteOK {
		metaSave.keyword = "delete"
		query = query.Delete()
	}
	if qCommand := queryInfo.Get("command", "").(string); qCommand != "" {
		command := map[string]interface{}{}
		err := json.Unmarshal([]byte(qCommand), &command)
		if err == nil {
			for key, value := range command {
				query = query.Command(key, value)
				break
			}
		}
	}

	if qWhere := queryInfo.Get("where", "").(string); qWhere != "" {
		whereAll := []map[string]interface{}{}
		err := json.Unmarshal([]byte(qWhere), &whereAll)
		if err == nil {
			allFilter := []*dbox.Filter{}

			for _, each := range whereAll {
				where, _ := toolkit.ToM(each)
				filter := d.filterParse(where)
				if filter != nil {
					allFilter = append(allFilter, filter)
				}
			}

			query = query.Where(allFilter...)
		}
	}

	return query, metaSave
}
func (d *DataSourceController) filterParse(where toolkit.M) *dbox.Filter {
	field := where.Get("field", "").(string)
	value := fmt.Sprintf("%v", where["value"])

	if key := where.Get("key", "").(string); key == "Eq" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Eq(field, valueInt)
		} else {
			return dbox.Eq(field, value)
		}
	} else if key == "Ne" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Ne(field, valueInt)
		} else {
			return dbox.Ne(field, value)
		}
	} else if key == "Lt" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Lt(field, valueInt)
		} else {
			return dbox.Lt(field, value)
		}
	} else if key == "Lte" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Lte(field, valueInt)
		} else {
			return dbox.Lte(field, value)
		}
	} else if key == "Gt" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Gt(field, valueInt)
		} else {
			return dbox.Gt(field, value)
		}
	} else if key == "Gte" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Gte(field, valueInt)
		} else {
			return dbox.Gte(field, value)
		}
	} else if key == "In" {
		valueArray := []interface{}{}
		for _, e := range strings.Split(value, ",") {
			valueArray = append(valueArray, strings.Trim(e, ""))
		}
		return dbox.In(field, valueArray...)
	} else if key == "Nin" {
		valueArray := []interface{}{}
		for _, e := range strings.Split(value, ",") {
			valueArray = append(valueArray, strings.Trim(e, ""))
		}
		return dbox.Nin(field, valueArray...)
	} else if key == "Contains" {
		return dbox.Contains(field, value)
	} else if key == "Or" {
		subs := where.Get("value", []interface{}{}).([]interface{})
		filtersToMerge := []*dbox.Filter{}
		for _, eachSub := range subs {
			eachWhere, _ := toolkit.ToM(eachSub)
			filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
		}
		return dbox.Or(filtersToMerge...)
	} else if key == "And" {
		subs := where.Get("value", []interface{}{}).([]interface{})
		filtersToMerge := []*dbox.Filter{}
		for _, eachSub := range subs {
			eachWhere, _ := toolkit.ToM(eachSub)
			filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
		}
		return dbox.And(filtersToMerge...)
	}

	return nil
}
Esempio n. 29
0
func foundCondition(dataCheck toolkit.M, cond toolkit.M) bool {
	resBool := false

	for key, val := range cond {
		if key == "$and" || key == "$or" {
			for i, sVal := range val.([]interface{}) {
				rVal := sVal.(map[string]interface{})
				mVal := toolkit.M{}
				for rKey, mapVal := range rVal {
					mVal.Set(rKey, mapVal)
				}

				xResBool := foundCondition(dataCheck, mVal)
				if key == "$and" {
					if i == 0 {
						resBool = xResBool
					} else {
						resBool = resBool && xResBool
					}
				} else {
					if i == 0 {
						resBool = xResBool
					} else {
						resBool = resBool || xResBool
					}
				}
			}
		} else {
			if reflect.ValueOf(val).Kind() == reflect.Map {
				mVal := val.(map[string]interface{})
				tomVal, _ := toolkit.ToM(mVal)
				switch {
				case tomVal.Has("$eq"):
					if tomVal["$eq"].(string) == dataCheck.Get(key, "").(string) {
						resBool = true
					}
				case tomVal.Has("$ne"):
					if tomVal["$ne"].(string) != dataCheck.Get(key, "").(string) {
						resBool = true
					}
				case tomVal.Has("$regex"):
					resBool, _ = regexp.MatchString(tomVal["$regex"].(string), dataCheck.Get(key, "").(string))
				case tomVal.Has("$gt"):
					if tomVal["$gt"].(string) > dataCheck.Get(key, "").(string) {
						resBool = true
					}
				case tomVal.Has("$gte"):
					if tomVal["$gte"].(string) >= dataCheck.Get(key, "").(string) {
						resBool = true
					}
				case tomVal.Has("$lt"):
					if tomVal["$lt"].(string) < dataCheck.Get(key, "").(string) {
						resBool = true
					}
				case tomVal.Has("$lte"):
					if tomVal["$lte"].(string) <= dataCheck.Get(key, "").(string) {
						resBool = true
					}
				}
			} else if reflect.ValueOf(val).Kind() == reflect.String && val != dataCheck.Get(key, "").(string) {
				resBool = true
			}
		}
	}

	return resBool
}
Esempio n. 30
0
func (q *Query) Exec(parm toolkit.M) error {
	var e error
	if parm == nil {
		parm = toolkit.M{}
	}

	dbname := q.Connection().Info().Database
	driverName := q.GetDriverDB()
	// driverName = "oracle"
	tablename := ""
	data := parm.Get("data", nil)
	// fmt.Println("Hasil ekstraksi Param : ", data)

	/*========================EXTRACT FIELD, DATA AND FORMAT DATE=============================*/

	var attributes string
	var values string
	var setUpdate, statement string
	var i int

	dataM, e := toolkit.ToM(data)
	if e != nil {
		return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
	}

	if data != nil {
		for field, val := range dataM {
			namaField := field
			dataValues := val
			stringValues := StringValue(dataValues, driverName)
			if i == 0 {
				attributes = "(" + namaField
				setUpdate = namaField + " = " + stringValues
				values = "(" + stringValues

			} else {
				attributes += ", " + namaField
				setUpdate += ", " + namaField + " = " + stringValues
				values += ", " + stringValues

			}
			i += 1
		}
		attributes += ")"
		values += ")"
	}

	/*=================================END OF EXTRACTION=======================================*/

	temp := ""
	quyerParts := q.Parts()
	c := crowd.From(&quyerParts)
	groupParts := c.Group(func(x interface{}) interface{} {
		qp := x.(*dbox.QueryPart)
		temp = toolkit.JsonString(qp)
		return qp.PartType
	}, nil).Exec()
	parts := map[interface{}]interface{}{}
	if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
		for _, kv := range groupParts.Result.Data().([]crowd.KV) {
			parts[kv.Key] = kv.Value
		}
	}

	fromParts, hasFrom := parts[dbox.QueryPartFrom]
	if !hasFrom {

		return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
	}
	tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)

	var where interface{}
	whereParts, hasWhere := parts[dbox.QueryPartWhere]
	if hasWhere {
		fb := q.Connection().Fb()
		for _, p := range whereParts.([]*dbox.QueryPart) {
			fs := p.Value.([]*dbox.Filter)
			for _, f := range fs {
				fb.AddFilter(f)
			}
		}

		where, e = fb.Build()
		if e != nil {

		} else {

		}

	}
	commandType := ""
	multi := false

	_, hasDelete := parts[dbox.QueryPartDelete]
	_, hasInsert := parts[dbox.QueryPartInsert]
	_, hasUpdate := parts[dbox.QueryPartUpdate]
	_, hasSave := parts[dbox.QueryPartSave]

	if hasDelete {
		commandType = dbox.QueryPartDelete
	} else if hasInsert {
		commandType = dbox.QueryPartInsert
	} else if hasUpdate {
		commandType = dbox.QueryPartUpdate
	} else if hasSave {
		commandType = dbox.QueryPartSave
	}

	var id string
	var idVal interface{}
	if data == nil {
		multi = true
	} else {
		if where == nil {
			id, idVal = toolkit.IdInfo(data)
			if id != "" {
				where = id + " = " + StringValue(idVal, "non")
			}
		} else {
			multi = true
		}
	}

	session := q.Session()
	sessionHive := q.SessionHive()

	if dbname != "" && tablename != "" && multi == true {

	}
	if commandType == dbox.QueryPartInsert {
		if attributes != "" && values != "" {
			if driverName == "hive" {
				statement = "INSERT INTO " + tablename + " VALUES " + values
				e = sessionHive.Exec(statement, nil)
			} else {
				statement = "INSERT INTO " + tablename + " " + attributes + " VALUES " + values
				_, e = session.Exec(statement)
			}

			if e != nil {
				fmt.Println(e.Error())
			}
		} else {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	} else if commandType == dbox.QueryPartUpdate {
		if setUpdate != "" {
			var statement string
			if where != nil {
				statement = "UPDATE " + tablename + " SET " + setUpdate +
					" WHERE " + cast.ToString(where)
			} else {
				statement = "UPDATE " + tablename + " SET " + setUpdate
			}
			if driverName == "hive" {
				e = sessionHive.Exec(statement, nil)
			} else {
				_, e = session.Exec(statement)
			}
			if e != nil {
				return errorlib.Error(packageName, modQuery+".Exec", commandType,
					cast.ToString(e.Error()))
			}
		} else {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	} else if commandType == dbox.QueryPartDelete {
		var statement string
		if where != nil {
			statement = "DELETE FROM " + tablename + " where " + cast.ToString(where)
		} else {
			statement = "DELETE FROM " + tablename
		}
		if driverName == "hive" {
			e = sessionHive.Exec(statement, nil)
		} else {
			_, e = session.Exec(statement)
		}
		if e != nil {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				cast.ToString(e.Error()))
		}
	} else if commandType == dbox.QueryPartSave {
		if attributes != "" && values != "" {
			var querystmt string
			if where != nil {
				querystmt = "select 1 as data from " + tablename + " where " + cast.ToString(where)
			}
			var rowCount int
			if driverName == "hive" {
				rowCount = 0
				// row := sessionHive.Exec(querystmt, nil)
				// rowCount = toolkit.ToInt(row[0], "auto")
			} else {
				if querystmt != "" {
					rows, _ := session.Query(querystmt)
					for rows.Next() {
						rows.Scan(&rowCount)
					}
				}
			}

			var statement string
			if rowCount == 0 || where == nil {
				if driverName == "hive" {
					statement = "INSERT INTO " + tablename + " VALUES " + values
				} else {
					statement = "INSERT INTO " + tablename + " " +
						attributes + " VALUES " + values
				}
			} else {
				statement = "UPDATE " + tablename + " SET " + setUpdate +
					" WHERE " + cast.ToString(where)
			}

			if driverName == "hive" {
				e = sessionHive.Exec(statement, nil)
			} else {
				_, e = session.Exec(statement)
			}
			if e != nil {
				return errorlib.Error(packageName, modQuery+".Exec", commandType,
					cast.ToString(e.Error()))
			}

		} else if values == "" {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	}
	if e != nil {
		return errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
	}
	return nil
}