Example #1
0
func getConfig() (err error) {
	var result interface{}

	if strings.Contains(configpath, "http") {
		res, err := http.Get(configpath)
		checkfatalerror(err)
		defer res.Body.Close()

		decoder := json.NewDecoder(res.Body)
		err = decoder.Decode(&result)
		checkfatalerror(err)
	} else {
		bytes, err := ioutil.ReadFile(configpath)
		checkfatalerror(err)

		err = json.Unmarshal(bytes, &result)
		checkfatalerror(err)
	}

	switch toolkit.TypeName(result) {
	case "[]interface {}":
		isFound := false
		for _, eachRaw := range result.([]interface{}) {
			each := eachRaw.(map[string]interface{})
			if each["_id"].(string) == _id {
				m := toolkit.M{}
				m, err = toolkit.ToM(each)
				checkfatalerror(err)

				config = m
				isFound = true
			}
		}

		if !isFound {
			checkfatalerror(errors.New(fmt.Sprintf("config with _id %s is not found\n%#v", _id, result)))
		}
	case "map[string]interface {}":
		m := toolkit.M{}
		m, err = toolkit.ToM(result)
		checkfatalerror(err)
		config = m
	default:
		checkfatalerror(errors.New(fmt.Sprintf("invalid config file\n%#v", result)))
	}

	return
}
Example #2
0
func StopProcess(datas []interface{}) (error, bool) {
	var (
		e          error
		isStopServ bool
	)
	for _, v := range datas {
		vToMap, _ := toolkit.ToM(v)
		if knot.SharedObject().Get(vToMap["nameid"].(string)) != nil {
			e, isStopServ = StopService(knot.SharedObject().Get(vToMap["nameid"].(string)).(*sdt.GrabService))
			if e != nil {
				return e, false
			}
		} else {
			return e, false
		}
	}

	if !isStopServ {
		isStopServ = true
	} else {
		isStopServ = false
	}

	return nil, isStopServ
}
Example #3
0
func (w *WebGrabberController) StopService(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	w.PrepareHistoryPath()

	payload := new(colonycore.WebGrabber)
	err := r.GetPayload(payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	err = colonycore.Get(payload, payload.ID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	o, err := toolkit.ToM(payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	err, isRun := modules.StopProcess([]interface{}{o})
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(isRun, nil, "")
}
Example #4
0
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	w.PrepareHistoryPath()

	payload := struct {
		ID   string `json:"_id"`
		Date string `json:"date"`
	}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	wg := new(colonycore.WebGrabber)
	err = colonycore.Get(wg, payload.ID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	o, err := toolkit.ToM(wg)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	history := modules.NewHistory(payload.ID)
	logs := history.GetLog([]interface{}{o}, payload.Date)

	return helper.CreateResult(true, logs, "")
}
Example #5
0
func getresultobj(strsplits []string, tval interface{}, val toolkit.M) interface{} {
	var xval interface{}
	if val == nil {
		val = toolkit.M{}
	}

	switch {
	case len(strsplits) == 1:
		xval = tval
	case len(strsplits) > 1:
		tm := toolkit.M{}
		if val.Has(strsplits[1]) {
			tm, _ = toolkit.ToM(val[strsplits[1]])
		}
		xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
	}

	return xval
	// if  {
	// 	val = tval
	// } else {
	// 	val = val.Set(k, v)
	// 	if val == nil {

	// 	} else {

	// 	}

	// 	val = getresultobj(strsplits[1:], tval)
	// }

	// return val
}
Example #6
0
func Process(datas []interface{}) (error, bool) {
	var (
		e error
		// isServRun bool
	)
	for _, v := range datas {
		vToMap, e := toolkit.ToM(v)
		if e != nil {
			return e, false
		}

		if vToMap["sourcetype"].(string) == "SourceType_Http" {
			grabs, e = GrabHtmlConfig(vToMap)
			if e != nil {
				return e, false
			}
		} else if vToMap["sourcetype"].(string) == "SourceType_DocExcel" {
			grabs, e = GrabDocConfig(vToMap)
			if e != nil {
				return e, false
			}
		}

	}

	e, isServRun := StartService(grabs)
	if e != nil {
		return e, false
	}

	return nil, isServRun
}
Example #7
0
func foundCondition(dataCheck toolkit.M, cond toolkit.M) bool {
	resBool := true

	for key, val := range cond {
		if key == "$and" || key == "$or" {
			for i, sVal := range val.([]interface{}) {
				rVal := sVal.(map[string]interface{})
				mVal := toolkit.M{}
				for rKey, mapVal := range rVal {
					mVal.Set(rKey, mapVal)
				}

				xResBool := foundCondition(dataCheck, mVal)
				if key == "$and" {
					resBool = resBool && xResBool
				} else {
					if i == 0 {
						resBool = xResBool
					} else {
						resBool = resBool || xResBool
					}
				}
			}
		} else {
			if reflect.ValueOf(val).Kind() == reflect.Map {
				mVal := val.(map[string]interface{})
				tomVal, _ := toolkit.ToM(mVal)
				switch {
				case tomVal.Has("$ne"):
					if tomVal["$ne"].(string) == dataCheck.Get(key, "").(string) {
						resBool = false
					}
				case tomVal.Has("$regex"):
					resBool, _ = regexp.MatchString(tomVal["$regex"].(string), dataCheck.Get(key, "").(string))
				case tomVal.Has("$gt"):
					if tomVal["$gt"].(string) >= dataCheck.Get(key, "").(string) {
						resBool = false
					}
				case tomVal.Has("$gte"):
					if tomVal["$gte"].(string) > dataCheck.Get(key, "").(string) {
						resBool = false
					}
				case tomVal.Has("$lt"):
					if tomVal["$lt"].(string) <= dataCheck.Get(key, "").(string) {
						resBool = false
					}
				case tomVal.Has("$lte"):
					if tomVal["$lte"].(string) < dataCheck.Get(key, "").(string) {
						resBool = false
					}
				}
			} else if reflect.ValueOf(val).Kind() == reflect.String && val != dataCheck.Get(key, "").(string) {
				resBool = false
			}
		}
	}

	return resBool
}
Example #8
0
func filterCondition(cond toolkit.M) *dbox.Filter {
	fb := new(dbox.Filter)

	for key, val := range cond {
		if key == "$and" || key == "$or" {
			afb := []*dbox.Filter{}
			for _, sVal := range val.([]interface{}) {
				rVal := sVal.(map[string]interface{})
				mVal := toolkit.M{}
				for rKey, mapVal := range rVal {
					mVal.Set(rKey, mapVal)
				}

				afb = append(afb, filterCondition(mVal))
			}

			if key == "$and" {
				fb = dbox.And(afb...)
			} else {
				fb = dbox.Or(afb...)
			}

		} else {
			if reflect.ValueOf(val).Kind() == reflect.Map {
				mVal := val.(map[string]interface{})
				tomVal, _ := toolkit.ToM(mVal)
				switch {
				case tomVal.Has("$eq"):
					fb = dbox.Eq(key, tomVal["$eq"].(string))
				case tomVal.Has("$ne"):
					fb = dbox.Ne(key, tomVal["$ne"].(string))
				case tomVal.Has("$regex"):
					fb = dbox.Contains(key, tomVal["$regex"].(string))
				case tomVal.Has("$gt"):
					fb = dbox.Gt(key, tomVal["$gt"].(string))
				case tomVal.Has("$gte"):
					fb = dbox.Gte(key, tomVal["$gte"].(string))
				case tomVal.Has("$lt"):
					fb = dbox.Lt(key, tomVal["$lt"].(string))
				case tomVal.Has("$lte"):
					fb = dbox.Lte(key, tomVal["$lte"].(string))
				}
			} else {
				fb = dbox.Eq(key, val)
			}
		}
	}

	return fb
}
Example #9
0
func (s *Server) Write(datakey DataKey, data []byte) error {
	handleError := func(e error) error {
		return errorlib.Error(packageName, modServer, "Write",
			fmt.Sprintf("[%s.%s.%s] ", datakey.Owner, datakey.Cluster, datakey.Key)+e.Error())
	}

	if s.ServerType == ServerTypeMaster || s.ServerType == ServerTypeReplicaMaster {
		dataSize := len(data)
		snode, e := s.getNode(ServerTypeNode, dataSize)
		if e != nil {
			return handleError(e)
		}

		snodes := []*Server{snode}
		srepma, e := s.getNode(ServerTypeReplicaMaster, dataSize)
		if srepma != nil {
			snodes = append(snodes, srepma)
		}

		for _, sn := range snodes {
			mDataKey, e := toolkit.ToM(datakey)
			if e != nil {
				return handleError(e)
			}
			_, e = sn.call(opWrite, "GET", mDataKey, data)
			if e != nil {
				return handleError(e)
			}

			e = s.writeDataMap(datakey, dataSize, sn)
			if e != nil {
				return handleError(e)
			}
		}
	} else {
		key := datakey.BuildKey()
		d := new(Data)
		d.Key = key
		d.Type = datakey.DataType
		if datakey.Expiry == 0 {
			d.Expiry = time.Now().Add(100 * 365 * 24 * 60 * time.Minute)
		} else {
			d.Expiry = time.Now().Add(time.Duration(datakey.Expiry))
		}
		d.Value = data
		s.Items[key] = d
	}
	return nil
}
Example #10
0
func (a *Login) GetACLConnectionInfo() (string, *dbox.ConnectionInfo) {
	conf, err := toolkit.ToM(GetConfig(CONF_DB_ACL))
	if err != nil {
		return "", nil
	}

	ci := dbox.ConnectionInfo{
		conf.GetString("host"),
		conf.GetString("db"),
		conf.GetString("user"),
		conf.GetString("pass"),
		toolkit.M{}.Set("timeout", 3),
	}

	return conf.GetString("driver"), &ci
}
Example #11
0
func (h *HistoryModule) GetLog(datas []interface{}, date string) interface{} {
	for _, v := range datas {
		vMap, _ := toolkit.ToM(v)

		logConf := vMap["logconf"].(map[string]interface{})
		dateNow := time.Now()
		dateNowFormat := dateNow.Format(logConf["filepattern"].(string))
		fileName := fmt.Sprintf("%s-%s", logConf["filename"], dateNowFormat)
		h.logPath = f.Join(logConf["logpath"].(string), fileName)
	}

	file, err := os.Open(h.logPath)
	if err != nil {
		return err.Error()
	}
	defer file.Close()

	getHours := strings.Split(date, ":")
	containString := getHours[0] + ":" + getHours[1]
	scanner := bufio.NewScanner(file)
	lines := 0
	containLines := 0

	var logs []interface{}
	for scanner.Scan() {
		lines++
		contains := strings.Contains(scanner.Text(), containString)
		if contains {
			containLines = lines
		}

		if lines == containLines {
			logs = append(logs, "<li>"+scanner.Text()+"</li>")
		}
	}

	if err := scanner.Err(); err != nil {
		return err.Error()
	}

	var addSlice = toolkit.M{}
	addSlice.Set("logs", logs)

	return addSlice
}
func (d *DataSourceController) TestConnection(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := map[string]interface{}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	driver := payload["Driver"].(string)
	host := payload["Host"].(string)
	database := payload["Database"].(string)
	username := payload["UserName"].(string)
	password := payload["Password"].(string)
	var settings toolkit.M = nil

	if settingsRaw := d.parseSettings(payload["Settings"], nil); settingsRaw != nil {
		settings, err = toolkit.ToM(settingsRaw)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}
	}

	if err := d.checkIfDriverIsSupported(driver); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	fakeDataConn := &colonycore.Connection{
		Database: database,
		Driver:   driver,
		Host:     host,
		UserName: username,
		Password: password,
		Settings: settings,
	}
	err = helper.ConnectUsingDataConn(fakeDataConn).CheckIfConnected()
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, nil, "")
}
Example #13
0
func (g *GrabModule) CheckStat(datas []interface{}) interface{} {
	var (
		grabStatus         interface{}
		lastDate, nextDate string
		//toolkit.M
	)
	var summaryNotes = toolkit.M{} //map[string]interface{}{}
	for _, v := range datas {
		vToMap, e := toolkit.ToM(v)
		if e != nil {
			return e.Error()
		}

		if knot.SharedObject().Get(vToMap["nameid"].(string)) != nil {
			i := knot.SharedObject().Get(vToMap["nameid"].(string)).(*sdt.GrabService)
			tLast := cast.Date2String(i.LastGrabExe, "YYYY/MM/dd HH:mm:ss") //i.LastGrabExe.Format("2006/01/02 15:04:05")
			if tLast != "0001/01/01 00:00:00" {
				lastDate = tLast
			}
			tNext := cast.Date2String(i.NextGrabExe, "YYYY/MM/dd HH:mm:ss") //i.NextGrabExe.Format("2006/01/02 15:04:05")
			if tNext != "0001/01/01 00:00:00" {
				nextDate = tNext
			}

			startdate := cast.Date2String(i.StartDate, "YYYY/MM/dd HH:mm:ss") //i.StartDate.Format("2006/01/02 15:04:05")
			enddate := cast.Date2String(i.EndDate, "YYYY/MM/dd HH:mm:ss")     //i.EndDate.Format("2006/01/02 15:04:05")
			summaryNotes.Set("startDate", startdate)
			summaryNotes.Set("endDate", enddate)
			summaryNotes.Set("grabCount", i.GrabCount)
			summaryNotes.Set("rowGrabbed", i.RowGrabbed)
			summaryNotes.Set("errorFound", i.ErrorFound)

			grabStatus = ReadLog(vToMap["logconf"], i.ServiceRunningStat, i.Name, lastDate, nextDate, i.LastGrabStat, summaryNotes)
		} else {
			summaryNotes.Set("errorFound", 0)
			grabStatus = ReadLog(vToMap["logconf"], false, vToMap["nameid"].(string), "", "", false, summaryNotes)
		}
	}

	return grabStatus
}
Example #14
0
func GetChildMenu(r *knot.WebContext, childMenu []colonycore.Menu) interface{} {
	sessionId := r.Session("sessionid", "")
	results := make([]toolkit.M, 0, 0)
	for _, m := range childMenu {
		result := toolkit.M{}
		acc := acl.HasAccess(toolkit.ToString(sessionId), acl.IDTypeSession, m.AccessId, acl.AccessRead)
		result, err := toolkit.ToM(m)
		if err != nil {
			fmt.Println(err)
		}
		if acc {
			if len(m.Childrens) > 0 {
				childs := GetChildMenu(r, m.Childrens)
				result.Set("childrens", childs)
			}
			result.Set("detail", 7)
			results = append(results, result)
		}
	}
	return results
}
Example #15
0
func checkisonprocess(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	if _, f := mapsnapshot[id]; !f {
		return
	}

	mtkdata := mapsnapshot[id]
	mapcron, _ := toolkit.ToM(intervalconf["cronconf"])
	if mtkdata.Grabstatus == "running" && len(mapcron) <= 0 {
		cond = true
	}

	mtkstarttime := sedotan.StringToDate(mtkdata.Starttime)

	timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
	timeoutsec := time.Second * time.Duration(timeoutint)

	if cond && (thistime.After(mtkstarttime.Add(timeoutsec)) && timeoutint > 0) {
		cond = false
	}

	return
}
Example #16
0
func (w *WebGrabberController) Stat(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	w.PrepareHistoryPath()

	payload := new(colonycore.WebGrabber)
	err := r.GetPayload(payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	err = colonycore.Get(payload, payload.ID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	o, err := toolkit.ToM(payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	grabStatus := modules.NewGrabService().CheckStat([]interface{}{o})
	return helper.CreateResult(true, grabStatus, "")
}
Example #17
0
func (a *GroupController) RefreshGroupLdap(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	payload := map[string]interface{}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	group := new(acl.Group)
	err = acl.FindByID(group, payload["groupid"].(string))
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}
	config, err := toolkit.ToM(group.MemberConf)
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}
	err = acl.RefreshUserLdapByGroup(payload["groupid"].(string), config)
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}

	return helper.CreateResult(true, nil, "success")
}
func (d *DataSourceController) parseQuery(query dbox.IQuery, queryInfo toolkit.M) (dbox.IQuery, MetaSave) {
	metaSave := MetaSave{}

	if qFrom := queryInfo.Get("from", "").(string); qFrom != "" {
		query = query.From(qFrom)
	}
	if qSelect := queryInfo.Get("select", "").(string); qSelect != "" {
		if qSelect != "*" {
			query = query.Select(strings.Split(qSelect, ",")...)
		}
	}
	if qTakeRaw, qTakeOK := queryInfo["take"]; qTakeOK {
		if qTake, ok := qTakeRaw.(float64); ok {
			query = query.Take(int(qTake))
		}
		if qTake, ok := qTakeRaw.(int); ok {
			query = query.Take(qTake)
		}
	}
	if qSkipRaw, qSkipOK := queryInfo["skip"]; qSkipOK {
		if qSkip, ok := qSkipRaw.(float64); ok {
			query = query.Take(int(qSkip))
		}
		if qSkip, ok := qSkipRaw.(int); ok {
			query = query.Take(qSkip)
		}
	}
	if qOrder := queryInfo.Get("order", "").(string); qOrder != "" {
		orderAll := map[string]string{}
		err := json.Unmarshal([]byte(qOrder), &orderAll)
		if err == nil {
			orderString := []string{}
			for key, val := range orderAll {
				orderString = append(orderString, key)
				orderString = append(orderString, val)
			}
			query = query.Order(orderString...)
		}
	}

	if qInsert := queryInfo.Get("insert", "").(string); qInsert != "" {
		if qInsert != "" {
			metaSave.keyword = "insert"
			metaSave.data = qInsert
			query = query.Insert()
		}
	}
	if qUpdate := queryInfo.Get("update", "").(string); qUpdate != "" {
		if qUpdate != "" {
			metaSave.keyword = "update"
			metaSave.data = qUpdate
			query = query.Update()
		}
	}
	if _, qDeleteOK := queryInfo["delete"]; qDeleteOK {
		metaSave.keyword = "delete"
		query = query.Delete()
	}
	if qCommand := queryInfo.Get("command", "").(string); qCommand != "" {
		command := map[string]interface{}{}
		err := json.Unmarshal([]byte(qCommand), &command)
		if err == nil {
			for key, value := range command {
				query = query.Command(key, value)
				break
			}
		}
	}

	if qWhere := queryInfo.Get("where", "").(string); qWhere != "" {
		whereAll := []map[string]interface{}{}
		err := json.Unmarshal([]byte(qWhere), &whereAll)
		if err == nil {
			allFilter := []*dbox.Filter{}

			for _, each := range whereAll {
				where, _ := toolkit.ToM(each)
				filter := d.filterParse(where)
				if filter != nil {
					allFilter = append(allFilter, filter)
				}
			}

			query = query.Where(allFilter...)
		}
	}

	return query, metaSave
}
func (d *DataSourceController) filterParse(where toolkit.M) *dbox.Filter {
	field := where.Get("field", "").(string)
	value := fmt.Sprintf("%v", where["value"])

	if key := where.Get("key", "").(string); key == "Eq" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Eq(field, valueInt)
		} else {
			return dbox.Eq(field, value)
		}
	} else if key == "Ne" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Ne(field, valueInt)
		} else {
			return dbox.Ne(field, value)
		}
	} else if key == "Lt" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Lt(field, valueInt)
		} else {
			return dbox.Lt(field, value)
		}
	} else if key == "Lte" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Lte(field, valueInt)
		} else {
			return dbox.Lte(field, value)
		}
	} else if key == "Gt" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Gt(field, valueInt)
		} else {
			return dbox.Gt(field, value)
		}
	} else if key == "Gte" {
		valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
		if errv == nil {
			return dbox.Gte(field, valueInt)
		} else {
			return dbox.Gte(field, value)
		}
	} else if key == "In" {
		valueArray := []interface{}{}
		for _, e := range strings.Split(value, ",") {
			valueArray = append(valueArray, strings.Trim(e, ""))
		}
		return dbox.In(field, valueArray...)
	} else if key == "Nin" {
		valueArray := []interface{}{}
		for _, e := range strings.Split(value, ",") {
			valueArray = append(valueArray, strings.Trim(e, ""))
		}
		return dbox.Nin(field, valueArray...)
	} else if key == "Contains" {
		return dbox.Contains(field, value)
	} else if key == "Or" {
		subs := where.Get("value", []interface{}{}).([]interface{})
		filtersToMerge := []*dbox.Filter{}
		for _, eachSub := range subs {
			eachWhere, _ := toolkit.ToM(eachSub)
			filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
		}
		return dbox.Or(filtersToMerge...)
	} else if key == "And" {
		subs := where.Get("value", []interface{}{}).([]interface{})
		filtersToMerge := []*dbox.Filter{}
		for _, eachSub := range subs {
			eachWhere, _ := toolkit.ToM(eachSub)
			filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
		}
		return dbox.And(filtersToMerge...)
	}

	return nil
}
Example #20
0
func (q *Query) ExecOut(parm toolkit.M) (int64, error) {
	var e error
	if parm == nil {
		parm = toolkit.M{}
	}

	driverName := q.GetDriverDB()
	// driverName = "oracle"
	tablename := ""
	data := parm.Get("data")

	var attributes string
	var values string
	var setUpdate string

	var dataM toolkit.M
	var dataMs []toolkit.M
	var returnId int64

	if toolkit.IsSlice(data) {
		e = toolkit.Unjson(toolkit.Jsonify(data), &dataMs)
		if e != nil {
			return returnId, errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
		}
	} else {
		dataM, e = toolkit.ToM(data)
		dataMs = append(dataMs, dataM)
		if e != nil {
			return returnId, errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
		}
	}

	for _, dataVal := range dataMs {
		temp := ""
		quyerParts := q.Parts()
		c := crowd.From(&quyerParts)
		groupParts := c.Group(func(x interface{}) interface{} {
			qp := x.(*dbox.QueryPart)
			temp = toolkit.JsonString(qp)
			return qp.PartType
		}, nil).Exec()
		parts := map[interface{}]interface{}{}
		if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
			for _, kv := range groupParts.Result.Data().([]crowd.KV) {
				parts[kv.Key] = kv.Value
			}
		}

		commandType := ""

		_, hasDelete := parts[dbox.QueryPartDelete]
		_, hasInsert := parts[dbox.QueryPartInsert]
		_, hasUpdate := parts[dbox.QueryPartUpdate]
		_, hasSave := parts[dbox.QueryPartSave]

		if hasDelete {
			commandType = dbox.QueryPartDelete
		} else if hasInsert {
			commandType = dbox.QueryPartInsert
		} else if hasUpdate {
			commandType = dbox.QueryPartUpdate
		} else if hasSave {
			commandType = dbox.QueryPartSave
		}

		if hasInsert || hasUpdate || hasSave {
			attributes, setUpdate, values = extractData(dataVal, driverName)
		} else if hasDelete {

		}

		fromParts, hasFrom := parts[dbox.QueryPartFrom]
		if !hasFrom {
			return returnId, errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
		}
		tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)

		var where interface{}
		whereParts, hasWhere := parts[dbox.QueryPartWhere]
		if hasWhere {
			fb := q.Connection().Fb()
			for _, p := range whereParts.([]*dbox.QueryPart) {
				fs := p.Value.([]*dbox.Filter)
				for _, f := range fs {
					fb.AddFilter(f)
				}
			}

			where, e = fb.Build()
			if e != nil {

			} else {

			}

		}

		var id string
		var idVal interface{}

		if where == nil {
			id, idVal = toolkit.IdInfo(dataVal)
			if id != "" {
				where = id + " = " + StringValue(idVal, "non")
			}
		}

		session := q.Session()
		sessionHive := q.SessionHive()

		if commandType == dbox.QueryPartInsert {
			if attributes != "" && values != "" {
				var statement string
				if driverName == "hive" {
					statement = "INSERT INTO " + tablename + " VALUES " + values
					e = sessionHive.Exec(statement, nil)
				} else {
					statement = "INSERT INTO " + tablename + " " + attributes + " VALUES " + values
					var res sql.Result
					res, e = session.Exec(statement)
					if res != nil {
						returnId, _ = res.LastInsertId()
					}
				}

				if e != nil {
					return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
						cast.ToString(e.Error()))
				}
			} else {
				return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
					"please provide the data")
			}
		} else if commandType == dbox.QueryPartUpdate {
			if setUpdate != "" {
				var statement string
				if where != nil {
					statement = "UPDATE " + tablename + " SET " + setUpdate +
						" WHERE " + cast.ToString(where)
				} else {
					statement = "UPDATE " + tablename + " SET " + setUpdate
				}
				if driverName == "hive" {
					e = sessionHive.Exec(statement, nil)
				} else {
					_, e = session.Exec(statement)
				}
				if e != nil {
					return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
						cast.ToString(e.Error()))
				}
			} else {
				return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
					"please provide the data")
			}

		} else if commandType == dbox.QueryPartDelete {
			var statement string
			if where != nil {
				statement = "DELETE FROM " + tablename + " where " + cast.ToString(where)
			} else {
				statement = "DELETE FROM " + tablename
			}
			if driverName == "hive" {
				e = sessionHive.Exec(statement, nil)
			} else {
				_, e = session.Exec(statement)
			}
			if e != nil {
				return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
					cast.ToString(e.Error()))
			}
		} else if commandType == dbox.QueryPartSave {
			if attributes != "" && values != "" {
				var querystmt string
				if where != nil {
					querystmt = "select 1 as data from " + tablename + " where " + cast.ToString(where)
				}
				var rowCount int
				if driverName == "hive" {
					rowCount = 0
					// row := sessionHive.Exec(querystmt, nil)
					// rowCount = toolkit.ToInt(row[0], "auto")
				} else {
					if querystmt != "" {
						rows, _ := session.Query(querystmt)
						for rows.Next() {
							rows.Scan(&rowCount)
						}
					}
				}

				var statement string
				if rowCount == 0 || where == nil {
					if driverName == "hive" {
						statement = "INSERT INTO " + tablename + " VALUES " + values
					} else {
						statement = "INSERT INTO " + tablename + " " +
							attributes + " VALUES " + values
					}
				} else {
					statement = "UPDATE " + tablename + " SET " + setUpdate +
						" WHERE " + cast.ToString(where)
				}

				if driverName == "hive" {
					e = sessionHive.Exec(statement, nil)
				} else {
					_, e = session.Exec(statement)
				}
				if e != nil {
					return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
						cast.ToString(e.Error()))
				}

			} else if values == "" {
				return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
					"please provide the data")
			}

		}
		if e != nil {
			return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
		}
	}
	return returnId, nil
}
Example #21
0
func (q *Query) Exec(parm toolkit.M) error {
	var e error
	if parm == nil {
		parm = toolkit.M{}
	}

	dbname := q.Connection().Info().Database
	driverName := q.GetDriverDB()
	// driverName = "oracle"
	tablename := ""
	data := parm.Get("data", nil)
	// fmt.Println("Hasil ekstraksi Param : ", data)

	/*========================EXTRACT FIELD, DATA AND FORMAT DATE=============================*/

	var attributes string
	var values string
	var setUpdate, statement string
	var i int

	dataM, e := toolkit.ToM(data)
	if e != nil {
		return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
	}

	if data != nil {
		for field, val := range dataM {
			namaField := field
			dataValues := val
			stringValues := StringValue(dataValues, driverName)
			if i == 0 {
				attributes = "(" + namaField
				setUpdate = namaField + " = " + stringValues
				values = "(" + stringValues

			} else {
				attributes += ", " + namaField
				setUpdate += ", " + namaField + " = " + stringValues
				values += ", " + stringValues

			}
			i += 1
		}
		attributes += ")"
		values += ")"
	}

	/*=================================END OF EXTRACTION=======================================*/

	temp := ""
	quyerParts := q.Parts()
	c := crowd.From(&quyerParts)
	groupParts := c.Group(func(x interface{}) interface{} {
		qp := x.(*dbox.QueryPart)
		temp = toolkit.JsonString(qp)
		return qp.PartType
	}, nil).Exec()
	parts := map[interface{}]interface{}{}
	if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
		for _, kv := range groupParts.Result.Data().([]crowd.KV) {
			parts[kv.Key] = kv.Value
		}
	}

	fromParts, hasFrom := parts[dbox.QueryPartFrom]
	if !hasFrom {

		return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
	}
	tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)

	var where interface{}
	whereParts, hasWhere := parts[dbox.QueryPartWhere]
	if hasWhere {
		fb := q.Connection().Fb()
		for _, p := range whereParts.([]*dbox.QueryPart) {
			fs := p.Value.([]*dbox.Filter)
			for _, f := range fs {
				fb.AddFilter(f)
			}
		}

		where, e = fb.Build()
		if e != nil {

		} else {

		}

	}
	commandType := ""
	multi := false

	_, hasDelete := parts[dbox.QueryPartDelete]
	_, hasInsert := parts[dbox.QueryPartInsert]
	_, hasUpdate := parts[dbox.QueryPartUpdate]
	_, hasSave := parts[dbox.QueryPartSave]

	if hasDelete {
		commandType = dbox.QueryPartDelete
	} else if hasInsert {
		commandType = dbox.QueryPartInsert
	} else if hasUpdate {
		commandType = dbox.QueryPartUpdate
	} else if hasSave {
		commandType = dbox.QueryPartSave
	}

	var id string
	var idVal interface{}
	if data == nil {
		multi = true
	} else {
		if where == nil {
			id, idVal = toolkit.IdInfo(data)
			if id != "" {
				where = id + " = " + StringValue(idVal, "non")
			}
		} else {
			multi = true
		}
	}

	session := q.Session()
	sessionHive := q.SessionHive()

	if dbname != "" && tablename != "" && multi == true {

	}
	if commandType == dbox.QueryPartInsert {
		if attributes != "" && values != "" {
			if driverName == "hive" {
				statement = "INSERT INTO " + tablename + " VALUES " + values
				e = sessionHive.Exec(statement, nil)
			} else {
				statement = "INSERT INTO " + tablename + " " + attributes + " VALUES " + values
				_, e = session.Exec(statement)
			}

			if e != nil {
				fmt.Println(e.Error())
			}
		} else {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	} else if commandType == dbox.QueryPartUpdate {
		if setUpdate != "" {
			var statement string
			if where != nil {
				statement = "UPDATE " + tablename + " SET " + setUpdate +
					" WHERE " + cast.ToString(where)
			} else {
				statement = "UPDATE " + tablename + " SET " + setUpdate
			}
			if driverName == "hive" {
				e = sessionHive.Exec(statement, nil)
			} else {
				_, e = session.Exec(statement)
			}
			if e != nil {
				return errorlib.Error(packageName, modQuery+".Exec", commandType,
					cast.ToString(e.Error()))
			}
		} else {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	} else if commandType == dbox.QueryPartDelete {
		var statement string
		if where != nil {
			statement = "DELETE FROM " + tablename + " where " + cast.ToString(where)
		} else {
			statement = "DELETE FROM " + tablename
		}
		if driverName == "hive" {
			e = sessionHive.Exec(statement, nil)
		} else {
			_, e = session.Exec(statement)
		}
		if e != nil {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				cast.ToString(e.Error()))
		}
	} else if commandType == dbox.QueryPartSave {
		if attributes != "" && values != "" {
			var querystmt string
			if where != nil {
				querystmt = "select 1 as data from " + tablename + " where " + cast.ToString(where)
			}
			var rowCount int
			if driverName == "hive" {
				rowCount = 0
				// row := sessionHive.Exec(querystmt, nil)
				// rowCount = toolkit.ToInt(row[0], "auto")
			} else {
				if querystmt != "" {
					rows, _ := session.Query(querystmt)
					for rows.Next() {
						rows.Scan(&rowCount)
					}
				}
			}

			var statement string
			if rowCount == 0 || where == nil {
				if driverName == "hive" {
					statement = "INSERT INTO " + tablename + " VALUES " + values
				} else {
					statement = "INSERT INTO " + tablename + " " +
						attributes + " VALUES " + values
				}
			} else {
				statement = "UPDATE " + tablename + " SET " + setUpdate +
					" WHERE " + cast.ToString(where)
			}

			if driverName == "hive" {
				e = sessionHive.Exec(statement, nil)
			} else {
				_, e = session.Exec(statement)
			}
			if e != nil {
				return errorlib.Error(packageName, modQuery+".Exec", commandType,
					cast.ToString(e.Error()))
			}

		} else if values == "" {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"please provide the data")
		}

	}
	if e != nil {
		return errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
	}
	return nil
}
Example #22
0
func fetchconfig() {
	var result interface{}

	if strings.Contains(configPath, "http") {
		res, configerr := http.Get(configPath)
		if configerr != nil {
			return
		}
		defer res.Body.Close()

		decoder := json.NewDecoder(res.Body)
		configerr = decoder.Decode(&result)
	} else {
		bytes, configerr := ioutil.ReadFile(configPath)
		if configerr != nil {
			return
		}

		configerr = json.Unmarshal(bytes, &result)
	}

	if configerr != nil {
		return
	}

	switch toolkit.TypeName(result) {
	case "[]interface {}":
		for i, each := range result.([]interface{}) {
			m, err := toolkit.ToM(each)

			if err == nil {
				err = checkconfig(m)
			}

			if err == nil {
				validconfig += 1
				if m.Get("running", false).(bool) {
					config = append(config, m)
				}
			} else {
				// tstring := fmt.Sprintf("%v;(Config index %d. %v)", configerr, i, err)
				// configerr = errors.New(fmt.Sprintf("%v", tstring))
				Log.AddLog(fmt.Sprintf("Config index %d. %v", i, err), "ERROR")
			}
		}
	case "map[string]interface {}":
		m, err := toolkit.ToM(result)
		if err == nil {
			err = checkconfig(m)
		}

		if err == nil {
			validconfig += 1
			if m.Get("running", false).(bool) {
				config = append(config, m)
			}
		} else {
			Log.AddLog(fmt.Sprintf("Fetch Config Error Found : %v", err), "ERROR")
		}
	default:
		Log.AddLog(fmt.Sprintf("invalid config file\n%#v", result), "ERROR")
	}
}
Example #23
0
func (q *Query) Exec(in toolkit.M) error {
	setting, e := q.prepare(in)
	commandType := setting["commandtype"].(string)

	if e != nil {
		return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
	}

	if commandType == dbox.QueryPartSelect {
		return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
	}

	q.Lock()
	defer q.Unlock()

	datam := toolkit.M{}

	if in.Has("data") {
		datam, e = toolkit.ToM(in["data"])
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
		}
	}

	if commandType != dbox.QueryPartDelete && len(datam) == 0 {
		return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working, need data to param input")
	}
	// fmt.Println("DEBUG-113")
	e = q.openFile()
	if e != nil {
		return err.Error(packageName, modQuery, "Exec ", e.Error())
	}

	where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
	if nameid := toolkit.IdField(datam); nameid != "" {
		if commandType == dbox.QueryPartInsert || commandType == dbox.QueryPartSave {
			where = make([]*dbox.Filter, 0, 0)
		}
		where = append(where, dbox.Eq(strings.ToLower(nameid), datam[nameid]))
	}

	q.indexes = make([]int, 0, 0)
	if len(where) > 0 {
		q.indexes, e = q.generateIndex(where)
		if e != nil {
			return err.Error(packageName, modQuery, "Exec: ", e.Error())
		}
	}

	if q.newheader && commandType != dbox.QueryPartSave && commandType != dbox.QueryPartInsert {
		return err.Error(packageName, modQuery, "Exec: ", "Only save and insert permited")
	}

	q.newFileWrite = false
	q.execOpr = false
	switch commandType {
	case dbox.QueryPartSave:
		if len(q.indexes) > 0 {
			q.newFileWrite = true
			e = q.execQueryPartUpdate(datam)
		} else {
			e = q.execQueryPartInsert(datam)
		}
	case dbox.QueryPartInsert:
		if len(q.indexes) > 0 {
			e = fmt.Errorf("Id found, cannot insert with same Id")
		} else {
			e = q.execQueryPartInsert(datam)
		}
	case dbox.QueryPartDelete:
		q.newFileWrite = true
		if len(q.indexes) > 0 {
			e = q.execQueryPartDelete()
		}
	case dbox.QueryPartUpdate:
		q.newFileWrite = true
		if len(q.indexes) > 0 {
			e = q.execQueryPartUpdate(datam)
		}
	}

	if e != nil {
		return err.Error(packageName, modQuery, "Exec: ", e.Error())
	}

	return nil
}
Example #24
0
func (q *Query) insertBulk(parm toolkit.M) error {

	var e error
	if parm == nil {
		parm = toolkit.M{}
	}

	driverName := q.GetDriverDB()
	// driverName = "oracle"
	tablename := ""
	data := parm.Get("data")

	var attributes string

	var dataM toolkit.M
	var dataMs []toolkit.M

	if toolkit.IsSlice(data) {
		e = toolkit.Unjson(toolkit.Jsonify(data), &dataMs)
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
		}
	} else {
		dataM, e = toolkit.ToM(data)
		dataMs = append(dataMs, dataM)
		if e != nil {
			return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
		}
	}

	temp := ""
	quyerParts := q.Parts()
	c := crowd.From(&quyerParts)
	groupParts := c.Group(func(x interface{}) interface{} {
		qp := x.(*dbox.QueryPart)
		temp = toolkit.JsonString(qp)
		return qp.PartType
	}, nil).Exec()
	parts := map[interface{}]interface{}{}
	if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
		for _, kv := range groupParts.Result.Data().([]crowd.KV) {
			parts[kv.Key] = kv.Value
		}
	}

	commandType := ""

	_, hasInsert := parts[dbox.QueryPartInsert]

	if hasInsert {
		commandType = dbox.QueryPartInsert
	} else {
		_, e = q.ExecOut(parm)
		return e
		// return errorlib.Error(packageName, "Query", modQuery+".InsertBulk", "Invalid Operation")
	}

	fromParts, hasFrom := parts[dbox.QueryPartFrom]

	if !hasFrom {
		return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
	}
	tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)

	session := q.Session()
	attributeList := extractFields(dataMs[0])

	var datas []string

	for _, dataVal := range dataMs {
		var values string
		tmp := toolkit.M{}

		for _, attr := range attributeList {
			tmp.Set(attr, dataVal.Get(attr))
		}

		values = extractDataBulk(attributeList, tmp, driverName)
		// toolkit.Printf("test: \n %v \n------\n %v \n------\n %v \n------\n %v \n", attributeList, dataVal, tmp, values)

		datas = append(datas, values)
	}

	attributes = "(" + strings.Join(attributeList, ",") + ")"

	if attributes != "" && nil != datas {
		var statement string
		if driverName == "hive" {
			/*statement = "INSERT INTO " + tablename + " VALUES " + values
			e = sessionHive.Exec(statement, nil)*/
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				"Not Implemented Yet for HIVE")
		} else {
			statement = fmt.Sprintf("INSERT INTO "+tablename+" "+attributes+" VALUES %s", strings.Join(datas, ","))
			_, e = session.Exec(statement)
		}

		if e != nil {
			return errorlib.Error(packageName, modQuery+".Exec", commandType,
				cast.ToString(e.Error()))
		}
	} else {
		return errorlib.Error(packageName, modQuery+".Exec", commandType,
			"please provide the data")
	}

	return nil
}
Example #25
0
func fetchConfig() (err error) {

	switch toolkit.ToString(config.Get("sourcetype", "")) {
	case "SourceType_HttpHtml":
		SourceType = SourceType_HttpHtml
	case "SourceType_HttpJson":
		SourceType = SourceType_HttpJson
	default:
		err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
		return
	}

	Log.AddLog("Start fetch grabconf", "INFO")
	if !config.Has("grabconf") {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
		return
	}

	tconfgrab := toolkit.M{}
	tconfgrab, err = toolkit.ToM(config["grabconf"])
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
		return
	}

	grabconfig := sedotan.Config{}
	if tconfgrab.Has("formvalues") {
		tfromvalues := toolkit.M{}
		tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
		if err != nil {
			err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
			return
		}
		grabconfig.SetFormValues(tfromvalues)
	}

	if tconfgrab.Has("loginvalues") {
		grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
	}

	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
		return
	}

	grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
	grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))

	grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
	grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
	grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))

	grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
	grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))

	Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")

	wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)

	Log.AddLog("Start fetch datasettings", "INFO")
	if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
	destDboxs = make(map[string]*DestInfo)

	for i, xVal := range config["datasettings"].([]interface{}) {
		err = nil
		tDataSetting := sedotan.DataSetting{}
		tDestDbox := DestInfo{}

		mVal := toolkit.M{}
		mVal, err = toolkit.ToM(xVal)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
			continue
		}

		tnameid := toolkit.ToString(mVal.Get("nameid", ""))
		if tnameid == "" {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
			continue
		}
		tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))

		// Fetch columnsettings
		if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
			continue
		}

		tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
		for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
			mValcs := toolkit.M{}
			mValcs, err = toolkit.ToM(Valcs)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
				continue
			}

			tgrabcolumn := sedotan.GrabColumn{}
			tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
			tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
			tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
			tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))

			tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
			tDataSetting.Column(tindex, &tgrabcolumn)
		}

		//Fetch Filter Condition
		if mVal.Has("filtercond") {
			tfiltercond := toolkit.M{}
			tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
			} else {
				tDataSetting.SetFilterCond(tfiltercond)
			}
		}

		//Fetch Connection Info
		tConnInfo := toolkit.M{}
		tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
		tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))

		tHost := toolkit.ToString(tConnInfo.Get("host", ""))
		tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
		tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
		tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
		tSettings := toolkit.M{}
		tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}

		tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.IConnection.Close()

		destDboxs[tnameid] = &tDestDbox
		wGrabber.DataSettings[tnameid] = &tDataSetting

	}
	err = nil

	if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	if !config.Has("histconf") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	Log.AddLog("Start fetch histconf", "INFO")
	histConf, err = toolkit.ToM(config.Get("histconf", nil))
	if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	return
}
Example #26
0
//Check Time run and record to snapshot
func checkistimerun(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	var mtklaststarttime, mtklastupdate time.Time
	tempss := Snapshot{Id: id,
		Starttime:      "",
		Laststartgrab:  "",
		Lastupdate:     "",
		Grabcount:      1,
		Rowgrabbed:     0,
		Errorfound:     0,
		Lastgrabstatus: "",
		Grabstatus:     "running",
		Cgtotal:        0,
		Cgprocess:      0,
		Note:           "",
		Pid:            0}
	// 	Lastgrabstatus string //[success|failed]
	// 	Grabstatus     string //[running|done]

	strintervalconf := intervalconf.Get("starttime", "").(string)
	intervalstart := sedotan.StringToDate(strintervalconf)
	strintervaltype := intervalconf.Get("intervaltype", "").(string)
	grabinterval := toolkit.ToInt(intervalconf.Get("grabinterval", 0), toolkit.RoundingAuto)

	mtkdata := Snapshot{}

	if _, f := mapsnapshot[id]; f {
		mtkdata = mapsnapshot[id]
		tempss.Starttime = mtkdata.Starttime
		//for data timeout
		mtklaststarttime = sedotan.StringToDate(mtkdata.Laststartgrab)
		mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)

		timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
		timeoutsec := time.Second * time.Duration(timeoutint)
		if mtkdata.Grabstatus == "running" && thistime.After(mtklaststarttime.Add(timeoutsec)) && timeoutint > 0 {
			mtkdata.Lastupdate = sedotan.DateToString(mtklaststarttime.Add(timeoutsec))
			mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
			mtkdata.Lastgrabstatus = "failed"
			mtkdata.Grabstatus = "done"
		}

		// set initial - found a long time ago snapshot
		if intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Starttime = sedotan.DateToString(thistime)

			mtkdata.Grabcount = 0
			mtkdata.Rowgrabbed = 0
			mtkdata.Errorfound = 0
			mtkdata.Lastgrabstatus = ""
		}
	}

	if mtkdata.Lastgrabstatus == "failed" {
		grabinterval = toolkit.ToInt(intervalconf.Get("timeoutinterval", 0), toolkit.RoundingAuto)
	}

	if strintervalconf != "" && intervalstart.Before(thistime) {
		_, fcond := mapsnapshot[id]

		switch {
		case !fcond || intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)):
			cond = true
		case intervalconf.Get("grabinterval", 0).(float64) > 0:
			var durationgrab time.Duration

			switch strintervaltype {
			case "seconds":
				durationgrab = time.Second * time.Duration(grabinterval)
			case "minutes":
				durationgrab = time.Minute * time.Duration(grabinterval)
			case "hours":
				durationgrab = time.Hour * time.Duration(grabinterval)
			}
			nextgrab := mtklastupdate.Add(durationgrab)

			if nextgrab.Before(thistime) && !mtklastupdate.IsZero() {
				cond = true
				tempss.Grabcount = mtkdata.Grabcount + 1
				tempss.Rowgrabbed = mtkdata.Rowgrabbed
				tempss.Errorfound = mtkdata.Errorfound
				tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
			}
		}
	}

	mapcronconf, _ := toolkit.ToM(intervalconf.Get("cronconf", nil))
	minutetime := sedotan.DateMinutePress(thistime) //review this and the usage and parsing in cron
	if len(mapcronconf) > 0 {
		//min hour dayofmonth month dayofweek
		cond = true
		arrstr := [6]string{"min", "hour", "dayofmonth", "month", "dayofweek"}
		// arrstr := [6]string{"month", "dayofmonth", "dayofweek", "hour", "min", "second"}
		for _, str := range arrstr {
			sval := toolkit.ToString(mapcronconf.Get(str, ""))
			ival := toolkit.ToInt(sval, toolkit.RoundingAuto)

			var valcom int
			switch str {
			// case "second":
			// 	valcom = thistime.Second()
			case "min":
				valcom = thistime.Minute()
			case "hour":
				valcom = thistime.Hour()
			case "dayofmonth":
				valcom = thistime.Day()
			case "month":
				valcom = toolkit.ToInt(thistime.Month(), toolkit.RoundingAuto)
			case "dayofweek":
				valcom = toolkit.ToInt(thistime.Weekday(), toolkit.RoundingAuto)
			}

			if sval != "*" && valcom != ival {
				cond = false
			} else {
				cond = cond && true
			}
		}

		if mtkdata.Laststartgrab != "" {
			cond = cond && minutetime.After(sedotan.StringToDate(mtkdata.Laststartgrab))
		}

		if cond && minutetime.Before(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Grabcount = mtkdata.Grabcount + 1
			tempss.Rowgrabbed = mtkdata.Rowgrabbed
			tempss.Errorfound = mtkdata.Errorfound
			tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
		}
	}

	if cond {
		mapsnapshot[id] = tempss
	}

	return
}
Example #27
0
func checkconfig(cm toolkit.M) (err error) {
	err = nil

	if !cm.Has("_id") {
		err = errors.New(fmt.Sprintf("_id is not found"))
		return
	}

	if !cm.Has("sourcetype") {
		err = errors.New(fmt.Sprintf("sourcetype is not found"))
		return
	}

	if cm.Has("grabconf") {
		_, err = toolkit.ToM(cm["grabconf"])
	} else {
		err = errors.New(fmt.Sprintf("grab config is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("intervalconf") {
		_, err = toolkit.ToM(cm["intervalconf"])
	} else {
		err = errors.New(fmt.Sprintf("interval configuration is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("logconf") {
		_, err = toolkit.ToM(cm["logconf"])
	} else {
		err = errors.New(fmt.Sprintf("log configuration is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("datasettings") {
		if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" {
			err = errors.New(fmt.Sprintf("data settings must be []interface {}"))
		}
	} else {
		err = errors.New(fmt.Sprintf("data settings is not found"))
	}

	if err != nil {
		return
	}

	if cm.Has("histconf") {
		_, err = toolkit.ToM(cm["histconf"])
	} else {
		err = errors.New(fmt.Sprintf("history configuration is not found"))
	}

	if err != nil {
		return
	}
	return
}
Example #28
0
func main() {
	var err error

	flagConfigPath := flag.String("config", "", "config file")
	flagSnapShot := flag.String("snapshot", "", "snapshot filepath")
	flagID := flag.String("id", "", "_id of the config (if array)")
	flagPID := flag.Int("pid", 0, "process id number for identify snapshot active")

	flag.Parse()
	tconfigPath := toolkit.ToString(*flagConfigPath)
	tsnapshot := toolkit.ToString(*flagSnapShot)
	_id = *flagID

	_id = strings.Replace(_id, `"`, "", -1)
	if _id == "" {
		sedotan.CheckError(errors.New("-id cannot be empty"))
	}

	configpath = strings.Replace(tconfigPath, `"`, "", -1)
	if configpath == "" {
		sedotan.CheckError(errors.New("-config cannot be empty"))
	}

	snapshot = strings.Replace(tsnapshot, `"`, "", -1)
	if snapshot == "" {
		sedotan.CheckError(errors.New("-snapshot cannot be empty"))
	}

	err = getsnapshot()
	if err != nil {
		sedotan.CheckError(errors.New(fmt.Sprintf("get snapshot error found : %v", err.Error())))
	}

	pid = *flagPID
	if pid == 0 {
		sedotan.CheckError(errors.New("-pid cannot be empty or zero value"))
	}

	err = getConfig()
	checkfatalerror(err)

	logconf, _ := toolkit.ToM(config.Get("logconf", toolkit.M{}))
	if !logconf.Has("logpath") || !logconf.Has("filename") || !logconf.Has("filepattern") {
		checkfatalerror(errors.New(fmt.Sprintf("config log is not complete")))
	}

	logpath := logconf["logpath"].(string)
	if EC_DATA_PATH != "" {
		logpath = filepath.Join(EC_DATA_PATH, "webgrabber", "log")
	}

	Log, err = toolkit.NewLog(false, true, logpath, (logconf["filename"].(string) + "-%s"), logconf["filepattern"].(string))
	checkfatalerror(err)
	Log.AddLog("Starting web grab data", "INFO")

	Log.AddLog("Start fetch the config", "INFO")
	err = fetchConfig()
	checkexiterror(err)
	Log.AddLog(fmt.Sprintf("Data grabber created : %v", toolkit.JsonString(wGrabber)), "INFO")
	Log.AddLog("Fetch the config success", "INFO")

	Log.AddLog("Get the data", "INFO")
	err = wGrabber.Grab(nil)
	if err != nil {
		checkexiterror(errors.New(fmt.Sprintf("Grab Failed : %v", err.Error())))
	}
	Log.AddLog("Get data grab success", "INFO")

	Log.AddLog("Start save data grab", "INFO")
	err = savedatagrab()
	if err != nil {
		checkexiterror(errors.New(fmt.Sprintf("Save data finish with error : %v", err.Error())))
	}
	Log.AddLog("Save data grab success", "INFO")

	err = getsnapshot()
	if err != nil {
		note := fmt.Sprintf("Unable to get last snapshot :%s", err.Error())
		Log.AddLog(note, "ERROR")
	}

	snapshotdata.Note = ""
	snapshotdata.Lastgrabstatus = "success"
	snapshotdata.Grabstatus = "done"

	if pid == snapshotdata.Pid {
		err = savesnapshot()
		if err != nil {
			checkexiterror(errors.New(fmt.Sprintf("Save snapshot error : %v", err.Error())))
		}
	}

	Log.AddLog("Finish grab data", "INFO")
}
Example #29
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) (
	*dbox.DataSet, error) {
	if closeWhenDone {
		c.Close()
	}

	e := c.prepIter()
	if e != nil {
		return nil, errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	}

	if c.jsonSelect == nil {
		return nil, errorlib.Error(packageName, modCursor, "Fetch", "Iter object is not yet initialized")
	}

	datas := []interface{}{}
	dec := json.NewDecoder(strings.NewReader(string(c.readFile)))
	dec.Decode(&datas)
	ds := dbox.NewDataSet(m)
	if n == 0 {
		whereFieldsToMap, _ := toolkit.ToM(c.whereFields)

		b := c.getCondition(whereFieldsToMap)
		var foundSelected = toolkit.M{}
		var foundData = []toolkit.M{}
		var getRemField = toolkit.M{}
		if c.isWhere {
			if b {
				for _, v := range datas {
					for i, subData := range v.(map[string]interface{}) {
						getRemField[i] = i //append(getRemField, i)
						for _, vWhere := range whereFieldsToMap {
							for _, subWhere := range vWhere.([]interface{}) {
								for _, subsubWhere := range subWhere.(map[string]interface{}) {
									if len(c.jsonSelect.([]string)) == 0 {
										if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
											ds.Data = append(ds.Data, v)
										}
									} else {
										if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
											foundData = append(foundData, v.(map[string]interface{}))
										}
									}
								}
							}
						}
					}
				}

				itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect.([]string))

				if len(foundData) > 0 {
					var found toolkit.M
					for _, found = range foundData {
						for _, remitem := range itemToRemove {
							found.Unset(remitem)
						}

						ds.Data = append(ds.Data, found)
					}
				}
			} else {
				for _, v := range datas {
					for _, v2 := range v.(map[string]interface{}) {
						for _, vWhere := range c.whereFields.(toolkit.M) {
							if reflect.ValueOf(v2).Kind() == reflect.String {
								if strings.ToLower(v2.(string)) == strings.ToLower(vWhere.(string)) {
									if len(c.jsonSelect.([]string)) == 0 {
										ds.Data = append(ds.Data, v)
									} else {
										// fmt.Println(c.jsonSelect.([]string)[0])
										// fmt.Println(v.(map[string]interface{}))
										foundData = append(foundData, v.(map[string]interface{}))
									}
								}
							}

						}
					}
				}

				if len(foundData) > 0 {

					for _, found := range foundData {
						for i, subData := range found {
							for _, selected := range c.jsonSelect.([]string) {
								if strings.ToLower(selected) == strings.ToLower(i) {
									foundSelected[i] = subData
								} else if selected == "*" {
									foundSelected[i] = subData
								}
							}
						}
					}
					ds.Data = append(ds.Data, foundSelected)
				}
			}
		} else {
			if c.jsonSelect.([]string)[0] != "*" {
				for _, v := range datas {
					for i, _ := range v.(map[string]interface{}) {
						getRemField[i] = i
					}
				}

				itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect.([]string))
				for _, found := range datas {
					toMap := toolkit.M(found.(map[string]interface{}))
					for _, remitem := range itemToRemove {
						toMap.Unset(remitem)
					}

					ds.Data = append(ds.Data, found)
				}
			} else {
				ds.Data = datas
			}
		}
	} else if n > 0 {
		fetched := 0
		fetching := true

		///read line
		fetchFile, e := os.OpenFile(c.tempPathFile, os.O_RDWR, 0)
		defer fetchFile.Close()
		if e != nil {
			return nil, errorlib.Error(packageName, modQuery+".Exec", "Fetch file", e.Error())
		}
		c.fetchSession = fetchFile

		scanner := bufio.NewScanner(fetchFile)
		lines := 0
		for scanner.Scan() {
			lines++
		}
		if lines > 0 {
			fetched = lines
			n = n + lines
		}
		for fetching {
			var dataM = toolkit.M{}

			if c.jsonSelect.([]string)[0] != "*" {
				for i := 0; i < len(c.jsonSelect.([]string)); i++ {

					dataM[c.jsonSelect.([]string)[i]] = datas[fetched].(map[string]interface{})[c.jsonSelect.([]string)[i]]

					if len(dataM) == len(c.jsonSelect.([]string)) {
						ds.Data = append(ds.Data, dataM)
					}
				}
			} else {
				ds.Data = append(ds.Data, datas[fetched])
			}
			io.WriteString(fetchFile, toolkit.JsonString(dataM)+"\n")

			fetched++
			if fetched == n {

				fetching = false
			}
		}
	}
	// c.Close()
	return ds, nil
}
Example #30
0
func main() {
	// runtime.GOMAXPROCS(runtime.NumCPU())
	var err error

	flagConfig := flag.String("config", "", "config file")
	flagDebugMode := flag.Bool("debug", false, "debug mode")
	flagLogPath := flag.String("logpath", "", "log path")

	flag.Parse()
	tconfig := toolkit.ToString(*flagConfig)
	tlogPath := toolkit.ToString(*flagLogPath)
	debugMode = *flagDebugMode

	configPath = strings.Replace(tconfig, `"`, "", -1)
	if tconfig == "" {
		sedotan.CheckError(errors.New("-config cannot be empty"))
	}

	logstdout := false
	logfile := true

	logPath := strings.Replace(tlogPath, `"`, "", -1)
	fmt.Println("Log Path, ", logPath)
	if logPath == "" {
		logPath, err = os.Getwd()
		if err != nil {
			logstdout = true
			logfile = false
			fmt.Println("cannot get log path")
		}
	}

	//Temporary :
	var snapshotpath string = filepath.Join(logPath, "daemonsnapshot.csv")
	// err = getDataSnapShot(snapshotpath)
	// sedotan.CheckError(err)

	Log, err = toolkit.NewLog(logstdout, logfile, logPath, "daemonlog-%s", "20060102")
	sedotan.CheckError(err)

	Log.AddLog(fmt.Sprintf("Start daemon grabbing, config path : %v", configPath), "INFO")

	for {
		err = nil

		daemoninterval := 1 * time.Second
		<-time.After(daemoninterval)
		thistime = sedotan.TimeNow()

		Log.AddLog(fmt.Sprintf("Run daemon"), "INFO")
		initiate()

		Log.AddLog(fmt.Sprintf("Fetch config grabbing started"), "INFO")
		fetchconfig()
		if configerr != nil {
			Log.AddLog(configerr.Error(), "ERROR")
			configerr = nil
		}

		Log.AddLog(fmt.Sprintf("Get data snapshot"), "INFO")
		err = getDataSnapShot(snapshotpath)
		if err != nil {
			Log.AddLog(fmt.Sprintf("Failed to start grabbing, snapshot error : %v", err.Error()), "ERROR")
			continue
		}

		if len(config) > 0 {
			Log.AddLog(fmt.Sprintf("Ready to start grabbing, found %v valid config and %v active config", validconfig, len(config)), "INFO")
		} else {
			Log.AddLog(fmt.Sprintf("Skip to start grabbing, found %v valid config and 0 active config", validconfig), "ERROR")
		}

		for _, econfig := range config {
			err = nil
			eid := econfig.Get("_id", "").(string)
			Log.AddLog(fmt.Sprintf("Check config for id : %v", eid), "INFO")
			intervalconf, _ := toolkit.ToM(econfig["intervalconf"])
			grabconf, _ := toolkit.ToM(econfig["grabconf"])

			var isonprocess bool = checkisonprocess(eid, intervalconf, grabconf)
			var isconfrun bool = econfig.Get("running", false).(bool) //check config status run/stop (isconfrun)
			var istimerun bool = checkistimerun(eid, intervalconf, grabconf)

			etype := econfig.Get("sourcetype", "").(string)
			//check grab status onprocess/done/na/error -> conf file / snapshot file ? (isonprocess)
			//check interval+time start/corn schedulling and check last running for interval(istimerun)
			// fmt.Printf("!%v && %v && %v \n", isonprocess, isconfrun, istimerun)
			if !isonprocess && isconfrun && istimerun {
				Log.AddLog(fmt.Sprintf("Start grabbing for id : %v", eid), "INFO")

				// save data snapshot using dbox save
				tsnapshot := mapsnapshot[eid]
				tsnapshot.Laststartgrab = sedotan.DateToString(thistime)
				tsnapshot.Pid = toolkit.RandInt(1000) + 1
				tsnapshot.Cgtotal = 0
				tsnapshot.Cgprocess = 0

				mapsnapshot[eid] = tsnapshot

				err = savesnapshot(eid, snapshotpath)
				if err != nil {
					Log.AddLog(fmt.Sprintf("Save snapshot id : %v, error found : %v", eid, err), "INFO")
					continue
				}
				// run grabbing
				go func(id string, etype string, thistime time.Time) {
					etype = strings.ToLower(etype)
					var cmd *exec.Cmd
					aCommand := make([]string, 0, 0)

					if runtime.GOOS == "windows" {
						aCommand = append(aCommand, "cmd")
						aCommand = append(aCommand, "/C")
					}

					apppath := ""
					// Check Type [SourceType_HttpHtml|SourceType_HttpJson|SourceType_DocExcel]
					switch {
					case strings.Contains(etype, "http"):
						if runtime.GOOS == "windows" {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw.exe")
						} else {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw")
						}
					case strings.Contains(etype, "doc"):
						if runtime.GOOS == "windows" {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans.exe")
						} else {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans")
						}
					}
					aCommand = append(aCommand, apppath)
					aCommand = append(aCommand, `-config="`+configPath+`"`)
					aCommand = append(aCommand, `-snapshot="`+snapshotpath+`"`)
					aCommand = append(aCommand, `-id="`+eid+`"`)
					aCommand = append(aCommand, `-pid=`+toolkit.ToString(mapsnapshot[eid].Pid))

					cmd = exec.Command(aCommand[0], aCommand[1:]...)

					Log.AddLog(fmt.Sprintf("[%v] run at %v, run : %v", eid, sedotan.DateToString(thistime), cmd.Args), "INFO")
					byteoutput, err := cmd.CombinedOutput()
					if err != nil {
						Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR")
					}
					Log.AddLog(fmt.Sprintf("[%v] run at %v, done with message : %v", eid, sedotan.DateToString(thistime), string(byteoutput)), "INFO")
				}(eid, etype, thistime)
			} else {
				Log.AddLog(fmt.Sprintf("Skip grabbing for id : %v", eid), "INFO")
			}
		}
	}
	// _, err := sedotan.Process(config)
	// sedotan.CheckError(err)
}