func getConfig() (err error) {
var result interface{}
if strings.Contains(configpath, "http") {
res, err := http.Get(configpath)
checkfatalerror(err)
defer res.Body.Close()
decoder := json.NewDecoder(res.Body)
err = decoder.Decode(&result)
checkfatalerror(err)
} else {
bytes, err := ioutil.ReadFile(configpath)
checkfatalerror(err)
err = json.Unmarshal(bytes, &result)
checkfatalerror(err)
}
switch toolkit.TypeName(result) {
case "[]interface {}":
isFound := false
for _, eachRaw := range result.([]interface{}) {
each := eachRaw.(map[string]interface{})
if each["_id"].(string) == _id {
m := toolkit.M{}
m, err = toolkit.ToM(each)
checkfatalerror(err)
config = m
isFound = true
}
}
if !isFound {
checkfatalerror(errors.New(fmt.Sprintf("config with _id %s is not found\n%#v", _id, result)))
}
case "map[string]interface {}":
m := toolkit.M{}
m, err = toolkit.ToM(result)
checkfatalerror(err)
config = m
default:
checkfatalerror(errors.New(fmt.Sprintf("invalid config file\n%#v", result)))
}
return
}
func StopProcess(datas []interface{}) (error, bool) {
var (
e error
isStopServ bool
)
for _, v := range datas {
vToMap, _ := toolkit.ToM(v)
if knot.SharedObject().Get(vToMap["nameid"].(string)) != nil {
e, isStopServ = StopService(knot.SharedObject().Get(vToMap["nameid"].(string)).(*sdt.GrabService))
if e != nil {
return e, false
}
} else {
return e, false
}
}
if !isStopServ {
isStopServ = true
} else {
isStopServ = false
}
return nil, isStopServ
}
func (w *WebGrabberController) StopService(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
w.PrepareHistoryPath()
payload := new(colonycore.WebGrabber)
err := r.GetPayload(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
err = colonycore.Get(payload, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o, err := toolkit.ToM(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
err, isRun := modules.StopProcess([]interface{}{o})
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(isRun, nil, "")
}
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
w.PrepareHistoryPath()
payload := struct {
ID string `json:"_id"`
Date string `json:"date"`
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
wg := new(colonycore.WebGrabber)
err = colonycore.Get(wg, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o, err := toolkit.ToM(wg)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
history := modules.NewHistory(payload.ID)
logs := history.GetLog([]interface{}{o}, payload.Date)
return helper.CreateResult(true, logs, "")
}
func getresultobj(strsplits []string, tval interface{}, val toolkit.M) interface{} {
var xval interface{}
if val == nil {
val = toolkit.M{}
}
switch {
case len(strsplits) == 1:
xval = tval
case len(strsplits) > 1:
tm := toolkit.M{}
if val.Has(strsplits[1]) {
tm, _ = toolkit.ToM(val[strsplits[1]])
}
xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
}
return xval
// if {
// val = tval
// } else {
// val = val.Set(k, v)
// if val == nil {
// } else {
// }
// val = getresultobj(strsplits[1:], tval)
// }
// return val
}
func Process(datas []interface{}) (error, bool) {
var (
e error
// isServRun bool
)
for _, v := range datas {
vToMap, e := toolkit.ToM(v)
if e != nil {
return e, false
}
if vToMap["sourcetype"].(string) == "SourceType_Http" {
grabs, e = GrabHtmlConfig(vToMap)
if e != nil {
return e, false
}
} else if vToMap["sourcetype"].(string) == "SourceType_DocExcel" {
grabs, e = GrabDocConfig(vToMap)
if e != nil {
return e, false
}
}
}
e, isServRun := StartService(grabs)
if e != nil {
return e, false
}
return nil, isServRun
}
func foundCondition(dataCheck toolkit.M, cond toolkit.M) bool {
resBool := true
for key, val := range cond {
if key == "$and" || key == "$or" {
for i, sVal := range val.([]interface{}) {
rVal := sVal.(map[string]interface{})
mVal := toolkit.M{}
for rKey, mapVal := range rVal {
mVal.Set(rKey, mapVal)
}
xResBool := foundCondition(dataCheck, mVal)
if key == "$and" {
resBool = resBool && xResBool
} else {
if i == 0 {
resBool = xResBool
} else {
resBool = resBool || xResBool
}
}
}
} else {
if reflect.ValueOf(val).Kind() == reflect.Map {
mVal := val.(map[string]interface{})
tomVal, _ := toolkit.ToM(mVal)
switch {
case tomVal.Has("$ne"):
if tomVal["$ne"].(string) == dataCheck.Get(key, "").(string) {
resBool = false
}
case tomVal.Has("$regex"):
resBool, _ = regexp.MatchString(tomVal["$regex"].(string), dataCheck.Get(key, "").(string))
case tomVal.Has("$gt"):
if tomVal["$gt"].(string) >= dataCheck.Get(key, "").(string) {
resBool = false
}
case tomVal.Has("$gte"):
if tomVal["$gte"].(string) > dataCheck.Get(key, "").(string) {
resBool = false
}
case tomVal.Has("$lt"):
if tomVal["$lt"].(string) <= dataCheck.Get(key, "").(string) {
resBool = false
}
case tomVal.Has("$lte"):
if tomVal["$lte"].(string) < dataCheck.Get(key, "").(string) {
resBool = false
}
}
} else if reflect.ValueOf(val).Kind() == reflect.String && val != dataCheck.Get(key, "").(string) {
resBool = false
}
}
}
return resBool
}
func filterCondition(cond toolkit.M) *dbox.Filter {
fb := new(dbox.Filter)
for key, val := range cond {
if key == "$and" || key == "$or" {
afb := []*dbox.Filter{}
for _, sVal := range val.([]interface{}) {
rVal := sVal.(map[string]interface{})
mVal := toolkit.M{}
for rKey, mapVal := range rVal {
mVal.Set(rKey, mapVal)
}
afb = append(afb, filterCondition(mVal))
}
if key == "$and" {
fb = dbox.And(afb...)
} else {
fb = dbox.Or(afb...)
}
} else {
if reflect.ValueOf(val).Kind() == reflect.Map {
mVal := val.(map[string]interface{})
tomVal, _ := toolkit.ToM(mVal)
switch {
case tomVal.Has("$eq"):
fb = dbox.Eq(key, tomVal["$eq"].(string))
case tomVal.Has("$ne"):
fb = dbox.Ne(key, tomVal["$ne"].(string))
case tomVal.Has("$regex"):
fb = dbox.Contains(key, tomVal["$regex"].(string))
case tomVal.Has("$gt"):
fb = dbox.Gt(key, tomVal["$gt"].(string))
case tomVal.Has("$gte"):
fb = dbox.Gte(key, tomVal["$gte"].(string))
case tomVal.Has("$lt"):
fb = dbox.Lt(key, tomVal["$lt"].(string))
case tomVal.Has("$lte"):
fb = dbox.Lte(key, tomVal["$lte"].(string))
}
} else {
fb = dbox.Eq(key, val)
}
}
}
return fb
}
func (s *Server) Write(datakey DataKey, data []byte) error {
handleError := func(e error) error {
return errorlib.Error(packageName, modServer, "Write",
fmt.Sprintf("[%s.%s.%s] ", datakey.Owner, datakey.Cluster, datakey.Key)+e.Error())
}
if s.ServerType == ServerTypeMaster || s.ServerType == ServerTypeReplicaMaster {
dataSize := len(data)
snode, e := s.getNode(ServerTypeNode, dataSize)
if e != nil {
return handleError(e)
}
snodes := []*Server{snode}
srepma, e := s.getNode(ServerTypeReplicaMaster, dataSize)
if srepma != nil {
snodes = append(snodes, srepma)
}
for _, sn := range snodes {
mDataKey, e := toolkit.ToM(datakey)
if e != nil {
return handleError(e)
}
_, e = sn.call(opWrite, "GET", mDataKey, data)
if e != nil {
return handleError(e)
}
e = s.writeDataMap(datakey, dataSize, sn)
if e != nil {
return handleError(e)
}
}
} else {
key := datakey.BuildKey()
d := new(Data)
d.Key = key
d.Type = datakey.DataType
if datakey.Expiry == 0 {
d.Expiry = time.Now().Add(100 * 365 * 24 * 60 * time.Minute)
} else {
d.Expiry = time.Now().Add(time.Duration(datakey.Expiry))
}
d.Value = data
s.Items[key] = d
}
return nil
}
func (a *Login) GetACLConnectionInfo() (string, *dbox.ConnectionInfo) {
conf, err := toolkit.ToM(GetConfig(CONF_DB_ACL))
if err != nil {
return "", nil
}
ci := dbox.ConnectionInfo{
conf.GetString("host"),
conf.GetString("db"),
conf.GetString("user"),
conf.GetString("pass"),
toolkit.M{}.Set("timeout", 3),
}
return conf.GetString("driver"), &ci
}
func (h *HistoryModule) GetLog(datas []interface{}, date string) interface{} {
for _, v := range datas {
vMap, _ := toolkit.ToM(v)
logConf := vMap["logconf"].(map[string]interface{})
dateNow := time.Now()
dateNowFormat := dateNow.Format(logConf["filepattern"].(string))
fileName := fmt.Sprintf("%s-%s", logConf["filename"], dateNowFormat)
h.logPath = f.Join(logConf["logpath"].(string), fileName)
}
file, err := os.Open(h.logPath)
if err != nil {
return err.Error()
}
defer file.Close()
getHours := strings.Split(date, ":")
containString := getHours[0] + ":" + getHours[1]
scanner := bufio.NewScanner(file)
lines := 0
containLines := 0
var logs []interface{}
for scanner.Scan() {
lines++
contains := strings.Contains(scanner.Text(), containString)
if contains {
containLines = lines
}
if lines == containLines {
logs = append(logs, "<li>"+scanner.Text()+"</li>")
}
}
if err := scanner.Err(); err != nil {
return err.Error()
}
var addSlice = toolkit.M{}
addSlice.Set("logs", logs)
return addSlice
}
func (d *DataSourceController) TestConnection(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
driver := payload["Driver"].(string)
host := payload["Host"].(string)
database := payload["Database"].(string)
username := payload["UserName"].(string)
password := payload["Password"].(string)
var settings toolkit.M = nil
if settingsRaw := d.parseSettings(payload["Settings"], nil); settingsRaw != nil {
settings, err = toolkit.ToM(settingsRaw)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
if err := d.checkIfDriverIsSupported(driver); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
fakeDataConn := &colonycore.Connection{
Database: database,
Driver: driver,
Host: host,
UserName: username,
Password: password,
Settings: settings,
}
err = helper.ConnectUsingDataConn(fakeDataConn).CheckIfConnected()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, nil, "")
}
func (g *GrabModule) CheckStat(datas []interface{}) interface{} {
var (
grabStatus interface{}
lastDate, nextDate string
//toolkit.M
)
var summaryNotes = toolkit.M{} //map[string]interface{}{}
for _, v := range datas {
vToMap, e := toolkit.ToM(v)
if e != nil {
return e.Error()
}
if knot.SharedObject().Get(vToMap["nameid"].(string)) != nil {
i := knot.SharedObject().Get(vToMap["nameid"].(string)).(*sdt.GrabService)
tLast := cast.Date2String(i.LastGrabExe, "YYYY/MM/dd HH:mm:ss") //i.LastGrabExe.Format("2006/01/02 15:04:05")
if tLast != "0001/01/01 00:00:00" {
lastDate = tLast
}
tNext := cast.Date2String(i.NextGrabExe, "YYYY/MM/dd HH:mm:ss") //i.NextGrabExe.Format("2006/01/02 15:04:05")
if tNext != "0001/01/01 00:00:00" {
nextDate = tNext
}
startdate := cast.Date2String(i.StartDate, "YYYY/MM/dd HH:mm:ss") //i.StartDate.Format("2006/01/02 15:04:05")
enddate := cast.Date2String(i.EndDate, "YYYY/MM/dd HH:mm:ss") //i.EndDate.Format("2006/01/02 15:04:05")
summaryNotes.Set("startDate", startdate)
summaryNotes.Set("endDate", enddate)
summaryNotes.Set("grabCount", i.GrabCount)
summaryNotes.Set("rowGrabbed", i.RowGrabbed)
summaryNotes.Set("errorFound", i.ErrorFound)
grabStatus = ReadLog(vToMap["logconf"], i.ServiceRunningStat, i.Name, lastDate, nextDate, i.LastGrabStat, summaryNotes)
} else {
summaryNotes.Set("errorFound", 0)
grabStatus = ReadLog(vToMap["logconf"], false, vToMap["nameid"].(string), "", "", false, summaryNotes)
}
}
return grabStatus
}
func GetChildMenu(r *knot.WebContext, childMenu []colonycore.Menu) interface{} {
sessionId := r.Session("sessionid", "")
results := make([]toolkit.M, 0, 0)
for _, m := range childMenu {
result := toolkit.M{}
acc := acl.HasAccess(toolkit.ToString(sessionId), acl.IDTypeSession, m.AccessId, acl.AccessRead)
result, err := toolkit.ToM(m)
if err != nil {
fmt.Println(err)
}
if acc {
if len(m.Childrens) > 0 {
childs := GetChildMenu(r, m.Childrens)
result.Set("childrens", childs)
}
result.Set("detail", 7)
results = append(results, result)
}
}
return results
}
func checkisonprocess(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
cond = false
if _, f := mapsnapshot[id]; !f {
return
}
mtkdata := mapsnapshot[id]
mapcron, _ := toolkit.ToM(intervalconf["cronconf"])
if mtkdata.Grabstatus == "running" && len(mapcron) <= 0 {
cond = true
}
mtkstarttime := sedotan.StringToDate(mtkdata.Starttime)
timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
timeoutsec := time.Second * time.Duration(timeoutint)
if cond && (thistime.After(mtkstarttime.Add(timeoutsec)) && timeoutint > 0) {
cond = false
}
return
}
func (w *WebGrabberController) Stat(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
w.PrepareHistoryPath()
payload := new(colonycore.WebGrabber)
err := r.GetPayload(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
err = colonycore.Get(payload, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o, err := toolkit.ToM(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
grabStatus := modules.NewGrabService().CheckStat([]interface{}{o})
return helper.CreateResult(true, grabStatus, "")
}
func (a *GroupController) RefreshGroupLdap(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
group := new(acl.Group)
err = acl.FindByID(group, payload["groupid"].(string))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
config, err := toolkit.ToM(group.MemberConf)
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
err = acl.RefreshUserLdapByGroup(payload["groupid"].(string), config)
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
return helper.CreateResult(true, nil, "success")
}
func (d *DataSourceController) parseQuery(query dbox.IQuery, queryInfo toolkit.M) (dbox.IQuery, MetaSave) {
metaSave := MetaSave{}
if qFrom := queryInfo.Get("from", "").(string); qFrom != "" {
query = query.From(qFrom)
}
if qSelect := queryInfo.Get("select", "").(string); qSelect != "" {
if qSelect != "*" {
query = query.Select(strings.Split(qSelect, ",")...)
}
}
if qTakeRaw, qTakeOK := queryInfo["take"]; qTakeOK {
if qTake, ok := qTakeRaw.(float64); ok {
query = query.Take(int(qTake))
}
if qTake, ok := qTakeRaw.(int); ok {
query = query.Take(qTake)
}
}
if qSkipRaw, qSkipOK := queryInfo["skip"]; qSkipOK {
if qSkip, ok := qSkipRaw.(float64); ok {
query = query.Take(int(qSkip))
}
if qSkip, ok := qSkipRaw.(int); ok {
query = query.Take(qSkip)
}
}
if qOrder := queryInfo.Get("order", "").(string); qOrder != "" {
orderAll := map[string]string{}
err := json.Unmarshal([]byte(qOrder), &orderAll)
if err == nil {
orderString := []string{}
for key, val := range orderAll {
orderString = append(orderString, key)
orderString = append(orderString, val)
}
query = query.Order(orderString...)
}
}
if qInsert := queryInfo.Get("insert", "").(string); qInsert != "" {
if qInsert != "" {
metaSave.keyword = "insert"
metaSave.data = qInsert
query = query.Insert()
}
}
if qUpdate := queryInfo.Get("update", "").(string); qUpdate != "" {
if qUpdate != "" {
metaSave.keyword = "update"
metaSave.data = qUpdate
query = query.Update()
}
}
if _, qDeleteOK := queryInfo["delete"]; qDeleteOK {
metaSave.keyword = "delete"
query = query.Delete()
}
if qCommand := queryInfo.Get("command", "").(string); qCommand != "" {
command := map[string]interface{}{}
err := json.Unmarshal([]byte(qCommand), &command)
if err == nil {
for key, value := range command {
query = query.Command(key, value)
break
}
}
}
if qWhere := queryInfo.Get("where", "").(string); qWhere != "" {
whereAll := []map[string]interface{}{}
err := json.Unmarshal([]byte(qWhere), &whereAll)
if err == nil {
allFilter := []*dbox.Filter{}
for _, each := range whereAll {
where, _ := toolkit.ToM(each)
filter := d.filterParse(where)
if filter != nil {
allFilter = append(allFilter, filter)
}
}
query = query.Where(allFilter...)
}
}
return query, metaSave
}
func (d *DataSourceController) filterParse(where toolkit.M) *dbox.Filter {
field := where.Get("field", "").(string)
value := fmt.Sprintf("%v", where["value"])
if key := where.Get("key", "").(string); key == "Eq" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Eq(field, valueInt)
} else {
return dbox.Eq(field, value)
}
} else if key == "Ne" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Ne(field, valueInt)
} else {
return dbox.Ne(field, value)
}
} else if key == "Lt" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Lt(field, valueInt)
} else {
return dbox.Lt(field, value)
}
} else if key == "Lte" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Lte(field, valueInt)
} else {
return dbox.Lte(field, value)
}
} else if key == "Gt" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Gt(field, valueInt)
} else {
return dbox.Gt(field, value)
}
} else if key == "Gte" {
valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"]))
if errv == nil {
return dbox.Gte(field, valueInt)
} else {
return dbox.Gte(field, value)
}
} else if key == "In" {
valueArray := []interface{}{}
for _, e := range strings.Split(value, ",") {
valueArray = append(valueArray, strings.Trim(e, ""))
}
return dbox.In(field, valueArray...)
} else if key == "Nin" {
valueArray := []interface{}{}
for _, e := range strings.Split(value, ",") {
valueArray = append(valueArray, strings.Trim(e, ""))
}
return dbox.Nin(field, valueArray...)
} else if key == "Contains" {
return dbox.Contains(field, value)
} else if key == "Or" {
subs := where.Get("value", []interface{}{}).([]interface{})
filtersToMerge := []*dbox.Filter{}
for _, eachSub := range subs {
eachWhere, _ := toolkit.ToM(eachSub)
filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
}
return dbox.Or(filtersToMerge...)
} else if key == "And" {
subs := where.Get("value", []interface{}{}).([]interface{})
filtersToMerge := []*dbox.Filter{}
for _, eachSub := range subs {
eachWhere, _ := toolkit.ToM(eachSub)
filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere))
}
return dbox.And(filtersToMerge...)
}
return nil
}
func (q *Query) ExecOut(parm toolkit.M) (int64, error) {
var e error
if parm == nil {
parm = toolkit.M{}
}
driverName := q.GetDriverDB()
// driverName = "oracle"
tablename := ""
data := parm.Get("data")
var attributes string
var values string
var setUpdate string
var dataM toolkit.M
var dataMs []toolkit.M
var returnId int64
if toolkit.IsSlice(data) {
e = toolkit.Unjson(toolkit.Jsonify(data), &dataMs)
if e != nil {
return returnId, errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return returnId, errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
}
}
for _, dataVal := range dataMs {
temp := ""
quyerParts := q.Parts()
c := crowd.From(&quyerParts)
groupParts := c.Group(func(x interface{}) interface{} {
qp := x.(*dbox.QueryPart)
temp = toolkit.JsonString(qp)
return qp.PartType
}, nil).Exec()
parts := map[interface{}]interface{}{}
if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
for _, kv := range groupParts.Result.Data().([]crowd.KV) {
parts[kv.Key] = kv.Value
}
}
commandType := ""
_, hasDelete := parts[dbox.QueryPartDelete]
_, hasInsert := parts[dbox.QueryPartInsert]
_, hasUpdate := parts[dbox.QueryPartUpdate]
_, hasSave := parts[dbox.QueryPartSave]
if hasDelete {
commandType = dbox.QueryPartDelete
} else if hasInsert {
commandType = dbox.QueryPartInsert
} else if hasUpdate {
commandType = dbox.QueryPartUpdate
} else if hasSave {
commandType = dbox.QueryPartSave
}
if hasInsert || hasUpdate || hasSave {
attributes, setUpdate, values = extractData(dataVal, driverName)
} else if hasDelete {
}
fromParts, hasFrom := parts[dbox.QueryPartFrom]
if !hasFrom {
return returnId, errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
}
tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)
var where interface{}
whereParts, hasWhere := parts[dbox.QueryPartWhere]
if hasWhere {
fb := q.Connection().Fb()
for _, p := range whereParts.([]*dbox.QueryPart) {
fs := p.Value.([]*dbox.Filter)
for _, f := range fs {
fb.AddFilter(f)
}
}
where, e = fb.Build()
if e != nil {
} else {
}
}
var id string
var idVal interface{}
if where == nil {
id, idVal = toolkit.IdInfo(dataVal)
if id != "" {
where = id + " = " + StringValue(idVal, "non")
}
}
session := q.Session()
sessionHive := q.SessionHive()
if commandType == dbox.QueryPartInsert {
if attributes != "" && values != "" {
var statement string
if driverName == "hive" {
statement = "INSERT INTO " + tablename + " VALUES " + values
e = sessionHive.Exec(statement, nil)
} else {
statement = "INSERT INTO " + tablename + " " + attributes + " VALUES " + values
var res sql.Result
res, e = session.Exec(statement)
if res != nil {
returnId, _ = res.LastInsertId()
}
}
if e != nil {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
} else if commandType == dbox.QueryPartUpdate {
if setUpdate != "" {
var statement string
if where != nil {
statement = "UPDATE " + tablename + " SET " + setUpdate +
" WHERE " + cast.ToString(where)
} else {
statement = "UPDATE " + tablename + " SET " + setUpdate
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
} else if commandType == dbox.QueryPartDelete {
var statement string
if where != nil {
statement = "DELETE FROM " + tablename + " where " + cast.ToString(where)
} else {
statement = "DELETE FROM " + tablename
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else if commandType == dbox.QueryPartSave {
if attributes != "" && values != "" {
var querystmt string
if where != nil {
querystmt = "select 1 as data from " + tablename + " where " + cast.ToString(where)
}
var rowCount int
if driverName == "hive" {
rowCount = 0
// row := sessionHive.Exec(querystmt, nil)
// rowCount = toolkit.ToInt(row[0], "auto")
} else {
if querystmt != "" {
rows, _ := session.Query(querystmt)
for rows.Next() {
rows.Scan(&rowCount)
}
}
}
var statement string
if rowCount == 0 || where == nil {
if driverName == "hive" {
statement = "INSERT INTO " + tablename + " VALUES " + values
} else {
statement = "INSERT INTO " + tablename + " " +
attributes + " VALUES " + values
}
} else {
statement = "UPDATE " + tablename + " SET " + setUpdate +
" WHERE " + cast.ToString(where)
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else if values == "" {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
}
if e != nil {
return returnId, errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
}
}
return returnId, nil
}
func (q *Query) Exec(parm toolkit.M) error {
var e error
if parm == nil {
parm = toolkit.M{}
}
dbname := q.Connection().Info().Database
driverName := q.GetDriverDB()
// driverName = "oracle"
tablename := ""
data := parm.Get("data", nil)
// fmt.Println("Hasil ekstraksi Param : ", data)
/*========================EXTRACT FIELD, DATA AND FORMAT DATE=============================*/
var attributes string
var values string
var setUpdate, statement string
var i int
dataM, e := toolkit.ToM(data)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
}
if data != nil {
for field, val := range dataM {
namaField := field
dataValues := val
stringValues := StringValue(dataValues, driverName)
if i == 0 {
attributes = "(" + namaField
setUpdate = namaField + " = " + stringValues
values = "(" + stringValues
} else {
attributes += ", " + namaField
setUpdate += ", " + namaField + " = " + stringValues
values += ", " + stringValues
}
i += 1
}
attributes += ")"
values += ")"
}
/*=================================END OF EXTRACTION=======================================*/
temp := ""
quyerParts := q.Parts()
c := crowd.From(&quyerParts)
groupParts := c.Group(func(x interface{}) interface{} {
qp := x.(*dbox.QueryPart)
temp = toolkit.JsonString(qp)
return qp.PartType
}, nil).Exec()
parts := map[interface{}]interface{}{}
if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
for _, kv := range groupParts.Result.Data().([]crowd.KV) {
parts[kv.Key] = kv.Value
}
}
fromParts, hasFrom := parts[dbox.QueryPartFrom]
if !hasFrom {
return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
}
tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)
var where interface{}
whereParts, hasWhere := parts[dbox.QueryPartWhere]
if hasWhere {
fb := q.Connection().Fb()
for _, p := range whereParts.([]*dbox.QueryPart) {
fs := p.Value.([]*dbox.Filter)
for _, f := range fs {
fb.AddFilter(f)
}
}
where, e = fb.Build()
if e != nil {
} else {
}
}
commandType := ""
multi := false
_, hasDelete := parts[dbox.QueryPartDelete]
_, hasInsert := parts[dbox.QueryPartInsert]
_, hasUpdate := parts[dbox.QueryPartUpdate]
_, hasSave := parts[dbox.QueryPartSave]
if hasDelete {
commandType = dbox.QueryPartDelete
} else if hasInsert {
commandType = dbox.QueryPartInsert
} else if hasUpdate {
commandType = dbox.QueryPartUpdate
} else if hasSave {
commandType = dbox.QueryPartSave
}
var id string
var idVal interface{}
if data == nil {
multi = true
} else {
if where == nil {
id, idVal = toolkit.IdInfo(data)
if id != "" {
where = id + " = " + StringValue(idVal, "non")
}
} else {
multi = true
}
}
session := q.Session()
sessionHive := q.SessionHive()
if dbname != "" && tablename != "" && multi == true {
}
if commandType == dbox.QueryPartInsert {
if attributes != "" && values != "" {
if driverName == "hive" {
statement = "INSERT INTO " + tablename + " VALUES " + values
e = sessionHive.Exec(statement, nil)
} else {
statement = "INSERT INTO " + tablename + " " + attributes + " VALUES " + values
_, e = session.Exec(statement)
}
if e != nil {
fmt.Println(e.Error())
}
} else {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
} else if commandType == dbox.QueryPartUpdate {
if setUpdate != "" {
var statement string
if where != nil {
statement = "UPDATE " + tablename + " SET " + setUpdate +
" WHERE " + cast.ToString(where)
} else {
statement = "UPDATE " + tablename + " SET " + setUpdate
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
} else if commandType == dbox.QueryPartDelete {
var statement string
if where != nil {
statement = "DELETE FROM " + tablename + " where " + cast.ToString(where)
} else {
statement = "DELETE FROM " + tablename
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else if commandType == dbox.QueryPartSave {
if attributes != "" && values != "" {
var querystmt string
if where != nil {
querystmt = "select 1 as data from " + tablename + " where " + cast.ToString(where)
}
var rowCount int
if driverName == "hive" {
rowCount = 0
// row := sessionHive.Exec(querystmt, nil)
// rowCount = toolkit.ToInt(row[0], "auto")
} else {
if querystmt != "" {
rows, _ := session.Query(querystmt)
for rows.Next() {
rows.Scan(&rowCount)
}
}
}
var statement string
if rowCount == 0 || where == nil {
if driverName == "hive" {
statement = "INSERT INTO " + tablename + " VALUES " + values
} else {
statement = "INSERT INTO " + tablename + " " +
attributes + " VALUES " + values
}
} else {
statement = "UPDATE " + tablename + " SET " + setUpdate +
" WHERE " + cast.ToString(where)
}
if driverName == "hive" {
e = sessionHive.Exec(statement, nil)
} else {
_, e = session.Exec(statement)
}
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else if values == "" {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
}
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType, e.Error())
}
return nil
}
func fetchconfig() {
var result interface{}
if strings.Contains(configPath, "http") {
res, configerr := http.Get(configPath)
if configerr != nil {
return
}
defer res.Body.Close()
decoder := json.NewDecoder(res.Body)
configerr = decoder.Decode(&result)
} else {
bytes, configerr := ioutil.ReadFile(configPath)
if configerr != nil {
return
}
configerr = json.Unmarshal(bytes, &result)
}
if configerr != nil {
return
}
switch toolkit.TypeName(result) {
case "[]interface {}":
for i, each := range result.([]interface{}) {
m, err := toolkit.ToM(each)
if err == nil {
err = checkconfig(m)
}
if err == nil {
validconfig += 1
if m.Get("running", false).(bool) {
config = append(config, m)
}
} else {
// tstring := fmt.Sprintf("%v;(Config index %d. %v)", configerr, i, err)
// configerr = errors.New(fmt.Sprintf("%v", tstring))
Log.AddLog(fmt.Sprintf("Config index %d. %v", i, err), "ERROR")
}
}
case "map[string]interface {}":
m, err := toolkit.ToM(result)
if err == nil {
err = checkconfig(m)
}
if err == nil {
validconfig += 1
if m.Get("running", false).(bool) {
config = append(config, m)
}
} else {
Log.AddLog(fmt.Sprintf("Fetch Config Error Found : %v", err), "ERROR")
}
default:
Log.AddLog(fmt.Sprintf("invalid config file\n%#v", result), "ERROR")
}
}
func (q *Query) Exec(in toolkit.M) error {
setting, e := q.prepare(in)
commandType := setting["commandtype"].(string)
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
if commandType == dbox.QueryPartSelect {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead")
}
q.Lock()
defer q.Unlock()
datam := toolkit.M{}
if in.Has("data") {
datam, e = toolkit.ToM(in["data"])
if e != nil {
return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error())
}
}
if commandType != dbox.QueryPartDelete && len(datam) == 0 {
return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working, need data to param input")
}
// fmt.Println("DEBUG-113")
e = q.openFile()
if e != nil {
return err.Error(packageName, modQuery, "Exec ", e.Error())
}
where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter)
if nameid := toolkit.IdField(datam); nameid != "" {
if commandType == dbox.QueryPartInsert || commandType == dbox.QueryPartSave {
where = make([]*dbox.Filter, 0, 0)
}
where = append(where, dbox.Eq(strings.ToLower(nameid), datam[nameid]))
}
q.indexes = make([]int, 0, 0)
if len(where) > 0 {
q.indexes, e = q.generateIndex(where)
if e != nil {
return err.Error(packageName, modQuery, "Exec: ", e.Error())
}
}
if q.newheader && commandType != dbox.QueryPartSave && commandType != dbox.QueryPartInsert {
return err.Error(packageName, modQuery, "Exec: ", "Only save and insert permited")
}
q.newFileWrite = false
q.execOpr = false
switch commandType {
case dbox.QueryPartSave:
if len(q.indexes) > 0 {
q.newFileWrite = true
e = q.execQueryPartUpdate(datam)
} else {
e = q.execQueryPartInsert(datam)
}
case dbox.QueryPartInsert:
if len(q.indexes) > 0 {
e = fmt.Errorf("Id found, cannot insert with same Id")
} else {
e = q.execQueryPartInsert(datam)
}
case dbox.QueryPartDelete:
q.newFileWrite = true
if len(q.indexes) > 0 {
e = q.execQueryPartDelete()
}
case dbox.QueryPartUpdate:
q.newFileWrite = true
if len(q.indexes) > 0 {
e = q.execQueryPartUpdate(datam)
}
}
if e != nil {
return err.Error(packageName, modQuery, "Exec: ", e.Error())
}
return nil
}
func (q *Query) insertBulk(parm toolkit.M) error {
var e error
if parm == nil {
parm = toolkit.M{}
}
driverName := q.GetDriverDB()
// driverName = "oracle"
tablename := ""
data := parm.Get("data")
var attributes string
var dataM toolkit.M
var dataMs []toolkit.M
if toolkit.IsSlice(data) {
e = toolkit.Unjson(toolkit.Jsonify(data), &dataMs)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
}
} else {
dataM, e = toolkit.ToM(data)
dataMs = append(dataMs, dataM)
if e != nil {
return errorlib.Error(packageName, modQuery, "Exec: data extraction", "Data encoding error: "+e.Error())
}
}
temp := ""
quyerParts := q.Parts()
c := crowd.From(&quyerParts)
groupParts := c.Group(func(x interface{}) interface{} {
qp := x.(*dbox.QueryPart)
temp = toolkit.JsonString(qp)
return qp.PartType
}, nil).Exec()
parts := map[interface{}]interface{}{}
if len(groupParts.Result.Data().([]crowd.KV)) > 0 {
for _, kv := range groupParts.Result.Data().([]crowd.KV) {
parts[kv.Key] = kv.Value
}
}
commandType := ""
_, hasInsert := parts[dbox.QueryPartInsert]
if hasInsert {
commandType = dbox.QueryPartInsert
} else {
_, e = q.ExecOut(parm)
return e
// return errorlib.Error(packageName, "Query", modQuery+".InsertBulk", "Invalid Operation")
}
fromParts, hasFrom := parts[dbox.QueryPartFrom]
if !hasFrom {
return errorlib.Error(packageName, "Query", modQuery, "Invalid table name")
}
tablename = fromParts.([]*dbox.QueryPart)[0].Value.(string)
session := q.Session()
attributeList := extractFields(dataMs[0])
var datas []string
for _, dataVal := range dataMs {
var values string
tmp := toolkit.M{}
for _, attr := range attributeList {
tmp.Set(attr, dataVal.Get(attr))
}
values = extractDataBulk(attributeList, tmp, driverName)
// toolkit.Printf("test: \n %v \n------\n %v \n------\n %v \n------\n %v \n", attributeList, dataVal, tmp, values)
datas = append(datas, values)
}
attributes = "(" + strings.Join(attributeList, ",") + ")"
if attributes != "" && nil != datas {
var statement string
if driverName == "hive" {
/*statement = "INSERT INTO " + tablename + " VALUES " + values
e = sessionHive.Exec(statement, nil)*/
return errorlib.Error(packageName, modQuery+".Exec", commandType,
"Not Implemented Yet for HIVE")
} else {
statement = fmt.Sprintf("INSERT INTO "+tablename+" "+attributes+" VALUES %s", strings.Join(datas, ","))
_, e = session.Exec(statement)
}
if e != nil {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
cast.ToString(e.Error()))
}
} else {
return errorlib.Error(packageName, modQuery+".Exec", commandType,
"please provide the data")
}
return nil
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_HttpHtml":
SourceType = SourceType_HttpHtml
case "SourceType_HttpJson":
SourceType = SourceType_HttpJson
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
grabconfig := sedotan.Config{}
if tconfgrab.Has("formvalues") {
tfromvalues := toolkit.M{}
tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
return
}
grabconfig.SetFormValues(tfromvalues)
}
if tconfgrab.Has("loginvalues") {
grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
}
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
return
}
grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))
grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))
grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))
Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")
wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tDataSetting := sedotan.DataSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))
// Fetch columnsettings
if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.GrabColumn{}
tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))
tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tDataSetting.Column(tindex, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tDataSetting.SetFilterCond(tfiltercond)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
wGrabber.DataSettings[tnameid] = &tDataSetting
}
err = nil
if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
return
}
//Check Time run and record to snapshot
func checkistimerun(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
cond = false
var mtklaststarttime, mtklastupdate time.Time
tempss := Snapshot{Id: id,
Starttime: "",
Laststartgrab: "",
Lastupdate: "",
Grabcount: 1,
Rowgrabbed: 0,
Errorfound: 0,
Lastgrabstatus: "",
Grabstatus: "running",
Cgtotal: 0,
Cgprocess: 0,
Note: "",
Pid: 0}
// Lastgrabstatus string //[success|failed]
// Grabstatus string //[running|done]
strintervalconf := intervalconf.Get("starttime", "").(string)
intervalstart := sedotan.StringToDate(strintervalconf)
strintervaltype := intervalconf.Get("intervaltype", "").(string)
grabinterval := toolkit.ToInt(intervalconf.Get("grabinterval", 0), toolkit.RoundingAuto)
mtkdata := Snapshot{}
if _, f := mapsnapshot[id]; f {
mtkdata = mapsnapshot[id]
tempss.Starttime = mtkdata.Starttime
//for data timeout
mtklaststarttime = sedotan.StringToDate(mtkdata.Laststartgrab)
mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
timeoutsec := time.Second * time.Duration(timeoutint)
if mtkdata.Grabstatus == "running" && thistime.After(mtklaststarttime.Add(timeoutsec)) && timeoutint > 0 {
mtkdata.Lastupdate = sedotan.DateToString(mtklaststarttime.Add(timeoutsec))
mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
mtkdata.Lastgrabstatus = "failed"
mtkdata.Grabstatus = "done"
}
// set initial - found a long time ago snapshot
if intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)) {
tempss.Starttime = sedotan.DateToString(thistime)
mtkdata.Grabcount = 0
mtkdata.Rowgrabbed = 0
mtkdata.Errorfound = 0
mtkdata.Lastgrabstatus = ""
}
}
if mtkdata.Lastgrabstatus == "failed" {
grabinterval = toolkit.ToInt(intervalconf.Get("timeoutinterval", 0), toolkit.RoundingAuto)
}
if strintervalconf != "" && intervalstart.Before(thistime) {
_, fcond := mapsnapshot[id]
switch {
case !fcond || intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)):
cond = true
case intervalconf.Get("grabinterval", 0).(float64) > 0:
var durationgrab time.Duration
switch strintervaltype {
case "seconds":
durationgrab = time.Second * time.Duration(grabinterval)
case "minutes":
durationgrab = time.Minute * time.Duration(grabinterval)
case "hours":
durationgrab = time.Hour * time.Duration(grabinterval)
}
nextgrab := mtklastupdate.Add(durationgrab)
if nextgrab.Before(thistime) && !mtklastupdate.IsZero() {
cond = true
tempss.Grabcount = mtkdata.Grabcount + 1
tempss.Rowgrabbed = mtkdata.Rowgrabbed
tempss.Errorfound = mtkdata.Errorfound
tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
}
}
}
mapcronconf, _ := toolkit.ToM(intervalconf.Get("cronconf", nil))
minutetime := sedotan.DateMinutePress(thistime) //review this and the usage and parsing in cron
if len(mapcronconf) > 0 {
//min hour dayofmonth month dayofweek
cond = true
arrstr := [6]string{"min", "hour", "dayofmonth", "month", "dayofweek"}
// arrstr := [6]string{"month", "dayofmonth", "dayofweek", "hour", "min", "second"}
for _, str := range arrstr {
sval := toolkit.ToString(mapcronconf.Get(str, ""))
ival := toolkit.ToInt(sval, toolkit.RoundingAuto)
var valcom int
switch str {
// case "second":
// valcom = thistime.Second()
case "min":
valcom = thistime.Minute()
case "hour":
valcom = thistime.Hour()
case "dayofmonth":
valcom = thistime.Day()
case "month":
valcom = toolkit.ToInt(thistime.Month(), toolkit.RoundingAuto)
case "dayofweek":
valcom = toolkit.ToInt(thistime.Weekday(), toolkit.RoundingAuto)
}
if sval != "*" && valcom != ival {
cond = false
} else {
cond = cond && true
}
}
if mtkdata.Laststartgrab != "" {
cond = cond && minutetime.After(sedotan.StringToDate(mtkdata.Laststartgrab))
}
if cond && minutetime.Before(sedotan.StringToDate(mtkdata.Starttime)) {
tempss.Grabcount = mtkdata.Grabcount + 1
tempss.Rowgrabbed = mtkdata.Rowgrabbed
tempss.Errorfound = mtkdata.Errorfound
tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
}
}
if cond {
mapsnapshot[id] = tempss
}
return
}
func checkconfig(cm toolkit.M) (err error) {
err = nil
if !cm.Has("_id") {
err = errors.New(fmt.Sprintf("_id is not found"))
return
}
if !cm.Has("sourcetype") {
err = errors.New(fmt.Sprintf("sourcetype is not found"))
return
}
if cm.Has("grabconf") {
_, err = toolkit.ToM(cm["grabconf"])
} else {
err = errors.New(fmt.Sprintf("grab config is not found"))
}
if err != nil {
return
}
if cm.Has("intervalconf") {
_, err = toolkit.ToM(cm["intervalconf"])
} else {
err = errors.New(fmt.Sprintf("interval configuration is not found"))
}
if err != nil {
return
}
if cm.Has("logconf") {
_, err = toolkit.ToM(cm["logconf"])
} else {
err = errors.New(fmt.Sprintf("log configuration is not found"))
}
if err != nil {
return
}
if cm.Has("datasettings") {
if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" {
err = errors.New(fmt.Sprintf("data settings must be []interface {}"))
}
} else {
err = errors.New(fmt.Sprintf("data settings is not found"))
}
if err != nil {
return
}
if cm.Has("histconf") {
_, err = toolkit.ToM(cm["histconf"])
} else {
err = errors.New(fmt.Sprintf("history configuration is not found"))
}
if err != nil {
return
}
return
}
func main() {
var err error
flagConfigPath := flag.String("config", "", "config file")
flagSnapShot := flag.String("snapshot", "", "snapshot filepath")
flagID := flag.String("id", "", "_id of the config (if array)")
flagPID := flag.Int("pid", 0, "process id number for identify snapshot active")
flag.Parse()
tconfigPath := toolkit.ToString(*flagConfigPath)
tsnapshot := toolkit.ToString(*flagSnapShot)
_id = *flagID
_id = strings.Replace(_id, `"`, "", -1)
if _id == "" {
sedotan.CheckError(errors.New("-id cannot be empty"))
}
configpath = strings.Replace(tconfigPath, `"`, "", -1)
if configpath == "" {
sedotan.CheckError(errors.New("-config cannot be empty"))
}
snapshot = strings.Replace(tsnapshot, `"`, "", -1)
if snapshot == "" {
sedotan.CheckError(errors.New("-snapshot cannot be empty"))
}
err = getsnapshot()
if err != nil {
sedotan.CheckError(errors.New(fmt.Sprintf("get snapshot error found : %v", err.Error())))
}
pid = *flagPID
if pid == 0 {
sedotan.CheckError(errors.New("-pid cannot be empty or zero value"))
}
err = getConfig()
checkfatalerror(err)
logconf, _ := toolkit.ToM(config.Get("logconf", toolkit.M{}))
if !logconf.Has("logpath") || !logconf.Has("filename") || !logconf.Has("filepattern") {
checkfatalerror(errors.New(fmt.Sprintf("config log is not complete")))
}
logpath := logconf["logpath"].(string)
if EC_DATA_PATH != "" {
logpath = filepath.Join(EC_DATA_PATH, "webgrabber", "log")
}
Log, err = toolkit.NewLog(false, true, logpath, (logconf["filename"].(string) + "-%s"), logconf["filepattern"].(string))
checkfatalerror(err)
Log.AddLog("Starting web grab data", "INFO")
Log.AddLog("Start fetch the config", "INFO")
err = fetchConfig()
checkexiterror(err)
Log.AddLog(fmt.Sprintf("Data grabber created : %v", toolkit.JsonString(wGrabber)), "INFO")
Log.AddLog("Fetch the config success", "INFO")
Log.AddLog("Get the data", "INFO")
err = wGrabber.Grab(nil)
if err != nil {
checkexiterror(errors.New(fmt.Sprintf("Grab Failed : %v", err.Error())))
}
Log.AddLog("Get data grab success", "INFO")
Log.AddLog("Start save data grab", "INFO")
err = savedatagrab()
if err != nil {
checkexiterror(errors.New(fmt.Sprintf("Save data finish with error : %v", err.Error())))
}
Log.AddLog("Save data grab success", "INFO")
err = getsnapshot()
if err != nil {
note := fmt.Sprintf("Unable to get last snapshot :%s", err.Error())
Log.AddLog(note, "ERROR")
}
snapshotdata.Note = ""
snapshotdata.Lastgrabstatus = "success"
snapshotdata.Grabstatus = "done"
if pid == snapshotdata.Pid {
err = savesnapshot()
if err != nil {
checkexiterror(errors.New(fmt.Sprintf("Save snapshot error : %v", err.Error())))
}
}
Log.AddLog("Finish grab data", "INFO")
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) (
*dbox.DataSet, error) {
if closeWhenDone {
c.Close()
}
e := c.prepIter()
if e != nil {
return nil, errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if c.jsonSelect == nil {
return nil, errorlib.Error(packageName, modCursor, "Fetch", "Iter object is not yet initialized")
}
datas := []interface{}{}
dec := json.NewDecoder(strings.NewReader(string(c.readFile)))
dec.Decode(&datas)
ds := dbox.NewDataSet(m)
if n == 0 {
whereFieldsToMap, _ := toolkit.ToM(c.whereFields)
b := c.getCondition(whereFieldsToMap)
var foundSelected = toolkit.M{}
var foundData = []toolkit.M{}
var getRemField = toolkit.M{}
if c.isWhere {
if b {
for _, v := range datas {
for i, subData := range v.(map[string]interface{}) {
getRemField[i] = i //append(getRemField, i)
for _, vWhere := range whereFieldsToMap {
for _, subWhere := range vWhere.([]interface{}) {
for _, subsubWhere := range subWhere.(map[string]interface{}) {
if len(c.jsonSelect.([]string)) == 0 {
if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
ds.Data = append(ds.Data, v)
}
} else {
if strings.ToLower(subData.(string)) == strings.ToLower(subsubWhere.(string)) {
foundData = append(foundData, v.(map[string]interface{}))
}
}
}
}
}
}
}
itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect.([]string))
if len(foundData) > 0 {
var found toolkit.M
for _, found = range foundData {
for _, remitem := range itemToRemove {
found.Unset(remitem)
}
ds.Data = append(ds.Data, found)
}
}
} else {
for _, v := range datas {
for _, v2 := range v.(map[string]interface{}) {
for _, vWhere := range c.whereFields.(toolkit.M) {
if reflect.ValueOf(v2).Kind() == reflect.String {
if strings.ToLower(v2.(string)) == strings.ToLower(vWhere.(string)) {
if len(c.jsonSelect.([]string)) == 0 {
ds.Data = append(ds.Data, v)
} else {
// fmt.Println(c.jsonSelect.([]string)[0])
// fmt.Println(v.(map[string]interface{}))
foundData = append(foundData, v.(map[string]interface{}))
}
}
}
}
}
}
if len(foundData) > 0 {
for _, found := range foundData {
for i, subData := range found {
for _, selected := range c.jsonSelect.([]string) {
if strings.ToLower(selected) == strings.ToLower(i) {
foundSelected[i] = subData
} else if selected == "*" {
foundSelected[i] = subData
}
}
}
}
ds.Data = append(ds.Data, foundSelected)
}
}
} else {
if c.jsonSelect.([]string)[0] != "*" {
for _, v := range datas {
for i, _ := range v.(map[string]interface{}) {
getRemField[i] = i
}
}
itemToRemove := removeDuplicatesUnordered(getRemField, c.jsonSelect.([]string))
for _, found := range datas {
toMap := toolkit.M(found.(map[string]interface{}))
for _, remitem := range itemToRemove {
toMap.Unset(remitem)
}
ds.Data = append(ds.Data, found)
}
} else {
ds.Data = datas
}
}
} else if n > 0 {
fetched := 0
fetching := true
///read line
fetchFile, e := os.OpenFile(c.tempPathFile, os.O_RDWR, 0)
defer fetchFile.Close()
if e != nil {
return nil, errorlib.Error(packageName, modQuery+".Exec", "Fetch file", e.Error())
}
c.fetchSession = fetchFile
scanner := bufio.NewScanner(fetchFile)
lines := 0
for scanner.Scan() {
lines++
}
if lines > 0 {
fetched = lines
n = n + lines
}
for fetching {
var dataM = toolkit.M{}
if c.jsonSelect.([]string)[0] != "*" {
for i := 0; i < len(c.jsonSelect.([]string)); i++ {
dataM[c.jsonSelect.([]string)[i]] = datas[fetched].(map[string]interface{})[c.jsonSelect.([]string)[i]]
if len(dataM) == len(c.jsonSelect.([]string)) {
ds.Data = append(ds.Data, dataM)
}
}
} else {
ds.Data = append(ds.Data, datas[fetched])
}
io.WriteString(fetchFile, toolkit.JsonString(dataM)+"\n")
fetched++
if fetched == n {
fetching = false
}
}
}
// c.Close()
return ds, nil
}
func main() {
// runtime.GOMAXPROCS(runtime.NumCPU())
var err error
flagConfig := flag.String("config", "", "config file")
flagDebugMode := flag.Bool("debug", false, "debug mode")
flagLogPath := flag.String("logpath", "", "log path")
flag.Parse()
tconfig := toolkit.ToString(*flagConfig)
tlogPath := toolkit.ToString(*flagLogPath)
debugMode = *flagDebugMode
configPath = strings.Replace(tconfig, `"`, "", -1)
if tconfig == "" {
sedotan.CheckError(errors.New("-config cannot be empty"))
}
logstdout := false
logfile := true
logPath := strings.Replace(tlogPath, `"`, "", -1)
fmt.Println("Log Path, ", logPath)
if logPath == "" {
logPath, err = os.Getwd()
if err != nil {
logstdout = true
logfile = false
fmt.Println("cannot get log path")
}
}
//Temporary :
var snapshotpath string = filepath.Join(logPath, "daemonsnapshot.csv")
// err = getDataSnapShot(snapshotpath)
// sedotan.CheckError(err)
Log, err = toolkit.NewLog(logstdout, logfile, logPath, "daemonlog-%s", "20060102")
sedotan.CheckError(err)
Log.AddLog(fmt.Sprintf("Start daemon grabbing, config path : %v", configPath), "INFO")
for {
err = nil
daemoninterval := 1 * time.Second
<-time.After(daemoninterval)
thistime = sedotan.TimeNow()
Log.AddLog(fmt.Sprintf("Run daemon"), "INFO")
initiate()
Log.AddLog(fmt.Sprintf("Fetch config grabbing started"), "INFO")
fetchconfig()
if configerr != nil {
Log.AddLog(configerr.Error(), "ERROR")
configerr = nil
}
Log.AddLog(fmt.Sprintf("Get data snapshot"), "INFO")
err = getDataSnapShot(snapshotpath)
if err != nil {
Log.AddLog(fmt.Sprintf("Failed to start grabbing, snapshot error : %v", err.Error()), "ERROR")
continue
}
if len(config) > 0 {
Log.AddLog(fmt.Sprintf("Ready to start grabbing, found %v valid config and %v active config", validconfig, len(config)), "INFO")
} else {
Log.AddLog(fmt.Sprintf("Skip to start grabbing, found %v valid config and 0 active config", validconfig), "ERROR")
}
for _, econfig := range config {
err = nil
eid := econfig.Get("_id", "").(string)
Log.AddLog(fmt.Sprintf("Check config for id : %v", eid), "INFO")
intervalconf, _ := toolkit.ToM(econfig["intervalconf"])
grabconf, _ := toolkit.ToM(econfig["grabconf"])
var isonprocess bool = checkisonprocess(eid, intervalconf, grabconf)
var isconfrun bool = econfig.Get("running", false).(bool) //check config status run/stop (isconfrun)
var istimerun bool = checkistimerun(eid, intervalconf, grabconf)
etype := econfig.Get("sourcetype", "").(string)
//check grab status onprocess/done/na/error -> conf file / snapshot file ? (isonprocess)
//check interval+time start/corn schedulling and check last running for interval(istimerun)
// fmt.Printf("!%v && %v && %v \n", isonprocess, isconfrun, istimerun)
if !isonprocess && isconfrun && istimerun {
Log.AddLog(fmt.Sprintf("Start grabbing for id : %v", eid), "INFO")
// save data snapshot using dbox save
tsnapshot := mapsnapshot[eid]
tsnapshot.Laststartgrab = sedotan.DateToString(thistime)
tsnapshot.Pid = toolkit.RandInt(1000) + 1
tsnapshot.Cgtotal = 0
tsnapshot.Cgprocess = 0
mapsnapshot[eid] = tsnapshot
err = savesnapshot(eid, snapshotpath)
if err != nil {
Log.AddLog(fmt.Sprintf("Save snapshot id : %v, error found : %v", eid, err), "INFO")
continue
}
// run grabbing
go func(id string, etype string, thistime time.Time) {
etype = strings.ToLower(etype)
var cmd *exec.Cmd
aCommand := make([]string, 0, 0)
if runtime.GOOS == "windows" {
aCommand = append(aCommand, "cmd")
aCommand = append(aCommand, "/C")
}
apppath := ""
// Check Type [SourceType_HttpHtml|SourceType_HttpJson|SourceType_DocExcel]
switch {
case strings.Contains(etype, "http"):
if runtime.GOOS == "windows" {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw.exe")
} else {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw")
}
case strings.Contains(etype, "doc"):
if runtime.GOOS == "windows" {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans.exe")
} else {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans")
}
}
aCommand = append(aCommand, apppath)
aCommand = append(aCommand, `-config="`+configPath+`"`)
aCommand = append(aCommand, `-snapshot="`+snapshotpath+`"`)
aCommand = append(aCommand, `-id="`+eid+`"`)
aCommand = append(aCommand, `-pid=`+toolkit.ToString(mapsnapshot[eid].Pid))
cmd = exec.Command(aCommand[0], aCommand[1:]...)
Log.AddLog(fmt.Sprintf("[%v] run at %v, run : %v", eid, sedotan.DateToString(thistime), cmd.Args), "INFO")
byteoutput, err := cmd.CombinedOutput()
if err != nil {
Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR")
}
Log.AddLog(fmt.Sprintf("[%v] run at %v, done with message : %v", eid, sedotan.DateToString(thistime), string(byteoutput)), "INFO")
}(eid, etype, thistime)
} else {
Log.AddLog(fmt.Sprintf("Skip grabbing for id : %v", eid), "INFO")
}
}
}
// _, err := sedotan.Process(config)
// sedotan.CheckError(err)
}
