/*
Write Write bytes of data into sebar storage.
- Data need to be defined as []byte on in["data"]
- To use memory or disk should be defined on in["storage"] as: MEM, DSK (sebar.StorageTypeMemory, sebar.StorageTypeMemory)
- If no in["storage"] or the value is not eq to either disk or memory, it will be defaulted to memory
*/
func (s *Storage) Write(in toolkit.M) *toolkit.Result {
r := toolkit.NewResult()
key := in.Get("key").(string)
storage := StorageTypeEnum(in.GetString("storage"))
if storage != StorageTypeMemory && storage != StorageTypeDisk {
storage = StorageTypeMemory
}
dataToWrite := in.Get("data").([]byte)
dataLen := len(dataToWrite)
// Validation
nodeCoordinator := s.NodeByID(s.Coordinator)
if nodeCoordinator == nil {
return r.SetErrorTxt(s.Address + " no Coordinator has been setup")
}
// Since all is ok commit the change
var ms *StorageMedia
if storage == StorageTypeMemory {
ms = s.MemoryStorage
} else {
ms = s.DiskStorage
}
ms.write(key, dataToWrite, nodeCoordinator)
s.Log.Info(toolkit.Sprintf("Writing %s (%s) to node %s", key, ParseSize(float64(dataLen)), s.Address))
return r
}
func (c *Coordinator) Get(in toolkit.M) *toolkit.Result {
result := toolkit.NewResult()
result.SetErrorTxt("Get command is still under development")
key := in.GetString("key")
owner, table, datakey := ParseKey(key)
key = MakeKey(owner, table, datakey)
return result
}
func (a *controller) Hi(in toolkit.M) *toolkit.Result {
r := toolkit.NewResult()
name := in.GetString("name")
r.SetBytes(struct {
HelloMessage string
TimeNow time.Time
Scores []Score
}{"Hello " + name, time.Now(), []Score{{"Bahasa Indonesia", 90}, {"Math", 85}}}, "gob")
return r
}
func (c *Coordinator) RequestJoin(in toolkit.M) *toolkit.Result {
var e error
r := toolkit.NewResult()
referenceID := in.GetString("auth_referenceid")
//toolkit.Println("User ID Request Join : " + referenceID)
secret := in.GetString("auth_secret")
nodeid := in.GetString("nodeid")
noderole := NodeRoleEnum(in.GetString("noderole"))
//--- init node
node := new(Node)
node.ID = nodeid
node.Role = noderole
node.UserID = referenceID
node.Secret = secret
//node.InitRPC()
e = c.AddNode(node)
if e != nil {
r.SetErrorTxt(e.Error())
}
//fmt.Printf("Nodes now:\n%s\n", toolkit.JsonString(c.nodes))
r.Data = node.Secret
return r
}
func (b *Broadcaster) Stop(k *knot.WebContext) interface{} {
m := toolkit.M{}
result := toolkit.NewResult()
k.GetPayload(&m)
defer func() {
if result.Status == toolkit.Status_OK {
k.Server.Stop()
}
}()
valid := b.validateToken("user", m.GetString("secret"),
m.GetString("userid"))
if !valid {
result.SetErrorTxt("Not authorised")
return result
}
return result
}
func convertDataType(typedt string, dtget string, toolmap toolkit.M) interface{} {
var resValueEachSF interface{}
switch typedt {
case "string":
resValueEachSF = fmt.Sprintf("%v", toolmap.Get(dtget))
case "int":
var resDefault int
resDefault = toolmap.GetInt(dtget)
resValueEachSF = resDefault
case "double":
var resDefault float32
resDefault = toolmap.GetFloat32(dtget)
resValueEachSF = resDefault
case "bool":
var resDefault bool
resDefault, _ = strconv.ParseBool(toolmap.GetString(dtget))
resValueEachSF = resDefault
}
return resValueEachSF
}
func (a *Client) Connect(address string, secret string, userid string) error {
//("Connecting to %s@%s \n", userid, address)
a.UserID = userid
client, e := rpc.Dial("tcp", address)
if e != nil {
return errorlib.Error(packageName, objClient, "Connect", "["+address+"] Unable to connect: "+e.Error())
}
a.client = client
a.LoginDate = time.Now().UTC()
r := a.Call("addsession", toolkit.M{}.Set("auth_secret", secret).Set("auth_referenceid", a.UserID))
if r.Status != toolkit.Status_OK {
return errors.New("[" + address + "] Connect: " + r.Message + " User:"******"gob", &m)
a.address = address
a.sessionID = m.GetString("referenceid")
a.secret = m.GetString("secret")
return nil
}
func (c *Coordinator) Set(in toolkit.M) *toolkit.Result {
var e error
if in == nil {
in = toolkit.M{}
}
result := toolkit.NewResult()
key := in.GetString("key")
// validation
if key == "" {
return result.SetErrorTxt("Key is empty")
}
data := in.Get("data", []byte{}).([]byte)
if len(data) == 0 {
return result.SetErrorTxt("Data is not valid")
}
// get available node
nodeIdx, e := c.getAvailableNode(data)
if e != nil {
return result.SetErrorTxt("Coordinator.Set: " + e.Error())
}
node := c.Node(RoleStorage, nodeIdx)
// write the data
delete(in, "auth_referenceid")
delete(in, "auth_secret")
in.Set("data", data)
rw := node.Call("write", in)
result.Data = rw.Data
result.Status = rw.Status
if result.Status != toolkit.Status_OK {
result.SetErrorTxt("Coordinator.Set: " + rw.Message)
}
return result
}
func (a *Client) Call(methodName string, in toolkit.M) *toolkit.Result {
if a.client == nil {
return toolkit.NewResult().SetErrorTxt("Unable to call, no connection handshake")
}
if in == nil {
in = toolkit.M{}
}
out := toolkit.NewResult()
in["method"] = methodName
if in.GetString("auth_referenceid") == "" {
in["auth_referenceid"] = a.sessionID
}
//fmt.Println("SessionID: " + a.sessionID)
if in.Has("auth_secret") == false {
in.Set("auth_secret", a.secret)
}
e := a.client.Call("Rpc.Do", in, out)
//_ = "breakpoint"
if e != nil {
return out.SetErrorTxt(a.address + "." + methodName + " Fail: " + e.Error())
}
return out
}
func (b *Broadcaster) Logout(k *knot.WebContext) interface{} {
r := toolkit.NewResult()
m := toolkit.M{}
k.GetPayload(&m)
if !b.validateToken("user", m.GetString("secret"), m.GetString("userid")) {
r.SetErrorTxt("Not authorised")
return r
}
delete(b.userTokens, m.GetString("userid"))
return r
}
func (r *Rpc) Do(in toolkit.M, out *toolkit.Result) error {
if r.Fns == nil {
r.Fns = map[string]*RpcFnInfo{}
}
in.Set("rpc", r)
method := in.GetString("method")
if method == "" {
return errors.New("Method is empty")
}
fninfo, fnExist := r.Fns[method]
if !fnExist {
return errors.New("Method " + method + " is not exist")
/*
" Available methodnames on " + r.Server.Address + " are: " + strings.Join(func() []string {
ret := []string{}
for name, _ := range r.Fns {
ret = append(ret, name)
}
return ret
}(), ", "))
*/
}
if fninfo.AuthRequired {
referenceID := in.GetString("auth_referenceid")
secret := in.GetString("auth_secret")
valid := r.Server.validateSecret(fninfo.AuthType, referenceID, secret)
if !valid {
return errors.New("Unauthorised to call " + method)
}
}
res := fninfo.Fn(in)
if res.Status != toolkit.Status_OK {
return errors.New("RPC Call error: " + res.Message)
}
//*out = toolkit.ToBytes(res.Data, MarshallingMethod())
*out = *res
return nil
}
func (this *DataBrowserController) GetFilter(k *knot.WebContext) interface{} {
k.Config.OutputType = knot.OutputJson
var e error
r := new(tk.Result)
d := DataBrowserInput{}
f := tk.M{}
ret := tk.M{}
_ = k.GetForms(&f)
_ = k.GetPayload(&d)
// tk.Printf("%#v \n", f)
r.Run(func(in interface{}) (interface{}, error) {
activeField := f.GetString("active_field")
if activeField != "" {
params, err := getDataBrowser(d)
// get datas
params.Set("@GROUP", activeField)
script := getSQLScript(SQLScript+"/databrowser_h3_filter.sql", params)
// tk.Printf("---\n%#v \n----\n", script)
cursor, err := this.DB().Connection.NewQuery().
Command("freequery", tk.M{}.Set("syntax", script)).
Cursor(nil)
defer cursor.Close()
// datas := []SPDataBrowser{}
tmpDatas := []tk.M{}
datas := []tk.M{}
err = cursor.Fetch(&tmpDatas, 0, true)
if e != nil && e.Error() == "No more data to fetched!" {
e = nil
}
if len(tmpDatas) > 0 {
for _, val := range tmpDatas {
tmp := tk.M{}
tmp.Set("_id", val.Get(strings.ToLower(activeField)))
tmp.Set(activeField, val.Get(strings.ToLower(activeField)))
datas = append(datas, tmp)
}
}
ret.Set("Data", datas)
e = err
}
return ret, e
}, nil)
return r
}
func (w *WidgetController) PreviewExample(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
data := toolkit.M{}
if err := r.GetPayload(&data); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
widgetSource := filepath.Join(EC_DATA_PATH, "widget", data.Get("_id", "").(string))
getFileIndex, err := colonycore.GetWidgetPath(widgetSource)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
widgetPath := filepath.Join(getFileIndex, "index.html")
content, err := ioutil.ReadFile(widgetPath)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
contentstring := string(content)
var datasource []string
for _, val := range data.Get("dataSource").([]interface{}) {
datasource = append(datasource, val.(string))
}
dataSourceArry := strings.Join(datasource, ",")
widgetData, err := w.FetchDataSources(dataSourceArry)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
widget := new(colonycore.Widget)
widget.ID = data.GetString("_id")
if err := widget.GetById(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
previewData := toolkit.M{}
previewData.Set("container", contentstring)
previewData.Set("dataSource", widgetData)
previewData.Set("widgetBasePath", strings.Replace(getFileIndex, EC_DATA_PATH+toolkit.PathSeparator+"widget", "", -1))
previewData.Set("settings", widget.Config)
if data.Get("mode", "").(string) == "save" {
dataWidget := colonycore.Widget{}
dataWidget.ID = data.Get("_id", "").(string)
if err := dataWidget.GetById(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// dataWidget.DataSourceId = datasource
configs := toolkit.Ms{}
for _, val := range data.Get("config", "").([]interface{}) {
configs = append(configs, val.(map[string]interface{}))
}
dataWidget.Config = configs
// dataWidget.URL = w.Server.Address
if err := dataWidget.Save(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
return helper.CreateResult(true, previewData, "")
}
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
dsOrigin := new(colonycore.DataSource)
err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dsDestination := new(colonycore.DataSource)
err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dataDS, _, conn, query, metaSave, err := new(DataSourceController).
ConnectToDataSource(dataGrabber.DataSourceOrigin)
if len(dataDS.QueryInfo) == 0 {
message := "Data source origin has invalid query"
logConf.AddLog(message, "ERROR")
return false, nil, message
}
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer conn.Close()
if metaSave.keyword != "" {
message := `Data source origin query is not "Select"`
logConf.AddLog(message, "ERROR")
return false, nil, message
}
cursor, err := query.Cursor(nil)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer cursor.Close()
data := []toolkit.M{}
err = cursor.Fetch(&data, 0, false)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
arrayContains := func(slice []string, key string) bool {
for _, each := range slice {
if each == key {
return true
}
}
return false
}
connDesc := new(colonycore.Connection)
err = colonycore.Get(connDesc, dsDestination.ConnectionID)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
transformedData := []toolkit.M{}
for _, each := range data {
eachTransformedData := toolkit.M{}
for _, eachMeta := range dsDestination.MetaData {
if arrayContains(dataGrabber.IgnoreFieldsDestination, eachMeta.ID) {
continue
}
fieldFrom := eachMeta.ID
checkMap:
for _, eachMap := range dataGrabber.Map {
if eachMap.FieldDestination == eachMeta.ID {
fieldFrom = eachMap.FieldOrigin
break checkMap
}
}
eachTransformedData.Set(eachMeta.ID, each.Get(fieldFrom))
}
transformedData = append(transformedData, eachTransformedData)
tableName := dsDestination.QueryInfo.GetString("from")
queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.GetString("_id")))
queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
err = queryWrapper.Save(tableName, eachTransformedData)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
}
message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
return true, transformedData, ""
}