func (g *Grabber) Grab(parm toolkit.M) error {
errorTxt := ""
sendConf, e := g.GetConfig()
if e != nil {
return fmt.Errorf("Unable to grab %s, GetConfig Error found %s", g.URL, e.Error())
}
r, e := toolkit.HttpCall(g.URL, g.CallType, g.DataByte(), sendConf)
if e != nil {
errorTxt = e.Error()
} else if r.StatusCode != 200 {
errorTxt = r.Status
}
if errorTxt != "" {
return fmt.Errorf("Unable to grab %s. %s", g.URL, errorTxt)
}
g.Response = r
g.bodyByte = toolkit.HttpContent(r)
//Logout ====
if sendConf.Has("cookie") {
tjar := sendConf.Get("cookie", nil).(*cookiejar.Jar)
if tjar != nil && g.LogoutUrl != "" {
_, e := toolkit.HttpCall(g.LogoutUrl, g.CallType, g.DataByte(), toolkit.M{}.Set("cookie", tjar))
if e != nil {
return fmt.Errorf("Unable to logout %s, grab logout Error found %s", g.LogoutUrl, e.Error())
}
}
}
return nil
}
/*
Execute command depend on type that has been declared before
*/
func (c *Command) Exec() (string, error) {
var (
res string
e error
httpRes *http.Response
)
res = "initial"
e = fmt.Errorf("Command %s %s can't be executed. No valid implementation can be found")
if c.Type == CommandType_Local {
ps := []string{}
if c.CommandParms != nil {
ps = c.CommandParms
}
res, e = toolkit.RunCommand(c.CommandText, ps...)
} else if c.Type == CommandType_SSH {
ps := []string{c.CommandText}
res, e = c.SshClient.RunCommandSsh(ps...)
} else if c.Type == CommandType_REST {
if c.RESTAuthType == RESTAuthType_None {
httpRes, e = toolkit.HttpCall(c.RESTUrl, c.RESTMethod, nil, nil)
} else if c.RESTAuthType == RESTAuthType_Basic {
var config = map[string]interface{}{"auth": "basic", "user": c.RESTUser, "password": c.RESTPassword}
httpRes, e = toolkit.HttpCall(c.RESTUrl, c.RESTMethod, nil, config)
// httpRes, e = toolkit.HttpCall(c.RESTUrl, c.RESTMethod, nil, true, c.RESTUser, c.RESTPassword)
}
res = toolkit.HttpContentString(httpRes)
}
return res, e
}
func (s *Subscriber) getMsgAsResult(key string) *toolkit.Result {
result := toolkit.NewResult()
if key == "" {
if len(s.messageKeys) > 0 {
key = s.messageKeys[0]
}
}
// if no key is provided, check from the latest
if key == "" {
result.Status = toolkit.Status_NOK
result.Message = "No key has been provided to receive the message"
} else {
msgQue, exist := s.MessageQues[key]
if !exist {
result.Status = toolkit.Status_NOK
result.Message = "Key " + key + " is not exist on message que or it has been collected. Available keys are: " + strings.Join(s.messageKeys, ",")
} else {
url := fmt.Sprintf("%s/getmsg", s.BroadcasterAddress)
r, e := toolkit.HttpCall(url, "POST",
toolkit.Jsonify(msgQue), nil)
if e != nil {
result.SetErrorTxt("Subscriber ReceiveMsg Call Error: " + e.Error())
} else if r.StatusCode != 200 {
result.SetErrorTxt("Subsciber ReceiveMsg Call Error: " + r.Status)
} else {
//var resultMsg toolkit.Result
e := toolkit.Unjson(toolkit.HttpContent(r),
&result)
if e != nil {
result.SetErrorTxt(fmt.Sprintf("Subsciber ReceiveMsg Decode Error: ", e.Error()))
}
/*else {
if resultMsg.Status == toolkit.Status_OK {
result.Data = resultMsg.Data
} else {
result.SetErrorTxt(resultMsg.Message)
}
}
*/
}
}
}
if result.Status == "OK" {
url := fmt.Sprintf("%s/msgreceived")
toolkit.HttpCall(url, "POST", toolkit.Jsonify(struct {
Key string
Subscriber string
}{key, s.Address}), nil)
}
return result
}
func main() {
UrlStr := "http://www.dce.com.cn/PublicWeb/MainServlet"
dataurl := toolkit.M{}
dataurl["Pu00231_Input.trade_date"] = "20151214"
dataurl["Pu00231_Input.variety"] = "i"
dataurl["Pu00231_Input.trade_type"] = "0"
dataurl["Submit"] = "Go"
dataurl["action"] = "Pu00231_result"
postdata := toolkit.M{}.Set("formvalues", dataurl)
// dataurl := url.Values{}
// dataurl.Add("Pu00231_Input.trade_date", "20151214")
// dataurl.Add("Pu00231_Input.variety", "i")
// dataurl.Add("Pu00231_Input.trade_type", "0")
// dataurl.Add("Submit", "Go")
// dataurl.Add("action", "Pu00231_result")
// tdata := toolkit.Jsonify(dataurl.Encode())
r, _ := toolkit.HttpCall(UrlStr, "POST", nil, postdata)
fmt.Println(string(toolkit.HttpContent(r)))
// rX, _ := http.NewRequest("POST", UrlStr, bytes.NewBuffer(tdata))
// fmt.Println(rX)
// client := &http.Client{}
// resp, _ := client.Do(rX)
// fmt.Println(string(toolkit.HttpContent(resp)))
}
func TestPut(t *testing.T) {
if e := getToken(); e != nil {
//he(t, e)
//return
}
surl := restPrefix + "/put"
for i := 1; i <= 1000; i++ {
randomTxt := createRandomString(toolkit.RandInt(22) + 10)
data := struct {
Token string
Data string
}{
token,
randomTxt,
}
fmt.Printf("Saving %d value %s", i, data.Data)
r, e := toolkit.HttpCall(surl, "XPUT", toolkit.GetEncodeByte(data), nil)
//e = nil
if e != nil {
fmt.Printf("... Fail: %s \n", e.Error())
} else {
if r.StatusCode != 200 {
fmt.Printf("... Fail: %d %s \n", r.StatusCode, r.Status)
} else {
fmt.Println("...Done")
}
fmt.Println("...Done")
}
}
}
func webcall(url, method string, data []byte) ([]byte, error) {
r, e := toolkit.HttpCall(url, method, data, nil)
if e != nil {
return nil, e
}
if r.StatusCode != 200 {
return nil, fmt.Errorf("Invalid Code: " + r.Status)
}
return toolkit.HttpContent(r), nil
}
func (p *Ping) checkHttpStatus() error {
r, e := toolkit.HttpCall(p.Host, "GET", nil, false, "", "")
if e != nil {
return fmt.Errorf("Unable to access %s, %s", p.Host, e.Error())
}
if r.StatusCode != 200 {
return fmt.Errorf("Unable to access %s, code: %d status: %s", p.Host, r.StatusCode, r.Status)
}
return nil
}
/*
Ping service over http status
*/
func (p *Ping) checkHttpStatus() error {
r, e := toolkit.HttpCall(p.Host, "GET", nil, nil)
p.LastStatus = "Fail"
if e != nil {
return fmt.Errorf("Unable to access %s, %s", p.Host, e.Error())
}
if r.StatusCode != 200 {
return fmt.Errorf("Unable to access %s, code: %d status: %s", p.Host, r.StatusCode, r.Status)
}
p.LastStatus = "OK"
return nil
}
func (m *MessageMonitor) distributeBroadcast() {
for len(m.Targets) != m.Success && time.Now().After(m.Expiry) == false {
wg := new(sync.WaitGroup)
for k, t := range m.Targets {
wg.Add(1)
go func(wg *sync.WaitGroup, k int, t string) {
defer wg.Done()
if m.Status[k] != "OK" && m.retry[k] < RetryWhenFail() {
var command, url string
/*
if m.Command != "" {
command = m.Command
} else {
command = "msg"
}
*/
sub := m.broadcaster.Subscibers[t]
if sub != nil {
command = "pushmessage"
url = fmt.Sprintf("http://%s/subscriber/%s", t, command)
r, ecall := toolkit.HttpCall(url, "POST",
toolkit.M{}.Set("secret", sub.Secret).Set("Data", m.Data).Set("Key", m.Key).ToBytes("json", nil),
nil)
if ecall != nil {
m.setSuccessFail(k, "CALL ERROR: "+url+" ERR:"+ecall.Error())
} else if r.StatusCode != 200 {
m.setSuccessFail(k, fmt.Sprintf("CALL STATUS ERROR: %s ERR: %s", url, r.Status))
} else {
var result toolkit.Result
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, &result)
if edecode != nil {
m.setSuccessFail(k, "DECODE ERROR: "+string(bs)+" ERR:"+edecode.Error())
} else {
m.setSuccessFail(k, toolkit.IfEq(result.Status, toolkit.Status_OK, "OK", result.Message).(string))
}
}
}
}
}(wg, k, t)
}
wg.Wait()
//time.Sleep(1 * time.Second)
//fmt.Printf("%d = %d \n", len(m.Targets), m.Success+m.Fail)
}
}
func (s *Server) call(op string, calltype string, qs toolkit.M, payload []byte) (*http.Response, error) {
url := s.Id
url += op
q := ""
for k, v := range qs {
if q == "" {
q = "?" + k + "=" + v.(string)
} else {
q += "&" + k + "=" + v.(string)
}
}
url += q
//rdr := bytes.NewReader(payload)
r, e := toolkit.HttpCall(url, calltype, payload, nil)
return r, e
}
func (w *WebGrabberController) FetchContent(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := new(colonycore.WebGrabber)
err := r.GetPayload(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
param := toolkit.M{} //.Set("formvalues", payload.Parameter)
res, err := toolkit.HttpCall(payload.URL, payload.CallType, nil, param)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
data := toolkit.HttpContentString(res)
return helper.CreateResult(true, data, "")
}
func (p *Ping) checkHttpBody() error {
r, e := toolkit.HttpCall(p.Host, "GET", nil, false, "", "")
if e != nil {
return e
}
body := toolkit.HttpContentString(r)
if p.HttpBodyType == HttpBody_Contains {
if !strings.Contains(body, p.HttpBodySearch) {
return fmt.Errorf("Phrase %s could not be found on response body", p.HttpBodySearch)
}
} else if p.HttpBodyType == HttpBody_Equals {
if body != p.HttpBodySearch {
return fmt.Errorf("Response is not valid. Expecting for %s", p.HttpBodySearch)
}
} else {
return fmt.Errorf("Invalid parameter")
}
return nil
}
func (s *Subscriber) AddChannel(c string) error {
url := fmt.Sprintf("%s/channelregister", s.BroadcasterAddress)
r, e := toolkit.HttpCall(url, "POST", toolkit.Jsonify(ChannelRegister{c, s.Address}), nil)
if e != nil {
return fmt.Errorf("Channel Register Call Error: %s", e.Error())
}
if r.StatusCode != 200 {
return fmt.Errorf("Channel Register Call Error: %s", r.Status)
}
result := new(toolkit.Result)
e = toolkit.Unjson(toolkit.HttpContent(r), &result)
if e != nil {
return fmt.Errorf("Channel Register Decode error: %s", e.Error())
}
if result.Status != toolkit.Status_OK {
return fmt.Errorf("Channel Register error: %s", result.Message)
}
return nil
}
func main() {
url := "http://www.dce.com.cn/PublicWeb/MainServlet"
formvalues := toolkit.M{}.Set("Pu00231_Input.trade_date", "20151214").
Set("Pu00231_Input.variety", "i").
Set("Pu00231_Input.trade_type", "0").
Set("Submit", "Go").
Set("action", "Pu00231_result")
config := toolkit.M{}.Set("formvalues", formvalues)
r, e := toolkit.HttpCall(url, "POST", nil, config)
if e != nil {
fmt.Printf("ERROR:\n%s\n", e.Error())
return
}
fmt.Printf("Result:\n%s\n", toolkit.HttpContentString(r))
}
func call(url, call string, data []byte, expectedStatus int) (*toolkit.Result, error) {
cfg := toolkit.M{}
if expectedStatus != 0 {
cfg.Set("expectedstatus", expectedStatus)
}
r, e := toolkit.HttpCall(url, call, data, cfg)
if e != nil {
return nil, fmt.Errorf(url + " Call Error: " + e.Error())
}
result := toolkit.NewResult()
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, result)
if edecode != nil {
return nil, fmt.Errorf(url + " Http Result Decode Error: " + edecode.Error() + "\nFollowing got: " + string(bs))
}
if result.Status == toolkit.Status_NOK {
return nil, fmt.Errorf(url + " Http Result Error: " + result.Message)
}
return result, nil
}
func (a *ConfigurationController) GetURL(k *knot.WebContext) interface{} {
d := struct {
URL string
Method string
Parameter tk.M
}{}
e := k.GetPayload(&d)
if e != nil {
fmt.Println(e)
}
k.Config.OutputType = knot.OutputJson
fmt.Println(d)
r, e := tk.HttpCall(d.URL, d.Method, nil, tk.M{}.Set("formvalues", d.Parameter))
fmt.Println(e)
if e != nil {
return e.Error()
} else {
return tk.HttpContentString(r)
}
}
func (w *WebGrabberController) FetchContent(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := struct {
URL string
Method string
Payloads toolkit.M
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
res, err := toolkit.HttpCall(payload.URL, payload.Method, nil, payload.Payloads)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
data := toolkit.HttpContentString(res)
return helper.CreateResult(true, data, "")
}
func (m *MessageMonitor) ditributeBroadcast() {
//for len(m.Targets) != m.Success && time.Now().After(m.Expiry) == false {
wg := new(sync.WaitGroup)
for k, t := range m.Targets {
wg.Add(1)
go func(wg *sync.WaitGroup, k int, t string) {
defer wg.Done()
if m.Status[k] != "OK" {
var command, url string
if m.Command != "" {
command = m.Command
} else {
command = "msg"
}
url = fmt.Sprintf("http://%s/%s", t, command)
r, ecall := toolkit.HttpCall(url, "POST",
toolkit.Jsonify(Message{Key: m.Key, Data: m.Data, Expiry: m.Expiry}), nil)
if ecall != nil {
m.setSuccessFail(k, "CALL ERROR: "+url+" ERR:"+ecall.Error())
} else if r.StatusCode != 200 {
m.setSuccessFail(k, fmt.Sprintf("CALL STATUS ERROR: %s ERR: %s", url, r.Status))
} else {
var result toolkit.Result
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, &result)
if edecode != nil {
m.setSuccessFail(k, "DECODE ERROR: "+string(bs)+" ERR:"+edecode.Error())
} else {
m.setSuccessFail(k, toolkit.IfEq(result.Status, toolkit.Status_OK, "OK", result.Message).(string))
}
}
}
}(wg, k, t)
}
wg.Wait()
//time.Sleep(1 * time.Millisecond)
//fmt.Printf("%d = %d \n", len(m.Targets), m.Success+m.Fail)
//}
}
func getToken() error {
surl := restPrefix + "auth/init"
b, e := toolkit.EncodeByte(struct {
Key string
Secret string
}{"ariefdarmawan", "Password.1"})
if e != nil {
return fmt.Errorf("Unable to connect: %s", e.Error())
}
r, e := toolkit.HttpCall(surl, "get", b, nil)
if e != nil {
return fmt.Errorf("Unable to connect: %s", e.Error())
}
token := toolkit.HttpContentM(r).Get("Token", "").(string)
if token == "" {
return fmt.Errorf("Wrong token returned")
}
return nil
}
func (m *MessageMonitor) distributeQue() {
//--- inform all targets that new message has been created
msg := toolkit.Jsonify(Message{Key: m.Key, Data: m.Data, Expiry: m.Expiry})
wg := new(sync.WaitGroup)
targetCount := len(m.Targets)
var newtargets []string
var failtargets []string
for _, t := range m.Targets {
wg.Add(1)
go func(wg *sync.WaitGroup, t string) {
defer wg.Done()
url := fmt.Sprintf("%s://%s/newkey", "http", t)
r, e := toolkit.HttpCall(url, "POST", msg, nil)
if e != nil {
m.Broadcaster.Log().Warning(fmt.Sprintf(
"Unable to inform %s for new que %s. %s",
url, m.Key, e.Error()))
failtargets = append(failtargets, t)
} else if r.StatusCode != 200 {
m.Broadcaster.Log().Warning(fmt.Sprintf(
"Unable to inform %s for new que %s %s",
url, m.Key, r.Status))
failtargets = append(failtargets, t)
} else {
newtargets = append(newtargets, t)
}
}(wg, t)
}
wg.Wait()
m.Targets = newtargets
m.Broadcaster.Log().Info(fmt.Sprintf("Ping %d servers for new message %s. Succcess: %d Fail: %d", targetCount, m.Key, len(newtargets), len(failtargets)))
//-- loop while not all target complete receival or expire
for len(m.Targets) != m.Success && time.Now().After(m.Expiry) == false {
time.Sleep(1 * time.Millisecond)
}
}
func (s *Subscriber) Start(address string) error {
s.Address = address
s.Actions = map[string]FnTrigger{}
s.MessageQues = map[string]*MessageQue{}
broadcasterUrl := s.BroadcasterAddress + "/nodeadd?node=" + s.Address
r, e := toolkit.HttpCall(broadcasterUrl, "GET", nil, nil)
if e != nil {
return fmt.Errorf("Unable to contact Broadcaster Server. %s", e.Error())
}
if sOk := toolkit.HttpContentString(r); sOk != "OK" {
return fmt.Errorf("Unable to add %s as subsriber to %s. Error: %s", s.Address, s.BroadcasterAddress, sOk)
}
//-- inform subscriber if newkey is available to be collected
s.Route("/newkey", func(k *knot.WebContext) interface{} {
result := toolkit.NewResult()
k.Config.OutputType = knot.OutputJson
msg := new(MessageQue)
e := k.GetPayload(&msg)
if e != nil {
result.Message = e.Error()
result.Status = toolkit.Status_NOK
} else {
msg.Data = nil
msg.Collected = false
_, exist := s.MessageQues[msg.Key]
if !exist {
s.messageKeys = append(s.messageKeys, msg.Key)
}
s.MessageQues[msg.Key] = msg
//s.messageKeys = append(s.messageKeys, msg.Key)
}
return result
})
s.Route("/getmsg", func(k *knot.WebContext) interface{} {
k.Config.OutputType = knot.OutputJson
key := k.Query("key")
result := s.getMsgAsResult(key)
return result
})
//-- replicate message to subscribe
s.Route("/msg", func(k *knot.WebContext) interface{} {
k.Config.OutputType = knot.OutputJson
result := toolkit.NewResult()
var payload Message
eDecode := k.GetPayload(&payload)
if eDecode != nil {
result.Status = toolkit.Status_NOK
result.Message = fmt.Sprintf("Subscriber Message Decode Error %s", eDecode.Error())
} else {
result.Data = payload
}
return result
})
//-- stop the server
s.Route("/stop", func(k *knot.WebContext) interface{} {
defer k.Server.Stop()
k.Config.OutputType = knot.OutputJson
result := new(toolkit.Result)
result.Status = "OK"
return result
})
go func() {
s.Server.Listen()
}()
return nil
}
func call(url string) (*http.Response, error) {
surl := "http://localhost:13000" + url
r, e := toolkit.HttpCall(surl, "GET", nil, nil)
return r, e
}