func (g *Grabber) Grab(parm toolkit.M) error {
errorTxt := ""
sendConf, e := g.GetConfig()
if e != nil {
return fmt.Errorf("Unable to grab %s, GetConfig Error found %s", g.URL, e.Error())
}
r, e := toolkit.HttpCall(g.URL, g.CallType, g.DataByte(), sendConf)
if e != nil {
errorTxt = e.Error()
} else if r.StatusCode != 200 {
errorTxt = r.Status
}
if errorTxt != "" {
return fmt.Errorf("Unable to grab %s. %s", g.URL, errorTxt)
}
g.Response = r
g.bodyByte = toolkit.HttpContent(r)
//Logout ====
if sendConf.Has("cookie") {
tjar := sendConf.Get("cookie", nil).(*cookiejar.Jar)
if tjar != nil && g.LogoutUrl != "" {
_, e := toolkit.HttpCall(g.LogoutUrl, g.CallType, g.DataByte(), toolkit.M{}.Set("cookie", tjar))
if e != nil {
return fmt.Errorf("Unable to logout %s, grab logout Error found %s", g.LogoutUrl, e.Error())
}
}
}
return nil
}
func main() {
UrlStr := "http://www.dce.com.cn/PublicWeb/MainServlet"
dataurl := toolkit.M{}
dataurl["Pu00231_Input.trade_date"] = "20151214"
dataurl["Pu00231_Input.variety"] = "i"
dataurl["Pu00231_Input.trade_type"] = "0"
dataurl["Submit"] = "Go"
dataurl["action"] = "Pu00231_result"
postdata := toolkit.M{}.Set("formvalues", dataurl)
// dataurl := url.Values{}
// dataurl.Add("Pu00231_Input.trade_date", "20151214")
// dataurl.Add("Pu00231_Input.variety", "i")
// dataurl.Add("Pu00231_Input.trade_type", "0")
// dataurl.Add("Submit", "Go")
// dataurl.Add("action", "Pu00231_result")
// tdata := toolkit.Jsonify(dataurl.Encode())
r, _ := toolkit.HttpCall(UrlStr, "POST", nil, postdata)
fmt.Println(string(toolkit.HttpContent(r)))
// rX, _ := http.NewRequest("POST", UrlStr, bytes.NewBuffer(tdata))
// fmt.Println(rX)
// client := &http.Client{}
// resp, _ := client.Do(rX)
// fmt.Println(string(toolkit.HttpContent(resp)))
}
func webcall(url, method string, data []byte) ([]byte, error) {
r, e := toolkit.HttpCall(url, method, data, nil)
if e != nil {
return nil, e
}
if r.StatusCode != 200 {
return nil, fmt.Errorf("Invalid Code: " + r.Status)
}
return toolkit.HttpContent(r), nil
}
func (s *Subscriber) getMsgAsResult(key string) *toolkit.Result {
result := toolkit.NewResult()
if key == "" {
if len(s.messageKeys) > 0 {
key = s.messageKeys[0]
}
}
// if no key is provided, check from the latest
if key == "" {
result.Status = toolkit.Status_NOK
result.Message = "No key has been provided to receive the message"
} else {
msgQue, exist := s.MessageQues[key]
if !exist {
result.Status = toolkit.Status_NOK
result.Message = "Key " + key + " is not exist on message que or it has been collected. Available keys are: " + strings.Join(s.messageKeys, ",")
} else {
url := fmt.Sprintf("%s/getmsg", s.BroadcasterAddress)
r, e := toolkit.HttpCall(url, "POST",
toolkit.Jsonify(msgQue), nil)
if e != nil {
result.SetErrorTxt("Subscriber ReceiveMsg Call Error: " + e.Error())
} else if r.StatusCode != 200 {
result.SetErrorTxt("Subsciber ReceiveMsg Call Error: " + r.Status)
} else {
//var resultMsg toolkit.Result
e := toolkit.Unjson(toolkit.HttpContent(r),
&result)
if e != nil {
result.SetErrorTxt(fmt.Sprintf("Subsciber ReceiveMsg Decode Error: ", e.Error()))
}
/*else {
if resultMsg.Status == toolkit.Status_OK {
result.Data = resultMsg.Data
} else {
result.SetErrorTxt(resultMsg.Message)
}
}
*/
}
}
}
if result.Status == "OK" {
url := fmt.Sprintf("%s/msgreceived")
toolkit.HttpCall(url, "POST", toolkit.Jsonify(struct {
Key string
Subscriber string
}{key, s.Address}), nil)
}
return result
}
func (m *MessageMonitor) distributeBroadcast() {
for len(m.Targets) != m.Success && time.Now().After(m.Expiry) == false {
wg := new(sync.WaitGroup)
for k, t := range m.Targets {
wg.Add(1)
go func(wg *sync.WaitGroup, k int, t string) {
defer wg.Done()
if m.Status[k] != "OK" && m.retry[k] < RetryWhenFail() {
var command, url string
/*
if m.Command != "" {
command = m.Command
} else {
command = "msg"
}
*/
sub := m.broadcaster.Subscibers[t]
if sub != nil {
command = "pushmessage"
url = fmt.Sprintf("http://%s/subscriber/%s", t, command)
r, ecall := toolkit.HttpCall(url, "POST",
toolkit.M{}.Set("secret", sub.Secret).Set("Data", m.Data).Set("Key", m.Key).ToBytes("json", nil),
nil)
if ecall != nil {
m.setSuccessFail(k, "CALL ERROR: "+url+" ERR:"+ecall.Error())
} else if r.StatusCode != 200 {
m.setSuccessFail(k, fmt.Sprintf("CALL STATUS ERROR: %s ERR: %s", url, r.Status))
} else {
var result toolkit.Result
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, &result)
if edecode != nil {
m.setSuccessFail(k, "DECODE ERROR: "+string(bs)+" ERR:"+edecode.Error())
} else {
m.setSuccessFail(k, toolkit.IfEq(result.Status, toolkit.Status_OK, "OK", result.Message).(string))
}
}
}
}
}(wg, k, t)
}
wg.Wait()
//time.Sleep(1 * time.Second)
//fmt.Printf("%d = %d \n", len(m.Targets), m.Success+m.Fail)
}
}
func (s *Subscriber) AddChannel(c string) error {
url := fmt.Sprintf("%s/channelregister", s.BroadcasterAddress)
r, e := toolkit.HttpCall(url, "POST", toolkit.Jsonify(ChannelRegister{c, s.Address}), nil)
if e != nil {
return fmt.Errorf("Channel Register Call Error: %s", e.Error())
}
if r.StatusCode != 200 {
return fmt.Errorf("Channel Register Call Error: %s", r.Status)
}
result := new(toolkit.Result)
e = toolkit.Unjson(toolkit.HttpContent(r), &result)
if e != nil {
return fmt.Errorf("Channel Register Decode error: %s", e.Error())
}
if result.Status != toolkit.Status_OK {
return fmt.Errorf("Channel Register error: %s", result.Message)
}
return nil
}
func call(url, call string, data []byte, expectedStatus int) (*toolkit.Result, error) {
cfg := toolkit.M{}
if expectedStatus != 0 {
cfg.Set("expectedstatus", expectedStatus)
}
r, e := toolkit.HttpCall(url, call, data, cfg)
if e != nil {
return nil, fmt.Errorf(url + " Call Error: " + e.Error())
}
result := toolkit.NewResult()
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, result)
if edecode != nil {
return nil, fmt.Errorf(url + " Http Result Decode Error: " + edecode.Error() + "\nFollowing got: " + string(bs))
}
if result.Status == toolkit.Status_NOK {
return nil, fmt.Errorf(url + " Http Result Error: " + result.Message)
}
return result, nil
}
func (m *MessageMonitor) ditributeBroadcast() {
//for len(m.Targets) != m.Success && time.Now().After(m.Expiry) == false {
wg := new(sync.WaitGroup)
for k, t := range m.Targets {
wg.Add(1)
go func(wg *sync.WaitGroup, k int, t string) {
defer wg.Done()
if m.Status[k] != "OK" {
var command, url string
if m.Command != "" {
command = m.Command
} else {
command = "msg"
}
url = fmt.Sprintf("http://%s/%s", t, command)
r, ecall := toolkit.HttpCall(url, "POST",
toolkit.Jsonify(Message{Key: m.Key, Data: m.Data, Expiry: m.Expiry}), nil)
if ecall != nil {
m.setSuccessFail(k, "CALL ERROR: "+url+" ERR:"+ecall.Error())
} else if r.StatusCode != 200 {
m.setSuccessFail(k, fmt.Sprintf("CALL STATUS ERROR: %s ERR: %s", url, r.Status))
} else {
var result toolkit.Result
bs := toolkit.HttpContent(r)
edecode := toolkit.Unjson(bs, &result)
if edecode != nil {
m.setSuccessFail(k, "DECODE ERROR: "+string(bs)+" ERR:"+edecode.Error())
} else {
m.setSuccessFail(k, toolkit.IfEq(result.Status, toolkit.Status_OK, "OK", result.Message).(string))
}
}
}
}(wg, k, t)
}
wg.Wait()
//time.Sleep(1 * time.Millisecond)
//fmt.Printf("%d = %d \n", len(m.Targets), m.Success+m.Fail)
//}
}