func (t *HttpFetcher) Fetch(task *tasks.FetcherTask) ([]byte, error) {
logger.Infof("%s HTTPFetcher config: %v", task.Id, t.httpFetcherConfig)
url := fmt.Sprintf("http://%s:%d%s",
task.Target,
t.Port,
t.Uri)
var (
resp *http.Response
err error
)
if t.ConnTimeout == DEFAULT_CONNECTION_TIMEOUT && t.ReadTimeout == DEFAULT_RW_TIMEOUT {
logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
task.Id, url, CONNECTION_TIMEOUT, RW_TIMEOUT)
resp, err = HttpClient.Get(url)
if err != nil {
return nil, err
}
} else {
connTimeout := time.Duration(t.ConnTimeout) * time.Millisecond
rwTimeout := time.Duration(t.ReadTimeout) * time.Millisecond
logger.Infof("%s requested URL: %s, nondefault timeouts: conn %v rw %v",
task.Id, url, connTimeout, rwTimeout)
httpCli := httpclient.NewClientWithTimeout(
connTimeout, rwTimeout)
httpCli.Transport.(*http.Transport).DisableKeepAlives = true
resp, err = httpCli.Get(url)
if err != nil {
return nil, err
}
}
defer resp.Body.Close()
return ioutil.ReadAll(resp.Body)
}
func (t *Timetail) Fetch(task *tasks.FetcherTask) ([]byte, error) {
period := t.Offset + (task.CurrTime - task.PrevTime)
url := fmt.Sprintf("http://%s:%d%s%s&time=%d",
task.Target,
t.Port,
t.Url,
t.Logname,
period)
logger.Infof("%s Requested URL: %s", task.Id, url)
var (
resp *http.Response
err error
)
if t.TimetailConfig.ConnTimeout == CONNECTION_TIMEOUT && t.TimetailConfig.ReadTimeout == RW_TIMEOUT {
logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
task.Id, url, D_CONNECTION_TIMEOUT, D_RW_TIMEOUT)
resp, err = HttpClient.Get(url)
} else {
connTimeout := time.Duration(t.TimetailConfig.ConnTimeout) * time.Millisecond
rwTimeout := time.Duration(t.TimetailConfig.ReadTimeout) * time.Millisecond
logger.Infof("%s requested URL: %s, timeouts conn %v rw %v",
task.Id, url, connTimeout, rwTimeout)
httpCli := httpclient.NewClientWithTimeout(
connTimeout, rwTimeout)
httpCli.Transport.(*http.Transport).DisableKeepAlives = true
resp, err = httpCli.Get(url)
}
if err != nil {
return nil, err
}
defer resp.Body.Close()
logger.Infof("%s Result for URL %s: %d", task.Id, url, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
return body, nil
}
func (r *RazladkiSender) Send(data tasks.DataType, timestamp uint64) error {
res, err := r.send(data, timestamp)
if err != nil {
return err
}
var buffer = new(bytes.Buffer)
if err = json.NewEncoder(buffer).Encode(res); err != nil {
return err
}
url := fmt.Sprintf("http://%s/save_new_data_json/%s", r.Host, r.Project)
logger.Infof("%s send to url %s, data %s", r.id, url, buffer.Bytes())
req, err := http.NewRequest("POST", url, buffer)
if err != nil {
return err
}
resp, err := RazladkiHttpClient.Do(req)
if err != nil {
logger.Errf("%s unable to do http request: %v", r.id, err)
return err
}
defer resp.Body.Close()
logger.Infof("%s response status %d %s", r.id, resp.StatusCode, resp.Status)
if resp.StatusCode != http.StatusOK {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("bad response code %d: %s", resp.StatusCode, resp.Status)
}
return fmt.Errorf("bad response code %d %s: %s", resp.StatusCode, resp.Status, b)
}
return nil
}
func (as *AgaveSender) sendPoint(url string) {
for _, host := range as.Hosts {
req, _ := http.NewRequest("GET",
fmt.Sprintf("http://%s%s", host, url),
nil)
req.Header = DEFAULT_HEADERS
logger.Debugf("%s %s", as.Id, req.URL)
resp, err := AgaveHttpClient.Do(req)
if err != nil {
logger.Errf("%s Unable to do request %s", as.Id, err)
continue
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Errf("%s %s %d %s", as.Id, req.URL, resp.StatusCode, err)
continue
}
logger.Infof("%s %s %d %s", as.Id, req.URL, resp.StatusCode, body)
}
}
func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) {
logger.Infof("%s start parsing", task.Id)
var (
blob []byte
err error
payload interface{}
wg sync.WaitGroup
)
blob, err = fetchDataFromTarget(&task)
if err != nil {
logger.Errf("%s error `%v` occured while fetching data", task.Id, err)
return nil, err
}
if !task.ParsingConfig.SkipParsingStage() {
logger.Infof("%s Send data to parsing", task.Id)
parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP)
if err != nil {
logger.Errf("%s error `%v` occured while resolving %s",
task.Id, err, common.PARSINGAPP)
return nil, err
}
taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob})
if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil {
logger.Errf("%s error `%v` occured while parsing data", task.Id, err)
return nil, err
}
}
payload = blob
if !task.ParsingConfig.Raw {
logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP)
datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP)
if err != nil {
logger.Errf("%s unable to get DG %v", task.Id, err)
return nil, err
}
var token string
if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil {
logger.Errf("%s unable to put data to DG %v", task.Id, err)
return nil, err
}
defer func() {
taskToDatagrid, _ := common.Pack([]interface{}{token})
datagrid.Do("enqueue", "drop", taskToDatagrid)
logger.Debugf("%s Drop table", task.Id)
}()
payload = token
}
pr := NewParsingResult()
for aggLogName, aggCfg := range task.AggregationConfigs {
for k, v := range aggCfg.Data {
aggType, err := v.Type()
if err != nil {
logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v)
return nil, err
}
logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v)
wg.Add(1)
go func(name string, dataName string, v interface{}, configName string) {
defer wg.Done()
app, err := ctx.Resolver.Resolve(ctx.Ctx, name)
if err != nil {
logger.Errf("%s %s %s", task.Id, name, err)
return
}
// Task structure
t, _ := common.Pack(map[string]interface{}{
"config": v,
"token": payload,
"prevtime": task.PrevTime,
"currtime": task.CurrTime,
"id": task.Id,
})
var rawRes []byte
if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil {
logger.Errf("%s Failed task: %v", task.Id, err)
return
}
logger.Debugf("result for %s %s: %v", configName, dataName, rawRes)
pr.Put(configName, dataName, rawRes)
}(aggType, k, v, aggLogName)
}
}
wg.Wait()
logger.Infof("%s Done", task.Id)
return pr, nil
}