Example #1
0
func (t *HttpFetcher) Fetch(task *tasks.FetcherTask) ([]byte, error) {
	logger.Infof("%s HTTPFetcher config: %v", task.Id, t.httpFetcherConfig)
	url := fmt.Sprintf("http://%s:%d%s",
		task.Target,
		t.Port,
		t.Uri)

	var (
		resp *http.Response
		err  error
	)
	if t.ConnTimeout == DEFAULT_CONNECTION_TIMEOUT && t.ReadTimeout == DEFAULT_RW_TIMEOUT {
		logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
			task.Id, url, CONNECTION_TIMEOUT, RW_TIMEOUT)
		resp, err = HttpClient.Get(url)
		if err != nil {
			return nil, err
		}
	} else {
		connTimeout := time.Duration(t.ConnTimeout) * time.Millisecond
		rwTimeout := time.Duration(t.ReadTimeout) * time.Millisecond
		logger.Infof("%s requested URL: %s, nondefault timeouts: conn %v rw %v",
			task.Id, url, connTimeout, rwTimeout)
		httpCli := httpclient.NewClientWithTimeout(
			connTimeout, rwTimeout)
		httpCli.Transport.(*http.Transport).DisableKeepAlives = true
		resp, err = httpCli.Get(url)
		if err != nil {
			return nil, err
		}
	}
	defer resp.Body.Close()

	return ioutil.ReadAll(resp.Body)
}
Example #2
0
func (t *Timetail) Fetch(task *tasks.FetcherTask) ([]byte, error) {
	period := t.Offset + (task.CurrTime - task.PrevTime)

	url := fmt.Sprintf("http://%s:%d%s%s&time=%d",
		task.Target,
		t.Port,
		t.Url,
		t.Logname,
		period)

	logger.Infof("%s Requested URL: %s", task.Id, url)

	var (
		resp *http.Response
		err  error
	)

	if t.TimetailConfig.ConnTimeout == CONNECTION_TIMEOUT && t.TimetailConfig.ReadTimeout == RW_TIMEOUT {
		logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
			task.Id, url, D_CONNECTION_TIMEOUT, D_RW_TIMEOUT)
		resp, err = HttpClient.Get(url)
	} else {
		connTimeout := time.Duration(t.TimetailConfig.ConnTimeout) * time.Millisecond
		rwTimeout := time.Duration(t.TimetailConfig.ReadTimeout) * time.Millisecond
		logger.Infof("%s requested URL: %s, timeouts conn %v rw %v",
			task.Id, url, connTimeout, rwTimeout)
		httpCli := httpclient.NewClientWithTimeout(
			connTimeout, rwTimeout)
		httpCli.Transport.(*http.Transport).DisableKeepAlives = true
		resp, err = httpCli.Get(url)
	}
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	logger.Infof("%s Result for URL %s: %d", task.Id, url, resp.StatusCode)

	body, err := ioutil.ReadAll(resp.Body)
	return body, nil
}
Example #3
0
func (r *RazladkiSender) Send(data tasks.DataType, timestamp uint64) error {
	res, err := r.send(data, timestamp)
	if err != nil {
		return err
	}

	var buffer = new(bytes.Buffer)
	if err = json.NewEncoder(buffer).Encode(res); err != nil {
		return err
	}

	url := fmt.Sprintf("http://%s/save_new_data_json/%s", r.Host, r.Project)
	logger.Infof("%s send to url %s, data %s", r.id, url, buffer.Bytes())
	req, err := http.NewRequest("POST", url, buffer)
	if err != nil {
		return err
	}

	resp, err := RazladkiHttpClient.Do(req)
	if err != nil {
		logger.Errf("%s unable to do http request: %v", r.id, err)
		return err
	}
	defer resp.Body.Close()

	logger.Infof("%s response status %d %s", r.id, resp.StatusCode, resp.Status)
	if resp.StatusCode != http.StatusOK {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("bad response code %d: %s", resp.StatusCode, resp.Status)
		}
		return fmt.Errorf("bad response code %d %s: %s", resp.StatusCode, resp.Status, b)
	}

	return nil
}
Example #4
0
func (as *AgaveSender) sendPoint(url string) {
	for _, host := range as.Hosts {
		req, _ := http.NewRequest("GET",
			fmt.Sprintf("http://%s%s", host, url),
			nil)
		req.Header = DEFAULT_HEADERS

		logger.Debugf("%s %s", as.Id, req.URL)
		resp, err := AgaveHttpClient.Do(req)
		if err != nil {
			logger.Errf("%s Unable to do request %s", as.Id, err)
			continue
		}
		defer resp.Body.Close()

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Errf("%s %s %d %s", as.Id, req.URL, resp.StatusCode, err)
			continue
		}

		logger.Infof("%s %s %d %s", as.Id, req.URL, resp.StatusCode, body)
	}
}
Example #5
0
func Parsing(ctx *ParsingContext, task tasks.ParsingTask) (*ParsingResult, error) {
	logger.Infof("%s start parsing", task.Id)

	var (
		blob    []byte
		err     error
		payload interface{}
		wg      sync.WaitGroup
	)

	blob, err = fetchDataFromTarget(&task)
	if err != nil {
		logger.Errf("%s error `%v` occured while fetching data", task.Id, err)
		return nil, err
	}

	if !task.ParsingConfig.SkipParsingStage() {
		logger.Infof("%s Send data to parsing", task.Id)

		parsingApp, err := ctx.Resolver.Resolve(ctx.Ctx, common.PARSINGAPP)
		if err != nil {
			logger.Errf("%s error `%v` occured while resolving %s",
				task.Id, err, common.PARSINGAPP)
			return nil, err
		}

		taskToParser, _ := common.Pack([]interface{}{task.Id, task.ParsingConfig.Parser, blob})
		if err := parsingApp.Do("enqueue", "parse", taskToParser).Wait(ctx.Ctx, &blob); err != nil {
			logger.Errf("%s error `%v` occured while parsing data", task.Id, err)
			return nil, err
		}
	}

	payload = blob

	if !task.ParsingConfig.Raw {
		logger.Debugf("%s Use %s for handle data", task.Id, common.DATABASEAPP)

		datagrid, err := ctx.Resolver.Resolve(ctx.Ctx, common.DATABASEAPP)
		if err != nil {
			logger.Errf("%s unable to get DG %v", task.Id, err)
			return nil, err
		}

		var token string
		if err := datagrid.Do("enqueue", "put", blob).Wait(ctx.Ctx, &token); err != nil {
			logger.Errf("%s unable to put data to DG %v", task.Id, err)
			return nil, err
		}

		defer func() {
			taskToDatagrid, _ := common.Pack([]interface{}{token})
			datagrid.Do("enqueue", "drop", taskToDatagrid)
			logger.Debugf("%s Drop table", task.Id)
		}()
		payload = token
	}

	pr := NewParsingResult()

	for aggLogName, aggCfg := range task.AggregationConfigs {
		for k, v := range aggCfg.Data {
			aggType, err := v.Type()
			if err != nil {
				logger.Errf("no type in configuration: %s %s %v", aggLogName, k, v)
				return nil, err
			}

			logger.Debugf("%s Send to %s %s type %s %v", task.Id, aggLogName, k, aggType, v)

			wg.Add(1)
			go func(name string, dataName string, v interface{}, configName string) {
				defer wg.Done()

				app, err := ctx.Resolver.Resolve(ctx.Ctx, name)
				if err != nil {
					logger.Errf("%s %s %s", task.Id, name, err)
					return
				}

				// Task structure
				t, _ := common.Pack(map[string]interface{}{
					"config":   v,
					"token":    payload,
					"prevtime": task.PrevTime,
					"currtime": task.CurrTime,
					"id":       task.Id,
				})

				var rawRes []byte
				if err := app.Do("enqueue", "aggregate_host", t).Wait(ctx.Ctx, &rawRes); err != nil {
					logger.Errf("%s Failed task: %v", task.Id, err)
					return
				}

				logger.Debugf("result for %s %s: %v", configName, dataName, rawRes)

				pr.Put(configName, dataName, rawRes)

			}(aggType, k, v, aggLogName)
		}
	}
	wg.Wait()

	logger.Infof("%s Done", task.Id)
	return pr, nil
}