func (t *HttpFetcher) Fetch(task *tasks.FetcherTask) ([]byte, error) {
logger.Infof("%s HTTPFetcher config: %v", task.Id, t.httpFetcherConfig)
url := fmt.Sprintf("http://%s:%d%s",
task.Target,
t.Port,
t.Uri)
var (
resp *http.Response
err error
)
if t.ConnTimeout == DEFAULT_CONNECTION_TIMEOUT && t.ReadTimeout == DEFAULT_RW_TIMEOUT {
logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
task.Id, url, CONNECTION_TIMEOUT, RW_TIMEOUT)
resp, err = HttpClient.Get(url)
if err != nil {
return nil, err
}
} else {
connTimeout := time.Duration(t.ConnTimeout) * time.Millisecond
rwTimeout := time.Duration(t.ReadTimeout) * time.Millisecond
logger.Infof("%s requested URL: %s, nondefault timeouts: conn %v rw %v",
task.Id, url, connTimeout, rwTimeout)
httpCli := httpclient.NewClientWithTimeout(
connTimeout, rwTimeout)
httpCli.Transport.(*http.Transport).DisableKeepAlives = true
resp, err = httpCli.Get(url)
if err != nil {
return nil, err
}
}
defer resp.Body.Close()
return ioutil.ReadAll(resp.Body)
}
func (t *Timetail) Fetch(task *tasks.FetcherTask) ([]byte, error) {
period := t.Offset + (task.CurrTime - task.PrevTime)
url := fmt.Sprintf("http://%s:%d%s%s&time=%d",
task.Target,
t.Port,
t.Url,
t.Logname,
period)
logger.Infof("%s Requested URL: %s", task.Id, url)
var (
resp *http.Response
err error
)
if t.TimetailConfig.ConnTimeout == CONNECTION_TIMEOUT && t.TimetailConfig.ReadTimeout == RW_TIMEOUT {
logger.Infof("%s requested URL: %s, default timeouts conn %v rw %v",
task.Id, url, D_CONNECTION_TIMEOUT, D_RW_TIMEOUT)
resp, err = HttpClient.Get(url)
} else {
connTimeout := time.Duration(t.TimetailConfig.ConnTimeout) * time.Millisecond
rwTimeout := time.Duration(t.TimetailConfig.ReadTimeout) * time.Millisecond
logger.Infof("%s requested URL: %s, timeouts conn %v rw %v",
task.Id, url, connTimeout, rwTimeout)
httpCli := httpclient.NewClientWithTimeout(
connTimeout, rwTimeout)
httpCli.Transport.(*http.Transport).DisableKeepAlives = true
resp, err = httpCli.Get(url)
}
if err != nil {
return nil, err
}
defer resp.Body.Close()
logger.Infof("%s Result for URL %s: %d", task.Id, url, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
return body, nil
}
panic(err)
}
if err := RegisterFetcherLoader("predefine", newPredefineFetcher); err != nil {
panic(err)
}
}
const (
fetcherCacheNamespace = "simpleFetcherCacheNamespace"
)
type FetcherLoader func(*Context, map[string]interface{}) (HostFetcher, error)
var (
fetchers map[string]FetcherLoader = make(map[string]FetcherLoader)
httpClient = httpclient.NewClientWithTimeout(1*time.Second, 3*time.Second)
)
func RegisterFetcherLoader(name string, f FetcherLoader) error {
_, ok := fetchers[name]
if ok {
return fmt.Errorf("HostFetcher `%s` is already registered", name)
}
fetchers[name] = f
return nil
}
func LoadHostFetcher(context *Context, config configs.PluginConfig) (HostFetcher, error) {
name, err := config.Type()
if err != nil {
"time"
"github.com/noxiouz/Combaine/common"
"github.com/noxiouz/Combaine/common/httpclient"
"github.com/noxiouz/Combaine/common/logger"
"github.com/noxiouz/Combaine/common/tasks"
)
const (
CONNECTION_TIMEOUT = 2000 // ms
RW_TIMEOUT = 3000 // ms
)
var (
RazladkiHttpClient = httpclient.NewClientWithTimeout(
time.Millisecond*CONNECTION_TIMEOUT,
time.Millisecond*RW_TIMEOUT)
)
type RazladkiConfig struct {
Items map[string]string `codec:"items"`
Project string `codec:"project"`
Host string `codec:"host"`
}
type RazladkiSender struct {
*RazladkiConfig
id string
}
type Meta struct {
func init() {
parsing.Register("http", NewHttpFetcher)
}
const (
DEFAULT_CONNECTION_TIMEOUT = 1000
DEFAULT_RW_TIMEOUT = 3000
)
var (
CONNECTION_TIMEOUT = DEFAULT_CONNECTION_TIMEOUT * time.Millisecond
RW_TIMEOUT = DEFAULT_RW_TIMEOUT * time.Millisecond
)
var HttpClient = httpclient.NewClientWithTimeout(CONNECTION_TIMEOUT, RW_TIMEOUT)
type HttpFetcher struct {
httpFetcherConfig
}
type httpFetcherConfig struct {
Port int `mapstructure:"port"`
Uri string `mapstructure:"uri"`
ConnTimeout int `mapstructure:"connection_timeout"`
ReadTimeout int `mapstructure:"read_timeout"`
}
func NewHttpFetcher(cfg map[string]interface{}) (t parsing.Fetcher, err error) {
var (
config httpFetcherConfig