Exemple #1
0
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	output := &ForwardOutput{
		logger:               logger,
		codec:                &_codec,
		bind:                 bind,
		retryInterval:        retryInterval,
		connectionTimeout:    connectionTimeout,
		writeTimeout:         writeTimeout,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		spoolerShutdownChan:  make(chan struct{}),
		isShuttingDown:       0,
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	output.journal = journalGroup.GetJournal("output")
	return output, nil
}
func NewTDOutput(
	logger *logging.Logger,
	endpoint string,
	connectionTimeout time.Duration,
	writeTimeout time.Duration,
	flushInterval time.Duration,
	parallelism int,
	journalGroupPath string,
	maxJournalChunkSize int64,
	apiKey string,
	databaseName string,
	tableName string,
	tempDir string,
	useSsl bool,
	rootCAs *x509.CertPool,
	httpProxy string,
	metadata string,
) (*TDOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	router := (td_client.EndpointRouter)(nil)
	if endpoint != "" {
		router = &td_client.FixedEndpointRouter{endpoint}
	}
	httpProxy_ := (interface{})(nil)
	if httpProxy != "" {
		httpProxy_ = httpProxy
	}
	client, err := td_client.NewTDClient(td_client.Settings{
		ApiKey:            apiKey,
		Router:            router,
		ConnectionTimeout: connectionTimeout,
		// ReadTimeout: readTimeout, // TODO
		SendTimeout: writeTimeout,
		Ssl:         useSsl,
		RootCAs:     rootCAs,
		Proxy:       httpProxy_,
	})
	if err != nil {
		return nil, err
	}
	output := &TDOutput{
		logger:               logger,
		codec:                &_codec,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		isShuttingDown:       0,
		client:               client,
		databaseName:         databaseName,
		tableName:            tableName,
		tempDir:              tempDir,
		sem:                  make(chan struct{}, parallelism),
		gcChan:               make(chan *os.File, 10),
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	return output, nil
}
Exemple #3
0
func (self *OutputForward) Init(config map[string]string) error {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	self.host = "localhost"
	self.port = 8888
	self.flush_interval = 10
	self.sync_interval = 2
	self.buffer_path = "/tmp/test"
	self.buffer_queue_limit = 64 * 1024 * 1024
	self.buffer_chunk_limit = 8 * 1024 * 1024
	self.connect_timeout = 10
	self.codec = &_codec

	value := config["host"]
	if len(value) > 0 {
		self.host = value
	}

	value = config["port"]
	if len(value) > 0 {
		self.port, _ = strconv.Atoi(value)
	}

	value = config["connect_timeout"]
	if len(value) > 0 {
		self.connect_timeout, _ = strconv.Atoi(value)
	}

	value = config["flush_interval"]
	if len(value) > 0 {
		self.flush_interval, _ = strconv.Atoi(value)
	}

	value = config["sync_interval"]
	if len(value) > 0 {
		sync_interval, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.sync_interval = sync_interval
	}

	value = config["buffer_path"]
	if len(value) > 0 {
		self.buffer_path = value
	}

	value = config["buffer_queue_limit"]
	if len(value) > 0 {
		buffer_queue_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
	}

	value = config["buffer_chunk_limit"]
	if len(value) > 0 {
		buffer_chunk_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
	}
	return nil
}