Exemple #1
0
func NewForwardInput(logger *logging.Logger, bind string, port Port) (*ForwardInput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	addr, err := net.ResolveTCPAddr("tcp", bind)
	if err != nil {
		logger.Error("%s", err.Error())
		return nil, err
	}
	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		logger.Error("%s", err.Error())
		return nil, err
	}
	return &ForwardInput{
		port:           port,
		logger:         logger,
		bind:           bind,
		listener:       listener,
		codec:          &_codec,
		clients:        make(map[*net.TCPConn]*forwardClient),
		clientsMtx:     sync.Mutex{},
		entries:        0,
		wg:             sync.WaitGroup{},
		acceptChan:     make(chan *net.TCPConn),
		shutdownChan:   make(chan struct{}),
		isShuttingDown: uintptr(0),
	}, nil
}
Exemple #2
0
func WriteFile(path string, obj interface{}) (err error) {
	mh := codec.MsgpackHandle{}
	mh.AsSymbols = codec.AsSymbolNone
	mh.RawToString = true
	buf := &bytes.Buffer{}
	encoder := codec.NewEncoder(buf, &mh)
	err = encoder.Encode(obj)
	if err != nil {
		return
	}
	err = ioutil.WriteFile(path, buf.Bytes(), os.FileMode(0777))
	return
}
Exemple #3
0
func newForwardInput(factory *ForwardInputFactory, logger *log.Logger, bind string, port ik.Port) (*ForwardInput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	listener, err := net.Listen("tcp", bind)
	if err != nil {
		logger.Fatal(err.Error())
		return nil, err
	}
	return &ForwardInput{
		factory:  factory,
		port:     port,
		logger:   logger,
		bind:     bind,
		listener: listener,
		codec:    &_codec,
		running:  false,
		clients:  make(map[net.Conn]*forwardClient),
	}, nil
}
Exemple #4
0
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	output := &ForwardOutput{
		logger:               logger,
		codec:                &_codec,
		bind:                 bind,
		retryInterval:        retryInterval,
		connectionTimeout:    connectionTimeout,
		writeTimeout:         writeTimeout,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		spoolerShutdownChan:  make(chan struct{}),
		isShuttingDown:       0,
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	output.journal = journalGroup.GetJournal("output")
	return output, nil
}
func NewTDOutput(
	logger *logging.Logger,
	endpoint string,
	connectionTimeout time.Duration,
	writeTimeout time.Duration,
	flushInterval time.Duration,
	parallelism int,
	journalGroupPath string,
	maxJournalChunkSize int64,
	apiKey string,
	databaseName string,
	tableName string,
	tempDir string,
	useSsl bool,
	rootCAs *x509.CertPool,
	httpProxy string,
	metadata string,
) (*TDOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	router := (td_client.EndpointRouter)(nil)
	if endpoint != "" {
		router = &td_client.FixedEndpointRouter{endpoint}
	}
	httpProxy_ := (interface{})(nil)
	if httpProxy != "" {
		httpProxy_ = httpProxy
	}
	client, err := td_client.NewTDClient(td_client.Settings{
		ApiKey:            apiKey,
		Router:            router,
		ConnectionTimeout: connectionTimeout,
		// ReadTimeout: readTimeout, // TODO
		SendTimeout: writeTimeout,
		Ssl:         useSsl,
		RootCAs:     rootCAs,
		Proxy:       httpProxy_,
	})
	if err != nil {
		return nil, err
	}
	output := &TDOutput{
		logger:               logger,
		codec:                &_codec,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		isShuttingDown:       0,
		client:               client,
		databaseName:         databaseName,
		tableName:            tableName,
		tempDir:              tempDir,
		sem:                  make(chan struct{}, parallelism),
		gcChan:               make(chan *os.File, 10),
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	return output, nil
}
Exemple #6
0
func (self *OutputForward) Init(config map[string]string) error {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	self.host = "localhost"
	self.port = 8888
	self.flush_interval = 10
	self.sync_interval = 2
	self.buffer_path = "/tmp/test"
	self.buffer_queue_limit = 64 * 1024 * 1024
	self.buffer_chunk_limit = 8 * 1024 * 1024
	self.connect_timeout = 10
	self.codec = &_codec

	value := config["host"]
	if len(value) > 0 {
		self.host = value
	}

	value = config["port"]
	if len(value) > 0 {
		self.port, _ = strconv.Atoi(value)
	}

	value = config["connect_timeout"]
	if len(value) > 0 {
		self.connect_timeout, _ = strconv.Atoi(value)
	}

	value = config["flush_interval"]
	if len(value) > 0 {
		self.flush_interval, _ = strconv.Atoi(value)
	}

	value = config["sync_interval"]
	if len(value) > 0 {
		sync_interval, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.sync_interval = sync_interval
	}

	value = config["buffer_path"]
	if len(value) > 0 {
		self.buffer_path = value
	}

	value = config["buffer_queue_limit"]
	if len(value) > 0 {
		buffer_queue_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
	}

	value = config["buffer_chunk_limit"]
	if len(value) > 0 {
		buffer_chunk_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
	}
	return nil
}