func NewForwardInput(logger *logging.Logger, bind string, port Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
addr, err := net.ResolveTCPAddr("tcp", bind)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
listener, err := net.ListenTCP("tcp", addr)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
return &ForwardInput{
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
clients: make(map[*net.TCPConn]*forwardClient),
clientsMtx: sync.Mutex{},
entries: 0,
wg: sync.WaitGroup{},
acceptChan: make(chan *net.TCPConn),
shutdownChan: make(chan struct{}),
isShuttingDown: uintptr(0),
}, nil
}
func WriteFile(path string, obj interface{}) (err error) {
mh := codec.MsgpackHandle{}
mh.AsSymbols = codec.AsSymbolNone
mh.RawToString = true
buf := &bytes.Buffer{}
encoder := codec.NewEncoder(buf, &mh)
err = encoder.Encode(obj)
if err != nil {
return
}
err = ioutil.WriteFile(path, buf.Bytes(), os.FileMode(0777))
return
}
func newForwardInput(factory *ForwardInputFactory, logger *log.Logger, bind string, port ik.Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
listener, err := net.Listen("tcp", bind)
if err != nil {
logger.Fatal(err.Error())
return nil, err
}
return &ForwardInput{
factory: factory,
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
running: false,
clients: make(map[net.Conn]*forwardClient),
}, nil
}
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
output := &ForwardOutput{
logger: logger,
codec: &_codec,
bind: bind,
retryInterval: retryInterval,
connectionTimeout: connectionTimeout,
writeTimeout: writeTimeout,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
spoolerShutdownChan: make(chan struct{}),
isShuttingDown: 0,
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
output.journal = journalGroup.GetJournal("output")
return output, nil
}
func NewTDOutput(
logger *logging.Logger,
endpoint string,
connectionTimeout time.Duration,
writeTimeout time.Duration,
flushInterval time.Duration,
parallelism int,
journalGroupPath string,
maxJournalChunkSize int64,
apiKey string,
databaseName string,
tableName string,
tempDir string,
useSsl bool,
rootCAs *x509.CertPool,
httpProxy string,
metadata string,
) (*TDOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
router := (td_client.EndpointRouter)(nil)
if endpoint != "" {
router = &td_client.FixedEndpointRouter{endpoint}
}
httpProxy_ := (interface{})(nil)
if httpProxy != "" {
httpProxy_ = httpProxy
}
client, err := td_client.NewTDClient(td_client.Settings{
ApiKey: apiKey,
Router: router,
ConnectionTimeout: connectionTimeout,
// ReadTimeout: readTimeout, // TODO
SendTimeout: writeTimeout,
Ssl: useSsl,
RootCAs: rootCAs,
Proxy: httpProxy_,
})
if err != nil {
return nil, err
}
output := &TDOutput{
logger: logger,
codec: &_codec,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
isShuttingDown: 0,
client: client,
databaseName: databaseName,
tableName: tableName,
tempDir: tempDir,
sem: make(chan struct{}, parallelism),
gcChan: make(chan *os.File, 10),
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
return output, nil
}
func (self *OutputForward) Init(config map[string]string) error {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
self.host = "localhost"
self.port = 8888
self.flush_interval = 10
self.sync_interval = 2
self.buffer_path = "/tmp/test"
self.buffer_queue_limit = 64 * 1024 * 1024
self.buffer_chunk_limit = 8 * 1024 * 1024
self.connect_timeout = 10
self.codec = &_codec
value := config["host"]
if len(value) > 0 {
self.host = value
}
value = config["port"]
if len(value) > 0 {
self.port, _ = strconv.Atoi(value)
}
value = config["connect_timeout"]
if len(value) > 0 {
self.connect_timeout, _ = strconv.Atoi(value)
}
value = config["flush_interval"]
if len(value) > 0 {
self.flush_interval, _ = strconv.Atoi(value)
}
value = config["sync_interval"]
if len(value) > 0 {
sync_interval, err := strconv.Atoi(value)
if err != nil {
return err
}
self.sync_interval = sync_interval
}
value = config["buffer_path"]
if len(value) > 0 {
self.buffer_path = value
}
value = config["buffer_queue_limit"]
if len(value) > 0 {
buffer_queue_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
}
value = config["buffer_chunk_limit"]
if len(value) > 0 {
buffer_chunk_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
}
return nil
}