func newTestJSONFormatter(directives []interface{}, start string) *JSON {
format := JSON{}
conf := core.NewPluginConfig("format.JSON")
conf.Stream = []string{core.LogInternalStream}
conf.Settings["JSONStartState"] = start
conf.Settings["JSONDirectives"] = directives
if err := format.Configure(conf); err != nil {
panic(err)
}
return &format
}
func TestConsumerInterface(t *testing.T) {
conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
consumers := shared.TypeRegistry.GetRegistered("consumer.")
if len(consumers) == 0 {
t.Error("No consumers defined")
}
for _, name := range consumers {
_, err := core.NewPluginWithType(name, conf)
if err != nil {
t.Errorf("Failed to create consumer %s: %s", name, err.Error())
}
}
}
func TestFormatters(t *testing.T) {
conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
formatters := shared.TypeRegistry.GetRegistered("format.")
if len(formatters) == 0 {
t.Error("No formatters defined")
}
for _, name := range formatters {
_, err := core.NewPluginWithType(name, conf)
if err != nil {
t.Errorf("Failed to create formatter %s: %s", name, err.Error())
}
}
}
func TestStreamInterface(t *testing.T) {
conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
conf.Stream = []string{"test"}
streams := shared.TypeRegistry.GetRegistered("stream.")
if len(streams) == 0 {
t.Error("No streams defined")
}
for _, name := range streams {
_, err := core.NewPluginWithType(name, conf)
if err != nil {
t.Errorf("Failed to create stream %s: %s", name, err.Error())
}
}
}
func TestFormatters(t *testing.T) {
conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
formatters := shared.RuntimeType.GetRegistered("format.")
if len(formatters) == 0 {
t.Error("No formatters defined")
}
for _, name := range formatters {
plugin, err := core.NewPluginWithType(name, conf)
if err != nil {
t.Errorf("Failed to create formatter %s: %s", name, err.Error())
} else {
if !testFormatter(plugin.(core.Formatter)) {
t.Errorf("Formatter %s tests failed", name)
}
}
}
}
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
// Make sure the log is printed to stdout if we are stuck here
logFallback := time.AfterFunc(time.Duration(3)*time.Second, func() {
Log.SetWriter(os.Stdout)
})
defer logFallback.Stop()
// Configure the multiplexer, create a byte pool and assign it to the log
shared.Metric.New(metricCons)
shared.Metric.New(metricProds)
shared.Metric.New(metricMessages)
shared.Metric.New(metricDropped)
shared.Metric.New(metricDiscarded)
shared.Metric.New(metricNoRoute)
shared.Metric.New(metricFiltered)
shared.Metric.New(metricMessagesSec)
shared.Metric.New(metricDroppedSec)
shared.Metric.New(metricDiscardedSec)
shared.Metric.New(metricNoRouteSec)
shared.Metric.New(metricFilteredSec)
shared.Metric.New(metricBlockedProducers)
shared.Metric.New(metricVersion)
shared.Metric.Set(metricVersion, gollumMajorVer*10000+gollumMinorVer*100+gollumPatchVer)
plex := multiplexer{
consumers: []core.Consumer{new(core.LogConsumer)},
consumerWorker: new(sync.WaitGroup),
producerWorker: new(sync.WaitGroup),
shutdownOrder: list.New(),
profile: profile,
state: multiplexerStateConfigure,
}
// Sort the plugins by interface type.
var consumerConfig, producerConfig, streamConfig []core.PluginConfig
for _, config := range conf.Plugins {
if !config.Enable {
continue // ### continue, disabled ###
}
Log.Debug.Print("Loading ", config.Typename)
pluginType := shared.TypeRegistry.GetTypeOf(config.Typename)
if pluginType == nil {
Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
continue // ### continue ###
}
validPlugin := false
if pluginType.Implements(consumerInterface) {
consumerConfig = append(consumerConfig, config)
validPlugin = true
}
if pluginType.Implements(producerInterface) {
producerConfig = append(producerConfig, config)
validPlugin = true
}
if pluginType.Implements(streamInterface) {
streamConfig = append(streamConfig, config)
validPlugin = true
}
if !validPlugin {
dumpFaultyPlugin(config.Typename, pluginType)
}
}
// Initialize the plugins in the order of stream, producer, consumer to
// match the order of reference between the different types.
for _, config := range streamConfig {
if len(config.Stream) == 0 {
Log.Error.Printf("Stream plugin %s has no streams set", config.Typename)
continue // ### continue ###
}
streamName := config.Stream[0]
if len(config.Stream) > 1 {
Log.Warning.Printf("Stream plugins may only bind to one stream. Plugin will bind to %s", streamName)
}
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Printf("Failed to configure stream %s: %s", streamName, err)
continue // ### continue ###
}
Log.Debug.Print("Configuring ", config.Typename, " for ", streamName)
core.StreamRegistry.Register(plugin.(core.Stream), core.GetStreamID(streamName))
}
// All producers are added to the wildcard stream so that consumers can send
// to all producers if required. The wildcard producer list is required
// to add producers listening to all streams to all streams that are used.
wildcardStream := core.StreamRegistry.GetStreamOrFallback(core.WildcardStreamID)
for _, config := range producerConfig {
for i := 0; i < config.Instances; i++ {
Log.Debug.Print("Configuring ", config.Typename)
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
continue // ### continue ###
}
producer, _ := plugin.(core.Producer)
streams := producer.Streams()
if len(streams) == 0 {
Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
continue // ### continue ###
}
for _, streamID := range streams {
if streamID == core.WildcardStreamID {
core.StreamRegistry.RegisterWildcardProducer(producer)
} else {
stream := core.StreamRegistry.GetStreamOrFallback(streamID)
stream.AddProducer(producer)
}
}
// Do not add internal streams to wildcard stream
for _, streamID := range streams {
if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
wildcardStream.AddProducer(producer)
break
}
}
plex.producers = append(plex.producers, producer)
shared.Metric.Inc(metricProds)
}
}
// Register dependencies by going over each producer and registering it to
// all producers listening to its DropStream
for _, prod := range plex.producers {
core.StreamRegistry.LinkDependencies(prod, prod.GetDropStreamID())
}
// Consumers are registered last so that the stream reference list can be
// built. This eliminates lookups when sending to specific streams.
logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))
for _, config := range consumerConfig {
for i := 0; i < config.Instances; i++ {
Log.Debug.Print("Configuring ", config.Typename)
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
continue // ### continue ###
}
consumer, _ := plugin.(core.Consumer)
plex.consumers = append(plex.consumers, consumer)
shared.Metric.Inc(metricCons)
}
}
// As consumers might create new fallback streams this is the first position
// where we can add the wildcard producers to all streams. No new streams
// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.
core.StreamRegistry.ForEachStream(
func(streamID core.MessageStreamID, stream core.Stream) {
core.StreamRegistry.AddWildcardProducersToStream(stream)
})
return plex
}
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
// Configure the multiplexer, create a byte pool and assign it to the log
shared.Metric.New(metricMsgSec)
shared.Metric.New(metricMsgSecAvg)
shared.Metric.New(metricCons)
shared.Metric.New(metricProds)
shared.Metric.New(metricMessages)
plex := multiplexer{
consumers: []core.Consumer{new(core.LogConsumer)},
consumerWorker: new(sync.WaitGroup),
producerWorker: new(sync.WaitGroup),
profile: profile,
state: multiplexerStateConfigure,
}
// Sort the plugins by interface type.
var consumerConfig, producerConfig, streamConfig []core.PluginConfig
consumerInterface := reflect.TypeOf((*core.Consumer)(nil)).Elem()
producerInterface := reflect.TypeOf((*core.Producer)(nil)).Elem()
streamInterface := reflect.TypeOf((*core.Stream)(nil)).Elem()
for _, config := range conf.Plugins {
if !config.Enable {
continue // ### continue, disabled ###
}
pluginType := shared.RuntimeType.GetTypeOf(config.Typename)
if pluginType == nil {
Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
continue // ### continue ###
}
validPlugin := false
if pluginType.Implements(consumerInterface) {
consumerConfig = append(consumerConfig, config)
validPlugin = true
}
if pluginType.Implements(producerInterface) {
producerConfig = append(producerConfig, config)
validPlugin = true
}
if pluginType.Implements(streamInterface) {
streamConfig = append(streamConfig, config)
validPlugin = true
}
if !validPlugin {
Log.Error.Print("Failed to load plugin ", config.Typename, ": Does not qualify for consumer, producer or stream interface")
consumerMatch, consumerMissing := shared.GetMissingMethods(pluginType, consumerInterface)
producerMatch, producerMissing := shared.GetMissingMethods(pluginType, producerInterface)
streamMatch, streamMissing := shared.GetMissingMethods(pluginType, streamInterface)
if consumerMatch > producerMatch {
if consumerMatch > streamMatch {
Log.Error.Print("Plugin looks like a consumer:")
for _, message := range consumerMissing {
Log.Error.Print(message)
}
} else {
Log.Error.Print("Plugin looks like a stream:")
for _, message := range streamMissing {
Log.Error.Print(message)
}
}
} else if producerMatch > streamMatch {
Log.Error.Print("Plugin looks like a producer:")
for _, message := range producerMissing {
Log.Error.Print(message)
}
} else {
Log.Error.Print("Plugin looks like a stream:")
for _, message := range streamMissing {
Log.Error.Print(message)
}
}
}
}
// Initialize the plugins in the order of stream, producer, consumer to
// match the order of reference between the different types.
for _, config := range streamConfig {
for _, streamName := range config.Stream {
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Print("Failed to configure stream plugin ", config.Typename, ": ", err)
continue // ### continue ###
}
core.StreamTypes.Register(plugin.(core.Stream), core.GetStreamID(streamName))
}
}
// All producers are added to the wildcard stream so that consumers can send
// to all producers if required. The wildcard producer list is required
// to add producers listening to all streams to all streams that are used.
wildcardStream := core.StreamTypes.GetStreamOrFallback(core.WildcardStreamID)
for _, config := range producerConfig {
for i := 0; i < config.Instances; i++ {
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
continue // ### continue ###
}
producer, _ := plugin.(core.Producer)
streams := producer.Streams()
if len(streams) == 0 {
Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
continue // ### continue ###
}
for _, streamID := range streams {
if streamID == core.WildcardStreamID {
core.StreamTypes.RegisterWildcardProducer(producer)
} else {
stream := core.StreamTypes.GetStreamOrFallback(streamID)
stream.AddProducer(producer)
}
}
// Do not add internal streams to wildcard stream
for _, streamID := range streams {
if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
wildcardStream.AddProducer(producer)
break
}
}
plex.producers = append(plex.producers, producer)
shared.Metric.Inc(metricProds)
}
}
// Consumers are registered last so that the stream reference list can be
// built. This eliminates lookups when sending to specific streams.
logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))
for _, config := range consumerConfig {
for i := 0; i < config.Instances; i++ {
plugin, err := core.NewPlugin(config)
if err != nil {
Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
continue // ### continue ###
}
consumer, _ := plugin.(core.Consumer)
plex.consumers = append(plex.consumers, consumer)
shared.Metric.Inc(metricCons)
}
}
// As consumers might create new fallback streams this is the first position
// where we can add the wildcard producers to all streams. No new streams
// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.
core.StreamTypes.ForEachStream(
func(streamID core.MessageStreamID, stream core.Stream) {
switch streamID {
case core.LogInternalStreamID, core.WildcardStreamID, core.DroppedStreamID:
// Internal streams are excluded for wildcard listeners
default:
core.StreamTypes.AddWildcardProducersToStream(stream)
}
})
return plex
}