Example #1
0
func newTestJSONFormatter(directives []interface{}, start string) *JSON {
	format := JSON{}
	conf := core.NewPluginConfig("format.JSON")
	conf.Stream = []string{core.LogInternalStream}

	conf.Settings["JSONStartState"] = start
	conf.Settings["JSONDirectives"] = directives

	if err := format.Configure(conf); err != nil {
		panic(err)
	}
	return &format
}
Example #2
0
func TestConsumerInterface(t *testing.T) {
	conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
	consumers := shared.TypeRegistry.GetRegistered("consumer.")

	if len(consumers) == 0 {
		t.Error("No consumers defined")
	}

	for _, name := range consumers {
		_, err := core.NewPluginWithType(name, conf)
		if err != nil {
			t.Errorf("Failed to create consumer %s: %s", name, err.Error())
		}
	}
}
Example #3
0
func TestFormatters(t *testing.T) {
	conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
	formatters := shared.TypeRegistry.GetRegistered("format.")

	if len(formatters) == 0 {
		t.Error("No formatters defined")
	}

	for _, name := range formatters {
		_, err := core.NewPluginWithType(name, conf)
		if err != nil {
			t.Errorf("Failed to create formatter %s: %s", name, err.Error())
		}
	}
}
Example #4
0
func TestStreamInterface(t *testing.T) {
	conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
	conf.Stream = []string{"test"}
	streams := shared.TypeRegistry.GetRegistered("stream.")

	if len(streams) == 0 {
		t.Error("No streams defined")
	}

	for _, name := range streams {
		_, err := core.NewPluginWithType(name, conf)
		if err != nil {
			t.Errorf("Failed to create stream %s: %s", name, err.Error())
		}
	}
}
Example #5
0
func TestFormatters(t *testing.T) {
	conf := core.NewPluginConfig(reflect.TypeOf(t).Name())
	formatters := shared.RuntimeType.GetRegistered("format.")

	if len(formatters) == 0 {
		t.Error("No formatters defined")
	}

	for _, name := range formatters {
		plugin, err := core.NewPluginWithType(name, conf)
		if err != nil {
			t.Errorf("Failed to create formatter %s: %s", name, err.Error())
		} else {
			if !testFormatter(plugin.(core.Formatter)) {
				t.Errorf("Formatter %s tests failed", name)
			}
		}
	}
}
Example #6
0
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
	// Make sure the log is printed to stdout if we are stuck here
	logFallback := time.AfterFunc(time.Duration(3)*time.Second, func() {
		Log.SetWriter(os.Stdout)
	})
	defer logFallback.Stop()

	// Configure the multiplexer, create a byte pool and assign it to the log
	shared.Metric.New(metricCons)
	shared.Metric.New(metricProds)
	shared.Metric.New(metricMessages)
	shared.Metric.New(metricDropped)
	shared.Metric.New(metricDiscarded)
	shared.Metric.New(metricNoRoute)
	shared.Metric.New(metricFiltered)
	shared.Metric.New(metricMessagesSec)
	shared.Metric.New(metricDroppedSec)
	shared.Metric.New(metricDiscardedSec)
	shared.Metric.New(metricNoRouteSec)
	shared.Metric.New(metricFilteredSec)
	shared.Metric.New(metricBlockedProducers)
	shared.Metric.New(metricVersion)

	shared.Metric.Set(metricVersion, gollumMajorVer*10000+gollumMinorVer*100+gollumPatchVer)

	plex := multiplexer{
		consumers:      []core.Consumer{new(core.LogConsumer)},
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		shutdownOrder:  list.New(),
		profile:        profile,
		state:          multiplexerStateConfigure,
	}

	// Sort the plugins by interface type.

	var consumerConfig, producerConfig, streamConfig []core.PluginConfig

	for _, config := range conf.Plugins {
		if !config.Enable {
			continue // ### continue, disabled ###
		}

		Log.Debug.Print("Loading ", config.Typename)

		pluginType := shared.TypeRegistry.GetTypeOf(config.Typename)
		if pluginType == nil {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
			continue // ### continue ###
		}

		validPlugin := false
		if pluginType.Implements(consumerInterface) {
			consumerConfig = append(consumerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(producerInterface) {
			producerConfig = append(producerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(streamInterface) {
			streamConfig = append(streamConfig, config)
			validPlugin = true
		}

		if !validPlugin {
			dumpFaultyPlugin(config.Typename, pluginType)
		}
	}

	// Initialize the plugins in the order of stream, producer, consumer to
	// match the order of reference between the different types.

	for _, config := range streamConfig {
		if len(config.Stream) == 0 {
			Log.Error.Printf("Stream plugin %s has no streams set", config.Typename)
			continue // ### continue ###
		}

		streamName := config.Stream[0]
		if len(config.Stream) > 1 {
			Log.Warning.Printf("Stream plugins may only bind to one stream. Plugin will bind to %s", streamName)
		}

		plugin, err := core.NewPlugin(config)
		if err != nil {
			Log.Error.Printf("Failed to configure stream %s: %s", streamName, err)
			continue // ### continue ###
		}

		Log.Debug.Print("Configuring ", config.Typename, " for ", streamName)
		core.StreamRegistry.Register(plugin.(core.Stream), core.GetStreamID(streamName))
	}

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all streams to all streams that are used.

	wildcardStream := core.StreamRegistry.GetStreamOrFallback(core.WildcardStreamID)

	for _, config := range producerConfig {
		for i := 0; i < config.Instances; i++ {
			Log.Debug.Print("Configuring ", config.Typename)
			plugin, err := core.NewPlugin(config)

			if err != nil {
				Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			producer, _ := plugin.(core.Producer)
			streams := producer.Streams()

			if len(streams) == 0 {
				Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
				continue // ### continue ###
			}

			for _, streamID := range streams {
				if streamID == core.WildcardStreamID {
					core.StreamRegistry.RegisterWildcardProducer(producer)
				} else {
					stream := core.StreamRegistry.GetStreamOrFallback(streamID)
					stream.AddProducer(producer)
				}
			}

			// Do not add internal streams to wildcard stream

			for _, streamID := range streams {
				if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
					wildcardStream.AddProducer(producer)
					break
				}
			}

			plex.producers = append(plex.producers, producer)
			shared.Metric.Inc(metricProds)
		}
	}

	// Register dependencies by going over each producer and registering it to
	// all producers listening to its DropStream

	for _, prod := range plex.producers {
		core.StreamRegistry.LinkDependencies(prod, prod.GetDropStreamID())
	}

	// Consumers are registered last so that the stream reference list can be
	// built. This eliminates lookups when sending to specific streams.

	logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
	logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))

	for _, config := range consumerConfig {
		for i := 0; i < config.Instances; i++ {
			Log.Debug.Print("Configuring ", config.Typename)
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			consumer, _ := plugin.(core.Consumer)
			plex.consumers = append(plex.consumers, consumer)
			shared.Metric.Inc(metricCons)
		}
	}

	// As consumers might create new fallback streams this is the first position
	// where we can add the wildcard producers to all streams. No new streams
	// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.

	core.StreamRegistry.ForEachStream(
		func(streamID core.MessageStreamID, stream core.Stream) {
			core.StreamRegistry.AddWildcardProducersToStream(stream)
		})

	return plex
}
Example #7
0
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
	// Configure the multiplexer, create a byte pool and assign it to the log

	shared.Metric.New(metricMsgSec)
	shared.Metric.New(metricMsgSecAvg)
	shared.Metric.New(metricCons)
	shared.Metric.New(metricProds)
	shared.Metric.New(metricMessages)

	plex := multiplexer{
		consumers:      []core.Consumer{new(core.LogConsumer)},
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		profile:        profile,
		state:          multiplexerStateConfigure,
	}

	// Sort the plugins by interface type.

	var consumerConfig, producerConfig, streamConfig []core.PluginConfig
	consumerInterface := reflect.TypeOf((*core.Consumer)(nil)).Elem()
	producerInterface := reflect.TypeOf((*core.Producer)(nil)).Elem()
	streamInterface := reflect.TypeOf((*core.Stream)(nil)).Elem()

	for _, config := range conf.Plugins {
		if !config.Enable {
			continue // ### continue, disabled ###
		}

		pluginType := shared.RuntimeType.GetTypeOf(config.Typename)
		if pluginType == nil {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
			continue // ### continue ###
		}

		validPlugin := false
		if pluginType.Implements(consumerInterface) {
			consumerConfig = append(consumerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(producerInterface) {
			producerConfig = append(producerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(streamInterface) {
			streamConfig = append(streamConfig, config)
			validPlugin = true
		}

		if !validPlugin {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Does not qualify for consumer, producer or stream interface")

			consumerMatch, consumerMissing := shared.GetMissingMethods(pluginType, consumerInterface)
			producerMatch, producerMissing := shared.GetMissingMethods(pluginType, producerInterface)
			streamMatch, streamMissing := shared.GetMissingMethods(pluginType, streamInterface)

			if consumerMatch > producerMatch {
				if consumerMatch > streamMatch {
					Log.Error.Print("Plugin looks like a consumer:")
					for _, message := range consumerMissing {
						Log.Error.Print(message)
					}
				} else {
					Log.Error.Print("Plugin looks like a stream:")
					for _, message := range streamMissing {
						Log.Error.Print(message)
					}
				}
			} else if producerMatch > streamMatch {
				Log.Error.Print("Plugin looks like a producer:")
				for _, message := range producerMissing {
					Log.Error.Print(message)
				}
			} else {
				Log.Error.Print("Plugin looks like a stream:")
				for _, message := range streamMissing {
					Log.Error.Print(message)
				}
			}
		}
	}

	// Initialize the plugins in the order of stream, producer, consumer to
	// match the order of reference between the different types.

	for _, config := range streamConfig {
		for _, streamName := range config.Stream {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure stream plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}
			core.StreamTypes.Register(plugin.(core.Stream), core.GetStreamID(streamName))
		}
	}

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all streams to all streams that are used.

	wildcardStream := core.StreamTypes.GetStreamOrFallback(core.WildcardStreamID)

	for _, config := range producerConfig {
		for i := 0; i < config.Instances; i++ {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			producer, _ := plugin.(core.Producer)
			streams := producer.Streams()

			if len(streams) == 0 {
				Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
				continue // ### continue ###
			}

			for _, streamID := range streams {
				if streamID == core.WildcardStreamID {
					core.StreamTypes.RegisterWildcardProducer(producer)
				} else {
					stream := core.StreamTypes.GetStreamOrFallback(streamID)
					stream.AddProducer(producer)
				}
			}

			// Do not add internal streams to wildcard stream

			for _, streamID := range streams {
				if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
					wildcardStream.AddProducer(producer)
					break
				}
			}

			plex.producers = append(plex.producers, producer)
			shared.Metric.Inc(metricProds)
		}
	}

	// Consumers are registered last so that the stream reference list can be
	// built. This eliminates lookups when sending to specific streams.

	logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
	logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))

	for _, config := range consumerConfig {
		for i := 0; i < config.Instances; i++ {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			consumer, _ := plugin.(core.Consumer)
			plex.consumers = append(plex.consumers, consumer)
			shared.Metric.Inc(metricCons)
		}
	}

	// As consumers might create new fallback streams this is the first position
	// where we can add the wildcard producers to all streams. No new streams
	// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.

	core.StreamTypes.ForEachStream(
		func(streamID core.MessageStreamID, stream core.Stream) {
			switch streamID {
			case core.LogInternalStreamID, core.WildcardStreamID, core.DroppedStreamID:
				// Internal streams are excluded for wildcard listeners
			default:
				core.StreamTypes.AddWildcardProducersToStream(stream)
			}
		})

	return plex
}