Example #1
0
func newStreamWithID(streamName string) streamWithID {
	streamID := core.GetStreamID(streamName)
	return streamWithID{
		id:     streamID,
		stream: core.StreamRegistry.GetStream(streamID),
	}
}
Example #2
0
// Configure initializes the basic members
func (prod *Null) Configure(conf core.PluginConfig) error {
	prod.control = make(chan core.PluginControl, 1)
	prod.streams = make([]core.MessageStreamID, len(conf.Stream))
	for i, stream := range conf.Stream {
		prod.streams[i] = core.GetStreamID(stream)
	}
	return nil
}
Example #3
0
// Format adds prefix and postfix to the message formatted by the base formatter
func (format *StreamRoute) Format(msg core.Message) ([]byte, core.MessageStreamID) {
	modMsg := msg
	prefixEnd := bytes.Index(msg.Data, format.delimiter)

	switch prefixEnd {
	case -1:
	case 0:
		modMsg.Data = msg.Data[1:]
	default:
		firstSpaceIdx := bytes.IndexByte(msg.Data, ' ')
		if (firstSpaceIdx < 0) || (firstSpaceIdx >= prefixEnd) {
			modMsg.StreamID = core.GetStreamID(string(msg.Data[:prefixEnd]))
			modMsg.Data = msg.Data[prefixEnd+1:]
		}
	}

	return format.base.Format(modMsg)
}
Example #4
0
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
	// Make sure the log is printed to stdout if we are stuck here
	logFallback := time.AfterFunc(time.Duration(3)*time.Second, func() {
		Log.SetWriter(os.Stdout)
	})
	defer logFallback.Stop()

	// Configure the multiplexer, create a byte pool and assign it to the log
	shared.Metric.New(metricCons)
	shared.Metric.New(metricProds)
	shared.Metric.New(metricMessages)
	shared.Metric.New(metricDropped)
	shared.Metric.New(metricDiscarded)
	shared.Metric.New(metricNoRoute)
	shared.Metric.New(metricFiltered)
	shared.Metric.New(metricMessagesSec)
	shared.Metric.New(metricDroppedSec)
	shared.Metric.New(metricDiscardedSec)
	shared.Metric.New(metricNoRouteSec)
	shared.Metric.New(metricFilteredSec)
	shared.Metric.New(metricBlockedProducers)
	shared.Metric.New(metricVersion)

	shared.Metric.Set(metricVersion, gollumMajorVer*10000+gollumMinorVer*100+gollumPatchVer)

	plex := multiplexer{
		consumers:      []core.Consumer{new(core.LogConsumer)},
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		shutdownOrder:  list.New(),
		profile:        profile,
		state:          multiplexerStateConfigure,
	}

	// Sort the plugins by interface type.

	var consumerConfig, producerConfig, streamConfig []core.PluginConfig

	for _, config := range conf.Plugins {
		if !config.Enable {
			continue // ### continue, disabled ###
		}

		Log.Debug.Print("Loading ", config.Typename)

		pluginType := shared.TypeRegistry.GetTypeOf(config.Typename)
		if pluginType == nil {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
			continue // ### continue ###
		}

		validPlugin := false
		if pluginType.Implements(consumerInterface) {
			consumerConfig = append(consumerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(producerInterface) {
			producerConfig = append(producerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(streamInterface) {
			streamConfig = append(streamConfig, config)
			validPlugin = true
		}

		if !validPlugin {
			dumpFaultyPlugin(config.Typename, pluginType)
		}
	}

	// Initialize the plugins in the order of stream, producer, consumer to
	// match the order of reference between the different types.

	for _, config := range streamConfig {
		if len(config.Stream) == 0 {
			Log.Error.Printf("Stream plugin %s has no streams set", config.Typename)
			continue // ### continue ###
		}

		streamName := config.Stream[0]
		if len(config.Stream) > 1 {
			Log.Warning.Printf("Stream plugins may only bind to one stream. Plugin will bind to %s", streamName)
		}

		plugin, err := core.NewPlugin(config)
		if err != nil {
			Log.Error.Printf("Failed to configure stream %s: %s", streamName, err)
			continue // ### continue ###
		}

		Log.Debug.Print("Configuring ", config.Typename, " for ", streamName)
		core.StreamRegistry.Register(plugin.(core.Stream), core.GetStreamID(streamName))
	}

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all streams to all streams that are used.

	wildcardStream := core.StreamRegistry.GetStreamOrFallback(core.WildcardStreamID)

	for _, config := range producerConfig {
		for i := 0; i < config.Instances; i++ {
			Log.Debug.Print("Configuring ", config.Typename)
			plugin, err := core.NewPlugin(config)

			if err != nil {
				Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			producer, _ := plugin.(core.Producer)
			streams := producer.Streams()

			if len(streams) == 0 {
				Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
				continue // ### continue ###
			}

			for _, streamID := range streams {
				if streamID == core.WildcardStreamID {
					core.StreamRegistry.RegisterWildcardProducer(producer)
				} else {
					stream := core.StreamRegistry.GetStreamOrFallback(streamID)
					stream.AddProducer(producer)
				}
			}

			// Do not add internal streams to wildcard stream

			for _, streamID := range streams {
				if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
					wildcardStream.AddProducer(producer)
					break
				}
			}

			plex.producers = append(plex.producers, producer)
			shared.Metric.Inc(metricProds)
		}
	}

	// Register dependencies by going over each producer and registering it to
	// all producers listening to its DropStream

	for _, prod := range plex.producers {
		core.StreamRegistry.LinkDependencies(prod, prod.GetDropStreamID())
	}

	// Consumers are registered last so that the stream reference list can be
	// built. This eliminates lookups when sending to specific streams.

	logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
	logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))

	for _, config := range consumerConfig {
		for i := 0; i < config.Instances; i++ {
			Log.Debug.Print("Configuring ", config.Typename)
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			consumer, _ := plugin.(core.Consumer)
			plex.consumers = append(plex.consumers, consumer)
			shared.Metric.Inc(metricCons)
		}
	}

	// As consumers might create new fallback streams this is the first position
	// where we can add the wildcard producers to all streams. No new streams
	// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.

	core.StreamRegistry.ForEachStream(
		func(streamID core.MessageStreamID, stream core.Stream) {
			core.StreamRegistry.AddWildcardProducersToStream(stream)
		})

	return plex
}
Example #5
0
// Create a new multiplexer based on a given config file.
func newMultiplexer(conf *core.Config, profile bool) multiplexer {
	// Configure the multiplexer, create a byte pool and assign it to the log

	shared.Metric.New(metricMsgSec)
	shared.Metric.New(metricMsgSecAvg)
	shared.Metric.New(metricCons)
	shared.Metric.New(metricProds)
	shared.Metric.New(metricMessages)

	plex := multiplexer{
		consumers:      []core.Consumer{new(core.LogConsumer)},
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		profile:        profile,
		state:          multiplexerStateConfigure,
	}

	// Sort the plugins by interface type.

	var consumerConfig, producerConfig, streamConfig []core.PluginConfig
	consumerInterface := reflect.TypeOf((*core.Consumer)(nil)).Elem()
	producerInterface := reflect.TypeOf((*core.Producer)(nil)).Elem()
	streamInterface := reflect.TypeOf((*core.Stream)(nil)).Elem()

	for _, config := range conf.Plugins {
		if !config.Enable {
			continue // ### continue, disabled ###
		}

		pluginType := shared.RuntimeType.GetTypeOf(config.Typename)
		if pluginType == nil {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Type not found")
			continue // ### continue ###
		}

		validPlugin := false
		if pluginType.Implements(consumerInterface) {
			consumerConfig = append(consumerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(producerInterface) {
			producerConfig = append(producerConfig, config)
			validPlugin = true
		}
		if pluginType.Implements(streamInterface) {
			streamConfig = append(streamConfig, config)
			validPlugin = true
		}

		if !validPlugin {
			Log.Error.Print("Failed to load plugin ", config.Typename, ": Does not qualify for consumer, producer or stream interface")

			consumerMatch, consumerMissing := shared.GetMissingMethods(pluginType, consumerInterface)
			producerMatch, producerMissing := shared.GetMissingMethods(pluginType, producerInterface)
			streamMatch, streamMissing := shared.GetMissingMethods(pluginType, streamInterface)

			if consumerMatch > producerMatch {
				if consumerMatch > streamMatch {
					Log.Error.Print("Plugin looks like a consumer:")
					for _, message := range consumerMissing {
						Log.Error.Print(message)
					}
				} else {
					Log.Error.Print("Plugin looks like a stream:")
					for _, message := range streamMissing {
						Log.Error.Print(message)
					}
				}
			} else if producerMatch > streamMatch {
				Log.Error.Print("Plugin looks like a producer:")
				for _, message := range producerMissing {
					Log.Error.Print(message)
				}
			} else {
				Log.Error.Print("Plugin looks like a stream:")
				for _, message := range streamMissing {
					Log.Error.Print(message)
				}
			}
		}
	}

	// Initialize the plugins in the order of stream, producer, consumer to
	// match the order of reference between the different types.

	for _, config := range streamConfig {
		for _, streamName := range config.Stream {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure stream plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}
			core.StreamTypes.Register(plugin.(core.Stream), core.GetStreamID(streamName))
		}
	}

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all streams to all streams that are used.

	wildcardStream := core.StreamTypes.GetStreamOrFallback(core.WildcardStreamID)

	for _, config := range producerConfig {
		for i := 0; i < config.Instances; i++ {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure producer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			producer, _ := plugin.(core.Producer)
			streams := producer.Streams()

			if len(streams) == 0 {
				Log.Error.Print("Producer plugin ", config.Typename, " has no streams set")
				continue // ### continue ###
			}

			for _, streamID := range streams {
				if streamID == core.WildcardStreamID {
					core.StreamTypes.RegisterWildcardProducer(producer)
				} else {
					stream := core.StreamTypes.GetStreamOrFallback(streamID)
					stream.AddProducer(producer)
				}
			}

			// Do not add internal streams to wildcard stream

			for _, streamID := range streams {
				if streamID != core.LogInternalStreamID && streamID != core.DroppedStreamID {
					wildcardStream.AddProducer(producer)
					break
				}
			}

			plex.producers = append(plex.producers, producer)
			shared.Metric.Inc(metricProds)
		}
	}

	// Consumers are registered last so that the stream reference list can be
	// built. This eliminates lookups when sending to specific streams.

	logConsumer, _ := plex.consumers[0].(*core.LogConsumer)
	logConsumer.Configure(core.NewPluginConfig("core.LogConsumer"))

	for _, config := range consumerConfig {
		for i := 0; i < config.Instances; i++ {
			plugin, err := core.NewPlugin(config)
			if err != nil {
				Log.Error.Print("Failed to configure consumer plugin ", config.Typename, ": ", err)
				continue // ### continue ###
			}

			consumer, _ := plugin.(core.Consumer)
			plex.consumers = append(plex.consumers, consumer)
			shared.Metric.Inc(metricCons)
		}
	}

	// As consumers might create new fallback streams this is the first position
	// where we can add the wildcard producers to all streams. No new streams
	// created beyond this point must use StreamRegistry.AddWildcardProducersToStream.

	core.StreamTypes.ForEachStream(
		func(streamID core.MessageStreamID, stream core.Stream) {
			switch streamID {
			case core.LogInternalStreamID, core.WildcardStreamID, core.DroppedStreamID:
				// Internal streams are excluded for wildcard listeners
			default:
				core.StreamTypes.AddWildcardProducersToStream(stream)
			}
		})

	return plex
}