Exemplo n.º 1
0
// calculate fetches the information ID associated with the given information
// sequence. In case the information sequence is not found within the underlying
// storage, a new information ID is generated and used to store the given
// information sequence. In any case the information ID is added to the given
// context.
func (c *clg) calculate(ctx spec.Context, informationSequence string) error {
	informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", informationSequence)
	informationID, err := c.Storage().General().Get(informationIDKey)
	if storage.IsNotFound(err) {
		// The given information sequence was never seen before. Thus we register it
		// now with its own very unique information ID.
		newID, err := c.Service().ID().New()
		if err != nil {
			return maskAny(err)
		}
		informationID = string(newID)

		err = c.Storage().General().Set(informationIDKey, informationID)
		if err != nil {
			return maskAny(err)
		}

		informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
		err = c.Storage().General().Set(informationSequenceKey, informationSequence)
		if err != nil {
			return maskAny(err)
		}
	} else if err != nil {
		return maskAny(err)
	}

	ctx.SetInformationID(informationID)

	return nil
}
Exemplo n.º 2
0
func Test_CLG_Input_SetInformationSequenceError(t *testing.T) {
	newCLG := MustNew()
	newCtx := context.MustNew()
	newServiceCollection := testMustNewServiceCollection(t)

	// Prepare the storage connection to fake a returned error.
	newInput := "test input"
	informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
	// Our test ID factory always returns the same ID. That way we are able to
	// check for the ID being used during the test.
	newID, err := newServiceCollection.ID().New()
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}
	informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID)

	c := redigomock.NewConn()
	c.Command("GET", "prefix:"+informationIDKey).ExpectError(redigo.ErrNil)
	c.Command("SET", "prefix:"+informationIDKey, string(newID)).Expect("OK")
	c.Command("SET", "prefix:"+informationSequenceKey, newInput).ExpectError(invalidConfigError)
	newStorageCollection := testMustNewStorageCollectionWithConn(t, c)

	// Set prepared storage to CLG we want to test.
	newCLG.(*clg).StorageCollection = newStorageCollection
	newCLG.(*clg).ServiceCollection = newServiceCollection

	// Execute CLG.
	err = newCLG.(*clg).calculate(newCtx, newInput)
	if !IsInvalidConfig(err) {
		t.Fatal("expected", true, "got", false)
	}
}
Exemplo n.º 3
0
func Test_CLG_Input_KnownInputSequence(t *testing.T) {
	newCLG := MustNew()
	newCtx := context.MustNew()
	newStorageCollection := testMustNewStorageCollection(t)

	// Create record for the test input.
	informationID := "123"
	newInput := "test input"
	informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
	err := newStorageCollection.General().Set(informationIDKey, informationID)
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}

	// Set prepared storage to CLG we want to test.
	newCLG.(*clg).StorageCollection = newStorageCollection

	// Execute CLG.
	err = newCLG.(*clg).calculate(newCtx, newInput)
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}

	// Check if the information ID was set to the context.
	injectedInformationID, _ := newCtx.GetInformationID()
	if informationID != injectedInformationID {
		t.Fatal("expected", informationID, "got", injectedInformationID)
	}
}
Exemplo n.º 4
0
func (t *tracker) CLGIDs(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
	destinationID := string(networkPayload.GetDestination())
	sourceIDs := networkPayload.GetSources()

	errors := make(chan error, len(sourceIDs))
	wg := sync.WaitGroup{}

	for _, s := range sourceIDs {
		wg.Add(1)
		go func(s string) {
			// Persist the single CLG ID connections.
			behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:o:tracker:behaviour-ids", s)
			err := t.Storage().General().PushToSet(behaviourIDKey, destinationID)
			if err != nil {
				errors <- maskAny(err)
			}
			wg.Done()
		}(string(s))
	}

	wg.Wait()

	select {
	case err := <-errors:
		if err != nil {
			return maskAny(err)
		}
	default:
		// Nothing do here. No error occurred. All good.
	}

	return nil
}
Exemplo n.º 5
0
func Test_CLG_Input_DataProperlyStored(t *testing.T) {
	newCLG := MustNew()
	newCtx := context.MustNew()
	newStorageCollection := testMustNewStorageCollection(t)
	newServiceCollection := testMustNewServiceCollection(t)

	// Note we do not create a record for the test input. This test is about an
	// unknown input sequence.
	newInput := "test input"
	// Our test ID factory always returns the same ID. That way we are able to
	// check for the ID being used during the test.
	newID, err := newServiceCollection.ID().New()
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}

	// Set prepared storage to CLG we want to test.
	newCLG.(*clg).ServiceCollection = newServiceCollection
	newCLG.(*clg).StorageCollection = newStorageCollection

	// Execute CLG.
	err = newCLG.(*clg).calculate(newCtx, newInput)
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}

	informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
	storedID, err := newStorageCollection.General().Get(informationIDKey)
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}
	if storedID != string(newID) {
		t.Fatal("expected", newID, "got", storedID)
	}

	informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID)
	storedInput, err := newStorageCollection.General().Get(informationSequenceKey)
	if err != nil {
		t.Fatal("expected", nil, "got", err)
	}
	if storedInput != newInput {
		t.Fatal("expected", newInput, "got", storedInput)
	}
}
Exemplo n.º 6
0
func (c *clg) calculate(ctx spec.Context) (string, error) {
	behaviourID, ok := ctx.GetBehaviourID()
	if !ok {
		return "", maskAnyf(invalidBehaviourIDError, "must not be empty")
	}

	behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:separator", behaviourID)
	separator, err := c.Storage().General().Get(behaviourIDKey)
	if storage.IsNotFound(err) {
		randomKey, err := c.Storage().Feature().GetRandom()
		if err != nil {
			return "", maskAny(err)
		}

		// Make sure the fetched feature is valid based on its key structure. The
		// key itself already contains the feature. Knowing the key structure makes
		// it possible to simply parse the feature from the key fetched from the
		// feature storage. These features are stored by the split-features CLG.
		// Changes in the key structure there must be aligned with implementation
		// details here. The current key structure looks as follows.
		//
		//     feature:%s:positions
		//
		if len(randomKey) != 22 {
			return "", maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", randomKey)
		}
		if !strings.HasPrefix(randomKey, "feature:") {
			return "", maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", randomKey)
		}
		if !strings.HasSuffix(randomKey, ":positions") {
			return "", maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", randomKey)
		}

		// Create a new separator from the fetched random feature. Note that a
		// feature is considered 4 characters long and the random factory takes a
		// max parameter, which is exlusive.
		feature := randomKey[8:12]
		featureIndex, err := c.Service().Random().CreateMax(5)
		if err != nil {
			return "", maskAny(err)
		}
		separator = string(feature[featureIndex])

		// Store the newly created separator using the CLGs own behaviour ID. In case
		// this CLG is asked again to return its separator, it will lookup its
		// separator in the general storage.
		err = c.Storage().General().Set(behaviourIDKey, separator)
		if err != nil {
			return "", maskAny(err)
		}
	} else if err != nil {
		return "", maskAny(err)
	}

	return separator, nil
}
Exemplo n.º 7
0
func (t *tracker) CLGNames(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
	destinationName := CLG.GetName()
	sourceIDs := networkPayload.GetSources()

	errors := make(chan error, len(sourceIDs))
	wg := sync.WaitGroup{}

	for _, s := range sourceIDs {
		wg.Add(1)
		go func(s string) {
			behaviourNameKey := key.NewNetworkKey("behaviour-id:%s:behaviour-name", s)
			name, err := t.Storage().General().Get(behaviourNameKey)
			if err != nil {
				errors <- maskAny(err)
			} else {
				// The errors channel is capable of buffering one error for each source
				// ID. The else clause is necessary to queue only one possible error for
				// each source ID. So in case the name lookup was successful, we are
				// able to actually persist the single CLG name connection.
				behaviourNameKey := key.NewNetworkKey("behaviour-name:%s:o:tracker:behaviour-names", name)
				err := t.Storage().General().PushToSet(behaviourNameKey, destinationName)
				if err != nil {
					errors <- maskAny(err)
				}
			}

			wg.Done()
		}(string(s))
	}

	wg.Wait()

	select {
	case err := <-errors:
		if err != nil {
			return maskAny(err)
		}
	default:
		// Nothing do here. No error occurred. All good.
	}

	return nil
}
Exemplo n.º 8
0
func (n *network) EventListener(canceler <-chan struct{}) error {
	invokeEventHandler := func() error {
		// Fetch the next network payload from the queue. This call blocks until one
		// network payload was fetched from the queue. As soon as we receive the
		// network payload, it is removed from the queue automatically, so it is not
		// handled twice.
		eventKey := key.NewNetworkKey("event:network-payload")
		element, err := n.Storage().General().PopFromList(eventKey)
		if err != nil {
			return maskAny(err)
		}
		networkPayload := networkpayload.MustNew()
		err = json.Unmarshal([]byte(element), &networkPayload)
		if err != nil {
			return maskAny(err)
		}

		// Lookup the CLG that is supposed to be executed. The CLG object is
		// referenced by name. When being executed it is referenced by its behaviour
		// ID. The behaviour ID represents a specific peer within a connection path.
		clgName, ok := networkPayload.GetContext().GetCLGName()
		if !ok {
			return maskAnyf(invalidCLGNameError, "must not be empty")
		}
		CLG, ok := n.CLGs[clgName]
		if !ok {
			return maskAnyf(clgNotFoundError, "name: %s", clgName)
		}

		// Invoke the event handler to execute the given CLG using the given network
		// payload. Here we execute one distinct behaviour within its own scope. The
		// CLG decides if and how it is activated, how it calculates its output, if
		// any, and where to forward signals to, if any.
		err = n.EventHandler(CLG, networkPayload)
		if err != nil {
			return maskAny(err)
		}

		return nil
	}

	for {
		select {
		case <-canceler:
			return maskAny(workerCanceledError)
		default:
			n.logNetworkError(invokeEventHandler())
		}
	}
}
Exemplo n.º 9
0
// calculate fetches the information sequence stored under a specific
// information ID. The information ID is provided by the given context.
func (c *clg) calculate(ctx spec.Context) (string, error) {
	informationID, ok := ctx.GetInformationID()
	if !ok {
		return "", maskAnyf(invalidInformationIDError, "must not be empty")
	}

	informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
	informationSequence, err := c.Storage().General().Get(informationSequenceKey)
	if err != nil {
		return "", maskAny(err)
	}

	return informationSequence, nil
}
Exemplo n.º 10
0
func (f *forwarder) Forward(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
	f.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: f, V: 13}, "call Forward")

	// This is the list of lookup functions which is executed seuqentially.
	lookups := []func(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error){
		f.GetNetworkPayloads,
		f.News,
	}

	// Execute one lookup after another. As soon as we find some behaviour IDs, we
	// use them to forward the given network payload.
	var newNetworkPayloads []objectspec.NetworkPayload
	var err error
	for _, lookup := range lookups {
		newNetworkPayloads, err = lookup(CLG, networkPayload)
		if IsNetworkPayloadsNotFound(err) {
			// There could no behaviour IDs be found by this lookup. Go on and try the
			// next one.
			continue
		} else if err != nil {
			return maskAny(err)
		}

		// The current lookup was successful. We do not need to execute any further
		// lookup, but can go on with the behaviour IDs found.
		break
	}

	// Forward the found network payloads to other CLGs by adding them to the
	// queue so other processes can fetch them.
	for _, np := range newNetworkPayloads {
		networkPayloadKey := key.NewNetworkKey("events:network-payload")
		b, err := json.Marshal(np)
		if err != nil {
			return maskAny(err)
		}
		// TODO store asynchronuously
		err = f.Storage().General().PushToSet(networkPayloadKey, string(b))
		if err != nil {
			return maskAny(err)
		}
	}

	return nil
}
Exemplo n.º 11
0
func (f *forwarder) GetNetworkPayloads(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) {
	ctx := networkPayload.GetContext()

	// Check if there are behaviour IDs known that we can use to forward the
	// current network payload to.
	behaviourID, ok := ctx.GetBehaviourID()
	if !ok {
		return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
	}
	behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
	newBehaviourIDs, err := f.Storage().General().GetAllFromSet(behaviourIDsKey)
	if storage.IsNotFound(err) {
		// No configuration of behaviour IDs is stored. Thus we return an error.
		// Eventually some other lookup is able to find sufficient network payloads.
		return nil, maskAny(networkPayloadsNotFoundError)
	} else if err != nil {
		return nil, maskAny(err)
	}

	// Create a list of new network payloads.
	var newNetworkPayloads []objectspec.NetworkPayload
	for _, behaviourID := range newBehaviourIDs {
		// Prepare a new context for the new network payload.
		newCtx := ctx.Clone()
		newCtx.SetBehaviourID(behaviourID)

		// Create a new network payload.
		newNetworkPayloadConfig := networkpayload.DefaultConfig()
		newNetworkPayloadConfig.Args = networkPayload.GetArgs()
		newNetworkPayloadConfig.Context = newCtx
		newNetworkPayloadConfig.Destination = string(behaviourID)
		newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()}
		newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
		if err != nil {
			return nil, maskAny(err)
		}

		newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload)
	}

	return newNetworkPayloads, nil
}
Exemplo n.º 12
0
func Test_CLG_Input_GetInformationIDError(t *testing.T) {
	newCLG := MustNew()
	newCtx := context.MustNew()

	newInput := "test input"
	informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)

	// Prepare the storage connection to fake a returned error.
	c := redigomock.NewConn()
	c.Command("GET", "prefix:"+informationIDKey).ExpectError(invalidConfigError)
	newStorageCollection := testMustNewStorageCollectionWithConn(t, c)

	// Set prepared storage to CLG we want to test.
	newCLG.SetStorageCollection(newStorageCollection)

	// Execute CLG.
	err := newCLG.(*clg).calculate(newCtx, newInput)
	if !IsInvalidConfig(err) {
		t.Fatal("expected", true, "got", false)
	}
}
Exemplo n.º 13
0
func (c *clg) calculate(ctx spec.Context, informationSequence, separator string) error {
	newConfig := featureset.DefaultConfig()
	newConfig.MaxLength = FeatureSize
	newConfig.MinLength = FeatureSize
	newConfig.Separator = separator
	newConfig.Sequences = []string{informationSequence}
	newFeatureSet, err := featureset.New(newConfig)
	if err != nil {
		return maskAny(err)
	}

	err = newFeatureSet.Scan()
	if err != nil {
		return maskAny(err)
	}

	features := newFeatureSet.GetFeatures()
	for _, f := range features {
		// Store the detected feature within the feature storage. It is important to
		// preserve the key structure used here to simply parse the stored features
		// in other places, like in the pair-syntactic and read-separator CLG.
		// Changes in the key structure there must be aligned with implementation
		// details here. The current key structure looks as follows.
		//
		//     feature:%s:positions
		//
		positionKey := key.NewNetworkKey("feature:%s:positions", f.GetSequence())
		raw, err := json.Marshal(f.GetPositions())
		if err != nil {
			return maskAny(err)
		}
		err = c.Storage().Feature().Set(positionKey, string(raw))
		if err != nil {
			return maskAny(err)
		}
	}

	return nil
}
Exemplo n.º 14
0
func (n *network) InputHandler(CLG systemspec.CLG, textInput objectspec.TextInput) error {
	// In case the text request defines the echo flag, we overwrite the given CLG
	// directly to the output CLG. This will cause the created network payload to
	// be forwarded to the output CLG without indirection. Note that this should
	// only be used for testing purposes to bypass more complex neural network
	// activities to directly respond with the received input.
	if textInput.GetEcho() {
		var ok bool
		CLG, ok = n.CLGs["output"]
		if !ok {
			return maskAnyf(clgNotFoundError, "name: %s", "output")
		}
	}

	// Create new IDs for the new CLG tree and the input CLG.
	clgTreeID, err := n.Service().ID().New()
	if err != nil {
		return maskAny(err)
	}
	behaviourID, err := n.Service().ID().New()
	if err != nil {
		return maskAny(err)
	}

	// Create a new context and adapt it using the information of the current scope.
	ctx := context.MustNew()
	ctx.SetBehaviourID(string(behaviourID))
	ctx.SetCLGName(CLG.GetName())
	ctx.SetCLGTreeID(string(clgTreeID))
	ctx.SetExpectation(textInput.GetExpectation())
	ctx.SetSessionID(textInput.GetSessionID())

	// We transform the received text request to a network payload to have a
	// conventional data structure within the neural network.
	newNetworkPayloadConfig := networkpayload.DefaultConfig()
	newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(textInput.GetInput())}
	newNetworkPayloadConfig.Context = ctx
	newNetworkPayloadConfig.Destination = behaviourID
	newNetworkPayloadConfig.Sources = []string{n.GetID()}
	newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
	if err != nil {
		return maskAny(err)
	}

	// Write the new CLG tree ID to reference the input CLG ID and add the CLG
	// tree ID to the new context.
	firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID)
	err = n.Storage().General().Set(firstBehaviourIDKey, string(behaviourID))
	if err != nil {
		return maskAny(err)
	}

	// Write the transformed network payload to the queue.
	eventKey := key.NewNetworkKey("event:network-payload")
	b, err := json.Marshal(newNetworkPayload)
	if err != nil {
		return maskAny(err)
	}
	err = n.Storage().General().PushToList(eventKey, string(b))
	if err != nil {
		return maskAny(err)
	}

	return nil
}
Exemplo n.º 15
0
func (a *activator) GetNetworkPayload(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
	// Fetch the combination of successful behaviour IDs which are known to be
	// useful for the activation of the requested CLG. The network payloads sent
	// by the CLGs being fetched here are known to be useful because they have
	// already been helpful for the execution of the current CLG tree.
	behaviourID, ok := queue[0].GetContext().GetBehaviourID()
	if !ok {
		return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
	}
	behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
	s, err := a.Storage().General().Get(behaviourIDsKey)
	if storage.IsNotFound(err) {
		// No successful combination of behaviour IDs is stored. Thus we return an
		// error. Eventually some other lookup is able to find a sufficient network
		// payload.
		return nil, maskAny(networkPayloadNotFoundError)
	} else if err != nil {
		return nil, maskAny(err)
	}
	behaviourIDs := strings.Split(s, ",")
	if len(behaviourIDs) == 0 {
		// No activation configuration of the requested CLG is stored. Thus we
		// return an error. Eventually some other lookup is able to find a
		// sufficient network payload.
		return nil, maskAny(networkPayloadNotFoundError)
	}

	// Check if there is a queued network payload for each behaviour ID we found in the
	// storage. Here it is important to obtain the order of the behaviour IDs
	// stored as connections. They represent the input interface of the requested
	// CLG. Thus there must not be any variation applied to the lookup here,
	// because we need the lookup to be reproducible.
	var matches []objectspec.NetworkPayload
	for _, behaviourID := range behaviourIDs {
		for _, np := range queue {
			// At this point there is only one source given. That is the CLG that
			// forwarded the current network payload to here. If this is not the case,
			// we return an error.
			sources := np.GetSources()
			if len(sources) != 1 {
				return nil, maskAnyf(invalidSourcesError, "there must be one source")
			}
			if behaviourID == string(sources[0]) {
				// The current behaviour ID belongs to the current network payload. We
				// add the matching network payload to our list and go on to find the
				// network payload belonging to the next behabiour ID.
				matches = append(matches, np)
				break
			}
		}
	}
	if len(behaviourIDs) != len(matches) {
		// No match using the stored configuration associated with the requested CLG
		// can be found. Thus we return an error. Eventually some other lookup is
		// able to find a sufficient network payload.
		return nil, maskAny(networkPayloadNotFoundError)
	}

	// The received network payloads are able to satisfy the interface of the
	// requested CLG. We merge the matching network payloads together and return
	// the result.
	newNetworkPayload, err := mergeNetworkPayloads(matches)
	if err != nil {
		return nil, maskAny(err)
	}

	return newNetworkPayload, nil
}
Exemplo n.º 16
0
func (c *clg) forwardNetworkPayload(ctx spec.Context, informationSequence string) error {
	// Find the original information sequence using the information ID from the
	// context.
	informationID, ok := ctx.GetInformationID()
	if !ok {
		return maskAnyf(invalidInformationIDError, "must not be empty")
	}
	informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
	informationSequence, err := c.Storage().General().Get(informationSequenceKey)
	if err != nil {
		return maskAny(err)
	}

	// Find the first behaviour ID using the CLG tree ID from the context. The
	// behaviour ID we are looking up here is the ID of the initial input CLG.
	clgTreeID, ok := ctx.GetCLGTreeID()
	if !ok {
		return maskAnyf(invalidCLGTreeIDError, "must not be empty")
	}
	firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID)
	inputBehaviourID, err := c.Storage().General().Get(firstBehaviourIDKey)
	if err != nil {
		return maskAny(err)
	}

	// Lookup the behaviour ID of the current output CLG. Below we are using this
	// to set the source of the new network payload accordingly.
	outputBehaviourID, ok := ctx.GetBehaviourID()
	if !ok {
		return maskAnyf(invalidBehaviourIDError, "must not be empty")
	}

	// Create a new contect using the given context and adapt the new context with
	// the information of the current scope.
	newCtx := ctx.Clone()
	newCtx.SetBehaviourID(inputBehaviourID)
	newCtx.SetCLGName("input")
	newCtx.SetCLGTreeID(clgTreeID)
	// We do not need to set the expectation because it never changes.
	// We do not need to set the session ID because it never changes.

	// Create a new network payload.
	newNetworkPayloadConfig := networkpayload.DefaultConfig()
	newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(informationSequence)}
	newNetworkPayloadConfig.Context = newCtx
	newNetworkPayloadConfig.Destination = string(inputBehaviourID)
	newNetworkPayloadConfig.Sources = []string{string(outputBehaviourID)}
	newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
	if err != nil {
		return maskAny(err)
	}

	// Write the transformed network payload to the queue.
	networkPayloadKey := key.NewNetworkKey("events:network-payload")
	b, err := json.Marshal(newNetworkPayload)
	if err != nil {
		return maskAny(err)
	}
	err = c.Storage().General().PushToList(networkPayloadKey, string(b))
	if err != nil {
		return maskAny(err)
	}

	return nil
}
Exemplo n.º 17
0
func (a *activator) New(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
	// Track the input types of the requested CLG as string slice to have
	// something that is easily comparable and efficient. By convention the first
	// input argument of each CLG is a context. We remove the first argument here,
	// because we want to match output interfaces against input interfaces. By
	// convention output interfaces of CLGs must not have a context as first
	// return value. Therefore we align the input and output values to make them
	// comparable.
	clgTypes := typesToStrings(getInputTypes(CLG.GetCalculate()))[1:]

	// Prepare the permutation list to find out which combination of payloads
	// satisfies the requested CLG's interface.
	newPermutationListConfig := permutation.DefaultListConfig()
	newPermutationListConfig.MaxGrowth = len(clgTypes)
	newPermutationListConfig.RawValues = queueToValues(queue)
	newPermutationList, err := permutation.NewList(newPermutationListConfig)
	if err != nil {
		return nil, maskAny(err)
	}

	// Permute the permutation list of the queued network payloads until we found
	// all the matching combinations.
	var possibleMatches [][]objectspec.NetworkPayload
	for {
		// Check if the current combination of network payloads already satisfies
		// the interface of the requested CLG. This is done in the first place to
		// also handle the very first combination of the permutation list.  In case
		// there does a combination of network payloads match the interface of the
		// requested CLG, we capture the found combination and try to find more
		// combinations in the upcoming loops.
		permutedValues := newPermutationList.GetPermutedValues()
		valueTypes := typesToStrings(valuesToTypes(permutedValues))
		if equalStrings(clgTypes, valueTypes) {
			possibleMatches = append(possibleMatches, valuesToQueue(permutedValues))
		}

		// Permute the list of the queued network payloads by one further
		// permutation step within the current iteration. As soon as the permutation
		// list cannot be permuted anymore, we stop the permutation loop to choose
		// one random combination of the tracked list in the next step below.
		err = a.Service().Permutation().PermuteBy(newPermutationList, 1)
		if permutation.IsMaxGrowthReached(err) {
			break
		} else if err != nil {
			return nil, maskAny(err)
		}
	}

	// We fetched all possible combinations if network payloads that match the
	// interface of the requested CLG. Now we need to select one random
	// combination to cover all possible combinations across all possible CLG
	// trees being created over time. This prevents us from choosing always only
	// the first matching combination, which would lack discoveries of all
	// potential combinations being created.
	matchIndex, err := a.Service().Random().CreateMax(len(possibleMatches))
	if err != nil {
		return nil, maskAny(err)
	}
	matches := possibleMatches[matchIndex]

	// The queued network payloads are able to satisfy the interface of the
	// requested CLG. We merge the matching network payloads together and return
	// the result after storing the created configuration of the requested CLG.
	newNetworkPayload, err := mergeNetworkPayloads(matches)
	if err != nil {
		return nil, maskAny(err)
	}

	// Persists the combination of permuted network payloads as configuration for
	// the requested CLG. This configuration is stored using references of the
	// behaviour IDs associated with CLGs that forwarded signals to this requested
	// CLG. Note that the order of behaviour IDs must be preserved, because it
	// represents the input interface of the requested CLG.
	behaviourID, ok := newNetworkPayload.GetContext().GetBehaviourID()
	if !ok {
		return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
	}
	behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
	var behaviourIDs []string
	for _, behaviourID := range newNetworkPayload.GetSources() {
		behaviourIDs = append(behaviourIDs, string(behaviourID))
	}
	err = a.Storage().General().Set(behaviourIDsKey, strings.Join(behaviourIDs, ","))
	if err != nil {
		return nil, maskAny(err)
	}

	return newNetworkPayload, nil
}
Exemplo n.º 18
0
func (f *forwarder) News(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) {
	ctx := networkPayload.GetContext()

	// Decide how many new behaviour IDs should be created. This defines the
	// number of signals being forwarded to other CLGs. Here we want to make a
	// pseudo random decision. CreateMax takes a max paramater which is exclusive.
	// Therefore we increment the configuration for the maximum signals desired by
	// one, to reflect the maximum setting properly.
	maxSignals, err := f.Service().Random().CreateMax(f.GetMaxSignals() + 1)
	if err != nil {
		return nil, maskAny(err)
	}

	// Create the desired number of behaviour IDs.
	var newBehaviourIDs []string
	for i := 0; i < maxSignals; i++ {
		newBehaviourID, err := f.Service().ID().New()
		if err != nil {
			return nil, maskAny(err)
		}
		newBehaviourIDs = append(newBehaviourIDs, string(newBehaviourID))
	}

	// TODO find a CLG name that can be connected to the current CLG for each new
	// behaviour ID and pair these combinations (network event tracker)

	// Store each new behaviour ID in the underlying storage.
	behaviourID, ok := ctx.GetBehaviourID()
	if !ok {
		return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
	}
	behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
	for _, behaviourID := range newBehaviourIDs {
		// TODO store asynchronuously
		err = f.Storage().General().PushToSet(behaviourIDsKey, behaviourID)
		if err != nil {
			return nil, maskAny(err)
		}
	}

	// Create a list of new network payloads.
	var newNetworkPayloads []objectspec.NetworkPayload
	for _, behaviourID := range newBehaviourIDs {
		// Prepare a new context for the new network payload.
		newCtx := ctx.Clone()
		newCtx.SetBehaviourID(behaviourID)
		// TODO set the paired CLG name to the new context

		// Create a new network payload.
		newNetworkPayloadConfig := networkpayload.DefaultConfig()
		newNetworkPayloadConfig.Args = networkPayload.GetArgs()
		newNetworkPayloadConfig.Context = newCtx
		newNetworkPayloadConfig.Destination = string(behaviourID)
		newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()}
		newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
		if err != nil {
			return nil, maskAny(err)
		}

		newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload)
	}

	return newNetworkPayloads, nil
}
Exemplo n.º 19
0
func (a *activator) Activate(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
	a.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: a, V: 13}, "call Activate")

	// Fetch the queued network payloads. queue is a string of comma separated
	// JSON objects representing a specific network payload.
	behaviourID, ok := networkPayload.GetContext().GetBehaviourID()
	if !ok {
		return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
	}
	queueKey := key.NewNetworkKey("activate:queue:behaviour-id:%s:network-payload", behaviourID)
	s, err := a.Storage().General().Get(queueKey)
	if err != nil {
		return nil, maskAny(err)
	}
	queue, err := stringToQueue(s)
	if err != nil {
		return nil, maskAny(err)
	}

	// Merge the given network payload with the queue that we just fetched from
	// storage. We store the extended queue directly after merging it with the
	// given network payload to definitely track the received network payload,
	// even if something goes wrong and we need to return an error on the code
	// below. In case the current queue exeeds a certain amount of payloads, it is
	// unlikely that the queue is going to be helpful when growing any further.
	// Thus we cut the queue at some point beyond the interface capabilities of
	// the requested CLG. Note that it is possible to have multiple network
	// payloads sent by the same CLG. That might happen in case a specific CLG
	// wants to fulfil the interface of the requested CLG on its own, even it is
	// not able to do so with the output of a single calculation.
	queue = append(queue, networkPayload)
	queueBuffer := len(getInputTypes(CLG.GetCalculate())) + 1
	if len(queue) > queueBuffer {
		queue = queue[1:]
	}
	err = a.persistQueue(queueKey, queue)
	if err != nil {
		return nil, maskAny(err)
	}

	// This is the list of lookup functions which is executed seuqentially.
	lookups := []func(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error){
		a.GetNetworkPayload,
		a.New,
	}

	// Execute one lookup after another. As soon as we find a network payload, we
	// return it.
	var newNetworkPayload objectspec.NetworkPayload
	for _, lookup := range lookups {
		newNetworkPayload, err = lookup(CLG, queue)
		if IsNetworkPayloadNotFound(err) {
			// There could no network payload be found by this lookup. Go on and try
			// the next one.
			continue
		} else if err != nil {
			return nil, maskAny(err)
		}

		// The current lookup was successful. We do not need to execute any further
		// lookup, but can go on with the network payload found.
		break
	}

	// Filter all network payloads from the queue that are merged into the new
	// network payload.
	var newQueue []objectspec.NetworkPayload
	for _, s := range newNetworkPayload.GetSources() {
		for _, np := range queue {
			// At this point there is only one source given. That is the CLG that
			// forwarded the current network payload to here. If this is not the case,
			// we return an error.
			sources := np.GetSources()
			if len(sources) != 1 {
				return nil, maskAnyf(invalidSourcesError, "there must be one source")
			}
			if s == sources[0] {
				// The current network payload is part of the merged network payload.
				// Thus we do not add it to the new queue.
				continue
			}
			newQueue = append(newQueue, np)
		}
	}

	// Update the modified queue in the underlying storage.
	err = a.persistQueue(queueKey, newQueue)
	if err != nil {
		return nil, maskAny(err)
	}

	// The current lookup was able to find a network payload. Thus we simply
	// return it.
	return newNetworkPayload, nil
}
Exemplo n.º 20
0
// TODO there is nothing that reads pairs
func (c *clg) calculate(ctx spec.Context) error {
	// The counter keeps track of the work already being done. We only increment
	// the counter in case we were not able to do our job. As soon as some
	// threshold is reached, we stop trying.
	var counter int

	for {
		// Fetch two random features from the feature storage. This is done by
		// fetching random keys. The keys itself already contain the features. Knowing
		// the key structure makes it possible to simply parse the features from the
		// keys fetched from the feature storage. These features are stored by the
		// split-features CLG. Changes in the key structure there must be aligned with
		// implementation details here. The current key structure looks as follows.
		//
		//     feature:%s:positions
		//
		key1, err := c.Storage().Feature().GetRandom()
		if err != nil {
			return maskAny(err)
		}
		key2, err := c.Storage().Feature().GetRandom()
		if err != nil {
			return maskAny(err)
		}

		// Validate the fetched keys.
		for _, k := range []string{key1, key2} {
			if len(k) != 22 {
				return maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", k)
			}
			if !strings.HasPrefix(key1, "feature:") {
				return maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", k)
			}
			if !strings.HasSuffix(key1, ":positions") {
				return maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", k)
			}
		}

		// Combine the fetched keys to a new pair.
		pair := key1[8:12] + key2[8:12]

		// Write the new pair into the general storage.
		pairIDKey := key.NewNetworkKey("pair:syntactic:feature:%s:pair-id", pair)
		_, err = c.Storage().General().Get(pairIDKey)
		if storage.IsNotFound(err) {
			// The created pair was not found within the feature storage. That means
			// we created a new one which we can store. Once we stored the new pair,
			// we break the outer loop to be done.
			newID, err := c.Service().ID().New()
			if err != nil {
				return maskAny(err)
			}
			pairID := string(newID)

			err = c.Storage().General().Set(pairIDKey, pairID)
			if err != nil {
				return maskAny(err)
			}

			break
		}

		if counter >= 3 {
			// We tried to create a new pair 3 times, but it looks like there is not
			// enough data. That is why the created pair was found all the time. It
			// looks like we cannot create any new pair and stop doing anything here.
			break
		}

		counter++
	}

	return nil
}