// calculate fetches the information ID associated with the given information
// sequence. In case the information sequence is not found within the underlying
// storage, a new information ID is generated and used to store the given
// information sequence. In any case the information ID is added to the given
// context.
func (c *clg) calculate(ctx spec.Context, informationSequence string) error {
informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", informationSequence)
informationID, err := c.Storage().General().Get(informationIDKey)
if storage.IsNotFound(err) {
// The given information sequence was never seen before. Thus we register it
// now with its own very unique information ID.
newID, err := c.Service().ID().New()
if err != nil {
return maskAny(err)
}
informationID = string(newID)
err = c.Storage().General().Set(informationIDKey, informationID)
if err != nil {
return maskAny(err)
}
informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
err = c.Storage().General().Set(informationSequenceKey, informationSequence)
if err != nil {
return maskAny(err)
}
} else if err != nil {
return maskAny(err)
}
ctx.SetInformationID(informationID)
return nil
}
func Test_CLG_Input_SetInformationSequenceError(t *testing.T) {
newCLG := MustNew()
newCtx := context.MustNew()
newServiceCollection := testMustNewServiceCollection(t)
// Prepare the storage connection to fake a returned error.
newInput := "test input"
informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
// Our test ID factory always returns the same ID. That way we are able to
// check for the ID being used during the test.
newID, err := newServiceCollection.ID().New()
if err != nil {
t.Fatal("expected", nil, "got", err)
}
informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID)
c := redigomock.NewConn()
c.Command("GET", "prefix:"+informationIDKey).ExpectError(redigo.ErrNil)
c.Command("SET", "prefix:"+informationIDKey, string(newID)).Expect("OK")
c.Command("SET", "prefix:"+informationSequenceKey, newInput).ExpectError(invalidConfigError)
newStorageCollection := testMustNewStorageCollectionWithConn(t, c)
// Set prepared storage to CLG we want to test.
newCLG.(*clg).StorageCollection = newStorageCollection
newCLG.(*clg).ServiceCollection = newServiceCollection
// Execute CLG.
err = newCLG.(*clg).calculate(newCtx, newInput)
if !IsInvalidConfig(err) {
t.Fatal("expected", true, "got", false)
}
}
func Test_CLG_Input_KnownInputSequence(t *testing.T) {
newCLG := MustNew()
newCtx := context.MustNew()
newStorageCollection := testMustNewStorageCollection(t)
// Create record for the test input.
informationID := "123"
newInput := "test input"
informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
err := newStorageCollection.General().Set(informationIDKey, informationID)
if err != nil {
t.Fatal("expected", nil, "got", err)
}
// Set prepared storage to CLG we want to test.
newCLG.(*clg).StorageCollection = newStorageCollection
// Execute CLG.
err = newCLG.(*clg).calculate(newCtx, newInput)
if err != nil {
t.Fatal("expected", nil, "got", err)
}
// Check if the information ID was set to the context.
injectedInformationID, _ := newCtx.GetInformationID()
if informationID != injectedInformationID {
t.Fatal("expected", informationID, "got", injectedInformationID)
}
}
func (t *tracker) CLGIDs(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
destinationID := string(networkPayload.GetDestination())
sourceIDs := networkPayload.GetSources()
errors := make(chan error, len(sourceIDs))
wg := sync.WaitGroup{}
for _, s := range sourceIDs {
wg.Add(1)
go func(s string) {
// Persist the single CLG ID connections.
behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:o:tracker:behaviour-ids", s)
err := t.Storage().General().PushToSet(behaviourIDKey, destinationID)
if err != nil {
errors <- maskAny(err)
}
wg.Done()
}(string(s))
}
wg.Wait()
select {
case err := <-errors:
if err != nil {
return maskAny(err)
}
default:
// Nothing do here. No error occurred. All good.
}
return nil
}
func Test_CLG_Input_DataProperlyStored(t *testing.T) {
newCLG := MustNew()
newCtx := context.MustNew()
newStorageCollection := testMustNewStorageCollection(t)
newServiceCollection := testMustNewServiceCollection(t)
// Note we do not create a record for the test input. This test is about an
// unknown input sequence.
newInput := "test input"
// Our test ID factory always returns the same ID. That way we are able to
// check for the ID being used during the test.
newID, err := newServiceCollection.ID().New()
if err != nil {
t.Fatal("expected", nil, "got", err)
}
// Set prepared storage to CLG we want to test.
newCLG.(*clg).ServiceCollection = newServiceCollection
newCLG.(*clg).StorageCollection = newStorageCollection
// Execute CLG.
err = newCLG.(*clg).calculate(newCtx, newInput)
if err != nil {
t.Fatal("expected", nil, "got", err)
}
informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
storedID, err := newStorageCollection.General().Get(informationIDKey)
if err != nil {
t.Fatal("expected", nil, "got", err)
}
if storedID != string(newID) {
t.Fatal("expected", newID, "got", storedID)
}
informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID)
storedInput, err := newStorageCollection.General().Get(informationSequenceKey)
if err != nil {
t.Fatal("expected", nil, "got", err)
}
if storedInput != newInput {
t.Fatal("expected", newInput, "got", storedInput)
}
}
func (c *clg) calculate(ctx spec.Context) (string, error) {
behaviourID, ok := ctx.GetBehaviourID()
if !ok {
return "", maskAnyf(invalidBehaviourIDError, "must not be empty")
}
behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:separator", behaviourID)
separator, err := c.Storage().General().Get(behaviourIDKey)
if storage.IsNotFound(err) {
randomKey, err := c.Storage().Feature().GetRandom()
if err != nil {
return "", maskAny(err)
}
// Make sure the fetched feature is valid based on its key structure. The
// key itself already contains the feature. Knowing the key structure makes
// it possible to simply parse the feature from the key fetched from the
// feature storage. These features are stored by the split-features CLG.
// Changes in the key structure there must be aligned with implementation
// details here. The current key structure looks as follows.
//
// feature:%s:positions
//
if len(randomKey) != 22 {
return "", maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", randomKey)
}
if !strings.HasPrefix(randomKey, "feature:") {
return "", maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", randomKey)
}
if !strings.HasSuffix(randomKey, ":positions") {
return "", maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", randomKey)
}
// Create a new separator from the fetched random feature. Note that a
// feature is considered 4 characters long and the random factory takes a
// max parameter, which is exlusive.
feature := randomKey[8:12]
featureIndex, err := c.Service().Random().CreateMax(5)
if err != nil {
return "", maskAny(err)
}
separator = string(feature[featureIndex])
// Store the newly created separator using the CLGs own behaviour ID. In case
// this CLG is asked again to return its separator, it will lookup its
// separator in the general storage.
err = c.Storage().General().Set(behaviourIDKey, separator)
if err != nil {
return "", maskAny(err)
}
} else if err != nil {
return "", maskAny(err)
}
return separator, nil
}
func (t *tracker) CLGNames(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
destinationName := CLG.GetName()
sourceIDs := networkPayload.GetSources()
errors := make(chan error, len(sourceIDs))
wg := sync.WaitGroup{}
for _, s := range sourceIDs {
wg.Add(1)
go func(s string) {
behaviourNameKey := key.NewNetworkKey("behaviour-id:%s:behaviour-name", s)
name, err := t.Storage().General().Get(behaviourNameKey)
if err != nil {
errors <- maskAny(err)
} else {
// The errors channel is capable of buffering one error for each source
// ID. The else clause is necessary to queue only one possible error for
// each source ID. So in case the name lookup was successful, we are
// able to actually persist the single CLG name connection.
behaviourNameKey := key.NewNetworkKey("behaviour-name:%s:o:tracker:behaviour-names", name)
err := t.Storage().General().PushToSet(behaviourNameKey, destinationName)
if err != nil {
errors <- maskAny(err)
}
}
wg.Done()
}(string(s))
}
wg.Wait()
select {
case err := <-errors:
if err != nil {
return maskAny(err)
}
default:
// Nothing do here. No error occurred. All good.
}
return nil
}
func (n *network) EventListener(canceler <-chan struct{}) error {
invokeEventHandler := func() error {
// Fetch the next network payload from the queue. This call blocks until one
// network payload was fetched from the queue. As soon as we receive the
// network payload, it is removed from the queue automatically, so it is not
// handled twice.
eventKey := key.NewNetworkKey("event:network-payload")
element, err := n.Storage().General().PopFromList(eventKey)
if err != nil {
return maskAny(err)
}
networkPayload := networkpayload.MustNew()
err = json.Unmarshal([]byte(element), &networkPayload)
if err != nil {
return maskAny(err)
}
// Lookup the CLG that is supposed to be executed. The CLG object is
// referenced by name. When being executed it is referenced by its behaviour
// ID. The behaviour ID represents a specific peer within a connection path.
clgName, ok := networkPayload.GetContext().GetCLGName()
if !ok {
return maskAnyf(invalidCLGNameError, "must not be empty")
}
CLG, ok := n.CLGs[clgName]
if !ok {
return maskAnyf(clgNotFoundError, "name: %s", clgName)
}
// Invoke the event handler to execute the given CLG using the given network
// payload. Here we execute one distinct behaviour within its own scope. The
// CLG decides if and how it is activated, how it calculates its output, if
// any, and where to forward signals to, if any.
err = n.EventHandler(CLG, networkPayload)
if err != nil {
return maskAny(err)
}
return nil
}
for {
select {
case <-canceler:
return maskAny(workerCanceledError)
default:
n.logNetworkError(invokeEventHandler())
}
}
}
// calculate fetches the information sequence stored under a specific
// information ID. The information ID is provided by the given context.
func (c *clg) calculate(ctx spec.Context) (string, error) {
informationID, ok := ctx.GetInformationID()
if !ok {
return "", maskAnyf(invalidInformationIDError, "must not be empty")
}
informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
informationSequence, err := c.Storage().General().Get(informationSequenceKey)
if err != nil {
return "", maskAny(err)
}
return informationSequence, nil
}
func (f *forwarder) Forward(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error {
f.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: f, V: 13}, "call Forward")
// This is the list of lookup functions which is executed seuqentially.
lookups := []func(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error){
f.GetNetworkPayloads,
f.News,
}
// Execute one lookup after another. As soon as we find some behaviour IDs, we
// use them to forward the given network payload.
var newNetworkPayloads []objectspec.NetworkPayload
var err error
for _, lookup := range lookups {
newNetworkPayloads, err = lookup(CLG, networkPayload)
if IsNetworkPayloadsNotFound(err) {
// There could no behaviour IDs be found by this lookup. Go on and try the
// next one.
continue
} else if err != nil {
return maskAny(err)
}
// The current lookup was successful. We do not need to execute any further
// lookup, but can go on with the behaviour IDs found.
break
}
// Forward the found network payloads to other CLGs by adding them to the
// queue so other processes can fetch them.
for _, np := range newNetworkPayloads {
networkPayloadKey := key.NewNetworkKey("events:network-payload")
b, err := json.Marshal(np)
if err != nil {
return maskAny(err)
}
// TODO store asynchronuously
err = f.Storage().General().PushToSet(networkPayloadKey, string(b))
if err != nil {
return maskAny(err)
}
}
return nil
}
func (f *forwarder) GetNetworkPayloads(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) {
ctx := networkPayload.GetContext()
// Check if there are behaviour IDs known that we can use to forward the
// current network payload to.
behaviourID, ok := ctx.GetBehaviourID()
if !ok {
return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
}
behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
newBehaviourIDs, err := f.Storage().General().GetAllFromSet(behaviourIDsKey)
if storage.IsNotFound(err) {
// No configuration of behaviour IDs is stored. Thus we return an error.
// Eventually some other lookup is able to find sufficient network payloads.
return nil, maskAny(networkPayloadsNotFoundError)
} else if err != nil {
return nil, maskAny(err)
}
// Create a list of new network payloads.
var newNetworkPayloads []objectspec.NetworkPayload
for _, behaviourID := range newBehaviourIDs {
// Prepare a new context for the new network payload.
newCtx := ctx.Clone()
newCtx.SetBehaviourID(behaviourID)
// Create a new network payload.
newNetworkPayloadConfig := networkpayload.DefaultConfig()
newNetworkPayloadConfig.Args = networkPayload.GetArgs()
newNetworkPayloadConfig.Context = newCtx
newNetworkPayloadConfig.Destination = string(behaviourID)
newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()}
newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
if err != nil {
return nil, maskAny(err)
}
newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload)
}
return newNetworkPayloads, nil
}
func Test_CLG_Input_GetInformationIDError(t *testing.T) {
newCLG := MustNew()
newCtx := context.MustNew()
newInput := "test input"
informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput)
// Prepare the storage connection to fake a returned error.
c := redigomock.NewConn()
c.Command("GET", "prefix:"+informationIDKey).ExpectError(invalidConfigError)
newStorageCollection := testMustNewStorageCollectionWithConn(t, c)
// Set prepared storage to CLG we want to test.
newCLG.SetStorageCollection(newStorageCollection)
// Execute CLG.
err := newCLG.(*clg).calculate(newCtx, newInput)
if !IsInvalidConfig(err) {
t.Fatal("expected", true, "got", false)
}
}
func (c *clg) calculate(ctx spec.Context, informationSequence, separator string) error {
newConfig := featureset.DefaultConfig()
newConfig.MaxLength = FeatureSize
newConfig.MinLength = FeatureSize
newConfig.Separator = separator
newConfig.Sequences = []string{informationSequence}
newFeatureSet, err := featureset.New(newConfig)
if err != nil {
return maskAny(err)
}
err = newFeatureSet.Scan()
if err != nil {
return maskAny(err)
}
features := newFeatureSet.GetFeatures()
for _, f := range features {
// Store the detected feature within the feature storage. It is important to
// preserve the key structure used here to simply parse the stored features
// in other places, like in the pair-syntactic and read-separator CLG.
// Changes in the key structure there must be aligned with implementation
// details here. The current key structure looks as follows.
//
// feature:%s:positions
//
positionKey := key.NewNetworkKey("feature:%s:positions", f.GetSequence())
raw, err := json.Marshal(f.GetPositions())
if err != nil {
return maskAny(err)
}
err = c.Storage().Feature().Set(positionKey, string(raw))
if err != nil {
return maskAny(err)
}
}
return nil
}
func (n *network) InputHandler(CLG systemspec.CLG, textInput objectspec.TextInput) error {
// In case the text request defines the echo flag, we overwrite the given CLG
// directly to the output CLG. This will cause the created network payload to
// be forwarded to the output CLG without indirection. Note that this should
// only be used for testing purposes to bypass more complex neural network
// activities to directly respond with the received input.
if textInput.GetEcho() {
var ok bool
CLG, ok = n.CLGs["output"]
if !ok {
return maskAnyf(clgNotFoundError, "name: %s", "output")
}
}
// Create new IDs for the new CLG tree and the input CLG.
clgTreeID, err := n.Service().ID().New()
if err != nil {
return maskAny(err)
}
behaviourID, err := n.Service().ID().New()
if err != nil {
return maskAny(err)
}
// Create a new context and adapt it using the information of the current scope.
ctx := context.MustNew()
ctx.SetBehaviourID(string(behaviourID))
ctx.SetCLGName(CLG.GetName())
ctx.SetCLGTreeID(string(clgTreeID))
ctx.SetExpectation(textInput.GetExpectation())
ctx.SetSessionID(textInput.GetSessionID())
// We transform the received text request to a network payload to have a
// conventional data structure within the neural network.
newNetworkPayloadConfig := networkpayload.DefaultConfig()
newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(textInput.GetInput())}
newNetworkPayloadConfig.Context = ctx
newNetworkPayloadConfig.Destination = behaviourID
newNetworkPayloadConfig.Sources = []string{n.GetID()}
newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
if err != nil {
return maskAny(err)
}
// Write the new CLG tree ID to reference the input CLG ID and add the CLG
// tree ID to the new context.
firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID)
err = n.Storage().General().Set(firstBehaviourIDKey, string(behaviourID))
if err != nil {
return maskAny(err)
}
// Write the transformed network payload to the queue.
eventKey := key.NewNetworkKey("event:network-payload")
b, err := json.Marshal(newNetworkPayload)
if err != nil {
return maskAny(err)
}
err = n.Storage().General().PushToList(eventKey, string(b))
if err != nil {
return maskAny(err)
}
return nil
}
func (a *activator) GetNetworkPayload(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
// Fetch the combination of successful behaviour IDs which are known to be
// useful for the activation of the requested CLG. The network payloads sent
// by the CLGs being fetched here are known to be useful because they have
// already been helpful for the execution of the current CLG tree.
behaviourID, ok := queue[0].GetContext().GetBehaviourID()
if !ok {
return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
}
behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
s, err := a.Storage().General().Get(behaviourIDsKey)
if storage.IsNotFound(err) {
// No successful combination of behaviour IDs is stored. Thus we return an
// error. Eventually some other lookup is able to find a sufficient network
// payload.
return nil, maskAny(networkPayloadNotFoundError)
} else if err != nil {
return nil, maskAny(err)
}
behaviourIDs := strings.Split(s, ",")
if len(behaviourIDs) == 0 {
// No activation configuration of the requested CLG is stored. Thus we
// return an error. Eventually some other lookup is able to find a
// sufficient network payload.
return nil, maskAny(networkPayloadNotFoundError)
}
// Check if there is a queued network payload for each behaviour ID we found in the
// storage. Here it is important to obtain the order of the behaviour IDs
// stored as connections. They represent the input interface of the requested
// CLG. Thus there must not be any variation applied to the lookup here,
// because we need the lookup to be reproducible.
var matches []objectspec.NetworkPayload
for _, behaviourID := range behaviourIDs {
for _, np := range queue {
// At this point there is only one source given. That is the CLG that
// forwarded the current network payload to here. If this is not the case,
// we return an error.
sources := np.GetSources()
if len(sources) != 1 {
return nil, maskAnyf(invalidSourcesError, "there must be one source")
}
if behaviourID == string(sources[0]) {
// The current behaviour ID belongs to the current network payload. We
// add the matching network payload to our list and go on to find the
// network payload belonging to the next behabiour ID.
matches = append(matches, np)
break
}
}
}
if len(behaviourIDs) != len(matches) {
// No match using the stored configuration associated with the requested CLG
// can be found. Thus we return an error. Eventually some other lookup is
// able to find a sufficient network payload.
return nil, maskAny(networkPayloadNotFoundError)
}
// The received network payloads are able to satisfy the interface of the
// requested CLG. We merge the matching network payloads together and return
// the result.
newNetworkPayload, err := mergeNetworkPayloads(matches)
if err != nil {
return nil, maskAny(err)
}
return newNetworkPayload, nil
}
func (c *clg) forwardNetworkPayload(ctx spec.Context, informationSequence string) error {
// Find the original information sequence using the information ID from the
// context.
informationID, ok := ctx.GetInformationID()
if !ok {
return maskAnyf(invalidInformationIDError, "must not be empty")
}
informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID)
informationSequence, err := c.Storage().General().Get(informationSequenceKey)
if err != nil {
return maskAny(err)
}
// Find the first behaviour ID using the CLG tree ID from the context. The
// behaviour ID we are looking up here is the ID of the initial input CLG.
clgTreeID, ok := ctx.GetCLGTreeID()
if !ok {
return maskAnyf(invalidCLGTreeIDError, "must not be empty")
}
firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID)
inputBehaviourID, err := c.Storage().General().Get(firstBehaviourIDKey)
if err != nil {
return maskAny(err)
}
// Lookup the behaviour ID of the current output CLG. Below we are using this
// to set the source of the new network payload accordingly.
outputBehaviourID, ok := ctx.GetBehaviourID()
if !ok {
return maskAnyf(invalidBehaviourIDError, "must not be empty")
}
// Create a new contect using the given context and adapt the new context with
// the information of the current scope.
newCtx := ctx.Clone()
newCtx.SetBehaviourID(inputBehaviourID)
newCtx.SetCLGName("input")
newCtx.SetCLGTreeID(clgTreeID)
// We do not need to set the expectation because it never changes.
// We do not need to set the session ID because it never changes.
// Create a new network payload.
newNetworkPayloadConfig := networkpayload.DefaultConfig()
newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(informationSequence)}
newNetworkPayloadConfig.Context = newCtx
newNetworkPayloadConfig.Destination = string(inputBehaviourID)
newNetworkPayloadConfig.Sources = []string{string(outputBehaviourID)}
newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
if err != nil {
return maskAny(err)
}
// Write the transformed network payload to the queue.
networkPayloadKey := key.NewNetworkKey("events:network-payload")
b, err := json.Marshal(newNetworkPayload)
if err != nil {
return maskAny(err)
}
err = c.Storage().General().PushToList(networkPayloadKey, string(b))
if err != nil {
return maskAny(err)
}
return nil
}
func (a *activator) New(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
// Track the input types of the requested CLG as string slice to have
// something that is easily comparable and efficient. By convention the first
// input argument of each CLG is a context. We remove the first argument here,
// because we want to match output interfaces against input interfaces. By
// convention output interfaces of CLGs must not have a context as first
// return value. Therefore we align the input and output values to make them
// comparable.
clgTypes := typesToStrings(getInputTypes(CLG.GetCalculate()))[1:]
// Prepare the permutation list to find out which combination of payloads
// satisfies the requested CLG's interface.
newPermutationListConfig := permutation.DefaultListConfig()
newPermutationListConfig.MaxGrowth = len(clgTypes)
newPermutationListConfig.RawValues = queueToValues(queue)
newPermutationList, err := permutation.NewList(newPermutationListConfig)
if err != nil {
return nil, maskAny(err)
}
// Permute the permutation list of the queued network payloads until we found
// all the matching combinations.
var possibleMatches [][]objectspec.NetworkPayload
for {
// Check if the current combination of network payloads already satisfies
// the interface of the requested CLG. This is done in the first place to
// also handle the very first combination of the permutation list. In case
// there does a combination of network payloads match the interface of the
// requested CLG, we capture the found combination and try to find more
// combinations in the upcoming loops.
permutedValues := newPermutationList.GetPermutedValues()
valueTypes := typesToStrings(valuesToTypes(permutedValues))
if equalStrings(clgTypes, valueTypes) {
possibleMatches = append(possibleMatches, valuesToQueue(permutedValues))
}
// Permute the list of the queued network payloads by one further
// permutation step within the current iteration. As soon as the permutation
// list cannot be permuted anymore, we stop the permutation loop to choose
// one random combination of the tracked list in the next step below.
err = a.Service().Permutation().PermuteBy(newPermutationList, 1)
if permutation.IsMaxGrowthReached(err) {
break
} else if err != nil {
return nil, maskAny(err)
}
}
// We fetched all possible combinations if network payloads that match the
// interface of the requested CLG. Now we need to select one random
// combination to cover all possible combinations across all possible CLG
// trees being created over time. This prevents us from choosing always only
// the first matching combination, which would lack discoveries of all
// potential combinations being created.
matchIndex, err := a.Service().Random().CreateMax(len(possibleMatches))
if err != nil {
return nil, maskAny(err)
}
matches := possibleMatches[matchIndex]
// The queued network payloads are able to satisfy the interface of the
// requested CLG. We merge the matching network payloads together and return
// the result after storing the created configuration of the requested CLG.
newNetworkPayload, err := mergeNetworkPayloads(matches)
if err != nil {
return nil, maskAny(err)
}
// Persists the combination of permuted network payloads as configuration for
// the requested CLG. This configuration is stored using references of the
// behaviour IDs associated with CLGs that forwarded signals to this requested
// CLG. Note that the order of behaviour IDs must be preserved, because it
// represents the input interface of the requested CLG.
behaviourID, ok := newNetworkPayload.GetContext().GetBehaviourID()
if !ok {
return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
}
behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
var behaviourIDs []string
for _, behaviourID := range newNetworkPayload.GetSources() {
behaviourIDs = append(behaviourIDs, string(behaviourID))
}
err = a.Storage().General().Set(behaviourIDsKey, strings.Join(behaviourIDs, ","))
if err != nil {
return nil, maskAny(err)
}
return newNetworkPayload, nil
}
func (f *forwarder) News(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) {
ctx := networkPayload.GetContext()
// Decide how many new behaviour IDs should be created. This defines the
// number of signals being forwarded to other CLGs. Here we want to make a
// pseudo random decision. CreateMax takes a max paramater which is exclusive.
// Therefore we increment the configuration for the maximum signals desired by
// one, to reflect the maximum setting properly.
maxSignals, err := f.Service().Random().CreateMax(f.GetMaxSignals() + 1)
if err != nil {
return nil, maskAny(err)
}
// Create the desired number of behaviour IDs.
var newBehaviourIDs []string
for i := 0; i < maxSignals; i++ {
newBehaviourID, err := f.Service().ID().New()
if err != nil {
return nil, maskAny(err)
}
newBehaviourIDs = append(newBehaviourIDs, string(newBehaviourID))
}
// TODO find a CLG name that can be connected to the current CLG for each new
// behaviour ID and pair these combinations (network event tracker)
// Store each new behaviour ID in the underlying storage.
behaviourID, ok := ctx.GetBehaviourID()
if !ok {
return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
}
behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID)
for _, behaviourID := range newBehaviourIDs {
// TODO store asynchronuously
err = f.Storage().General().PushToSet(behaviourIDsKey, behaviourID)
if err != nil {
return nil, maskAny(err)
}
}
// Create a list of new network payloads.
var newNetworkPayloads []objectspec.NetworkPayload
for _, behaviourID := range newBehaviourIDs {
// Prepare a new context for the new network payload.
newCtx := ctx.Clone()
newCtx.SetBehaviourID(behaviourID)
// TODO set the paired CLG name to the new context
// Create a new network payload.
newNetworkPayloadConfig := networkpayload.DefaultConfig()
newNetworkPayloadConfig.Args = networkPayload.GetArgs()
newNetworkPayloadConfig.Context = newCtx
newNetworkPayloadConfig.Destination = string(behaviourID)
newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()}
newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig)
if err != nil {
return nil, maskAny(err)
}
newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload)
}
return newNetworkPayloads, nil
}
func (a *activator) Activate(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) (objectspec.NetworkPayload, error) {
a.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: a, V: 13}, "call Activate")
// Fetch the queued network payloads. queue is a string of comma separated
// JSON objects representing a specific network payload.
behaviourID, ok := networkPayload.GetContext().GetBehaviourID()
if !ok {
return nil, maskAnyf(invalidBehaviourIDError, "must not be empty")
}
queueKey := key.NewNetworkKey("activate:queue:behaviour-id:%s:network-payload", behaviourID)
s, err := a.Storage().General().Get(queueKey)
if err != nil {
return nil, maskAny(err)
}
queue, err := stringToQueue(s)
if err != nil {
return nil, maskAny(err)
}
// Merge the given network payload with the queue that we just fetched from
// storage. We store the extended queue directly after merging it with the
// given network payload to definitely track the received network payload,
// even if something goes wrong and we need to return an error on the code
// below. In case the current queue exeeds a certain amount of payloads, it is
// unlikely that the queue is going to be helpful when growing any further.
// Thus we cut the queue at some point beyond the interface capabilities of
// the requested CLG. Note that it is possible to have multiple network
// payloads sent by the same CLG. That might happen in case a specific CLG
// wants to fulfil the interface of the requested CLG on its own, even it is
// not able to do so with the output of a single calculation.
queue = append(queue, networkPayload)
queueBuffer := len(getInputTypes(CLG.GetCalculate())) + 1
if len(queue) > queueBuffer {
queue = queue[1:]
}
err = a.persistQueue(queueKey, queue)
if err != nil {
return nil, maskAny(err)
}
// This is the list of lookup functions which is executed seuqentially.
lookups := []func(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error){
a.GetNetworkPayload,
a.New,
}
// Execute one lookup after another. As soon as we find a network payload, we
// return it.
var newNetworkPayload objectspec.NetworkPayload
for _, lookup := range lookups {
newNetworkPayload, err = lookup(CLG, queue)
if IsNetworkPayloadNotFound(err) {
// There could no network payload be found by this lookup. Go on and try
// the next one.
continue
} else if err != nil {
return nil, maskAny(err)
}
// The current lookup was successful. We do not need to execute any further
// lookup, but can go on with the network payload found.
break
}
// Filter all network payloads from the queue that are merged into the new
// network payload.
var newQueue []objectspec.NetworkPayload
for _, s := range newNetworkPayload.GetSources() {
for _, np := range queue {
// At this point there is only one source given. That is the CLG that
// forwarded the current network payload to here. If this is not the case,
// we return an error.
sources := np.GetSources()
if len(sources) != 1 {
return nil, maskAnyf(invalidSourcesError, "there must be one source")
}
if s == sources[0] {
// The current network payload is part of the merged network payload.
// Thus we do not add it to the new queue.
continue
}
newQueue = append(newQueue, np)
}
}
// Update the modified queue in the underlying storage.
err = a.persistQueue(queueKey, newQueue)
if err != nil {
return nil, maskAny(err)
}
// The current lookup was able to find a network payload. Thus we simply
// return it.
return newNetworkPayload, nil
}
// TODO there is nothing that reads pairs
func (c *clg) calculate(ctx spec.Context) error {
// The counter keeps track of the work already being done. We only increment
// the counter in case we were not able to do our job. As soon as some
// threshold is reached, we stop trying.
var counter int
for {
// Fetch two random features from the feature storage. This is done by
// fetching random keys. The keys itself already contain the features. Knowing
// the key structure makes it possible to simply parse the features from the
// keys fetched from the feature storage. These features are stored by the
// split-features CLG. Changes in the key structure there must be aligned with
// implementation details here. The current key structure looks as follows.
//
// feature:%s:positions
//
key1, err := c.Storage().Feature().GetRandom()
if err != nil {
return maskAny(err)
}
key2, err := c.Storage().Feature().GetRandom()
if err != nil {
return maskAny(err)
}
// Validate the fetched keys.
for _, k := range []string{key1, key2} {
if len(k) != 22 {
return maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", k)
}
if !strings.HasPrefix(key1, "feature:") {
return maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", k)
}
if !strings.HasSuffix(key1, ":positions") {
return maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", k)
}
}
// Combine the fetched keys to a new pair.
pair := key1[8:12] + key2[8:12]
// Write the new pair into the general storage.
pairIDKey := key.NewNetworkKey("pair:syntactic:feature:%s:pair-id", pair)
_, err = c.Storage().General().Get(pairIDKey)
if storage.IsNotFound(err) {
// The created pair was not found within the feature storage. That means
// we created a new one which we can store. Once we stored the new pair,
// we break the outer loop to be done.
newID, err := c.Service().ID().New()
if err != nil {
return maskAny(err)
}
pairID := string(newID)
err = c.Storage().General().Set(pairIDKey, pairID)
if err != nil {
return maskAny(err)
}
break
}
if counter >= 3 {
// We tried to create a new pair 3 times, but it looks like there is not
// enough data. That is why the created pair was found all the time. It
// looks like we cannot create any new pair and stop doing anything here.
break
}
counter++
}
return nil
}