// calculate fetches the information ID associated with the given information // sequence. In case the information sequence is not found within the underlying // storage, a new information ID is generated and used to store the given // information sequence. In any case the information ID is added to the given // context. func (c *clg) calculate(ctx spec.Context, informationSequence string) error { informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", informationSequence) informationID, err := c.Storage().General().Get(informationIDKey) if storage.IsNotFound(err) { // The given information sequence was never seen before. Thus we register it // now with its own very unique information ID. newID, err := c.Service().ID().New() if err != nil { return maskAny(err) } informationID = string(newID) err = c.Storage().General().Set(informationIDKey, informationID) if err != nil { return maskAny(err) } informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID) err = c.Storage().General().Set(informationSequenceKey, informationSequence) if err != nil { return maskAny(err) } } else if err != nil { return maskAny(err) } ctx.SetInformationID(informationID) return nil }
func Test_CLG_Input_SetInformationSequenceError(t *testing.T) { newCLG := MustNew() newCtx := context.MustNew() newServiceCollection := testMustNewServiceCollection(t) // Prepare the storage connection to fake a returned error. newInput := "test input" informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput) // Our test ID factory always returns the same ID. That way we are able to // check for the ID being used during the test. newID, err := newServiceCollection.ID().New() if err != nil { t.Fatal("expected", nil, "got", err) } informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID) c := redigomock.NewConn() c.Command("GET", "prefix:"+informationIDKey).ExpectError(redigo.ErrNil) c.Command("SET", "prefix:"+informationIDKey, string(newID)).Expect("OK") c.Command("SET", "prefix:"+informationSequenceKey, newInput).ExpectError(invalidConfigError) newStorageCollection := testMustNewStorageCollectionWithConn(t, c) // Set prepared storage to CLG we want to test. newCLG.(*clg).StorageCollection = newStorageCollection newCLG.(*clg).ServiceCollection = newServiceCollection // Execute CLG. err = newCLG.(*clg).calculate(newCtx, newInput) if !IsInvalidConfig(err) { t.Fatal("expected", true, "got", false) } }
func Test_CLG_Input_KnownInputSequence(t *testing.T) { newCLG := MustNew() newCtx := context.MustNew() newStorageCollection := testMustNewStorageCollection(t) // Create record for the test input. informationID := "123" newInput := "test input" informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput) err := newStorageCollection.General().Set(informationIDKey, informationID) if err != nil { t.Fatal("expected", nil, "got", err) } // Set prepared storage to CLG we want to test. newCLG.(*clg).StorageCollection = newStorageCollection // Execute CLG. err = newCLG.(*clg).calculate(newCtx, newInput) if err != nil { t.Fatal("expected", nil, "got", err) } // Check if the information ID was set to the context. injectedInformationID, _ := newCtx.GetInformationID() if informationID != injectedInformationID { t.Fatal("expected", informationID, "got", injectedInformationID) } }
func (t *tracker) CLGIDs(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error { destinationID := string(networkPayload.GetDestination()) sourceIDs := networkPayload.GetSources() errors := make(chan error, len(sourceIDs)) wg := sync.WaitGroup{} for _, s := range sourceIDs { wg.Add(1) go func(s string) { // Persist the single CLG ID connections. behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:o:tracker:behaviour-ids", s) err := t.Storage().General().PushToSet(behaviourIDKey, destinationID) if err != nil { errors <- maskAny(err) } wg.Done() }(string(s)) } wg.Wait() select { case err := <-errors: if err != nil { return maskAny(err) } default: // Nothing do here. No error occurred. All good. } return nil }
func Test_CLG_Input_DataProperlyStored(t *testing.T) { newCLG := MustNew() newCtx := context.MustNew() newStorageCollection := testMustNewStorageCollection(t) newServiceCollection := testMustNewServiceCollection(t) // Note we do not create a record for the test input. This test is about an // unknown input sequence. newInput := "test input" // Our test ID factory always returns the same ID. That way we are able to // check for the ID being used during the test. newID, err := newServiceCollection.ID().New() if err != nil { t.Fatal("expected", nil, "got", err) } // Set prepared storage to CLG we want to test. newCLG.(*clg).ServiceCollection = newServiceCollection newCLG.(*clg).StorageCollection = newStorageCollection // Execute CLG. err = newCLG.(*clg).calculate(newCtx, newInput) if err != nil { t.Fatal("expected", nil, "got", err) } informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput) storedID, err := newStorageCollection.General().Get(informationIDKey) if err != nil { t.Fatal("expected", nil, "got", err) } if storedID != string(newID) { t.Fatal("expected", newID, "got", storedID) } informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", newID) storedInput, err := newStorageCollection.General().Get(informationSequenceKey) if err != nil { t.Fatal("expected", nil, "got", err) } if storedInput != newInput { t.Fatal("expected", newInput, "got", storedInput) } }
func (c *clg) calculate(ctx spec.Context) (string, error) { behaviourID, ok := ctx.GetBehaviourID() if !ok { return "", maskAnyf(invalidBehaviourIDError, "must not be empty") } behaviourIDKey := key.NewNetworkKey("behaviour-id:%s:separator", behaviourID) separator, err := c.Storage().General().Get(behaviourIDKey) if storage.IsNotFound(err) { randomKey, err := c.Storage().Feature().GetRandom() if err != nil { return "", maskAny(err) } // Make sure the fetched feature is valid based on its key structure. The // key itself already contains the feature. Knowing the key structure makes // it possible to simply parse the feature from the key fetched from the // feature storage. These features are stored by the split-features CLG. // Changes in the key structure there must be aligned with implementation // details here. The current key structure looks as follows. // // feature:%s:positions // if len(randomKey) != 22 { return "", maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", randomKey) } if !strings.HasPrefix(randomKey, "feature:") { return "", maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", randomKey) } if !strings.HasSuffix(randomKey, ":positions") { return "", maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", randomKey) } // Create a new separator from the fetched random feature. Note that a // feature is considered 4 characters long and the random factory takes a // max parameter, which is exlusive. feature := randomKey[8:12] featureIndex, err := c.Service().Random().CreateMax(5) if err != nil { return "", maskAny(err) } separator = string(feature[featureIndex]) // Store the newly created separator using the CLGs own behaviour ID. In case // this CLG is asked again to return its separator, it will lookup its // separator in the general storage. err = c.Storage().General().Set(behaviourIDKey, separator) if err != nil { return "", maskAny(err) } } else if err != nil { return "", maskAny(err) } return separator, nil }
func (t *tracker) CLGNames(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error { destinationName := CLG.GetName() sourceIDs := networkPayload.GetSources() errors := make(chan error, len(sourceIDs)) wg := sync.WaitGroup{} for _, s := range sourceIDs { wg.Add(1) go func(s string) { behaviourNameKey := key.NewNetworkKey("behaviour-id:%s:behaviour-name", s) name, err := t.Storage().General().Get(behaviourNameKey) if err != nil { errors <- maskAny(err) } else { // The errors channel is capable of buffering one error for each source // ID. The else clause is necessary to queue only one possible error for // each source ID. So in case the name lookup was successful, we are // able to actually persist the single CLG name connection. behaviourNameKey := key.NewNetworkKey("behaviour-name:%s:o:tracker:behaviour-names", name) err := t.Storage().General().PushToSet(behaviourNameKey, destinationName) if err != nil { errors <- maskAny(err) } } wg.Done() }(string(s)) } wg.Wait() select { case err := <-errors: if err != nil { return maskAny(err) } default: // Nothing do here. No error occurred. All good. } return nil }
func (n *network) EventListener(canceler <-chan struct{}) error { invokeEventHandler := func() error { // Fetch the next network payload from the queue. This call blocks until one // network payload was fetched from the queue. As soon as we receive the // network payload, it is removed from the queue automatically, so it is not // handled twice. eventKey := key.NewNetworkKey("event:network-payload") element, err := n.Storage().General().PopFromList(eventKey) if err != nil { return maskAny(err) } networkPayload := networkpayload.MustNew() err = json.Unmarshal([]byte(element), &networkPayload) if err != nil { return maskAny(err) } // Lookup the CLG that is supposed to be executed. The CLG object is // referenced by name. When being executed it is referenced by its behaviour // ID. The behaviour ID represents a specific peer within a connection path. clgName, ok := networkPayload.GetContext().GetCLGName() if !ok { return maskAnyf(invalidCLGNameError, "must not be empty") } CLG, ok := n.CLGs[clgName] if !ok { return maskAnyf(clgNotFoundError, "name: %s", clgName) } // Invoke the event handler to execute the given CLG using the given network // payload. Here we execute one distinct behaviour within its own scope. The // CLG decides if and how it is activated, how it calculates its output, if // any, and where to forward signals to, if any. err = n.EventHandler(CLG, networkPayload) if err != nil { return maskAny(err) } return nil } for { select { case <-canceler: return maskAny(workerCanceledError) default: n.logNetworkError(invokeEventHandler()) } } }
// calculate fetches the information sequence stored under a specific // information ID. The information ID is provided by the given context. func (c *clg) calculate(ctx spec.Context) (string, error) { informationID, ok := ctx.GetInformationID() if !ok { return "", maskAnyf(invalidInformationIDError, "must not be empty") } informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID) informationSequence, err := c.Storage().General().Get(informationSequenceKey) if err != nil { return "", maskAny(err) } return informationSequence, nil }
func (f *forwarder) Forward(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) error { f.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: f, V: 13}, "call Forward") // This is the list of lookup functions which is executed seuqentially. lookups := []func(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error){ f.GetNetworkPayloads, f.News, } // Execute one lookup after another. As soon as we find some behaviour IDs, we // use them to forward the given network payload. var newNetworkPayloads []objectspec.NetworkPayload var err error for _, lookup := range lookups { newNetworkPayloads, err = lookup(CLG, networkPayload) if IsNetworkPayloadsNotFound(err) { // There could no behaviour IDs be found by this lookup. Go on and try the // next one. continue } else if err != nil { return maskAny(err) } // The current lookup was successful. We do not need to execute any further // lookup, but can go on with the behaviour IDs found. break } // Forward the found network payloads to other CLGs by adding them to the // queue so other processes can fetch them. for _, np := range newNetworkPayloads { networkPayloadKey := key.NewNetworkKey("events:network-payload") b, err := json.Marshal(np) if err != nil { return maskAny(err) } // TODO store asynchronuously err = f.Storage().General().PushToSet(networkPayloadKey, string(b)) if err != nil { return maskAny(err) } } return nil }
func (f *forwarder) GetNetworkPayloads(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) { ctx := networkPayload.GetContext() // Check if there are behaviour IDs known that we can use to forward the // current network payload to. behaviourID, ok := ctx.GetBehaviourID() if !ok { return nil, maskAnyf(invalidBehaviourIDError, "must not be empty") } behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID) newBehaviourIDs, err := f.Storage().General().GetAllFromSet(behaviourIDsKey) if storage.IsNotFound(err) { // No configuration of behaviour IDs is stored. Thus we return an error. // Eventually some other lookup is able to find sufficient network payloads. return nil, maskAny(networkPayloadsNotFoundError) } else if err != nil { return nil, maskAny(err) } // Create a list of new network payloads. var newNetworkPayloads []objectspec.NetworkPayload for _, behaviourID := range newBehaviourIDs { // Prepare a new context for the new network payload. newCtx := ctx.Clone() newCtx.SetBehaviourID(behaviourID) // Create a new network payload. newNetworkPayloadConfig := networkpayload.DefaultConfig() newNetworkPayloadConfig.Args = networkPayload.GetArgs() newNetworkPayloadConfig.Context = newCtx newNetworkPayloadConfig.Destination = string(behaviourID) newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()} newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig) if err != nil { return nil, maskAny(err) } newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload) } return newNetworkPayloads, nil }
func Test_CLG_Input_GetInformationIDError(t *testing.T) { newCLG := MustNew() newCtx := context.MustNew() newInput := "test input" informationIDKey := key.NewNetworkKey("information-sequence:%s:information-id", newInput) // Prepare the storage connection to fake a returned error. c := redigomock.NewConn() c.Command("GET", "prefix:"+informationIDKey).ExpectError(invalidConfigError) newStorageCollection := testMustNewStorageCollectionWithConn(t, c) // Set prepared storage to CLG we want to test. newCLG.SetStorageCollection(newStorageCollection) // Execute CLG. err := newCLG.(*clg).calculate(newCtx, newInput) if !IsInvalidConfig(err) { t.Fatal("expected", true, "got", false) } }
func (c *clg) calculate(ctx spec.Context, informationSequence, separator string) error { newConfig := featureset.DefaultConfig() newConfig.MaxLength = FeatureSize newConfig.MinLength = FeatureSize newConfig.Separator = separator newConfig.Sequences = []string{informationSequence} newFeatureSet, err := featureset.New(newConfig) if err != nil { return maskAny(err) } err = newFeatureSet.Scan() if err != nil { return maskAny(err) } features := newFeatureSet.GetFeatures() for _, f := range features { // Store the detected feature within the feature storage. It is important to // preserve the key structure used here to simply parse the stored features // in other places, like in the pair-syntactic and read-separator CLG. // Changes in the key structure there must be aligned with implementation // details here. The current key structure looks as follows. // // feature:%s:positions // positionKey := key.NewNetworkKey("feature:%s:positions", f.GetSequence()) raw, err := json.Marshal(f.GetPositions()) if err != nil { return maskAny(err) } err = c.Storage().Feature().Set(positionKey, string(raw)) if err != nil { return maskAny(err) } } return nil }
func (n *network) InputHandler(CLG systemspec.CLG, textInput objectspec.TextInput) error { // In case the text request defines the echo flag, we overwrite the given CLG // directly to the output CLG. This will cause the created network payload to // be forwarded to the output CLG without indirection. Note that this should // only be used for testing purposes to bypass more complex neural network // activities to directly respond with the received input. if textInput.GetEcho() { var ok bool CLG, ok = n.CLGs["output"] if !ok { return maskAnyf(clgNotFoundError, "name: %s", "output") } } // Create new IDs for the new CLG tree and the input CLG. clgTreeID, err := n.Service().ID().New() if err != nil { return maskAny(err) } behaviourID, err := n.Service().ID().New() if err != nil { return maskAny(err) } // Create a new context and adapt it using the information of the current scope. ctx := context.MustNew() ctx.SetBehaviourID(string(behaviourID)) ctx.SetCLGName(CLG.GetName()) ctx.SetCLGTreeID(string(clgTreeID)) ctx.SetExpectation(textInput.GetExpectation()) ctx.SetSessionID(textInput.GetSessionID()) // We transform the received text request to a network payload to have a // conventional data structure within the neural network. newNetworkPayloadConfig := networkpayload.DefaultConfig() newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(textInput.GetInput())} newNetworkPayloadConfig.Context = ctx newNetworkPayloadConfig.Destination = behaviourID newNetworkPayloadConfig.Sources = []string{n.GetID()} newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig) if err != nil { return maskAny(err) } // Write the new CLG tree ID to reference the input CLG ID and add the CLG // tree ID to the new context. firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID) err = n.Storage().General().Set(firstBehaviourIDKey, string(behaviourID)) if err != nil { return maskAny(err) } // Write the transformed network payload to the queue. eventKey := key.NewNetworkKey("event:network-payload") b, err := json.Marshal(newNetworkPayload) if err != nil { return maskAny(err) } err = n.Storage().General().PushToList(eventKey, string(b)) if err != nil { return maskAny(err) } return nil }
func (a *activator) GetNetworkPayload(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) { // Fetch the combination of successful behaviour IDs which are known to be // useful for the activation of the requested CLG. The network payloads sent // by the CLGs being fetched here are known to be useful because they have // already been helpful for the execution of the current CLG tree. behaviourID, ok := queue[0].GetContext().GetBehaviourID() if !ok { return nil, maskAnyf(invalidBehaviourIDError, "must not be empty") } behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID) s, err := a.Storage().General().Get(behaviourIDsKey) if storage.IsNotFound(err) { // No successful combination of behaviour IDs is stored. Thus we return an // error. Eventually some other lookup is able to find a sufficient network // payload. return nil, maskAny(networkPayloadNotFoundError) } else if err != nil { return nil, maskAny(err) } behaviourIDs := strings.Split(s, ",") if len(behaviourIDs) == 0 { // No activation configuration of the requested CLG is stored. Thus we // return an error. Eventually some other lookup is able to find a // sufficient network payload. return nil, maskAny(networkPayloadNotFoundError) } // Check if there is a queued network payload for each behaviour ID we found in the // storage. Here it is important to obtain the order of the behaviour IDs // stored as connections. They represent the input interface of the requested // CLG. Thus there must not be any variation applied to the lookup here, // because we need the lookup to be reproducible. var matches []objectspec.NetworkPayload for _, behaviourID := range behaviourIDs { for _, np := range queue { // At this point there is only one source given. That is the CLG that // forwarded the current network payload to here. If this is not the case, // we return an error. sources := np.GetSources() if len(sources) != 1 { return nil, maskAnyf(invalidSourcesError, "there must be one source") } if behaviourID == string(sources[0]) { // The current behaviour ID belongs to the current network payload. We // add the matching network payload to our list and go on to find the // network payload belonging to the next behabiour ID. matches = append(matches, np) break } } } if len(behaviourIDs) != len(matches) { // No match using the stored configuration associated with the requested CLG // can be found. Thus we return an error. Eventually some other lookup is // able to find a sufficient network payload. return nil, maskAny(networkPayloadNotFoundError) } // The received network payloads are able to satisfy the interface of the // requested CLG. We merge the matching network payloads together and return // the result. newNetworkPayload, err := mergeNetworkPayloads(matches) if err != nil { return nil, maskAny(err) } return newNetworkPayload, nil }
func (c *clg) forwardNetworkPayload(ctx spec.Context, informationSequence string) error { // Find the original information sequence using the information ID from the // context. informationID, ok := ctx.GetInformationID() if !ok { return maskAnyf(invalidInformationIDError, "must not be empty") } informationSequenceKey := key.NewNetworkKey("information-id:%s:information-sequence", informationID) informationSequence, err := c.Storage().General().Get(informationSequenceKey) if err != nil { return maskAny(err) } // Find the first behaviour ID using the CLG tree ID from the context. The // behaviour ID we are looking up here is the ID of the initial input CLG. clgTreeID, ok := ctx.GetCLGTreeID() if !ok { return maskAnyf(invalidCLGTreeIDError, "must not be empty") } firstBehaviourIDKey := key.NewNetworkKey("clg-tree-id:%s:first-behaviour-id", clgTreeID) inputBehaviourID, err := c.Storage().General().Get(firstBehaviourIDKey) if err != nil { return maskAny(err) } // Lookup the behaviour ID of the current output CLG. Below we are using this // to set the source of the new network payload accordingly. outputBehaviourID, ok := ctx.GetBehaviourID() if !ok { return maskAnyf(invalidBehaviourIDError, "must not be empty") } // Create a new contect using the given context and adapt the new context with // the information of the current scope. newCtx := ctx.Clone() newCtx.SetBehaviourID(inputBehaviourID) newCtx.SetCLGName("input") newCtx.SetCLGTreeID(clgTreeID) // We do not need to set the expectation because it never changes. // We do not need to set the session ID because it never changes. // Create a new network payload. newNetworkPayloadConfig := networkpayload.DefaultConfig() newNetworkPayloadConfig.Args = []reflect.Value{reflect.ValueOf(informationSequence)} newNetworkPayloadConfig.Context = newCtx newNetworkPayloadConfig.Destination = string(inputBehaviourID) newNetworkPayloadConfig.Sources = []string{string(outputBehaviourID)} newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig) if err != nil { return maskAny(err) } // Write the transformed network payload to the queue. networkPayloadKey := key.NewNetworkKey("events:network-payload") b, err := json.Marshal(newNetworkPayload) if err != nil { return maskAny(err) } err = c.Storage().General().PushToList(networkPayloadKey, string(b)) if err != nil { return maskAny(err) } return nil }
func (a *activator) New(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error) { // Track the input types of the requested CLG as string slice to have // something that is easily comparable and efficient. By convention the first // input argument of each CLG is a context. We remove the first argument here, // because we want to match output interfaces against input interfaces. By // convention output interfaces of CLGs must not have a context as first // return value. Therefore we align the input and output values to make them // comparable. clgTypes := typesToStrings(getInputTypes(CLG.GetCalculate()))[1:] // Prepare the permutation list to find out which combination of payloads // satisfies the requested CLG's interface. newPermutationListConfig := permutation.DefaultListConfig() newPermutationListConfig.MaxGrowth = len(clgTypes) newPermutationListConfig.RawValues = queueToValues(queue) newPermutationList, err := permutation.NewList(newPermutationListConfig) if err != nil { return nil, maskAny(err) } // Permute the permutation list of the queued network payloads until we found // all the matching combinations. var possibleMatches [][]objectspec.NetworkPayload for { // Check if the current combination of network payloads already satisfies // the interface of the requested CLG. This is done in the first place to // also handle the very first combination of the permutation list. In case // there does a combination of network payloads match the interface of the // requested CLG, we capture the found combination and try to find more // combinations in the upcoming loops. permutedValues := newPermutationList.GetPermutedValues() valueTypes := typesToStrings(valuesToTypes(permutedValues)) if equalStrings(clgTypes, valueTypes) { possibleMatches = append(possibleMatches, valuesToQueue(permutedValues)) } // Permute the list of the queued network payloads by one further // permutation step within the current iteration. As soon as the permutation // list cannot be permuted anymore, we stop the permutation loop to choose // one random combination of the tracked list in the next step below. err = a.Service().Permutation().PermuteBy(newPermutationList, 1) if permutation.IsMaxGrowthReached(err) { break } else if err != nil { return nil, maskAny(err) } } // We fetched all possible combinations if network payloads that match the // interface of the requested CLG. Now we need to select one random // combination to cover all possible combinations across all possible CLG // trees being created over time. This prevents us from choosing always only // the first matching combination, which would lack discoveries of all // potential combinations being created. matchIndex, err := a.Service().Random().CreateMax(len(possibleMatches)) if err != nil { return nil, maskAny(err) } matches := possibleMatches[matchIndex] // The queued network payloads are able to satisfy the interface of the // requested CLG. We merge the matching network payloads together and return // the result after storing the created configuration of the requested CLG. newNetworkPayload, err := mergeNetworkPayloads(matches) if err != nil { return nil, maskAny(err) } // Persists the combination of permuted network payloads as configuration for // the requested CLG. This configuration is stored using references of the // behaviour IDs associated with CLGs that forwarded signals to this requested // CLG. Note that the order of behaviour IDs must be preserved, because it // represents the input interface of the requested CLG. behaviourID, ok := newNetworkPayload.GetContext().GetBehaviourID() if !ok { return nil, maskAnyf(invalidBehaviourIDError, "must not be empty") } behaviourIDsKey := key.NewNetworkKey("activate:configuration:behaviour-id:%s:behaviour-ids", behaviourID) var behaviourIDs []string for _, behaviourID := range newNetworkPayload.GetSources() { behaviourIDs = append(behaviourIDs, string(behaviourID)) } err = a.Storage().General().Set(behaviourIDsKey, strings.Join(behaviourIDs, ",")) if err != nil { return nil, maskAny(err) } return newNetworkPayload, nil }
func (f *forwarder) News(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) ([]objectspec.NetworkPayload, error) { ctx := networkPayload.GetContext() // Decide how many new behaviour IDs should be created. This defines the // number of signals being forwarded to other CLGs. Here we want to make a // pseudo random decision. CreateMax takes a max paramater which is exclusive. // Therefore we increment the configuration for the maximum signals desired by // one, to reflect the maximum setting properly. maxSignals, err := f.Service().Random().CreateMax(f.GetMaxSignals() + 1) if err != nil { return nil, maskAny(err) } // Create the desired number of behaviour IDs. var newBehaviourIDs []string for i := 0; i < maxSignals; i++ { newBehaviourID, err := f.Service().ID().New() if err != nil { return nil, maskAny(err) } newBehaviourIDs = append(newBehaviourIDs, string(newBehaviourID)) } // TODO find a CLG name that can be connected to the current CLG for each new // behaviour ID and pair these combinations (network event tracker) // Store each new behaviour ID in the underlying storage. behaviourID, ok := ctx.GetBehaviourID() if !ok { return nil, maskAnyf(invalidBehaviourIDError, "must not be empty") } behaviourIDsKey := key.NewNetworkKey("forward:configuration:behaviour-id:%s:behaviour-ids", behaviourID) for _, behaviourID := range newBehaviourIDs { // TODO store asynchronuously err = f.Storage().General().PushToSet(behaviourIDsKey, behaviourID) if err != nil { return nil, maskAny(err) } } // Create a list of new network payloads. var newNetworkPayloads []objectspec.NetworkPayload for _, behaviourID := range newBehaviourIDs { // Prepare a new context for the new network payload. newCtx := ctx.Clone() newCtx.SetBehaviourID(behaviourID) // TODO set the paired CLG name to the new context // Create a new network payload. newNetworkPayloadConfig := networkpayload.DefaultConfig() newNetworkPayloadConfig.Args = networkPayload.GetArgs() newNetworkPayloadConfig.Context = newCtx newNetworkPayloadConfig.Destination = string(behaviourID) newNetworkPayloadConfig.Sources = []string{networkPayload.GetDestination()} newNetworkPayload, err := networkpayload.New(newNetworkPayloadConfig) if err != nil { return nil, maskAny(err) } newNetworkPayloads = append(newNetworkPayloads, newNetworkPayload) } return newNetworkPayloads, nil }
func (a *activator) Activate(CLG systemspec.CLG, networkPayload objectspec.NetworkPayload) (objectspec.NetworkPayload, error) { a.Log.WithTags(systemspec.Tags{C: nil, L: "D", O: a, V: 13}, "call Activate") // Fetch the queued network payloads. queue is a string of comma separated // JSON objects representing a specific network payload. behaviourID, ok := networkPayload.GetContext().GetBehaviourID() if !ok { return nil, maskAnyf(invalidBehaviourIDError, "must not be empty") } queueKey := key.NewNetworkKey("activate:queue:behaviour-id:%s:network-payload", behaviourID) s, err := a.Storage().General().Get(queueKey) if err != nil { return nil, maskAny(err) } queue, err := stringToQueue(s) if err != nil { return nil, maskAny(err) } // Merge the given network payload with the queue that we just fetched from // storage. We store the extended queue directly after merging it with the // given network payload to definitely track the received network payload, // even if something goes wrong and we need to return an error on the code // below. In case the current queue exeeds a certain amount of payloads, it is // unlikely that the queue is going to be helpful when growing any further. // Thus we cut the queue at some point beyond the interface capabilities of // the requested CLG. Note that it is possible to have multiple network // payloads sent by the same CLG. That might happen in case a specific CLG // wants to fulfil the interface of the requested CLG on its own, even it is // not able to do so with the output of a single calculation. queue = append(queue, networkPayload) queueBuffer := len(getInputTypes(CLG.GetCalculate())) + 1 if len(queue) > queueBuffer { queue = queue[1:] } err = a.persistQueue(queueKey, queue) if err != nil { return nil, maskAny(err) } // This is the list of lookup functions which is executed seuqentially. lookups := []func(CLG systemspec.CLG, queue []objectspec.NetworkPayload) (objectspec.NetworkPayload, error){ a.GetNetworkPayload, a.New, } // Execute one lookup after another. As soon as we find a network payload, we // return it. var newNetworkPayload objectspec.NetworkPayload for _, lookup := range lookups { newNetworkPayload, err = lookup(CLG, queue) if IsNetworkPayloadNotFound(err) { // There could no network payload be found by this lookup. Go on and try // the next one. continue } else if err != nil { return nil, maskAny(err) } // The current lookup was successful. We do not need to execute any further // lookup, but can go on with the network payload found. break } // Filter all network payloads from the queue that are merged into the new // network payload. var newQueue []objectspec.NetworkPayload for _, s := range newNetworkPayload.GetSources() { for _, np := range queue { // At this point there is only one source given. That is the CLG that // forwarded the current network payload to here. If this is not the case, // we return an error. sources := np.GetSources() if len(sources) != 1 { return nil, maskAnyf(invalidSourcesError, "there must be one source") } if s == sources[0] { // The current network payload is part of the merged network payload. // Thus we do not add it to the new queue. continue } newQueue = append(newQueue, np) } } // Update the modified queue in the underlying storage. err = a.persistQueue(queueKey, newQueue) if err != nil { return nil, maskAny(err) } // The current lookup was able to find a network payload. Thus we simply // return it. return newNetworkPayload, nil }
// TODO there is nothing that reads pairs func (c *clg) calculate(ctx spec.Context) error { // The counter keeps track of the work already being done. We only increment // the counter in case we were not able to do our job. As soon as some // threshold is reached, we stop trying. var counter int for { // Fetch two random features from the feature storage. This is done by // fetching random keys. The keys itself already contain the features. Knowing // the key structure makes it possible to simply parse the features from the // keys fetched from the feature storage. These features are stored by the // split-features CLG. Changes in the key structure there must be aligned with // implementation details here. The current key structure looks as follows. // // feature:%s:positions // key1, err := c.Storage().Feature().GetRandom() if err != nil { return maskAny(err) } key2, err := c.Storage().Feature().GetRandom() if err != nil { return maskAny(err) } // Validate the fetched keys. for _, k := range []string{key1, key2} { if len(k) != 22 { return maskAnyf(invalidFeatureKeyError, "key '%s' must have length '22'", k) } if !strings.HasPrefix(key1, "feature:") { return maskAnyf(invalidFeatureKeyError, "key '%s' prefix must be 'feature:'", k) } if !strings.HasSuffix(key1, ":positions") { return maskAnyf(invalidFeatureKeyError, "key '%s' suffix must be ':positions'", k) } } // Combine the fetched keys to a new pair. pair := key1[8:12] + key2[8:12] // Write the new pair into the general storage. pairIDKey := key.NewNetworkKey("pair:syntactic:feature:%s:pair-id", pair) _, err = c.Storage().General().Get(pairIDKey) if storage.IsNotFound(err) { // The created pair was not found within the feature storage. That means // we created a new one which we can store. Once we stored the new pair, // we break the outer loop to be done. newID, err := c.Service().ID().New() if err != nil { return maskAny(err) } pairID := string(newID) err = c.Storage().General().Set(pairIDKey, pairID) if err != nil { return maskAny(err) } break } if counter >= 3 { // We tried to create a new pair 3 times, but it looks like there is not // enough data. That is why the created pair was found all the time. It // looks like we cannot create any new pair and stop doing anything here. break } counter++ } return nil }