Ejemplo n.º 1
0
func (s *HandlersSuite) TestHealingHistoryHandler(c *check.C) {
	evt1, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeNode, Value: "addr1"},
		InternalKind: "healer",
		CustomData:   map[string]interface{}{"node": cluster.Node{Address: "addr1"}},
		Allowed:      event.Allowed(permission.PermPool),
	})
	c.Assert(err, check.IsNil)
	evt1.DoneCustomData(nil, cluster.Node{Address: "addr2"})
	time.Sleep(10 * time.Millisecond)
	evt2, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeNode, Value: "addr3"},
		InternalKind: "healer",
		CustomData:   map[string]interface{}{"node": cluster.Node{Address: "addr3"}},
		Allowed:      event.Allowed(permission.PermPool),
	})
	c.Assert(err, check.IsNil)
	evt2.DoneCustomData(errors.New("some error"), cluster.Node{})
	time.Sleep(10 * time.Millisecond)
	evt3, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeContainer, Value: "1234"},
		InternalKind: "healer",
		CustomData:   container.Container{ID: "1234"},
		Allowed:      event.Allowed(permission.PermApp),
	})
	c.Assert(err, check.IsNil)
	evt3.DoneCustomData(nil, container.Container{ID: "9876"})
	recorder := httptest.NewRecorder()
	request, err := http.NewRequest("GET", "/docker/healing", nil)
	c.Assert(err, check.IsNil)
	request.Header.Set("Authorization", "bearer "+s.token.GetValue())
	server := api.RunServer(true)
	server.ServeHTTP(recorder, request)
	c.Assert(recorder.Code, check.Equals, http.StatusOK)
	c.Assert(recorder.Header().Get("Content-Type"), check.Equals, "application/json")
	body := recorder.Body.Bytes()
	var healings []healer.HealingEvent
	err = json.Unmarshal(body, &healings)
	c.Assert(err, check.IsNil)
	c.Assert(healings, check.HasLen, 3)
	c.Assert(healings[2].StartTime.UTC().Format(time.Stamp), check.Equals, evt1.StartTime.UTC().Format(time.Stamp))
	c.Assert(healings[2].EndTime.UTC().Format(time.Stamp), check.Equals, evt1.EndTime.UTC().Format(time.Stamp))
	c.Assert(healings[2].FailingNode.Address, check.Equals, "addr1")
	c.Assert(healings[2].CreatedNode.Address, check.Equals, "addr2")
	c.Assert(healings[2].Error, check.Equals, "")
	c.Assert(healings[2].Successful, check.Equals, true)
	c.Assert(healings[2].Action, check.Equals, "node-healing")
	c.Assert(healings[1].FailingNode.Address, check.Equals, "addr3")
	c.Assert(healings[1].CreatedNode.Address, check.Equals, "")
	c.Assert(healings[1].Error, check.Equals, "some error")
	c.Assert(healings[1].Successful, check.Equals, false)
	c.Assert(healings[1].Action, check.Equals, "node-healing")
	c.Assert(healings[0].FailingContainer.ID, check.Equals, "1234")
	c.Assert(healings[0].CreatedContainer.ID, check.Equals, "9876")
	c.Assert(healings[0].Successful, check.Equals, true)
	c.Assert(healings[0].Error, check.Equals, "")
	c.Assert(healings[0].Action, check.Equals, "container-healing")
}
Ejemplo n.º 2
0
func (s *HandlersSuite) TestAutoScaleHistoryHandler(c *check.C) {
	evt1, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: poolMetadataName, Value: "poolx"},
		InternalKind: autoScaleEventKind,
		Allowed:      event.Allowed(permission.PermPool),
	})
	c.Assert(err, check.IsNil)
	evt1.Logf("my evt1")
	err = evt1.DoneCustomData(nil, evtCustomData{
		Result: &scalerResult{ToAdd: 1, Reason: "r1"},
	})
	c.Assert(err, check.IsNil)
	time.Sleep(100 * time.Millisecond)
	evt2, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: poolMetadataName, Value: "pooly"},
		InternalKind: autoScaleEventKind,
		Allowed:      event.Allowed(permission.PermPool),
	})
	c.Assert(err, check.IsNil)
	evt2.Logf("my evt2")
	err = evt2.DoneCustomData(nil, evtCustomData{
		Result: &scalerResult{ToRebalance: true, Reason: "r2"},
	})
	c.Assert(err, check.IsNil)
	recorder := httptest.NewRecorder()
	request, err := http.NewRequest("GET", "/docker/autoscale", nil)
	c.Assert(err, check.IsNil)
	request.Header.Set("Authorization", "bearer "+s.token.GetValue())
	server := api.RunServer(true)
	server.ServeHTTP(recorder, request)
	c.Assert(recorder.Code, check.Equals, http.StatusOK)
	c.Assert(recorder.Header().Get("Content-Type"), check.Equals, "application/json")
	body := recorder.Body.Bytes()
	history := []autoScaleEvent{}
	err = json.Unmarshal(body, &history)
	c.Assert(err, check.IsNil)
	c.Assert(history, check.HasLen, 2)
	c.Assert(evt1.StartTime.Sub(history[1].StartTime) < time.Second, check.Equals, true)
	c.Assert(evt2.StartTime.Sub(history[0].StartTime) < time.Second, check.Equals, true)
	c.Assert(history[1].MetadataValue, check.Equals, "poolx")
	c.Assert(history[0].MetadataValue, check.Equals, "pooly")
	c.Assert(history[1].Action, check.Equals, "add")
	c.Assert(history[0].Action, check.Equals, "rebalance")
	c.Assert(history[1].Reason, check.Equals, "r1")
	c.Assert(history[0].Reason, check.Equals, "r2")
	c.Assert(history[1].Log, check.Equals, "my evt1\n")
	c.Assert(history[0].Log, check.Equals, "my evt2\n")
}
Ejemplo n.º 3
0
func (h *NodeHealer) tryHealingNode(node provision.Node, reason string, lastCheck *NodeChecks) error {
	_, hasIaas := node.Metadata()["iaas"]
	if !hasIaas {
		log.Debugf("node %q doesn't have IaaS information, healing (%s) won't run on it.", node.Address(), reason)
		return nil
	}
	poolName := node.Metadata()[poolMetadataName]
	evt, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeNode, Value: node.Address()},
		InternalKind: "healer",
		CustomData: NodeHealerCustomData{
			Node:      provision.NodeToSpec(node),
			Reason:    reason,
			LastCheck: lastCheck,
		},
		Allowed: event.Allowed(permission.PermPoolReadEvents, permission.Context(permission.CtxPool, poolName)),
	})
	if err != nil {
		if _, ok := err.(event.ErrEventLocked); ok {
			// Healing in progress.
			return nil
		}
		return errors.Wrap(err, "Error trying to insert node healing event, healing aborted")
	}
	var createdNode *provision.NodeSpec
	var evtErr error
	defer func() {
		var updateErr error
		if evtErr == nil && createdNode == nil {
			updateErr = evt.Abort()
		} else {
			updateErr = evt.DoneCustomData(evtErr, createdNode)
		}
		if updateErr != nil {
			log.Errorf("error trying to update healing event: %s", updateErr)
		}
	}()
	_, err = node.Provisioner().GetNode(node.Address())
	if err != nil {
		if err == provision.ErrNodeNotFound {
			return nil
		}
		evtErr = errors.Wrap(err, "unable to check if node still exists")
		return evtErr
	}
	shouldHeal, err := h.shouldHealNode(node)
	if err != nil {
		evtErr = errors.Wrap(err, "unable to check if node should be healed")
		return evtErr
	}
	if !shouldHeal {
		return nil
	}
	log.Errorf("initiating healing process for node %q due to: %s", node.Address(), reason)
	createdNode, evtErr = h.healNode(node)
	return evtErr
}
Ejemplo n.º 4
0
func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)
	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	healer.started = time.Now().Add(-3 * time.Second)
	conf := healerConfig()
	err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")
	for i := 0; i < 3; i++ {
		var evt *event.Event
		evt, err = event.NewInternal(&event.Opts{
			Target:       event.Target{Type: "node", Value: nodes[0].Address()},
			InternalKind: "healer",
			Allowed:      event.Allowed(permission.PermPoolReadEvents),
		})
		c.Assert(err, check.IsNil)
		err = evt.Done(nil)
		c.Assert(err, check.IsNil)
	}
	err = healer.tryHealingNode(nodes[0], "myreason", nil)
	c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}
Ejemplo n.º 5
0
func (h *ContainerHealer) healContainerIfNeeded(cont container.Container) error {
	if cont.LastSuccessStatusUpdate.IsZero() {
		if !cont.MongoID.Time().Before(time.Now().Add(-h.maxUnresponsiveTime)) {
			return nil
		}
	}
	isAsExpected, err := h.isAsExpected(cont)
	if err != nil {
		log.Errorf("Containers healing: couldn't verify running processes in container %q: %s", cont.ID, err)
	}
	if isAsExpected {
		cont.SetStatus(h.provisioner, cont.ExpectedStatus(), true)
		return nil
	}
	locked := h.locker.Lock(cont.AppName)
	if !locked {
		return errors.Errorf("Containers healing: unable to heal %q couldn't lock app %s", cont.ID, cont.AppName)
	}
	defer h.locker.Unlock(cont.AppName)
	// Sanity check, now we have a lock, let's find out if the container still exists
	_, err = h.provisioner.GetContainer(cont.ID)
	if err != nil {
		if _, isNotFound := err.(*provision.UnitNotFoundError); isNotFound {
			return nil
		}
		return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't verify it still exists", cont.ID)
	}
	a, err := app.GetByName(cont.AppName)
	if err != nil {
		return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't get app %q", cont.ID, cont.AppName)
	}
	log.Errorf("Initiating healing process for container %q, unresponsive since %s.", cont.ID, cont.LastSuccessStatusUpdate)
	evt, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeContainer, Value: cont.ID},
		InternalKind: "healer",
		CustomData:   cont,
		Allowed: event.Allowed(permission.PermAppReadEvents, append(permission.Contexts(permission.CtxTeam, a.Teams),
			permission.Context(permission.CtxApp, a.Name),
			permission.Context(permission.CtxPool, a.Pool),
		)...),
	})
	if err != nil {
		return errors.Wrap(err, "Error trying to insert container healing event, healing aborted")
	}
	newCont, healErr := h.healContainer(cont)
	if healErr != nil {
		healErr = errors.Errorf("Error healing container %q: %s", cont.ID, healErr.Error())
	}
	err = evt.DoneCustomData(healErr, newCont)
	if err != nil {
		log.Errorf("Error trying to update containers healing event: %s", err)
	}
	return healErr
}
Ejemplo n.º 6
0
func (s *S) TestListHealingHistory(c *check.C) {
	evt1, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: "node", Value: "addr1"},
		InternalKind: "healer",
		Allowed:      event.Allowed(permission.PermPoolReadEvents),
	})
	c.Assert(err, check.IsNil)
	err = evt1.Done(nil)
	c.Assert(err, check.IsNil)
	time.Sleep(100 * time.Millisecond)
	evt2, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: "container", Value: "cont1"},
		InternalKind: "healer",
		Allowed:      event.Allowed(permission.PermPoolReadEvents),
	})
	c.Assert(err, check.IsNil)
	err = evt2.Done(nil)
	c.Assert(err, check.IsNil)
	evts, err := ListHealingHistory("")
	c.Assert(err, check.IsNil)
	c.Assert(evts, check.DeepEquals, []HealingEvent{
		{
			ID:         evt2.UniqueID,
			StartTime:  mongoTime(evt2.StartTime),
			EndTime:    mongoTime(evt2.EndTime),
			Action:     "container-healing",
			Successful: true,
			Error:      "",
		},
		{
			ID:         evt1.UniqueID,
			StartTime:  mongoTime(evt1.StartTime),
			EndTime:    mongoTime(evt1.EndTime),
			Action:     "node-healing",
			Successful: true,
			Error:      "",
		},
	})
}
Ejemplo n.º 7
0
func (s *S) TestRunContainerHealerThrottled(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := newFakeAppInDB("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
	for i := 0; i < 3; i++ {
		var evt *event.Event
		evt, err = event.NewInternal(&event.Opts{
			Target:       event.Target{Type: "container", Value: toMoveCont.ID},
			InternalKind: "healer",
			CustomData:   toMoveCont,
			Allowed:      event.Allowed(permission.PermAppReadEvents),
		})
		c.Assert(err, check.IsNil)
		err = evt.DoneCustomData(nil, nil)
		c.Assert(err, check.IsNil)
	}
	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
	err = healer.healContainerIfNeeded(toMoveCont)
	c.Assert(err, check.ErrorMatches, "Error trying to insert container healing event, healing aborted: event throttled, limit for healer on container \".*?\" is 3 every 5m0s")
}
Ejemplo n.º 8
0
func (a *autoScaleConfig) runScalerInNodes(pool string, nodes []*cluster.Node) {
	evt, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypePool, Value: pool},
		InternalKind: autoScaleEventKind,
		Allowed:      event.Allowed(permission.PermPoolReadEvents, permission.Context(permission.CtxPool, pool)),
	})
	if err != nil {
		if _, ok := err.(event.ErrEventLocked); ok {
			a.logDebug("skipping already running for: %s", pool)
		} else {
			a.logError("error creating scale event %s: %s", pool, err.Error())
		}
		return
	}
	evt.SetLogWriter(a.writer)
	var retErr error
	var sResult *scalerResult
	var evtNodes []cluster.Node
	var rule *autoScaleRule
	defer func() {
		if retErr != nil {
			evt.Logf(retErr.Error())
		}
		if (sResult == nil && retErr == nil) || (sResult != nil && sResult.NoAction()) {
			evt.Logf("nothing to do for %q: %q", poolMetadataName, pool)
			evt.Abort()
		} else {
			evt.DoneCustomData(retErr, evtCustomData{
				Result: sResult,
				Nodes:  evtNodes,
				Rule:   rule,
			})
		}
	}()
	rule, err = autoScaleRuleForMetadata(pool)
	if err == mgo.ErrNotFound {
		rule, err = autoScaleRuleForMetadata("")
	}
	if err != nil {
		if err != mgo.ErrNotFound {
			retErr = errors.Wrapf(err, "unable to fetch auto scale rules for %s", pool)
			return
		}
		evt.Logf("no auto scale rule for %s", pool)
		return
	}
	if !rule.Enabled {
		evt.Logf("auto scale rule disabled for %s", pool)
		return
	}
	scaler, err := a.scalerForRule(rule)
	if err != nil {
		retErr = errors.Wrapf(err, "error getting scaler for %s", pool)
		return
	}
	evt.Logf("running scaler %T for %q: %q", scaler, poolMetadataName, pool)
	sResult, err = scaler.scale(pool, nodes)
	if err != nil {
		if _, ok := err.(errAppNotLocked); ok {
			evt.Logf("aborting scaler for now, gonna retry later: %s", err)
			return
		}
		retErr = errors.Wrapf(err, "error scaling group %s", pool)
		return
	}
	if sResult.ToAdd > 0 {
		evt.Logf("running event \"add\" for %q: %#v", pool, sResult)
		evtNodes, err = a.addMultipleNodes(evt, nodes, sResult.ToAdd)
		if err != nil {
			if len(evtNodes) == 0 {
				retErr = err
				return
			}
			evt.Logf("not all required nodes were created: %s", err)
		}
	} else if len(sResult.ToRemove) > 0 {
		evt.Logf("running event \"remove\" for %q: %#v", pool, sResult)
		evtNodes = sResult.ToRemove
		err = a.removeMultipleNodes(evt, sResult.ToRemove)
		if err != nil {
			retErr = err
			return
		}
	}
	if !rule.PreventRebalance {
		err := a.rebalanceIfNeeded(evt, pool, nodes, sResult)
		if err != nil {
			if sResult.IsRebalanceOnly() {
				retErr = err
			} else {
				evt.Logf("unable to rebalance: %s", err.Error())
			}
		}
	}
}
Ejemplo n.º 9
0
func (s *S) TestListFilterMany(c *check.C) {
	var allEvts []event.Event
	var create = func(opts *event.Opts) {
		evt, err := event.New(opts)
		c.Assert(err, check.IsNil)
		allEvts = append(allEvts, *evt)
	}
	var createi = func(opts *event.Opts) {
		evt, err := event.NewInternal(opts)
		c.Assert(err, check.IsNil)
		allEvts = append(allEvts, *evt)
	}
	var checkFilters = func(f *event.Filter, expected interface{}) {
		evts, err := event.List(f)
		c.Assert(err, check.IsNil)
		c.Assert(evts, eventtest.EvtEquals, expected)
	}
	create(&event.Opts{
		Target:  event.Target{Type: "app", Value: "myapp"},
		Kind:    permission.PermAppUpdateEnvSet,
		Owner:   s.token,
		Allowed: event.Allowed(permission.PermAppReadEvents, permission.Context(permission.CtxApp, "myapp")),
	})
	time.Sleep(100 * time.Millisecond)
	t0 := time.Now().UTC()
	create(&event.Opts{
		Target:  event.Target{Type: "app", Value: "myapp2"},
		Kind:    permission.PermAppUpdateEnvSet,
		Owner:   s.token,
		Allowed: event.Allowed(permission.PermAppAdmin),
	})
	t05 := time.Now().UTC()
	time.Sleep(100 * time.Millisecond)
	create(&event.Opts{
		Target:  event.Target{Type: "app2", Value: "myapp"},
		Kind:    permission.PermAppUpdateEnvSet,
		Owner:   s.token,
		Allowed: event.Allowed(permission.PermAppAdmin),
	})
	t1 := time.Now().UTC()
	time.Sleep(100 * time.Millisecond)
	createi(&event.Opts{
		Target:       event.Target{Type: "node", Value: "http://10.0.1.1"},
		InternalKind: "healer",
		Allowed:      event.Allowed(permission.PermAppAdmin),
	})
	createi(&event.Opts{
		Target:       event.Target{Type: "node", Value: "http://10.0.1.2"},
		InternalKind: "healer",
		Allowed:      event.Allowed(permission.PermAppAdmin),
	})
	createi(&event.Opts{
		Target:       event.Target{Type: "nodex", Value: "http://10.0.1.3"},
		InternalKind: "healer",
		Allowed:      event.Allowed(permission.PermAppAdmin),
	})
	err := event.MarkAsRemoved(event.Target{Type: "nodex", Value: "http://10.0.1.3"})
	c.Assert(err, check.IsNil)
	allEvts[len(allEvts)-2].Done(nil)
	allEvts[len(allEvts)-3].Done(errors.New("my err"))
	checkFilters(&event.Filter{Sort: "_id"}, allEvts[:len(allEvts)-1])
	checkFilters(&event.Filter{Running: boolPtr(false), Sort: "_id"}, allEvts[len(allEvts)-3:len(allEvts)-1])
	checkFilters(&event.Filter{Running: boolPtr(true), Sort: "_id"}, allEvts[:len(allEvts)-3])
	checkFilters(&event.Filter{ErrorOnly: true, Sort: "_id"}, allEvts[len(allEvts)-3])
	checkFilters(&event.Filter{Target: event.Target{Type: "app"}, Sort: "_id"}, []event.Event{allEvts[0], allEvts[1]})
	checkFilters(&event.Filter{Target: event.Target{Type: "app", Value: "myapp"}}, allEvts[0])
	checkFilters(&event.Filter{KindType: event.KindTypeInternal, Sort: "_id"}, allEvts[3:len(allEvts)-1])
	checkFilters(&event.Filter{KindType: event.KindTypePermission, Sort: "_id"}, allEvts[:3])
	checkFilters(&event.Filter{KindType: event.KindTypePermission, KindName: "kind"}, nil)
	checkFilters(&event.Filter{KindType: event.KindTypeInternal, KindName: "healer", Sort: "_id"}, allEvts[3:len(allEvts)-1])
	checkFilters(&event.Filter{OwnerType: event.OwnerTypeUser, Sort: "_id"}, allEvts[:3])
	checkFilters(&event.Filter{OwnerType: event.OwnerTypeInternal, Sort: "_id"}, allEvts[3:len(allEvts)-1])
	checkFilters(&event.Filter{OwnerType: event.OwnerTypeUser, OwnerName: s.token.GetUserName(), Sort: "_id"}, allEvts[:3])
	checkFilters(&event.Filter{Since: t0, Sort: "_id"}, allEvts[1:len(allEvts)-1])
	checkFilters(&event.Filter{Until: t05, Sort: "_id"}, allEvts[:2])
	checkFilters(&event.Filter{Since: t0, Until: t1, Sort: "_id"}, allEvts[1:3])
	checkFilters(&event.Filter{Limit: 2, Sort: "_id"}, allEvts[:2])
	checkFilters(&event.Filter{Limit: 1, Sort: "-_id"}, allEvts[len(allEvts)-2])
	checkFilters(&event.Filter{Target: event.Target{Type: "nodex"}}, allEvts[:0])
	checkFilters(&event.Filter{Target: event.Target{Type: "nodex"}, IncludeRemoved: true}, allEvts[5:6])
	checkFilters(&event.Filter{AllowedTargets: []event.TargetFilter{}, Sort: "_id"}, allEvts[:0])
	checkFilters(&event.Filter{AllowedTargets: []event.TargetFilter{
		{Type: "app"},
	}, Sort: "_id"}, allEvts[:2])
	checkFilters(&event.Filter{AllowedTargets: []event.TargetFilter{
		{Type: "app", Values: []string{}},
	}, Sort: "_id"}, allEvts[:0])
	checkFilters(&event.Filter{AllowedTargets: []event.TargetFilter{
		{Type: "app", Values: []string{"myapp", "myapp2"}},
	}, Sort: "_id"}, allEvts[:2])
	checkFilters(&event.Filter{AllowedTargets: []event.TargetFilter{
		{Type: "app", Values: []string{"myapp"}},
		{Type: "node", Values: []string{"http://10.0.1.2"}},
	}, Sort: "_id"}, []event.Event{allEvts[0], allEvts[4]})
	checkFilters(&event.Filter{Permissions: []permission.Permission{
		{Scheme: permission.PermAll, Context: permission.Context(permission.CtxGlobal, "")},
	}, Sort: "_id"}, allEvts[:len(allEvts)-1])
	checkFilters(&event.Filter{Permissions: []permission.Permission{
		{Scheme: permission.PermAll},
	}, Sort: "_id"}, allEvts[:0])
	checkFilters(&event.Filter{Permissions: []permission.Permission{
		{Scheme: permission.PermAppRead, Context: permission.Context(permission.CtxApp, "myapp")},
		{Scheme: permission.PermAppRead, Context: permission.Context(permission.CtxApp, "invalid-app")},
	}, Sort: "_id"}, allEvts[:1])
	checkFilters(&event.Filter{Permissions: []permission.Permission{
		{Scheme: permission.PermAppRead, Context: permission.Context(permission.CtxApp, "invalid-app")},
	}, Sort: "_id"}, allEvts[:0])
}