Example #1
0
func (s *endpointServiceImpl) send(ctx context.Context, data []byte) error {
	ctx = log.SetField(ctx, "endpointURL", s.url)
	return retryCall(ctx, "endpoint.send", func() error {
		startTime := clock.Now(ctx)

		log.Debugf(ctx, "Pushing message to endpoint.")
		req, err := http.NewRequest("POST", s.url, bytes.NewReader(data))
		if err != nil {
			log.Errorf(log.SetError(ctx, err), "Failed to create HTTP request.")
			return err
		}
		req.Header.Add("content-type", protobufContentType)
		req.Header.Add("user-agent", monitoringEndpointUserAgent)

		resp, err := s.client.Do(req)
		if err != nil {
			// Treat a client error as transient.
			log.Warningf(log.SetError(ctx, err), "Failed proxy client request.")
			return errors.WrapTransient(err)
		}
		defer resp.Body.Close()

		// Read the full response body. This will enable us to re-use the
		// connection.
		bodyData, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Errorf(log.SetError(ctx, err), "Error during endpoint connection.")
			return errors.WrapTransient(err)
		}

		log.Fields{
			"status":        resp.Status,
			"statusCode":    resp.StatusCode,
			"headers":       resp.Header,
			"contentLength": resp.ContentLength,
			"body":          string(bodyData),
			"duration":      clock.Now(ctx).Sub(startTime),
		}.Debugf(ctx, "Received HTTP response from endpoint.")

		if http.StatusOK <= resp.StatusCode && resp.StatusCode < http.StatusMultipleChoices {
			log.Debugf(ctx, "Message pushed successfully.")
			return nil
		}

		err = fmt.Errorf("http: server error (%d)", resp.StatusCode)
		if resp.StatusCode >= http.StatusInternalServerError {
			err = errors.WrapTransient(err)
		}

		log.Fields{
			log.ErrorKey: err,
			"status":     resp.Status,
			"statusCode": resp.StatusCode,
		}.Warningf(ctx, "Proxy error.")
		return err
	})
}
Example #2
0
// Next implements the Iterator interface.
func (i *Limited) Next(ctx context.Context, _ error) time.Duration {
	if i.Retries == 0 {
		return Stop
	}
	i.Retries--

	// If there is a maximum total time, enforce it.
	if i.MaxTotal > 0 {
		now := clock.Now(ctx)
		if i.startTime.IsZero() {
			i.startTime = now
		}

		var elapsed time.Duration
		if now.After(i.startTime) {
			elapsed = now.Sub(i.startTime)
		}

		// Remaining time is the difference between total allowed time and elapsed
		// time.
		remaining := i.MaxTotal - elapsed
		if remaining <= 0 {
			// No more time!
			i.Retries = 0
			return Stop
		}
	}

	return i.Delay
}
Example #3
0
func (m *memcacheImpl) Increment(key string, delta int64, initialValue *uint64) (uint64, error) {
	now := clock.Now(m.ctx)

	m.data.lock.Lock()
	defer m.data.lock.Unlock()

	cur := uint64(0)
	if initialValue == nil {
		curItm, err := m.data.retrieveLocked(now, key)
		if err != nil {
			return 0, err
		}
		if len(curItm.value) != 8 {
			return 0, errors.New("memcache Increment: got invalid current value")
		}
		cur = binary.LittleEndian.Uint64(curItm.value)
	} else {
		cur = *initialValue
	}
	if delta < 0 {
		if uint64(-delta) > cur {
			cur = 0
		} else {
			cur -= uint64(-delta)
		}
	} else {
		cur += uint64(delta)
	}

	newval := make([]byte, 8)
	binary.LittleEndian.PutUint64(newval, cur)
	m.data.setItemLocked(now, m.NewItem(key).SetValue(newval))

	return cur, nil
}
Example #4
0
// pullAckMessages pulls a set of messages from the configured Subscription.
// If no messages are available, errNoMessages will be returned.
//
// handler is a method that returns true if there was a transient failure,
// indicating that the messages shouldn't be ACK'd.
func (p *pubsubClient) pullAckMessages(ctx context.Context, handler func([]*pubsub.Message)) error {
	var err error
	var msgs []*pubsub.Message
	ackCount := 0

	// Report the duration of a Pull/ACK cycle.
	startTime := clock.Now(ctx)
	defer func() {
		duration := clock.Now(ctx).Sub(startTime)
		log.Fields{
			"count":    len(msgs),
			"ackCount": ackCount,
			"duration": duration,
		}.Infof(ctx, "Pull/ACK cycle complete.")
	}()

	err = retryCall(ctx, "Pull()", func() error {
		var err error
		msgs, err = p.service.Pull(p.subscription, p.batchSize)
		return p.wrapTransient(err)
	})
	log.Fields{
		log.ErrorKey: err,
		"duration":   clock.Now(ctx).Sub(startTime),
		"count":      len(msgs),
	}.Debugf(ctx, "Pull() complete.")

	if err != nil {
		return err
	}

	if len(msgs) == 0 {
		return errNoMessages
	}

	defer func() {
		ackCount, err = p.ackMessages(ctx, msgs)
		if err != nil {
			log.Warningf(log.SetError(ctx, err), "Failed to ACK messages!")
		}
	}()
	handler(msgs)
	return nil
}
Example #5
0
func (m *memcacheImpl) SetMulti(items []mc.Item, cb mc.RawCB) error {
	now := clock.Now(m.ctx)
	doCBs(items, cb, func(itm mc.Item) error {
		m.data.lock.Lock()
		defer m.data.lock.Unlock()
		m.data.setItemLocked(now, itm)
		return nil
	})
	return nil
}
Example #6
0
func (m *memcacheImpl) AddMulti(items []mc.Item, cb mc.RawCB) error {
	now := clock.Now(m.ctx)
	doCBs(items, cb, func(itm mc.Item) error {
		m.data.lock.Lock()
		defer m.data.lock.Unlock()
		if !m.data.hasItemLocked(now, itm.Key()) {
			m.data.setItemLocked(now, itm)
			return nil
		}
		return mc.ErrNotStored
	})
	return nil
}
Example #7
0
func (l *boundCloudLogger) LogCall(level log.Level, calldepth int, f string, args []interface{}) {
	if len(f) == 0 || !log.IsLogging(l.ctx, level) {
		return
	}

	l.logger.logC <- &logEntry{
		timestamp: clock.Now(l.ctx),
		level:     level,
		fmt:       f,
		args:      args,
		fields:    log.GetFields(l.ctx),
	}
}
Example #8
0
func (t *taskQueueData) prepTask(c context.Context, ns string, task *tq.Task, queueName string) (*tq.Task, error) {
	toSched := task.Duplicate()

	if toSched.Path == "" {
		toSched.Path = "/_ah/queue/" + queueName
	}

	if toSched.ETA.IsZero() {
		toSched.ETA = clock.Now(c).Add(toSched.Delay)
	} else if toSched.Delay != 0 {
		panic("taskqueue: both Delay and ETA are set")
	}
	toSched.Delay = 0

	switch toSched.Method {
	// Methods that can have payloads.
	case "":
		toSched.Method = "POST"
		fallthrough
	case "POST", "PUT", "PULL":
		break

	// Methods that can not have payloads.
	case "GET", "HEAD", "DELETE":
		toSched.Payload = nil

	default:
		return nil, fmt.Errorf("taskqueue: bad method %q", toSched.Method)
	}

	if _, ok := toSched.Header[currentNamespace]; !ok {
		if ns != "" {
			if toSched.Header == nil {
				toSched.Header = http.Header{}
			}
			toSched.Header[currentNamespace] = []string{ns}
		}
	}
	// TODO(riannucci): implement DefaultNamespace

	if toSched.Name == "" {
		toSched.Name = mkName(c, "", t.named[queueName])
	} else {
		if !validTaskName.MatchString(toSched.Name) {
			return nil, errors.New("INVALID_TASK_NAME")
		}
	}

	return toSched, nil
}
Example #9
0
// ackMessages ACKs the supplied messages. If a message is nil, it will be
// ignored.
func (p *pubsubClient) ackMessages(ctx context.Context, messages []*pubsub.Message) (int, error) {
	messageIds := make([]string, 0, len(messages))
	skipped := 0
	for _, msg := range messages {
		if msg != nil {
			messageIds = append(messageIds, msg.AckID)
		} else {
			skipped++
		}
	}
	if len(messageIds) == 0 {
		return 0, nil
	}

	startTime := clock.Now(ctx)
	ctx = log.SetFields(ctx, log.Fields{
		"count":   len(messageIds),
		"skipped": skipped,
	})
	err := retryCall(ctx, "Ack()", func() error {
		return p.wrapTransient(p.service.Ack(p.subscription, messageIds))
	})
	duration := clock.Now(ctx).Sub(startTime)

	if err != nil {
		log.Fields{
			log.ErrorKey: err,
			"duration":   duration,
		}.Errorf(ctx, "Failed to ACK messages.")
		return 0, err
	}

	log.Fields{
		"duration": duration,
	}.Debugf(ctx, "Successfully ACK messages.")
	return len(messageIds), nil
}
Example #10
0
func (m *memcacheImpl) CompareAndSwapMulti(items []mc.Item, cb mc.RawCB) error {
	now := clock.Now(m.ctx)
	doCBs(items, cb, func(itm mc.Item) error {
		m.data.lock.Lock()
		defer m.data.lock.Unlock()

		if cur, err := m.data.retrieveLocked(now, itm.Key()); err == nil {
			casid := uint64(0)
			if mi, ok := itm.(*mcItem); ok && mi != nil {
				casid = mi.CasID
			}

			if cur.casID == casid {
				m.data.setItemLocked(now, itm)
			} else {
				return mc.ErrCASConflict
			}
			return nil
		}
		return mc.ErrNotStored
	})
	return nil
}
Example #11
0
func (l *boundCloudLogger) LogCall(level logging.Level, calldepth int, f string, args []interface{}) {
	if len(f) == 0 {
		return
	}

	text := fmt.Sprintf(f, args...)
	fields := logging.GetFields(l.ctx)
	if len(fields) > 0 {
		text = text + " " + fields.FieldString(true)
	}

	// Add logging fields to labels.
	entry := cloudlogging.Entry{
		Timestamp:   clock.Now(l.ctx),
		Severity:    l.getSeverity(level),
		Labels:      make(map[string]string, len(fields)),
		TextPayload: text,
	}

	// Populate Labels.
	for k, v := range fields {
		val := ""
		if l.FieldConverter != nil {
			val = l.FieldConverter(v)
		} else {
			val = fmt.Sprintf("%v", v)
		}
		entry.Labels[k] = val
	}

	// Generate an InsertID, if we're configured with a base.
	if l.InsertIDBase != "" {
		entry.InsertID = l.generateInsertID()
	}

	l.client.PushEntries([]*cloudlogging.Entry{&entry})
}
Example #12
0
func (m *memcacheImpl) DeleteMulti(keys []string, cb mc.RawCB) error {
	now := clock.Now(m.ctx)

	errs := make([]error, len(keys))

	for i, k := range keys {
		errs[i] = func() error {
			m.data.lock.Lock()
			defer m.data.lock.Unlock()
			_, err := m.data.retrieveLocked(now, k)
			if err != nil {
				return err
			}
			m.data.delItemLocked(k)
			return nil
		}()
	}

	for _, e := range errs {
		cb(e)
	}

	return nil
}
Example #13
0
func (m *memcacheImpl) GetMulti(keys []string, cb mc.RawItemCB) error {
	now := clock.Now(m.ctx)

	itms := make([]mc.Item, len(keys))
	errs := make([]error, len(keys))

	for i, k := range keys {
		itms[i], errs[i] = func() (mc.Item, error) {
			m.data.lock.RLock()
			defer m.data.lock.RUnlock()
			val, err := m.data.retrieveLocked(now, k)
			if err != nil {
				return nil, err
			}
			return val.toUserItem(k), nil
		}()
	}

	for i, itm := range itms {
		cb(itm, errs[i])
	}

	return nil
}
Example #14
0
// IsGloballyEnabled checks to see if this filter is enabled globally.
//
// This checks InstanceEnabledStatic, as well as polls the datastore entity
//   /dscache,1 (a GlobalConfig instance)
// Once every GlobalEnabledCheckInterval.
//
// For correctness, any error encountered returns true. If this assumed false,
// then Put operations might incorrectly invalidate the cache.
func IsGloballyEnabled(c context.Context) bool {
	if !InstanceEnabledStatic {
		return false
	}

	now := clock.Now(c)

	globalEnabledLock.RLock()
	nextCheck := globalEnabledNextCheck
	enabledVal := globalEnabled
	globalEnabledLock.RUnlock()

	if now.Before(nextCheck) {
		return enabledVal
	}

	globalEnabledLock.Lock()
	defer globalEnabledLock.Unlock()
	// just in case we raced
	if now.Before(globalEnabledNextCheck) {
		return globalEnabled
	}

	// always go to the default namespace
	c, err := info.Get(c).Namespace("")
	if err != nil {
		return true
	}
	cfg := &GlobalConfig{Enable: true}
	if err := datastore.Get(c).Get(cfg); err != nil && err != datastore.ErrNoSuchEntity {
		return true
	}
	globalEnabled = cfg.Enable
	globalEnabledNextCheck = now.Add(GlobalEnabledCheckInterval)
	return globalEnabled
}
Example #15
0
// Get gets a *"math/rand".Rand from the context. If one hasn't been
// set, this creates a new Rand object with a Source initialized from the
// current time clock.Now(c).UnixNano().
func Get(c context.Context) *rand.Rand {
	if f, ok := c.Value(mathRandKey).(Factory); ok && f != nil {
		return f(c)
	}
	return rand.New(rand.NewSource(clock.Now(c).UnixNano()))
}
Example #16
0
func TestTaskQueue(t *testing.T) {
	t.Parallel()

	Convey("TaskQueue", t, func() {
		now := time.Date(2000, time.January, 1, 1, 1, 1, 1, time.UTC)
		c, tc := testclock.UseTime(context.Background(), now)
		c = mathrand.Set(c, rand.New(rand.NewSource(clock.Now(c).UnixNano())))
		c = Use(c)

		tq := tqS.Get(c)
		tqt := tq.Testable()
		So(tqt, ShouldNotBeNil)

		So(tq, ShouldNotBeNil)

		Convey("implements TQMultiReadWriter", func() {
			Convey("Add", func() {
				t := tq.NewTask("/hello/world")

				Convey("works", func() {
					t.Delay = 4 * time.Second
					t.Header = http.Header{}
					t.Header.Add("Cat", "tabby")
					t.Payload = []byte("watwatwat")
					t.RetryOptions = &tqS.RetryOptions{AgeLimit: 7 * time.Second}
					So(tq.Add(t, ""), ShouldBeNil)

					name := "Z_UjshxM9ecyMQfGbZmUGOEcgxWU0_5CGLl_-RntudwAw2DqQ5-58bzJiWQN4OKzeuUb9O4JrPkUw2rOvk2Ax46THojnQ6avBQgZdrKcJmrwQ6o4qKfJdiyUbGXvy691yRfzLeQhs6cBhWrgf3wH-VPMcA4SC-zlbJ2U8An7I0zJQA5nBFnMNoMgT-2peGoay3rCSbj4z9VFFm9kS_i6JCaQH518ujLDSNCYdjTq6B6lcWrZAh0U_q3a1S2nXEwrKiw_t9MTNQFgAQZWyGBbvZQPmeRYtu8SPaWzTfd25v_YWgBuVL2rRSPSMvlDwE04nNdtvVzE8vNNiA1zRimmdzKeqATQF9_ReUvj4D7U8dcS703DZWfKMBLgBffY9jqCassOOOw77V72Oq5EVauUw3Qw0L6bBsfM9FtahTKUdabzRZjXUoze3EK4KXPt3-wdidau-8JrVf2XFocjjZbwHoxcGvbtT3b4nGLDlgwdC00bwaFBZWff"
					So(tqt.GetScheduledTasks()["default"][name], ShouldResemble, &tqS.Task{
						ETA:          now.Add(4 * time.Second),
						Header:       http.Header{"Cat": []string{"tabby"}},
						Method:       "POST",
						Name:         name,
						Path:         "/hello/world",
						Payload:      []byte("watwatwat"),
						RetryOptions: &tqS.RetryOptions{AgeLimit: 7 * time.Second},
					})
				})

				Convey("picks up namespace", func() {
					c, err := info.Get(c).Namespace("coolNamespace")
					So(err, ShouldBeNil)
					tq = tqS.Get(c)

					t := tq.NewTask("")
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Header, ShouldResemble, http.Header{
						"X-Appengine-Current-Namespace": {"coolNamespace"},
					})

				})

				Convey("cannot add to bad queues", func() {
					So(tq.Add(nil, "waaat").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")

					Convey("but you can add Queues when testing", func() {
						tqt.CreateQueue("waaat")
						So(tq.Add(t, "waaat"), ShouldBeNil)

						Convey("you just can't add them twice", func() {
							So(func() { tqt.CreateQueue("waaat") }, ShouldPanic)
						})
					})
				})

				Convey("supplies a URL if it's missing", func() {
					t.Path = ""
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Path, ShouldEqual, "/_ah/queue/default")
				})

				Convey("cannot add twice", func() {
					t.Name = "bob"
					So(tq.Add(t, ""), ShouldBeNil)

					// can't add the same one twice!
					So(tq.Add(t, ""), ShouldEqual, tqS.ErrTaskAlreadyAdded)
				})

				Convey("cannot add deleted task", func() {
					t.Name = "bob"
					So(tq.Add(t, ""), ShouldBeNil)

					So(tq.Delete(t, ""), ShouldBeNil)

					// can't add a deleted task!
					So(tq.Add(t, ""), ShouldEqual, tqS.ErrTaskAlreadyAdded)
				})

				Convey("cannot set ETA+Delay", func() {
					t.ETA = clock.Now(c).Add(time.Hour)
					tc.Add(time.Second)
					t.Delay = time.Hour
					So(func() {
						So(tq.Add(t, ""), ShouldBeNil)
					}, ShouldPanic)
				})

				Convey("must use a reasonable method", func() {
					t.Method = "Crystal"
					So(tq.Add(t, "").Error(), ShouldContainSubstring, "bad method")
				})

				Convey("payload gets dumped for non POST/PUT methods", func() {
					t.Method = "HEAD"
					t.Payload = []byte("coool")
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Payload, ShouldBeNil)
				})

				Convey("invalid names are rejected", func() {
					t.Name = "happy times"
					So(tq.Add(t, "").Error(), ShouldContainSubstring, "INVALID_TASK_NAME")
				})

				Convey("AddMulti also works", func() {
					t2 := t.Duplicate()
					t2.Path = "/hi/city"

					expect := []*tqS.Task{t, t2}

					So(tq.AddMulti(expect, "default"), ShouldBeNil)
					So(len(expect), ShouldEqual, 2)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 2)

					for i := range expect {
						Convey(fmt.Sprintf("task %d: %s", i, expect[i].Path), func() {
							So(expect[i].Method, ShouldEqual, "POST")
							So(expect[i].ETA, ShouldHappenOnOrBefore, now)
							So(len(expect[i].Name), ShouldEqual, 500)
						})
					}

					Convey("stats work too", func() {
						delay := -time.Second * 400

						t := tq.NewTask("/somewhere")
						t.Delay = delay
						So(tq.Add(t, ""), ShouldBeNil)

						stats, err := tq.Stats("")
						So(err, ShouldBeNil)
						So(stats[0].Tasks, ShouldEqual, 3)
						So(stats[0].OldestETA, ShouldHappenOnOrBefore, clock.Now(c).Add(delay))

						_, err = tq.Stats("noexist")
						So(err.Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
					})

					Convey("can purge all tasks", func() {
						So(tq.Add(&tqS.Task{Path: "/wut/nerbs"}, ""), ShouldBeNil)
						So(tq.Purge(""), ShouldBeNil)

						So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

						Convey("purging a queue which DNE fails", func() {
							So(tq.Purge("noexist").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
						})
					})

				})
			})

			Convey("Delete", func() {
				t := &tqS.Task{Path: "/hello/world"}
				So(tq.Add(t, ""), ShouldBeNil)

				Convey("works", func() {
					err := tq.Delete(t, "")
					So(err, ShouldBeNil)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 1)
					So(tqt.GetTombstonedTasks()["default"][t.Name], ShouldResemble, t)
				})

				Convey("cannot delete a task twice", func() {
					So(tq.Delete(t, ""), ShouldBeNil)

					So(tq.Delete(t, "").Error(), ShouldContainSubstring, "TOMBSTONED_TASK")

					Convey("but you can if you do a reset", func() {
						tqt.ResetTasks()

						So(tq.Add(t, ""), ShouldBeNil)
						So(tq.Delete(t, ""), ShouldBeNil)
					})
				})

				Convey("cannot delete from bogus queues", func() {
					err := tq.Delete(t, "wat")
					So(err.Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
				})

				Convey("cannot delete a missing task", func() {
					t.Name = "tarntioarenstyw"
					err := tq.Delete(t, "")
					So(err.Error(), ShouldContainSubstring, "UNKNOWN_TASK")
				})

				Convey("DeleteMulti also works", func() {
					t2 := t.Duplicate()
					t2.Name = ""
					t2.Path = "/hi/city"
					So(tq.Add(t2, ""), ShouldBeNil)

					Convey("usually works", func() {
						So(tq.DeleteMulti([]*tqS.Task{t, t2}, ""), ShouldBeNil)
						So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 2)
					})
				})
			})
		})

		Convey("works with transactions", func() {
			t := &tqS.Task{Path: "/hello/world"}
			So(tq.Add(t, ""), ShouldBeNil)

			t2 := &tqS.Task{Path: "/hi/city"}
			So(tq.Add(t2, ""), ShouldBeNil)

			So(tq.Delete(t2, ""), ShouldBeNil)

			Convey("can view regular tasks", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tqt := tqS.GetRaw(c).Testable()

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("can add a new task", func() {
				t3 := &tqS.Task{Path: "/sandwitch/victory"}

				err := dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tq := tqS.Get(c)
					tqt := tq.Testable()

					So(tq.Add(t3, ""), ShouldBeNil)
					So(t3.Name, ShouldEqual, "")

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"][0], ShouldResemble, t3)
					return nil
				}, nil)
				So(err, ShouldBeNil)

				for _, tsk := range tqt.GetScheduledTasks()["default"] {
					if tsk.Name == t.Name {
						So(tsk, ShouldResemble, t)
					} else {
						tsk.Name = ""
						So(tsk, ShouldResemble, t3)
					}
				}

				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

			Convey("can add a new task (but reset the state in a test)", func() {
				t3 := &tqS.Task{Path: "/sandwitch/victory"}

				ttq := tqS.Interface(nil)

				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					ttq = tqS.Get(c)
					tqt := ttq.Testable()

					So(ttq.Add(t3, ""), ShouldBeNil)

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"][0], ShouldResemble, t3)

					tqt.ResetTasks()

					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

					return nil
				}, nil), ShouldBeNil)

				So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
				So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
				So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

				Convey("and reusing a closed context is bad times", func() {
					So(ttq.Add(nil, "").Error(), ShouldContainSubstring, "expired")
				})
			})

			Convey("you can AddMulti as well", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tq := tqS.Get(c)
					tqt := tq.Testable()

					t.Name = ""
					tasks := []*tqS.Task{t.Duplicate(), t.Duplicate(), t.Duplicate()}
					So(tq.AddMulti(tasks, ""), ShouldBeNil)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 1)
					So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 3)
					return nil
				}, nil), ShouldBeNil)
				So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 4)
				So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)
			})

			Convey("unless you add too many things", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					for i := 0; i < 5; i++ {
						So(tqS.Get(c).Add(t.Duplicate(), ""), ShouldBeNil)
					}
					So(tqS.Get(c).Add(t, "").Error(), ShouldContainSubstring, "BAD_REQUEST")
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("unless you Add to a bad queue", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.Get(c).Add(t, "meat").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")

					Convey("unless you add it!", func() {
						tqS.GetRaw(c).Testable().CreateQueue("meat")
						So(tqS.Get(c).Add(t, "meat"), ShouldBeNil)
					})

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("No other features are available, however", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.Get(c).Delete(t, "").Error(), ShouldContainSubstring, "cannot DeleteMulti from a transaction")
					So(tqS.Get(c).Purge("").Error(), ShouldContainSubstring, "cannot Purge from a transaction")
					_, err := tqS.Get(c).Stats("")
					So(err.Error(), ShouldContainSubstring, "cannot Stats from a transaction")
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("can get the non-transactional taskqueue context though", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.GetNoTxn(c).Delete(t, ""), ShouldBeNil)
					So(tqS.GetNoTxn(c).Purge(""), ShouldBeNil)
					_, err := tqS.GetNoTxn(c).Stats("")
					So(err, ShouldBeNil)
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("adding a new task only happens if we don't errout", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					t3 := tq.NewTask("/sandwitch/victory")
					So(tqS.Get(c).Add(t3, ""), ShouldBeNil)
					return fmt.Errorf("nooooo")
				}, nil), ShouldErrLike, "nooooo")

				So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

			Convey("likewise, a panic doesn't schedule anything", func() {
				func() {
					defer func() { _ = recover() }()
					So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
						tq := tqS.Get(c)

						So(tq.Add(tq.NewTask("/sandwitch/victory"), ""), ShouldBeNil)

						panic(fmt.Errorf("nooooo"))
					}, nil), ShouldBeNil)
				}()

				So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

		})
	})
}