func (c *conn) serve() {
defer c.cancelAll()
for {
t, err := c.readBuf()
if err != nil {
if err != os.EOF {
log.Println(err)
}
return
}
verb := pb.GetInt32((*int32)(t.Verb))
f, ok := ops[verb]
if !ok {
var r R
r.ErrCode = newResponse_Err(response_UNKNOWN_VERB)
c.respond(t, Valid|Done, nil, &r)
continue
}
tag := pb.GetInt32((*int32)(t.Tag))
tx := newTxn()
c.tl.Lock()
c.tx[tag] = tx
c.tl.Unlock()
f(c, t, tx)
}
}
func (c *conn) readResponses() {
defer c.close()
for {
r, err := c.readR()
if err != nil {
c.clk.Lock()
if c.err == nil {
c.err = err
}
c.clk.Unlock()
return
}
if r.ErrCode != nil && *r.ErrCode == proto.Response_REDIRECT {
c.redirectAddr = pb.GetString(r.ErrDetail)
c.redirected = true
}
tag := pb.GetInt32(r.Tag)
flags := pb.GetInt32(r.Flags)
c.cblk.Lock()
ch, ok := c.cb[tag]
if ok && ch == nil {
c.cblk.Unlock()
continue
}
if flags&Done != 0 {
c.cb[tag] = nil, false
}
c.cblk.Unlock()
if !ok {
log.Printf(
"%v unexpected: tag=%d flags=%d rev=%d path=%q value=%v len=%d err_code=%v err_detail=%q",
ch,
tag,
flags,
pb.GetInt64(r.Rev),
pb.GetString(r.Path),
r.Value,
pb.GetInt32(r.Len),
pb.GetInt32((*int32)(r.ErrCode)),
pb.GetString(r.ErrDetail),
)
continue
}
if flags&Valid != 0 {
ch <- r
}
if flags&Done != 0 {
close(ch)
}
}
}
func (c *conn) getdir(t *T, tx txn) {
path := pb.GetString(t.Path)
if g := c.getterFor(t); g != nil {
go func() {
ents, rev := g.Get(path)
if rev == store.Missing {
c.respond(t, Valid|Done, nil, noEnt)
return
}
if rev != store.Dir {
c.respond(t, Valid|Done, nil, notDir)
return
}
offset := int(pb.GetInt32(t.Offset))
limit := int(pb.GetInt32(t.Limit))
if limit <= 0 {
limit = len(ents)
}
if offset < 0 {
offset = 0
}
end := offset + limit
if end > len(ents) {
end = len(ents)
}
for _, e := range ents[offset:end] {
select {
case <-tx.cancel:
c.closeTxn(*t.Tag)
return
default:
}
c.respond(t, Valid, tx.cancel, &R{Path: &e})
}
c.respond(t, Done, nil, &R{})
}()
}
}
func (c *conn) walk(t *T, tx txn) {
pat := pb.GetString(t.Path)
glob, err := store.CompileGlob(pat)
if err != nil {
c.respond(t, Valid|Done, nil, errResponse(err))
return
}
offset := pb.GetInt32(t.Offset)
if offset < 0 {
c.respond(t, Valid|Done, nil, erange)
return
}
if g := c.getterFor(t); g != nil {
var r R
f := func(path, body string, rev int64) (stop bool) {
if offset == 0 {
r.Path = &path
r.Value = []byte(body)
r.Rev = &rev
return true
}
offset--
return false
}
if store.Walk(g, glob, f) {
c.respond(t, Set|Valid|Done, nil, &r)
} else {
c.respond(t, Valid|Done, nil, erange)
}
}
}
func (c *conn) getdir(t *T, tx txn) {
path := pb.GetString(t.Path)
if g := c.getterFor(t); g != nil {
ents, rev := g.Get(path)
if rev == store.Missing {
c.respond(t, Valid|Done, nil, noEnt)
return
}
if rev != store.Dir {
c.respond(t, Valid|Done, nil, notDir)
return
}
sort.SortStrings(ents)
offset := int(pb.GetInt32(t.Offset))
if offset < 0 || offset >= len(ents) {
c.respond(t, Valid|Done, nil, erange)
return
}
e := ents[offset]
c.respond(t, Valid|Done, tx.cancel, &R{Path: &e})
}
}
func (c *conn) events(t *T) (*Watch, os.Error) {
cb, err := c.send(t)
if err != nil {
return nil, err
}
evs := make(chan *Event)
w := &Watch{evs, c, cb, *t.Tag}
go func() {
for r := range cb {
var ev Event
if err := r.err(); err != nil {
ev.Err = err
} else {
ev.Rev = pb.GetInt64(r.Rev)
ev.Path = pb.GetString(r.Path)
ev.Body = r.Value
ev.Flag = pb.GetInt32(r.Flags)
}
evs <- &ev
}
close(evs)
}()
return w, nil
}
func (c *conn) walk(t *T, tx txn) {
pat := pb.GetString(t.Path)
glob, err := store.CompileGlob(pat)
if err != nil {
c.respond(t, Valid|Done, nil, errResponse(err))
return
}
offset := pb.GetInt32(t.Offset)
var limit int32 = math.MaxInt32
if t.Limit != nil {
limit = pb.GetInt32(t.Limit)
}
if g := c.getterFor(t); g != nil {
go func() {
f := func(path, body string, rev int64) (stop bool) {
select {
case <-tx.cancel:
c.closeTxn(*t.Tag)
return true
default:
}
if offset <= 0 && limit > 0 {
var r R
r.Path = &path
r.Value = []byte(body)
r.Rev = &rev
c.respond(t, Valid|Set, tx.cancel, &r)
limit--
}
offset--
return false
}
stopped := store.Walk(g, glob, f)
if !stopped {
c.respond(t, Done, nil, &R{})
}
}()
}
}
func (t *txn) run() {
verb := proto.GetInt32((*int32)(t.req.Verb))
if f, ok := ops[verb]; ok {
f(t)
} else {
t.respondErrCode(response_UNKNOWN_VERB)
}
}
func (cl *Client) Stat(path string, rev *int64) (int32, int64, os.Error) {
r, err := cl.retry(&T{Verb: stat, Path: &path, Rev: rev})
if err != nil {
return 0, 0, err
}
return pb.GetInt32(r.Len), pb.GetInt64(r.Rev), nil
}
// protoToItem converts a protocol buffer item to a Go struct.
func protoToItem(p *pb.MemcacheGetResponse_Item) *Item {
return &Item{
Key: string(p.Key),
Value: p.Value,
Flags: proto.GetUint32(p.Flags),
Expiration: proto.GetInt32(p.ExpiresInSeconds),
casID: proto.GetUint64(p.CasId),
}
}
func (c *conn) respond(t *T, flag int32, cc chan bool, r *R) {
r.Tag = t.Tag
r.Flags = pb.Int32(flag)
tag := pb.GetInt32(t.Tag)
if flag&Done != 0 {
c.closeTxn(tag)
}
if c.poisoned {
select {
case cc <- true:
default:
}
return
}
buf, err := pb.Marshal(r)
c.wl.Lock()
defer c.wl.Unlock()
if err != nil {
c.poisoned = true
select {
case cc <- true:
default:
}
log.Println(err)
return
}
err = binary.Write(c.c, binary.BigEndian, int32(len(buf)))
if err != nil {
c.poisoned = true
select {
case cc <- true:
default:
}
log.Println(err)
return
}
for len(buf) > 0 {
n, err := c.c.Write(buf)
if err != nil {
c.poisoned = true
select {
case cc <- true:
default:
}
log.Println(err)
return
}
buf = buf[n:]
}
}
// Stat returns metadata about the file or directory at path,
// in revision *storeRev. If storeRev is nil, uses the current
// revision.
func (c *Conn) Stat(path string, storeRev *int64) (len int, fileRev int64, err os.Error) {
var t txn
t.req.Verb = newRequest_Verb(request_STAT)
t.req.Path = &path
t.req.Rev = storeRev
err = c.call(&t)
if err != nil {
return 0, 0, err
}
return int(proto.GetInt32(t.resp.Len)), proto.GetInt64(t.resp.Rev), nil
}
func (t *Iterator) next() (*Key, *pb.EntityProto, error) {
if t.err != nil {
return nil, nil, t.err
}
// Issue datastore_v3/Next RPCs as necessary.
for len(t.res.Result) == 0 {
if !proto.GetBool(t.res.MoreResults) {
t.err = Done
return nil, nil, t.err
}
t.offset -= proto.GetInt32(t.res.SkippedResults)
if t.offset < 0 {
t.offset = 0
}
if err := callNext(t.c, &t.res, t.offset, t.limit, zeroLimitMeansUnlimited); err != nil {
t.err = err
return nil, nil, t.err
}
// For an Iterator, a zero limit means unlimited.
if t.limit == 0 {
continue
}
t.limit -= int32(len(t.res.Result))
if t.limit > 0 {
continue
}
t.limit = 0
if proto.GetBool(t.res.MoreResults) {
t.err = errors.New("datastore: internal error: limit exhausted but more_results is true")
return nil, nil, t.err
}
}
// Pop the EntityProto from the front of t.res.Result and
// extract its key.
var e *pb.EntityProto
e, t.res.Result = t.res.Result[0], t.res.Result[1:]
if e.Key == nil {
return nil, nil, errors.New("datastore: internal error: server did not return a key")
}
k, err := protoToKey(e.Key)
if err != nil || k.Incomplete() {
return nil, nil, errors.New("datastore: internal error: server returned an invalid key")
}
if proto.GetBool(t.res.KeysOnly) {
return k, nil, nil
}
return k, e, nil
}
func (c *conn) cancel(t *T, tx txn) {
tag := pb.GetInt32(t.OtherTag)
c.tl.Lock()
otx, ok := c.tx[tag]
c.tl.Unlock()
if ok {
select {
case otx.cancel <- true:
default:
}
<-otx.done
c.respond(t, Valid|Done, nil, &R{})
} else {
c.respond(t, Valid|Done, nil, badTag)
}
}
// Count returns the number of results for the query.
func (q *Query) Count(c appengine.Context) (int, error) {
// Check that the query is well-formed.
if q.err != nil {
return 0, q.err
}
// Run a copy of the query, with keysOnly true, and an adjusted offset.
// We also set the limit to zero, as we don't want any actual entity data,
// just the number of skipped results.
newQ := *q
newQ.keysOnly = true
newQ.limit = 0
if q.limit == 0 {
// If the original query was unlimited, set the new query's offset to maximum.
newQ.offset = math.MaxInt32
} else {
newQ.offset = q.offset + q.limit
if newQ.offset < 0 {
// Do the best we can, in the presence of overflow.
newQ.offset = math.MaxInt32
}
}
req := &pb.Query{}
if err := newQ.toProto(req, c.FullyQualifiedAppID(), zeroLimitMeansZero); err != nil {
return 0, err
}
res := &pb.QueryResult{}
if err := c.Call("datastore_v3", "RunQuery", req, res, nil); err != nil {
return 0, err
}
// n is the count we will return. For example, suppose that our original
// query had an offset of 4 and a limit of 2008: the count will be 2008,
// provided that there are at least 2012 matching entities. However, the
// RPCs will only skip 1000 results at a time. The RPC sequence is:
// call RunQuery with (offset, limit) = (2012, 0) // 2012 == newQ.offset
// response has (skippedResults, moreResults) = (1000, true)
// n += 1000 // n == 1000
// call Next with (offset, limit) = (1012, 0) // 1012 == newQ.offset - n
// response has (skippedResults, moreResults) = (1000, true)
// n += 1000 // n == 2000
// call Next with (offset, limit) = (12, 0) // 12 == newQ.offset - n
// response has (skippedResults, moreResults) = (12, false)
// n += 12 // n == 2012
// // exit the loop
// n -= 4 // n == 2008
var n int32
for {
// The QueryResult should have no actual entity data, just skipped results.
if len(res.Result) != 0 {
return 0, errors.New("datastore: internal error: Count request returned too much data")
}
n += proto.GetInt32(res.SkippedResults)
if !proto.GetBool(res.MoreResults) {
break
}
if err := callNext(c, res, newQ.offset-n, 0, zeroLimitMeansZero); err != nil {
return 0, err
}
}
n -= q.offset
if n < 0 {
// If the offset was greater than the number of matching entities,
// return 0 instead of negative.
n = 0
}
return int(n), nil
}
