func (je *JoinEngine) Yield(s *protocol.Series) (bool, error) {
log4go.Fine("JoinEngine.Yield(): %s", s)
idx := je.tableIdx[s.GetName()]
state := &je.tablesState[idx]
// If the state for this table didn't contain a point already,
// increment the number of tables ready to emit a point by
// incrementing `pts`
if state.lastPoint == nil {
je.pts++
}
state.lastPoint = s.Points[len(s.Points)-1]
// update the fields for this table. the fields shouldn't change
// after the first point, so we only need to set them once
if state.lastFields == nil {
for _, f := range s.Fields {
state.lastFields = append(state.lastFields, s.GetName()+"."+f)
}
}
log4go.Fine("JoinEngine: pts = %d", je.pts)
// if the number of tables ready to emit a point isn't equal to the
// total number of tables being joined, then return
if je.pts != len(je.tablesState) {
return true, nil
}
// we arbitrarily use the timestamp of the first table's point as
// the timestamp of the resulting point. may be we should use the
// smalles (or largest) timestamp.
ts := je.tablesState[0].lastPoint.Timestamp
newSeries := &protocol.Series{
Name: &je.name,
Fields: je.fields(),
Points: []*protocol.Point{
{
Timestamp: ts,
Values: je.values(),
},
},
}
// filter the point. the user may have a where clause with the join,
// e.g. `select * from join(foo1, foo2) where foo1.val > 10`. we
// can't evaluate the where clause until after join happens
filteredSeries, err := Filter(je.query, newSeries)
if err != nil {
return false, err
}
if len(filteredSeries.Points) > 0 {
return je.next.Yield(newSeries)
}
return true, nil
}
// When a new waiter arrives, we check if it matches existing tuples. If so,
// we return those tuples. If not, the waiter is added to the list of waiters
// that will match against future tuples.
func (t *TupleSpaceImpl) processNewWaiter(waiter *tupleWaiter) {
t.waitersLock.Lock()
defer t.waitersLock.Unlock()
atomic.AddInt64(&t.stats.WaitersSeen, 1)
one := waiter.actions&ActionOne != 0
take := waiter.actions&ActionTake != 0
limit := 0
if one {
limit = 1
}
stored, deletes, err := t.store.Match(waiter.match, limit)
if err != nil {
waiter.err <- err
return
}
matches := make([]Tuple, 0, len(stored))
taken := 0
for _, entry := range stored {
matches = append(matches, entry.Tuple)
if take {
taken++
deletes = append(deletes, entry)
}
if one {
break
}
}
if len(deletes) > 0 {
log.Fine("Deleting %d tuples. %d taken by %s, %d expired", len(deletes), taken, waiter, len(deletes)-taken)
t.store.Delete(deletes)
}
if len(matches) > 0 {
log.Fine("Waiter %s immediately returned %d matching tuples", waiter, len(matches))
if len(deletes) != 0 {
atomic.AddInt64(&t.stats.TuplesTaken, int64(len(deletes)))
} else {
atomic.AddInt64(&t.stats.TuplesRead, int64(len(matches)))
}
waiter.matches <- matches
} else {
log.Fine("Adding new waiter %s", waiter)
t.waiters[waiter] = nil
}
}
// SendMany tuples into the tuplespace. The "tuples" argument must be an array
// of values.
func (t *TupleSpaceClient) SendMany(tuples interface{}, timeout time.Duration) error {
req := &tuplespace.SendRequest{
Tuples: tuples,
Timeout: timeout,
}
log.Fine("TupleSpaceClient.SendMany(%+v, %s)", tuples, timeout)
resp := &tuplespace.SendResponse{}
return t.do("POST", req, resp)
}
func (self *ShardDatastore) ReturnShard(id uint32) {
self.shardsLock.Lock()
defer self.shardsLock.Unlock()
log.Fine("Returning shard. id = %d", id)
self.shardRefCounts[id] -= 1
if self.shardRefCounts[id] != 0 {
return
}
log.Fine("Checking if shard should be deleted. id = %d", id)
if _, ok := self.shardsToDelete[id]; ok {
self.deleteShard(id)
return
}
log.Fine("Checking if shard should be closed. id = %d", id)
if self.shardsToClose[id] {
self.closeShard(id)
}
}
func (t *TupleSpaceClient) read(match *tuplespace.TupleMatcher, timeout time.Duration, actions int, out interface{}) error {
method := "GET"
if actions&tuplespace.ActionTake != 0 {
method = "DELETE"
}
req := &tuplespace.ReadRequest{
Match: match.String(),
Timeout: timeout,
}
req.All = actions&tuplespace.ActionOne == 0
log.Fine("TupleSpaceClient.read(%+v)", req)
return t.do(method, req, out)
}
func (m *MemoryStore) Put(tuples []tuplespace.Tuple, timeout time.Time) ([]*tuplespace.TupleEntry, error) {
log.Fine("Putting %d tuples", len(tuples))
entries := make([]*tuplespace.TupleEntry, 0, len(tuples))
for _, tuple := range tuples {
m.id++
entry := &tuplespace.TupleEntry{
ID: m.id,
Tuple: tuple,
Timeout: timeout,
}
entries = append(entries, entry)
m.tuples[entry.ID] = entry
}
return entries, nil
}
func (self *ShardDatastore) DeleteShard(shardId uint32) {
self.shardsLock.Lock()
defer self.shardsLock.Unlock()
// If someone has a reference to the shard we can't delete it
// now. We have to wait until it's returned and delete
// it. ReturnShard will take care of that as soon as the reference
// count becomes 0.
if refc := self.shardRefCounts[shardId]; refc > 0 {
log.Fine("Cannot delete shard: shardId = %d, shardRefCounts[shardId] = %d", shardId, refc)
self.shardsToDelete[shardId] = struct{}{}
return
}
// otherwise, close the shard and delete it now
self.deleteShard(shardId)
}
// Purge expired waiters.
func (t *TupleSpaceImpl) purge() {
t.waitersLock.Lock()
defer t.waitersLock.Unlock()
now := time.Now()
waiters := 0
for waiter := range t.waiters {
if !waiter.timeout.IsZero() && waiter.timeout.Before(now) {
delete(t.waiters, waiter)
waiter.err <- ReaderTimeout
waiters++
}
}
if waiters > 0 {
log.Fine("Purged %d waiters", waiters)
}
}
func (m *MemoryStore) Match(match *tuplespace.TupleMatcher, limit int) ([]*tuplespace.TupleEntry, []*tuplespace.TupleEntry, error) {
now := time.Now()
matches := make([]*tuplespace.TupleEntry, 0, 32)
deletes := make([]*tuplespace.TupleEntry, 0, 32)
log.Fine("Matching %s against %d tuples limit %d", match, len(m.tuples), limit)
for _, entry := range m.tuples {
if entry.IsExpired(now) {
deletes = append(deletes, entry)
continue
}
if match.Match(entry.Tuple) {
matches = append(matches, entry)
if len(matches) == limit {
break
}
}
}
if len(deletes) > 0 {
m.Delete(deletes)
}
return matches, nil, nil
}
func (self *Coordinator) CommitSeriesData(db string, serieses []*protocol.Series, sync bool) error {
now := common.CurrentTime()
shardToSerieses := map[uint32]map[string]*protocol.Series{}
shardIdToShard := map[uint32]*cluster.ShardData{}
for _, series := range serieses {
if len(series.Points) == 0 {
return fmt.Errorf("Can't write series with zero points.")
}
for _, point := range series.Points {
if point.Timestamp == nil {
point.Timestamp = &now
}
}
// sort the points by timestamp
// TODO: this isn't needed anymore
series.SortPointsTimeDescending()
sn := series.GetName()
// use regular for loop since we update the iteration index `i' as
// we batch points that have the same timestamp
for i := 0; i < len(series.Points); {
if len(series.GetName()) == 0 {
return fmt.Errorf("Series name cannot be empty")
}
ts := series.Points[i].GetTimestamp()
shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, sn, ts)
if err != nil {
return err
}
log.Fine("GetShardToWriteToBySeriesAndTime(%s, %s, %d) = (%s, %v)", db, sn, ts, shard, err)
firstIndex := i
for ; i < len(series.Points) && series.Points[i].GetTimestamp() == ts; i++ {
// add all points with the same timestamp
}
// if shard == nil, then the points shouldn't be writte. This
// will happen if the points had timestamps earlier than the
// retention period
if shard == nil {
continue
}
newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[firstIndex:i:i]}
shardIdToShard[shard.Id()] = shard
shardSerieses := shardToSerieses[shard.Id()]
if shardSerieses == nil {
shardSerieses = map[string]*protocol.Series{}
shardToSerieses[shard.Id()] = shardSerieses
}
seriesName := series.GetName()
s := shardSerieses[seriesName]
if s == nil {
shardSerieses[seriesName] = newSeries
continue
}
shardSerieses[seriesName] = common.MergeSeries(s, newSeries)
}
}
for id, serieses := range shardToSerieses {
shard := shardIdToShard[id]
seriesesSlice := make([]*protocol.Series, 0, len(serieses))
for _, s := range serieses {
seriesesSlice = append(seriesesSlice, s)
self.monitor.RecordWrite(uint64(len(s.Points)))
}
err := self.write(db, seriesesSlice, shard, sync)
if err != nil {
log.Error("COORD error writing: ", err)
return err
}
}
return nil
}
func (cme *CommonMergeEngine) Yield(s *protocol.Series) (bool, error) {
log4go.Fine("CommonMergeEngine.Yield(): %s", s)
stream := cme.streams[s.GetShardId()]
stream.Yield(s)
return cme.merger.Update()
}