func (je *JoinEngine) Yield(s *protocol.Series) (bool, error) { log4go.Fine("JoinEngine.Yield(): %s", s) idx := je.tableIdx[s.GetName()] state := &je.tablesState[idx] // If the state for this table didn't contain a point already, // increment the number of tables ready to emit a point by // incrementing `pts` if state.lastPoint == nil { je.pts++ } state.lastPoint = s.Points[len(s.Points)-1] // update the fields for this table. the fields shouldn't change // after the first point, so we only need to set them once if state.lastFields == nil { for _, f := range s.Fields { state.lastFields = append(state.lastFields, s.GetName()+"."+f) } } log4go.Fine("JoinEngine: pts = %d", je.pts) // if the number of tables ready to emit a point isn't equal to the // total number of tables being joined, then return if je.pts != len(je.tablesState) { return true, nil } // we arbitrarily use the timestamp of the first table's point as // the timestamp of the resulting point. may be we should use the // smalles (or largest) timestamp. ts := je.tablesState[0].lastPoint.Timestamp newSeries := &protocol.Series{ Name: &je.name, Fields: je.fields(), Points: []*protocol.Point{ { Timestamp: ts, Values: je.values(), }, }, } // filter the point. the user may have a where clause with the join, // e.g. `select * from join(foo1, foo2) where foo1.val > 10`. we // can't evaluate the where clause until after join happens filteredSeries, err := Filter(je.query, newSeries) if err != nil { return false, err } if len(filteredSeries.Points) > 0 { return je.next.Yield(newSeries) } return true, nil }
// When a new waiter arrives, we check if it matches existing tuples. If so, // we return those tuples. If not, the waiter is added to the list of waiters // that will match against future tuples. func (t *TupleSpaceImpl) processNewWaiter(waiter *tupleWaiter) { t.waitersLock.Lock() defer t.waitersLock.Unlock() atomic.AddInt64(&t.stats.WaitersSeen, 1) one := waiter.actions&ActionOne != 0 take := waiter.actions&ActionTake != 0 limit := 0 if one { limit = 1 } stored, deletes, err := t.store.Match(waiter.match, limit) if err != nil { waiter.err <- err return } matches := make([]Tuple, 0, len(stored)) taken := 0 for _, entry := range stored { matches = append(matches, entry.Tuple) if take { taken++ deletes = append(deletes, entry) } if one { break } } if len(deletes) > 0 { log.Fine("Deleting %d tuples. %d taken by %s, %d expired", len(deletes), taken, waiter, len(deletes)-taken) t.store.Delete(deletes) } if len(matches) > 0 { log.Fine("Waiter %s immediately returned %d matching tuples", waiter, len(matches)) if len(deletes) != 0 { atomic.AddInt64(&t.stats.TuplesTaken, int64(len(deletes))) } else { atomic.AddInt64(&t.stats.TuplesRead, int64(len(matches))) } waiter.matches <- matches } else { log.Fine("Adding new waiter %s", waiter) t.waiters[waiter] = nil } }
// SendMany tuples into the tuplespace. The "tuples" argument must be an array // of values. func (t *TupleSpaceClient) SendMany(tuples interface{}, timeout time.Duration) error { req := &tuplespace.SendRequest{ Tuples: tuples, Timeout: timeout, } log.Fine("TupleSpaceClient.SendMany(%+v, %s)", tuples, timeout) resp := &tuplespace.SendResponse{} return t.do("POST", req, resp) }
func (self *ShardDatastore) ReturnShard(id uint32) { self.shardsLock.Lock() defer self.shardsLock.Unlock() log.Fine("Returning shard. id = %d", id) self.shardRefCounts[id] -= 1 if self.shardRefCounts[id] != 0 { return } log.Fine("Checking if shard should be deleted. id = %d", id) if _, ok := self.shardsToDelete[id]; ok { self.deleteShard(id) return } log.Fine("Checking if shard should be closed. id = %d", id) if self.shardsToClose[id] { self.closeShard(id) } }
func (t *TupleSpaceClient) read(match *tuplespace.TupleMatcher, timeout time.Duration, actions int, out interface{}) error { method := "GET" if actions&tuplespace.ActionTake != 0 { method = "DELETE" } req := &tuplespace.ReadRequest{ Match: match.String(), Timeout: timeout, } req.All = actions&tuplespace.ActionOne == 0 log.Fine("TupleSpaceClient.read(%+v)", req) return t.do(method, req, out) }
func (m *MemoryStore) Put(tuples []tuplespace.Tuple, timeout time.Time) ([]*tuplespace.TupleEntry, error) { log.Fine("Putting %d tuples", len(tuples)) entries := make([]*tuplespace.TupleEntry, 0, len(tuples)) for _, tuple := range tuples { m.id++ entry := &tuplespace.TupleEntry{ ID: m.id, Tuple: tuple, Timeout: timeout, } entries = append(entries, entry) m.tuples[entry.ID] = entry } return entries, nil }
func (self *ShardDatastore) DeleteShard(shardId uint32) { self.shardsLock.Lock() defer self.shardsLock.Unlock() // If someone has a reference to the shard we can't delete it // now. We have to wait until it's returned and delete // it. ReturnShard will take care of that as soon as the reference // count becomes 0. if refc := self.shardRefCounts[shardId]; refc > 0 { log.Fine("Cannot delete shard: shardId = %d, shardRefCounts[shardId] = %d", shardId, refc) self.shardsToDelete[shardId] = struct{}{} return } // otherwise, close the shard and delete it now self.deleteShard(shardId) }
// Purge expired waiters. func (t *TupleSpaceImpl) purge() { t.waitersLock.Lock() defer t.waitersLock.Unlock() now := time.Now() waiters := 0 for waiter := range t.waiters { if !waiter.timeout.IsZero() && waiter.timeout.Before(now) { delete(t.waiters, waiter) waiter.err <- ReaderTimeout waiters++ } } if waiters > 0 { log.Fine("Purged %d waiters", waiters) } }
func (m *MemoryStore) Match(match *tuplespace.TupleMatcher, limit int) ([]*tuplespace.TupleEntry, []*tuplespace.TupleEntry, error) { now := time.Now() matches := make([]*tuplespace.TupleEntry, 0, 32) deletes := make([]*tuplespace.TupleEntry, 0, 32) log.Fine("Matching %s against %d tuples limit %d", match, len(m.tuples), limit) for _, entry := range m.tuples { if entry.IsExpired(now) { deletes = append(deletes, entry) continue } if match.Match(entry.Tuple) { matches = append(matches, entry) if len(matches) == limit { break } } } if len(deletes) > 0 { m.Delete(deletes) } return matches, nil, nil }
func (self *Coordinator) CommitSeriesData(db string, serieses []*protocol.Series, sync bool) error { now := common.CurrentTime() shardToSerieses := map[uint32]map[string]*protocol.Series{} shardIdToShard := map[uint32]*cluster.ShardData{} for _, series := range serieses { if len(series.Points) == 0 { return fmt.Errorf("Can't write series with zero points.") } for _, point := range series.Points { if point.Timestamp == nil { point.Timestamp = &now } } // sort the points by timestamp // TODO: this isn't needed anymore series.SortPointsTimeDescending() sn := series.GetName() // use regular for loop since we update the iteration index `i' as // we batch points that have the same timestamp for i := 0; i < len(series.Points); { if len(series.GetName()) == 0 { return fmt.Errorf("Series name cannot be empty") } ts := series.Points[i].GetTimestamp() shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, sn, ts) if err != nil { return err } log.Fine("GetShardToWriteToBySeriesAndTime(%s, %s, %d) = (%s, %v)", db, sn, ts, shard, err) firstIndex := i for ; i < len(series.Points) && series.Points[i].GetTimestamp() == ts; i++ { // add all points with the same timestamp } // if shard == nil, then the points shouldn't be writte. This // will happen if the points had timestamps earlier than the // retention period if shard == nil { continue } newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[firstIndex:i:i]} shardIdToShard[shard.Id()] = shard shardSerieses := shardToSerieses[shard.Id()] if shardSerieses == nil { shardSerieses = map[string]*protocol.Series{} shardToSerieses[shard.Id()] = shardSerieses } seriesName := series.GetName() s := shardSerieses[seriesName] if s == nil { shardSerieses[seriesName] = newSeries continue } shardSerieses[seriesName] = common.MergeSeries(s, newSeries) } } for id, serieses := range shardToSerieses { shard := shardIdToShard[id] seriesesSlice := make([]*protocol.Series, 0, len(serieses)) for _, s := range serieses { seriesesSlice = append(seriesesSlice, s) self.monitor.RecordWrite(uint64(len(s.Points))) } err := self.write(db, seriesesSlice, shard, sync) if err != nil { log.Error("COORD error writing: ", err) return err } } return nil }
func (cme *CommonMergeEngine) Yield(s *protocol.Series) (bool, error) { log4go.Fine("CommonMergeEngine.Yield(): %s", s) stream := cme.streams[s.GetShardId()] stream.Yield(s) return cme.merger.Update() }