Пример #1
0
func (je *JoinEngine) Yield(s *protocol.Series) (bool, error) {
	log4go.Fine("JoinEngine.Yield(): %s", s)
	idx := je.tableIdx[s.GetName()]
	state := &je.tablesState[idx]
	// If the state for this table didn't contain a point already,
	// increment the number of tables ready to emit a point by
	// incrementing `pts`
	if state.lastPoint == nil {
		je.pts++
	}
	state.lastPoint = s.Points[len(s.Points)-1]
	// update the fields for this table. the fields shouldn't change
	// after the first point, so we only need to set them once
	if state.lastFields == nil {
		for _, f := range s.Fields {
			state.lastFields = append(state.lastFields, s.GetName()+"."+f)
		}
	}

	log4go.Fine("JoinEngine: pts = %d", je.pts)
	// if the number of tables ready to emit a point isn't equal to the
	// total number of tables being joined, then return
	if je.pts != len(je.tablesState) {
		return true, nil
	}

	// we arbitrarily use the timestamp of the first table's point as
	// the timestamp of the resulting point. may be we should use the
	// smalles (or largest) timestamp.
	ts := je.tablesState[0].lastPoint.Timestamp
	newSeries := &protocol.Series{
		Name:   &je.name,
		Fields: je.fields(),
		Points: []*protocol.Point{
			{
				Timestamp: ts,
				Values:    je.values(),
			},
		},
	}

	// filter the point. the user may have a where clause with the join,
	// e.g. `select * from join(foo1, foo2) where foo1.val > 10`. we
	// can't evaluate the where clause until after join happens
	filteredSeries, err := Filter(je.query, newSeries)
	if err != nil {
		return false, err
	}

	if len(filteredSeries.Points) > 0 {
		return je.next.Yield(newSeries)
	}
	return true, nil
}
Пример #2
0
// When a new waiter arrives, we check if it matches existing tuples. If so,
// we return those tuples. If not, the waiter is added to the list of waiters
// that will match against future tuples.
func (t *TupleSpaceImpl) processNewWaiter(waiter *tupleWaiter) {
	t.waitersLock.Lock()
	defer t.waitersLock.Unlock()

	atomic.AddInt64(&t.stats.WaitersSeen, 1)
	one := waiter.actions&ActionOne != 0
	take := waiter.actions&ActionTake != 0
	limit := 0
	if one {
		limit = 1
	}
	stored, deletes, err := t.store.Match(waiter.match, limit)
	if err != nil {
		waiter.err <- err
		return
	}

	matches := make([]Tuple, 0, len(stored))
	taken := 0
	for _, entry := range stored {
		matches = append(matches, entry.Tuple)
		if take {
			taken++
			deletes = append(deletes, entry)
		}
		if one {
			break
		}
	}

	if len(deletes) > 0 {
		log.Fine("Deleting %d tuples. %d taken by %s, %d expired", len(deletes), taken, waiter, len(deletes)-taken)
		t.store.Delete(deletes)
	}

	if len(matches) > 0 {
		log.Fine("Waiter %s immediately returned %d matching tuples", waiter, len(matches))
		if len(deletes) != 0 {
			atomic.AddInt64(&t.stats.TuplesTaken, int64(len(deletes)))
		} else {
			atomic.AddInt64(&t.stats.TuplesRead, int64(len(matches)))
		}
		waiter.matches <- matches
	} else {
		log.Fine("Adding new waiter %s", waiter)
		t.waiters[waiter] = nil
	}
}
Пример #3
0
// SendMany tuples into the tuplespace. The "tuples" argument must be an array
// of values.
func (t *TupleSpaceClient) SendMany(tuples interface{}, timeout time.Duration) error {
	req := &tuplespace.SendRequest{
		Tuples:  tuples,
		Timeout: timeout,
	}
	log.Fine("TupleSpaceClient.SendMany(%+v, %s)", tuples, timeout)
	resp := &tuplespace.SendResponse{}
	return t.do("POST", req, resp)
}
Пример #4
0
func (self *ShardDatastore) ReturnShard(id uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	log.Fine("Returning shard. id = %d", id)
	self.shardRefCounts[id] -= 1
	if self.shardRefCounts[id] != 0 {
		return
	}

	log.Fine("Checking if shard should be deleted. id = %d", id)
	if _, ok := self.shardsToDelete[id]; ok {
		self.deleteShard(id)
		return
	}

	log.Fine("Checking if shard should be closed. id = %d", id)
	if self.shardsToClose[id] {
		self.closeShard(id)
	}
}
Пример #5
0
func (t *TupleSpaceClient) read(match *tuplespace.TupleMatcher, timeout time.Duration, actions int, out interface{}) error {
	method := "GET"
	if actions&tuplespace.ActionTake != 0 {
		method = "DELETE"
	}
	req := &tuplespace.ReadRequest{
		Match:   match.String(),
		Timeout: timeout,
	}
	req.All = actions&tuplespace.ActionOne == 0
	log.Fine("TupleSpaceClient.read(%+v)", req)
	return t.do(method, req, out)
}
Пример #6
0
func (m *MemoryStore) Put(tuples []tuplespace.Tuple, timeout time.Time) ([]*tuplespace.TupleEntry, error) {
	log.Fine("Putting %d tuples", len(tuples))
	entries := make([]*tuplespace.TupleEntry, 0, len(tuples))
	for _, tuple := range tuples {
		m.id++
		entry := &tuplespace.TupleEntry{
			ID:      m.id,
			Tuple:   tuple,
			Timeout: timeout,
		}
		entries = append(entries, entry)
		m.tuples[entry.ID] = entry
	}
	return entries, nil
}
Пример #7
0
func (self *ShardDatastore) DeleteShard(shardId uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	// If someone has a reference to the shard we can't delete it
	// now. We have to wait until it's returned and delete
	// it. ReturnShard will take care of that as soon as the reference
	// count becomes 0.
	if refc := self.shardRefCounts[shardId]; refc > 0 {
		log.Fine("Cannot delete shard: shardId = %d, shardRefCounts[shardId] = %d", shardId, refc)
		self.shardsToDelete[shardId] = struct{}{}
		return
	}

	// otherwise, close the shard and delete it now
	self.deleteShard(shardId)
}
Пример #8
0
// Purge expired waiters.
func (t *TupleSpaceImpl) purge() {
	t.waitersLock.Lock()
	defer t.waitersLock.Unlock()
	now := time.Now()
	waiters := 0
	for waiter := range t.waiters {
		if !waiter.timeout.IsZero() && waiter.timeout.Before(now) {
			delete(t.waiters, waiter)
			waiter.err <- ReaderTimeout
			waiters++
		}
	}
	if waiters > 0 {
		log.Fine("Purged %d waiters", waiters)
	}
}
Пример #9
0
func (m *MemoryStore) Match(match *tuplespace.TupleMatcher, limit int) ([]*tuplespace.TupleEntry, []*tuplespace.TupleEntry, error) {
	now := time.Now()
	matches := make([]*tuplespace.TupleEntry, 0, 32)
	deletes := make([]*tuplespace.TupleEntry, 0, 32)

	log.Fine("Matching %s against %d tuples limit %d", match, len(m.tuples), limit)
	for _, entry := range m.tuples {
		if entry.IsExpired(now) {
			deletes = append(deletes, entry)
			continue
		}
		if match.Match(entry.Tuple) {
			matches = append(matches, entry)
			if len(matches) == limit {
				break
			}
		}
	}

	if len(deletes) > 0 {
		m.Delete(deletes)
	}
	return matches, nil, nil
}
Пример #10
0
func (self *Coordinator) CommitSeriesData(db string, serieses []*protocol.Series, sync bool) error {
	now := common.CurrentTime()

	shardToSerieses := map[uint32]map[string]*protocol.Series{}
	shardIdToShard := map[uint32]*cluster.ShardData{}

	for _, series := range serieses {
		if len(series.Points) == 0 {
			return fmt.Errorf("Can't write series with zero points.")
		}

		for _, point := range series.Points {
			if point.Timestamp == nil {
				point.Timestamp = &now
			}
		}

		// sort the points by timestamp
		// TODO: this isn't needed anymore
		series.SortPointsTimeDescending()

		sn := series.GetName()
		// use regular for loop since we update the iteration index `i' as
		// we batch points that have the same timestamp
		for i := 0; i < len(series.Points); {
			if len(series.GetName()) == 0 {
				return fmt.Errorf("Series name cannot be empty")
			}

			ts := series.Points[i].GetTimestamp()
			shard, err := self.clusterConfiguration.GetShardToWriteToBySeriesAndTime(db, sn, ts)
			if err != nil {
				return err
			}
			log.Fine("GetShardToWriteToBySeriesAndTime(%s, %s, %d) = (%s, %v)", db, sn, ts, shard, err)
			firstIndex := i
			for ; i < len(series.Points) && series.Points[i].GetTimestamp() == ts; i++ {
				// add all points with the same timestamp
			}

			// if shard == nil, then the points shouldn't be writte. This
			// will happen if the points had timestamps earlier than the
			// retention period
			if shard == nil {
				continue
			}
			newSeries := &protocol.Series{Name: series.Name, Fields: series.Fields, Points: series.Points[firstIndex:i:i]}

			shardIdToShard[shard.Id()] = shard
			shardSerieses := shardToSerieses[shard.Id()]
			if shardSerieses == nil {
				shardSerieses = map[string]*protocol.Series{}
				shardToSerieses[shard.Id()] = shardSerieses
			}
			seriesName := series.GetName()
			s := shardSerieses[seriesName]
			if s == nil {
				shardSerieses[seriesName] = newSeries
				continue
			}
			shardSerieses[seriesName] = common.MergeSeries(s, newSeries)
		}
	}

	for id, serieses := range shardToSerieses {
		shard := shardIdToShard[id]

		seriesesSlice := make([]*protocol.Series, 0, len(serieses))
		for _, s := range serieses {
			seriesesSlice = append(seriesesSlice, s)

			self.monitor.RecordWrite(uint64(len(s.Points)))
		}

		err := self.write(db, seriesesSlice, shard, sync)
		if err != nil {
			log.Error("COORD error writing: ", err)
			return err
		}
	}

	return nil
}
Пример #11
0
func (cme *CommonMergeEngine) Yield(s *protocol.Series) (bool, error) {
	log4go.Fine("CommonMergeEngine.Yield(): %s", s)
	stream := cme.streams[s.GetShardId()]
	stream.Yield(s)
	return cme.merger.Update()
}