// CreateShardGroup creates a shard group on a database and policy for a given timestamp.
func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) error {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Verify that shard group doesn't already exist for this timestamp.
if rpi.ShardGroupByTimestamp(timestamp) != nil {
return nil
}
// Create the shard group.
data.MaxShardGroupID++
sgi := ShardGroupInfo{}
sgi.ID = data.MaxShardGroupID
sgi.StartTime = timestamp.Truncate(rpi.ShardGroupDuration).UTC()
sgi.EndTime = sgi.StartTime.Add(rpi.ShardGroupDuration).UTC()
data.MaxShardID++
sgi.Shards = []ShardInfo{
{ID: data.MaxShardID},
}
// Retention policy has a new shard group, so update the policy. Shard
// Groups must be stored in sorted order, as other parts of the system
// assume this to be the case.
rpi.ShardGroups = append(rpi.ShardGroups, sgi)
sort.Sort(ShardGroupInfos(rpi.ShardGroups))
return nil
}
// createInternalStorage ensures the internal storage has been created.
func (m *Monitor) createInternalStorage() {
if m.storeCreated {
return
}
if _, err := m.MetaClient.CreateDatabase(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', failed to create storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy)
rpi.Duration = MonitorRetentionPolicyDuration
rpi.ReplicaN = 1
if _, err := m.MetaClient.CreateRetentionPolicy(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', failed to create internal storage: %s",
rpi.Name, err.Error())
return
}
if err := m.MetaClient.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil {
m.Logger.Printf("failed to set default retention policy on '%s', failed to create internal storage: %s",
m.storeDatabase, err.Error())
return
}
err := m.MetaClient.DropRetentionPolicy(m.storeDatabase, "default")
if err != nil && err.Error() != influxdb.ErrRetentionPolicyNotFound("default").Error() {
m.Logger.Printf("failed to delete retention policy 'default', failed to created internal storage: %s", err.Error())
return
}
// Mark storage creation complete.
m.storeCreated = true
}
// CreateSubscription adds a named subscription to a database and retention policy.
func (data *Data) CreateSubscription(database, rp, name, mode string, destinations []string) error {
for _, d := range destinations {
if err := validateURL(d); err != nil {
return err
}
}
rpi, err := data.RetentionPolicy(database, rp)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(rp)
}
// Ensure the name doesn't already exist.
for i := range rpi.Subscriptions {
if rpi.Subscriptions[i].Name == name {
return ErrSubscriptionExists
}
}
// Append new query.
rpi.Subscriptions = append(rpi.Subscriptions, SubscriptionInfo{
Name: name,
Mode: mode,
Destinations: destinations,
})
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
// holds the start time ranges for required shard groups
timeRanges := map[time.Time]*meta.ShardGroupInfo{}
rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
}
if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
for _, p := range wp.Points {
timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
}
// holds all the shard groups and shards that are required for writes
for t := range timeRanges {
sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, t)
if err != nil {
return nil, err
}
timeRanges[t] = sg
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
func TestMetaService_Subscriptions_Drop(t *testing.T) {
t.Parallel()
d, s, c := newServiceAndClient()
defer os.RemoveAll(d)
defer s.Close()
defer c.Close()
// Create a database to use
if res := c.ExecuteStatement(mustParseStatement("CREATE DATABASE db0")); res.Err != nil {
t.Fatal(res.Err)
}
db, err := c.Database("db0")
if err != nil {
t.Fatal(err)
} else if db.Name != "db0" {
t.Fatalf("db name wrong: %s", db.Name)
}
// DROP SUBSCRIPTION returns ErrSubscriptionNotFound when the
// subscription is unknown.
res := c.ExecuteStatement(mustParseStatement(`DROP SUBSCRIPTION foo ON db0."default"`))
if got, exp := res.Err, meta.ErrSubscriptionNotFound; got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// Create a subscription.
if res = c.ExecuteStatement(mustParseStatement(`CREATE SUBSCRIPTION sub0 ON db0."default" DESTINATIONS ALL 'udp://example.com:9090'`)); res.Err != nil {
t.Fatal(res.Err)
}
// DROP SUBSCRIPTION returns an influxdb.ErrDatabaseNotFound when
// the database is unknown.
res = c.ExecuteStatement(mustParseStatement(`DROP SUBSCRIPTION sub0 ON foo."default"`))
if got, exp := res.Err, influxdb.ErrDatabaseNotFound("foo"); got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// DROP SUBSCRIPTION returns an influxdb.ErrRetentionPolicyNotFound
// when the retention policy is unknown.
res = c.ExecuteStatement(mustParseStatement(`DROP SUBSCRIPTION sub0 ON db0."foo_policy"`))
if got, exp := res.Err, influxdb.ErrRetentionPolicyNotFound("foo_policy"); got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// DROP SUBSCRIPTION drops the subsciption if it can find it.
res = c.ExecuteStatement(mustParseStatement(`DROP SUBSCRIPTION sub0 ON db0."default"`))
if got := res.Err; got != nil {
t.Fatalf("got: %s, exp: %v", got, nil)
}
if res = c.ExecuteStatement(mustParseStatement(`SHOW SUBSCRIPTIONS`)); res.Err != nil {
t.Fatal(res.Err)
} else if got, exp := len(res.Series), 0; got != exp {
t.Fatalf("got %d series, expected %d", got, exp)
}
}
// ShardGroupByTimestamp returns the shard group on a database and policy for a given timestamp.
func (data *Data) ShardGroupByTimestamp(database, policy string, timestamp time.Time) (*ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
return rpi.ShardGroupByTimestamp(timestamp), nil
}
// UpdateRetentionPolicy updates an existing retention policy.
func (data *Data) UpdateRetentionPolicy(database, name string, rpu *RetentionPolicyUpdate, makeDefault bool) error {
// Find database.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
}
// Find policy.
rpi := di.RetentionPolicy(name)
if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Ensure new policy doesn't match an existing policy.
if rpu.Name != nil && *rpu.Name != name && di.RetentionPolicy(*rpu.Name) != nil {
return ErrRetentionPolicyNameExists
}
// Enforce duration of at least MinRetentionPolicyDuration
if rpu.Duration != nil && *rpu.Duration < MinRetentionPolicyDuration && *rpu.Duration != 0 {
return ErrRetentionPolicyDurationTooLow
}
// Enforce duration is at least the shard duration
if (rpu.Duration != nil && *rpu.Duration > 0 &&
((rpu.ShardGroupDuration != nil && *rpu.Duration < *rpu.ShardGroupDuration) ||
(rpu.ShardGroupDuration == nil && *rpu.Duration < rpi.ShardGroupDuration))) ||
(rpu.Duration == nil && rpi.Duration > 0 &&
rpu.ShardGroupDuration != nil && rpi.Duration < *rpu.ShardGroupDuration) {
return ErrIncompatibleDurations
}
// Update fields.
if rpu.Name != nil {
rpi.Name = *rpu.Name
}
if rpu.Duration != nil {
rpi.Duration = *rpu.Duration
}
if rpu.ReplicaN != nil {
rpi.ReplicaN = *rpu.ReplicaN
}
if rpu.ShardGroupDuration != nil {
rpi.ShardGroupDuration = normalisedShardDuration(*rpu.ShardGroupDuration, rpi.Duration)
}
if di.DefaultRetentionPolicy != rpi.Name && makeDefault {
di.DefaultRetentionPolicy = rpi.Name
}
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
// holds the start time ranges for required shard groups
timeRanges := map[time.Time]*meta.ShardGroupInfo{}
rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
} else if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
// Find the minimum time for a point if the retention policy has a shard
// group duration. We will automatically drop any points before this time.
// There is a chance of a time on the edge of the shard group duration to
// sneak through even after it has been removed, but the circumstances are
// rare enough and don't matter enough that we don't account for this
// edge case.
min := time.Unix(0, models.MinNanoTime)
if rp.Duration > 0 {
min = time.Now().Add(-rp.Duration)
}
for _, p := range wp.Points {
if p.Time().Before(min) {
continue
}
timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
}
// holds all the shard groups and shards that are required for writes
for t := range timeRanges {
sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, t)
if err != nil {
return nil, err
}
timeRanges[t] = sg
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg, ok := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
if !ok {
w.statMap.Add(statWriteDrop, 1)
continue
}
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// SetDefaultRetentionPolicy sets the default retention policy for a database.
func (data *Data) SetDefaultRetentionPolicy(database, name string) error {
// Find database and verify policy exists.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
} else if di.RetentionPolicy(name) == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Set default policy.
di.DefaultRetentionPolicy = name
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
} else if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
// Holds all the shard groups and shards that are required for writes.
list := make(sgList, 0, 8)
min := time.Unix(0, models.MinNanoTime)
if rp.Duration > 0 {
min = time.Now().Add(-rp.Duration)
}
for _, p := range wp.Points {
// Either the point is outside the scope of the RP, or we already have
// a suitable shard group for the point.
if p.Time().Before(min) || list.Covers(p.Time()) {
continue
}
// No shard groups overlap with the point's time, so we will create
// a new shard group for this point.
sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, p.Time())
if err != nil {
return nil, err
}
if sg == nil {
return nil, errors.New("nil shard group")
}
list = list.Append(*sg)
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg := list.ShardGroupAt(p.Time())
if sg == nil {
// We didn't create a shard group because the point was outside the
// scope of the RP.
atomic.AddInt64(&w.stats.WriteDropped, 1)
continue
}
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// DropSubscription removes a subscription.
func (data *Data) DropSubscription(database, rp, name string) error {
rpi, err := data.RetentionPolicy(database, rp)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(rp)
}
for i := range rpi.Subscriptions {
if rpi.Subscriptions[i].Name == name {
rpi.Subscriptions = append(rpi.Subscriptions[:i], rpi.Subscriptions[i+1:]...)
return nil
}
}
return ErrSubscriptionNotFound
}
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data
// for the specified time range. Shard groups are sorted by start time.
func (data *Data) ShardGroupsByTimeRange(database, policy string, tmin, tmax time.Time) ([]ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups))
for _, g := range rpi.ShardGroups {
if g.Deleted() || !g.Overlaps(tmin, tmax) {
continue
}
groups = append(groups, g)
}
return groups, nil
}
// writeDatabaseInfo will write the relative paths of all shards in the retention policy on
// this server into the connection
func (s *Service) writeRetentionPolicyInfo(conn net.Conn, database, retentionPolicy string) error {
res := Response{}
db, err := s.MetaClient.Database(database)
if err != nil {
return err
}
if db == nil {
return influxdb.ErrDatabaseNotFound(database)
}
var ret *meta.RetentionPolicyInfo
for _, rp := range db.RetentionPolicies {
if rp.Name == retentionPolicy {
ret = &rp
break
}
}
if ret == nil {
return influxdb.ErrRetentionPolicyNotFound(retentionPolicy)
}
for _, sg := range ret.ShardGroups {
for _, sh := range sg.Shards {
// ignore if the shard isn't on the server
if s.TSDBStore.Shard(sh.ID) == nil {
continue
}
path, err := s.TSDBStore.ShardRelativePath(sh.ID)
if err != nil {
return err
}
res.Paths = append(res.Paths, path)
}
}
if err := json.NewEncoder(conn).Encode(res); err != nil {
return fmt.Errorf("encode resonse: %s", err.Error())
}
return nil
}
func TestMetaService_Subscriptions_Drop(t *testing.T) {
t.Parallel()
d, s, c := newServiceAndClient()
defer os.RemoveAll(d)
defer s.Close()
defer c.Close()
// Create a database to use
if _, err := c.CreateDatabase("db0"); err != nil {
t.Fatal(err)
}
// DROP SUBSCRIPTION returns ErrSubscriptionNotFound when the
// subscription is unknown.
err := c.DropSubscription("db0", "default", "foo")
if got, exp := err, meta.ErrSubscriptionNotFound; got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// Create a subscription.
if err := c.CreateSubscription("db0", "default", "sub0", "ALL", []string{"udp://example.com:9090"}); err != nil {
t.Fatal(err)
}
// DROP SUBSCRIPTION returns an influxdb.ErrDatabaseNotFound when
// the database is unknown.
err = c.DropSubscription("foo", "default", "sub0")
if got, exp := err, influxdb.ErrDatabaseNotFound("foo"); got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// DROP SUBSCRIPTION returns an influxdb.ErrRetentionPolicyNotFound
// when the retention policy is unknown.
err = c.DropSubscription("db0", "foo_policy", "sub0")
if got, exp := err, influxdb.ErrRetentionPolicyNotFound("foo_policy"); got.Error() != exp.Error() {
t.Fatalf("got: %s, exp: %s", got, exp)
}
// DROP SUBSCRIPTION drops the subsciption if it can find it.
err = c.DropSubscription("db0", "default", "sub0")
if got := err; got != nil {
t.Fatalf("got: %s, exp: %v", got, nil)
}
}
// DeleteShardGroup removes a shard group from a database and retention policy by id.
func (data *Data) DeleteShardGroup(database, policy string, id uint64) error {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Find shard group by ID and set its deletion timestamp.
for i := range rpi.ShardGroups {
if rpi.ShardGroups[i].ID == id {
rpi.ShardGroups[i].DeletedAt = time.Now().UTC()
return nil
}
}
return ErrShardGroupNotFound
}
// UpdateRetentionPolicy updates an existing retention policy.
func (data *Data) UpdateRetentionPolicy(database, name string, rpu *RetentionPolicyUpdate) error {
// Find database.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
}
// Find policy.
rpi := di.RetentionPolicy(name)
if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Ensure new policy doesn't match an existing policy.
if rpu.Name != nil && *rpu.Name != name && di.RetentionPolicy(*rpu.Name) != nil {
return ErrRetentionPolicyNameExists
}
// Enforce duration of at least MinRetentionPolicyDuration
if rpu.Duration != nil && *rpu.Duration < MinRetentionPolicyDuration && *rpu.Duration != 0 {
return ErrRetentionPolicyDurationTooLow
}
// Update fields.
if rpu.Name != nil {
rpi.Name = *rpu.Name
}
if rpu.Duration != nil {
rpi.Duration = *rpu.Duration
}
if rpu.ReplicaN != nil {
rpi.ReplicaN = *rpu.ReplicaN
}
if rpu.ShardGroupDuration != nil {
rpi.ShardGroupDuration = *rpu.ShardGroupDuration
} else {
rpi.ShardGroupDuration = shardGroupDuration(rpi.Duration)
}
return nil
}
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data
// for the specified time range. Shard groups are sorted by start time.
func (c *Client) ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []ShardGroupInfo, err error) {
c.mu.RLock()
defer c.mu.RUnlock()
// Find retention policy.
rpi, err := c.cacheData.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups))
for _, g := range rpi.ShardGroups {
if g.Deleted() || !g.Overlaps(min, max) {
continue
}
groups = append(groups, g)
}
return groups, nil
}
// CreateShardGroup creates a shard group on a database and policy for a given timestamp.
func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) error {
// Ensure there are nodes in the metadata.
if len(data.DataNodes) == 0 {
return nil
}
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Verify that shard group doesn't already exist for this timestamp.
if rpi.ShardGroupByTimestamp(timestamp) != nil {
return nil
}
// Require at least one replica but no more replicas than nodes.
replicaN := rpi.ReplicaN
if replicaN == 0 {
replicaN = 1
} else if replicaN > len(data.DataNodes) {
replicaN = len(data.DataNodes)
}
// Determine shard count by node count divided by replication factor.
// This will ensure nodes will get distributed across nodes evenly and
// replicated the correct number of times.
shardN := len(data.DataNodes) / replicaN
// Create the shard group.
data.MaxShardGroupID++
sgi := ShardGroupInfo{}
sgi.ID = data.MaxShardGroupID
sgi.StartTime = timestamp.Truncate(rpi.ShardGroupDuration).UTC()
sgi.EndTime = sgi.StartTime.Add(rpi.ShardGroupDuration).UTC()
// Create shards on the group.
sgi.Shards = make([]ShardInfo, shardN)
for i := range sgi.Shards {
data.MaxShardID++
sgi.Shards[i] = ShardInfo{ID: data.MaxShardID}
}
// Assign data nodes to shards via round robin.
// Start from a repeatably "random" place in the node list.
nodeIndex := int(data.Index % uint64(len(data.DataNodes)))
for i := range sgi.Shards {
si := &sgi.Shards[i]
for j := 0; j < replicaN; j++ {
nodeID := data.DataNodes[nodeIndex%len(data.DataNodes)].ID
si.Owners = append(si.Owners, ShardOwner{NodeID: nodeID})
nodeIndex++
}
}
// Retention policy has a new shard group, so update the policy. Shard
// Groups must be stored in sorted order, as other parts of the system
// assume this to be the case.
rpi.ShardGroups = append(rpi.ShardGroups, sgi)
sort.Sort(ShardGroupInfos(rpi.ShardGroups))
return nil
}