// CreateSubscription adds a named subscription to a database and retention policy.
func (data *Data) CreateSubscription(database, rp, name, mode string, destinations []string) error {
rpi, err := data.RetentionPolicy(database, rp)
if err != nil {
return err
}
if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(rp)
}
// Ensure the name doesn't already exist.
for i := range rpi.Subscriptions {
if rpi.Subscriptions[i].Name == name {
return ErrSubscriptionExists
}
}
// Append new query.
rpi.Subscriptions = append(rpi.Subscriptions, SubscriptionInfo{
Name: name,
Mode: mode,
Destinations: destinations,
})
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
// holds the start time ranges for required shard groups
timeRanges := map[time.Time]*meta.ShardGroupInfo{}
rp, err := w.MetaStore.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
}
if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
for _, p := range wp.Points {
timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
}
// holds all the shard groups and shards that are required for writes
for t := range timeRanges {
sg, err := w.MetaStore.CreateShardGroupIfNotExists(wp.Database, wp.RetentionPolicy, t)
if err != nil {
return nil, err
}
timeRanges[t] = sg
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// createInternalStorage ensures the internal storage has been created.
func (m *Monitor) createInternalStorage() {
if m.storeCreated {
return
}
if _, err := m.MetaClient.CreateDatabase(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', failed to create storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy)
rpi.Duration = MonitorRetentionPolicyDuration
rpi.ReplicaN = 1
if _, err := m.MetaClient.CreateRetentionPolicy(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', failed to create internal storage: %s",
rpi.Name, err.Error())
return
}
if err := m.MetaClient.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil {
m.Logger.Printf("failed to set default retention policy on '%s', failed to create internal storage: %s",
m.storeDatabase, err.Error())
return
}
err := m.MetaClient.DropRetentionPolicy(m.storeDatabase, "default")
if err != nil && err.Error() != influxdb.ErrRetentionPolicyNotFound("default").Error() {
m.Logger.Printf("failed to delete retention policy 'default', failed to created internal storage: %s", err.Error())
return
}
// Mark storage creation complete.
m.storeCreated = true
}
// Ensure an error is returned when deleting a non-existent policy.
func TestData_DropRetentionPolicy_ErrRetentionPolicyNotFound(t *testing.T) {
var data meta.Data
if err := data.CreateDatabase("db0"); err != nil {
t.Fatal(err)
}
expErr := influxdb.ErrRetentionPolicyNotFound("rp0")
if err := data.DropRetentionPolicy("db0", "rp0"); err.Error() != expErr.Error() {
t.Fatal(err)
}
}
// ShardGroupByTimestamp returns the shard group on a database and policy for a given timestamp.
func (data *Data) ShardGroupByTimestamp(database, policy string, timestamp time.Time) (*ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
return rpi.ShardGroupByTimestamp(timestamp), nil
}
// SetDefaultRetentionPolicy sets the default retention policy for a database.
func (data *Data) SetDefaultRetentionPolicy(database, name string) error {
// Find database and verify policy exists.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
} else if di.RetentionPolicy(name) == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Set default policy.
di.DefaultRetentionPolicy = name
return nil
}
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data
// for the specified time range. Shard groups are sorted by start time.
func (data *Data) ShardGroupsByTimeRange(database, policy string, tmin, tmax time.Time) ([]ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups))
for _, g := range rpi.ShardGroups {
if g.Deleted() || !g.Overlaps(tmin, tmax) {
continue
}
groups = append(groups, g)
}
return groups, nil
}
// writeDatabaseInfo will write the relative paths of all shards in the retention policy on
// this server into the connection
func (s *Service) writeRetentionPolicyInfo(conn net.Conn, database, retentionPolicy string) error {
res := Response{}
db, err := s.MetaClient.Database(database)
if err != nil {
return err
}
if db == nil {
return influxdb.ErrDatabaseNotFound(database)
}
var ret *meta.RetentionPolicyInfo
for _, rp := range db.RetentionPolicies {
if rp.Name == retentionPolicy {
ret = &rp
break
}
}
if ret == nil {
return influxdb.ErrRetentionPolicyNotFound(retentionPolicy)
}
for _, sg := range ret.ShardGroups {
for _, sh := range sg.Shards {
// ignore if the shard isn't on the server
if s.TSDBStore.Shard(sh.ID) == nil {
continue
}
path, err := s.TSDBStore.ShardRelativePath(sh.ID)
if err != nil {
return err
}
res.Paths = append(res.Paths, path)
}
}
if err := json.NewEncoder(conn).Encode(res); err != nil {
return fmt.Errorf("encode resonse: %s", err.Error())
}
return nil
}
// DeleteShardGroup removes a shard group from a database and retention policy by id.
func (data *Data) DeleteShardGroup(database, policy string, id uint64) error {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Find shard group by ID and set its deletion timestamp.
for i := range rpi.ShardGroups {
if rpi.ShardGroups[i].ID == id {
rpi.ShardGroups[i].DeletedAt = time.Now().UTC()
return nil
}
}
return ErrShardGroupNotFound
}
// DropRetentionPolicy removes a retention policy from a database by name.
func (data *Data) DropRetentionPolicy(database, name string) error {
// Find database.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
}
// Prohibit dropping the default retention policy.
if di.DefaultRetentionPolicy == name {
return ErrRetentionPolicyDefault
}
// Remove from list.
for i := range di.RetentionPolicies {
if di.RetentionPolicies[i].Name == name {
di.RetentionPolicies = append(di.RetentionPolicies[:i], di.RetentionPolicies[i+1:]...)
return nil
}
}
return influxdb.ErrRetentionPolicyNotFound(name)
}
// UpdateRetentionPolicy updates an existing retention policy.
func (data *Data) UpdateRetentionPolicy(database, name string, rpu *RetentionPolicyUpdate) error {
// Find database.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
}
// Find policy.
rpi := di.RetentionPolicy(name)
if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Ensure new policy doesn't match an existing policy.
if rpu.Name != nil && *rpu.Name != name && di.RetentionPolicy(*rpu.Name) != nil {
return ErrRetentionPolicyNameExists
}
// Enforce duration of at least MinRetentionPolicyDuration
if rpu.Duration != nil && *rpu.Duration < MinRetentionPolicyDuration && *rpu.Duration != 0 {
return ErrRetentionPolicyDurationTooLow
}
// Update fields.
if rpu.Name != nil {
rpi.Name = *rpu.Name
}
if rpu.Duration != nil {
rpi.Duration = *rpu.Duration
rpi.ShardGroupDuration = shardGroupDuration(rpi.Duration)
}
if rpu.ReplicaN != nil {
rpi.ReplicaN = *rpu.ReplicaN
}
return nil
}
// CreateShardGroup creates a shard group on a database and policy for a given timestamp.
func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) error {
// Ensure there are nodes in the metadata.
if len(data.DataNodes) == 0 {
return nil
}
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Verify that shard group doesn't already exist for this timestamp.
if rpi.ShardGroupByTimestamp(timestamp) != nil {
return nil
}
// Require at least one replica but no more replicas than nodes.
replicaN := rpi.ReplicaN
if replicaN == 0 {
replicaN = 1
} else if replicaN > len(data.DataNodes) {
replicaN = len(data.DataNodes)
}
// Determine shard count by node count divided by replication factor.
// This will ensure nodes will get distributed across nodes evenly and
// replicated the correct number of times.
shardN := len(data.DataNodes) / replicaN
// Create the shard group.
data.MaxShardGroupID++
sgi := ShardGroupInfo{}
sgi.ID = data.MaxShardGroupID
sgi.StartTime = timestamp.Truncate(rpi.ShardGroupDuration).UTC()
sgi.EndTime = sgi.StartTime.Add(rpi.ShardGroupDuration).UTC()
// Create shards on the group.
sgi.Shards = make([]ShardInfo, shardN)
for i := range sgi.Shards {
data.MaxShardID++
sgi.Shards[i] = ShardInfo{ID: data.MaxShardID}
}
// Assign data nodes to shards via round robin.
// Start from a repeatably "random" place in the node list.
nodeIndex := int(data.Index % uint64(len(data.DataNodes)))
for i := range sgi.Shards {
si := &sgi.Shards[i]
for j := 0; j < replicaN; j++ {
nodeID := data.DataNodes[nodeIndex%len(data.DataNodes)].ID
si.Owners = append(si.Owners, ShardOwner{NodeID: nodeID})
nodeIndex++
}
}
// Retention policy has a new shard group, so update the policy. Shard
// Groups must be stored in sorted order, as other parts of the system
// assume this to be the case.
rpi.ShardGroups = append(rpi.ShardGroups, sgi)
sort.Sort(ShardGroupInfos(rpi.ShardGroups))
return nil
}