// CreateShardGroup creates a shard group on a database and policy for a given timestamp.
func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) error {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Verify that shard group doesn't already exist for this timestamp.
if rpi.ShardGroupByTimestamp(timestamp) != nil {
return nil
}
// Create the shard group.
data.MaxShardGroupID++
sgi := ShardGroupInfo{}
sgi.ID = data.MaxShardGroupID
sgi.StartTime = timestamp.Truncate(rpi.ShardGroupDuration).UTC()
sgi.EndTime = sgi.StartTime.Add(rpi.ShardGroupDuration).UTC()
data.MaxShardID++
sgi.Shards = []ShardInfo{
{ID: data.MaxShardID},
}
// Retention policy has a new shard group, so update the policy. Shard
// Groups must be stored in sorted order, as other parts of the system
// assume this to be the case.
rpi.ShardGroups = append(rpi.ShardGroups, sgi)
sort.Sort(ShardGroupInfos(rpi.ShardGroups))
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
// holds the start time ranges for required shard groups
timeRanges := map[time.Time]*meta.ShardGroupInfo{}
rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
}
if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
for _, p := range wp.Points {
timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
}
// holds all the shard groups and shards that are required for writes
for t := range timeRanges {
sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, t)
if err != nil {
return nil, err
}
timeRanges[t] = sg
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// createInternalStorage ensures the internal storage has been created.
func (m *Monitor) createInternalStorage() {
if m.storeCreated {
return
}
if _, err := m.MetaClient.CreateDatabase(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', failed to create storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy)
rpi.Duration = MonitorRetentionPolicyDuration
rpi.ReplicaN = 1
if _, err := m.MetaClient.CreateRetentionPolicy(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', failed to create internal storage: %s",
rpi.Name, err.Error())
return
}
if err := m.MetaClient.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil {
m.Logger.Printf("failed to set default retention policy on '%s', failed to create internal storage: %s",
m.storeDatabase, err.Error())
return
}
err := m.MetaClient.DropRetentionPolicy(m.storeDatabase, "default")
if err != nil && err.Error() != influxdb.ErrRetentionPolicyNotFound("default").Error() {
m.Logger.Printf("failed to delete retention policy 'default', failed to created internal storage: %s", err.Error())
return
}
// Mark storage creation complete.
m.storeCreated = true
}
// ShardGroupByTimestamp returns the shard group on a database and policy for a given timestamp.
func (data *Data) ShardGroupByTimestamp(database, policy string, timestamp time.Time) (*ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
return rpi.ShardGroupByTimestamp(timestamp), nil
}
// SetDefaultRetentionPolicy sets the default retention policy for a database.
func (data *Data) SetDefaultRetentionPolicy(database, name string) error {
// Find database and verify policy exists.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
} else if di.RetentionPolicy(name) == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Set default policy.
di.DefaultRetentionPolicy = name
return nil
}
// DropSubscription removes a subscription.
func (data *Data) DropSubscription(database, rp, name string) error {
rpi, err := data.RetentionPolicy(database, rp)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(rp)
}
for i := range rpi.Subscriptions {
if rpi.Subscriptions[i].Name == name {
rpi.Subscriptions = append(rpi.Subscriptions[:i], rpi.Subscriptions[i+1:]...)
return nil
}
}
return ErrSubscriptionNotFound
}
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data
// for the specified time range. Shard groups are sorted by start time.
func (data *Data) ShardGroupsByTimeRange(database, policy string, tmin, tmax time.Time) ([]ShardGroupInfo, error) {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups))
for _, g := range rpi.ShardGroups {
if g.Deleted() || !g.Overlaps(tmin, tmax) {
continue
}
groups = append(groups, g)
}
return groups, nil
}
// writeDatabaseInfo will write the relative paths of all shards in the retention policy on
// this server into the connection
func (s *Service) writeRetentionPolicyInfo(conn net.Conn, database, retentionPolicy string) error {
res := Response{}
db, err := s.MetaClient.Database(database)
if err != nil {
return err
}
if db == nil {
return influxdb.ErrDatabaseNotFound(database)
}
var ret *meta.RetentionPolicyInfo
for _, rp := range db.RetentionPolicies {
if rp.Name == retentionPolicy {
ret = &rp
break
}
}
if ret == nil {
return influxdb.ErrRetentionPolicyNotFound(retentionPolicy)
}
for _, sg := range ret.ShardGroups {
for _, sh := range sg.Shards {
// ignore if the shard isn't on the server
if s.TSDBStore.Shard(sh.ID) == nil {
continue
}
path, err := s.TSDBStore.ShardRelativePath(sh.ID)
if err != nil {
return err
}
res.Paths = append(res.Paths, path)
}
}
if err := json.NewEncoder(conn).Encode(res); err != nil {
return fmt.Errorf("encode resonse: %s", err.Error())
}
return nil
}
// DeleteShardGroup removes a shard group from a database and retention policy by id.
func (data *Data) DeleteShardGroup(database, policy string, id uint64) error {
// Find retention policy.
rpi, err := data.RetentionPolicy(database, policy)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(policy)
}
// Find shard group by ID and set its deletion timestamp.
for i := range rpi.ShardGroups {
if rpi.ShardGroups[i].ID == id {
rpi.ShardGroups[i].DeletedAt = time.Now().UTC()
return nil
}
}
return ErrShardGroupNotFound
}
// UpdateRetentionPolicy updates an existing retention policy.
func (data *Data) UpdateRetentionPolicy(database, name string, rpu *RetentionPolicyUpdate) error {
// Find database.
di := data.Database(database)
if di == nil {
return influxdb.ErrDatabaseNotFound(database)
}
// Find policy.
rpi := di.RetentionPolicy(name)
if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(name)
}
// Ensure new policy doesn't match an existing policy.
if rpu.Name != nil && *rpu.Name != name && di.RetentionPolicy(*rpu.Name) != nil {
return ErrRetentionPolicyNameExists
}
// Enforce duration of at least MinRetentionPolicyDuration
if rpu.Duration != nil && *rpu.Duration < MinRetentionPolicyDuration && *rpu.Duration != 0 {
return ErrRetentionPolicyDurationTooLow
}
// Update fields.
if rpu.Name != nil {
rpi.Name = *rpu.Name
}
if rpu.Duration != nil {
rpi.Duration = *rpu.Duration
}
if rpu.ReplicaN != nil {
rpi.ReplicaN = *rpu.ReplicaN
}
if rpu.ShardGroupDuration != nil {
rpi.ShardGroupDuration = *rpu.ShardGroupDuration
} else {
rpi.ShardGroupDuration = shardGroupDuration(rpi.Duration)
}
return nil
}
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data
// for the specified time range. Shard groups are sorted by start time.
func (c *Client) ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []ShardGroupInfo, err error) {
c.mu.RLock()
defer c.mu.RUnlock()
// Find retention policy.
rpi, err := c.cacheData.RetentionPolicy(database, policy)
if err != nil {
return nil, err
} else if rpi == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(policy)
}
groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups))
for _, g := range rpi.ShardGroups {
if g.Deleted() || !g.Overlaps(min, max) {
continue
}
groups = append(groups, g)
}
return groups, nil
}
// CreateSubscription adds a named subscription to a database and retention policy.
func (data *Data) CreateSubscription(database, rp, name, mode string, destinations []string) error {
rpi, err := data.RetentionPolicy(database, rp)
if err != nil {
return err
} else if rpi == nil {
return influxdb.ErrRetentionPolicyNotFound(rp)
}
// Ensure the name doesn't already exist.
for i := range rpi.Subscriptions {
if rpi.Subscriptions[i].Name == name {
return ErrSubscriptionExists
}
}
// Append new query.
rpi.Subscriptions = append(rpi.Subscriptions, SubscriptionInfo{
Name: name,
Mode: mode,
Destinations: destinations,
})
return nil
}