// CreateShardGroup creates a shard group on a database and policy for a given timestamp. func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) error { // Find retention policy. rpi, err := data.RetentionPolicy(database, policy) if err != nil { return err } else if rpi == nil { return influxdb.ErrRetentionPolicyNotFound(policy) } // Verify that shard group doesn't already exist for this timestamp. if rpi.ShardGroupByTimestamp(timestamp) != nil { return nil } // Create the shard group. data.MaxShardGroupID++ sgi := ShardGroupInfo{} sgi.ID = data.MaxShardGroupID sgi.StartTime = timestamp.Truncate(rpi.ShardGroupDuration).UTC() sgi.EndTime = sgi.StartTime.Add(rpi.ShardGroupDuration).UTC() data.MaxShardID++ sgi.Shards = []ShardInfo{ {ID: data.MaxShardID}, } // Retention policy has a new shard group, so update the policy. Shard // Groups must be stored in sorted order, as other parts of the system // assume this to be the case. rpi.ShardGroups = append(rpi.ShardGroups, sgi) sort.Sort(ShardGroupInfos(rpi.ShardGroups)) return nil }
// MapShards maps the points contained in wp to a ShardMapping. If a point // maps to a shard group or shard that does not currently exist, it will be // created before returning the mapping. func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) { // holds the start time ranges for required shard groups timeRanges := map[time.Time]*meta.ShardGroupInfo{} rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy) if err != nil { return nil, err } if rp == nil { return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy) } for _, p := range wp.Points { timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil } // holds all the shard groups and shards that are required for writes for t := range timeRanges { sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, t) if err != nil { return nil, err } timeRanges[t] = sg } mapping := NewShardMapping() for _, p := range wp.Points { sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] sh := sg.ShardFor(p.HashID()) mapping.MapPoint(&sh, p) } return mapping, nil }
// createInternalStorage ensures the internal storage has been created. func (m *Monitor) createInternalStorage() { if m.storeCreated { return } if _, err := m.MetaClient.CreateDatabase(m.storeDatabase); err != nil { m.Logger.Printf("failed to create database '%s', failed to create storage: %s", m.storeDatabase, err.Error()) return } rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy) rpi.Duration = MonitorRetentionPolicyDuration rpi.ReplicaN = 1 if _, err := m.MetaClient.CreateRetentionPolicy(m.storeDatabase, rpi); err != nil { m.Logger.Printf("failed to create retention policy '%s', failed to create internal storage: %s", rpi.Name, err.Error()) return } if err := m.MetaClient.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil { m.Logger.Printf("failed to set default retention policy on '%s', failed to create internal storage: %s", m.storeDatabase, err.Error()) return } err := m.MetaClient.DropRetentionPolicy(m.storeDatabase, "default") if err != nil && err.Error() != influxdb.ErrRetentionPolicyNotFound("default").Error() { m.Logger.Printf("failed to delete retention policy 'default', failed to created internal storage: %s", err.Error()) return } // Mark storage creation complete. m.storeCreated = true }
// ShardGroupByTimestamp returns the shard group on a database and policy for a given timestamp. func (data *Data) ShardGroupByTimestamp(database, policy string, timestamp time.Time) (*ShardGroupInfo, error) { // Find retention policy. rpi, err := data.RetentionPolicy(database, policy) if err != nil { return nil, err } else if rpi == nil { return nil, influxdb.ErrRetentionPolicyNotFound(policy) } return rpi.ShardGroupByTimestamp(timestamp), nil }
// SetDefaultRetentionPolicy sets the default retention policy for a database. func (data *Data) SetDefaultRetentionPolicy(database, name string) error { // Find database and verify policy exists. di := data.Database(database) if di == nil { return influxdb.ErrDatabaseNotFound(database) } else if di.RetentionPolicy(name) == nil { return influxdb.ErrRetentionPolicyNotFound(name) } // Set default policy. di.DefaultRetentionPolicy = name return nil }
// DropSubscription removes a subscription. func (data *Data) DropSubscription(database, rp, name string) error { rpi, err := data.RetentionPolicy(database, rp) if err != nil { return err } else if rpi == nil { return influxdb.ErrRetentionPolicyNotFound(rp) } for i := range rpi.Subscriptions { if rpi.Subscriptions[i].Name == name { rpi.Subscriptions = append(rpi.Subscriptions[:i], rpi.Subscriptions[i+1:]...) return nil } } return ErrSubscriptionNotFound }
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data // for the specified time range. Shard groups are sorted by start time. func (data *Data) ShardGroupsByTimeRange(database, policy string, tmin, tmax time.Time) ([]ShardGroupInfo, error) { // Find retention policy. rpi, err := data.RetentionPolicy(database, policy) if err != nil { return nil, err } else if rpi == nil { return nil, influxdb.ErrRetentionPolicyNotFound(policy) } groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups)) for _, g := range rpi.ShardGroups { if g.Deleted() || !g.Overlaps(tmin, tmax) { continue } groups = append(groups, g) } return groups, nil }
// writeDatabaseInfo will write the relative paths of all shards in the retention policy on // this server into the connection func (s *Service) writeRetentionPolicyInfo(conn net.Conn, database, retentionPolicy string) error { res := Response{} db, err := s.MetaClient.Database(database) if err != nil { return err } if db == nil { return influxdb.ErrDatabaseNotFound(database) } var ret *meta.RetentionPolicyInfo for _, rp := range db.RetentionPolicies { if rp.Name == retentionPolicy { ret = &rp break } } if ret == nil { return influxdb.ErrRetentionPolicyNotFound(retentionPolicy) } for _, sg := range ret.ShardGroups { for _, sh := range sg.Shards { // ignore if the shard isn't on the server if s.TSDBStore.Shard(sh.ID) == nil { continue } path, err := s.TSDBStore.ShardRelativePath(sh.ID) if err != nil { return err } res.Paths = append(res.Paths, path) } } if err := json.NewEncoder(conn).Encode(res); err != nil { return fmt.Errorf("encode resonse: %s", err.Error()) } return nil }
// DeleteShardGroup removes a shard group from a database and retention policy by id. func (data *Data) DeleteShardGroup(database, policy string, id uint64) error { // Find retention policy. rpi, err := data.RetentionPolicy(database, policy) if err != nil { return err } else if rpi == nil { return influxdb.ErrRetentionPolicyNotFound(policy) } // Find shard group by ID and set its deletion timestamp. for i := range rpi.ShardGroups { if rpi.ShardGroups[i].ID == id { rpi.ShardGroups[i].DeletedAt = time.Now().UTC() return nil } } return ErrShardGroupNotFound }
// UpdateRetentionPolicy updates an existing retention policy. func (data *Data) UpdateRetentionPolicy(database, name string, rpu *RetentionPolicyUpdate) error { // Find database. di := data.Database(database) if di == nil { return influxdb.ErrDatabaseNotFound(database) } // Find policy. rpi := di.RetentionPolicy(name) if rpi == nil { return influxdb.ErrRetentionPolicyNotFound(name) } // Ensure new policy doesn't match an existing policy. if rpu.Name != nil && *rpu.Name != name && di.RetentionPolicy(*rpu.Name) != nil { return ErrRetentionPolicyNameExists } // Enforce duration of at least MinRetentionPolicyDuration if rpu.Duration != nil && *rpu.Duration < MinRetentionPolicyDuration && *rpu.Duration != 0 { return ErrRetentionPolicyDurationTooLow } // Update fields. if rpu.Name != nil { rpi.Name = *rpu.Name } if rpu.Duration != nil { rpi.Duration = *rpu.Duration } if rpu.ReplicaN != nil { rpi.ReplicaN = *rpu.ReplicaN } if rpu.ShardGroupDuration != nil { rpi.ShardGroupDuration = *rpu.ShardGroupDuration } else { rpi.ShardGroupDuration = shardGroupDuration(rpi.Duration) } return nil }
// ShardGroupsByTimeRange returns a list of all shard groups on a database and policy that may contain data // for the specified time range. Shard groups are sorted by start time. func (c *Client) ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []ShardGroupInfo, err error) { c.mu.RLock() defer c.mu.RUnlock() // Find retention policy. rpi, err := c.cacheData.RetentionPolicy(database, policy) if err != nil { return nil, err } else if rpi == nil { return nil, influxdb.ErrRetentionPolicyNotFound(policy) } groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups)) for _, g := range rpi.ShardGroups { if g.Deleted() || !g.Overlaps(min, max) { continue } groups = append(groups, g) } return groups, nil }
// CreateSubscription adds a named subscription to a database and retention policy. func (data *Data) CreateSubscription(database, rp, name, mode string, destinations []string) error { rpi, err := data.RetentionPolicy(database, rp) if err != nil { return err } else if rpi == nil { return influxdb.ErrRetentionPolicyNotFound(rp) } // Ensure the name doesn't already exist. for i := range rpi.Subscriptions { if rpi.Subscriptions[i].Name == name { return ErrSubscriptionExists } } // Append new query. rpi.Subscriptions = append(rpi.Subscriptions, SubscriptionInfo{ Name: name, Mode: mode, Destinations: destinations, }) return nil }