// createInternalStorage ensures the internal storage has been created.
func (m *Monitor) createInternalStorage() {
if !m.MetaStore.IsLeader() || m.storeCreated {
return
}
if _, err := m.MetaStore.CreateDatabaseIfNotExists(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', failed to create storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy)
rpi.Duration = MonitorRetentionPolicyDuration
rpi.ReplicaN = 1
if _, err := m.MetaStore.CreateRetentionPolicyIfNotExists(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', failed to create internal storage: %s",
rpi.Name, err.Error())
return
}
if err := m.MetaStore.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil {
m.Logger.Printf("failed to set default retention policy on '%s', failed to create internal storage: %s",
m.storeDatabase, err.Error())
return
}
if err := m.MetaStore.DropRetentionPolicy(m.storeDatabase, "default"); err != nil && err != meta.ErrRetentionPolicyNotFound {
m.Logger.Printf("failed to delete retention policy 'default', failed to created internal storage: %s", err.Error())
return
}
// Mark storage creation complete.
m.storeCreated = true
}
// storeStatistics writes the statistics to an InfluxDB system.
func (m *Monitor) storeStatistics() {
defer m.wg.Done()
m.Logger.Printf("Storing statistics in database '%s' retention policy '%s', at interval %s",
m.storeDatabase, m.storeRetentionPolicy, m.storeInterval)
if err := m.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
m.Logger.Printf("failed to detect a cluster leader, terminating storage: %s", err.Error())
return
}
if _, err := m.MetaStore.CreateDatabaseIfNotExists(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', terminating storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(m.storeRetentionPolicy)
rpi.Duration = m.storeRetentionDuration
rpi.ReplicaN = m.storeReplicationFactor
if _, err := m.MetaStore.CreateRetentionPolicyIfNotExists(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', terminating storage: %s",
m.storeRetentionPolicy, err.Error())
return
}
tick := time.NewTicker(m.storeInterval)
defer tick.Stop()
for {
select {
case <-tick.C:
stats, err := m.Statistics()
if err != nil {
m.Logger.Printf("failed to retrieve registered statistics: %s", err)
continue
}
points := make(tsdb.Points, 0, len(stats))
for _, s := range stats {
points = append(points, tsdb.NewPoint(s.Name, s.Tags, s.Values, time.Now()))
}
err = m.PointsWriter.WritePoints(&cluster.WritePointsRequest{
Database: m.storeDatabase,
RetentionPolicy: m.storeRetentionPolicy,
ConsistencyLevel: cluster.ConsistencyLevelOne,
Points: points,
})
if err != nil {
m.Logger.Printf("failed to store statistics: %s", err)
}
case <-m.done:
m.Logger.Printf("terminating storage of statistics")
return
}
}
}
// storeStatistics writes the statistics to an InfluxDB system.
func (m *Monitor) storeStatistics() {
defer m.wg.Done()
m.Logger.Printf("Storing statistics in database '%s' retention policy '%s', at interval %s",
m.storeDatabase, m.storeRetentionPolicy, m.storeInterval)
if err := m.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
m.Logger.Printf("failed to detect a cluster leader, terminating storage: %s", err.Error())
return
}
// Get cluster-level metadata. Nothing different is going to happen if errors occur.
clusterID, _ := m.MetaStore.ClusterID()
nodeID := m.MetaStore.NodeID()
hostname, _ := os.Hostname()
clusterTags := map[string]string{
"clusterID": fmt.Sprintf("%d", clusterID),
"nodeID": fmt.Sprintf("%d", nodeID),
"hostname": hostname,
}
if _, err := m.MetaStore.CreateDatabaseIfNotExists(m.storeDatabase); err != nil {
m.Logger.Printf("failed to create database '%s', terminating storage: %s",
m.storeDatabase, err.Error())
return
}
rpi := meta.NewRetentionPolicyInfo(MonitorRetentionPolicy)
rpi.Duration = MonitorRetentionPolicyDuration
rpi.ReplicaN = 1
if _, err := m.MetaStore.CreateRetentionPolicyIfNotExists(m.storeDatabase, rpi); err != nil {
m.Logger.Printf("failed to create retention policy '%s', terminating storage: %s",
rpi.Name, err.Error())
return
}
if err := m.MetaStore.SetDefaultRetentionPolicy(m.storeDatabase, rpi.Name); err != nil {
m.Logger.Printf("failed to set default retention policy on '%s', terminating storage: %s",
m.storeDatabase, err.Error())
return
}
if err := m.MetaStore.DropRetentionPolicy(m.storeDatabase, "default"); err != nil && err != meta.ErrRetentionPolicyNotFound {
m.Logger.Printf("failed to delete retention policy 'default', terminating storage: %s", err.Error())
return
}
tick := time.NewTicker(m.storeInterval)
defer tick.Stop()
for {
select {
case <-tick.C:
stats, err := m.Statistics(clusterTags)
if err != nil {
m.Logger.Printf("failed to retrieve registered statistics: %s", err)
continue
}
points := make(models.Points, 0, len(stats))
for _, s := range stats {
points = append(points, models.NewPoint(s.Name, s.Tags, s.Values, time.Now()))
}
err = m.PointsWriter.WritePoints(&cluster.WritePointsRequest{
Database: m.storeDatabase,
RetentionPolicy: m.storeRetentionPolicy,
ConsistencyLevel: cluster.ConsistencyLevelOne,
Points: points,
})
if err != nil {
m.Logger.Printf("failed to store statistics: %s", err)
}
case <-m.done:
m.Logger.Printf("terminating storage of statistics")
return
}
}
}