func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
inst := &driver{
IoNotSupported: &volume.IoNotSupported{},
DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
}
inst.buseDevices = make(map[string]*buseDev)
if err := os.MkdirAll(BuseMountPath, 0744); err != nil {
return nil, err
}
volumeInfo, err := inst.DefaultEnumerator.Enumerate(
&api.VolumeLocator{},
nil,
)
if err == nil {
for _, info := range volumeInfo {
if info.Status == api.VolumeStatus_VOLUME_STATUS_NONE {
info.Status = api.VolumeStatus_VOLUME_STATUS_UP
inst.UpdateVol(info)
}
}
} else {
dlog.Println("Could not enumerate Volumes, ", err)
}
c, err := cluster.Inst()
if err != nil {
dlog.Println("BUSE initializing in single node mode")
} else {
dlog.Println("BUSE initializing in clustered mode")
c.AddEventListener(inst)
}
dlog.Println("BUSE initialized and driver mounted at: ", BuseMountPath)
return inst, nil
}
func watchTree(t *testing.T) {
fmt.Println("watchTree")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
tree := "tree"
watchData := watchData{
t: t,
key: tree + "/key",
stop: "stop",
iterations: 2,
}
kv.Delete(watchData.key)
time.Sleep(time.Second)
err := kv.WatchTree(tree, 0, &watchData, watchFn)
if err != nil {
fmt.Printf("Cannot test watchKey: %v\n", err)
return
}
go watchUpdate(&watchData)
for watchData.watchStopped == false {
time.Sleep(time.Millisecond * 100)
}
}
func deleteTree(t *testing.T) {
fmt.Println("deleteTree")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
prefix := "tree"
keys := map[string]string{
prefix + "/1cbc9a98-072a-4793-8608-01ab43db96c8": "bar",
prefix + "/foo": "baz",
}
for key, val := range keys {
_, err := kv.Put(key, []byte(val), 10)
assert.NoError(t, err, "Unexpected error on Put")
}
for key, _ := range keys {
_, err := kv.Get(key)
assert.NoError(t, err, "Unexpected error on Get")
}
err := kv.DeleteTree(prefix)
assert.NoError(t, err, "Unexpected error on DeleteTree")
for key, _ := range keys {
_, err := kv.Get(key)
assert.Error(t, err, "Get should fail on all keys after DeleteTree")
}
}
func TestRaiseWithGenerateIdAndErase(t *testing.T) {
// RaiseWithGenerateId api.Alerts Id : 1
raiseAlerts, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_VOLUMES, Severity: api.SeverityType_NOTIFY}, mockGenerateId)
require.NoError(t, err, "Failed in raising an alert")
kv := kvdb.Instance()
var alert api.Alerts
_, err = kv.GetVal(getResourceKey(api.ResourceType_VOLUMES)+strconv.FormatInt(raiseAlerts.Id, 10), &alert)
require.NoError(t, err, "Failed to retrieve alert from kvdb")
require.NotNil(t, alert, "api.Alerts object null in kvdb")
require.Equal(t, raiseAlerts.Id, alert.Id, "api.Alerts Id mismatch")
require.Equal(t, api.ResourceType_VOLUMES, alert.Resource, "api.Alerts Resource mismatch")
require.Equal(t, api.SeverityType_NOTIFY, alert.Severity, "api.Alerts Severity mismatch")
// RaiseWithGenerateId api.Alerts with no Resource
_, err = kva.RaiseWithGenerateId(api.Alerts{Severity: api.SeverityType_NOTIFY}, mockGenerateId)
require.Error(t, err, "An error was expected")
require.Equal(t, ErrResourceNotFound, err, "Error mismatch")
// Erase api.Alerts Id : 1
err = kva.Erase(api.ResourceType_VOLUMES, raiseAlerts.Id)
require.NoError(t, err, "Failed to erase an alert")
_, err = kv.GetVal(getResourceKey(api.ResourceType_VOLUMES)+"1", &alert)
require.Error(t, err, "api.Alerts not erased from kvdb")
}
func update(t *testing.T) {
fmt.Println("update")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
key := "update/foo"
kv.Delete(key)
kvp, err := kv.Update(key, []byte("bar"), 0)
assert.Error(t, err, "Update should error on non-existent key")
defer func() {
kv.Delete(key)
}()
kvp, err = kv.Create(key, []byte("bar"), 0)
assert.NoError(t, err, "Unexpected error on create")
kvp, err = kv.Update(key, []byte("bar"), 0)
assert.NoError(t, err, "Unexpected error on update")
assert.Equal(t, kvp.Action, kvdb.KVSet,
"Expected action KVSet, actual %v", kvp.Action)
}
func cas(t *testing.T) {
fmt.Println("cas")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
key := "foo/docker"
val := "great"
defer func() {
kv.Delete(key)
}()
kvPair, err := kv.Put(key, []byte(val), 10)
assert.NoError(t, err, "Unxpected error in Put")
kvPair, err = kv.Get(key)
assert.NoError(t, err, "Failed in Get")
_, err = kv.CompareAndSet(kvPair, kvdb.KVFlags(0), []byte("badval"))
assert.Error(t, err, "CompareAndSet should fail on an incorrect previous value")
kvPair.ModifiedIndex++
_, err = kv.CompareAndSet(kvPair, kvdb.KVModifiedIndex, nil)
assert.Error(t, err, "CompareAndSet should fail on an incorrect modified index")
kvPair.ModifiedIndex--
kvPair, err = kv.CompareAndSet(kvPair, kvdb.KVModifiedIndex, nil)
assert.NoError(t, err, "CompareAndSet should succeed on an correct modified index")
kvPair, err = kv.CompareAndSet(kvPair, kvdb.KVFlags(0), []byte(val))
assert.NoError(t, err, "CompareAndSet should succeed on an correct value")
kvPair, err = kv.CompareAndSet(kvPair, kvdb.KVModifiedIndex, []byte(val))
assert.NoError(t, err, "CompareAndSet should succeed on an correct value and modified index")
}
func lock(t *testing.T) {
fmt.Println("lock")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
key := "locktest"
kvPair, err := kv.Lock(key, 100)
assert.NoError(t, err, "Unexpected error in lock")
stash := *kvPair
stash.Value = []byte("hoohah")
err = kv.Unlock(&stash)
assert.Error(t, err, "Unlock should fail for bad KVPair")
err = kv.Unlock(kvPair)
assert.NoError(t, err, "Unexpected error from Unlock")
kvPair, err = kv.Lock(key, 20)
assert.NoError(t, err, "Failed to lock after unlock")
err = kv.Unlock(kvPair)
assert.NoError(t, err, "Unexpected error from Unlock")
}
func readDatabase() (Database, error) {
kvdb := kv.Instance()
db := Database{Status: api.StatusInit,
NodeEntries: make(map[string]NodeEntry)}
kv, err := kvdb.Get("cluster/database")
if err != nil && !strings.Contains(err.Error(), "Key not found") {
log.Warn("Warning, could not read cluster database")
goto done
}
if kv == nil || bytes.Compare(kv.Value, []byte("{}")) == 0 {
log.Info("Cluster is uninitialized...")
err = nil
goto done
} else {
err = json.Unmarshal(kv.Value, &db)
if err != nil {
log.Warn("Fatal, Could not parse cluster database ", kv)
goto done
}
}
done:
return db, err
}
// Watch on all alerts
func (kva *KvAlerts) Watch(alertsWatcherFunc AlertsWatcherFunc) error {
prefix := alertsKey
alertsWatcher = watcher{status: watchBootstrap, cb: alertsWatcherFunc, kvcb: kvdbWatch}
if err := subscribeWatch(); err != nil {
return err
}
// Subscribe for a watch is in a goroutine. Bootsrap by writing to the key and waiting for an update
retries := 0
kv := kvdb.Instance()
for alertsWatcher.status == watchBootstrap {
_, err := kv.Put(prefix+bootstrap, time.Now(), 1)
if err != nil {
return err
}
if alertsWatcher.status == watchBootstrap {
retries++
time.Sleep(time.Millisecond * 100)
}
}
if alertsWatcher.status != watchReady {
return fmt.Errorf("Failed to watch on %v", prefix)
}
return nil
}
func getInterface(t *testing.T) {
fmt.Println("getInterface")
expected := struct {
N int
S string
}{
N: 10,
S: "Ten",
}
actual := expected
actual.N = 0
actual.S = "zero"
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
key := "DEADBEEF"
_, err := kv.Delete(key)
_, err = kv.Put(key, &expected, 0)
assert.NoError(t, err, "Failed in Put")
_, err = kv.GetVal(key, &actual)
assert.NoError(t, err, "Failed in Get")
assert.Equal(t, expected, actual, "Expected %#v but got %#v",
expected, actual)
}
func readDatabase() (Database, error) {
kvdb := kvdb.Instance()
db := Database{
Status: api.Status_STATUS_INIT,
NodeEntries: make(map[string]NodeEntry),
}
kv, err := kvdb.Get(ClusterDBKey)
if err != nil && !strings.Contains(err.Error(), "Key not found") {
dlog.Warnln("Warning, could not read cluster database")
return db, err
}
if kv == nil || bytes.Compare(kv.Value, []byte("{}")) == 0 {
dlog.Infoln("Cluster is uninitialized...")
return db, nil
}
if err := json.Unmarshal(kv.Value, &db); err != nil {
dlog.Warnln("Fatal, Could not parse cluster database ", kv)
return db, err
}
return db, nil
}
// Erase erases an alert
func (kva *KvAlerts) Erase(resourceType api.ResourceType, alertId int64) error {
kv := kvdb.Instance()
if resourceType == api.ResourceType_UNKNOWN_RESOURCE {
return ErrResourceNotFound
}
var err error
_, err = kv.Delete(getResourceKey(resourceType) + strconv.FormatInt(alertId, 10))
return err
}
func subscribeWatch() error {
watchIndex := alertsWatchIndex
if watchIndex != 0 {
watchIndex = alertsWatchIndex + 1
}
kv := kvdb.Instance()
if err := kv.WatchTree(alertsKey, watchIndex, nil, alertsWatcher.kvcb); err != nil {
return err
}
return nil
}
func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
restUrl, ok := params["restUrl"]
if !ok {
return nil, fmt.Errorf("rest api 'url' configuration parameter must be set")
}
user, ok := params["user"]
if !ok {
return nil, fmt.Errorf("rest auth 'user' must be set")
}
pass, ok := params["password"]
if !ok {
return nil, fmt.Errorf("rest auth 'password' must be set")
}
consistency_group, ok := params["consistency_group"]
if !ok {
return nil, fmt.Errorf("'consistency_group' configuration parameter must be set")
}
project, ok := params["project"]
if !ok {
return nil, fmt.Errorf("'project' configuration parameter must be set")
}
varray, ok := params["varray"]
if !ok {
return nil, fmt.Errorf("'varray' configuration parameter must be set")
}
vpool, ok := params["vpool"]
if !ok {
return nil, fmt.Errorf("'vpool' configuration parameter must be set")
}
d := &driver{
DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
consistency_group: consistency_group,
project: project,
varray: varray,
vpool: vpool,
url: restUrl,
creds: url.UserPassword(user, pass),
httpClient: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
},
}
return d, nil
}
func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
root, ok := params[RootParam]
if !ok {
return nil, fmt.Errorf("Root directory should be specified with key %q", RootParam)
}
home := path.Join(root, Volumes)
d, err := btrfs.Init(home, nil)
if err != nil {
return nil, err
}
s := volume.NewDefaultEnumerator(Name, kvdb.Instance())
return &driver{btrfs: d, root: root, DefaultEnumerator: s}, nil
}
// Init aws volume driver metadata.
func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
zone, err := metadata("placement/availability-zone")
if err != nil {
return nil, err
}
instance, err := metadata("instance-id")
if err != nil {
return nil, err
}
logrus.Infof("AWS instance %v zone %v", instance, zone)
accessKey, ok := params["AWS_ACCESS_KEY_ID"]
if !ok {
if accessKey = os.Getenv("AWS_ACCESS_KEY_ID"); accessKey == "" {
return nil, fmt.Errorf("AWS_ACCESS_KEY_ID environment variable must be set")
}
}
secretKey, ok := params["AWS_SECRET_ACCESS_KEY"]
if !ok {
if secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY"); secretKey == "" {
return nil, fmt.Errorf("AWS_SECRET_ACCESS_KEY environment variable must be set")
}
}
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
region := zone[:len(zone)-1]
d := &Driver{
ec2: ec2.New(
session.New(
&aws.Config{
Region: ®ion,
Credentials: creds,
},
),
),
md: &Metadata{
zone: zone,
instance: instance,
},
IoNotSupported: &volume.IoNotSupported{},
DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
}
devPrefix, letters, err := d.freeDevices()
if err != nil {
return nil, err
}
d.SingleLetter, err = device.NewSingleLetter(devPrefix, letters)
if err != nil {
return nil, err
}
return d, nil
}
func TestClear(t *testing.T) {
// RaiseWithGenerateId an alert
var alert api.Alerts
kv := kvdb.Instance()
raiseAlerts, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_NODE, Severity: api.SeverityType_ALARM}, mockGenerateId)
err = kva.Clear(api.ResourceType_NODE, raiseAlerts.Id)
require.NoError(t, err, "Failed to clear alert")
_, err = kv.GetVal(getResourceKey(api.ResourceType_NODE)+strconv.FormatInt(raiseAlerts.Id, 10), &alert)
require.Equal(t, true, alert.Cleared, "Failed to clear alert")
err = kva.Erase(api.ResourceType_NODE, raiseAlerts.Id)
}
// Retrieve retrieves a specific alert
func (kva *KvAlerts) Retrieve(resourceType api.ResourceType, alertId int64) (api.Alerts, error) {
var (
alert api.Alerts
err error
)
if resourceType == api.ResourceType_UNKNOWN_RESOURCE {
return api.Alerts{}, ErrResourceNotFound
}
kv := kvdb.Instance()
_, err = kv.GetVal(getResourceKey(resourceType)+strconv.FormatInt(alertId, 10), &alert)
return alert, err
}
func writeDatabase(db *Database) error {
kvdb := kvdb.Instance()
b, err := json.Marshal(db)
if err != nil {
dlog.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
return err
}
if _, err := kvdb.Put(ClusterDBKey, b, 0); err != nil {
dlog.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
return err
}
dlog.Infoln("Cluster database updated.")
return nil
}
func writeDatabase(db *Database) error {
kvdb := kvdb.Instance()
b, err := json.Marshal(db)
if err != nil {
logrus.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
return err
}
if _, err := kvdb.Put("cluster/database", b, 0); err != nil {
logrus.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
return err
}
logrus.Info("Cluster database updated.")
return nil
}
func newVolumeDriver(
name string,
baseDirPath string,
provider Provider,
) *volumeDriver {
return &volumeDriver{
name,
baseDirPath,
provider,
&volume.DefaultBlockDriver{},
volume.NewDefaultEnumerator(
name,
kvdb.Instance(),
),
&volume.SnapshotNotSupported{},
}
}
func raiseAnAlerts(a api.Alerts, generateId func() (int64, error)) (api.Alerts, error) {
kv := kvdb.Instance()
if a.Resource == api.ResourceType_UNKNOWN_RESOURCE {
return api.Alerts{}, ErrResourceNotFound
}
alertId, err := generateId()
if err != nil {
return a, err
}
a.Id = alertId
a.Timestamp = prototime.Now()
a.Cleared = false
_, err = kv.Create(getResourceKey(a.Resource)+strconv.FormatInt(a.Id, 10), &a, 0)
return a, err
}
func (kv *MemKV) Put(
key string,
value interface{},
ttl uint64) (*kvdb.KVPair, error) {
var kvp *kvdb.KVPair
key = kv.domain + key
kv.mutex.Lock()
defer kv.mutex.Unlock()
index := atomic.AddUint64(&kv.index, 1)
if ttl != 0 {
time.AfterFunc(time.Second*time.Duration(ttl), func() {
kvdb.Instance().Delete(key)
})
}
b, err := kv.toBytes(value)
if err != nil {
return nil, err
}
if old, ok := kv.m[key]; ok {
old.Value = b
old.Action = kvdb.KVSet
old.ModifiedIndex = index
old.KVDBIndex = index
kvp = old
} else {
kvp = &kvdb.KVPair{
Key: key,
Value: b,
TTL: int64(ttl),
KVDBIndex: index,
ModifiedIndex: index,
CreatedIndex: index,
Action: kvdb.KVCreate,
}
kv.m[key] = kvp
}
kv.normalize(kvp)
go kv.fireCB(key, kvp, nil)
return kvp, nil
}
func getResourceSpecificAlerts(resourceType api.ResourceType) ([]*api.Alerts, error) {
kv := kvdb.Instance()
allAlerts := []*api.Alerts{}
kvp, err := kv.Enumerate(getResourceKey(resourceType))
if err != nil {
return nil, err
}
for _, v := range kvp {
var elem *api.Alerts
err = json.Unmarshal(v.Value, &elem)
if err != nil {
return nil, err
}
allAlerts = append(allAlerts, elem)
}
return allAlerts, nil
}
func TestSetup(t *testing.T) {
kv := kvdb.Instance()
if kv == nil {
kv, err := kvdb.New(mem.Name, "openstorage/1", []string{}, nil)
if err != nil {
t.Fatalf("Failed to set default KV store : (%v): %v", mem.Name, err)
}
err = kvdb.SetInstance(kv)
if err != nil {
t.Fatalf("Failed to set default KV store: (%v): %v", mem.Name, err)
}
}
var err error
kva, err = New("alerts_kvdb")
if err != nil {
t.Fatalf("Failed to create new Kvapi.Alerts object")
}
}
// Clear clears an alert
func (kva *KvAlerts) Clear(resourceType api.ResourceType, alertId int64) error {
kv := kvdb.Instance()
var (
err error
alert api.Alerts
)
if resourceType == api.ResourceType_UNKNOWN_RESOURCE {
return ErrResourceNotFound
}
_, err = kv.GetVal(getResourceKey(resourceType)+strconv.FormatInt(alertId, 10), &alert)
if err != nil {
return err
}
alert.Cleared = true
_, err = kv.Update(getResourceKey(resourceType)+strconv.FormatInt(alertId, 10), &alert, 0)
return err
}
func TestEnumerateAlerts(t *testing.T) {
// RaiseWithGenerateId a few alerts
raiseAlert1, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_VOLUMES, Severity: api.SeverityType_NOTIFY}, mockGenerateId)
raiseAlert2, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_VOLUMES, Severity: api.SeverityType_NOTIFY}, mockGenerateId)
raiseAlert3, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_VOLUMES, Severity: api.SeverityType_WARNING}, mockGenerateId)
raiseAlert4, err := kva.RaiseWithGenerateId(api.Alerts{Resource: api.ResourceType_NODE, Severity: api.SeverityType_WARNING}, mockGenerateId)
enAlerts, err := kva.Enumerate(api.Alerts{Resource: api.ResourceType_VOLUMES})
require.NoError(t, err, "Failed to enumerate alerts")
require.Equal(t, 3, len(enAlerts), "Enumerated incorrect number of alerts")
enAlerts, err = kva.Enumerate(api.Alerts{Resource: api.ResourceType_VOLUMES, Severity: api.SeverityType_WARNING})
require.NoError(t, err, "Failed to enumerate alerts")
require.Equal(t, 1, len(enAlerts), "Enumerated incorrect number of alerts")
require.Equal(t, api.SeverityType_WARNING, enAlerts[0].Severity, "Severity mismatch")
enAlerts, err = kva.Enumerate(api.Alerts{})
require.NoError(t, err, "Failed to enumerate alerts")
require.Equal(t, 4, len(enAlerts), "Enumerated incorrect number of alerts")
enAlerts, err = kva.Enumerate(api.Alerts{Severity: api.SeverityType_WARNING})
require.NoError(t, err, "Failed to enumerate alerts")
require.Equal(t, 2, len(enAlerts), "Enumerated incorrect number of alerts")
require.Equal(t, api.SeverityType_WARNING, enAlerts[0].Severity, "Severity mismatch")
// Add a dummy event into kvdb two hours ago
kv := kvdb.Instance()
currentTime := time.Now()
delayedTime := currentTime.Add(-1 * time.Duration(2) * time.Hour)
alert := api.Alerts{Timestamp: prototime.TimeToTimestamp(delayedTime), Id: 10, Resource: api.ResourceType_VOLUMES}
_, err = kv.Put(getResourceKey(api.ResourceType_VOLUMES)+strconv.FormatInt(10, 10), &alert, 0)
enAlerts, err = kva.EnumerateWithinTimeRange(currentTime.Add(-1*time.Duration(10)*time.Second), currentTime, api.ResourceType_VOLUMES)
require.NoError(t, err, "Failed to enumerate results")
require.Equal(t, 3, len(enAlerts), "Enumerated incorrect number of alerts")
err = kva.Erase(api.ResourceType_VOLUMES, raiseAlert1.Id)
err = kva.Erase(api.ResourceType_VOLUMES, raiseAlert2.Id)
err = kva.Erase(api.ResourceType_VOLUMES, raiseAlert3.Id)
err = kva.Erase(api.ResourceType_NODE, raiseAlert4.Id)
}
func getNextIdFromKVDB() (int64, error) {
kv := kvdb.Instance()
nextAlertsId := 0
kvp, err := kv.Create(getNextAlertsIdKey(), strconv.FormatInt(int64(nextAlertsId+1), 10), 0)
for err != nil {
kvp, err = kv.GetVal(getNextAlertsIdKey(), &nextAlertsId)
if err != nil {
err = ErrNotInitialized
return int64(nextAlertsId), err
}
prevValue := kvp.Value
kvp.Value = []byte(strconv.FormatInt(int64(nextAlertsId+1), 10))
kvp, err = kv.CompareAndSet(kvp, kvdb.KVFlags(0), prevValue)
}
return int64(nextAlertsId), err
}
func watchKey(t *testing.T) {
fmt.Println("watchKey")
kv := kvdb.Instance()
assert.NotNil(t, kv, "Default KVDB is not set")
watchData := watchData{
t: t,
key: "tree/key",
stop: "stop",
iterations: 2,
}
kv.Delete(watchData.key)
kv.WatchKey(watchData.key, 0, &watchData, watchFn)
go watchUpdate(&watchData)
for watchData.watchStopped == false {
time.Sleep(time.Millisecond * 100)
}
}
// SetSize sets the maximum number of nodes in a cluster.
func (c *ClusterManager) SetSize(size int) error {
kvdb := kvdb.Instance()
kvlock, err := kvdb.Lock(clusterLockKey, 20)
if err != nil {
dlog.Warnln("Unable to obtain cluster lock for updating config", err)
return nil
}
defer kvdb.Unlock(kvlock)
db, err := readDatabase()
if err != nil {
return err
}
db.Size = size
err = writeDatabase(&db)
return err
}