func TestClusterHAFetching(t *testing.T) {
logger := logformat.NewVaultLogger(log.LevelTrace)
redirect := "http://127.0.0.1:8200"
c, err := NewCore(&CoreConfig{
Physical: physical.NewInmemHA(logger),
HAPhysical: physical.NewInmemHA(logger),
RedirectAddr: redirect,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, _ := TestCoreInit(t, c)
if _, err := TestCoreUnseal(c, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := c.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
TestWaitActive(t, c)
cluster, err := c.Cluster()
if err != nil {
t.Fatal(err)
}
// Test whether expected values are found
if cluster == nil || cluster.Name == "" || cluster.ID == "" {
t.Fatalf("cluster information missing: cluster:%#v", cluster)
}
}
func TestCore_Standby_SeparateHA(t *testing.T) {
logger = logformat.NewVaultLogger(log.LevelTrace)
testCore_Standby_Common(t, physical.NewInmemHA(logger), physical.NewInmemHA(logger))
}
func TestCore_Standby(t *testing.T) {
logger = logformat.NewVaultLogger(log.LevelTrace)
inmha := physical.NewInmemHA(logger)
testCore_Standby_Common(t, inmha, inmha)
}
func TestCore_CleanLeaderPrefix(t *testing.T) {
// Create the first core and initialize it
logger = logformat.NewVaultLogger(log.LevelTrace)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
redirectOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
TestWaitActive(t, core)
// Ensure that the original clean function has stopped running
time.Sleep(2 * time.Second)
// Put several random entries
for i := 0; i < 5; i++ {
keyUUID, err := uuid.GenerateUUID()
if err != nil {
t.Fatal(err)
}
valueUUID, err := uuid.GenerateUUID()
if err != nil {
t.Fatal(err)
}
core.barrier.Put(&Entry{
Key: coreLeaderPrefix + keyUUID,
Value: []byte(valueUUID),
})
}
entries, err := core.barrier.List(coreLeaderPrefix)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(entries) != 6 {
t.Fatalf("wrong number of core leader prefix entries, got %d", len(entries))
}
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create a second core, attached to same in-memory store
redirectOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := TestCoreUnseal(core2, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Seal the first core, should step down
err = core.Seal(root)
if err != nil {
t.Fatalf("err: %v", err)
}
// Core should be in standby
standby, err = core.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Wait for core2 to become active
TestWaitActive(t, core2)
// Check the leader is local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
// Give time for the entries to clear out; it is conservative at 1/second
time.Sleep(10 * leaderPrefixCleanDelay)
entries, err = core2.barrier.List(coreLeaderPrefix)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(entries) != 1 {
t.Fatalf("wrong number of core leader prefix entries, got %d", len(entries))
}
}
func TestCore_StepDown(t *testing.T) {
// Create the first core and initialize it
logger = logformat.NewVaultLogger(log.LevelTrace)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
redirectOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
TestWaitActive(t, core)
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create the second core and initialize it
redirectOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := TestCoreUnseal(core2, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
req := &logical.Request{
ClientToken: root,
Path: "sys/step-down",
}
// Create an identifier for the request
req.ID, err = uuid.GenerateUUID()
if err != nil {
t.Fatalf("failed to generate identifier for the request: path: %s err: %v", req.Path, err)
}
// Step down core
err = core.StepDown(req)
if err != nil {
t.Fatal("error stepping down core 1")
}
// Give time to switch leaders
time.Sleep(5 * time.Second)
// Core1 should be in standby
standby, err = core.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is core2
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
// Check the leader is not local
isLeader, advertise, err = core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != redirectOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
// Step down core2
err = core2.StepDown(req)
if err != nil {
t.Fatal("error stepping down core 1")
}
// Give time to switch leaders -- core 1 will still be waiting on its
// cooling off period so give it a full 10 seconds to recover
time.Sleep(10 * time.Second)
// Core2 should be in standby
standby, err = core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is core1
isLeader, advertise, err = core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
}
func TestCore_Standby_Rekey(t *testing.T) {
// Create the first core and initialize it
logger := log.New(os.Stderr, "", log.LstdFlags)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
redirectOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Wait for core to become active
TestWaitActive(t, core)
// Create a second core, attached to same in-memory store
redirectOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := TestCoreUnseal(core2, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Rekey the master key
newConf := &SealConfig{
SecretShares: 1,
SecretThreshold: 1,
}
err = core.RekeyInit(newConf, false)
if err != nil {
t.Fatalf("err: %v", err)
}
// Fetch new config with generated nonce
rkconf, err := core.RekeyConfig(false)
if err != nil {
t.Fatalf("err: %v", err)
}
if rkconf == nil {
t.Fatalf("bad: no rekey config received")
}
result, err := core.RekeyUpdate(key, rkconf.Nonce, false)
if err != nil {
t.Fatalf("err: %v", err)
}
if result == nil {
t.Fatalf("rekey failed")
}
// Seal the first core, should step down
err = core.Seal(root)
if err != nil {
t.Fatalf("err: %v", err)
}
// Wait for core2 to become active
TestWaitActive(t, core2)
// Rekey the master key again
err = core2.RekeyInit(newConf, false)
if err != nil {
t.Fatalf("err: %v", err)
}
// Fetch new config with generated nonce
rkconf, err = core2.RekeyConfig(false)
if err != nil {
t.Fatalf("err: %v", err)
}
if rkconf == nil {
t.Fatalf("bad: no rekey config received")
}
result, err = core2.RekeyUpdate(result.SecretShares[0], rkconf.Nonce, false)
if err != nil {
t.Fatalf("err: %v", err)
}
if result == nil {
t.Fatalf("rekey failed")
}
}
func TestCore_Standby_SeparateHA(t *testing.T) {
logger = log.New(os.Stderr, "", log.LstdFlags)
testCore_Standby_Common(t, physical.NewInmemHA(logger), physical.NewInmemHA(logger))
}
func TestLogical_StandbyRedirect(t *testing.T) {
ln1, addr1 := TestListener(t)
defer ln1.Close()
ln2, addr2 := TestListener(t)
defer ln2.Close()
// Create an HA Vault
inm := physical.NewInmemHA()
conf := &vault.CoreConfig{
Physical: inm,
AdvertiseAddr: addr1,
DisableMlock: true,
}
core1, err := vault.NewCore(conf)
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := vault.TestCoreInit(t, core1)
if _, err := core1.Unseal(vault.TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Create a second HA Vault
conf2 := &vault.CoreConfig{
Physical: inm,
AdvertiseAddr: addr2,
DisableMlock: true,
}
core2, err := vault.NewCore(conf2)
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(vault.TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
TestServerWithListener(t, ln1, addr1, core1)
TestServerWithListener(t, ln2, addr2, core2)
TestServerAuth(t, addr1, root)
// WRITE to STANDBY
resp := testHttpPut(t, addr2+"/v1/secret/foo", map[string]interface{}{
"data": "bar",
})
testResponseStatus(t, resp, 307)
//// READ to standby
resp, err = http.Get(addr2 + "/v1/auth/token/lookup-self")
if err != nil {
t.Fatalf("err: %s", err)
}
var actual map[string]interface{}
expected := map[string]interface{}{
"renewable": false,
"lease_duration": float64(0),
"data": map[string]interface{}{
"meta": nil,
"num_uses": float64(0),
"path": "auth/token/root",
"policies": []interface{}{"root"},
"display_name": "root",
"id": root,
},
"auth": nil,
}
testResponseStatus(t, resp, 200)
testResponseBody(t, resp, &actual)
delete(actual, "lease_id")
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("bad: %#v %#v", actual, expected)
}
//// DELETE to standby
resp = testHttpDelete(t, addr2+"/v1/secret/foo")
testResponseStatus(t, resp, 307)
}
func TestCore_StepDown(t *testing.T) {
// Create the first core and initialize it
logger = log.New(os.Stderr, "", log.LstdFlags)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
advertiseOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
AdvertiseAddr: advertiseOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := core.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
testWaitActive(t, core)
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create the second core and initialize it
advertiseOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
AdvertiseAddr: advertiseOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Step down core
err = core.StepDown(root)
if err != nil {
t.Fatal("error stepping down core 1")
}
// Give time to switch leaders
time.Sleep(2 * time.Second)
// Core1 should be in standby
standby, err = core.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is core2
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
// Check the leader is not local
isLeader, advertise, err = core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != advertiseOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
// Step down core2
err = core2.StepDown(root)
if err != nil {
t.Fatal("error stepping down core 1")
}
// Give time to switch leaders -- core 1 will still be waiting on its
// cooling off period so give it a full 10 seconds to recover
time.Sleep(10 * time.Second)
// Core2 should be in standby
standby, err = core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is core1
isLeader, advertise, err = core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
}
func TestCore_Standby(t *testing.T) {
logger = log.New(os.Stderr, "", log.LstdFlags)
inmha := physical.NewInmemHA(logger)
testCore_Standby_Common(t, inmha, inmha)
}
func TestCore_Standby_SeparateHA(t *testing.T) {
testCore_Standby_Common(t, physical.NewInmemHA(), physical.NewInmemHA())
}
func TestCore_Standby(t *testing.T) {
inmha := physical.NewInmemHA()
testCore_Standby_Common(t, inmha, inmha)
}
func TestCluster(t *testing.T, handlers []http.Handler, base *CoreConfig, unsealStandbys bool) []*TestClusterCore {
if handlers == nil || len(handlers) != 3 {
t.Fatal("handlers must be size 3")
}
//
// TLS setup
//
block, _ := pem.Decode([]byte(TestClusterCACert))
if block == nil {
t.Fatal("error decoding cluster CA cert")
}
caBytes := block.Bytes
caCert, err := x509.ParseCertificate(caBytes)
if err != nil {
t.Fatal(err)
}
serverCert, err := tls.X509KeyPair([]byte(TestClusterServerCert), []byte(TestClusterServerKey))
if err != nil {
t.Fatal(err)
}
rootCAs := x509.NewCertPool()
rootCAs.AppendCertsFromPEM([]byte(TestClusterCACert))
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{serverCert},
RootCAs: rootCAs,
ClientCAs: rootCAs,
ClientAuth: tls.RequireAndVerifyClientCert,
}
tlsConfig.BuildNameToCertificate()
// Sanity checking
block, _ = pem.Decode([]byte(TestClusterServerCert))
if block == nil {
t.Fatal(err)
}
parsedServerCert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
t.Fatal(err)
}
chains, err := parsedServerCert.Verify(x509.VerifyOptions{
DNSName: "127.0.0.1",
Roots: rootCAs,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
})
if err != nil {
t.Fatal(err)
}
if chains == nil || len(chains) == 0 {
t.Fatal("no verified chains for server auth")
}
chains, err = parsedServerCert.Verify(x509.VerifyOptions{
DNSName: "127.0.0.1",
Roots: rootCAs,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Fatal(err)
}
if chains == nil || len(chains) == 0 {
t.Fatal("no verified chains for chains auth")
}
logger := logformat.NewVaultLogger(log.LevelTrace)
//
// Listener setup
//
ln, err := net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("127.0.0.1"),
Port: 0,
})
if err != nil {
t.Fatal(err)
}
c1lns := []*TestListener{&TestListener{
Listener: tls.NewListener(ln, tlsConfig),
Address: ln.Addr().(*net.TCPAddr),
},
}
ln, err = net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("127.0.0.1"),
Port: 0,
})
if err != nil {
t.Fatal(err)
}
c1lns = append(c1lns, &TestListener{
Listener: tls.NewListener(ln, tlsConfig),
Address: ln.Addr().(*net.TCPAddr),
})
server1 := &http.Server{
Handler: handlers[0],
}
for _, ln := range c1lns {
go server1.Serve(ln)
}
ln, err = net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("127.0.0.1"),
Port: 0,
})
if err != nil {
t.Fatal(err)
}
c2lns := []*TestListener{&TestListener{
Listener: tls.NewListener(ln, tlsConfig),
Address: ln.Addr().(*net.TCPAddr),
},
}
server2 := &http.Server{
Handler: handlers[1],
}
for _, ln := range c2lns {
go server2.Serve(ln)
}
ln, err = net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("127.0.0.1"),
Port: 0,
})
if err != nil {
t.Fatal(err)
}
c3lns := []*TestListener{&TestListener{
Listener: tls.NewListener(ln, tlsConfig),
Address: ln.Addr().(*net.TCPAddr),
},
}
server3 := &http.Server{
Handler: handlers[2],
}
for _, ln := range c3lns {
go server3.Serve(ln)
}
// Create three cores with the same physical and different redirect/cluster addrs
coreConfig := &CoreConfig{
Physical: physical.NewInmem(logger),
HAPhysical: physical.NewInmemHA(logger),
LogicalBackends: make(map[string]logical.Factory),
CredentialBackends: make(map[string]logical.Factory),
AuditBackends: make(map[string]audit.Factory),
RedirectAddr: fmt.Sprintf("https://127.0.0.1:%d", c1lns[0].Address.Port),
ClusterAddr: fmt.Sprintf("https://127.0.0.1:%d", c1lns[0].Address.Port+1),
DisableMlock: true,
}
if base != nil {
// Used to set something non-working to test fallback
switch base.ClusterAddr {
case "empty":
coreConfig.ClusterAddr = ""
case "":
default:
coreConfig.ClusterAddr = base.ClusterAddr
}
if base.LogicalBackends != nil {
for k, v := range base.LogicalBackends {
coreConfig.LogicalBackends[k] = v
}
}
if base.CredentialBackends != nil {
for k, v := range base.CredentialBackends {
coreConfig.CredentialBackends[k] = v
}
}
if base.AuditBackends != nil {
for k, v := range base.AuditBackends {
coreConfig.AuditBackends[k] = v
}
}
}
c1, err := NewCore(coreConfig)
if err != nil {
t.Fatalf("err: %v", err)
}
coreConfig.RedirectAddr = fmt.Sprintf("https://127.0.0.1:%d", c2lns[0].Address.Port)
if coreConfig.ClusterAddr != "" {
coreConfig.ClusterAddr = fmt.Sprintf("https://127.0.0.1:%d", c2lns[0].Address.Port+1)
}
c2, err := NewCore(coreConfig)
if err != nil {
t.Fatalf("err: %v", err)
}
coreConfig.RedirectAddr = fmt.Sprintf("https://127.0.0.1:%d", c3lns[0].Address.Port)
if coreConfig.ClusterAddr != "" {
coreConfig.ClusterAddr = fmt.Sprintf("https://127.0.0.1:%d", c3lns[0].Address.Port+1)
}
c3, err := NewCore(coreConfig)
if err != nil {
t.Fatalf("err: %v", err)
}
//
// Clustering setup
//
clusterAddrGen := func(lns []*TestListener) []*net.TCPAddr {
ret := make([]*net.TCPAddr, len(lns))
for i, ln := range lns {
ret[i] = &net.TCPAddr{
IP: ln.Address.IP,
Port: ln.Address.Port + 1,
}
}
return ret
}
c2.SetClusterListenerAddrs(clusterAddrGen(c2lns))
c2.SetClusterSetupFuncs(WrapHandlerForClustering(handlers[1], logger))
c3.SetClusterListenerAddrs(clusterAddrGen(c3lns))
c3.SetClusterSetupFuncs(WrapHandlerForClustering(handlers[2], logger))
key, root := TestCoreInitClusterWrapperSetup(t, c1, clusterAddrGen(c1lns), WrapHandlerForClustering(handlers[0], logger))
if _, err := c1.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := c1.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
TestWaitActive(t, c1)
if unsealStandbys {
if _, err := c2.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
if _, err := c3.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Let them come fully up to standby
time.Sleep(2 * time.Second)
// Ensure cluster connection info is populated
isLeader, _, err := c2.Leader()
if err != nil {
t.Fatal(err)
}
if isLeader {
t.Fatal("c2 should not be leader")
}
isLeader, _, err = c3.Leader()
if err != nil {
t.Fatal(err)
}
if isLeader {
t.Fatal("c3 should not be leader")
}
}
return []*TestClusterCore{
&TestClusterCore{
Core: c1,
Listeners: c1lns,
Root: root,
Key: TestKeyCopy(key),
CACertBytes: caBytes,
CACert: caCert,
TLSConfig: tlsConfig,
},
&TestClusterCore{
Core: c2,
Listeners: c2lns,
Root: root,
Key: TestKeyCopy(key),
CACertBytes: caBytes,
CACert: caCert,
TLSConfig: tlsConfig,
},
&TestClusterCore{
Core: c3,
Listeners: c3lns,
Root: root,
Key: TestKeyCopy(key),
CACertBytes: caBytes,
CACert: caCert,
TLSConfig: tlsConfig,
},
}
}
func TestCore_Standby_Rotate(t *testing.T) {
// Create the first core and initialize it
logger = logformat.NewVaultLogger(log.LevelTrace)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
redirectOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Wait for core to become active
TestWaitActive(t, core)
// Create a second core, attached to same in-memory store
redirectOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := TestCoreUnseal(core2, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Rotate the encryption key
req := &logical.Request{
Operation: logical.UpdateOperation,
Path: "sys/rotate",
ClientToken: root,
}
_, err = core.HandleRequest(req)
if err != nil {
t.Fatalf("err: %v", err)
}
// Seal the first core, should step down
err = core.Seal(root)
if err != nil {
t.Fatalf("err: %v", err)
}
// Wait for core2 to become active
TestWaitActive(t, core2)
// Read the key status
req = &logical.Request{
Operation: logical.ReadOperation,
Path: "sys/key-status",
ClientToken: root,
}
resp, err := core2.HandleRequest(req)
if err != nil {
t.Fatalf("err: %v", err)
}
// Verify the response
if resp.Data["term"] != 2 {
t.Fatalf("bad: %#v", resp)
}
}
func TestCore_Standby(t *testing.T) {
// Create the first core and initialize it
inm := physical.NewInmemHA()
advertiseOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
AdvertiseAddr: advertiseOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := core.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
testWaitActive(t, core)
// Put a secret
req := &logical.Request{
Operation: logical.WriteOperation,
Path: "secret/foo",
Data: map[string]interface{}{
"foo": "bar",
},
ClientToken: root,
}
_, err = core.HandleRequest(req)
if err != nil {
t.Fatalf("err: %v", err)
}
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create a second core, attached to same in-memory store
advertiseOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
AdvertiseAddr: advertiseOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Request should fail in standby mode
_, err = core2.HandleRequest(req)
if err != ErrStandby {
t.Fatalf("err: %v", err)
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Seal the first core, should step down
err = core.Seal(root)
if err != nil {
t.Fatalf("err: %v", err)
}
// Core should be in standby
standby, err = core.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Wait for core2 to become active
testWaitActive(t, core2)
// Read the secret
req = &logical.Request{
Operation: logical.ReadOperation,
Path: "secret/foo",
ClientToken: root,
}
resp, err := core2.HandleRequest(req)
if err != nil {
t.Fatalf("err: %v", err)
}
// Verify the response
if resp.Data["foo"] != "bar" {
t.Fatalf("bad: %#v", resp)
}
// Check the leader is local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal2 {
t.Fatalf("Bad advertise: %v", advertise)
}
}
func TestCore_Standby_Seal(t *testing.T) {
// Create the first core and initialize it
logger = logformat.NewVaultLogger(log.LevelTrace)
inm := physical.NewInmem(logger)
inmha := physical.NewInmemHA(logger)
redirectOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
TestWaitActive(t, core)
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create the second core and initialize it
redirectOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
RedirectAddr: redirectOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := TestCoreUnseal(core2, TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != redirectOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Seal the standby core with the correct token. Shouldn't go down
err = core2.Seal(root)
if err == nil {
t.Fatal("should not be sealed")
}
keyUUID, err := uuid.GenerateUUID()
if err != nil {
t.Fatal(err)
}
// Seal the standby core with an invalid token. Shouldn't go down
err = core2.Seal(keyUUID)
if err == nil {
t.Fatal("should not be sealed")
}
}
func TestCore_Standby_Rekey(t *testing.T) {
// Create the first core and initialize it
inm := physical.NewInmemHA()
advertiseOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
AdvertiseAddr: advertiseOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := core.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Wait for core to become active
testWaitActive(t, core)
// Create a second core, attached to same in-memory store
advertiseOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
AdvertiseAddr: advertiseOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Rekey the master key
newConf := &SealConfig{
SecretShares: 1,
SecretThreshold: 1,
}
err = core.RekeyInit(newConf)
if err != nil {
t.Fatalf("err: %v", err)
}
result, err := core.RekeyUpdate(key)
if err != nil {
t.Fatalf("err: %v", err)
}
if result == nil {
t.Fatalf("rekey failed")
}
// Seal the first core, should step down
err = core.Seal(root)
if err != nil {
t.Fatalf("err: %v", err)
}
// Wait for core2 to become active
testWaitActive(t, core2)
// Rekey the master key again
err = core2.RekeyInit(newConf)
if err != nil {
t.Fatalf("err: %v", err)
}
result, err = core2.RekeyUpdate(result.SecretShares[0])
if err != nil {
t.Fatalf("err: %v", err)
}
if result == nil {
t.Fatalf("rekey failed")
}
}
func TestLogical_StandbyRedirect(t *testing.T) {
ln1, addr1 := TestListener(t)
defer ln1.Close()
ln2, addr2 := TestListener(t)
defer ln2.Close()
// Create an HA Vault
inmha := physical.NewInmemHA(logger)
conf := &vault.CoreConfig{
Physical: inmha,
HAPhysical: inmha,
RedirectAddr: addr1,
DisableMlock: true,
}
core1, err := vault.NewCore(conf)
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := vault.TestCoreInit(t, core1)
if _, err := core1.Unseal(vault.TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Attempt to fix raciness in this test by giving the first core a chance
// to grab the lock
time.Sleep(time.Second)
// Create a second HA Vault
conf2 := &vault.CoreConfig{
Physical: inmha,
HAPhysical: inmha,
RedirectAddr: addr2,
DisableMlock: true,
}
core2, err := vault.NewCore(conf2)
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(vault.TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
TestServerWithListener(t, ln1, addr1, core1)
TestServerWithListener(t, ln2, addr2, core2)
TestServerAuth(t, addr1, root)
// WRITE to STANDBY
resp := testHttpPut(t, root, addr2+"/v1/secret/foo", map[string]interface{}{
"data": "bar",
})
testResponseStatus(t, resp, 307)
//// READ to standby
resp = testHttpGet(t, root, addr2+"/v1/auth/token/lookup-self")
var actual map[string]interface{}
var nilWarnings interface{}
expected := map[string]interface{}{
"renewable": false,
"lease_duration": json.Number("0"),
"data": map[string]interface{}{
"meta": nil,
"num_uses": json.Number("0"),
"path": "auth/token/root",
"policies": []interface{}{"root"},
"display_name": "root",
"orphan": true,
"id": root,
"ttl": json.Number("0"),
"creation_ttl": json.Number("0"),
"explicit_max_ttl": json.Number("0"),
},
"warnings": nilWarnings,
"wrap_info": nil,
"auth": nil,
}
testResponseStatus(t, resp, 200)
testResponseBody(t, resp, &actual)
actualDataMap := actual["data"].(map[string]interface{})
delete(actualDataMap, "creation_time")
delete(actualDataMap, "accessor")
actual["data"] = actualDataMap
expected["request_id"] = actual["request_id"]
delete(actual, "lease_id")
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("bad: got %#v; expected %#v", actual, expected)
}
//// DELETE to standby
resp = testHttpDelete(t, root, addr2+"/v1/secret/foo")
testResponseStatus(t, resp, 307)
}
func TestCore_Standby_Seal(t *testing.T) {
// Create the first core and initialize it
inm := physical.NewInmem()
inmha := physical.NewInmemHA()
advertiseOriginal := "http://127.0.0.1:8200"
core, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
AdvertiseAddr: advertiseOriginal,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
key, root := TestCoreInit(t, core)
if _, err := core.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err := core.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Wait for core to become active
testWaitActive(t, core)
// Ensure that the original clean function has stopped running
time.Sleep(2 * time.Second)
// Check the leader is local
isLeader, advertise, err := core.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if !isLeader {
t.Fatalf("should be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Create the second core and initialize it
advertiseOriginal2 := "http://127.0.0.1:8500"
core2, err := NewCore(&CoreConfig{
Physical: inm,
HAPhysical: inmha,
AdvertiseAddr: advertiseOriginal2,
DisableMlock: true,
})
if err != nil {
t.Fatalf("err: %v", err)
}
if _, err := core2.Unseal(TestKeyCopy(key)); err != nil {
t.Fatalf("unseal err: %s", err)
}
// Verify unsealed
sealed, err = core2.Sealed()
if err != nil {
t.Fatalf("err checking seal status: %s", err)
}
if sealed {
t.Fatal("should not be sealed")
}
// Core2 should be in standby
standby, err := core2.Standby()
if err != nil {
t.Fatalf("err: %v", err)
}
if !standby {
t.Fatalf("should be standby")
}
// Check the leader is not local
isLeader, advertise, err = core2.Leader()
if err != nil {
t.Fatalf("err: %v", err)
}
if isLeader {
t.Fatalf("should not be leader")
}
if advertise != advertiseOriginal {
t.Fatalf("Bad advertise: %v", advertise)
}
// Seal the standby core with the correct token. Shouldn't go down
err = core2.Seal(root)
if err == nil {
t.Fatal("should not be sealed")
}
keyUUID, err := uuid.GenerateUUID()
if err != nil {
t.Fatal(err)
}
// Seal the standby core with an invalid token. Shouldn't go down
err = core2.Seal(keyUUID)
if err == nil {
t.Fatal("should not be sealed")
}
}