func TestDecrypt(t *testing.T) { tempdir, err := ioutil.TempDir("", "rafttool") require.NoError(t, err) defer os.RemoveAll(tempdir) kek := []byte("kek") dek := []byte("dek") unlockKey := encryption.HumanReadableKey(kek) // write a key to disk, else we won't be able to decrypt anything paths := certPaths(tempdir) krw := ca.NewKeyReadWriter(paths.Node, kek, manager.RaftDEKData{EncryptionKeys: raft.EncryptionKeys{CurrentDEK: dek}}) cert, key, err := testutils.CreateRootCertAndKey("not really a root, just need cert and key") require.NoError(t, err) require.NoError(t, krw.Write(cert, key, nil)) // create the encrypted v3 directory origSnapshot := raftpb.Snapshot{ Data: []byte("snapshot"), Metadata: raftpb.SnapshotMetadata{ Index: 1, Term: 1, }, } e, d := encryption.Defaults(dek) writeFakeRaftData(t, tempdir, &origSnapshot, storage.NewWALFactory(e, d), storage.NewSnapFactory(e, d)) outdir := filepath.Join(tempdir, "outdir") // if we use the wrong unlock key, we can't actually decrypt anything. The output directory won't get created. err = decryptRaftData(tempdir, outdir, "") require.IsType(t, ca.ErrInvalidKEK{}, err) require.False(t, fileutil.Exist(outdir)) // Using the right unlock key, we produce data that is unencrypted require.NoError(t, decryptRaftData(tempdir, outdir, unlockKey)) require.True(t, fileutil.Exist(outdir)) // The snapshot directory is readable by the regular snapshotter snapshot, err := storage.OriginalSnap.New(filepath.Join(outdir, "snap-decrypted")).Load() require.NoError(t, err) require.NotNil(t, snapshot) require.Equal(t, origSnapshot, *snapshot) // The wals are readable by the regular wal walreader, err := storage.OriginalWAL.Open(filepath.Join(outdir, "wal-decrypted"), walpb.Snapshot{Index: 1, Term: 1}) require.NoError(t, err) metadata, _, entries, err := walreader.ReadAll() require.NoError(t, err) require.Equal(t, []byte("v3metadata"), metadata) require.Len(t, entries, 5) }
func decryptRaftData(swarmdir, outdir, unlockKey string) error { krw, err := getKRW(swarmdir, unlockKey) if err != nil { return err } deks, err := getDEKData(krw) if err != nil { return err } _, d := encryption.Defaults(deks.CurrentDEK) if deks.PendingDEK == nil { _, d2 := encryption.Defaults(deks.PendingDEK) d = storage.MultiDecrypter{d, d2} } snapDir := filepath.Join(outdir, "snap-decrypted") if err := moveDirAside(snapDir); err != nil { return err } if err := storage.MigrateSnapshot( filepath.Join(swarmdir, "raft", "snap-v3-encrypted"), snapDir, storage.NewSnapFactory(encryption.NoopCrypter, d), storage.OriginalSnap); err != nil { return err } var walsnap walpb.Snapshot snap, err := storage.OriginalSnap.New(snapDir).Load() if err != nil && !os.IsNotExist(err) { return err } if snap != nil { walsnap.Index = snap.Metadata.Index walsnap.Term = snap.Metadata.Term } walDir := filepath.Join(outdir, "wal-decrypted") if err := moveDirAside(walDir); err != nil { return err } return storage.MigrateWALs(context.Background(), filepath.Join(swarmdir, "raft", "wal-v3-encrypted"), walDir, storage.NewWALFactory(encryption.NoopCrypter, d), storage.OriginalWAL, walsnap) }
// Tests locking and unlocking the manager and key rotations func TestManagerLockUnlock(t *testing.T) { ctx := context.Background() temp, err := ioutil.TempFile("", "test-manager-lock") require.NoError(t, err) require.NoError(t, temp.Close()) require.NoError(t, os.Remove(temp.Name())) defer os.RemoveAll(temp.Name()) stateDir, err := ioutil.TempDir("", "test-raft") require.NoError(t, err) defer os.RemoveAll(stateDir) tc := testutils.NewTestCA(t) defer tc.Stop() managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole) require.NoError(t, err) _, _, err = managerSecurityConfig.KeyReader().Read() require.NoError(t, err) m, err := New(&Config{ RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, ControlAPI: temp.Name(), StateDir: stateDir, SecurityConfig: managerSecurityConfig, // start off without any encryption }) require.NoError(t, err) require.NotNil(t, m) done := make(chan error) defer close(done) go func() { done <- m.Run(ctx) }() opts := []grpc.DialOption{ grpc.WithTimeout(10 * time.Second), grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds), } conn, err := grpc.Dial(m.Addr(), opts...) require.NoError(t, err) defer func() { require.NoError(t, conn.Close()) }() // check that there is no kek currently - we are using the API because this // lets us wait until the manager is up and listening, as well var cluster *api.Cluster client := api.NewControlClient(conn) require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { resp, err := client.ListClusters(ctx, &api.ListClustersRequest{}) if err != nil { return err } if len(resp.Clusters) == 0 { return fmt.Errorf("no clusters yet") } cluster = resp.Clusters[0] return nil }, 1*time.Second)) require.Nil(t, cluster.UnlockKeys) // tls key is unencrypted, but there is a DEK key, err := ioutil.ReadFile(tc.Paths.Node.Key) require.NoError(t, err) keyBlock, _ := pem.Decode(key) require.NotNil(t, keyBlock) require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) require.Len(t, keyBlock.Headers, 2) currentDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) require.NoError(t, err) require.NotEmpty(t, currentDEK) // update the lock key - this may fail due to update out of sequence errors, so try again for { getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) require.NoError(t, err) cluster = getResp.Cluster spec := cluster.Spec.Copy() spec.EncryptionConfig.AutoLockManagers = true updateResp, err := client.UpdateCluster(ctx, &api.UpdateClusterRequest{ ClusterID: cluster.ID, ClusterVersion: &cluster.Meta.Version, Spec: spec, }) if grpc.ErrorDesc(err) == "update out of sequence" { continue } // if there is any other type of error, this should fail if err == nil { cluster = updateResp.Cluster } break } require.NoError(t, err) caConn := api.NewCAClient(conn) unlockKeyResp, err := caConn.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) require.NoError(t, err) // this should update the TLS key, rotate the DEK, and finish snapshotting var updatedKey []byte require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { updatedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) require.NoError(t, err) // this should never error due to atomic writes if bytes.Equal(key, updatedKey) { return fmt.Errorf("TLS key should have been re-encrypted at least") } keyBlock, _ = pem.Decode(updatedKey) require.NotNil(t, keyBlock) // this should never error due to atomic writes if !x509.IsEncryptedPEMBlock(keyBlock) { return fmt.Errorf("Key not encrypted") } // we don't check that the TLS key has been rotated, because that may take // a little bit, and is best effort only currentDEKString, ok := keyBlock.Headers[pemHeaderRaftDEK] require.True(t, ok) // there should never NOT be a current header nowCurrentDEK, err := decodePEMHeaderValue(currentDEKString, unlockKeyResp.UnlockKey) require.NoError(t, err) // it should always be encrypted if bytes.Equal(currentDEK, nowCurrentDEK) { return fmt.Errorf("snapshot has not been finished yet") } currentDEK = nowCurrentDEK return nil }, 1*time.Second)) _, ok := keyBlock.Headers[pemHeaderRaftPendingDEK] require.False(t, ok) // once the snapshot is do _, ok = keyBlock.Headers[pemHeaderRaftDEKNeedsRotation] require.False(t, ok) // verify that the snapshot is readable with the new DEK encrypter, decrypter := encryption.Defaults(currentDEK) // we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove // the manager, they'll be deleted. snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load() require.NoError(t, err) require.NotNil(t, snapshot) // update the lock key to nil for i := 0; i < 3; i++ { getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) require.NoError(t, err) cluster = getResp.Cluster spec := cluster.Spec.Copy() spec.EncryptionConfig.AutoLockManagers = false _, err = client.UpdateCluster(ctx, &api.UpdateClusterRequest{ ClusterID: cluster.ID, ClusterVersion: &cluster.Meta.Version, Spec: spec, }) if grpc.ErrorDesc(err) == "update out of sequence" { continue } require.NoError(t, err) } // this should update the TLS key var unlockedKey []byte require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { unlockedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) if err != nil { return err } if bytes.Equal(unlockedKey, updatedKey) { return fmt.Errorf("TLS key should have been rotated") } return nil }, 1*time.Second)) // the new key should not be encrypted, and the DEK should also be unencrypted // but not rotated keyBlock, _ = pem.Decode(unlockedKey) require.NotNil(t, keyBlock) require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) unencryptedDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) require.NoError(t, err) require.NotNil(t, unencryptedDEK) require.Equal(t, currentDEK, unencryptedDEK) m.Stop(ctx) // After stopping we should MAY receive an error from ListenAndServe if // all this happened before WaitForLeader completed, so don't check the // error. <-done }