func main() { // This script generates CSR, private key and a verifier ID. // Keyserver admin will sign the CSR and provide the cert privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { panic(err) } pk, sk, err := ed25519.GenerateKey(rand.Reader) if err != nil { panic(err) } sv := &proto.PublicKey{PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pk[:]}} hostname := fmt.Sprintf("verifier %x", proto.KeyID(sv)) fmt.Println("verifier ID: " + hostname + "\nID(uint): " + strconv.FormatUint(proto.KeyID(sv), 10)) err = ioutil.WriteFile("signing.ed25519secret", []byte(sk[:]), 0644) if err != nil { panic(err) } certTemplate := &x509.CertificateRequest{ Subject: pkix.Name{CommonName: hostname}, } csrDER, err := x509.CreateCertificateRequest(rand.Reader, certTemplate, privKey) if err != nil { panic(err) } csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrDER}) // openssl req -in csr.pem -noout -text err = ioutil.WriteFile("csr.pem", csrPEM, 0644) if err != nil { panic(err) } keyF, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE, 0644) defer keyF.Close() if err != nil { panic(err) } skDer, err := x509.MarshalECPrivateKey(privKey) if err != nil { panic(err) } err = pem.Encode(keyF, &pem.Block{ Type: "EC PRIVATE KEY", Headers: make(map[string]string), Bytes: skDer, }) if err != nil { panic(err) } }
// setupVerifier initializes a verifier, but does not start it and does not // wait for it to sign anything. func setupVerifier(t *testing.T, keyserverVerif *proto.AuthorizationPolicy, keyserverAddr string, caCert *x509.Certificate, caPool *x509.CertPool, caKey *ecdsa.PrivateKey) (cfg *proto.VerifierConfig, getKey func(string) (crypto.PrivateKey, error), db kv.DB, sv *proto.PublicKey, teardown func()) { dir, err := ioutil.TempDir("", "verifier") if err != nil { t.Fatal(err) } teardown = chain(func() { os.RemoveAll(dir) }) ldb, err := leveldb.OpenFile(dir, nil) if err != nil { teardown() t.Fatal(err) } teardown = chain(func() { ldb.Close() }, teardown) pk, sk, err := ed25519.GenerateKey(rand.Reader) if err != nil { teardown() t.Fatal(err) } sv = &proto.PublicKey{PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pk[:]}} cert := tlstestutil.Cert(t, caCert, caKey, fmt.Sprintf("verifier %x", proto.KeyID(sv)), nil) getKey = func(keyid string) (crypto.PrivateKey, error) { switch keyid { case "signing": return sk, nil case "tls": return cert.PrivateKey, nil default: panic("unknown key requested in tests [" + keyid + "]") } } cfg = &proto.VerifierConfig{ Realm: testingRealm, KeyserverAddr: keyserverAddr, InitialKeyserverAuth: *keyserverVerif, SigningKeyID: "signing", ID: proto.KeyID(sv), TLS: &proto.TLSConfig{RootCAs: [][]byte{caCert.Raw}, Certificates: []*proto.CertificateAndKeyID{{cert.Certificate, "tls", nil}}}, } db = leveldbkv.Wrap(ldb) return }
// setupRealm initializes nReplicas keyserver replicas and nVerifiers // verifiers, and then waits until each one of them has signed an epoch. func setupRealm(t *testing.T, nReplicas, nVerifiers int) ( kss []*Keyserver, caPool *x509.CertPool, clks []*clock.Mock, verifiers []uint64, clientKeyGetter func(string) (crypto.PrivateKey, error), clientConfig *proto.Config, teardown func(), ) { cfgs, gks, ck, clientConfig, caCert, caPool, caKey, teardown := setupKeyservers(t, nReplicas) logs, dbs, clks, _, teardown2 := setupRaftLogCluster(t, nReplicas, 0) teardown = chain(teardown2, teardown) var ksDone sync.WaitGroup ksDone.Add(nReplicas) for i := range dbs { var ksDoneOnce sync.Once dbs[i] = tracekv.WithSimpleTracing(dbs[i], func(update tracekv.Update) { // We are waiting for an epoch to be ratified (in case there are no // verifiers, blocking on them does not help). if update.IsDeletion || len(update.Key) < 1 || update.Key[0] != tableVerifierLogPrefix { return } ksDoneOnce.Do(func() { ksDone.Done() }) }) } type doneVerifier struct { teardown func() id uint64 } ksBarrier := make(chan struct{}) doneVerifiers := make(chan doneVerifier, nVerifiers) vpks := make([]*proto.PublicKey, nVerifiers) for i := 0; i < nVerifiers; i++ { vrBarrier := make(chan struct{}) var verifierTeardown func() var vcfg *proto.VerifierConfig var doneOnce sync.Once dbs[0] = tracekv.WithSimpleTracing(dbs[0], func(update tracekv.Update) { // We are waiting for epoch 1 to be ratified by the verifier and // reach the client because before that lookups requiring this // verifier will immediately fail. if len(update.Key) < 1 || update.Key[0] != tableRatificationsPrefix { return } <-vrBarrier epoch := binary.BigEndian.Uint64(update.Key[1 : 1+8]) id := binary.BigEndian.Uint64(update.Key[1+8 : 1+8+8]) if id == vcfg.ID && epoch == 1 { doneOnce.Do(func() { doneVerifiers <- doneVerifier{verifierTeardown, vcfg.ID} }) } }) go func(i int) { var vdb kv.DB <-ksBarrier var getKey func(string) (crypto.PrivateKey, error) vcfg, getKey, vdb, vpks[i], verifierTeardown = setupVerifier(t, clientConfig.Realms[0].VerificationPolicy, kss[i%nReplicas].verifierListen.Addr().String(), caCert, caPool, caKey) vr, err := verifier.Start(vcfg, vdb, getKey) if err != nil { t.Fatal(err) } verifierTeardown = chain(vr.Stop, verifierTeardown) close(vrBarrier) }(i) } for i := range cfgs { ks, err := Open(cfgs[i], dbs[i], logs[i], clientConfig.Realms[0].VerificationPolicy, clks[i], gks[i], nil) if err != nil { t.Fatal(err) } ks.insecureSkipEmailProof = true ks.Start() teardown = chain(ks.Stop, teardown) kss = append(kss, ks) } close(ksBarrier) ksDoneCh := make(chan struct{}) go func() { ksDone.Wait() close(ksDoneCh) }() loop: for { select { case <-time.After(poll): for _, clk := range clks { clk.Add(tick) } case <-ksDoneCh: break loop } } for i := 0; i < nVerifiers; i++ { v := <-doneVerifiers verifiers = append(verifiers, v.id) teardown = chain(v.teardown, teardown) } pol := copyAuthorizationPolicy(clientConfig.Realms[0].VerificationPolicy) pol.PolicyType = &proto.AuthorizationPolicy_Quorum{Quorum: &proto.QuorumExpr{ Subexpressions: []*proto.QuorumExpr{pol.GetQuorum()}, Threshold: uint32(1 + nVerifiers), Candidates: verifiers, }} for i := 0; i < nVerifiers; i++ { pol.PublicKeys[proto.KeyID(vpks[i])] = vpks[i] } clientConfig.Realms[0].VerificationPolicy = pol return kss, caPool, clks, verifiers, ck, clientConfig, teardown }
// setupKeyservers initializes everything needed to start a set of keyserver // replicas, but does not actually start them yet func setupKeyservers(t *testing.T, nReplicas int) ( cfgs []*proto.ReplicaConfig, serverKeyGetters []func(string) (crypto.PrivateKey, error), clientKeyGetter func(string) (crypto.PrivateKey, error), clientConfig *proto.Config, caCert *x509.Certificate, caPool *x509.CertPool, caKey *ecdsa.PrivateKey, teardown func(), ) { caCert, caPool, caKey = tlstestutil.CA(t, nil) vrfPublic, vrfSecret, err := vrf.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } teardown = func() {} clientCert := tlstestutil.Cert(t, caCert, caKey, "client", nil) clientKeyGetter = func(id string) (crypto.PrivateKey, error) { switch id { case "client": return clientCert.PrivateKey, nil default: t.Fatalf("unknown key %s", id) return nil, nil } } pks := make(map[uint64]*proto.PublicKey) replicaIDs := []uint64{} pol := &proto.AuthorizationPolicy{} realmConfig := &proto.RealmConfig{ RealmName: testingRealm, Domains: []string{realmDomain}, VRFPublic: vrfPublic, VerificationPolicy: pol, EpochTimeToLive: proto.DurationStamp(time.Hour), ClientTLS: &proto.TLSConfig{ RootCAs: [][]byte{caCert.Raw}, Certificates: []*proto.CertificateAndKeyID{{clientCert.Certificate, "client", nil}}, }, } clientConfig = &proto.Config{ Realms: []*proto.RealmConfig{realmConfig}, } for n := 0; n < nReplicas; n++ { pk, sk, err := ed25519.GenerateKey(rand.Reader) if err != nil { teardown() t.Fatal(err) } pked := &proto.PublicKey{PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pk[:]}} replicaID := proto.KeyID(pked) pks[replicaID] = pked replicaIDs = append(replicaIDs, replicaID) cert := tlstestutil.Cert(t, caCert, caKey, "127.0.0.1", nil) pcerts := []*proto.CertificateAndKeyID{{cert.Certificate, "tls", nil}} cfgs = append(cfgs, &proto.ReplicaConfig{ KeyserverConfig: proto.KeyserverConfig{ Realm: testingRealm, ServerID: replicaID, VRFKeyID: "vrf", MinEpochInterval: proto.DurationStamp(tick), MaxEpochInterval: proto.DurationStamp(tick), ProposalRetryInterval: proto.DurationStamp(poll), }, SigningKeyID: "signing", ReplicaID: replicaID, PublicAddr: "localhost:0", VerifierAddr: "localhost:0", HKPAddr: "localhost:0", PublicTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, VerifierTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, HKPTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, ClientTimeout: proto.DurationStamp(time.Hour), LaggingVerifierScan: 1000 * 1000 * 1000, }) serverKeyGetters = append(serverKeyGetters, func(keyid string) (crypto.PrivateKey, error) { switch keyid { case "vrf": return vrfSecret, nil case "signing": return sk, nil case "tls": return cert.PrivateKey, nil default: panic("unknown key requested in test") } }) } pol.PublicKeys = pks pol.PolicyType = &proto.AuthorizationPolicy_Quorum{Quorum: majorityQuorum(replicaIDs)} return }
func RunWithConfig(cfg *proto.ReplicaConfig) { // TODO: since we only want to support precisely this ratification policy, // this should be moved into server.go ratificationPolicy := &proto.AuthorizationPolicy{ PublicKeys: make(map[uint64]*proto.PublicKey), PolicyType: &proto.AuthorizationPolicy_Quorum{Quorum: &proto.QuorumExpr{ Threshold: uint32(majority(len(cfg.KeyserverConfig.InitialReplicas)))}, }, } replicaIDs := []uint64{} for _, replica := range cfg.KeyserverConfig.InitialReplicas { replicaIDs = append(replicaIDs, replica.ID) replicaExpr := &proto.QuorumExpr{ Threshold: 1, } for _, pk := range replica.PublicKeys { pkid := proto.KeyID(pk) ratificationPolicy.PublicKeys[pkid] = pk replicaExpr.Candidates = append(replicaExpr.Candidates, pkid) } ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions = append(ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions, replicaExpr) } leveldb, err := leveldb.OpenFile(cfg.LevelDBPath, nil) if err != nil { log.Fatalf("Couldn't open DB in directory %s: %s", cfg.LevelDBPath, err) } db := leveldbkv.Wrap(leveldb) clk := clock.New() raftListener, err := net.Listen("tcp", cfg.RaftAddr) if err != nil { log.Fatalf("Couldn't bind to Raft node address %s: %s", cfg.RaftAddr, err) } defer raftListener.Close() raftTLS, err := cfg.RaftTLS.Config(getKey) if err != nil { log.Fatalf("Bad Raft TLS configuration: %s", err) } raftCreds := credentials.NewTLS(raftTLS) raftServer := grpc.NewServer(grpc.Creds(raftCreds)) go raftServer.Serve(raftListener) defer raftServer.Stop() dialRaftPeer := func(id uint64) raftproto.RaftClient { // TODO use current, not initial, config for _, replica := range cfg.KeyserverConfig.InitialReplicas { if replica.ID == id { conn, err := grpc.Dial(replica.RaftAddr, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{RootCAs: raftTLS.RootCAs}))) if err != nil { log.Panicf("Raft GRPC dial failed: %s", err) } return raftproto.NewRaftClient(conn) } } log.Panicf("No raft peer %x in configuration", id) return nil } raft := raftlog.New( cfg.ReplicaID, replicaIDs, db, []byte{tableReplicationLogPrefix}, clk, cfg.RaftHeartbeat.Duration(), raftServer, dialRaftPeer, ) defer raft.Stop() server, err := Open(cfg, db, raft, ratificationPolicy, clk, getKey, net.LookupTXT) if err != nil { log.Fatalf("Failed to initialize keyserver: %s", err) } server.Start() defer server.Stop() ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt) <-ch }
func main() { if len(os.Args) < 4 { fmt.Printf("usage: %s cacert cakey host0 host1 host2...\n", os.Args[0]) return } caCertFile := os.Args[1] caKeyFile := os.Args[2] caCertPem, err := ioutil.ReadFile(caCertFile) if err != nil { log.Fatalf("Failed to read from %v: %v", caCertFile, err) } caCertBlock, _ := pem.Decode(caCertPem) if caCertBlock == nil { log.Fatalf("Failed to parse PEM: %v", string(caCertPem)) } caCert, err := x509.ParseCertificate(caCertBlock.Bytes) if err != nil { log.Fatalf("Failed to parse X.509 cert: %v", err) } caKeyPem, err := ioutil.ReadFile(caKeyFile) if err != nil { log.Fatalf("Failed to read from %v: %v", caKeyFile, err) } caKeyBlock, _ := pem.Decode(caKeyPem) if caKeyBlock == nil { log.Fatalf("Failed to parse PEM: %v", string(caKeyPem)) } caKey, err := x509.ParseECPrivateKey(caKeyBlock.Bytes) if err != nil { log.Fatalf("Failed to parse EC private key: %v", err) } hosts := os.Args[3:] vrfPublic, vrfSecret, err := vrf.GenerateKey(rand.Reader) if err != nil { log.Panic(err) } var pks []*[ed25519.PublicKeySize]byte var sks []*[ed25519.PrivateKeySize]byte var replicas []*proto.Replica for _, host := range hosts { pk, sk, err := ed25519.GenerateKey(rand.Reader) if err != nil { log.Panic(err) } pks = append(pks, pk) sks = append(sks, sk) ppk := &proto.PublicKey{ PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pk[:]}, } replicas = append(replicas, &proto.Replica{ ID: proto.KeyID(ppk), PublicKeys: []*proto.PublicKey{ppk}, RaftAddr: fmt.Sprintf("%s:%d", host, raftPort), }) } // TODO: non-silly way of generating ID var serverID [8]byte sha3.ShakeSum128(serverID[:], vrfPublic) ksConfig := proto.KeyserverConfig{ Realm: realm, ServerID: binary.LittleEndian.Uint64(serverID[:]), VRFKeyID: "vrf.vrfsecret", MinEpochInterval: proto.DurationStamp(1 * time.Second), MaxEpochInterval: proto.DurationStamp(1 * time.Minute), ProposalRetryInterval: proto.DurationStamp(1 * time.Second), InitialReplicas: replicas, RegistrationPolicy: []*proto.RegistrationPolicy{ &proto.RegistrationPolicy{PolicyType: &proto.RegistrationPolicy_EmailProofByDKIM{ EmailProofByDKIM: &proto.EmailProofByDKIM{ AllowedDomains: []string{dkimProofDomain}, ToAddr: "*****@*****.**", SubjectPrefix: dkimProofPrefix}, }, }, &proto.RegistrationPolicy{PolicyType: &proto.RegistrationPolicy_EmailProofByOIDC{ EmailProofByOIDC: &proto.EmailProofByOIDC{ OIDCConfig: []*proto.OIDCConfig{ &proto.OIDCConfig{AllowedDomains: []string{"yahoo.com"}, DiscoveryURL: "https://login.yahoo.com", Issuer: "https://api.login.yahoo.com", ClientID: "foobar"}, }, }, }, }, }, } var cfgs []*proto.ReplicaConfig for i, host := range hosts { pk, sk := pks[i], sks[i] pked := &proto.PublicKey{PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pk[:]}} replicaID := proto.KeyID(pked) cert, err := make_cert(caCert, caKey, host) if err != nil { log.Fatal(err) } pcerts := []*proto.CertificateAndKeyID{{cert.Certificate, "tls.key.pem", nil}} heartbeat := proto.DurationStamp(1 * time.Second) cfg := &proto.ReplicaConfig{ KeyserverConfig: ksConfig, SigningKeyID: "signing.ed25519secret", ReplicaID: replicaID, PublicAddr: fmt.Sprintf("%s:%d", host, publicPort), VerifierAddr: fmt.Sprintf("%s:%d", host, verifierPort), HKPAddr: fmt.Sprintf("%s:%d", host, hkpPort), HTTPFrontAddr: fmt.Sprintf("%s:%d", host, httpFrontPort), RaftAddr: fmt.Sprintf("%s:%d", host, raftPort), PublicTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, VerifierTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, HKPTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, HTTPFrontTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}, ClientCAs: [][]byte{caCert.Raw}, ClientAuth: proto.REQUIRE_AND_VERIFY_CLIENT_CERT}, RaftTLS: proto.TLSConfig{Certificates: pcerts, RootCAs: [][]byte{caCert.Raw}}, LevelDBPath: "db", // TODO RaftHeartbeat: heartbeat, ClientTimeout: proto.DurationStamp(1 * time.Minute), LaggingVerifierScan: 1000, } cfgs = append(cfgs, cfg) if _, err := os.Stat(host + "/"); os.IsNotExist(err) { os.Mkdir(host, 0700) } tlsKeyF, err := os.OpenFile(path.Join(host, "tls.key.pem"), os.O_WRONLY|os.O_CREATE, 0600) defer tlsKeyF.Close() if err != nil { log.Panic(err) } pkDer, err := x509.MarshalECPrivateKey(cert.PrivateKey.(*ecdsa.PrivateKey)) if err != nil { log.Panic(err) } err = pem.Encode(tlsKeyF, &pem.Block{ Type: "EC PRIVATE KEY", Headers: make(map[string]string), Bytes: pkDer, }) if err != nil { log.Panic(err) } err = ioutil.WriteFile(path.Join(host, "vrf.vrfsecret"), vrfSecret[:], 0600) if err != nil { log.Panic(err) } err = ioutil.WriteFile(path.Join(host, "signing.ed25519secret"), sk[:], 0600) if err != nil { log.Panic(err) } configF, err := os.OpenFile(path.Join(host, "config.json"), os.O_WRONLY|os.O_CREATE, 0600) defer configF.Close() if err != nil { log.Panic(err) } err = new(jsonpb.Marshaler).Marshal(configF, cfg) if err != nil { log.Panic(err) } } err = ioutil.WriteFile("vrf.vrfpublic", vrfPublic[:], 0644) if err != nil { log.Panic(err) } clientCert, err := make_cert(caCert, caKey, "client") if err != nil { log.Fatal(err) } verificationPolicy := &proto.AuthorizationPolicy{ PublicKeys: make(map[uint64]*proto.PublicKey), PolicyType: &proto.AuthorizationPolicy_Quorum{ Quorum: &proto.QuorumExpr{Threshold: uint32(majority(len(hosts)))}, }, } replicaIDs := []uint64{} for i, cfg := range cfgs { replicaIDs = append(replicaIDs, cfg.ReplicaID) pk := &proto.PublicKey{ PubkeyType: &proto.PublicKey_Ed25519{Ed25519: pks[i][:]}, } pkid := proto.KeyID(pk) verificationPolicy.PublicKeys[pkid] = pk replicaExpr := &proto.QuorumExpr{ Threshold: 1, Candidates: []uint64{pkid}, } verificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions = append(verificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions, replicaExpr) } clientConfig := &proto.Config{ Realms: []*proto.RealmConfig{ &proto.RealmConfig{ Domains: []string{"yahoo-inc.com"}, Addr: cfgs[0].PublicAddr, VRFPublic: vrfPublic[:], VerificationPolicy: verificationPolicy, EpochTimeToLive: proto.DurationStamp(3 * time.Minute), TreeNonce: nil, ClientTLS: &proto.TLSConfig{ RootCAs: [][]byte{caCert.Raw}, Certificates: []*proto.CertificateAndKeyID{{clientCert.Certificate, "client.key.pem", nil}}, }, }, }, } clientConfigF, err := os.OpenFile("clientconfig.json", os.O_WRONLY|os.O_CREATE, 0644) defer clientConfigF.Close() if err != nil { log.Panic(err) } err = new(jsonpb.Marshaler).Marshal(clientConfigF, clientConfig) if err != nil { log.Panic(err) } clientKeyF, err := os.OpenFile("client.key.pem", os.O_WRONLY|os.O_CREATE, 0644) defer clientKeyF.Close() if err != nil { log.Panic(err) } skDer, err := x509.MarshalECPrivateKey(clientCert.PrivateKey.(*ecdsa.PrivateKey)) if err != nil { log.Panic(err) } err = pem.Encode(clientKeyF, &pem.Block{ Type: "EC PRIVATE KEY", Headers: make(map[string]string), Bytes: skDer, }) if err != nil { log.Panic(err) } }