// Join an existing cluster
func (s *Server) Join(primary string) error {
cs, err := transport.Encode(s.listen)
if err != nil {
return err
}
//log.Printf("My listen is: " + cs)
csPrimary, err := transport.Encode(primary)
if err != nil {
return err
}
//log.Printf("My listen is: " + s.listen)
//log.Printf("Attempting to Join: " + csPrimary)
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: cs,
}
b := util.JSONEncode(command)
for {
_, err := s.client.SafePost(csPrimary, "/join", "application/json", b)
return err
}
}
// Join an existing cluster
func (s *Server) Join(primary string) error {
join := &Join{Self: s.cluster.self}
b := util.JSONEncode(join)
cs, err := transport.Encode(primary)
if err != nil {
return err
}
for {
body, err := s.client.SafePost(cs, "/join", b)
if err != nil {
log.Printf("Unable to join cluster: %s", err)
time.Sleep(1 * time.Second)
continue
}
resp := &JoinResponse{}
if err = util.JSONDecode(body, &resp); err != nil {
return err
}
s.cluster.Join(resp.Self, resp.Members)
return nil
}
}
func (s *Server) connectionString() string {
cs,err := transport.Encode(s.listen)
if err != nil {
log.Fatal(err)
}
return cs
}
// Join an existing cluster
func (s *Server) Join(primary string) error {
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
}
var b bytes.Buffer
json.NewEncoder(&b).Encode(command)
cs, err := transport.Encode(primary)
if err != nil {
return err
}
log.Printf("Server %v with cs=%v is trying to join %v on %s/join", s.raftServer.Name(), s.connectionString(), primary, cs)
for {
_, err := s.client.SafePost(cs, "/join", &b)
if err != nil {
log.Printf("Unable to join cluster: %s", err)
time.Sleep(1 * time.Second)
continue
}
return nil
}
}
// This is the only user-facing function, and accordingly the body is
// a raw string rather than JSON.
func (s *Server) sqlHandler(w http.ResponseWriter, req *http.Request) {
if s.block {
time.Sleep(1000000 * time.Second)
}
query, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Printf("Couldn't read body: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
}
if s.leader != s.listen {
cs, errLeader := transport.Encode(s.leader)
if errLeader != nil {
http.Error(w, "Only the primary can service queries, but this is a secondary", http.StatusBadRequest)
log.Printf("Leader ain't present?: %s", errLeader)
return
}
//_, errLeaderHealthCheck := s.client.SafeGet(cs, "/healthcheck")
//if errLeaderHealthCheck != nil {
// http.Error(w, "Primary is down", http.StatusBadRequest)
// return
//}
body, errLResp := s.client.SafePost(cs, "/sql", bytes.NewBufferString(string(query)))
if errLResp != nil {
s.block = true
http.Error(w, "Can't forward request to primary, gotta block now", http.StatusBadRequest)
return
// log.Printf("Didn't get reply from leader: %s", errLResp)
}
formatted := fmt.Sprintf("%s", body)
resp := []byte(formatted)
w.Write(resp)
return
} else {
log.Debugf("Primary Received query: %#v", string(query))
resp, err := s.execute(query)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
}
w.Write(resp)
return
}
}
// This is the only user-facing function, and accordingly the body is
// a raw string rather than JSON.
func (s *Server) sqlHandler(w http.ResponseWriter, req *http.Request) {
query, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Printf("Couldn't read body: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Printf("[%s]Received query: %#v", string(s.raftServer.State()), string(query))
if s.raftServer.State() != "leader" {
if s.raftServer.Leader() == "" {
w.WriteHeader(http.StatusBadRequest)
return
}
leader, _ := transport.Encode(s.raftServer.Leader())
log.Printf("Relaying query to leader: %v", s.raftServer.Leader())
relayResp, err := s.client.SafePost(leader, "/sql", bytes.NewReader(query))
if err != nil {
http.Error(w, "Only the primary can service queries, relaying failed", http.StatusBadRequest)
return
}
//buf := new(bytes.Buffer)
//buf.ReadFrom(relayResp)
w.Write(relayResp.(*bytes.Buffer).Bytes())
return
}
// Execute the command against the Raft server.
resp, err := s.raftServer.Do(command.NewWriteCommand(string(query)))
if err != nil {
log.Printf("Current leader: %#v resp=%v err=%v", string(s.raftServer.Leader()), resp, err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if formatted, ok := resp.(string); ok {
log.Printf("Returning response to %#v: %#v", string(query), resp)
w.Write([]byte(formatted))
} else {
w.WriteHeader(http.StatusBadRequest)
}
}
// Creates a new server.
func New(path, listen string) (*Server, error) {
cs, err := transport.Encode(listen)
if err != nil {
return nil, err
}
sqlPath := filepath.Join(path, "storage.sql")
util.EnsureAbsent(sqlPath)
s := &Server{
path: path,
listen: listen,
sql: sql.NewSQL(sqlPath),
router: mux.NewRouter(),
client: transport.NewClient(),
cluster: NewCluster(path, cs),
}
return s, nil
}
// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
cs, err := transport.Encode(leader)
if err != nil {
log.Fatal(err)
}
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
}
var b bytes.Buffer
json.NewEncoder(&b).Encode(command)
log.Println(fmt.Sprintf("%s is joining cluster %s from %s (connection string = %s)", s.name, leader, s.listen, cs))
_, err = s.client.SafePost(cs, "/join", &b)
if err != nil {
return err
}
return nil
}
// Creates a new server.
func New(path, listen, join string) (*Server, error) {
cs, err := transport.Encode(listen)
if err != nil {
return nil, err
}
sqlPath := filepath.Join(path, "storage.sql")
util.EnsureAbsent(sqlPath)
s := &Server{
path: path,
listen: listen,
name: path,
connectionString: cs,
sql: sql.NewSQL(sqlPath),
router: mux.NewRouter(),
client: transport.NewClient(),
execCommand: make(chan *ExecCommand),
}
return s, nil
}
// Creates a new server.
func New(path, listen string) (*Server, error) {
cs, err := transport.Encode(listen)
if err != nil {
return nil, err
}
log.Printf("My connection string is %s", cs)
sqlPath := filepath.Join(path, "storage.sql")
util.EnsureAbsent(sqlPath)
s := &Server{
path: path,
name: strings.Replace(listen, "/", "-", -1),
listen: listen,
connection_string: cs,
sql: sql.NewSQL(sqlPath),
router: mux.NewRouter(),
client: transport.NewClient(),
}
return s, nil
}
// Creates a new server.
func New(path, listen string) (*Server, error) {
cs, err := transport.Encode(listen)
if err != nil {
return nil, err
}
log.Printf("Starting server at" + cs)
sqlPath := filepath.Join(path, "storage.sql")
util.EnsureAbsent(sqlPath)
s := &Server{
path: path,
listen: listen,
sql: sql.NewSQL(sqlPath),
router: mux.NewRouter(),
client: transport.NewClient(),
block: false,
}
return s, nil
}
// Join an existing cluster
func (s *Server) Join(primary string) error {
cs, err := transport.Encode(primary)
if err != nil {
return err
}
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
}
for {
b := util.JSONEncode(command)
_, err = s.client.SafePost(cs, "/join", b)
if err != nil {
log.Printf("Unable to join cluster: %s", err)
time.Sleep(1 * time.Second)
continue
}
return nil
}
return nil
}
// Join an existing cluster
func (s *Server) Join(primary string) error {
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString,
}
b := util.JSONEncode(command)
log.Printf("Joining Cluster: %s", primary)
cs, err := transport.Encode(primary)
if err != nil {
return err
}
for i := 0; i < 5; i += 1 {
log.Printf("Join -> %s Command: %s", cs, command)
_, err = s.client.SafePost(cs, "/join", b)
if err != nil {
log.Printf("Unable to join cluster: %s", err)
} else {
break
}
}
return err
}
// Returns the connection string.
func (s *Server) connectionString() string {
cs, _ := transport.Encode(s.listen)
log.Println("ConnectionString: " + cs)
return cs
}
// This is the only user-facing function, and accordingly the body is
// a raw string rather than JSON.
func (s *Server) sqlHandler(w http.ResponseWriter, req *http.Request) {
// read the query
query, err := ioutil.ReadAll(req.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if s.raftServer.State() == raft.Leader {
// just execute the query if we're the leader
resp, err := s.execSQL(string(query))
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Write(resp)
return
}
// if we're not the leader forward the request to some other host
// TODO: there must be a better way to do all this
// get a list of available mates
var _mates []string
var mates []string
_mates = make([]string, 0, len(s.raftServer.Peers())+1)
mates = make([]string, 0, len(s.raftServer.Peers())+1)
for _, peer := range s.raftServer.Peers() {
_mates = append(_mates, peer.Name)
mates = append(mates, peer.Name)
}
// shuffle the hosts
perm := rand.Perm(len(_mates))
for i, v := range perm {
mates[v] = _mates[i]
}
// if we know the leader, always try it first and only fallback
// to other hosts in the cluster if we don't (but maybe they do)
if s.raftServer.Leader() != "" {
mates = append(mates, s.raftServer.Leader())
}
// reverse the peers so that the leader is always first
for i, j := 0, len(mates)-1; i < j; i, j = i+1, j-1 {
mates[i], mates[j] = mates[j], mates[i]
}
for _, who := range mates {
// proxy to whoever is available
cs, err := transport.Encode(who)
if err != nil {
continue
}
// the number of retries is totally arbitrary
if output, ok := s.proxy(cs, query, len(mates)); ok {
w.Write(output)
return
}
}
http.Error(w, "error", http.StatusBadRequest)
}
// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
var err error
log.Printf("Initializing Raft Server: %s", s.path)
// Initialize and start Raft server.
transporter := raft.NewHTTPTransporter("/raft")
transporter.Transport.Dial = transport.UnixDialer
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
s.raftServer.Start()
s.raftServer.SetHeartbeatTimeout(1 * time.Millisecond)
s.raftServer.SetElectionTimeout(500 * time.Millisecond)
fn := func(e raft.Event) {
log.Printf("%s %v -> %v\n", e.Type(), e.PrevValue(), e.Value())
}
s.raftServer.AddEventListener(raft.StateChangeEventType, fn)
s.raftServer.AddEventListener(raft.LeaderChangeEventType, fn)
s.raftServer.AddEventListener(raft.TermChangeEventType, fn)
if leader != "" {
// Join to leader if specified.
log.Println("Attempting to join leader:", leader)
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
}
//time.Sleep(1 * time.Second)
if err := s.Join(leader); err != nil {
log.Println("Join failed")
log.Fatal(err)
}
log.Printf("Node %s joined leader %s" , s.connectionString(), leader)
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
log.Println("Initializing new cluster")
cs, err := transport.Encode(s.listen)
if err != nil {
return err
}
_, err = s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: cs,
})
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Recovered from log")
}
log.Println("Initializing HTTP server")
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error
rand.Seed(int64(time.Now().Nanosecond()))
s.primary = primary
s.name = "name-" + s.listen
raft.RegisterCommand(&BatchCommand{})
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
httpTransport := transport.NewClient().GetHTTPClient()
//log.Printf(("Initializing Raft Server")
transporter := NewHTTPTransporter("/raft", *httpTransport)
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.db, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
s.raftServer.Start()
s.raftServer.SetElectionTimeout(400 * time.Millisecond)
s.raftServer.AddEventListener("addPeer", func(e raft.Event) {
//log.Printf("Joined!")
s.joined = true
})
s.raftServer.AddEventListener("leaderChange", func(e raft.Event) {
leader := e.Value().(string)
if leader == s.name {
//log.Printf("Leader Changed to %v", leader)
s.leaderNotify <- leader
}
})
if primary == "" {
cs, err := transport.Encode(s.listen)
if err != nil {
log.Fatal(err)
}
//log.Printf(("Starting as Leader")
_, err = s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: cs,
})
//log.Printf(("I am Leader")
if err != nil {
log.Fatal(err)
}
} else {
//log.Printf("Waiting 100 milliseconds to join Primary")
time.AfterFunc(10*time.Millisecond, func() {
maxTries := 25
tries := 0
for !s.joined {
//log.Printf("Trying to Join")
tries++
//log.Printf("Attempting to Join")
s.Join(primary)
// if err != nil {
// //log.Printf("Failed to join")
// } else {
// //log.Printf("Joined!")
// break
// }
if tries > maxTries {
log.Fatal("Could not join!")
}
time.Sleep(JOIN_TIMEOUT)
}
})
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/healthcheck", s.healthcheckHandler).Methods("GET")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
s.router.HandleFunc("/forward", s.forwardHandler).Methods("POST")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
//log.Printf(("Serving?")
go s.healthChecker()
go s.processQueue()
go s.processNotifications()
s.httpServer.Serve(l)
return nil
}