func (s *Server) joinHandler(w http.ResponseWriter, req *http.Request) {
//log.Printf("JOIN REQUEST")
command := &raft.DefaultJoinCommand{}
if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
//log.Printf("FAILED TO DECODE JOIN REQUEST")
log.Fatal(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if s.raftServer.Leader() != s.name {
err := s.ForwardJoin(command)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
//log.Printf(("Got join request from: %v", command)
if _, err := s.raftServer.Do(command); err != nil {
log.Fatal(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//s.raftServer.AddPeer(command.Name, command.ConnectionString)
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error = nil
// Initialize and start Raft server.
transporter := raft.NewHTTPTransporter("/raft")
transporter.Transport.Dial = transport.UnixDialer
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
s.raftServer.SetElectionTimeout(750 * time.Millisecond)
transporter.Install(s.raftServer, s)
s.raftServer.Start()
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
if primary == "" {
if s.raftServer.IsLogEmpty() {
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
})
if err != nil {
log.Fatal(err)
}
}
} else {
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
}
// namePath, _ := filepath.Abs()
if err := s.Join(primary); err != nil {
log.Fatal(err)
}
log.Println("Joined leader:", s.raftServer.Leader())
}
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
func (s *Server) connectionString() string {
cs,err := transport.Encode(s.listen)
if err != nil {
log.Fatal(err)
}
return cs
}
// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
cs, err := transport.Encode(leader)
if err != nil {
log.Fatal(err)
}
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
}
var b bytes.Buffer
json.NewEncoder(&b).Encode(command)
log.Println(fmt.Sprintf("%s is joining cluster %s from %s (connection string = %s)", s.name, leader, s.listen, cs))
_, err = s.client.SafePost(cs, "/join", &b)
if err != nil {
return err
}
return nil
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
if primary == "" {
s.cluster.Init()
} else {
s.Join(primary)
go func() {
for {
if s.healthcheckPrimary() {
time.Sleep(10 * time.Millisecond)
continue
}
s.cluster.PerformFailover()
if s.cluster.State() == "primary" {
break
}
}
}()
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/replicate", s.replicationHandler).Methods("POST")
s.router.HandleFunc("/healthcheck", s.healthcheckHandler).Methods("GET")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
if primary == "" {
s.leader = s.listen
} else {
s.leader = primary
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/healthcheck", s.healthcheckHandler).Methods("GET")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
func main() {
//defer profile.Start(profile.CPUProfile).Stop()
var raftdebug, verbose bool
var listen, join, directory string
flag.BoolVar(&verbose, "v", false, "Enable debug output")
flag.BoolVar(&raftdebug, "r", false, "Enable raft debugging")
flag.StringVar(&listen, "l", "127.0.0.1:4000", "Socket to listen on (Unix or TCP)")
flag.StringVar(&join, "join", "", "Cluster to join")
flag.StringVar(&directory, "d", "", "Storage directory")
dir := filepath.Dir(os.Args[0])
base := "./" + filepath.Base(os.Args[0])
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `Usage: %s [options]
SQLCluster is a highly-available SQL store. It accepts commands over
HTTP, and returns the output of commands together with a
SequenceNumber which indicates the ordering in which requests are
being applied. (SequenceNumber is useful for validation, and can
otherwise be ignored.)
Run a cluster as follows:
cd %s
%s -d /tmp/sqlcluster/node0 &
%s -d /tmp/sqlcluster/node1 -l 127.0.0.1:4001 --join 127.0.0.1:4000 &
%s -d /tmp/sqlcluster/node2 -l 127.0.0.1:4002 --join 127.0.0.1:4000
You can then issue queries using your favorite HTTP client.
curl 127.0.0.1:4000/sql -d 'CREATE TABLE hello (world int)'
curl 127.0.0.1:4000/sql -d 'INSERT INTO hello (world) VALUES (1), (2)'
curl 127.0.0.1:4000/sql -d 'SELECT * FROM hello'
This should return the following sequence of outputs:
- SequenceNumber: 0
- SequenceNumber: 1
- SequenceNumber: 2
1
2
By default, SQLCluster will listen on a TCP port. However, if you
specify a listen address that begins with a / or ., that will be
interpeted as Unix path for SQLCluster to listen on. (Note that
Octopus runs using Unix sockets only, but it will probably be more
convient for you to develop using TCP.)
OPTIONS:
`, os.Args[0], dir, base, base, base)
flag.PrintDefaults()
}
flag.Parse()
if flag.NArg() != 0 {
flag.Usage()
os.Exit(1)
}
log.SetVerbose(verbose)
if directory == "" {
var err error
directory, err = ioutil.TempDir("/tmp", "node")
if err != nil {
log.Fatalf("Could not create temporary base directory: %s", err)
}
defer os.RemoveAll(directory)
log.Printf("Storing state in tmpdir %s", directory)
} else {
if err := os.MkdirAll(directory, os.ModeDir|0755); err != nil {
log.Fatalf("Error while creating storage directory: %s", err)
}
}
log.Printf("Changing directory to %s", directory)
if err := os.Chdir(directory); err != nil {
log.Fatalf("Error while changing to storage directory: %s", err)
}
// Make sure we don't leave stranded sqlclusters lying around
go func() {
for {
time.Sleep(2 * time.Second)
if os.Getppid() == 1 {
log.Fatal("Parent process exited; terminating")
}
}
}()
runtime.GOMAXPROCS(4)
rand.Seed(time.Now().UnixNano())
raft.RegisterCommand(&server.WriteCommand{})
if raftdebug {
raft.SetLogLevel(raft.Debug)
}
// raft.SetLogLevel(raft.Trace)
// Start the server
go func() {
s, err := server.New(directory, listen)
if err != nil {
log.Fatal(err)
}
if err := s.ListenAndServe(join); err != nil {
log.Fatal(err)
}
}()
// Exit cleanly so we can remove the tmpdir
sigchan := make(chan os.Signal)
signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
<-sigchan
}
// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
var err error
log.Printf("Initializing Raft Server: %s", s.path)
// Initialize and start Raft server.
transporter := raft.NewHTTPTransporter("/raft")
transporter.Transport.Dial = transport.UnixDialer
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
s.raftServer.Start()
s.raftServer.SetHeartbeatTimeout(1 * time.Millisecond)
s.raftServer.SetElectionTimeout(500 * time.Millisecond)
fn := func(e raft.Event) {
log.Printf("%s %v -> %v\n", e.Type(), e.PrevValue(), e.Value())
}
s.raftServer.AddEventListener(raft.StateChangeEventType, fn)
s.raftServer.AddEventListener(raft.LeaderChangeEventType, fn)
s.raftServer.AddEventListener(raft.TermChangeEventType, fn)
if leader != "" {
// Join to leader if specified.
log.Println("Attempting to join leader:", leader)
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
}
//time.Sleep(1 * time.Second)
if err := s.Join(leader); err != nil {
log.Println("Join failed")
log.Fatal(err)
}
log.Printf("Node %s joined leader %s" , s.connectionString(), leader)
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
log.Println("Initializing new cluster")
cs, err := transport.Encode(s.listen)
if err != nil {
return err
}
_, err = s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: cs,
})
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Recovered from log")
}
log.Println("Initializing HTTP server")
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error
rand.Seed(int64(time.Now().Nanosecond()))
s.primary = primary
s.name = "name-" + s.listen
raft.RegisterCommand(&BatchCommand{})
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
httpTransport := transport.NewClient().GetHTTPClient()
//log.Printf(("Initializing Raft Server")
transporter := NewHTTPTransporter("/raft", *httpTransport)
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.db, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
s.raftServer.Start()
s.raftServer.SetElectionTimeout(400 * time.Millisecond)
s.raftServer.AddEventListener("addPeer", func(e raft.Event) {
//log.Printf("Joined!")
s.joined = true
})
s.raftServer.AddEventListener("leaderChange", func(e raft.Event) {
leader := e.Value().(string)
if leader == s.name {
//log.Printf("Leader Changed to %v", leader)
s.leaderNotify <- leader
}
})
if primary == "" {
cs, err := transport.Encode(s.listen)
if err != nil {
log.Fatal(err)
}
//log.Printf(("Starting as Leader")
_, err = s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: cs,
})
//log.Printf(("I am Leader")
if err != nil {
log.Fatal(err)
}
} else {
//log.Printf("Waiting 100 milliseconds to join Primary")
time.AfterFunc(10*time.Millisecond, func() {
maxTries := 25
tries := 0
for !s.joined {
//log.Printf("Trying to Join")
tries++
//log.Printf("Attempting to Join")
s.Join(primary)
// if err != nil {
// //log.Printf("Failed to join")
// } else {
// //log.Printf("Joined!")
// break
// }
if tries > maxTries {
log.Fatal("Could not join!")
}
time.Sleep(JOIN_TIMEOUT)
}
})
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/healthcheck", s.healthcheckHandler).Methods("GET")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
s.router.HandleFunc("/forward", s.forwardHandler).Methods("POST")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
//log.Printf(("Serving?")
go s.healthChecker()
go s.processQueue()
go s.processNotifications()
s.httpServer.Serve(l)
return nil
}
// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
var err error
// Initialize and start HTTP server.
log.Println("Initializing HTTP server")
s.httpServer = &http.Server{
Handler: s.router,
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
// Start Unix transport
log.Println(s.name, "listening at", s.listen)
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
// initialize raft
transporter := raft.NewHTTPTransporter("/raft")
// swap the dialer with the unix dialer that also allows unix-sockets to
// be passed around
transporter.Transport.Dial = transport.UnixDialer
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
// this seems to yield good results, but it's definitely not the most
// empirical of the measurements
s.raftServer.SetElectionTimeout(800 * time.Millisecond)
s.raftServer.SetHeartbeatTimeout(150 * time.Millisecond)
s.raftServer.Start()
if leader != "" {
// Join the leader if specified.
log.Println("Attempting to join the leader:", leader)
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
return nil
}
// retry the join until we actually join the cluster
// (it may take a while, until octopus sets the sockets up)
for {
if err := s.Join(leader); err == nil {
break
}
log.Fatal(err)
time.Sleep(10 * time.Millisecond)
}
log.Println("joined.")
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
log.Println("Initializing new cluster")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connectionString(),
})
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Recovered from log")
}
log.Println(s.name, "IS READY TO ACCEPT REQUESTS")
return s.httpServer.Serve(l)
}
func EnsureAbsent(path string) {
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
log.Fatal(err)
}
}
// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
var err error
leader = strings.Replace(leader, "/", "-", -1)
log.Printf("Initializing Raft Server: %s", s.path)
// Initialize and start Raft server.
transporter := raft.NewHTTPTransporter("/raft")
transporter.Transport.Dial = transport.UnixDialer
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
s.raftServer.SetElectionTimeout(200 * time.Millisecond) // default 150ms
s.raftServer.SetHeartbeatTimeout(80 * time.Millisecond) // default 50ms
transporter.Install(s.raftServer, s)
s.raftServer.Start()
if leader != "" {
// Join to leader if specified.
log.Println("Attempting to join leader:", leader)
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
}
for tries := 0; tries < 10; tries += 1 {
err := s.Join(leader)
if err == nil {
break
}
log.Printf("Join attempt %d failed; sleeping", tries)
time.Sleep(200 * time.Millisecond)
}
if err != nil {
log.Fatal(err)
}
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
log.Println("Initializing new cluster")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: s.connection_string,
})
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Recovered from log")
}
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
s.router.HandleFunc("/sql", s.sqlHandler).Methods("POST")
s.router.HandleFunc("/join", s.joinHandler).Methods("POST")
s.router.HandleFunc("/forward", s.forwardHandler).Methods("GET")
// Start Unix transport
l, err := transport.Listen(s.listen)
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}
// Starts the server.
func (s *Server) ListenAndServe(primary string) error {
var err error
// Initialize and start HTTP server.
s.httpServer = &http.Server{
Handler: s.router,
}
s.httpServer2 = &http.Server{
Handler: s.router,
}
transporter := raft.NewHTTPTransporter("/raft")
s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.sql, "")
if err != nil {
log.Fatal(err)
}
transporter.Install(s.raftServer, s)
s.raftServer.Start()
if primary != "" {
log.Println("Attempting to join primary:", primary)
if !s.raftServer.IsLogEmpty() {
log.Fatal("Cannot join with an existing log")
}
if err := s.Join(primary); err != nil {
log.Fatal(err)
}
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
log.Println("Initializing new cluster")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.name,
ConnectionString: s.connectionString,
})
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Recovered from log")
}
log.Println("Initializing HTTP server")
s.router.HandleFunc("/sql", transport.MakeGzipHandler(s.sqlHandler)).Methods("POST")
s.router.HandleFunc("/forward", s.forwardHandler).Methods("GET")
s.router.HandleFunc("/healthcheck", transport.MakeGzipHandler(s.healthcheckHandler)).Methods("GET")
s.router.HandleFunc("/join", transport.MakeGzipHandler(s.joinHandler)).Methods("POST")
log.Println("Listening at:", s.connectionString)
// Start Unix transport
l, err := transport.Listen(s.listen)
for retry := 0; retry < 5 && err != nil; retry += 1 {
time.Sleep(100 * time.Millisecond)
l, err = transport.Listen(s.listen)
}
if err != nil {
log.Fatal(err)
}
return s.httpServer.Serve(l)
}