func (lfh *LocalFileHeader) Save() {
// TODO: Fix this double marshalling!
bytes, err := json.Marshal(lfh.header)
if err != nil {
log.Error("Couldn't marshal header: %s", err)
}
file, err := os.Create(lfh.headerpath)
if err != nil {
log.Error("Couldn't save header file for %s: %s", lfh.headerpath, err)
} else {
n, err := file.Write(lfh.header.ToJSON())
if n != len(bytes) {
log.Error("Didn't write all header data: written %n bytes", n)
}
if err != nil {
log.Error("Couldn't save file header: %s", err)
}
}
file.Close()
}
func (fss *FsService) replicationWatcher() {
for fss.running {
if fss.replQueue.Len() > 0 {
fss.replQueueMutex.Lock()
if fss.replQueue.Len() > 0 {
next := fss.replQueue.Front()
fss.replQueue.Remove(next)
fss.replQueueMutex.Unlock()
path := next.Value.(*Path)
localheader := fss.headers.GetFileHeader(path)
log.Info("%d: FSS: Starting replica download for path %s version %d...", fss.cluster.MyNode.Id, path, localheader.header.Version)
// TODO: Make sure we don't download the same replica twice...
// check if the file doesn't already exist locally
file := OpenFile(fss, localheader, 0)
if !file.Exists() {
// TODO: Use config to get temp path
tempfile := fmt.Sprintf("%s/%d.%d.%d.data", os.TempDir(), path.Hash(), time.Nanoseconds(), localheader.header.Version)
fd, err := os.Create(tempfile)
if err == nil {
_, err = fss.Read(path, 0, -1, 0, fd, nil)
fd.Close()
if err == nil {
os.Rename(tempfile, file.datapath)
log.Info("%d: FSS: Successfully replicated %s version %d locally", fss.cluster.MyNode.Id, path, localheader.header.Version)
} else {
log.Error("%d: FSS: Couldn't replicate file %s locally because couldn't read: %s", fss.cluster.MyNode.Id, path, err)
}
} else {
log.Error("%d: FSS: Couldn't open temporary file %s to download replica localy for path %s", fss.cluster.MyNode.Id, tempfile, path)
os.Remove(tempfile)
}
} else {
log.Info("%d: FSS: Local replica for %s version %d already exist", fss.cluster.MyNode.Id, path, localheader.header.Version)
}
} else {
fss.replQueueMutex.Unlock()
}
} else {
// if no more replica in the queue, stop replica force
if fss.replQueue.Len() == 0 {
fss.replForce = false
}
}
// TODO: Put that in configuration
if !fss.replForce {
time.Sleep(100 * 1000 * 1000)
}
}
}
func (s *Server) acceptUDP() {
// Looping for new messages
for {
buf := make([]byte, MAX_MSG_SIZE)
n, adr, err := s.udpsock.ReadFrom(buf)
if s.comm.running {
if err != nil {
log.Error("Error while reading UDP (read %d) from %s: %s\n", n, adr, err)
} else {
abcon := net.Conn(s.udpsock)
connection := NewConnection(s.comm.pool, P_UDP, D_Inbound, abcon)
read := io.Reader(bytes.NewBuffer(buf))
msg := s.comm.NewMessage()
msg.connection = connection
err := msg.readMessage(read)
if err != nil {
log.Error("Couldn't handle message received from UDP because of errors: %s %s\n", msg, err)
} else {
go s.comm.handleMessage(msg)
}
}
} else {
log.Info("Dropping connection because communications have been paused")
}
}
}
func (fss *FsService) sendToReplicaNode(resolv *cluster.ResolveResult, req_cb func(node *cluster.Node) *comm.Message) chan os.Error {
toSyncCount := resolv.Count() - 1 // minus one for the master
var syncError os.Error = nil
myNodeId := fss.cluster.MyNode.Id
errChan := make(chan os.Error, 1) // channel used to return data to the messageor
c := make(chan bool, toSyncCount) // channel used to wait for all replicas
if toSyncCount > 0 {
go func() {
for i := 0; i < resolv.Count(); i++ {
node := resolv.Get(i)
if node.Status == cluster.Status_Online && node.Id != myNodeId {
// get the new message
req := req_cb(node)
req.Timeout = 1000 // TODO: Config
req.OnResponse = func(message *comm.Message) {
log.Debug("%d: FSS: Received acknowledge message for message %s\n", fss.cluster.MyNode.Id, req)
c <- true
}
req.OnTimeout = func(last bool) (retry bool, handled bool) {
// TODO: Retry it!
syncError = comm.ErrorTimeout
log.Error("%d: FSS: Couldn't send message to replicate node %s because of a timeout for message %s\n", fss.cluster.MyNode.Id, node, req)
c <- true
return true, false
}
req.OnError = func(message *comm.Message, syncError os.Error) {
log.Error("%d: FSS: Received an error while sending to replica %s for message %s: %d %s\n", fss.cluster.MyNode.Id, req, node, syncError)
c <- true
}
fss.comm.SendNode(node, req)
}
}
// wait for nodes to sync the handoff
for i := 0; i < toSyncCount; i++ {
<-c
}
errChan <- syncError
}()
} else {
errChan <- nil
}
return errChan
}
func (c Config) Save(path string) {
bytes, err := json.Marshal(c)
if err != nil {
log.Error("Couldn't save config: %s", err)
}
fc, err := os.Create(path)
_, err = fc.Write(bytes)
if err != nil {
log.Error("Couldn't write config: %s", err)
}
}
func LoadConfig(path string) Config {
config := new(Config)
fc, err := os.Open(path)
if err != nil {
log.Error("Couldn't load config file: %s", err)
}
buf := new(bytes.Buffer)
buf.ReadFrom(fc)
strOrigConfig := string(buf.Bytes())
strFinConfig := ""
lines := strings.Split(strOrigConfig, "\n", -1)
for _, line := range lines {
trimedLine := strings.TrimSpace(line)
if len(trimedLine) < 2 || trimedLine[:2] != "//" {
strFinConfig += trimedLine
}
}
err = json.Unmarshal([]byte(strFinConfig), config)
if err != nil {
log.Fatal("Couldn't unmarshal config: %s %s", strFinConfig, err)
}
return *config
}
func (cs *ClusterService) RemoteContactMaster(msg *comm.Message) {
myNode := cs.cluster.MyNode
log.Debug("%d: Got a ContactMaster request: %s", myNode.Id, msg)
if cs.state == state_online {
masters := cs.cluster.Rings.GetRing(cs.masterRing).ResolveToken(master_token)
// make sure I'm the master, and online
if myNode.Status == cluster.Status_Online && masters.IsFirst(myNode) {
node := cluster.NewEmptyNode()
err := node.Unserialize(msg.Message)
if err != nil {
cs.comm.RespondError(msg, os.NewError("Couldn't unmarshal node data"))
log.Error("Couldn't unmarshal node data: %s", err)
return
}
node.Status = cluster.Status_Online
cs.cluster.MergeNode(node, true)
// TODO: Send the cluster back to the node
resp := cs.comm.NewMsgMessage(cs.serviceId)
cs.comm.RespondSource(msg, resp)
//log.Fatal("BLABLA", node.Id)
// TODO: LOCK SO THAT WE DON'T MAKE IT ONLINE TWICE
// TODO: Accept the node
// TODO: Check its rings
// TODO: Broadcast the change
} else {
cs.comm.RedirectFirst(masters, msg)
}
}
}
func (s *segment) replay(db *Db) (err os.Error) {
log.Info("Replaying segment %s", s)
entrych, errch := s.iter(0)
end := false
count := 0
for !end {
select {
case entry, ok := <-entrych:
if ok {
count++
err = entry.mutation.execute(db, true) // execute mutation (for replay)
if err != nil {
log.Error("Got an error replaying a mutation: %s", err)
return
}
} else {
end = true
}
case segerr, ok := <-errch:
if ok {
return segerr
} else {
end = true
}
}
}
log.Info("Segment %s replayed: %d mutations replayed", s, count)
return
}
func (f *FileHeader) ToJSON() []byte {
bytes, err := json.Marshal(f)
if err != nil {
log.Error("Couldn't marshal header: %s", err)
}
return bytes
}
func (fss *FsService) Unlock(key string) {
mutex, found := fss.mutexes[key]
if !found {
log.Error("FSS: Couldn't find file mutex to unlock for key %s\n", key)
return
}
mutex.Unlock()
}
func (f *File) Size() int64 {
dir, err := os.Stat(f.datapath)
if err != nil {
log.Error("Cannot stat data file: %s\n", f.path)
} else {
return dir.Size
}
return 0
}
// Returns an API Server
func NewServer(handler Handler, adr string) *Server {
server := new(Server)
server.handler = handler
server.servmux = http.NewServeMux()
con, err := net.Listen("tcp", adr)
if err != nil {
log.Error("API: Couldn't create listener socket: %s\n", err)
}
// Add handling function at root, delegating everything to the handler
server.servmux.HandleFunc("/", func(httpresp http.ResponseWriter, httpreq *http.Request) {
log.Debug("API: Connection on url %s\n", httpreq.URL)
resp := &ResponseWriter{httpresp}
var req *Request
if req = NewRequest(resp, httpreq); req == nil {
log.Error("API: Couldn't create request object")
return
}
handler.Handle(resp, req)
// TODO: Remove that! Shouldn't be here!!
// Read the rest, so we don't cause Broken Pipe on the other end if we don't read to the end
ioutil.ReadAll(req.Body)
})
// Start serving the API on another thread
go func() {
log.Debug("API: Starting API server on adr %s\n", adr)
err = http.Serve(con, server.servmux)
con.Close()
if err != nil {
log.Fatal("API: Serve error: ", err.String())
}
}()
return server
}
// Returns an interface marshaled in JSON
func (r *ResponseWriter) ReturnJSON(v interface{}) {
bytes, err := json.Marshal(v)
if err != nil {
log.Error("API: Couldn't marshall JSON response: %s\n", err)
}
r.Header().Set("Content-Type", "application/json")
//r.SetHeader("Content-Type", "application/json")
r.Write(bytes)
}
func LoadFileHeaderFromJSON(bytes []byte) *FileHeader {
fh := new(FileHeader)
errtok := json.Unmarshal(bytes, fh)
if errtok != nil {
fh = new(FileHeader)
log.Error("Couldn't load file header: %s, %s", errtok, bytes)
}
return fh
}
func (api *api) get(resp *rest.ResponseWriter, req *rest.Request, path *Path) {
log.Debug("FSS API: Received a read request for path %s\n", path)
// TODO: Handle offset
// TODO: Handle version
// TODO: Handle size
_, err := api.fss.Read(path, 0, -1, 0, resp, nil)
log.Debug("API: Fs Read data returned\n")
if err != nil && err != os.EOF {
log.Error("API: Fs Read returned an error for %s: %s\n", path, err)
resp.ReturnError(err.String())
}
}
func (api *api) head(resp *rest.ResponseWriter, req *rest.Request, path *Path) {
log.Debug("FSS API: Received a head request for path %s\n", path)
header, err := api.fss.HeaderJSON(path, nil)
resp.Write(header)
log.Debug("API: Fs Header data returned\n")
if err != nil {
log.Error("API: Fs header returned an error: %s\n", err)
resp.ReturnError(err.String())
}
}
func (s *Server) acceptTCP() {
for {
conn, err := s.tcpsock.Accept()
if s.comm.running {
if err != nil {
log.Error("Couldn't accept TCP connexion: %s\n", err)
}
go s.handleTCPConnection(conn)
} else {
log.Info("Dropping connection because communications have been paused")
}
}
}
func (p *Pool) GetMsgConnection(node *cluster.Node) *Connection {
// TODO: IMPLEMENT THE POOLING
adr := net.UDPAddr{node.Address, int(node.UdpPort)}
con, err := net.DialUDP("udp", nil, &adr) // TODO: should use local address instead of nil (implicitly local)
if err != nil {
log.Error("NETPOOL: Couldn't create a connection\n", err)
return nil
}
abcon := net.Conn(con)
connection := NewConnection(p, P_UDP, D_Outbound, abcon)
return connection
}
func (s *Server) handleTCPConnection(conn net.Conn) {
connection := NewConnection(s.comm.pool, P_TCP, D_Inbound, conn)
reader := io.Reader(conn)
msg := s.comm.NewMessage()
msg.connection = connection
err := msg.readMessage(reader)
if err != nil {
log.Error("Couldn't handle message received from TCP because of errors: %s %s\n", msg, err)
conn.Close() // Close the connection to make sure we don't cause error
} else {
s.comm.handleMessage(msg)
}
}
// Returns a new request composed of the original http.Request structure
func NewRequest(r *ResponseWriter, httpreq *http.Request) *Request {
req := new(Request)
req.Request = httpreq
// try to parse the URL
params, err := http.ParseQuery(httpreq.URL.RawQuery)
if err != nil {
log.Error("API: Couldn't parse GET parameters: %s\n", err)
r.ReturnError("Parameters error")
return nil
}
req.Params = params
return req
}
func (api *api) delete(resp *rest.ResponseWriter, req *rest.Request, path *Path) {
log.Debug("FSS API: Received a delete request for %s\n", path)
recursive := false
mrec, ok := req.Params["recursive"]
if ok {
recursive = (mrec[0] == "1") || (mrec[0] == "true")
}
err := api.fss.Delete(path, recursive, nil)
if err != nil {
log.Error("API: Fs Write returned an error: %s\n", err)
resp.ReturnError(err.String())
}
}
func (fss *FsService) RemoteChildRemove(message *comm.Message) {
str, _ := message.Message.ReadString() // path
path := NewPath(str)
child, _ := message.Message.ReadString() // child
log.Debug("FSS: Received message to remove the child %s from %s\n", child, path)
// resolve path
mynode := fss.cluster.MyNode
resolv := fss.ring.Resolve(path.String())
// only the master has the lock
if resolv.IsFirst(mynode) {
fss.Lock(path.String())
}
localheader := fss.headers.GetFileHeader(path)
localheader.header.RemoveChild(child)
if resolv.IsFirst(mynode) {
// replicate to nodes
syncChan := fss.sendToReplicaNode(resolv, func(node *cluster.Node) *comm.Message {
msg := fss.comm.NewMsgMessage(fss.serviceId)
msg.Function = "RemoteChildRemove"
msg.Message.WriteString(path.String()) // path
msg.Message.WriteString(child) // child name
return msg
})
// wait for replicas sync
syncError := <-syncChan
// check for sync error
if syncError != nil {
log.Error("FSS: Couldn't replicate remove child to nodes: %s\n", syncError)
fss.comm.RespondError(message, os.NewError("Couldn't replicate remove child to all nodes"))
}
// unlock
fss.Unlock(path.String())
}
// Send an acknowledgement
msg := fss.comm.NewMsgMessage(fss.serviceId)
fss.comm.RespondSource(message, msg)
}
func (api *api) post(resp *rest.ResponseWriter, req *rest.Request, path *Path) {
log.Debug("FSS API: Received a write request for %d bytes\n", req.ContentLength)
mimetype := "application/octet-stream"
mtar, ok := req.Params["type"]
if ok {
mimetype = mtar[0]
}
err := api.fss.Write(path, req.ContentLength, mimetype, req.Body, nil)
if err != nil {
log.Error("API: Fs Write returned an error: %s\n", err)
resp.ReturnError(err.String())
}
log.Debug("API: Fs Write returned\n")
}
func (comm *Comm) handleMessage(message *Message) {
// TODO: We should make sure we don't handle a message twice (since UDP can duplicate packets)
if message.FunctionId == FUNC_ERROR {
log.Info("Comm: Received an error: %s\n", message)
}
// Check if the message needed an acknowledgement or check if
// it has an error callback
handled := false
if message.SourceNode().Equals(comm.Cluster.MyNode) {
handled = comm.handleTracker(message)
}
// Service #0 is net and there is no implementation yet.
// Function = RESPONSE should be handled by callback
if message.ServiceId != 0 && message.FunctionId != FUNC_RESPONSE && !handled {
serviceWrapper := comm.GetWrapper(message.ServiceId)
if serviceWrapper.service != nil {
// if its an error
if message.FunctionId == FUNC_ERROR {
err := ReadErrorPayload(message)
message.SeekZero()
serviceWrapper.service.HandleUnmanagedError(message, err)
} else {
// call the right function
handled := serviceWrapper.callFunction(message.FunctionId, message)
if !handled {
serviceWrapper.service.HandleUnmanagedMessage(message)
}
}
} else {
log.Error("Comm: Couldn't find service for message %s\n", message)
}
}
}
func (cs *ClusterService) loadCluster() {
cs.clusterMutex.Lock()
// Load data
log.Debug("cls: Loading cluster data...")
stat, err := os.Stat(cs.clsDataPath)
if err == nil && stat.IsRegular() {
file, err := os.Open(cs.clsDataPath)
if err == nil {
typedFile := typedio.NewReader(file)
cs.clusterVersion, _ = typedFile.ReadInt64() // cluster version
cs.diskVerson = cs.clusterVersion
nbNodes, _ := typedFile.ReadUint16() // nodes count
var i uint16
for i = 0; i < nbNodes; i++ {
node := cluster.NewEmptyNode()
node.Unserialize(typedFile)
node.Status = cluster.Status_Offline
cs.cluster.MergeNode(node, false) // merge node, doesn't notify
}
} else {
log.Error("cls: Error while opening data file", err)
}
}
// replay commit log
log.Info("cls: Replaying commit log...")
cs.commitlog.Replay()
// TODO: Load cluster data
cs.clusterMutex.Unlock()
}
func NewLocalFileHeader(headerpath string) *LocalFileHeader {
lfh := new(LocalFileHeader)
lfh.headerpath = headerpath
file, err := os.Open(lfh.headerpath)
// no error
if err == nil {
lfh.header = LoadFileHeader(file)
} else {
if ptherror, ok := err.(*os.PathError); ok && ptherror.Error == os.ENOENT {
// if file doesn't exists, we don't show an error
lfh.header = NewFileHeader()
} else {
lfh.header = NewFileHeader()
log.Error("Couldn't load header file for %s: %d", lfh.headerpath, err)
}
}
file.Close()
return lfh
}
func (cs *ClusterService) HandleUnmanagedMessage(msg *comm.Message) {
log.Error("CS: Got an unmanaged message: %s", msg)
}
func (r *Message) writeMessage(writer io.Writer) (err os.Error) {
twriter := typedio.NewWriter(writer)
err = twriter.WriteUint16(r.Id) // message id
if err != nil {
return
}
// prepare flags
var flags byte
if r.InitId != 0 {
flags = flags | prm_has_init_msg_id
}
if r.Type == T_DATA {
flags = flags | prm_is_data
}
if r.srcNodeAdhoc {
flags = flags | prm_src_node_adhoc
}
if r.middleNodePresent {
flags = flags | prm_has_middle_node
}
if r.middleNodeAdhoc {
flags = flags | prm_middle_node_adhoc
}
err = twriter.WriteUint8(flags) // flags
if err != nil {
return
}
if r.InitId != 0 {
err = twriter.WriteUint16(r.InitId) // initial message id
if err != nil {
return
}
}
err = twriter.WriteUint8(r.ServiceId) // service id
if err != nil {
return
}
msgSize := uint16(r.Message.Size)
err = twriter.WriteUint16(msgSize) // message size
if err != nil {
return
}
if r.Type == T_DATA {
twriter.WriteInt64(r.DataSize) // data size
if err != nil {
return
}
}
if r.srcNodeAdhoc { // source node information
err = twriter.WriteString(r.srcNodeAdr.String()) // addr
if err != nil {
return
}
err = twriter.WriteUint16(r.srcNodeTcpPort) // tcp port
if err != nil {
return
}
err = twriter.WriteUint16(r.srcNodeUdpPort) // udp port
if err != nil {
return
}
} else {
err = twriter.WriteUint16(r.srcNodeId) // node id
if err != nil {
return
}
}
if r.middleNodePresent { // middle node information
if r.middleNodeAdhoc {
err = twriter.WriteString(r.middleNodeAdr.String()) // addr
if err != nil {
return
}
err = twriter.WriteUint16(r.middleNodeTcpPort) // tcp port
if err != nil {
return
}
err = twriter.WriteUint16(r.middleNodeUdpPort) // udp port
if err != nil {
return
}
} else {
err = twriter.WriteUint16(r.middleNodeId) // node id
if err != nil {
return
}
}
}
err = twriter.WriteUint8(r.FunctionId) // function id
if err != nil {
return
}
// Write message
r.Message.Seek(0, 0)
w, err := io.Copyn(writer, r.Message, r.Message.Size) // message
if err != nil {
log.Error("Couldn't write message message to writer: %s\n", err)
return err
}
if w != int64(msgSize) {
log.Error("Couldn't write the whole message message to write: written %d out of %d\n", w, msgSize)
return os.NewError("Message write truncated")
}
// Write data
if r.Type == T_DATA {
io.Copyn(writer, r.Data, r.DataSize) // data
}
return nil
}
func (r *Message) readMessage(reader io.Reader) (err os.Error) {
treader := typedio.NewReader(reader)
r.Id, err = treader.ReadUint16() // message id
if err != nil {
return
}
flags, err := treader.ReadUint8() // flags
if err != nil {
return
}
hasInitId := false
if flags&prm_has_init_msg_id == prm_has_init_msg_id {
hasInitId = true
}
if flags&prm_is_data == prm_is_data {
r.Type = T_DATA
} else {
r.Type = T_MSG
}
if flags&prm_src_node_adhoc == prm_src_node_adhoc {
r.srcNodeAdhoc = true
}
if flags&prm_has_middle_node == prm_has_middle_node {
r.middleNodePresent = true
}
if flags&prm_middle_node_adhoc == prm_middle_node_adhoc {
r.middleNodeAdhoc = true
}
if hasInitId {
r.InitId, err = treader.ReadUint16() // initial message id
if err != nil {
return
}
}
r.ServiceId, err = treader.ReadUint8() // service id
if err != nil {
return
}
msgSize, err := treader.ReadUint16() // message size
if err != nil {
return
}
if r.Type == T_DATA {
r.DataSize, err = treader.ReadInt64() // data size
if err != nil {
return
}
}
if r.srcNodeAdhoc { // source node information
adr, err := treader.ReadString() // addr
if err != nil {
return
}
r.srcNodeAdr = net.ParseIP(adr)
r.srcNodeTcpPort, err = treader.ReadUint16() // tcp port
if err != nil {
return
}
r.srcNodeUdpPort, err = treader.ReadUint16() // udp port
if err != nil {
return
}
} else {
r.srcNodeId, err = treader.ReadUint16() // node id
if err != nil {
return
}
}
// TODO: Adhoc
if r.middleNodePresent {
if r.middleNodeAdhoc {
adr, err := treader.ReadString() // addr
if err != nil {
return
}
r.middleNodeAdr = net.ParseIP(adr)
r.middleNodeTcpPort, err = treader.ReadUint16() // tcp port
if err != nil {
return
}
r.middleNodeUdpPort, err = treader.ReadUint16() // udp port
if err != nil {
return
}
} else {
r.middleNodeId, err = treader.ReadUint16() // node id
if err != nil {
return
}
}
}
r.FunctionId, err = treader.ReadUint8() // function id
if err != nil {
return
}
// Load message
r.Message = buffer.NewWithSize(int64(msgSize), false) // message
n, err := io.Copyn(r.Message, reader, int64(msgSize))
r.Message.Seek(0, 0)
if err != nil {
log.Error("COMM: Got an error reading message from message: %s", err)
return err
}
if n != int64(msgSize) {
log.Error("COMM: Couldn't read the whole message. Read %d out of %d", n, msgSize)
return os.NewError("Message truncated")
}
// release the connection if its a message
if r.Type == T_MSG {
r.Release()
} else {
r.Data = reader
}
return nil
}
func (cs *ClusterService) HandleUnmanagedError(errorMessage *comm.Message, error os.Error) {
log.Error("CS: Got an unmanaged error: %s", errorMessage)
}