// find and record the next hop route for c.
// Additionally, all hops along this route are also the shortest path, so
// record those as well to save on effort.
func (n *Node) updateRoute(c string) {
if len(n.effectiveNetwork) == 0 {
return
}
if log.WillLog(log.DEBUG) {
log.Debug("updating route for %v", c)
}
routes := make(map[string]string) // a key node has a value of the previous hop, the key exists if it's been visited
routes[n.name] = n.name // the route to ourself is pretty easy to calculate
// dijkstra's algorithm is well suited in go - we can use a buffered
// channel of nodes to order our search. We start by putting ourselves
// in the queue (channel) and follow all nodes connected to it. If we
// haven't visited that node before, it goes in the queue.
q := make(chan string, len(n.effectiveNetwork))
q <- n.name
for len(q) != 0 {
v := <-q
if log.WillLog(log.DEBUG) {
log.Debug("visiting %v", v)
}
for _, a := range n.effectiveNetwork[v] {
if _, ok := routes[a]; !ok {
q <- a
if log.WillLog(log.DEBUG) {
log.Debug("previous hop for %v is %v", a, v)
}
routes[a] = v // this is the route to node a from v
}
}
if v == c {
break
}
}
for k, v := range routes {
curr := v
prev := k
r := k
if curr == n.name {
r += "<-" + routes[curr]
}
for curr != n.name {
prev = curr
curr = routes[curr]
r += "<-" + routes[curr]
}
if log.WillLog(log.DEBUG) {
log.Debug("full route for %v is %v", k, r)
}
n.routes[k] = prev
}
}
// Send a message according to the parameters set in the message.
// Users will generally use the Set and Broadcast functions instead of Send.
// The returned error is always nil if the message type is broadcast.
// If an error is encountered, Send returns immediately.
func (n *Node) Send(m *Message) ([]string, error) {
if log.WillLog(log.DEBUG) {
log.Debug("Send: %v", m)
}
// force updating the network if needed on Send()
n.checkUpdateNetwork()
routeSlices, err := n.getRoutes(m)
if err != nil {
return nil, err
}
if log.WillLog(log.DEBUG) {
log.Debug("routeSlices: %v", routeSlices)
}
errChan := make(chan error)
for k, v := range routeSlices {
go func(client string, recipients []string) {
mOne := &Message{
Recipients: recipients,
Source: m.Source,
CurrentRoute: m.CurrentRoute,
Command: m.Command,
Body: m.Body,
}
err := n.clientSend(client, mOne)
if err != nil {
errChan <- err
} else {
errChan <- nil
}
}(k, v)
}
// wait on all of the client sends to complete
var ret string
for i := 0; i < len(routeSlices); i++ {
r := <-errChan
if r != nil {
ret += r.Error() + "\n"
}
}
// Rebuild the recipients from the routeSlices so that the caller can know
// which recipients were actually valid.
recipients := []string{}
for _, r := range routeSlices {
recipients = append(recipients, r...)
}
if ret == "" {
return recipients, nil
}
return recipients, errors.New(ret)
}
// generateEffectiveNetwork returns the subset of the current known topology that contains
// pairs of known connections. That is, if node A says it's connected to node B
// but node B does not say it's connected to node A, then that connection will
// not be listed in the effective mesh list.
// generateEffectiveNetwork expects to be called with meshLock obtained.
func (n *Node) generateEffectiveNetwork() {
log.Debugln("generateEffectiveNetwork")
for {
emesh := make(mesh)
for k, v := range n.network {
effectiveNetworkLoop:
for _, i := range v { // for each connection i to node k, see if i also reports being connected to k
for _, j := range emesh[i] { // do we already have this connection noted? if so, move on. This should happen zero or one times for each node.
if j == k {
continue effectiveNetworkLoop
}
}
for _, j := range n.network[i] { // go through all of node i's connections looking for k
if j == k {
if log.WillLog(log.DEBUG) {
log.Debug("found pair %v <-> %v", k, i)
}
// note the connection in the adjacency list for both i and k
emesh[k] = append(emesh[k], i)
emesh[i] = append(emesh[i], k)
break
}
}
}
}
n.effectiveNetwork = emesh
// now generate routes to each of the nodes from us based on the effectiveNetwork
n.routes = make(map[string]string)
// attempt to learn routes to each node from this node,
// assuming that all nodes in the effective network are
// routable. It's possible that the effective network
// represents partitioned meshes, so if we cannot find a route
// to a node, remove it from the known network and start this
// entire process over.
stable := true
for h, _ := range n.effectiveNetwork {
if _, ok := n.routes[h]; !ok {
n.updateRoute(h)
if _, ok := n.routes[h]; !ok {
log.Debug("removing unroutable node %v", h)
delete(n.network, h)
stable = false
}
}
}
if stable {
break
}
}
if log.WillLog(log.DEBUG) {
log.Debug("new effectiveNetwork: %v", n.effectiveNetwork)
}
}
// Transfer a single filepart to a temporary transfer directory.
func (iom *IOMeshage) Xfer(filename string, part int64, from string) error {
TID := genTID()
c := make(chan *IOMMessage)
err := iom.registerTID(TID, c)
defer iom.unregisterTID(TID)
if err != nil {
// a collision in int64, we should tell someone about this
log.Fatalln(err)
}
m := &IOMMessage{
From: iom.node.Name(),
Type: TYPE_XFER,
Filename: filename,
TID: TID,
Part: part,
}
_, err = iom.node.Set([]string{from}, m)
if err != nil {
return err
}
// wait for a response, or a timeout
select {
case resp := <-c:
if log.WillLog(log.DEBUG) {
log.Debugln("got part: ", resp.Part)
}
if resp.ACK {
if log.WillLog(log.DEBUG) {
log.Debugln("got part from: ", resp.From)
}
// write the part out to disk
iom.transferLock.RLock()
defer iom.transferLock.RUnlock()
if t, ok := iom.transfers[filename]; ok {
outfile := fmt.Sprintf("%v/%v.part_%v", t.Dir, filepath.Base(filename), part)
err := ioutil.WriteFile(outfile, resp.Data, 0664)
if err != nil {
return err
}
} else {
return fmt.Errorf("no transfer temporary directory to write to!")
}
} else {
return fmt.Errorf("received NACK from xfer node")
}
case <-time.After(timeout):
return fmt.Errorf("timeout")
}
return nil
}
func (n *Node) handleMSA(m *Message) {
log.Debug("handleMSA: %v", m)
n.meshLock.Lock()
defer n.meshLock.Unlock()
if len(n.network[m.Source]) == len(m.Body.([]string)) {
diff := false
for i, v := range n.network[m.Source] {
if m.Body.([]string)[i] != v {
diff = true
break
}
}
if !diff {
log.Debugln("MSA discarded, client data hasn't changed")
return
}
}
n.network[m.Source] = m.Body.([]string)
if log.WillLog(log.DEBUG) {
log.Debug("new network is: %v", n.network)
}
n.updateNetwork = true
}
func (n *Node) clientSend(host string, m *Message) error {
if log.WillLog(log.DEBUG) {
log.Debug("clientSend %s: %v", host, m)
}
if c, ok := n.clients[host]; ok {
c.lock.Lock()
defer c.lock.Unlock()
err := c.enc.Encode(m)
if err != nil {
c.conn.Close()
return err
}
// wait for a response
for {
select {
case ID := <-c.ack:
if ID == m.ID {
return nil
}
case <-time.After(n.timeout):
c.conn.Close()
return errors.New("timeout")
}
}
}
return fmt.Errorf("no such client %s", host)
}
// Handle incoming "get file info" messages by looking up if we have the file
// and responding with the number of parts or a NACK. Also process directories
// and globs, populating the Glob field of the IOMMessage if needed.
func (iom *IOMeshage) handleInfo(m *IOMMessage) {
// do we have this file, rooted at iom.base?
resp := IOMMessage{
From: iom.node.Name(),
Type: TYPE_RESPONSE,
Filename: m.Filename,
TID: m.TID,
}
glob, parts, err := iom.fileInfo(filepath.Join(iom.base, m.Filename))
if err != nil {
resp.ACK = false
} else if len(glob) == 1 && glob[0] == m.Filename {
resp.ACK = true
resp.Part = parts
fi, err := os.Stat(filepath.Join(iom.base, m.Filename))
if err != nil {
resp.ACK = false
} else {
resp.Perm = fi.Mode() & os.ModePerm
}
if log.WillLog(log.DEBUG) {
log.Debugln("handleInfo found file with parts: ", resp.Part)
}
} else {
// populate Glob
resp.ACK = true
resp.Glob = glob
}
_, err = iom.node.Set([]string{m.From}, resp)
if err != nil {
log.Errorln("handleInfo: sending message: ", err)
}
}
// Get file info and return the number of parts in the file. If the filename is
// a directory or glob, return the list of files the directory/glob contains.
func (iom *IOMeshage) fileInfo(filename string) ([]string, int64, error) {
glob, err := filepath.Glob(filename)
if err != nil {
return nil, 0, err
}
if len(glob) > 1 {
// globs are recursive, figure out any directories
var globsRet []string
for _, v := range glob {
rGlob, _, err := iom.fileInfo(v)
if err != nil {
return nil, 0, err
}
globsRet = append(globsRet, rGlob...)
}
return globsRet, 0, nil
}
f, err := os.Open(filename)
if err != nil {
return nil, 0, err
}
defer f.Close()
// is this a directory
fi, err := f.Stat()
if err != nil {
if log.WillLog(log.DEBUG) {
log.Debugln("fileInfo error stat: ", err)
}
return nil, 0, err
}
if fi.IsDir() {
// walk the directory and populate glob
glob = []string{}
err := filepath.Walk(filename, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
rel, err := filepath.Rel(iom.base, path)
if err != nil {
return err
}
glob = append(glob, rel)
return nil
})
if err != nil {
return nil, 0, err
}
return glob, 0, nil
}
// we do have the file, calculate the number of parts
parts := (fi.Size() + PART_SIZE - 1) / PART_SIZE // integer divide with ceiling instead of floor
rel, err := filepath.Rel(iom.base, filename)
return []string{rel}, parts, nil
}
func (n *Node) flood(m *Message) {
if log.WillLog(log.DEBUG) {
log.Debug("flood: %v", m)
}
n.clientLock.Lock()
defer n.clientLock.Unlock()
floodLoop:
for k, _ := range n.clients {
for _, j := range m.CurrentRoute {
if k == j {
continue floodLoop
}
}
go func(j string, m *Message) {
err := n.clientSend(j, m)
if err != nil {
// is j still a client?
if n.hasClient(j) {
log.Error("flood to client %v: %v", j, err)
}
}
}(k, m)
}
}
// MSA issues a Meshage State Annoucement, which contains a list of all the nodes connected to the broadcaster
func (n *Node) MSA() {
log.Debugln("MSA")
// rate limit MSA spam to once per MSA timeout / 2
n.msaLock.Lock()
defer n.msaLock.Unlock()
if time.Now().Sub(n.lastMSA) < (n.msaTimeout / 2) {
return
}
n.lastMSA = time.Now()
n.clientLock.Lock()
var clients []string
for k, _ := range n.clients {
clients = append(clients, k)
}
n.clientLock.Unlock()
sort.Strings(clients)
n.meshLock.Lock()
diff := false
if len(n.network[n.name]) != len(clients) {
diff = true
} else {
for i, v := range n.network[n.name] {
if clients[i] != v {
diff = true
break
}
}
}
if diff {
log.Debugln("client list changed, recalculating topology")
n.network[n.name] = clients
n.updateNetwork = true
}
n.meshLock.Unlock()
if log.WillLog(log.DEBUG) {
log.Debug("client list: %v", clients)
}
if len(clients) == 0 {
log.Debugln("not issuing MSA, no connected clients")
return
}
m := &Message{
Source: n.name,
CurrentRoute: []string{n.name},
ID: n.sequence(),
Command: MSA,
Body: clients,
}
n.flood(m)
}
// clientHandler is called as a goroutine after a successful handshake. It
// begins by issuing an MSA. When the receiver exits, another MSA is issued
// without the client.
func (n *Node) clientHandler(host string) {
log.Debug("clientHandler: %v", host)
c, err := n.getClient(host)
if err != nil {
log.Error("client %v vanished -- %v", host, err)
return
}
n.MSA()
for {
var m Message
c.conn.SetReadDeadline(time.Now().Add(deadlineMultiplier * n.msaTimeout))
err := c.dec.Decode(&m)
if err != nil {
if err != io.EOF && !strings.Contains(err.Error(), "connection reset by peer") {
log.Error("client %v decode: %v", host, err)
}
break
}
if log.WillLog(log.DEBUG) {
log.Debug("decoded message: %v: %v", c.name, &m)
}
if m.Command == ACK {
c.ack <- m.ID
} else {
// send an ack
a := Message{
Command: ACK,
ID: m.ID,
}
c.conn.SetWriteDeadline(time.Now().Add(deadlineMultiplier * n.msaTimeout))
err := c.enc.Encode(a)
if err != nil {
if err != io.EOF {
log.Error("client %v encode ACK: %v", host, err)
}
break
}
n.messagePump <- &m
}
}
log.Info("client %v disconnected", host)
// client has disconnected
c.conn.Close()
n.clientLock.Lock()
delete(n.clients, c.name)
n.clientLock.Unlock()
go n.checkDegree()
n.MSA()
}
// Handle incoming responses (ACK, file transfer, etc.). It's possible for an
// incoming response to be invalid, such as when a message times out and the
// receiver is no longer expecting the message to arrive. If so, drop the
// message. Responses are sent along registered channels, which are closed when
// the receiver gives up. If we try to send on a closed channel, recover and
// move on.
func (iom *IOMeshage) handleResponse(m *IOMMessage) {
if c, ok := iom.TIDs[m.TID]; ok {
defer func() {
recover()
if log.WillLog(log.DEBUG) {
log.Debugln("send on closed channel recovered")
}
}()
c <- m
} else {
log.Errorln("dropping message for invalid TID: ", m.TID)
}
}
// messageHandler accepts messages from all connected clients and forwards them to the
// appropriate handlers, and to the receiver channel should the message be intended for this
// node.
func (n *Node) messageHandler() {
log.Debugln("messageHandler")
for {
m := <-n.messagePump
if log.WillLog(log.DEBUG) {
log.Debug("messageHandler: %v", m)
}
m.CurrentRoute = append(m.CurrentRoute, n.name)
switch m.Command {
case MSA:
n.sequenceLock.Lock()
if m.ID == 1 && n.sequences[m.Source] > LOLLIPOP_LENGTH {
n.sequences[m.Source] = 0
}
if m.ID > n.sequences[m.Source] {
n.sequences[m.Source] = m.ID
go n.handleMSA(m)
go n.flood(m)
} else {
log.Debug("dropping broadcast: %v:%v", m.Source, m.ID)
}
n.sequenceLock.Unlock()
case MESSAGE:
var newRecipients []string
runLocal := false
for _, i := range m.Recipients {
if i == n.name {
runLocal = true
} else {
newRecipients = append(newRecipients, i)
}
}
m.Recipients = newRecipients
go n.Send(m)
if runLocal {
go n.handleMessage(m)
} else {
if n.Snoop != nil {
go n.Snoop(m)
}
}
default:
log.Errorln("invalid message command: ", m.Command)
}
}
}
// Handle incoming responses (ACK, file transfer, etc.). It's possible for an
// incoming response to be invalid, such as when a message times out and the
// receiver is no longer expecting the message to arrive. If so, drop the
// message. Responses are sent along registered channels, which are closed when
// the receiver gives up. If we try to send on a closed channel, recover and
// move on.
func (iom *IOMeshage) handleResponse(m *IOMMessage) {
iom.tidLock.Lock()
c, ok := iom.TIDs[m.TID]
iom.tidLock.Unlock()
if !ok {
log.Errorln("dropping message for invalid TID: ", m.TID)
return
}
defer func() {
recover()
if log.WillLog(log.DEBUG) {
log.Debugln("send on closed channel recovered")
}
}()
c <- m
}
// Message pump for incoming iomeshage messages.
func (iom *IOMeshage) handleMessages() {
for {
message := (<-iom.Messages).Body.(IOMMessage)
m := &message
if log.WillLog(log.DEBUG) {
log.Debug("got iomessage from %v, type %v", m.From, m.Type)
}
switch m.Type {
case TYPE_INFO:
go iom.handleInfo(m)
case TYPE_WHOHAS:
go iom.handleWhohas(m)
case TYPE_XFER:
go iom.handleXfer(m)
case TYPE_RESPONSE:
go iom.handleResponse(m)
default:
log.Errorln("iomeshage: received invalid message type: ", m.Type)
}
}
}
// search the mesh for the file/glob/directory, returning a slice of string
// matches. The search includes local matches.
func (iom *IOMeshage) Info(file string) []string {
var ret []string
// search locally
files, _, _ := iom.fileInfo(filepath.Join(iom.base, file))
ret = append(ret, files...)
// search the mesh
TID := genTID()
c := make(chan *IOMMessage)
err := iom.registerTID(TID, c)
defer iom.unregisterTID(TID)
if err != nil {
// a collision in int64, we should tell someone about this
log.Fatalln(err)
}
m := &IOMMessage{
From: iom.node.Name(),
Type: TYPE_INFO,
Filename: file,
TID: TID,
}
recipients, err := iom.node.Broadcast(m)
if err != nil {
log.Errorln(err)
return nil
}
if log.WillLog(log.DEBUG) {
log.Debug("sent info request to %v nodes", len(recipients))
}
// wait for n responses, or a timeout
for i := 0; i < len(recipients); i++ {
select {
case resp := <-c:
if log.WillLog(log.DEBUG) {
log.Debugln("got response: ", resp)
}
if resp.ACK {
if log.WillLog(log.DEBUG) {
log.Debugln("got info from: ", resp.From)
}
if len(resp.Glob) == 0 {
// exact match unless the exact match is the original glob
if !strings.Contains(resp.Filename, "*") {
ret = append(ret, resp.Filename)
}
} else {
ret = append(ret, resp.Glob...)
}
}
case <-time.After(timeout):
log.Errorln(fmt.Errorf("timeout"))
return nil
}
}
return ret
}
// Respond to message m with an ACK if a filepart exists, and optionally the
// contents of that filepart.
func (iom *IOMeshage) handlePart(m *IOMMessage, xfer bool) {
// do we have this file, rooted at iom.base?
resp := IOMMessage{
From: iom.node.Name(),
Type: TYPE_RESPONSE,
Filename: m.Filename,
TID: m.TID,
}
iom.drainLock.RLock()
defer iom.drainLock.RUnlock()
_, _, err := iom.fileInfo(filepath.Join(iom.base, m.Filename))
if err != nil {
resp.ACK = false
} else {
resp.ACK = true
resp.Part = m.Part
if xfer {
resp.Data = iom.readPart(m.Filename, m.Part)
}
if log.WillLog(log.DEBUG) {
log.Debugln("handlePart found file with parts: ", resp.Part)
}
}
if resp.ACK {
_, err = iom.node.Set([]string{m.From}, resp)
if err != nil {
log.Errorln("handlePart: sending message: ", err)
}
return
}
// we don't have the file in a complete state at least, do we have that specific part in flight somewhere?
// we consider a part to be transferrable IFF it exists on disk and is marked as being fully received.
iom.transferLock.RLock()
if t, ok := iom.transfers[m.Filename]; ok {
// we are currently transferring parts of the file
if t.Parts[m.Part] {
partname := fmt.Sprintf("%v/%v.part_%v", t.Dir, t.Filename, m.Part)
_, _, err := iom.fileInfo(partname)
if err == nil {
// we have it
resp.ACK = true
resp.Part = m.Part
if xfer {
resp.Data = iom.readPart(partname, 0)
log.Debug("sending partial %v", partname)
}
} else {
resp.ACK = false
}
}
}
iom.transferLock.RUnlock()
_, err = iom.node.Set([]string{m.From}, resp)
if err != nil {
log.Errorln("handlePart: sending message: ", err)
}
}
// Get a file with numParts parts. getParts will randomize the order of the
// parts to maximize the distributed transfer behavior of iomeshage when used
// at scale.
func (iom *IOMeshage) getParts(filename string, numParts int64, perm os.FileMode) {
// corner case - empty file
if numParts == 0 {
log.Debug("file %v has 0 parts, creating empty file", filename)
// create subdirectories
fullPath := filepath.Join(iom.base, filename)
err := os.MkdirAll(filepath.Dir(fullPath), 0755)
if err != nil {
log.Errorln(err)
return
}
f, err := os.Create(fullPath)
if err != nil {
log.Errorln(err)
return
}
f.Close()
log.Debug("changing permissions: %v %v", fullPath, perm)
err = os.Chmod(fullPath, perm)
if err != nil {
log.Errorln(err)
}
return
}
// create a random list of parts to grab
var parts []int64
var i int64
for i = 0; i < numParts; i++ {
parts = append(parts, i)
}
// fisher-yates shuffle
s := rand.NewSource(time.Now().UnixNano())
r := rand.New(s)
for i = numParts - 1; i > 0; i-- {
j := r.Int63n(i + 1)
t := parts[j]
parts[j] = parts[i]
parts[i] = t
}
// create a transfer object
tdir, err := ioutil.TempDir(iom.base, "transfer_")
if err != nil {
log.Errorln(err)
return
}
iom.transferLock.Lock()
iom.transfers[filename] = &Transfer{
Dir: tdir,
Filename: filename,
Parts: make(map[int64]bool),
NumParts: len(parts),
Inflight: -1,
Queued: true,
}
iom.transferLock.Unlock()
defer iom.destroyTempTransfer(filename)
// get in line
iom.queue <- true
defer func() {
<-iom.queue
}()
iom.transferLock.Lock()
iom.transfers[filename].Queued = false
iom.transferLock.Unlock()
for _, p := range parts {
// did I already get this part via another node's request?
iom.transferLock.Lock()
if iom.transfers[filename].Parts[p] {
iom.transferLock.Unlock()
continue
}
iom.transfers[filename].Inflight = p
iom.transferLock.Unlock()
// attempt to get this part up to MAX_ATTEMPTS attempts
for attempt := 0; attempt < MAX_ATTEMPTS; attempt++ {
TID := genTID()
c := make(chan *IOMMessage)
err := iom.registerTID(TID, c)
if err != nil {
// a collision in int64, we should tell someone about this
log.Fatalln(err)
}
if log.WillLog(log.DEBUG) {
log.Debug("transferring filepart %v:%v, attempt %v", filename, p, attempt)
}
if attempt > 0 {
// we're most likely issuing multiple attempts because of heavy traffic, wait a bit for things to calm down
time.Sleep(timeout)
}
m := &IOMMessage{
From: iom.node.Name(),
Type: TYPE_WHOHAS,
Filename: filename,
TID: TID,
Part: p,
}
recipients, err := iom.node.Broadcast(m)
if err != nil {
log.Errorln(err)
iom.unregisterTID(TID)
continue
}
if log.WillLog(log.DEBUG) {
log.Debug("sent info request to %v nodes", len(recipients))
}
var info *IOMMessage
var gotPart bool
var timeoutCount int
// wait for n responses, or a timeout
IOMESHAGE_WHOHAS_LOOP:
for i := 0; i < len(recipients); i++ {
select {
case resp := <-c:
if log.WillLog(log.DEBUG) {
log.Debugln("got response: ", resp)
}
if resp.ACK {
if log.WillLog(log.DEBUG) {
log.Debugln("got partInfo from: ", resp.From)
}
info = resp
gotPart = true
break IOMESHAGE_WHOHAS_LOOP
}
case <-time.After(timeout):
log.Errorln("timeout")
timeoutCount++
if timeoutCount == MAX_ATTEMPTS {
log.Debugln("too many timeouts")
break IOMESHAGE_WHOHAS_LOOP
}
continue
}
}
if !gotPart {
log.Errorln("part not found")
iom.unregisterTID(TID)
continue
}
if log.WillLog(log.DEBUG) {
log.Debug("found part %v on node %v", info.Part, info.From)
}
// transfer this part
err = iom.Xfer(m.Filename, info.Part, info.From)
if err != nil {
log.Errorln(err)
iom.unregisterTID(TID)
continue
}
iom.transferLock.Lock()
iom.transfers[filename].Parts[p] = true
iom.transferLock.Unlock()
iom.unregisterTID(TID)
break
}
iom.transferLock.RLock()
if !iom.transfers[filename].Parts[p] {
log.Error("could not transfer filepart %v:%v after %v attempts", filename, p, MAX_ATTEMPTS)
iom.transferLock.RUnlock()
return
}
iom.transferLock.RUnlock()
}
// copy the parts into the whole file
iom.transferLock.RLock()
t := iom.transfers[filename]
iom.transferLock.RUnlock()
tfile, err := ioutil.TempFile(t.Dir, "cat_")
if err != nil {
log.Errorln(err)
}
for i = 0; i < numParts; i++ {
fname := fmt.Sprintf("%v/%v.part_%v", t.Dir, filepath.Base(filename), i)
fpart, err := os.Open(fname)
if err != nil {
log.Errorln(err)
tfile.Close()
return
}
io.Copy(tfile, fpart)
fpart.Close()
}
name := tfile.Name()
tfile.Close()
// create subdirectories
fullPath := filepath.Join(iom.base, filename)
err = os.MkdirAll(filepath.Dir(fullPath), 0755)
if err != nil {
log.Errorln(err)
return
}
os.Rename(name, fullPath)
log.Debug("changing permissions: %v %v", fullPath, perm)
err = os.Chmod(fullPath, perm)
if err != nil {
log.Errorln(err)
}
}
// Retrieve a file from the shortest path node that has it. Get blocks until
// the file transfer is begins or errors out. If the file specified is a
// directory, the entire directory will be recursively transferred.
// If the file already exists on this node, Get will return immediately with no
// error.
func (iom *IOMeshage) Get(file string) error {
// is this a directory or a glob
fi, err := os.Stat(filepath.Join(iom.base, file))
if err == nil && !fi.IsDir() {
return nil
}
// is this file already in flight?
iom.transferLock.RLock()
if _, ok := iom.transfers[file]; ok {
iom.transferLock.RUnlock()
return fmt.Errorf("file already in flight")
}
iom.transferLock.RUnlock()
// find the file somewhere in the mesh
TID := genTID()
c := make(chan *IOMMessage)
err = iom.registerTID(TID, c)
defer iom.unregisterTID(TID)
if err != nil {
// a collision in int64, we should tell someone about this
log.Fatalln(err)
}
m := &IOMMessage{
From: iom.node.Name(),
Type: TYPE_INFO,
Filename: file,
TID: TID,
}
recipients, err := iom.node.Broadcast(m)
if err != nil {
return err
}
if log.WillLog(log.DEBUG) {
log.Debug("sent info request to %v nodes", len(recipients))
}
var info *IOMMessage
var gotInfo bool
// wait for n responses, or a timeout
for i := 0; i < len(recipients); i++ {
select {
case resp := <-c:
if log.WillLog(log.DEBUG) {
log.Debugln("got response: ", resp)
}
if resp.ACK {
if log.WillLog(log.DEBUG) {
log.Debugln("got info from: ", resp.From)
}
info = resp
gotInfo = true
}
case <-time.After(timeout):
return fmt.Errorf("timeout")
}
}
if !gotInfo {
return fmt.Errorf("file not found")
}
// is this a single file or a directory/blob?
if len(info.Glob) == 0 {
if log.WillLog(log.DEBUG) {
log.Debug("found file on node %v with %v parts", info.From, info.Part)
}
go iom.getParts(info.Filename, info.Part, info.Perm)
} else {
// call Get on each of the constituent files, queued in a random order
// fisher-yates shuffle
s := rand.NewSource(time.Now().UnixNano())
r := rand.New(s)
for i := int64(len(info.Glob)) - 1; i > 0; i-- {
j := r.Int63n(i + 1)
t := info.Glob[j]
info.Glob[j] = info.Glob[i]
info.Glob[i] = t
}
for _, v := range info.Glob {
err := iom.Get(v)
if err != nil {
return err
}
}
}
return nil
}