func (cw *streamWriter) run() {
var msgc chan raftpb.Message
var heartbeatc <-chan time.Time
var t streamType
var enc encoder
var flusher http.Flusher
tickc := time.Tick(ConnReadTimeout / 3)
for {
select {
case <-heartbeatc:
start := time.Now()
if err := enc.encode(linkHeartbeatMessage); err != nil {
reportSentFailure(string(t), linkHeartbeatMessage)
cw.status.deactivate(failureType{source: t.String(), action: "heartbeat"}, err.Error())
cw.close()
heartbeatc, msgc = nil, nil
continue
}
flusher.Flush()
reportSentDuration(string(t), linkHeartbeatMessage, time.Since(start))
case m := <-msgc:
start := time.Now()
if err := enc.encode(m); err != nil {
reportSentFailure(string(t), m)
cw.status.deactivate(failureType{source: t.String(), action: "write"}, err.Error())
cw.close()
heartbeatc, msgc = nil, nil
cw.r.ReportUnreachable(m.To)
continue
}
flusher.Flush()
reportSentDuration(string(t), m, time.Since(start))
case conn := <-cw.connc:
cw.close()
t = conn.t
switch conn.t {
case streamTypeMsgAppV2:
enc = newMsgAppV2Encoder(conn.Writer, cw.fs)
case streamTypeMessage:
enc = &messageEncoder{w: conn.Writer}
default:
plog.Panicf("unhandled stream type %s", conn.t)
}
flusher = conn.Flusher
cw.mu.Lock()
cw.status.activate()
cw.closer = conn.Closer
cw.working = true
cw.mu.Unlock()
heartbeatc, msgc = tickc, cw.msgc
case <-cw.stopc:
cw.close()
close(cw.done)
return
}
}
}
func (*messageWriter) writeNewMessage(newMessage *messages.MessageOutput, w *http.ResponseWriter, flusher http.Flusher) {
_, err := fmt.Fprintf(*w, "id: %d\n", newMessage.Id)
if err != nil {
log.Printf("Error while pushing: %v", err)
}
fmt.Fprintf(*w, "event: %s\n", "msg")
fmt.Fprintf(*w, "data: %s\n\n", newMessage.Data)
flusher.Flush()
}
func flushLoop(f http.Flusher, interval time.Duration, done <-chan struct{}) {
for {
select {
case <-time.After(interval):
f.Flush()
case <-done:
return
}
}
}
//write a string and flush it to the http Flushwriter
func writeStringAndFlush(w http.ResponseWriter, flusher http.Flusher, line string) error {
_, err := w.Write([]byte(line))
if err != nil {
return err
}
flusher.Flush()
return nil
}
func (b *Broker) sendEvents(w http.ResponseWriter, f http.Flusher, client chan string) {
for {
ev, open := <-client
if !open {
break
}
fmt.Fprintf(w, "data: Event: %s\n\n", ev)
f.Flush()
}
}
func consumerHandler(w http.ResponseWriter, r *http.Request, es *eventSource) {
if r.Header.Get("Accept") != "text/event-stream" {
http.Error(w, "eventsource: should accept text/event-stream", http.StatusNotAcceptable)
return
}
var (
fl http.Flusher
con *consumer
id int
err error
ok bool
)
id, err = lastEventId(r)
if err != nil {
http.Error(w, "eventsource: bad last-event-id header", http.StatusBadRequest)
return
}
fl, ok = w.(http.Flusher)
if !ok {
http.Error(w, "eventsouce: response does not implement http.Flusher", http.StatusInternalServerError)
return
}
h := w.Header()
h.Set("Content-Type", "text/event-stream")
h.Set("Cache-Control", "no-cache")
h.Set("Connection", "keep-alive")
con = es.Subscribe(id)
defer es.Unsubscribe(con)
for {
select {
case b := <-con.ch:
if _, err := w.Write(b); err == nil {
fl.Flush()
} else {
log.Print(err)
return
}
case <-con.quit:
close(con.ch)
close(con.quit)
return
}
}
}
func (server *Server) sendEvent(w http.ResponseWriter, f http.Flusher, event es.Event) bool {
_, err := fmt.Fprintf(w, "event: %s\n", event.Event)
if nil != err {
return false
}
if "" != event.Data {
_, err = fmt.Fprintf(w, "data: %s\n", event.Data)
if nil != err {
return false
}
}
_, err = fmt.Fprint(w, "\n")
if nil != err {
return false
}
f.Flush()
return true
}
func (cw *streamWriter) run() {
var msgc chan raftpb.Message
var heartbeatc <-chan time.Time
var t streamType
var msgAppTerm uint64
var enc encoder
var flusher http.Flusher
tickc := time.Tick(ConnReadTimeout / 3)
for {
select {
case <-heartbeatc:
start := time.Now()
if err := enc.encode(linkHeartbeatMessage); err != nil {
reportSentFailure(string(t), linkHeartbeatMessage)
cw.status.deactivate(failureType{source: t.String(), action: "heartbeat"}, err.Error())
cw.close()
heartbeatc, msgc = nil, nil
continue
}
flusher.Flush()
reportSentDuration(string(t), linkHeartbeatMessage, time.Since(start))
case m := <-msgc:
if t == streamTypeMsgApp && m.Term != msgAppTerm {
// TODO: reasonable retry logic
if m.Term > msgAppTerm {
cw.close()
heartbeatc, msgc = nil, nil
// TODO: report to raft at peer level
cw.r.ReportUnreachable(m.To)
}
continue
}
start := time.Now()
if err := enc.encode(m); err != nil {
reportSentFailure(string(t), m)
cw.status.deactivate(failureType{source: t.String(), action: "write"}, err.Error())
cw.close()
heartbeatc, msgc = nil, nil
cw.r.ReportUnreachable(m.To)
continue
}
flusher.Flush()
reportSentDuration(string(t), m, time.Since(start))
case conn := <-cw.connc:
cw.close()
t = conn.t
switch conn.t {
case streamTypeMsgApp:
var err error
msgAppTerm, err = strconv.ParseUint(conn.termStr, 10, 64)
if err != nil {
plog.Panicf("could not parse term %s to uint (%v)", conn.termStr, err)
}
enc = &msgAppEncoder{w: conn.Writer, fs: cw.fs}
case streamTypeMsgAppV2:
enc = newMsgAppV2Encoder(conn.Writer, cw.fs)
case streamTypeMessage:
enc = &messageEncoder{w: conn.Writer}
default:
plog.Panicf("unhandled stream type %s", conn.t)
}
flusher = conn.Flusher
cw.mu.Lock()
cw.status.activate()
cw.closer = conn.Closer
cw.working = true
cw.mu.Unlock()
heartbeatc, msgc = tickc, cw.msgc
case <-cw.stopc:
cw.close()
close(cw.done)
return
}
}
}
func (cw *streamWriter) run() {
var (
msgc chan raftpb.Message
heartbeatc <-chan time.Time
t streamType
enc encoder
flusher http.Flusher
batched int
)
tickc := time.Tick(ConnReadTimeout / 3)
unflushed := 0
plog.Infof("started streaming with peer %s (writer)", cw.peerID)
for {
select {
case <-heartbeatc:
err := enc.encode(linkHeartbeatMessage)
unflushed += linkHeartbeatMessage.Size()
if err == nil {
flusher.Flush()
batched = 0
sentBytes.WithLabelValues(cw.peerID.String()).Add(float64(unflushed))
unflushed = 0
continue
}
cw.status.deactivate(failureType{source: t.String(), action: "heartbeat"}, err.Error())
cw.close()
plog.Warningf("lost the TCP streaming connection with peer %s (%s writer)", cw.peerID, t)
heartbeatc, msgc = nil, nil
case m := <-msgc:
err := enc.encode(m)
if err == nil {
unflushed += m.Size()
if len(msgc) == 0 || batched > streamBufSize/2 {
flusher.Flush()
sentBytes.WithLabelValues(cw.peerID.String()).Add(float64(unflushed))
unflushed = 0
batched = 0
} else {
batched++
}
continue
}
cw.status.deactivate(failureType{source: t.String(), action: "write"}, err.Error())
cw.close()
plog.Warningf("lost the TCP streaming connection with peer %s (%s writer)", cw.peerID, t)
heartbeatc, msgc = nil, nil
cw.r.ReportUnreachable(m.To)
case conn := <-cw.connc:
if cw.close() {
plog.Warningf("closed an existing TCP streaming connection with peer %s (%s writer)", cw.peerID, t)
}
t = conn.t
switch conn.t {
case streamTypeMsgAppV2:
enc = newMsgAppV2Encoder(conn.Writer, cw.fs)
case streamTypeMessage:
enc = &messageEncoder{w: conn.Writer}
default:
plog.Panicf("unhandled stream type %s", conn.t)
}
flusher = conn.Flusher
unflushed = 0
cw.mu.Lock()
cw.status.activate()
cw.closer = conn.Closer
cw.working = true
cw.mu.Unlock()
plog.Infof("established a TCP streaming connection with peer %s (%s writer)", cw.peerID, t)
heartbeatc, msgc = tickc, cw.msgc
case <-cw.stopc:
if cw.close() {
plog.Infof("closed the TCP streaming connection with peer %s (%s writer)", cw.peerID, t)
}
close(cw.done)
plog.Infof("stopped streaming with peer %s (writer)", cw.peerID)
return
}
}
}
func (*messageWriter) writeTimeout(w http.ResponseWriter, flusher http.Flusher) {
fmt.Fprintf(w, "event: %s\n", "timeout")
fmt.Fprintf(w, "data: %s\n\n", "30s")
flusher.Flush()
}
func Sync(mfuser, mfpass, ssuser, sspass string, o io.Writer) bool {
useflush := false
var flusher http.Flusher
if strings.Contains(reflect.TypeOf(o).String(), "http") {
flusher = o.(http.Flusher)
useflush = true
}
// sendspace api key 4R6XNCX3EA
// mediafire api key 5v1vtcl1k8285gtbpc891jtijjxnit5v1cgopla8
// mediafire app id 47574
mfapi := "5v1vtcl1k8285gtbpc891jtijjxnit5v1cgopla8"
mfapp := "47574"
ssapi := "4R6XNCX3EA"
// create channels for goroutines
c := make(chan string)
f := make(chan string)
numfilestosync := 0
// sign in to Mediafire and get a file path list
mfsessionkey, err := MfGetSessionToken(mfapp, mfapi, mfuser, mfpass)
if err != nil {
fmt.Fprintln(o, err)
return false
}
go func() {
MfFindFolders(mfsessionkey, "myfiles", "/", 0)
c <- "Mediafire file list ok"
}()
// sign in to Sendspace and get a file path list
sstoken, err := SsGetToken(ssapi)
if err != nil {
fmt.Fprintln(o, err)
return false
}
sssessionkey, err := SsLogin(sstoken, ssuser, sspass)
if err != nil {
fmt.Fprintln(o, err)
return false
}
go func() {
SsFindFolders(sssessionkey, "0", "/")
c <- "Sendspace file list ok"
}()
// wait for both file path lists before going further
for i := 0; i < 2; i++ {
msg := <-c
fmt.Fprintln(o, msg)
if useflush {
flusher.Flush()
}
}
// copy Sendspace data to the Mediafire structs, if exists. don't care about the rest
for _, s := range SsPaths {
for i, m := range MfPaths {
if s.Path == m.Path {
MfPaths[i].SsD = s.Descriptor
MfPaths[i].SsP = s.Parent
}
}
}
// create folder tree on Sendspace as it is on Mediafire
for i, m := range MfPaths {
// (m.SsD == "") <==> folder does not exist on Sendspace
if m.SsD == "" && isFolderPath(m.Path) {
SsMkFolder(m.Filename, getSsD(MfPaths, m.Parent), sssessionkey, i, m.Path)
fmt.Fprintln(o, m.Path+" -- CREATED")
} else if isFolderPath(m.Path) {
fmt.Fprintln(o, m.Path+" -- exists")
}
if useflush {
flusher.Flush()
}
}
// create files
for _, m := range MfPaths {
if !isFolderPath(m.Path) {
numfilestosync += 1
go func(m MfPath) {
// (m.SsD == "") <==> file does not exist on Sendspace
if m.SsD == "" {
fmt.Fprintln(o, m.Path+" -- SYNCING")
pr, pw := io.Pipe()
go func() {
// get a download link & download
dlink := MfGetDownloadlink(mfsessionkey, m.Descriptor)
dload, err := http.Get(dlink)
if err != nil {
log.Fatal(err)
}
// copy from download to pipe
_, err = io.Copy(pw, dload.Body)
if err != nil {
log.Fatal(err)
}
//fmt.Fprintln(m.Path + " DOWLOADED")
dload.Body.Close()
pw.Close()
c <- m.Filename + " downloaded"
}()
go func() {
// get upload info for the file
uq := SsGetUploadInfo(sssessionkey)
urlToCall, err := url.Parse(uq.Res.Upload.Url)
if err != nil {
log.Fatal(err)
}
urlParams := urlToCall.Query()
urlParams.Add("folder_id", getSsD(MfPaths, m.Parent))
urlParams.Add("userfile", m.Filename)
urlToCall.RawQuery = urlParams.Encode()
// have to do string concat as urlParams.Add encodes % => %25 and extra info fails then
callUrl := urlToCall.String() + "&extra_info=" + uq.Res.Upload.Extra
// create a writer for the upload
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, err := bodyWriter.CreateFormFile("userfile", m.Filename)
if err != nil {
log.Fatal(err)
}
// iocopy from the pipe
_, err = io.Copy(fileWriter, pr)
if err != nil {
log.Fatal(err)
}
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
pr.Close()
resp, err := http.Post(callUrl, contentType, bodyBuf)
defer resp.Body.Close()
if err != nil {
log.Fatal(err)
}
resp_body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
if strings.Contains(string(resp_body), "fail") {
fmt.Fprintln(o, string(resp_body))
return
} else {
//fmt.Fprintln(m.Path + " UPLOADED")
}
c <- m.Filename + " uploaded"
}()
// wait for both download & upload routines to complete
for i := 0; i < 2; i++ {
_ = <-c
}
// signal sync channel
f <- m.Path + " -- DONE"
} else {
f <- m.Path + " -- exists"
}
}(m)
}
}
// wait for all file syncs/exists to complete and print
for j := 0; j < numfilestosync; j++ {
msg := <-f
fmt.Fprintln(o, msg)
if useflush {
flusher.Flush()
}
}
fmt.Fprintln(o, "Sync complete!")
return true
}
func (cw *streamWriter) run() {
var msgc chan raftpb.Message
var heartbeatc <-chan time.Time
var t streamType
var msgAppTerm uint64
var enc encoder
var flusher http.Flusher
tickc := time.Tick(ConnReadTimeout / 3)
for {
select {
case <-heartbeatc:
start := time.Now()
if err := enc.encode(linkHeartbeatMessage); err != nil {
reportSentFailure(string(t), linkHeartbeatMessage)
log.Printf("rafthttp: failed to heartbeat on stream %s due to %v. waiting for a new stream to be established.", t, err)
cw.resetCloser()
heartbeatc, msgc = nil, nil
continue
}
flusher.Flush()
reportSentDuration(string(t), linkHeartbeatMessage, time.Since(start))
case m := <-msgc:
if t == streamTypeMsgApp && m.Term != msgAppTerm {
// TODO: reasonable retry logic
if m.Term > msgAppTerm {
cw.resetCloser()
heartbeatc, msgc = nil, nil
// TODO: report to raft at peer level
cw.r.ReportUnreachable(m.To)
}
continue
}
start := time.Now()
if err := enc.encode(m); err != nil {
reportSentFailure(string(t), m)
log.Printf("rafthttp: failed to send message on stream %s due to %v. waiting for a new stream to be established.", t, err)
cw.resetCloser()
heartbeatc, msgc = nil, nil
cw.r.ReportUnreachable(m.To)
continue
}
flusher.Flush()
reportSentDuration(string(t), m, time.Since(start))
case conn := <-cw.connc:
cw.resetCloser()
t = conn.t
switch conn.t {
case streamTypeMsgApp:
var err error
msgAppTerm, err = strconv.ParseUint(conn.termStr, 10, 64)
if err != nil {
log.Panicf("rafthttp: unexpected parse term %s error: %v", conn.termStr, err)
}
enc = &msgAppEncoder{w: conn.Writer, fs: cw.fs}
case streamTypeMessage:
enc = &messageEncoder{w: conn.Writer}
default:
log.Panicf("rafthttp: unhandled stream type %s", conn.t)
}
flusher = conn.Flusher
cw.mu.Lock()
cw.closer = conn.Closer
cw.working = true
cw.mu.Unlock()
heartbeatc, msgc = tickc, cw.msgc
case <-cw.stopc:
cw.resetCloser()
close(cw.done)
return
}
}
}