func NewWebSocketHandler() *webSocketHandler { wsh := webSocketHandler{ io.NewParallelMultiWriter(nil), // Yes, we block if too much is going on, which might slow down the entire operation make(chan string), make(chan webClient), websocket.Upgrader{ ReadBufferSize: 1024, WriteBufferSize: 1024, // at least when testing, origin checks failed as host is 127.0.0.1, which // is compared with 'localhost' CheckOrigin: func(r *http.Request) bool { return true }, }, api.StatisticsFilter{ FirstStatisticsAfter: 125 * time.Millisecond, }, } go wsh.eventHandler() return &wsh }
// Drains FileInfos from files channel, reads them using ctrl and generates hashes. // Creates a Result using makeResult() and sends it down the results channel. // If wctrls is set, we will setup parallel writer which writes the bytes used for hashing // to all controllers at the same time, which will be as slow as the slowest device func Gather(files <-chan FileInfo, results chan<- Result, stats *Stats, makeResult func(*FileInfo, *FileInfo, error) Result, rctrl *gio.ReadChannelController, wctrls gio.RootedWriteControllers) { if rctrl == nil { panic("ReadChannelController and WaitGroup must be set") } sha1gen := HashStatAdapter{sha1.New(), stats} md5gen := HashStatAdapter{md5.New(), stats} const nHashes = 2 atomic.AddUint32(&stats.NumHashers, uint32(nHashes)) isWriting := len(wctrls) > 0 numDestinations := wctrls.Trees() // The hgher this number, the less syscall and communication overhead we will have. // As we expect mostly larger files, we go for bigger buffers var buf [512 * 1024]byte var multiWriter io.Writer var channelWriters []gio.ChannelWriter var lazyWriters []gio.LazyFileWriteCloser // We keep an index of failed destinations, skipping them in write mode after first failure var isFailedDestination []bool numFailedDestinations := 0 // Build the multi-writer which will dispatch all writes to a write controller if isWriting { // We have one controller per device, each as a number of streams writers := make([]io.Writer, numDestinations+nHashes) // Writer with full checking enabled - it will never show anything for the hashes, but might // report errrs for the real writers // We place the hashes last, as the writers will be changed in each iteration writers[len(writers)-1] = &sha1gen writers[len(writers)-2] = &md5gen multiWriter = gio.NewParallelMultiWriter(writers) // Keeps all Writers we are going to prepare per source file channelWriters = make([]gio.ChannelWriter, numDestinations) lazyWriters = make([]gio.LazyFileWriteCloser, numDestinations) isFailedDestination = make([]bool, numDestinations) // Init them, per controller, and set them to be used by the multi-writer right away // These never change, only their Destination path does ofs := 0 for _, wctrl := range wctrls { ofse := ofs + len(wctrl.Trees) wctrl.Ctrl.InitChannelWriters(channelWriters[ofs:ofse]) for x := ofs; x < ofse; x++ { channelWriters[x].SetWriter(&lazyWriters[x]) } ofs = ofse } } else { multiWriter = gio.NewUncheckedParallelMultiWriter(&sha1gen, &md5gen) } // umf == unmodifiedFileInfo sendResults := func(f *FileInfo, umf *FileInfo, err error) { if !isWriting { // NOTE: in seal mode, we might want to communicate this ... after all, we don't get the seal done results <- makeResult(f, umf, err) } else { // Each parallel writer has it's own result, we just send it off pmw := multiWriter.(*gio.ParallelMultiWriter) // Make sure we keep the source intact forig := *f for i := 0; i < numDestinations; i++ { w, e := pmw.WriterAtIndex(i) if e != nil && !isFailedDestination[i] { isFailedDestination[i] = true numFailedDestinations += 1 } // If the reader had an error, no write may succeed. We just don't overwrite write errors if e == nil && err != nil { e = err } // Could be a previously unset writer if wc, ok := w.(gio.WriteCloser); ok { wc.Close() // we may change the same instance, as it will be copied into the Result structure later on f.Path = wc.Writer().(*gio.LazyFileWriteCloser).Path() // it doesn't matter here if there actually is an error, aggregator will handle it results <- makeResult(f, &forig, e) } } // for each write controller to write to // If all of our destinations are in fail state, let the gatherer know we can't do anything if numFailedDestinations == numDestinations { atomic.AddUint32(&stats.StopTheEngines, 1) } } // handle write mode } // sendResults // NOTE: This loop must not return ! It must be finished !! for f := range files { // In hash-only mode, there is only one result var err error if isWriting { // in write mode, there are as many results as we have destinations // All that's left to be done is to fill our ChannelWriters with new lazyFileWriters // that write to the given paths // Current writer id, absolute to this write operation awid := 0 for _, wctrl := range wctrls { // We just create one ChannelWriter per destination, and let the writers // deal with the parallelization and blocking fawid := awid // the first index pmw := multiWriter.(*gio.ParallelMultiWriter) for x := 0; x < len(wctrl.Trees); x++ { // reset previous errror by resetting the writer pointer // If the destination is known to have an error, disable its writer if isFailedDestination[awid] { pmw.SetWriterAtIndex(awid, nil) } else { pmw.SetWriterAtIndex(awid, &channelWriters[awid]) } lazyWriters[awid].SetPath(filepath.Join(wctrl.Trees[awid-fawid], f.RelaPath), f.Mode) awid += 1 } } // for each device's write controller if awid != numDestinations { panic("Mismatched writers") } } // handle write mode preparations // let the other end open the file and close it as well reader := rctrl.NewChannelReaderFromPath(f.Path, f.Mode, buf[:]) sha1gen.Reset() md5gen.Reset() var written int64 written, err = reader.WriteTo(multiWriter) if err != nil { // This should actually never fail, the way we are implemented. // If it does, it's the WriteTo() implementation, and as we are decoupled from it, // let's make the check here anyway ... sendResults(&f, &f, err) continue } // Always keep the hash as far as we have it - it's a value to preserve umf := f f.Sha1 = sha1gen.Sum(nil) f.MD5 = md5gen.Sum(nil) if written != f.Size { err = &FileSizeMismatch{f.Path, f.Size, written} sendResults(&f, &umf, err) continue } // all good sendResults(&f, &umf, nil) } // end for each file to process // Keep the count valid ... atomic.AddUint32(&stats.NumHashers, ^uint32(nHashes-1)) }