// ParseFrom read from bufio.Reader and parse needle. // used in scan block by bufio.Reader. func (n *Needle) ParseFrom(rd *bufio.Reader) (err error) { var ( dataOffset int32 footerOffset int32 endOffset int32 data []byte ) // header if data, err = rd.Peek(_headerSize); err != nil { return } if err = n.parseHeader(data); err != nil { return } dataOffset = _headerSize footerOffset = dataOffset + n.Size endOffset = footerOffset + n.FooterSize // no discard, get all needle buffer if data, err = rd.Peek(int(n.TotalSize)); err != nil { return } if err = n.parseData(data[dataOffset:footerOffset]); err != nil { return } // footer if err = n.parseFooter(data[footerOffset:endOffset]); err != nil { return } n.buffer = data _, err = rd.Discard(int(n.TotalSize)) return }
func (u *udpgwPacket) read(r *bufio.Reader) error { bs, err := r.Peek(2) if nil != err { return err } u.length = binary.LittleEndian.Uint16(bs) //binary.Read(r, binary.BigEndian, &u.length) r.Discard(2) //log.Printf("###First %d %d %d %p", u.length, binary.BigEndian.Uint16(bs), len(bs), r) _, err = r.Peek(int(u.length)) if nil != err { //log.Printf("### %v", err) return err } bodylen := u.length binary.Read(r, binary.BigEndian, &u.flags) binary.Read(r, binary.BigEndian, &u.conid) bodylen -= 3 if bodylen > 0 { if (u.flags & flagIPv6) != 0 { u.addr.ip = make(net.IP, 16) } else { u.addr.ip = make(net.IP, 4) } r.Read(u.addr.ip) bodylen -= uint16(len(u.addr.ip)) binary.Read(r, binary.BigEndian, &u.addr.port) bodylen -= 2 u.content = make([]byte, int(bodylen)) r.Read(u.content) } return nil }
func (h Handler) Set(cmd common.SetRequest, src *bufio.Reader) error { // TODO: should there be a unique flags value for regular data? // Write command header if err := binprot.WriteSetCmd(h.rw.Writer, cmd.Key, cmd.Flags, cmd.Exptime, uint32(len(cmd.Data))); err != nil { return err } // Write value h.rw.Write(cmd.Data) if err := h.rw.Flush(); err != nil { return err } // Read server's response resHeader, err := readResponseHeader(h.rw.Reader) if err != nil { // Discard request body if _, ioerr := src.Discard(len(cmd.Data)); ioerr != nil { return ioerr } // Discard response body if _, ioerr := h.rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return ioerr } return err } return nil }
func (img *Image) readNextStream(r *bufio.Reader) (*Stream, error) { lengthBuf, err := r.Peek(8) if err != nil { return nil, &ParseError{Oper: "stream length check", Err: err} } left := int(binary.LittleEndian.Uint64(lengthBuf)) if left < streamentrySize { return nil, &ParseError{Oper: "stream entry", Err: errors.New("size too short")} } var sentry streamentry err = binary.Read(r, binary.LittleEndian, &sentry) if err != nil { return nil, &ParseError{Oper: "stream entry", Err: err} } left -= streamentrySize if left < int(sentry.NameLength) { return nil, &ParseError{Oper: "stream entry", Err: errors.New("size too short for name")} } names := make([]uint16, sentry.NameLength/2) err = binary.Read(r, binary.LittleEndian, names) if err != nil { return nil, &ParseError{Oper: "file name", Err: err} } left -= int(sentry.NameLength) name := string(utf16.Decode(names)) var offset resourceDescriptor if sentry.Hash != (SHA1Hash{}) { var ok bool offset, ok = img.wim.fileData[sentry.Hash] if !ok { return nil, &ParseError{Oper: "stream entry", Path: name, Err: fmt.Errorf("could not find file data matching hash %v", sentry.Hash)} } } s := &Stream{ StreamHeader: StreamHeader{ Hash: sentry.Hash, Size: offset.OriginalSize, Name: name, }, wim: img.wim, offset: offset, } _, err = r.Discard(left) if err != nil { return nil, err } return s, nil }
// Recovery recovery needles map from super block. func (b *SuperBlock) Recovery(offset uint32, fn func(*Needle, uint32) error) ( err error) { var ( n = &Needle{} rd *bufio.Reader data []byte ) log.Infof("block: %s recovery from offset: %d", b.File, offset) if offset == 0 { offset = NeedleOffset(superBlockHeaderOffset) } b.Offset = offset if _, err = b.r.Seek(blockOffset(b.Offset), os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.File) return } rd = bufio.NewReaderSize(b.r, NeedleMaxSize) for { if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = n.ParseHeader(data); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } if data, err = rd.Peek(n.DataSize); err != nil { break } if err = n.ParseData(data); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } if log.V(1) { log.Infof("block add offset: %d, size: %d to needles cache", b.Offset, n.TotalSize) log.Info(n.String()) } if err = fn(n, b.Offset); err != nil { break } b.Offset += NeedleOffset(int64(n.TotalSize)) } if err == io.EOF { // reset b.w offset, discard left space which can't parse to a needle if _, err = b.w.Seek(blockOffset(b.Offset), os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.File, err) } else { log.Infof("block: %s:%d*8 recovery [ok]", b.File, b.Offset) return } } log.Infof("block: %s recovery [failed]", b.File) return }
func getLocalIntoBuf(rw *bufio.Reader, metaData metadata, tokenBuf, dataBuf []byte, chunkNum, totalDataLength int) (opcodeNoop bool, err error) { resHeader, err := binprot.ReadResponseHeader(rw) if err != nil { return false, err } // it feels a bit dirty knowing about batch gets here, but it's the most logical place to put // a check for an opcode that signals the end of a batch get or GAT. This code is a bit too big // to copy-paste in multiple places. if resHeader.Opcode == binprot.OpcodeNoop { return true, nil } err = binprot.DecodeError(resHeader) if err != nil { // Discard the message body on error if _, ioerr := rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return false, ioerr } return false, err } serverFlags := make([]byte, 4) binary.Read(rw, binary.BigEndian, &serverFlags) // Read in token if requested if tokenBuf != nil { if _, err := io.ReadFull(rw, tokenBuf); err != nil { return false, err } } // indices for slicing, end exclusive start, end := chunkSliceIndices(int(metaData.ChunkSize), chunkNum, int(metaData.Length)) // read data directly into buf chunkBuf := dataBuf[start:end] // Read in value if _, err := io.ReadFull(rw, chunkBuf); err != nil { return false, err } // consume padding at end of chunk if needed if len(chunkBuf) < totalDataLength { if _, ioerr := rw.Discard(totalDataLength - len(chunkBuf)); ioerr != nil { return false, ioerr } } return false, nil }
// Recovery recovery needles map from super block. func (b *SuperBlock) Recovery(needles map[int64]NeedleCache, indexer *Indexer, offset int64) (err error) { var ( rd *bufio.Reader data []byte size int32 noffset uint32 n = &Needle{} ) log.Printf("start super block recovery, offset: %d\n", offset) if _, err = b.r.Seek(offset, os.SEEK_SET); err != nil { return } rd = bufio.NewReaderSize(b.r, NeedleMaxSize) for { // header if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = ParseNeedleHeader(data, n); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } // data if data, err = rd.Peek(n.DataSize); err != nil { break } if err = ParseNeedleData(data, n); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } log.Print(n.String()) size = int32(NeedleHeaderSize + n.DataSize) noffset += NeedleOffset(int(size)) needles[n.Key] = NewNeedleCache(size, noffset) indexer.Add(n.Key, noffset, size) } if err == io.EOF { err = nil } // reset b.w offset, discard left space which can't parse to a needle if _, err = b.w.Seek(BlockOffset(noffset), os.SEEK_SET); err != nil { return } return }
// Recovery recovery needle cache meta data in memory, index file will stop // at the right parse data offset. func (i *Indexer) Recovery(fn func(*Index) error) (noffset uint32, err error) { var ( rd *bufio.Reader data []byte offset int64 ix = &Index{} ) log.Infof("index: %s recovery", i.File) if offset, err = i.f.Seek(0, os.SEEK_SET); err != nil { log.Errorf("index: %s Seek() error(%v)", i.File, err) return } rd = bufio.NewReaderSize(i.f, NeedleMaxSize) for { if data, err = rd.Peek(indexSize); err != nil { break } ix.parse(data) if ix.Size > NeedleMaxSize || ix.Size < 1 { log.Errorf("index parse size: %d error", ix.Size) err = ErrIndexSize break } if _, err = rd.Discard(indexSize); err != nil { break } if log.V(1) { log.Info(ix.String()) } offset += int64(indexSize) if err = fn(ix); err != nil { break } // save this for recovery supper block noffset = ix.Offset + NeedleOffset(int64(ix.Size)) } if err == io.EOF { // reset b.w offset, discard left space which can't parse to a needle if _, err = i.f.Seek(offset, os.SEEK_SET); err != nil { log.Errorf("index: %s Seek() error(%v)", i.File, err) } else { log.Infof("index: %s recovery [ok]", i.File) return } } log.Errorf("index: %s recovery [failed], error(%v)", i.File, err) return }
func consumeResponse(r *bufio.Reader) error { res, err := readRes(r) if err != nil { return err } apperr := statusToError(res.Status) // read body in regardless of the error in the header r.Discard(int(res.BodyLen)) if apperr != nil && srsErr(apperr) { return apperr } return err }
func handleSet(msg string, reader *bufio.Reader) string { m := setFormat.FindStringSubmatch(msg) if len(m) < 5 { return fmt.Sprintf("CLIENT_ERROR couln't extract values from %v\r\n", msg) } key, _, _, sizeStr := m[1], m[2], m[3], m[4] size, _ := strconv.ParseInt(sizeStr, 10, 16) buf := make([]byte, size) _, err := reader.Read(buf) reader.Discard(2) if err != nil { log.Printf("Error reading buffer: %v\n", err) } lock.Lock() kvs[key] = string(buf) lock.Unlock() return "STORED\r\n" }
func parseBulkString(reader *bufio.Reader) (RESPType, BulkString, error) { i, err := readInt(reader) if err != nil { return BulkStringType, BulkString(nil), err } if i == -1 { return BulkStringType, BulkString(nil), nil } b, err := reader.Peek(int(i)) if err != nil { return BulkStringType, BulkString(nil), err } reader.Discard(int(i) + 2) // +2 for \r\n return BulkStringType, BulkString(b), nil }
func (i *Indexer) Recovery(needles map[int64]NeedleCache) (noffset uint32, err error) { var ( rd *bufio.Reader data []byte offset int64 ix = &Index{} ) if offset, err = i.f.Seek(0, os.SEEK_SET); err != nil { return } rd = bufio.NewReaderSize(i.f, i.bufSize) for { // parse data if data, err = rd.Peek(indexSize); err != nil { break } ix.Key = BigEndian.Int64(data) ix.Offset = BigEndian.Uint32(data[indexOffsetOffset:]) ix.Size = BigEndian.Int32(data[indexSizeOffset:]) // check if ix.Offset%NeedlePaddingSize != 0 { err = ErrIndexOffset break } if ix.Size > NeedleMaxSize { err = ErrIndexSize break } if _, err = rd.Discard(indexSize); err != nil { break } log.Print(ix.String()) offset += int64(indexSize) needles[ix.Key] = NewNeedleCache(ix.Size, ix.Offset) // save this for recovery supper block noffset = ix.Offset + NeedleOffset(int(ix.Size)) } // reset b.w offset, discard left space which can't parse to a needle log.Printf("seek offset: %d\n", offset) if _, err = i.f.Seek(offset, os.SEEK_SET); err != nil { return } return }
func consumeBatchResponse(r *bufio.Reader) error { opcode := uint8(Get) var apperr error for opcode != Noop { res, err := readRes(r) if err != nil { return err } opcode = res.Opcode apperr = statusToError(res.Status) // read body in regardless of the error in the header r.Discard(int(res.BodyLen)) } return apperr }
// Recovery recovery needle cache meta data in memory, index file will stop // at the right parse data offset. func (i *Indexer) Recovery(needles map[int64]NeedleCache) (noffset uint32, err error) { var ( rd *bufio.Reader data []byte offset int64 ix = &Index{} ) log.Infof("index: %s recovery", i.file) if offset, err = i.f.Seek(0, os.SEEK_SET); err != nil { log.Errorf("index: %s Seek() error(%v)", i.file, err) return } rd = bufio.NewReaderSize(i.f, NeedleMaxSize) for { // parse data if data, err = rd.Peek(indexSize); err != nil { break } ix.parse(data) // check if ix.Size > NeedleMaxSize || ix.Size < 1 { log.Errorf("index parse size: %d > %d or %d < 1", ix.Size, NeedleMaxSize, ix.Size) break } if _, err = rd.Discard(indexSize); err != nil { break } log.V(1).Info(ix.String()) offset += int64(indexSize) needles[ix.Key] = NewNeedleCache(ix.Offset, ix.Size) // save this for recovery supper block noffset = ix.Offset + NeedleOffset(int64(ix.Size)) } if err != io.EOF { return } // reset b.w offset, discard left space which can't parse to a needle if _, err = i.f.Seek(offset, os.SEEK_SET); err != nil { log.Errorf("index: %s Seek() error(%v)", i.file, err) } log.Infof("index: %s recovery [ok]", i.file) return }
//当error为ErrUnexpectedRESPEOF时,说明还缺少数据,并不是一个完整的bulk string //还缺少的数据由第二个参数返回,其余n的值则都是0 func parseBulkString(line []byte, br *bufio.Reader) (RESP_BULK_STRING, int, error) { neededDataLen := 0 n, err := parseLen(line[0:]) if n < 0 || err != nil { return "", neededDataLen, err } brSize := br.Buffered() //n+2: bulkString + "\r\n" if n+2 > brSize { neededDataLen = n + 2 - brSize return "", neededDataLen, ErrUnexpectedRESPEOF } p, _ := br.Peek(n) br.Discard(n) if line, err := readLine(br); err != nil { return "", neededDataLen, err } else if len(line) != 0 { return "", neededDataLen, errors.New("bad bulk string format") } return RESP_BULK_STRING(p), neededDataLen, nil }
// Parse Parse needle from bufio. func (n *Needle) ParseFrom(rd *bufio.Reader) (err error) { var data []byte // header if data, err = rd.Peek(HeaderSize); err != nil { return } if err = n.ParseHeader(data); err != nil { return } if _, err = rd.Discard(HeaderSize); err != nil { return } // data if data, err = rd.Peek(int(n.Size)); err != nil { return } if err = n.ParseData(data); err != nil { return } if _, err = rd.Discard(int(n.Size)); err != nil { return } // footer if data, err = rd.Peek(int(n.FooterSize)); err != nil { return } if err = n.ParseFooter(data); err != nil { return } _, err = rd.Discard(int(n.FooterSize)) return }
// Dump parse supper block file and dump print for debug. // ONLY DEBUG!!!! func (b *SuperBlock) Dump() (err error) { var ( rd *bufio.Reader data []byte n = &Needle{} ) if _, err = b.r.Seek(0, os.SEEK_SET); err != nil { return } rd = bufio.NewReaderSize(b.r, NeedleMaxSize) for { // header if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = ParseNeedleHeader(data, n); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } // data if data, err = rd.Peek(n.DataSize); err != nil { break } if err = ParseNeedleData(data, n); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } log.Print(n.String()) } if err == io.EOF { err = nil } return }
func (img *Image) readNextEntry(r *bufio.Reader) (*File, error) { lengthBuf, err := r.Peek(8) if err != nil { return nil, &ParseError{Oper: "directory length check", Err: err} } left := int(binary.LittleEndian.Uint64(lengthBuf)) if left == 0 { return nil, io.EOF } if left < direntrySize { return nil, &ParseError{Oper: "directory entry", Err: errors.New("size too short")} } var dentry direntry err = binary.Read(r, binary.LittleEndian, &dentry) if err != nil { return nil, &ParseError{Oper: "directory entry", Err: err} } left -= direntrySize namesLen := int(dentry.FileNameLength + 2 + dentry.ShortNameLength) if left < namesLen { return nil, &ParseError{Oper: "directory entry", Err: errors.New("size too short for names")} } names := make([]uint16, namesLen/2) err = binary.Read(r, binary.LittleEndian, names) if err != nil { return nil, &ParseError{Oper: "file name", Err: err} } left -= namesLen var name, shortName string if dentry.FileNameLength > 0 { name = string(utf16.Decode(names[:dentry.FileNameLength/2])) } if dentry.ShortNameLength > 0 { shortName = string(utf16.Decode(names[dentry.FileNameLength/2+1:])) } var offset resourceDescriptor zerohash := SHA1Hash{} if dentry.Hash != zerohash { var ok bool offset, ok = img.wim.fileData[dentry.Hash] if !ok { return nil, &ParseError{Oper: "directory entry", Path: name, Err: fmt.Errorf("could not find file data matching hash %#v", dentry)} } } f := &File{ FileHeader: FileHeader{ Attributes: dentry.Attributes, CreationTime: dentry.CreationTime, LastAccessTime: dentry.LastAccessTime, LastWriteTime: dentry.LastWriteTime, Hash: dentry.Hash, Size: offset.OriginalSize, Name: name, ShortName: shortName, }, offset: offset, img: img, subdirOffset: dentry.SubdirOffset, } isDir := false if dentry.Attributes&FILE_ATTRIBUTE_REPARSE_POINT == 0 { f.LinkID = dentry.ReparseHardLink if dentry.Attributes&FILE_ATTRIBUTE_DIRECTORY != 0 { isDir = true } } else { f.ReparseTag = uint32(dentry.ReparseHardLink) f.ReparseReserved = uint32(dentry.ReparseHardLink >> 32) } if isDir && f.subdirOffset == 0 { return nil, &ParseError{Oper: "directory entry", Path: name, Err: errors.New("no subdirectory data for directory")} } else if !isDir && f.subdirOffset != 0 { return nil, &ParseError{Oper: "directory entry", Path: name, Err: errors.New("unexpected subdirectory data for non-directory")} } if dentry.SecurityID != 0xffffffff { f.SecurityDescriptor = img.sds[dentry.SecurityID] } _, err = r.Discard(left) if err != nil { return nil, err } if dentry.StreamCount > 0 { var streams []*Stream for i := uint16(0); i < dentry.StreamCount; i++ { s, err := img.readNextStream(r) if err != nil { return nil, err } // The first unnamed stream should be treated as the file stream. if i == 0 && s.Name == "" { f.Hash = s.Hash f.Size = s.Size f.offset = s.offset } else if s.Name != "" { streams = append(streams, s) } } f.Streams = streams } if dentry.Attributes&FILE_ATTRIBUTE_REPARSE_POINT != 0 && f.Size == 0 { return nil, &ParseError{Oper: "directory entry", Path: name, Err: errors.New("reparse point is missing reparse stream")} } return f, nil }
func getLocalIntoBuf(rw *bufio.Reader, metaData metadata, tokenBuf, dataBuf []byte, chunkNum, totalDataLength int) (opcodeNoop bool, err error) { resHeader, err := binprot.ReadResponseHeader(rw) if err != nil { return false, err } defer binprot.PutResponseHeader(resHeader) // it feels a bit dirty knowing about batch gets here, but it's the most logical place to put // a check for an opcode that signals the end of a batch get or GAT. This code is a bit too big // to copy-paste in multiple places. if resHeader.Opcode == binprot.OpcodeNoop { return true, nil } err = binprot.DecodeError(resHeader) if err != nil { // read in the message "Not found" after a miss n, ioerr := rw.Discard(int(resHeader.TotalBodyLength)) metrics.IncCounterBy(common.MetricBytesReadLocal, uint64(n)) if ioerr != nil { return false, ioerr } return false, err } // we currently do nothing with the flags //buf := make([]byte, 4) //n, err := io.ReadAtLeast(rw, buf, 4) //metrics.IncCounterBy(common.MetricBytesReadLocal, uint64(n)) //if err != nil { // return emptyMeta, err //} //serverFlags := binary.BigEndian.Uint32(buf) // instead of reading and parsing flags, just discard rw.Discard(4) metrics.IncCounterBy(common.MetricBytesReadLocal, 4) // Read in token if requested if tokenBuf != nil { n, err := io.ReadAtLeast(rw, tokenBuf, tokenSize) metrics.IncCounterBy(common.MetricBytesReadLocal, uint64(n)) if err != nil { return false, err } } // indices for slicing, end exclusive start, end := chunkSliceIndices(int(metaData.ChunkSize), chunkNum, int(metaData.Length)) // read data directly into buf chunkBuf := dataBuf[start:end] // Read in value n, err := io.ReadAtLeast(rw, chunkBuf, len(chunkBuf)) metrics.IncCounterBy(common.MetricBytesReadLocal, uint64(n)) if err != nil { return false, err } // consume padding at end of chunk if needed if len(chunkBuf) < totalDataLength { n, ioerr := rw.Discard(totalDataLength - len(chunkBuf)) metrics.IncCounterBy(common.MetricBytesReadLocal, uint64(n)) if ioerr != nil { return false, ioerr } } return false, nil }
// Compress compress the orig block, copy to disk dst block. func (b *SuperBlock) Compress(offset int64, v *Volume) (noffset int64, err error) { var ( data []byte r *os.File rd *bufio.Reader n = &Needle{} ) log.Infof("block: %s compress", b.file) if r, err = os.OpenFile(b.file, os.O_RDONLY, 0664); err != nil { log.Errorf("os.OpenFile(\"%s\", os.O_RDONLY, 0664) error(%v)", b.file, err) return } if offset == 0 { offset = superBlockHeaderOffset } if _, err = r.Seek(offset, os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.file, err) return } rd = bufio.NewReaderSize(r, NeedleMaxSize) for { // header if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = n.ParseHeader(data); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } // data if data, err = rd.Peek(n.DataSize); err != nil { break } if err = n.ParseData(data); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } offset += int64(NeedleHeaderSize + n.DataSize) log.V(1).Info(n.String()) // skip delete needle if n.Flag == NeedleStatusDel { continue } // multi append if err = v.Write(n.Key, n.Cookie, n.Data); err != nil { break } } if err != io.EOF { return } if err = v.Flush(); err != nil { return } if err = r.Close(); err != nil { return } noffset = offset return }
// Recovery recovery needles map from super block. func (b *SuperBlock) Recovery(needles map[int64]NeedleCache, indexer *Indexer, offset int64) (err error) { var ( size int32 data []byte rd *bufio.Reader n = &Needle{} nc NeedleCache noffset uint32 ) log.Infof("block: %s recovery from offset: %d", b.file, offset) if offset == 0 { offset = superBlockHeaderOffset } noffset = NeedleOffset(offset) if _, err = b.r.Seek(offset, os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.file) return } rd = bufio.NewReaderSize(b.r, NeedleMaxSize) for { // header if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = n.ParseHeader(data); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } // data if data, err = rd.Peek(n.DataSize); err != nil { break } if err = n.ParseData(data); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } size = int32(NeedleHeaderSize + n.DataSize) if n.Flag == NeedleStatusOK { if err = indexer.Add(n.Key, noffset, size); err != nil { break } nc = NewNeedleCache(noffset, size) } else { nc = NewNeedleCache(NeedleCacheDelOffset, size) } needles[n.Key] = nc log.V(1).Infof("block add offset: %d, size: %d to needles cache", noffset, size) log.V(1).Info(n.String()) noffset += NeedleOffset(int64(size)) } if err == io.EOF { err = nil } // reset b.w offset, discard left space which can't parse to a needle if _, err = b.w.Seek(BlockOffset(noffset), os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.file, err) } return }
func discard(r *bufio.Reader, i int) (int, error) { return r.Discard(i) }
func discard(r *bufio.Reader, n int) { r.Discard(n) }
func (h Handler) Set(cmd common.SetRequest, src *bufio.Reader) error { // For writing chunks, the specialized chunked reader is appropriate. // for unchunked, a limited reader will be needed since the text protocol // includes a /r/n at the end and there's no EOF to be had with a long-lived // connection. limChunkReader := newChunkLimitedReader(bytes.NewBuffer(cmd.Data), int64(chunkSize), int64(len(cmd.Data))) numChunks := int(math.Ceil(float64(len(cmd.Data)) / float64(chunkSize))) token := <-tokens metaKey := metaKey(cmd.Key) metaData := metadata{ Length: uint32(len(cmd.Data)), OrigFlags: cmd.Flags, NumChunks: uint32(numChunks), ChunkSize: chunkSize, Token: token, } metaDataBuf := new(bytes.Buffer) binary.Write(metaDataBuf, binary.BigEndian, metaData) // Write metadata key // TODO: should there be a unique flags value for chunked data? if err := binprot.WriteSetCmd(h.rw.Writer, metaKey, cmd.Flags, cmd.Exptime, metadataSize); err != nil { return err } // Write value if _, err := io.Copy(h.rw.Writer, metaDataBuf); err != nil { return err } if err := h.rw.Flush(); err != nil { return err } // Read server's response resHeader, err := readResponseHeader(h.rw.Reader) if err != nil { // Discard request body if _, ioerr := src.Discard(len(cmd.Data)); ioerr != nil { return ioerr } // Discard response body if _, ioerr := h.rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return ioerr } return err } // Write all the data chunks // TODO: Clean up if a data chunk write fails // Failure can mean the write failing at the I/O level // or at the memcached level, e.g. response == ERROR chunkNum := 0 for limChunkReader.More() { // Build this chunk's key key := chunkKey(cmd.Key, chunkNum) // Write the key if err := binprot.WriteSetCmd(h.rw.Writer, key, cmd.Flags, cmd.Exptime, fullDataSize); err != nil { return err } // Write token if _, err := h.rw.Write(token[:]); err != nil { return err } // Write value if _, err := io.Copy(h.rw.Writer, limChunkReader); err != nil { return err } // There's some additional overhead here calling Flush() because it causes a write() syscall // The set case is already a slow path and is async from the client perspective for our use // case so this is not a problem. if err := h.rw.Flush(); err != nil { return err } // Read server's response resHeader, err = readResponseHeader(h.rw.Reader) if err != nil { // Reset the ReadWriter to prevent sending garbage to memcached // otherwise we get disconnected h.reset() // Discard request body // This is more complicated code but more straightforward than attempting to get at // the underlying reader and discard directly, since we don't exactly know how many // bytes were sent already for limChunkReader.More() { if _, ioerr := io.Copy(ioutil.Discard, limChunkReader); ioerr != nil { return ioerr } limChunkReader.NextChunk() } // Discard repsonse body if _, ioerr := h.rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return ioerr } return err } // Reset for next iteration limChunkReader.NextChunk() chunkNum++ } return nil }
func HandleSet(cmd common.SetRequest, src *bufio.Reader, rw *bufio.ReadWriter) error { // For writing chunks, the specialized chunked reader is appropriate. // for unchunked, a limited reader will be needed since the text protocol // includes a /r/n at the end and there's no EOF to be had with a long-l`ived // connection. limChunkReader := stream.NewChunkLimitedReader(src, int64(CHUNK_SIZE), int64(cmd.Length)) numChunks := int(math.Ceil(float64(cmd.Length) / float64(CHUNK_SIZE))) token := <-tokens metaKey := metaKey(cmd.Key) metaData := common.Metadata{ Length: cmd.Length, OrigFlags: cmd.Flags, NumChunks: uint32(numChunks), ChunkSize: CHUNK_SIZE, Token: token, } metaDataBuf := new(bytes.Buffer) binary.Write(metaDataBuf, binary.BigEndian, metaData) // Write metadata key // TODO: should there be a unique flags value for chunked data? localCmd := binprot.SetCmd(metaKey, cmd.Flags, cmd.Exptime, common.METADATA_SIZE) if err := setLocal(rw.Writer, localCmd, metaDataBuf); err != nil { return err } // Read server's response resHeader, err := readResponseHeader(rw.Reader) if err != nil { // Discard request body if _, ioerr := src.Discard(int(cmd.Length)); ioerr != nil { return ioerr } // Discard response body if _, ioerr := rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return ioerr } return err } // Write all the data chunks // TODO: Clean up if a data chunk write fails // Failure can mean the write failing at the I/O level // or at the memcached level, e.g. response == ERROR chunkNum := 0 for limChunkReader.More() { // Build this chunk's key key := chunkKey(cmd.Key, chunkNum) // Write the key localCmd = binprot.SetCmd(key, cmd.Flags, cmd.Exptime, FULL_DATA_SIZE) if err = setLocalWithToken(rw.Writer, localCmd, token, limChunkReader); err != nil { return err } // Read server's response resHeader, err = readResponseHeader(rw.Reader) if err != nil { // Discard request body for limChunkReader.More() { if _, ioerr := io.Copy(ioutil.Discard, limChunkReader); ioerr != nil { return ioerr } limChunkReader.NextChunk() } // Discard repsonse body if _, ioerr := rw.Discard(int(resHeader.TotalBodyLength)); ioerr != nil { return ioerr } return err } // Reset for next iteration limChunkReader.NextChunk() chunkNum++ } return nil }
func discardBytes(buf *bufio.Reader, n int) { // the Discard method was introduced in Go 1.5 buf.Discard(n) }
// Compact compact the orig block, copy to disk dst block. func (b *SuperBlock) Compact(offset int64, fn func(*Needle) error) ( noffset int64, err error) { if b.LastErr != nil { err = b.LastErr return } var ( r *os.File rd *bufio.Reader data []byte n = &Needle{} ) log.Infof("block: %s compact", b.File) if r, err = os.OpenFile(b.File, os.O_RDONLY, 0664); err != nil { log.Errorf("os.OpenFile(\"%s\") error(%v)", b.File, err) return } if offset == 0 { offset = superBlockHeaderOffset } if _, err = r.Seek(offset, os.SEEK_SET); err != nil { log.Errorf("block: %s Seek() error(%v)", b.File, err) return } rd = bufio.NewReaderSize(r, NeedleMaxSize) for { if data, err = rd.Peek(NeedleHeaderSize); err != nil { break } if err = n.ParseHeader(data); err != nil { break } if _, err = rd.Discard(NeedleHeaderSize); err != nil { break } if data, err = rd.Peek(n.DataSize); err != nil { break } if err = n.ParseData(data); err != nil { break } if _, err = rd.Discard(n.DataSize); err != nil { break } offset += int64(n.TotalSize) if log.V(1) { log.Info(n.String()) } // skip delete needle if n.Flag == NeedleStatusDel { continue } if err = fn(n); err != nil { break } } if err != io.EOF { return } if err = r.Close(); err != nil { return } noffset = offset return }
func mustDiscard(r *bufio.Reader, n int) { if _, err := r.Discard(n); err != nil { panic(fmt.Sprintf("bufio.Reader.Discard(%d) failed: %s", n, err)) } }
//读取一个完整的回复数据 func readReply(io *bufio.Reader) (reply interface{}, err error) { if io == nil { panic(ErrNotConnected) } b := readToCRLF(io) switch v := string(b[1:]); b[0] { case ok_byte: switch { case v == okReply: reply = okReply case v == pongReply: reply = pongReply default: reply = v } case err_byte: err = Error(v) case num_byte: i, err := strconv.Atoi(v) checkError(err) reply = i case size_byte: var size int size, err = strconv.Atoi(v) checkError(err) if size < 0 { reply = nil } s, err := io.Peek(size) checkError(err) l, err := io.Discard(size) checkError(err) readToCRLF(io) reply = string(s[0:l]) case array_byte: var size int size, err = strconv.Atoi(v) checkError(err) r := make([]interface{}, size) for i := 0; i < size; i++ { r[i], err = readReply(io) if err != nil { return nil, err } } reply = r default: panic(ErrUnexpectedReplyType) } return }