Пример #1
0
func ScanDirtyData(indexFileContent []byte) (dirtys DirtyDatas) {
	m := NewCompactMap()
	for i := 0; i+16 <= len(indexFileContent); i += 16 {
		bytes := indexFileContent[i : i+16]
		key := util.BytesToUint64(bytes[:8])
		offset := util.BytesToUint32(bytes[8:12])
		size := util.BytesToUint32(bytes[12:16])
		k := Key(key)
		if offset != 0 && size != 0 {
			m.Set(k, offset, size)
		} else {
			if nv, ok := m.Get(k); ok {
				//mark old needle data as dirty data
				if int64(nv.Size)-NeedleHeaderSize > 0 {
					dirtys = append(dirtys, DirtyData{
						Offset: int64(nv.Offset)*8 + NeedleHeaderSize,
						Size:   nv.Size,
					})
				}
			}
			m.Delete(k)
		}
	}
	dirtys.Sort()
	return dirtys
}
Пример #2
0
func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
	bytes := make([]byte, 8)
	util.Uint64toBytes(bytes, key)
	data, err := m.db.Get(bytes, nil)
	if err != nil || len(data) != 8 {
		return nil, false
	}
	offset := util.BytesToUint32(data[0:4])
	size := util.BytesToUint32(data[4:8])
	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
}
Пример #3
0
func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, offset, size uint32)) error {
	values := make(url.Values)
	values.Add("volume", vid)
	line := make([]byte, 16)
	err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) {
		key := util.BytesToUint64(bytes[:8])
		offset := util.BytesToUint32(bytes[8:12])
		size := util.BytesToUint32(bytes[12:16])
		eachEntryFn(key, offset, size)
	})
	if err != nil {
		return err
	}
	return nil
}
Пример #4
0
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
	bytes, block, err := ReadNeedleBlob(r, offset, size)
	if err != nil {
		return err
	}
	n.rawBlock = block
	n.ParseNeedleHeader(bytes)
	if n.Size != size {
		return fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)
	}
	switch version {
	case Version1:
		n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
	case Version2:
		n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
	}
	if size == 0 {
		return nil
	}
	checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
	newChecksum := NewCRC(n.Data)
	if checksum != newChecksum.Value() {
		return errors.New("CRC error! Data On Disk Corrupted")
	}
	n.Checksum = newChecksum
	return nil
}
Пример #5
0
func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
	bytes := make([]byte, 8)
	var data []byte
	util.Uint64toBytes(bytes, key)
	err := m.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(boltdbBucket)
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", boltdbBucket)
		}

		data = bucket.Get(bytes)
		return nil
	})

	if err != nil || len(data) != 8 {
		return nil, false
	}
	offset := util.BytesToUint32(data[0:4])
	size := util.BytesToUint32(data[4:8])
	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
}
Пример #6
0
func LoadNewNeedleMap(file *os.File) CompactMap {
	m := NewCompactMap()
	bytes := make([]byte, 16*1024)
	count, e := file.Read(bytes)
	if count > 0 {
		fstat, _ := file.Stat()
		glog.V(0).Infoln("Loading index file", fstat.Name(), "size", fstat.Size())
	}
	for count > 0 && e == nil {
		for i := 0; i < count; i += 16 {
			key := util.BytesToUint64(bytes[i : i+8])
			offset := util.BytesToUint32(bytes[i+8 : i+12])
			size := util.BytesToUint32(bytes[i+12 : i+16])
			if offset > 0 {
				m.Set(Key(key), offset, size)
			} else {
				//delete(m, key)
			}
		}

		count, e = file.Read(bytes)
	}
	return m
}
Пример #7
0
//n should be a needle already read the header
//the input stream will read until next file entry
func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error) {
	if bodyLength <= 0 {
		return nil
	}
	switch version {
	case Version1:
		bytes := make([]byte, bodyLength)
		if _, err = r.ReadAt(bytes, offset); err != nil {
			return
		}
		n.Data = bytes[:n.Size]
		n.Checksum = NewCRC(n.Data)
		checksum := util.BytesToUint32(bytes[n.Size : n.Size+NeedleChecksumSize])

		if n.Checksum.Value() != checksum {
			glog.V(0).Infof("CRC error! Data On Disk Corrupted, needle id = %x", n.Id)
		}
	case Version2:
		bytes := make([]byte, bodyLength)
		if _, err = r.ReadAt(bytes, offset); err != nil {
			return
		}
		n.readNeedleDataVersion2(bytes[0:n.Size])
		if n.DataSize == 0 {
			return
		}
		n.Checksum = NewCRC(n.Data)
		checksum := util.BytesToUint32(bytes[n.Size : n.Size+NeedleChecksumSize])
		if n.Checksum.Value() != checksum {
			glog.V(0).Infof("CRC error! Data On Disk Corrupted, needle id = %x", n.Id)
		}
	default:
		err = fmt.Errorf("Unsupported Version! (%d)", version)
	}
	return
}
Пример #8
0
func (n *Needle) readNeedleDataVersion2(bytes []byte) {
	index, lenBytes := 0, len(bytes)
	if index < lenBytes {
		n.DataSize = util.BytesToUint32(bytes[index : index+4])
		index = index + 4
		if int(n.DataSize)+index > lenBytes {
			// this if clause is due to bug #87 and #93, fixed in v0.69
			// remove this clause later
			return
		}
		n.Data = bytes[index : index+int(n.DataSize)]
		index = index + int(n.DataSize)
		n.Flags = bytes[index]
		index = index + 1
	}
	if index < lenBytes && n.HasName() {
		n.NameSize = uint8(bytes[index])
		index = index + 1
		n.Name = bytes[index : index+int(n.NameSize)]
		index = index + int(n.NameSize)
	}
	if index < lenBytes && n.HasMime() {
		n.MimeSize = uint8(bytes[index])
		index = index + 1
		n.Mime = bytes[index : index+int(n.MimeSize)]
		index = index + int(n.MimeSize)
	}
	if index < lenBytes && n.HasLastModifiedDate() {
		n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
		index = index + LastModifiedBytesLength
	}
	if index < lenBytes && n.HasTtl() {
		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
		index = index + TtlBytesLength
	}
}
Пример #9
0
func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) {
	// start to read index file
	var readerOffset int64
	bytes := make([]byte, 16)
	count, _ := idxFile.ReadAt(bytes, readerOffset)
	readerOffset += int64(count)

	// start to read dat file
	offset := int64(storage.SuperBlockSize)
	version := storage.Version2
	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
	if err != nil {
		fmt.Printf("cannot read needle header: %v", err)
		return
	}
	fmt.Printf("Needle %+v, rest %d\n", n, rest)
	for n != nil && count > 0 {
		// parse index file entry
		key := util.BytesToUint64(bytes[0:8])
		offsetFromIndex := util.BytesToUint32(bytes[8:12])
		sizeFromIndex := util.BytesToUint32(bytes[12:16])
		count, _ = idxFile.ReadAt(bytes, readerOffset)
		readerOffset += int64(count)

		if offsetFromIndex != 0 && offset != int64(offsetFromIndex)*8 {
			//t := offset
			offset = int64(offsetFromIndex) * 8
			//fmt.Printf("Offset change %d => %d\n", t, offset)
		}

		fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)

		padding := storage.NeedlePaddingSize - ((sizeFromIndex + storage.NeedleHeaderSize + storage.NeedleChecksumSize) % storage.NeedlePaddingSize)
		rest = sizeFromIndex + storage.NeedleChecksumSize + padding

		func() {
			defer func() {
				if r := recover(); r != nil {
					fmt.Println("Recovered in f", r)
				}
			}()
			if err = n.ReadNeedleBody(datFile, version, offset+int64(storage.NeedleHeaderSize), rest); err != nil {
				fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
			}
		}()

		if n.Size <= n.DataSize {
			continue
		}
		visitNeedle(n, offset)

		offset += int64(storage.NeedleHeaderSize) + int64(rest)
		//fmt.Printf("==> new entry offset %d\n", offset)
		if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil {
			if err == io.EOF {
				return
			}

			fmt.Printf("cannot read needle header: %v\n", err)
			return
		}
		//fmt.Printf("new entry needle size:%d rest:%d\n", n.Size, rest)
	}

}
Пример #10
0
func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
	key = util.BytesToUint64(bytes[:8])
	offset = util.BytesToUint32(bytes[8:12])
	size = util.BytesToUint32(bytes[12:16])
	return
}
Пример #11
0
func (n *Needle) ParseNeedleHeader(bytes []byte) {
	n.Cookie = util.BytesToUint32(bytes[0:4])
	n.Id = util.BytesToUint64(bytes[4:12])
	n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
}