func findPart(part []byte, currentLevel []*PatternNode) ([]*PatternNode, int) {
nextLevel := make([]*PatternNode, 0, 64)
hash := xxhash.Checksum32(part)
for _, node := range currentLevel {
for _, child := range node.Children {
match := false
if child.Hash == asteriskHash || child.Hash == hash {
match = true
} else if len(child.InnerParts) > 0 {
for _, innerPart := range child.InnerParts {
innerMatch, _ := path.Match(innerPart, string(part))
if innerMatch {
match = true
break
}
}
}
if match {
nextLevel = append(nextLevel, child)
}
}
}
return nextLevel, len(nextLevel)
}
func checksumString(s string) string {
b := make([]byte, 4)
cs := xxhash.Checksum32([]byte(s))
binary.LittleEndian.PutUint32(b, cs)
return base64.RawURLEncoding.EncodeToString(b)
}
// secondaryIndex will return the secondary index of any given index.
func (gt *GokooTable) secondaryIndex(i1 int, f []byte) int {
// get the xxhash of the fingerprint
i2 := int(xxhash.Checksum32(f))
// XOR the primary index with the hash of the fingerprint
i2 = i1 ^ i2
// return the alternative index
return i2 % gt.nBuckets
}
// TODO: possibly optimize by having caller pass in a buffer reference?
// also need to give user the address so they can keep track of it
// returns single messages sequentially
func (rd *Reader) Read() ([]byte, error) {
var dlen, xx32 uint32
buf := make([]byte, 4)
// read 4 bytes length
for cnt := 0; cnt < 4; {
rx, err := rd.rd.Read(buf[cnt:])
if err == io.EOF {
offset, _ := rd.fp.Seek(0, os.SEEK_CUR)
//TODO test this reader changing slab file code, seems brittle
// issues with reader outpacing writer?? file locks? ugh?
rd.base += uint64(offset)
err := rd.Seek(rd.topic, rd.base)
if err != nil {
return nil, err
}
continue
} else if err != nil {
return nil, err
}
cnt += rx
}
dlen = binary.LittleEndian.Uint32(buf)
// read 4 bytes crc
for cnt := 0; cnt < 4; {
rx, err := rd.rd.Read(buf[cnt:])
if err != nil {
return nil, err
}
cnt += rx
}
xx32 = binary.LittleEndian.Uint32(buf)
// read data payload
buf = make([]byte, dlen)
for cnt := 0; uint32(cnt) < dlen; {
rx, err := rd.rd.Read(buf[cnt:])
if err != nil {
return nil, err
}
cnt += rx
}
// check crc
if xx32 != xxhash.Checksum32(buf) {
return buf, ErrBadChecksum
}
return buf, nil
}
func (t *PatternStorage) buildTree(patterns []string) error {
newTree := &PatternNode{}
for _, pattern := range patterns {
currentNode := newTree
parts := strings.Split(pattern, ".")
for _, part := range parts {
found := false
for _, child := range currentNode.Children {
if part == child.Part {
currentNode = child
found = true
break
}
}
if !found {
newNode := &PatternNode{Part: part}
if currentNode.Prefix == "" {
newNode.Prefix = part
} else {
newNode.Prefix = fmt.Sprintf("%s.%s", currentNode.Prefix, part)
}
if part == "*" || !strings.ContainsAny(part, "{*") {
newNode.Hash = xxhash.Checksum32([]byte(part))
} else {
if strings.Contains(part, "{") && strings.Contains(part, "}") {
prefix, bigSuffix := split2(part, "{")
inner, suffix := split2(bigSuffix, "}")
innerParts := strings.Split(inner, ",")
newNode.InnerParts = make([]string, 0, len(innerParts))
for _, innerPart := range innerParts {
newNode.InnerParts = append(newNode.InnerParts, fmt.Sprintf("%s%s%s", prefix, innerPart, suffix))
}
} else {
newNode.InnerParts = []string{part}
}
}
currentNode.Children = append(currentNode.Children, newNode)
currentNode = newNode
}
}
}
t.PatternTree = newTree
return nil
}
func (wt *Writer) Write(d []byte) error {
var dlen, xx32 uint32
buf := make([]byte, 4)
dlen = uint32(len(d))
xx32 = xxhash.Checksum32(d)
wt.Lock()
// FIXME -- make a function like WriteAll() to write until all written
// e.g.
// for cnt = 0; cnt < len(key); {
// tx, _ := fp.Write(key[cnt:])
// cnt += tx
// }
// write header
binary.LittleEndian.PutUint32(buf, dlen)
tx, err := wt.wt.Write(buf)
if err != nil {
return err
}
binary.LittleEndian.PutUint32(buf, xx32)
tx, err = wt.wt.Write(buf)
if err != nil {
return err
}
// write payload
tx, err = wt.wt.Write(d)
if err != nil {
return err
}
// update address
wt.address = wt.address + uint64(8+tx)
// roll over slab file if it is big enough
if (wt.address - wt.base) > wt.slabSizeHint {
wt.Flush()
wt.fp.Close()
wt.create()
}
wt.Unlock()
return nil
}
func (z *reader) nextBlock() {
// Read block size
var blockSize uint32
z.err = z.read(&blockSize)
if z.err != nil {
return
}
uncompressedFlag := (blockSize >> 31) != 0
blockSize &= 0x7FFFFFFF
if blockSize == lz4EOM {
z.err = io.EOF
return
}
if blockSize > z.maxBlockSize {
z.err = errors.New("lz4: invalid block size")
return
}
// Read block data
block := make([]byte, blockSize)
_, z.err = io.ReadFull(z.r, block)
if z.err != nil {
return
}
if z.blockChecksumFlag {
// Check block checksum
var checksum uint32
z.err = z.read(&checksum)
if z.err != nil {
return
}
if checksum != xxhash.Checksum32(block) {
z.err = errors.New("lz4: invalid block checksum detected")
return
}
}
// Decompress
data := make([]byte, z.maxBlockSize)
if !uncompressedFlag {
n, err := lz4Decompress(block, data, z.maxBlockSize)
if err != nil {
z.err = err
return
}
data = data[0:n]
} else {
copy(data, block)
data = data[0:blockSize]
}
if z.contentChecksumFlag {
z.h.Write(data)
}
// Add block to our history
z.buf = append(z.buf, data...)
}
package filter
import (
"fmt"
"log"
"path"
"strings"
"time"
"github.com/vova616/xxhash"
)
var asteriskHash = xxhash.Checksum32([]byte("*"))
// PatternStorage contains pattern tree
type PatternStorage struct {
PatternTree *PatternNode
}
type PatternNode struct {
Children []*PatternNode
Part string
Hash uint32
Prefix string
InnerParts []string
}
// NewPatternStorage creates new PatternStorage struct
func NewPatternStorage() *PatternStorage {
return &PatternStorage{}
}