Beispiel #1
0
func findPart(part []byte, currentLevel []*PatternNode) ([]*PatternNode, int) {
	nextLevel := make([]*PatternNode, 0, 64)
	hash := xxhash.Checksum32(part)
	for _, node := range currentLevel {
		for _, child := range node.Children {
			match := false

			if child.Hash == asteriskHash || child.Hash == hash {
				match = true
			} else if len(child.InnerParts) > 0 {
				for _, innerPart := range child.InnerParts {
					innerMatch, _ := path.Match(innerPart, string(part))
					if innerMatch {
						match = true
						break
					}
				}
			}

			if match {
				nextLevel = append(nextLevel, child)
			}
		}
	}
	return nextLevel, len(nextLevel)
}
Beispiel #2
0
func checksumString(s string) string {
	b := make([]byte, 4)
	cs := xxhash.Checksum32([]byte(s))

	binary.LittleEndian.PutUint32(b, cs)
	return base64.RawURLEncoding.EncodeToString(b)
}
Beispiel #3
0
// secondaryIndex will return the secondary index of any given index.
func (gt *GokooTable) secondaryIndex(i1 int, f []byte) int {

	// get the xxhash of the fingerprint
	i2 := int(xxhash.Checksum32(f))

	// XOR the primary index with the hash of the fingerprint
	i2 = i1 ^ i2

	// return the alternative index
	return i2 % gt.nBuckets
}
Beispiel #4
0
// TODO: possibly optimize by having caller pass in a buffer reference?
//       also need to give user the address so they can keep track of it
// returns single messages sequentially
func (rd *Reader) Read() ([]byte, error) {
	var dlen, xx32 uint32
	buf := make([]byte, 4)

	// read 4 bytes length
	for cnt := 0; cnt < 4; {
		rx, err := rd.rd.Read(buf[cnt:])
		if err == io.EOF {
			offset, _ := rd.fp.Seek(0, os.SEEK_CUR)
			//TODO test this reader changing slab file code, seems brittle
			// issues with reader outpacing writer?? file locks? ugh?
			rd.base += uint64(offset)
			err := rd.Seek(rd.topic, rd.base)
			if err != nil {
				return nil, err
			}
			continue
		} else if err != nil {
			return nil, err
		}
		cnt += rx
	}
	dlen = binary.LittleEndian.Uint32(buf)

	// read 4 bytes crc
	for cnt := 0; cnt < 4; {
		rx, err := rd.rd.Read(buf[cnt:])
		if err != nil {
			return nil, err
		}
		cnt += rx
	}
	xx32 = binary.LittleEndian.Uint32(buf)

	// read data payload
	buf = make([]byte, dlen)
	for cnt := 0; uint32(cnt) < dlen; {
		rx, err := rd.rd.Read(buf[cnt:])
		if err != nil {
			return nil, err
		}
		cnt += rx
	}

	// check crc
	if xx32 != xxhash.Checksum32(buf) {
		return buf, ErrBadChecksum
	}

	return buf, nil
}
Beispiel #5
0
func (t *PatternStorage) buildTree(patterns []string) error {
	newTree := &PatternNode{}

	for _, pattern := range patterns {
		currentNode := newTree
		parts := strings.Split(pattern, ".")
		for _, part := range parts {
			found := false
			for _, child := range currentNode.Children {
				if part == child.Part {
					currentNode = child
					found = true
					break
				}
			}
			if !found {
				newNode := &PatternNode{Part: part}

				if currentNode.Prefix == "" {
					newNode.Prefix = part
				} else {
					newNode.Prefix = fmt.Sprintf("%s.%s", currentNode.Prefix, part)
				}

				if part == "*" || !strings.ContainsAny(part, "{*") {
					newNode.Hash = xxhash.Checksum32([]byte(part))
				} else {
					if strings.Contains(part, "{") && strings.Contains(part, "}") {
						prefix, bigSuffix := split2(part, "{")
						inner, suffix := split2(bigSuffix, "}")
						innerParts := strings.Split(inner, ",")

						newNode.InnerParts = make([]string, 0, len(innerParts))
						for _, innerPart := range innerParts {
							newNode.InnerParts = append(newNode.InnerParts, fmt.Sprintf("%s%s%s", prefix, innerPart, suffix))
						}
					} else {
						newNode.InnerParts = []string{part}
					}

				}
				currentNode.Children = append(currentNode.Children, newNode)
				currentNode = newNode
			}
		}
	}

	t.PatternTree = newTree

	return nil
}
Beispiel #6
0
func (wt *Writer) Write(d []byte) error {
	var dlen, xx32 uint32
	buf := make([]byte, 4)

	dlen = uint32(len(d))
	xx32 = xxhash.Checksum32(d)

	wt.Lock()

	// FIXME -- make a function like WriteAll() to write until all written
	// e.g.
	// for cnt = 0; cnt < len(key); {
	//     tx, _ := fp.Write(key[cnt:])
	//     cnt += tx
	// }

	// write header
	binary.LittleEndian.PutUint32(buf, dlen)
	tx, err := wt.wt.Write(buf)
	if err != nil {
		return err
	}

	binary.LittleEndian.PutUint32(buf, xx32)
	tx, err = wt.wt.Write(buf)
	if err != nil {
		return err
	}

	// write payload
	tx, err = wt.wt.Write(d)
	if err != nil {
		return err
	}

	// update address
	wt.address = wt.address + uint64(8+tx)

	// roll over slab file if it is big enough
	if (wt.address - wt.base) > wt.slabSizeHint {
		wt.Flush()
		wt.fp.Close()
		wt.create()
	}

	wt.Unlock()

	return nil
}
Beispiel #7
0
func (z *reader) nextBlock() {
	// Read block size
	var blockSize uint32
	z.err = z.read(&blockSize)
	if z.err != nil {
		return
	}

	uncompressedFlag := (blockSize >> 31) != 0
	blockSize &= 0x7FFFFFFF

	if blockSize == lz4EOM {
		z.err = io.EOF
		return
	}

	if blockSize > z.maxBlockSize {
		z.err = errors.New("lz4: invalid block size")
		return
	}

	// Read block data
	block := make([]byte, blockSize)
	_, z.err = io.ReadFull(z.r, block)
	if z.err != nil {
		return
	}

	if z.blockChecksumFlag {
		// Check block checksum
		var checksum uint32
		z.err = z.read(&checksum)
		if z.err != nil {
			return
		}
		if checksum != xxhash.Checksum32(block) {
			z.err = errors.New("lz4: invalid block checksum detected")
			return
		}
	}

	// Decompress
	data := make([]byte, z.maxBlockSize)
	if !uncompressedFlag {
		n, err := lz4Decompress(block, data, z.maxBlockSize)
		if err != nil {
			z.err = err
			return
		}
		data = data[0:n]
	} else {
		copy(data, block)
		data = data[0:blockSize]
	}

	if z.contentChecksumFlag {
		z.h.Write(data)
	}

	// Add block to our history
	z.buf = append(z.buf, data...)
}
Beispiel #8
0
package filter

import (
	"fmt"
	"log"
	"path"
	"strings"
	"time"

	"github.com/vova616/xxhash"
)

var asteriskHash = xxhash.Checksum32([]byte("*"))

// PatternStorage contains pattern tree
type PatternStorage struct {
	PatternTree *PatternNode
}

type PatternNode struct {
	Children   []*PatternNode
	Part       string
	Hash       uint32
	Prefix     string
	InnerParts []string
}

// NewPatternStorage creates new PatternStorage struct
func NewPatternStorage() *PatternStorage {
	return &PatternStorage{}
}