Beispiel #1
0
func TestParseHex(t *testing.T) {
	ts, _, _, err := process("x-fmt/8", bsStub1.Hex, false)
	if err != nil {
		t.Error("Parse items: Error", err)
	}
	if len(ts) != 6 {
		t.Error("Parse items: Expecting 6 patterns, got", len(ts))
	}
	tok := ts[5]
	if tok.Min() != 10 || tok.Max() != 10 {
		t.Error("Parse items: Expecting 10,10, got", tok.Min(), tok.Max())
	}
	tok = ts[3]
	if tok.Min() != 2 || tok.Max() != 2 {
		t.Error("Parse items: Expecting 2,2, got", tok.Min(), tok.Max())
	}
	if !tok.Pat().Equals(Range{[]byte{0}, []byte{3}}) {
		t.Error("Parse items: Expecting [00:03], got", tok.Pat())
	}
	ts, _, _, _ = process("fmt/390", bsStub5.Hex, false)
	tok = ts[12]
	if tok.Min() != 5 || tok.Max() != -1 {
		t.Error("Parse items: Expecting 5-0, got", tok.Min(), tok.Max())
	}
	if !tok.Pat().Equals(patterns.Sequence(processText("7E41"))) {
		t.Error("Parse items: Expecting 7E41, got", tok.Pat())
	}
	ts, _, _, _ = process("x-fmt/317", bsStub6.Hex, false)
	seqs := ts[2].Pat().Sequences()
	if !seqs[0].Equals(patterns.Sequence(processText("0D0A"))) {
		t.Error("Parse items: Expecting [13 10], got", []byte(seqs[0]))
	}

}
func TestMask(t *testing.T) {
	apple := Mask{
		pat: patterns.Sequence{'a', 'p', 'p', 0, 0, 'l', 'e'},
		val: []byte{255, 255, 255, 0, 0, 255, 255},
	}
	apple2 := Mask{
		pat: patterns.Sequence{'a', 'p', 'p', 0, 0, 'l', 'e'},
		val: []byte{255, 255, 255, 0, 0, 255, 255},
	}
	if !apple.Equals(apple2) {
		t.Error("Mask fail: Equality")
	}
	if r, _ := apple.Test([]byte("apPyzle")); r {
		t.Error("Mask fail: shouldn't match")
	}
	if r, l := apple.Test([]byte("appyzle")); !r || l != 7 {
		t.Error("Mask fail: should match")
	}
	if r, l := apple.TestR([]byte("appyzle")); !r || l != 7 {
		t.Error("Mask fail: should match reverse")
	}
	saver := persist.NewLoadSaver(nil)
	apple.Save(saver)
	loader := persist.NewLoadSaver(saver.Bytes())
	_ = loader.LoadByte()
	p := loadMask(loader)
	if !p.Equals(apple) {
		t.Errorf("expecting %s, got %s", apple, p)
	}
	seqsTest := Mask{
		pat: patterns.Sequence("ap"),
		val: []byte{0xFF, 0xFE},
	}
	if seqs := seqsTest.Sequences(); len(seqs) != 2 || seqs[1][1] != 'q' {
		t.Error(seqs)
	}
	pats, ints := unmask(apple)
	if len(ints) != 2 || ints[0] != 0 || ints[1] != 2 {
		t.Errorf("Unmask fail, got ints %v", ints)
	}
	if len(pats) != 2 || !pats[0].Equals(patterns.Sequence("app")) || !pats[1].Equals(patterns.Sequence("le")) {
		t.Errorf("Unmask fail, got pats %v", pats)
	}
	pats, ints = unmask(Mask{
		pat: patterns.Sequence{'A', 'C', '0', '0', '0', '0'},
		val: []byte{0xFF, 0xFF, 0xF0, 0xF0, 0xF0, 0xF0},
	})
	if len(ints) != 2 || ints[0] != 0 || ints[1] != 0 {
		t.Errorf("Unmask fail, got ints %v", ints)
	}
	if len(pats) != 2 || !pats[0].Equals(patterns.Sequence("AC")) || !pats[1].Equals(Mask{
		pat: patterns.Sequence{'0', '0', '0', '0'},
		val: []byte{0xF0, 0xF0, 0xF0, 0xF0},
	}) {
		t.Errorf("Unmask fail, got pats %v", pats)
	}
}
Beispiel #3
0
func TestProcessGroup(t *testing.T) {
	// try PRONOM form
	l := lexPRONOM("test", "(FF|10[!00:10])")
	<-l.items // discard group entry
	pat, err := processGroup(l)
	if err != nil {
		t.Fatal(err)
	}
	expect := patterns.Choice{
		patterns.Sequence([]byte{255}),
		patterns.List{
			patterns.Sequence([]byte{16}),
			patterns.Not{Range{[]byte{0}, []byte{16}}},
		},
	}
	if !pat.Equals(expect) {
		t.Errorf("expecting %v, got %v", expect, pat)
	}
	// try container form
	l = lexPRONOM("test2", "[10 'cats']")
	<-l.items
	pat, err = processGroup(l)
	if err != nil {
		t.Fatal(err)
	}
	expect = patterns.Choice{
		patterns.Sequence([]byte{16}),
		patterns.Sequence([]byte("cats")),
	}
	if !pat.Equals(expect) {
		t.Errorf("expecting %v, got %v", expect, pat)
	}
	// try simple
	l = lexPRONOM("test3", "[00:10]")
	<-l.items
	pat, err = processGroup(l)
	if err != nil {
		t.Fatal(err)
	}
	rng := Range{[]byte{0}, []byte{16}}
	if !pat.Equals(rng) {
		t.Errorf("expecting %v, got %v", expect, rng)
	}
}
Beispiel #4
0
func magics(m []string) ([]frames.Signature, error) {
	hx, ascii, hxx, asciix, err := characterise(m)
	if err != nil {
		return nil, err
	}
	if len(hx) > 0 {
		sigs := make([]frames.Signature, len(hx))
		for i, v := range hx {
			byts, offs, masks, err := dehex(v, hxx[i])
			if err != nil {
				return nil, err
			}
			sigs[i] = make(frames.Signature, len(byts))
			for ii, vv := range byts {
				rel := frames.BOF
				if ii > 0 {
					rel = frames.PREV
				}
				var pat patterns.Pattern
				if masks[ii] {
					pat = patterns.Mask(vv[0])
				} else {
					pat = patterns.Sequence(vv)
				}
				sigs[i][ii] = frames.NewFrame(rel, pat, offs[ii], offs[ii])
			}
		}
		return sigs, nil
	} else if len(ascii) > 0 {
		sigs := make([]frames.Signature, len(ascii))
		for i, v := range ascii {
			pat := patterns.Sequence(v)
			sigs[i] = frames.Signature{frames.NewFrame(frames.BOF, pat, asciix[i], asciix[i])}
		}
		return sigs, nil
	}
	return nil, nil
}
Beispiel #5
0
func process(puid, seq string, eof bool) (frames.Signature, int, int, error) {
	typ := frames.PREV
	if eof {
		typ = frames.SUCC
	}
	var min, max int
	l := lexPRONOM(puid, seq)
	sig := frames.Signature{}
	for i := l.nextItem(); i.typ != itemEOF; i = l.nextItem() {
		switch i.typ {
		case itemError:
			return nil, 0, 0, errors.New("parse error " + puid + ": " + i.String())
		case itemWildSingle:
			min++
			max++
		case itemWildStart:
			min, _ = decodeNum(i.val)
		case itemCurlyRight: //detect {n} wildcards by checking if the max value has been set
			if max == 0 {
				max = min
			}
		case itemWildEnd:
			if i.val == "*" {
				max = -1
			} else {
				max, _ = decodeNum(i.val)
			}
		case itemWild:
			max = -1
		case itemEnterGroup:
			pat, err := processGroup(l)
			if err != nil {
				return nil, 0, 0, errors.New("parse error " + puid + ": " + err.Error())
			}
			sig = append(sig, frames.NewFrame(typ, pat, min, max))
			min, max = 0, 0
		case itemUnprocessedText:
			sig = append(sig, frames.NewFrame(typ, patterns.Sequence(processText(i.val)), min, max))
			min, max = 0, 0
		}
	}
	return sig, min, max, nil
}
Beispiel #6
0
func toPattern(m mappings.Match) (patterns.Pattern, int, int, error) {
	min, max, err := toOffset(m.Offset)
	if err != nil {
		return nil, min, max, err
	}
	var pat patterns.Pattern
	switch m.Typ {
	case "byte":
		i, err := strconv.ParseInt(m.Value, 0, 16)
		if err != nil {
			return nil, min, max, err
		}
		pat = Int8(i)
	case "big16":
		i, err := strconv.ParseInt(m.Value, 0, 32)
		if err != nil {
			return nil, min, max, err
		}
		pat = Big16(i)
	case "little16":
		i, err := strconv.ParseInt(m.Value, 0, 32)
		if err != nil {
			return nil, min, max, err
		}
		pat = Little16(i)
	case "host16":
		i, err := strconv.ParseInt(m.Value, 0, 32)
		if err != nil {
			return nil, min, max, err
		}
		pat = Host16(i)
	case "big32":
		i, err := strconv.ParseInt(m.Value, 0, 64)
		if err != nil {
			return nil, min, max, err
		}
		pat = Big32(i)
	case "little32":
		i, err := strconv.ParseInt(m.Value, 0, 64)
		if err != nil {
			return nil, min, max, err
		}
		pat = Little32(i)
	case "host32":
		i, err := strconv.ParseInt(m.Value, 0, 64)
		if err != nil {
			return nil, min, max, err
		}
		pat = Host32(i)
	case "string", "": // if no type given, assume string
		pat = patterns.Sequence(unquote(m.Value))
	case "stringignorecase":
		pat = IgnoreCase(unquote(m.Value))
	case "unicodeLE":
		uints := utf16.Encode([]rune(string(unquote(m.Value))))
		buf := make([]byte, len(uints)*2)
		for i, u := range uints {
			binary.LittleEndian.PutUint16(buf[i*2:], u)
		}
		pat = patterns.Sequence(buf)
	case "regex":
		return nil, min, max, nil // ignore regex magic
	default:
		return nil, min, max, errors.New("unknown magic type: " + m.Typ + " val: " + m.Value)
	}
	if len(m.Mask) > 0 {
		pat = Mask{pat, unquote(m.Mask)}
	}
	return pat, min, max, err
}
Beispiel #7
0
// groups are chunks of PRONOM/Droid patterns delimited by parentheses or brackets
// these chunks represent any non-sequence pattern (choices, ranges, bitmasks, not-patterns etc.)
func processGroup(l *lexer) (patterns.Pattern, error) {
	var (
		list                    patterns.List   // bucket to stuff patterns into
		choice                  patterns.Choice // bucket to stuff choices into
		val                     []byte          // bucket to stuff text values
		not, mask, anyMask, rng bool            // retains state from previous tokens
	)
	// when commit a pattern (to the list), go back to zero state
	reset := func() {
		val = []byte{}
		not, mask, anyMask, rng = false, false, false, false
	}
	// make a pattern based on the current state
	makePat := func() patterns.Pattern {
		if len(val) == 0 {
			return nil
		}
		var pat patterns.Pattern
		switch {
		case mask:
			pat = patterns.Mask(val[0])
		case anyMask:
			pat = patterns.AnyMask(val[0])
		default:
			pat = patterns.Sequence(val)
		}
		if not {
			pat = patterns.Not{pat}
		}
		reset()
		return pat
	}
	// add patterns to the choice
	addChoice := func() (patterns.Choice, error) {
		switch len(list) {
		case 0:
			return nil, errors.New(l.name + " has choice marker without preceding pattern")
		case 1:
			choice = append(choice, list[0])
		default:
			choice = append(choice, list)
		}
		list = patterns.List{}
		return choice, nil
	}
	for {
		i := <-l.items
		switch i.typ {
		default:
			return nil, errors.New(l.name + " encountered unexpected token " + i.val)
		case itemEnterGroup: // recurse e.g. for a range nested within a choice
			if pat := makePat(); pat != nil {
				list = append(list, pat)
			}
			pat, err := processGroup(l)
			if err != nil {
				return nil, err
			}
			list = append(list, pat)
		case itemExitGroup:
			if pat := makePat(); pat != nil {
				list = append(list, pat)
			}
			if len(choice) > 0 {
				return addChoice()
			} else {
				switch len(list) {
				case 0:
					return nil, errors.New(l.name + " has group with no legal pattern")
				case 1:
					return list[0], nil
				default:
					return list, nil
				}
			}
		case itemRangeMarker:
			rng = true
		case itemChoiceMarker:
			if pat := makePat(); pat != nil {
				list = append(list, pat)
			}
			_, err := addChoice()
			if err != nil {
				return nil, err
			}
		case itemNotMarker:
			not = true
		case itemMaskMarker:
			mask = true
		case itemAnyMaskMarker:
			anyMask = true
		case itemUnprocessedText:
			v := processText(i.val)
			// if it is a range, we need values before and after the range marker, so add it here
			if rng {
				r := Range{val, v}
				if not {
					list = append(list, patterns.Not{r})
				} else {
					list = append(list, r)
				}
				reset()
			} else {
				val = v
			}
		}
	}
}
Beispiel #8
0
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
		Window{PREV, 46, 1439, patterns.Sequence{255, 254}},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence{16}, patterns.Sequence{17}, patterns.Sequence{18}, patterns.Sequence{19}, patterns.Sequence{20}}},
	},
	418: {
		Fixed{BOF, 0, patterns.Sequence("%!PS-Adobe-2.0")},
		Window{PREV, 16, 512, patterns.Sequence("%%DocumentNeededResources:")},
		Window{PREV, 1, 512, patterns.Sequence("%%+ procset Adobe_Illustrator")},
		Fixed{PREV, 0, patterns.Choice{patterns.Sequence("_AI3"), patterns.Sequence("A_AI3")}},
	},
	363: {
		Window{BOF, 0, 320, patterns.Sequence("@@@@@@@@@@@@@@@@@@@@@@")},
		Fixed{BOF, 3200, patterns.Sequence{0, 0}},
		Fixed{PREV, 15, patterns.Not{patterns.Sequence{0}}},
		Fixed{PREV, 3, patterns.Not{patterns.Sequence{0}}},
		Fixed{PREV, 2, patterns.Choice{
			patterns.Sequence{1, 0},
			patterns.List{
				patterns.Sequence{0},
				patterns.Sequence{8}, // Actual signature has range here
			},