Beispiel #1
0
// The LowerCase option defines a Profile's case mapping rule. Options can be
// provided to determine the type of case folding used.
func LowerCase(opts ...cases.Option) Option {
	return func(o *options) {
		if len(opts) == 0 {
			o.cases = cases.Lower(language.Und, cases.HandleFinalSigma(false))
			return
		}

		opts = append([]cases.Option{cases.HandleFinalSigma(false)}, opts...)
		o.cases = cases.Lower(language.Und, opts...)
	}
}
// NewTransformer creates a new transform.Transformer that performs the PRECIS
// preparation and enforcement steps on the given UTF-8 encoded bytes.
func (p Profile) NewTransformer() *Transformer {
	var ts []transform.Transformer

	if p.options.allowwidechars {
		ts = append(ts, width.Fold)
	}

	ts = append(ts, checker{p: p})

	if p.options.width != nil {
		ts = append(ts, width.Fold)
	}

	for _, f := range p.options.additional {
		ts = append(ts, f())
	}

	if p.options.cases {
		ts = append(ts, transform.Chain(
			cases.Upper(language.Und), cases.Lower(language.Und),
		))
	}

	ts = append(ts, p.options.norm)

	// TODO: Apply directionality rule (blocking on the Bidi package)
	// TODO: Add the disallow empty rule with a dummy transformer?

	return &Transformer{transform.Chain(ts...)}
}
Beispiel #3
0
func Example() {
	src := []string{
		"hello world!",
		"i with dot",
		"'n ijsberg",
		"here comes O'Brian",
	}
	for _, c := range []cases.Caser{
		cases.Lower(language.Und),
		cases.Upper(language.Turkish),
		cases.Title(language.Dutch),
		cases.Title(language.Und, cases.NoLower),
	} {
		fmt.Println()
		for _, s := range src {
			fmt.Println(c.String(s))
		}
	}

	// Output:
	// hello world!
	// i with dot
	// 'n ijsberg
	// here comes o'brian
	//
	// HELLO WORLD!
	// İ WİTH DOT
	// 'N İJSBERG
	// HERE COMES O'BRİAN
	//
	// Hello World!
	// I With Dot
	// 'n IJsberg
	// Here Comes O'brian
	//
	// Hello World!
	// I With Dot
	// 'N Ijsberg
	// Here Comes O'Brian
}
Beispiel #4
0
	}
	x := b.next & 1
	if b.buf[x] == nil {
		b.buf[x] = make([]byte, 0, 8+len(b.src)+len(b.src)>>2)
	}
	span := append(b.buf[x][:0], b.src[:n]...)
	b.src, _, err = transform.Append(t, span, b.src[n:])
	b.buf[x] = b.src
	b.next++
	return err
}

// Pre-allocate transformers when possible. In some cases this avoids allocation.
var (
	foldWidthT transform.SpanningTransformer = width.Fold
	lowerCaseT transform.SpanningTransformer = cases.Lower(language.Und, cases.HandleFinalSigma(false))
)

// TODO: make this a method on profile.

func (b *buffers) enforce(p *Profile, src []byte, comparing bool) (str []byte, err error) {
	b.src = src

	ascii := true
	for _, c := range src {
		if c >= utf8.RuneSelf {
			ascii = false
			break
		}
	}
	// ASCII fast path.
Beispiel #5
0
package runes

import (
	"strings"
	"testing"
	"unicode"

	"golang.org/x/text/cases"
	"golang.org/x/text/language"
	"golang.org/x/text/transform"
)

var (
	toUpper = cases.Upper(language.Und)
	toLower = cases.Lower(language.Und)
)

func TestPredicate(t *testing.T) {
	testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
		return If(Predicate(func(r rune) bool {
			return unicode.Is(rt, r)
		}), t, f)
	})
}

func TestIn(t *testing.T) {
	testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
		return If(In(rt), t, f)
	})
}
Beispiel #6
0
func (b *buffers) enforce(p *Profile, src []byte, comparing bool) (str []byte, err error) {
	b.src = src

	// These transforms are applied in the order defined in
	// https://tools.ietf.org/html/rfc7564#section-7

	// TODO: allow different width transforms options.
	if p.options.foldWidth || (p.options.ignorecase && comparing) {
		// TODO: use Span, once available.
		if err = b.apply(width.Fold); err != nil {
			return nil, err
		}
	}
	for _, f := range p.options.additional {
		if err = b.apply(f()); err != nil {
			return nil, err
		}
	}
	if p.options.cases != nil {
		if err = b.apply(p.options.cases); err != nil {
			return nil, err
		}
	}
	if comparing && p.options.ignorecase {
		if err = b.apply(cases.Lower(language.Und, cases.HandleFinalSigma(false))); err != nil {
			return nil, err
		}
	}
	if n := p.norm.QuickSpan(b.src); n < len(b.src) {
		x := b.next & 1
		n = copy(b.buf[x], b.src[:n])
		b.src, _, err = transform.Append(p.norm, b.buf[x][:n], b.src[n:])
		b.buf[x] = b.src
		b.next++
		if err != nil {
			return nil, err
		}
	}
	if p.options.bidiRule {
		if err := b.apply(bidirule.New()); err != nil {
			return nil, err
		}
	}
	c := checker{p: p}
	if _, err := c.span(b.src, true); err != nil {
		return nil, err
	}
	if p.disallow != nil {
		for i := 0; i < len(b.src); {
			r, size := utf8.DecodeRune(b.src[i:])
			if p.disallow.Contains(r) {
				return nil, errDisallowedRune
			}
			i += size
		}
	}

	// TODO: Add the disallow empty rule with a dummy transformer?

	if p.options.disallowEmpty && len(b.src) == 0 {
		return nil, errEmptyString
	}
	return b.src, nil
}