Example #1
0
func (s *TokenSuite) TestMatchNonPunctuation(c *C) {
	token := punkt.MakeToken("foo")
	c.Assert(token.MatchNonPunctuation(), Equals, true)

	token = punkt.MakeToken("!")
	c.Assert(token.MatchNonPunctuation(), Equals, false)
}
Example #2
0
func (s *TokenSuite) TestEndWithPeriod(c *C) {
	token := punkt.MakeToken("Test")
	c.Check(token.EndsWithPeriod(), Equals, false)

	token = punkt.MakeToken("Test.")
	c.Check(token.EndsWithPeriod(), Equals, true)
}
Example #3
0
func (s *TokenSuite) TestIsAlpha(c *C) {
	token := punkt.MakeToken("foo")
	c.Assert(token.MatchAlpha(), Equals, true)

	token = punkt.MakeToken("!")
	c.Assert(token.MatchAlpha(), Equals, false)
}
Example #4
0
func (s *TokenSuite) TestIsInitial(c *C) {
	token := punkt.MakeToken("C.")
	c.Assert(token.MatchInitial(), Equals, true)

	token = punkt.MakeToken("B.M.")
	c.Assert(token.MatchInitial(), Equals, false)
}
Example #5
0
func (s *TokenSuite) TestTypeWithoutSentencePeriod(c *C) {
	token := punkt.MakeToken("Test")
	c.Check(token.TypeWithoutSentencePeriod(), Equals, "test")

	token = punkt.MakeToken("Test.")
	token.SetSentenceBreak(true)
	c.Check(token.TypeWithoutSentencePeriod(), Equals, "test")
}
Example #6
0
func (s *TokenSuite) TestIsEllipsis(c *C) {
	token := punkt.MakeToken("...")
	c.Assert(token.MatchEllipsis(), Equals, true)

	token = punkt.MakeToken("..")
	c.Assert(token.MatchEllipsis(), Equals, true)

	token = punkt.MakeToken("..foo")
	c.Assert(token.MatchEllipsis(), Equals, false)
}
Example #7
0
func (s *TokenSuite) TestFirstLower(c *C) {
	token := punkt.MakeToken("Test")
	c.Assert(token.FirstLower(), Equals, false)

	token = punkt.MakeToken("índico")
	c.Assert(token.FirstLower(), Equals, true)

	token = punkt.MakeToken("test.")
	c.Assert(token.FirstLower(), Equals, true)
}
Example #8
0
func (s *TokenSuite) TestTypeWithoutPeriod(c *C) {
	token := punkt.MakeToken("Test")
	c.Check(token.TypeWithoutPeriod(), Equals, "test")

	token = punkt.MakeToken("Test.")
	c.Check(token.TypeWithoutPeriod(), Equals, "test")

	token = punkt.MakeToken("123.")
	c.Check(token.TypeWithoutPeriod(), Equals, "##number##")
}
Example #9
0
func (s *TokenSuite) TestTypeAttributes(c *C) {
	token := punkt.MakeToken("Test")
	c.Check(token.Type, Equals, "test")

	token = punkt.MakeToken("Test.")
	c.Check(token.Type, Equals, "test.")

	token = punkt.MakeToken("Índico")
	c.Check(token.Type, Equals, "índico") // does this need to be plain ASCII?
}
Example #10
0
func (s *TokenSuite) TestFlags(c *C) {
	token := punkt.MakeToken("Test")

	token.SetAbbr(true)
	c.Check(token.IsAbbr(), Equals, true)
	token.SetAbbr(false)
	c.Check(token.IsAbbr(), Equals, false)
}
Example #11
0
func (s *TokenSuite) TestString(c *C) {
	token := punkt.MakeToken("foo")
	token.SetAbbr(true)
	token.SetSentenceBreak(true)
	token.SetEllipsis(true)

	tokenStr := fmt.Sprintf("%v", token)
	c.Assert(tokenStr, Equals, "<<foo<A><E><S>>>")
}
Example #12
0
func (s *TokenSuite) TestTokenInit(c *C) {
	token := punkt.MakeToken("Test")

	c.Check(token.IsAbbr(), Equals, false)
}