func (s *ShingleFilter) shingleCurrentRingState() analysis.TokenStream { rv := make(analysis.TokenStream, 0) for shingleN := s.min; shingleN <= s.max; shingleN++ { // if there are enough items in the ring // to produce a shingle of this size if s.itemsInRing >= shingleN { thisShingleRing := s.ring.Move(-(shingleN - 1)) shingledBytes := make([]byte, 0) pos := 0 start := -1 end := 0 for i := 0; i < shingleN; i++ { if i != 0 { shingledBytes = append(shingledBytes, []byte(s.tokenSeparator)...) } curr := thisShingleRing.Value.(*analysis.Token) if pos == 0 && curr.Position != 0 { pos = curr.Position } if start == -1 && curr.Start != -1 { start = curr.Start } if curr.End != -1 { end = curr.End } shingledBytes = append(shingledBytes, curr.Term...) thisShingleRing = thisShingleRing.Next() } token := analysis.Token{ Type: analysis.Shingle, Term: shingledBytes, } if pos != 0 { token.Position = pos } if start != -1 { token.Start = start } if end != -1 { token.End = end } rv = append(rv, &token) } } return rv }