Example #1
0
/*
	Tokenize the source code described by the given range into raw
	lexical tokens.

	Parameter TU the translation unit whose text is being tokenized.

	Parameter Range the source range in which text should be tokenized. All of the
	tokens produced by tokenization will fall within this source range,

	Parameter Tokens this pointer will be set to point to the array of tokens
	that occur within the given source range. The returned pointer must be
	freed with clang_disposeTokens() before the translation unit is destroyed.

	Parameter NumTokens will be set to the number of tokens in the *Tokens
	array.
*/
func (tu TranslationUnit) Tokenize(r SourceRange) []Token {
	var cp_tokens *C.CXToken
	var tokens []Token
	var numTokens C.uint

	C.clang_tokenize(tu.c, r.c, &cp_tokens, &numTokens)

	gos_tokens := (*reflect.SliceHeader)(unsafe.Pointer(&tokens))
	gos_tokens.Cap = int(numTokens)
	gos_tokens.Len = int(numTokens)
	gos_tokens.Data = uintptr(unsafe.Pointer(cp_tokens))

	return tokens
}
Example #2
0
/**
 * \brief Tokenize the source code described by the given range into raw
 * lexical tokens.
 *
 * \param TU the translation unit whose text is being tokenized.
 *
 * \param Range the source range in which text should be tokenized. All of the
 * tokens produced by tokenization will fall within this source range,
 *
 * \param Tokens this pointer will be set to point to the array of tokens
 * that occur within the given source range. The returned pointer must be
 * freed with clang_disposeTokens() before the translation unit is destroyed.
 *
 * \param NumTokens will be set to the number of tokens in the \c *Tokens
 * array.
 *
 */
func Tokenize(tu TranslationUnit, src SourceRange) Tokens {
	tokens := Tokens{}
	tokens.tu = tu.c
	C.clang_tokenize(tu.c, src.c, &tokens.c, &tokens.n)
	return tokens
}