Golang Tokenize Examples

Example #1

File: parser.go Project: ukode/mojo

// Make a new Parser in preparation for calling Parse().
func MakeParser(fileName, fileContents string,
	descriptorToPopulate *mojom.MojomDescriptor) Parser {
	if descriptorToPopulate == nil {
		panic("descriptorToPopulate must not be nil")
	}
	inputStream := lexer.Tokenize(fileContents)
	parser := Parser{inputStream: inputStream,
		mojomDescriptor: descriptorToPopulate}
	parser.mojomDescriptor = descriptorToPopulate
	parser.mojomFile = parser.mojomDescriptor.AddMojomFile(fileName)
	return parser
}

Example #2

File: parser.go Project: willbittner/mojo

// Make a new Parser in preparation for calling Parse().
func MakeParser(canonicalFileName, specifiedName, fileContents string,
	descriptorToPopulate *mojom.MojomDescriptor, importedFrom *mojom.MojomFile) Parser {
	if descriptorToPopulate == nil {
		panic("descriptorToPopulate must not be nil")
	}
	inputStream := lexer.Tokenize(fileContents)
	parser := Parser{inputStream: inputStream,
		mojomDescriptor: descriptorToPopulate}
	parser.mojomDescriptor = descriptorToPopulate
	parser.mojomFile = parser.mojomDescriptor.AddMojomFile(canonicalFileName, specifiedName,
		importedFrom, fileContents)
	return parser
}