func ExampleFieldsFunc() {
f := func(c rune) bool {
return !unicode.IsLetter(c) && !unicode.IsNumber(c)
}
fmt.Printf("Fields are: %q", bytes.FieldsFunc([]byte(" foo1;bar2,baz3..."), f))
// Output: Fields are: ["foo1" "bar2" "baz3"]
}
func main() {
languages := []byte("golang haskell ruby python")
individualLanguages := bytes.Fields(languages)
log.Printf("Fields split %q on whitespace into %q", languages, individualLanguages)
vowelsAndSpace := "aeiouy "
split := bytes.FieldsFunc(languages, func(r rune) bool {
return strings.ContainsRune(vowelsAndSpace, r)
})
log.Printf("FieldsFunc split %q on vowels and space into %q", languages, split)
space := []byte{' '}
splitLanguages := bytes.Split(languages, space)
log.Printf("Split split %q on a single space into %q", languages, splitLanguages)
numberOfSubslices := 2 // Not number of splits
singleSplit := bytes.SplitN(languages, space, numberOfSubslices)
log.Printf("SplitN split %q on a single space into %d subslices: %q", languages, numberOfSubslices, singleSplit)
splitAfterLanguages := bytes.SplitAfter(languages, space)
log.Printf("SplitAfter split %q AFTER a single space (keeping the space) into %q", languages, splitAfterLanguages)
splitAfterNLanguages := bytes.SplitAfterN(languages, space, numberOfSubslices)
log.Printf("SplitAfterN split %q AFTER a single space (keeping the space) into %d subslices: %q", languages, numberOfSubslices, splitAfterNLanguages)
languagesBackTogether := bytes.Join(individualLanguages, space)
log.Printf("Languages are back togeher again! %q == %q? %v", languagesBackTogether, languages, bytes.Equal(languagesBackTogether, languages))
}
//func latestsVertion(htmldoc bufio.Reader) (vertionNumber string, err error) {
func latestsVersion(htmldoc []byte) (vertionNumber []byte) {
maxvertionNumber := []byte{}
for _, line := range bytes.Fields(htmldoc) {
queryTokens := bytes.FieldsFunc(line, isNotDigit)
if len(queryTokens) > 0 && bytes.Compare(queryTokens[0], maxvertionNumber) > 0 {
maxvertionNumber = queryTokens[0]
}
}
return maxvertionNumber
}
func sequenceKeys(in []byte) [][]byte {
var ret [][]byte
s := bytes.FieldsFunc(in, extractSeq)
if len(s) > 1 {
ret = collapseChords(combine(expandSeqs(s)))
} else {
ret = s
}
return ret
}
func (t *Tag) ReadTags(r io.ReadSeeker) (e error) {
if t.tags == nil {
t.tags = make(map[string]TAG_TYPE)
}
bf := make([]byte, 1024)
size := 0
for {
n, e := ReadLine(r, bf)
if e != nil && e != io.EOF {
return e
}
if n == 0 {
if e == io.EOF {
break
} else {
continue
}
}
sz := bytes.FieldsFunc(bf[:n], ascIsSpace)
if len(sz) < 2 {
return StringError("file format error, only 1 column")
}
if size == 0 {
size = len(sz)
} else if size != len(sz) {
return StringError(fmt.Sprintf("file format error, columns not same %d:%d:%s:%s", size, len(sz), sz, bf[0:n]))
}
tag := string(sz[len(sz)-1])
val, isexist := t.tags[tag]
if isexist {
continue
}
val = t.id
t.id += 1
t.tags[tag] = val
if e == io.EOF {
break
}
}
return nil
}
func isCPUCol(col []byte) bool {
// Sometimes we have multiple subsystems in one line, as in this example
// from:
// https://source.datanerd.us/newrelic/cross_agent_tests/blob/master/docker_container_id/docker-1.1.2-native-driver-systemd.txt
//
// 3:cpuacct,cpu:/system.slice/docker-67f98c9e6188f9c1818672a15dbe46237b6ee7e77f834d40d41c5fb3c2f84a2f.scope
splitCSV := func(r rune) bool { return r == ',' }
subsysCPU := []byte("cpu")
for _, subsys := range bytes.FieldsFunc(col, splitCSV) {
if bytes.Equal(subsysCPU, subsys) {
return true
}
}
return false
}
func getPktHeaders(buf []byte) (headers map[string]string) {
headers = make(map[string]string)
end := pktHeaderEnd(buf)
if end == -1 {
return
}
buf = buf[:end] //not including end-mark
bs := bytes.FieldsFunc(buf, isLineBreak)
for _, v := range bs {
i := bytes.IndexByte(v, ':')
if i == -1 {
continue
}
headers[string(v[:i])] = string(v[i+1:])
}
return
}
func lines(filename string, spans spans, opts ...opt) spanCase {
buf, err := ioutil.ReadFile(dir + filename)
if err != nil {
panic(err)
}
for _, opt := range opts {
buf = opt(buf)
}
return spanCase{
fname: filename,
buf: buf,
blocks: bytes.FieldsFunc(buf, func(r rune) bool {
return r == '\n' || r == '\r'
}),
spans: spans,
}
}
// parseLine parses "xx: yyyy [zz: tttt] @" as the
// 4 numbers xx, yyyy, zz, tttt.
func parseLine(s []byte) (a, b, c, d int64, err error) {
// s is in correct format if splitting at digits
// yields ": ", " [", ": ", "] @".
seps := bytes.FieldsFunc(s, isDigit)
switch {
case !bytes.Equal(seps[0], []byte(": ")),
!bytes.Equal(seps[1], []byte(" [")),
!bytes.Equal(seps[2], []byte(": ")),
!bytes.Equal(seps[3], []byte("] @")):
err = errBadLineHeader(0)
return
}
for i, x := range s {
if x == ':' {
s = s[i+2:]
break
}
a = a*10 + int64(x-'0')
}
for i, x := range s {
if x == ' ' {
s = s[i+2:]
break
}
b = b*10 + int64(x-'0')
}
for i, x := range s {
if x == ':' {
s = s[i+2:]
break
}
c = c*10 + int64(x-'0')
}
for i, x := range s {
if x == ']' {
s = s[i+2:]
break
}
d = d*10 + int64(x-'0')
}
return
}
func parseModuleIdentificatioNRequest(logStruct *S7Log, s7Packet *S7Packet) error {
if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
return errS7PacketTooShort
}
fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
return int(c) == 0
})
for i := len(fields) - 1; i >= 0; i-- {
switch i {
case 0:
logStruct.ModuleId = string(fields[i][1:]) // exclude index byte
case 5:
logStruct.Hardware = string(fields[i][1:])
case 6:
logStruct.Firmware = string(fields[i][1:])
}
}
return nil
}
func parseComponentIdentificationResponse(logStruct *S7Log, s7Packet *S7Packet) error {
if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
return errS7PacketTooShort
}
fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
return int(c) == 0
})
for i := len(fields) - 1; i >= 0; i-- {
switch i {
case 0:
logStruct.System = string(fields[i][1:]) // exclude index byte
case 1:
logStruct.Module = string(fields[i][1:])
case 2:
logStruct.PlantId = string(fields[i][1:])
case 3:
logStruct.Copyright = string(fields[i][1:])
case 4:
logStruct.SerialNumber = string(fields[i][1:])
case 5:
logStruct.ModuleType = string(fields[i][1:])
case 6:
logStruct.ReservedForOS = string(fields[i][1:])
case 7:
logStruct.MemorySerialNumber = string(fields[i][1:])
case 8:
logStruct.CpuProfile = string(fields[i][1:])
case 9:
logStruct.OEMId = string(fields[i][1:])
case 10:
logStruct.Location = string(fields[i][1:])
}
}
return nil
}
var argKey = argDef{
parse: func(parser *parser, hdr, buf *streambuf.Buffer) error {
keys, err := parseKeyArg(buf)
parser.message.keys = keys
return err
},
serialize: serializeKeys,
}
var argMultiKeys = argDef{
parse: func(parser *parser, hdr, buf *streambuf.Buffer) error {
msg := parser.message
rest := buf.Bytes()
buf.Advance(len(rest))
raw_keys := bytes.FieldsFunc(rest, func(b rune) bool {
return b == ' '
})
if len(raw_keys) == 0 {
return ErrExpectedKeys
}
msg.keys = make([]memcacheString, len(raw_keys))
for i, raw_key := range raw_keys {
msg.keys[i] = memcacheString{raw_key}
}
return nil
},
serialize: serializeKeys,
}
var argFlags = argDef{
parse: textUintArg(setFlags),
func readHits(buf *bytes.Buffer) (hits []Hit, err error) {
hits = make([]Hit, 0, 10)
for {
line, err := buf.ReadBytes('\n')
if err == io.EOF && len(line) == 0 {
break
}
if err != nil && err != io.EOF {
return nil, err
}
line = trim(line)
if len(line) == 0 {
panic(fmt.Sprintf("BUG: Empty line in hit list."))
}
hit := Hit{}
// Grrrr. Variable length fields with fixed width fields without
// a delimiter.
numRest := bytes.SplitN(line, []byte{' '}, 2)
hit.Num, err = strconv.Atoi(str(numRest[0]))
if err != nil {
return nil, err
}
// Thankfully, the hit name is the only variable length field.
hit.Name = str(numRest[1][0:32])
// So now we can split the rest by whitespace.
// Except for when there isn't any whitespace to delimit columns!
// Oh my. *facepalm*
delim := func(r rune) bool {
return unicode.IsSpace(r) || r == '('
}
rest := bytes.FieldsFunc(numRest[1][32:], delim)
hit.Prob, err = readFloat(rest[0])
if err != nil {
return nil, err
}
hit.Prob /= 100.0
hit.EValue, err = readFloat(rest[1])
if err != nil {
return nil, err
}
hit.PValue, err = readFloat(rest[2])
if err != nil {
return nil, err
}
hit.ViterbiScore, err = readFloat(rest[3])
if err != nil {
return nil, err
}
hit.SSScore, err = readFloat(rest[4])
if err != nil {
return nil, err
}
hit.NumAlignedCols, err = strconv.Atoi(str(rest[5]))
if err != nil {
return nil, err
}
// query/template range look like '{start}-{end}' where '{...}' is
// an integer.
queryRange := bytes.Split(rest[6], []byte{'-'})
templateRange := bytes.Split(rest[7], []byte{'-'})
hit.QueryStart, err = strconv.Atoi(str(queryRange[0]))
if err != nil {
return nil, err
}
hit.QueryEnd, err = strconv.Atoi(str(queryRange[1]))
if err != nil {
return nil, err
}
hit.TemplateStart, err = strconv.Atoi(str(templateRange[0]))
if err != nil {
return nil, err
}
hit.TemplateEnd, err = strconv.Atoi(str(templateRange[1]))
if err != nil {
return nil, err
}
numPart := rest[8][0 : len(rest[8])-1] // i.e., remove parens in '(52)'.
hit.NumTemplateCols, err = strconv.Atoi(str(numPart))
if err != nil {
return nil, err
}
hits = append(hits, hit)
}
return hits, nil
}
func LoadModel(modelfile string) (*Model, error) {
f, e := os.Open(modelfile)
if e != nil {
return nil, e
}
ml := new(Model)
bf := make([]byte, 1024)
n, err := ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
ml.version = string(bf[:n])
ml.template = new(Template)
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
listb := bytes.FieldsFunc(bf[:n], ascIsSpace)
if len(listb) < 2 {
return nil, StringError("template nums error in modle file")
}
fmt.Println(string(listb[0]), string(listb[1]))
un, e0 := strconv.Atoi(string(listb[0]))
bn, e1 := strconv.Atoi(string(listb[1]))
if e0 != nil {
return nil, e0
}
if e1 != nil {
return nil, e1
}
fmt.Println(listb, len(listb[0]), len(listb[1]))
for i := 0; i < un; i++ {
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
ml.template.unigram = append(ml.template.unigram, string(bf[:n]))
}
for i := 0; i < bn; i++ {
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
ml.template.bigram = append(ml.template.bigram, string(bf[:n]))
}
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
tagnum, e2 := strconv.Atoi(string(bf[:n]))
if e2 != nil {
return nil, e2
}
ml.tags = new(Tag)
ml.tags.tags = make(map[string]TAG_TYPE)
for i := 0; i < tagnum; i++ {
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
listtags := bytes.FieldsFunc(bf[:n], ascIsSpace)
if len(listtags) < 2 {
return nil, StringError("read tag types failed")
}
tid, e := strconv.Atoi(string(listtags[1]))
if e != nil {
return nil, e
}
ml.tags.tags[string(listtags[0])] = TAG_TYPE(tid)
}
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
la, e3 := strconv.Atoi(string(bf[:n]))
if e3 != nil {
return nil, e3
}
fmt.Println(la)
ml.alpha = make([]float64, 0, la)
i := 0
for i = 0; i < la; {
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
listalpha := bytes.FieldsFunc(bf[:n], ascIsSpace)
lla := len(listalpha)
fmt.Println(listalpha, lla)
for j := 0; j < lla; j++ {
sf, e := strconv.ParseFloat(string(listalpha[j]), 64)
if e != nil {
return nil, e
}
ml.alpha = append(ml.alpha, sf)
}
i += lla
}
if i != la {
return nil, StringError("alpha size error")
}
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
ls, e4 := strconv.Atoi(string(bf[:n]))
if e4 != nil {
return nil, e3
}
ml.set = GetFeatureSet(ml.tags.GetTagNum())
for i = 0; i < ls; i++ {
n, err = ReadLine(f, bf)
if err != nil && err != io.EOF {
return nil, err
}
listalpha := bytes.FieldsFunc(bf[:n], ascIsSpace)
if len(listalpha) != 2 {
return nil, StringError("model format error")
}
sid, e := strconv.ParseInt(string(listalpha[1]), 10, 64)
if e != nil {
return nil, e
}
ml.set.dict[string(listalpha[0])] = &feature{id: ID_TYPE(sid)}
}
f.Close()
return ml, nil
}