func loadSiteList(fpath string) (lst []string, err error) {
var exists bool
if exists, err = isFileExists(fpath); err != nil {
errl.Printf("Error loading domaint list: %v\n", err)
}
if !exists {
return
}
f, err := os.Open(fpath)
if err != nil {
errl.Println("Error opening domain list:", err)
return
}
defer f.Close()
fr := bufio.NewReader(f)
lst = make([]string, 0)
var site string
for {
site, err = ReadLine(fr)
if err == io.EOF {
return lst, nil
} else if err != nil {
errl.Printf("Error reading domain list %s: %v\n", fpath, err)
return
}
if site == "" {
continue
}
lst = append(lst, strings.TrimSpace(site))
}
return
}
func TestReadLine(t *testing.T) {
testData := []struct {
text string
lines []string
}{
{"one\ntwo", []string{"one", "two"}},
{"three\r\nfour\n", []string{"three", "four"}},
{"five\r\nsix\r\n", []string{"five", "six"}},
{"seven", []string{"seven"}},
{"eight\n", []string{"eight"}},
{"\r\n", []string{""}},
{"\n", []string{""}},
}
for _, td := range testData {
raw := strings.NewReader(td.text)
rd := bufio.NewReader(raw)
for i, line := range td.lines {
l, err := ReadLine(rd)
if err != nil {
t.Fatalf("%d read error %v got: %s\ntext: %s\n", i+1, err, l, td.text)
}
if line != l {
t.Fatalf("%d read got: %s (%d) should be: %s (%d)\n", i+1, l, len(l), line, len(line))
}
}
_, err := ReadLine(rd)
if err != io.EOF {
t.Error("ReadLine past end should return EOF")
}
}
}
func parseConfig(path string) {
// fmt.Println("rcFile:", path)
f, err := os.Open(expandTilde(path))
if err != nil {
if os.IsNotExist(err) {
fmt.Printf("Config file %s not found, using default options\n", path)
} else {
fmt.Println("Error opening config file:", err)
}
return
}
defer f.Close()
fr := bufio.NewReader(f)
parser := reflect.ValueOf(configParser{})
zeroMethod := reflect.Value{}
var line string
var n int
for {
n++
line, err = ReadLine(fr)
if err == io.EOF {
return
} else if err != nil {
errl.Printf("Error reading rc file: %v\n", err)
os.Exit(1)
}
line = strings.TrimSpace(line)
if line == "" || line[0] == '#' {
continue
}
v := strings.Split(line, "=")
if len(v) != 2 {
fmt.Println("Config error: syntax error on line", n)
os.Exit(1)
}
key, val := strings.TrimSpace(v[0]), strings.TrimSpace(v[1])
if val == "" {
continue
}
methodName := "Parse" + strings.ToUpper(key[0:1]) + key[1:]
method := parser.MethodByName(methodName)
if method == zeroMethod {
fmt.Printf("Config error: no such option \"%s\"\n", key)
os.Exit(1)
}
args := []reflect.Value{reflect.ValueOf(val)}
method.Call(args)
}
}
func parseConfig(path string) {
// fmt.Println("rcFile:", path)
f, err := os.Open(expandTilde(path))
if err != nil {
if os.IsNotExist(err) {
fmt.Printf("Config file %s not found, using default options\n", path)
} else {
fmt.Println("Error opening config file:", err)
}
return
}
defer f.Close()
fr := bufio.NewReader(f)
parser := reflect.ValueOf(configParser{})
zeroMethod := reflect.Value{}
var line string
var n int
for {
n++
line, err = ReadLine(fr)
if err == io.EOF {
return
} else if err != nil {
Fatalf("Error reading rc file: %v\n", err)
}
line = strings.TrimSpace(line)
if line == "" || line[0] == '#' {
continue
}
v := strings.Split(line, "=")
if len(v) != 2 {
Fatal("config syntax error on line", n)
}
key, val := strings.TrimSpace(v[0]), strings.TrimSpace(v[1])
methodName := "Parse" + strings.ToUpper(key[0:1]) + key[1:]
method := parser.MethodByName(methodName)
if method == zeroMethod {
Fatalf("no such option \"%s\"\n", key)
}
// for backward compatibility, allow empty string in shadowMethod and logFile
if val == "" && key != "shadowMethod" && key != "logFile" {
Fatalf("empty %s, please comment or remove unused option\n", key)
}
args := []reflect.Value{reflect.ValueOf(val)}
method.Call(args)
}
}
func TestParseHeader(t *testing.T) {
var testData = []struct {
raw string
newraw string
header *Header
}{
{"content-length: 64\r\nConnection: keep-alive\r\n\r\n",
"content-length: 64\r\nConnection: keep-alive\r\n",
&Header{ContLen: 64, Chunking: false, ConnectionKeepAlive: true}},
{"Connection: keep-alive\r\nKeep-Alive: timeout=10\r\nTransfer-Encoding: chunked\r\nTE: trailers\r\n\r\n",
"Connection: keep-alive\r\nTransfer-Encoding: chunked\r\n",
&Header{ContLen: -1, Chunking: true, ConnectionKeepAlive: true,
KeepAlive: 10 * time.Second}},
/*
{"Connection: keep-alive\r\nKeep-Alive: max=5,\r\n timeout=10\r\n\r\n", // test multi-line header
"Connection: keep-alive\r\n",
&Header{ContLen: -1, Chunking: false, ConnectionKeepAlive: true,
KeepAlive: 10 * time.Second}},
{"Connection: \r\n keep-alive\r\n\r\n", // test multi-line header
"Connection: keep-alive\r\n",
&Header{ContLen: -1, Chunking: false, ConnectionKeepAlive: true}},
*/
}
for _, td := range testData {
var h Header
var newraw bytes.Buffer
h.parseHeader(bufio.NewReader(strings.NewReader(td.raw)), &newraw, nil)
if h.ContLen != td.header.ContLen {
t.Errorf("%q parsed content length wrong, should be %d, get %d\n",
td.raw, td.header.ContLen, h.ContLen)
}
if h.Chunking != td.header.Chunking {
t.Errorf("%q parsed chunking wrong, should be %t, get %t\n",
td.raw, td.header.Chunking, h.Chunking)
}
if h.ConnectionKeepAlive != td.header.ConnectionKeepAlive {
t.Errorf("%q parsed connection wrong, should be %v, get %v\n",
td.raw, td.header.ConnectionKeepAlive, h.ConnectionKeepAlive)
}
if h.KeepAlive != td.header.KeepAlive {
t.Errorf("%q parsed keep alive wrong, should be %v, get %v\n",
td.raw, td.header.KeepAlive, h.KeepAlive)
}
if newraw.String() != td.newraw {
t.Errorf("%q parsed raw wrong\nshould be: %q\ngot: %q\n",
td.raw, td.newraw, newraw.Bytes())
}
}
}
func loadUserPasswdFile(file string) {
if file == "" {
return
}
f, err := os.Open(file)
if err != nil {
Fatal("error opening user passwd fle:", err)
}
r := bufio.NewReader(f)
s := bufio.NewScanner(r)
for s.Scan() {
addUserPasswd(s.Text())
}
f.Close()
}
func TestCopyN(t *testing.T) {
testStr := "go is really a nice language"
for _, step := range []int{4, 9, 17, 32} {
src := bufio.NewReader(strings.NewReader(testStr))
dst := new(bytes.Buffer)
err := copyN(dst, src, len(testStr), step)
if err != nil {
t.Error("unexpected err:", err)
break
}
if dst.String() != testStr {
t.Errorf("step %d want %q, got: %q\n", step, testStr, dst.Bytes())
}
}
}
func (segmenter *Segmenter) LoadDictionary(fileName string) error {
segmenter.dict = &Dictionary{root: new(node)}
file, err := os.Open(fileName)
if err != nil {
return err
}
defer file.Close()
bufReader := bufio.NewReader(file)
var tokens []*Token
for {
var (
text string
frequency uint64
post string
)
n, _ := fmt.Fscanln(bufReader, &text, &frequency, &post)
if n == 0 { // EOF
break
} else if n < 2 {
continue
} else if n == 2 {
post = ""
}
lexeme := segmenter.splitTextToLexeme(text)
if lexeme == nil {
continue
}
token := Token{lexeme: lexeme, post: post, frequency: frequency, distance: 1.0 / frequency}
segmenter.dict.insertToken(&token)
tokens = append(tokens, &token)
}
for _, token := range tokens {
segs := segmenter.viterbi(token)
token.segs = segs
}
return nil
}
