// Load model from a text file, which must be in the following format:
//
// word_0 N(word_0, topic_0) N(word_0, topic_1) ...
// word_1 N(word_1, topic_0) N(word_1, topic_1) ...
// ...
//
// where each line in the file is the topic histogram of a word,
// word_x is a string containing no whitespaces, and N(word_x,topic_y)
// is an integer, counting the number of times that word_x is assigned
// topic_y. Fields in a line are separated by one or more whitespaces.
//
func LoadModel(filename string) (model *Model, err os.Error) {
file, err := os.Open(filename, 0, 0)
if err != nil {
return nil, os.NewError("Cannot open file: " + filename)
}
defer file.Close()
num_topics := 0
model = new(Model)
model.topic_histograms = make(map[string]Histogram)
reader := line.NewReader(bufio.NewReader(file), kMaxModelFileLineLength)
l, is_prefix, err := reader.ReadLine()
for err == nil {
line := string(l)
if is_prefix {
return nil, os.NewError("Encountered a long line:" + line)
}
fields := strings.Fields(line)
if len(fields) < 3 {
return nil, os.NewError("Invalid line: " + line)
}
if _, present := model.topic_histograms[fields[0]]; present {
return nil, os.NewError("Found duplicated word: " + fields[0])
}
if num_topics == 0 {
num_topics = len(fields) - 1
model.global_histogram = NewHistogram(num_topics)
model.zero_histogram = NewHistogram(num_topics)
} else if len(fields)-1 != num_topics {
return nil, os.NewError("Inconsistent num_topics: " + line)
}
hist := NewHistogram(num_topics)
var conv_err os.Error
for i := 0; i < num_topics; i++ {
hist[i], conv_err = strconv.Atoi(fields[i+1])
if conv_err != nil {
return nil, os.NewError("Failed conversion to int: " + fields[i+1])
}
model.global_histogram[i] += hist[i]
}
model.topic_histograms[fields[0]] = hist
l, _, err = reader.ReadLine()
}
if err != os.EOF {
return nil, os.NewError("Error reading: " + filename + err.String())
}
if len(model.topic_histograms) <= 0 {
return nil, os.NewError("No valid line in file: " + filename)
}
return model, nil
}
func LoadCorpus(filename string, num_topics int) (corpus *Corpus, err os.Error) {
file, err := os.Open(filename, 0, 0)
if err != nil {
return nil, os.NewError("Cannot open file: " + filename)
}
defer file.Close()
corpus = NewCorpus()
reader := line.NewReader(bufio.NewReader(file), kMaxCorpusFileLineLength)
l, is_prefix, err := reader.ReadLine()
for err == nil {
line := string(l)
if is_prefix {
return nil, os.NewError("Encountered a long line:" + line)
}
if len(l) > 1 { // skip empty lines
doc, err := NewDocument(line, num_topics)
if err == nil {
*corpus = append(*corpus, doc)
} else {
panic("Cannot create document from: " + line + " due to " + err.String())
}
}
l, _, err = reader.ReadLine()
}
if err != os.EOF {
return nil, os.NewError("Error reading: " + filename + err.String())
}
return corpus, nil
}
func newconn(c net.Conn) {
// find out the desired destination on a new connect
connre := regexp.MustCompile("CONNECT (.*) HTTP/")
r := line.NewReader(c, 512)
l, isprefix, err := r.ReadLine()
if err != nil || isprefix == true {
c.Close()
return
}
m := connre.FindStringSubmatch(string(l))
if m == nil {
io.WriteString(c, "HTTP/1.0 502 No luck, Chuck\r\n\r\n")
c.Close()
return
}
// wait until we get a blank line (end of HTTP headers)
for {
l, _, _ := r.ReadLine()
if l == nil {
return
}
if len(l) == 0 {
break
}
}
if l != nil {
if command == "" {
forward(c, m[1])
} else {
forward2cmd(c, m[1])
}
}
}
func (s *Server) Gophermap(ctx *Context, gmap *os.File, dir *os.File) (ok bool, err os.Error) {
cwd := dir.Name()[len(s.Cwd):]
linereader := line.NewReader(bufio.NewReader(gmap), 512)
for {
if read, _, err := linereader.ReadLine(); err == nil {
entry := bytes.NewBuffer(read).String()
if strings.Index(entry, "\t") == -1 {
ctx.Write(s.InfoLine(entry))
} else {
entries := s.ParseGophermapLine(ctx, entry)
for e := 0; e < entries.Len(); e++ {
ctx.Write(entries[e].(*gophermapEntry).String())
}
}
} else {
if err != os.EOF {
return false, err
}
break
}
}
ctx.Write(".")
s.Logger.Printf("Served gophermapped directory `%s`\n", cwd)
return true, nil
}
func (s *Server) handle(ctx *Context) (err os.Error) {
defer ctx.conn.Close()
linereader := line.NewReader(bufio.NewReader(ctx.conn), 512)
read, _, err := linereader.ReadLine()
if err != nil {
s.Logger.Println("Malformed request from client")
return
}
clientRequest := bytes.NewBuffer(read).String()
s.Logger.Printf("REQUEST: %s\n", clientRequest)
ctx.Request = "/" + strings.Trim(path.Clean(clientRequest), "/")
absReqPath := path.Clean(fmt.Sprintf("%s%s", s.Cwd, ctx.Request))
if !strings.HasPrefix(absReqPath, s.Cwd) {
s.Logger.Printf("Requested file not in document root")
return
}
var requestedFile *os.File
if requestedFile, err = os.Open(absReqPath, 0, 0); err != nil {
if patherr, ok := err.(*os.PathError); ok {
switch true {
case patherr.Error == os.ENOENT:
ctx.Error(fmt.Sprintf("Resource `%s' not found", clientRequest))
s.Logger.Printf("ERROR: Resource `%s' not found\n", ctx.Request)
return
case patherr == os.EPERM || patherr == os.EACCES:
ctx.Error(fmt.Sprintf("Resource `%s' not found", clientRequest))
s.Logger.Printf("ERROR: Access denied for file `%s'\n", ctx.Request)
return
default:
s.Logger.Printf("ERROR: %s\n", err)
return
}
} else {
s.Logger.Printf("ERROR: %s\n", err)
return
}
}
stats, err := requestedFile.Stat()
if err != nil {
s.Logger.Printf("ERROR: Could not stat file `%s': %s\n", absReqPath, err)
return
}
if stats.IsDirectory() {
s.Directory(ctx, requestedFile)
} else if stats.IsRegular() {
s.Textfile(ctx, requestedFile)
} else {
ctx.Write(s.InfoLine("STUMPED"))
}
return
}
func main() {
if flag.NArg() < 1 {
fmt.Println("usage: doozer-report FILE")
os.Exit(1)
}
fn := flag.Arg(0)
f, err := os.OpenFile(fn, os.O_RDONLY, 0)
if err != nil {
panic(err)
}
defer f.Close()
ln := 0
ts := Int64Array(make([]int64, 0))
r := line.NewReader(f, math.MaxInt8)
for {
ln += 1
line, _, err := r.ReadLine()
if err == os.EOF {
break
} else if err != nil {
panic(err)
}
fds := bytes.Split(line, Delim, -1)
if len(fds) < 5 {
fmt.Fprintf(os.Stderr, "%s:%d:invalid record: %s\n", fn, ln, line)
continue
}
ns, err := strconv.Atoi64(string(fds[4]))
if err != nil {
// TODO: be more gracefull
fmt.Fprintf(os.Stderr, "%s:%d: %s\n", fn, ln, err.String())
continue
}
ts = append(ts, ns)
}
sort.Sort(ts)
n := float64(len(ts))
ftp := int((n * 50 / 100) + 0.5)
nnp := int((n * 99 / 100) + 0.5)
fmt.Printf("50%% %d.%03d sec\n", ts[ftp]/1e9, ts[ftp]%1e9/1e6)
fmt.Printf("99%% %d.%03d sec\n", ts[nnp]/1e9, ts[nnp]%1e9/1e6)
}
func main() {
threads := flag.Int("threads", 0, "number of parallel threads in a run. If 0, use CPU count.")
sleepTime := flag.Float64("sleep", 4.0, "amount of sleep between runs.")
runs := flag.Int("runs", 10, "number of runs.")
flag.Parse()
if *threads == 0 {
*threads = fuse.CountCpus()
runtime.GOMAXPROCS(*threads)
}
filename := flag.Args()[0]
f, err := os.Open(filename)
if err != nil {
panic("err" + err.String())
}
linelen := 1000
reader := line.NewReader(f, linelen)
files := make([]string, 0)
for {
l, _, err := reader.ReadLine()
if err != nil {
break
}
files = append(files, string(l))
}
totalRuns := *runs + 1
results := make([]float64, 0)
for j := 0; j < totalRuns; j++ {
result := BulkStat(*threads, files)
if j > 0 {
results = append(results, result)
} else {
fmt.Println("Ignoring first run to preheat caches.")
}
if j < totalRuns-1 {
fmt.Printf("Sleeping %.2f seconds\n", *sleepTime)
time.Sleep(int64(*sleepTime * 1e9))
}
}
Analyze(results)
}
"fmt"
"log"
"os"
"strconv"
"strings"
)
const (
MAX_LINE = 1 << 15
)
type ProblemReader struct {
r *line.Reader
}
var In = ProblemReader{line.NewReader(os.Stdin, MAX_LINE)}
func (in *ProblemReader) SolveProblems(solve func(*ProblemReader) string) {
cases := in.Num()
for j := 0; j < cases; j++ {
fmt.Printf("Case #%d: %s\n", j+1, solve(in))
}
}
/* Read nums from in */
func (in *ProblemReader) Nums() (nums []int) {
words := in.Words()
nums = make([]int, len(words))
for pos, word := range words {
// Iplements half of HexFormat; q.v.
func (TiText) ReadHex(r io.Reader) (Hexfile, os.Error) {
resp := RecordSequence{}
// 16 bytes per line ought to be enough for anybody.
line_reader := line.NewReader(r, 64)
addr := 0
for {
line, is_prefix, err := line_reader.ReadLine()
if line == nil && err == os.EOF {
break
} else if err != nil {
return nil, err
}
for is_prefix {
var line_part []byte
line_part, is_prefix, err = line_reader.ReadLine()
if line == nil && err == os.EOF {
break
} else if err != nil {
return nil, err
}
line = append(line, line_part...)
}
line_s := string(line)
reader := strings.NewReader(line_s)
switch line_s[0] {
case '@':
reader.ReadByte()
if len(line) < 2 {
return nil, StrError("Format error: short address")
}
addr, err = decodeInt(reader, -1)
if err != nil {
return nil, err
}
// TODO(thequux): Check for trailing junk
case 'q':
return resp, nil
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
fallthrough
case 'a', 'b', 'c', 'd', 'e', 'f':
fallthrough
case 'A', 'B', 'C', 'D', 'E', 'F':
buf := make([]byte, 0, len(line)/3)
for {
if n, err := decodeInt(reader, -1); err != nil {
if err == os.EOF {
break
}
return nil, err
} else {
buf = append(buf, byte(n))
}
}
resp = append(resp, Record{addr, buf})
addr += len(buf)
default:
return nil, StrError("Invalid format")
}
}
return resp, nil
}
// Decode reads a PGP armored block from the given Reader. It will ignore
// leading garbage. If it doesn't find a block, it will return nil, os.EOF. The
// given Reader is not usable after calling this function: an arbitary amount
// of data may have been read past the end of the block.
func Decode(in io.Reader) (p *Block, err os.Error) {
r := line.NewReader(in, 100)
var line []byte
ignoreNext := false
TryNextBlock:
p = nil
// Skip leading garbage
for {
ignoreThis := ignoreNext
line, ignoreNext, err = r.ReadLine()
if err != nil {
return
}
if ignoreNext || ignoreThis {
continue
}
line = bytes.TrimSpace(line)
if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) {
break
}
}
p = new(Block)
p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)])
p.Header = make(map[string]string)
nextIsContinuation := false
var lastKey string
// Read headers
for {
isContinuation := nextIsContinuation
line, nextIsContinuation, err = r.ReadLine()
if err != nil {
p = nil
return
}
if isContinuation {
p.Header[lastKey] += string(line)
continue
}
line = bytes.TrimSpace(line)
if len(line) == 0 {
break
}
i := bytes.Index(line, []byte(": "))
if i == -1 {
goto TryNextBlock
}
lastKey = string(line[:i])
p.Header[lastKey] = string(line[i+2:])
}
p.lReader.in = r
p.oReader.currentCRC = crc24Init
p.oReader.lReader = &p.lReader
p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader)
p.Body = &p.oReader
return
}
func newIntelHEX(r io.Reader) *IntelHEX {
out := make(chan *Record)
between := make(chan []byte)
reader := line.NewReader(r, 64)
decodeHex := func(b []byte, n int) (int, []byte) {
res := 0
for _, ch := range b[:n] {
digit := int(0)
switch {
case '0' <= ch && ch <= '9':
digit = int(ch - '0')
case 'A' <= ch && ch <= 'F':
ch |= 0x20
fallthrough
case 'a' <= ch && ch <= 'f':
digit = int(ch - 'a' + 10)
default:
return res, nil
}
res = res<<4 + digit
}
return res, b[n:]
}
byte_addition := func(b []byte) (res int) {
res = 0
var c int
for len(b) > 0 {
c, b = decodeHex(b, 2)
res += c
}
return
}
go func() {
for {
lp, is_prefix, err := reader.ReadLine()
//fmt.Printf("lp is %s (%d)\n", lp, len(lp))
if is_prefix {
for is_prefix {
var extra []byte
cp := make([]byte, len(lp))
copy(cp, lp)
extra, is_prefix, err = reader.ReadLine()
//fmt.Printf("extra \"%s\"\n", extra)
if err != nil {
goto cleanup // umad?
}
lp = append(cp, extra...)
//fmt.Printf("extended: \"%s\"\n", lp)
}
}
if !is_prefix {
//fmt.Printf("sending \"%s\"\n", lp)
between <- []byte(string(lp)) // truncate any left over space from []byte
}
cleanup:
if err != nil {
if err == os.EOF {
fmt.Printf("a exiting...\n")
close(between)
return
}
panic(err)
}
}
}()
go func() {
addrMod := 0
wordsz := 2
for str := range between {
if str == nil {
fmt.Printf("b exiting...\n")
out <- &Record{
Address: 0,
Data: nil,
}
return
}
if str[0] != ':' {
panic("not even the \":\"?\n")
}
//fmt.Printf("b recvd \"%s\"\n", str)
os.Stdout.Sync()
str = str[1:]
//fmt.Printf("checksuming \"%s\"\n", str[:len(str)-2])
xsum := 0x100 - (byte_addition(str[:len(str)-2]) & 0xFF)
checksum, _ := decodeHex(str[len(str)-2:], 2)
// checksum, _ := decodeHex(str[bytecnt*wordsz:], 2)
if xsum != checksum {
panic("checksum failed")
}
bytecnt, str := decodeHex(str, 2)
addr, str := decodeHex(str, 4)
rectyp, str := decodeHex(str, 2)
// fmt.Printf("{%d %d %d (%x v %x)}\n", bytecnt, addr, rectyp, xsum, checksum)
switch rectyp {
case IH_DATA:
data, err := hex.DecodeString(string(str[:bytecnt*wordsz]))
if err != nil {
panic(err)
}
addr += int(addrMod)
out <- &Record{
Address: int(addr),
Data: data,
}
case IH_ESAR: // switch wordsz to 4
wordsz = 4
mod, _ := decodeHex(str, bytecnt*wordsz)
addrMod = mod << 4
case IH_ELAR:
mod, _ := decodeHex(str, bytecnt*wordsz)
addrMod = mod << 16
case IH_SSAR:
continue // do we care about CS:IP registers?
case IH_SLAR:
continue // " " EIP?
case IH_EOF:
out <- &Record{
Address: 0,
Data: nil,
}
return
}
}
}()
return &IntelHEX{
stream: out,
}
}
// Read a private key (file) string and create a public key. Return the private key.
func (k *RR_DNSKEY) ReadPrivateKey(q io.Reader) (PrivateKey, os.Error) {
p := new(rsa.PrivateKey)
p.Primes = []*big.Int{nil, nil}
var left, right string
r := line.NewReader(q, 300)
line, _, err := r.ReadLine()
for err == nil {
n, _ := fmt.Sscanf(string(line), "%s %s+\n", &left, &right)
if n > 0 {
switch left {
case "Private-key-format:":
if right != "v1.3" {
return nil, ErrPrivKey
}
case "Algorithm:":
a, _ := strconv.Atoi(right)
if a == 0 {
return nil, ErrAlg
}
k.Algorithm = uint8(a)
case "Modulus:", "PublicExponent:", "PrivateExponent:", "Prime1:", "Prime2:":
v, err := packBase64([]byte(right))
if err != nil {
return nil, err
}
if left == "Modulus:" {
p.PublicKey.N = big.NewInt(0)
p.PublicKey.N.SetBytes(v)
}
if left == "PublicExponent:" {
i := big.NewInt(0)
i.SetBytes(v)
// Int64 should be large enough
p.PublicKey.E = int(i.Int64())
}
if left == "PrivateExponent:" {
p.D = big.NewInt(0)
p.D.SetBytes(v)
}
if left == "Prime1:" {
p.Primes[0] = big.NewInt(0)
p.Primes[0].SetBytes(v)
}
if left == "Prime2:" {
p.Primes[1] = big.NewInt(0)
p.Primes[1].SetBytes(v)
}
case "Exponent1:", "Exponent2:", "Coefficient:":
/* not used in Go (yet) */
case "Created:", "Publish:", "Activate:":
/* not used in Go (yet) */
default:
return nil, ErrKey
}
}
line, _, err = r.ReadLine()
}
if !k.setPublicKeyRSA(p.PublicKey.E, p.PublicKey.N) {
return nil, ErrKey
}
return p, nil
}