Exemple #1
0
// Load model from a text file, which must be in the following format:
//
// word_0   N(word_0, topic_0)  N(word_0, topic_1) ...
// word_1   N(word_1, topic_0)  N(word_1, topic_1) ...
// ...
//
// where each line in the file is the topic histogram of a word,
// word_x is a string containing no whitespaces, and N(word_x,topic_y)
// is an integer, counting the number of times that word_x is assigned
// topic_y.  Fields in a line are separated by one or more whitespaces.
//
func LoadModel(filename string) (model *Model, err os.Error) {
	file, err := os.Open(filename, 0, 0)
	if err != nil {
		return nil, os.NewError("Cannot open file: " + filename)
	}
	defer file.Close()

	num_topics := 0
	model = new(Model)
	model.topic_histograms = make(map[string]Histogram)

	reader := line.NewReader(bufio.NewReader(file), kMaxModelFileLineLength)
	l, is_prefix, err := reader.ReadLine()
	for err == nil {
		line := string(l)

		if is_prefix {
			return nil, os.NewError("Encountered a long line:" + line)
		}

		fields := strings.Fields(line)
		if len(fields) < 3 {
			return nil, os.NewError("Invalid line: " + line)
		}

		if _, present := model.topic_histograms[fields[0]]; present {
			return nil, os.NewError("Found duplicated word: " + fields[0])
		}

		if num_topics == 0 {
			num_topics = len(fields) - 1
			model.global_histogram = NewHistogram(num_topics)
			model.zero_histogram = NewHistogram(num_topics)
		} else if len(fields)-1 != num_topics {
			return nil, os.NewError("Inconsistent num_topics: " + line)
		}

		hist := NewHistogram(num_topics)
		var conv_err os.Error
		for i := 0; i < num_topics; i++ {
			hist[i], conv_err = strconv.Atoi(fields[i+1])
			if conv_err != nil {
				return nil, os.NewError("Failed conversion to int: " + fields[i+1])
			}
			model.global_histogram[i] += hist[i]
		}
		model.topic_histograms[fields[0]] = hist

		l, _, err = reader.ReadLine()
	}

	if err != os.EOF {
		return nil, os.NewError("Error reading: " + filename + err.String())
	}
	if len(model.topic_histograms) <= 0 {
		return nil, os.NewError("No valid line in file: " + filename)
	}

	return model, nil
}
Exemple #2
0
func LoadCorpus(filename string, num_topics int) (corpus *Corpus, err os.Error) {
	file, err := os.Open(filename, 0, 0)
	if err != nil {
		return nil, os.NewError("Cannot open file: " + filename)
	}
	defer file.Close()

	corpus = NewCorpus()
	reader := line.NewReader(bufio.NewReader(file), kMaxCorpusFileLineLength)
	l, is_prefix, err := reader.ReadLine()
	for err == nil {
		line := string(l)

		if is_prefix {
			return nil, os.NewError("Encountered a long line:" + line)
		}

		if len(l) > 1 { // skip empty lines
			doc, err := NewDocument(line, num_topics)
			if err == nil {
				*corpus = append(*corpus, doc)
			} else {
				panic("Cannot create document from: " + line + " due to " + err.String())
			}
		}

		l, _, err = reader.ReadLine()
	}

	if err != os.EOF {
		return nil, os.NewError("Error reading: " + filename + err.String())
	}
	return corpus, nil
}
func newconn(c net.Conn) {
	// find out the desired destination on a new connect
	connre := regexp.MustCompile("CONNECT (.*) HTTP/")
	r := line.NewReader(c, 512)
	l, isprefix, err := r.ReadLine()
	if err != nil || isprefix == true {
		c.Close()
		return
	}
	m := connre.FindStringSubmatch(string(l))
	if m == nil {
		io.WriteString(c, "HTTP/1.0 502 No luck, Chuck\r\n\r\n")
		c.Close()
		return
	}
	// wait until we get a blank line (end of HTTP headers)
	for {
		l, _, _ := r.ReadLine()
		if l == nil {
			return
		}
		if len(l) == 0 {
			break
		}
	}
	if l != nil {
		if command == "" {
			forward(c, m[1])
		} else {
			forward2cmd(c, m[1])
		}
	}
}
Exemple #4
0
func (s *Server) Gophermap(ctx *Context, gmap *os.File, dir *os.File) (ok bool, err os.Error) {
	cwd := dir.Name()[len(s.Cwd):]
	linereader := line.NewReader(bufio.NewReader(gmap), 512)
	for {
		if read, _, err := linereader.ReadLine(); err == nil {
			entry := bytes.NewBuffer(read).String()
			if strings.Index(entry, "\t") == -1 {
				ctx.Write(s.InfoLine(entry))
			} else {
				entries := s.ParseGophermapLine(ctx, entry)
				for e := 0; e < entries.Len(); e++ {
					ctx.Write(entries[e].(*gophermapEntry).String())
				}
			}
		} else {
			if err != os.EOF {
				return false, err
			}
			break
		}

	}
	ctx.Write(".")
	s.Logger.Printf("Served gophermapped directory `%s`\n", cwd)
	return true, nil
}
Exemple #5
0
func (s *Server) handle(ctx *Context) (err os.Error) {
	defer ctx.conn.Close()
	linereader := line.NewReader(bufio.NewReader(ctx.conn), 512)
	read, _, err := linereader.ReadLine()
	if err != nil {
		s.Logger.Println("Malformed request from client")
		return
	}
	clientRequest := bytes.NewBuffer(read).String()
	s.Logger.Printf("REQUEST: %s\n", clientRequest)
	ctx.Request = "/" + strings.Trim(path.Clean(clientRequest), "/")
	absReqPath := path.Clean(fmt.Sprintf("%s%s", s.Cwd, ctx.Request))
	if !strings.HasPrefix(absReqPath, s.Cwd) {
		s.Logger.Printf("Requested file not in document root")
		return
	}
	var requestedFile *os.File
	if requestedFile, err = os.Open(absReqPath, 0, 0); err != nil {
		if patherr, ok := err.(*os.PathError); ok {
			switch true {
			case patherr.Error == os.ENOENT:
				ctx.Error(fmt.Sprintf("Resource `%s' not found", clientRequest))
				s.Logger.Printf("ERROR: Resource `%s' not found\n", ctx.Request)
				return
			case patherr == os.EPERM || patherr == os.EACCES:
				ctx.Error(fmt.Sprintf("Resource `%s' not found", clientRequest))
				s.Logger.Printf("ERROR: Access denied for file `%s'\n", ctx.Request)
				return
			default:
				s.Logger.Printf("ERROR: %s\n", err)
				return
			}
		} else {
			s.Logger.Printf("ERROR: %s\n", err)
			return
		}
	}
	stats, err := requestedFile.Stat()
	if err != nil {
		s.Logger.Printf("ERROR: Could not stat file `%s': %s\n", absReqPath, err)
		return
	}
	if stats.IsDirectory() {
		s.Directory(ctx, requestedFile)
	} else if stats.IsRegular() {
		s.Textfile(ctx, requestedFile)
	} else {
		ctx.Write(s.InfoLine("STUMPED"))
	}
	return
}
func main() {
	if flag.NArg() < 1 {
		fmt.Println("usage: doozer-report FILE")
		os.Exit(1)
	}

	fn := flag.Arg(0)
	f, err := os.OpenFile(fn, os.O_RDONLY, 0)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	ln := 0
	ts := Int64Array(make([]int64, 0))
	r := line.NewReader(f, math.MaxInt8)
	for {
		ln += 1
		line, _, err := r.ReadLine()
		if err == os.EOF {
			break
		} else if err != nil {
			panic(err)
		}

		fds := bytes.Split(line, Delim, -1)

		if len(fds) < 5 {
			fmt.Fprintf(os.Stderr, "%s:%d:invalid record: %s\n", fn, ln, line)
			continue
		}

		ns, err := strconv.Atoi64(string(fds[4]))
		if err != nil {
			// TODO: be more gracefull
			fmt.Fprintf(os.Stderr, "%s:%d: %s\n", fn, ln, err.String())
			continue
		}

		ts = append(ts, ns)
	}

	sort.Sort(ts)

	n := float64(len(ts))
	ftp := int((n * 50 / 100) + 0.5)
	nnp := int((n * 99 / 100) + 0.5)

	fmt.Printf("50%% %d.%03d sec\n", ts[ftp]/1e9, ts[ftp]%1e9/1e6)
	fmt.Printf("99%% %d.%03d sec\n", ts[nnp]/1e9, ts[nnp]%1e9/1e6)
}
Exemple #7
0
func main() {
	threads := flag.Int("threads", 0, "number of parallel threads in a run. If 0, use CPU count.")
	sleepTime := flag.Float64("sleep", 4.0, "amount of sleep between runs.")
	runs := flag.Int("runs", 10, "number of runs.")

	flag.Parse()
	if *threads == 0 {
		*threads = fuse.CountCpus()
		runtime.GOMAXPROCS(*threads)
	}
	filename := flag.Args()[0]
	f, err := os.Open(filename)
	if err != nil {
		panic("err" + err.String())
	}

	linelen := 1000
	reader := line.NewReader(f, linelen)

	files := make([]string, 0)
	for {
		l, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		files = append(files, string(l))
	}

	totalRuns := *runs + 1

	results := make([]float64, 0)
	for j := 0; j < totalRuns; j++ {
		result := BulkStat(*threads, files)
		if j > 0 {
			results = append(results, result)
		} else {
			fmt.Println("Ignoring first run to preheat caches.")
		}

		if j < totalRuns-1 {
			fmt.Printf("Sleeping %.2f seconds\n", *sleepTime)
			time.Sleep(int64(*sleepTime * 1e9))
		}
	}

	Analyze(results)
}
Exemple #8
0
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
)

const (
	MAX_LINE = 1 << 15
)

type ProblemReader struct {
	r *line.Reader
}

var In = ProblemReader{line.NewReader(os.Stdin, MAX_LINE)}

func (in *ProblemReader) SolveProblems(solve func(*ProblemReader) string) {
	cases := in.Num()

	for j := 0; j < cases; j++ {
		fmt.Printf("Case #%d: %s\n", j+1, solve(in))
	}
}

/* Read nums from in */
func (in *ProblemReader) Nums() (nums []int) {
	words := in.Words()
	nums = make([]int, len(words))

	for pos, word := range words {
Exemple #9
0
// Iplements half of HexFormat; q.v.
func (TiText) ReadHex(r io.Reader) (Hexfile, os.Error) {
	resp := RecordSequence{}

	// 16 bytes per line ought to be enough for anybody.
	line_reader := line.NewReader(r, 64)
	addr := 0

	for {
		line, is_prefix, err := line_reader.ReadLine()
		if line == nil && err == os.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		for is_prefix {
			var line_part []byte
			line_part, is_prefix, err = line_reader.ReadLine()
			if line == nil && err == os.EOF {
				break
			} else if err != nil {
				return nil, err
			}
			line = append(line, line_part...)
		}
		line_s := string(line)
		reader := strings.NewReader(line_s)
		switch line_s[0] {
		case '@':
			reader.ReadByte()
			if len(line) < 2 {
				return nil, StrError("Format error: short address")
			}
			addr, err = decodeInt(reader, -1)
			if err != nil {
				return nil, err
			}
			// TODO(thequux): Check for trailing junk
		case 'q':
			return resp, nil
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			fallthrough
		case 'a', 'b', 'c', 'd', 'e', 'f':
			fallthrough
		case 'A', 'B', 'C', 'D', 'E', 'F':
			buf := make([]byte, 0, len(line)/3)
			for {

				if n, err := decodeInt(reader, -1); err != nil {
					if err == os.EOF {
						break
					}
					return nil, err
				} else {
					buf = append(buf, byte(n))
				}
			}
			resp = append(resp, Record{addr, buf})
			addr += len(buf)
		default:
			return nil, StrError("Invalid format")
		}
	}
	return resp, nil
}
Exemple #10
0
// Decode reads a PGP armored block from the given Reader. It will ignore
// leading garbage. If it doesn't find a block, it will return nil, os.EOF. The
// given Reader is not usable after calling this function: an arbitary amount
// of data may have been read past the end of the block.
func Decode(in io.Reader) (p *Block, err os.Error) {
	r := line.NewReader(in, 100)
	var line []byte
	ignoreNext := false

TryNextBlock:
	p = nil

	// Skip leading garbage
	for {
		ignoreThis := ignoreNext
		line, ignoreNext, err = r.ReadLine()
		if err != nil {
			return
		}
		if ignoreNext || ignoreThis {
			continue
		}
		line = bytes.TrimSpace(line)
		if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) {
			break
		}
	}

	p = new(Block)
	p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)])
	p.Header = make(map[string]string)
	nextIsContinuation := false
	var lastKey string

	// Read headers
	for {
		isContinuation := nextIsContinuation
		line, nextIsContinuation, err = r.ReadLine()
		if err != nil {
			p = nil
			return
		}
		if isContinuation {
			p.Header[lastKey] += string(line)
			continue
		}
		line = bytes.TrimSpace(line)
		if len(line) == 0 {
			break
		}

		i := bytes.Index(line, []byte(": "))
		if i == -1 {
			goto TryNextBlock
		}
		lastKey = string(line[:i])
		p.Header[lastKey] = string(line[i+2:])
	}

	p.lReader.in = r
	p.oReader.currentCRC = crc24Init
	p.oReader.lReader = &p.lReader
	p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader)
	p.Body = &p.oReader

	return
}
Exemple #11
0
func newIntelHEX(r io.Reader) *IntelHEX {
	out := make(chan *Record)
	between := make(chan []byte)
	reader := line.NewReader(r, 64)
	decodeHex := func(b []byte, n int) (int, []byte) {
		res := 0
		for _, ch := range b[:n] {
			digit := int(0)
			switch {
			case '0' <= ch && ch <= '9':
				digit = int(ch - '0')
			case 'A' <= ch && ch <= 'F':
				ch |= 0x20
				fallthrough
			case 'a' <= ch && ch <= 'f':
				digit = int(ch - 'a' + 10)
			default:
				return res, nil
			}
			res = res<<4 + digit
		}
		return res, b[n:]
	}
	byte_addition := func(b []byte) (res int) {
		res = 0
		var c int
		for len(b) > 0 {
			c, b = decodeHex(b, 2)
			res += c
		}
		return
	}

	go func() {
		for {
			lp, is_prefix, err := reader.ReadLine()
			//fmt.Printf("lp is %s (%d)\n", lp, len(lp))
			if is_prefix {
				for is_prefix {
					var extra []byte
					cp := make([]byte, len(lp))
					copy(cp, lp)
					extra, is_prefix, err = reader.ReadLine()
					//fmt.Printf("extra \"%s\"\n", extra)
					if err != nil {
						goto cleanup // umad?
					}
					lp = append(cp, extra...)
					//fmt.Printf("extended: \"%s\"\n", lp)
				}
			}
			if !is_prefix {
				//fmt.Printf("sending \"%s\"\n", lp)
				between <- []byte(string(lp)) // truncate any left over space from []byte
			}
		cleanup:
			if err != nil {
				if err == os.EOF {
					fmt.Printf("a exiting...\n")
					close(between)
					return
				}
				panic(err)
			}
		}
	}()

	go func() {
		addrMod := 0
		wordsz := 2
		for str := range between {
			if str == nil {
				fmt.Printf("b exiting...\n")
				out <- &Record{
					Address: 0,
					Data:    nil,
				}
				return
			}
			if str[0] != ':' {
				panic("not even the \":\"?\n")
			}
			//fmt.Printf("b recvd \"%s\"\n", str)
			os.Stdout.Sync()

			str = str[1:]
			//fmt.Printf("checksuming \"%s\"\n", str[:len(str)-2])
			xsum := 0x100 - (byte_addition(str[:len(str)-2]) & 0xFF)
			checksum, _ := decodeHex(str[len(str)-2:], 2)
			// checksum, _ := decodeHex(str[bytecnt*wordsz:], 2)
			if xsum != checksum {
				panic("checksum failed")
			}

			bytecnt, str := decodeHex(str, 2)
			addr, str := decodeHex(str, 4)
			rectyp, str := decodeHex(str, 2)
			// fmt.Printf("{%d %d %d (%x v %x)}\n", bytecnt, addr, rectyp, xsum, checksum)

			switch rectyp {
			case IH_DATA:
				data, err := hex.DecodeString(string(str[:bytecnt*wordsz]))
				if err != nil {
					panic(err)
				}

				addr += int(addrMod)
				out <- &Record{
					Address: int(addr),
					Data:    data,
				}
			case IH_ESAR: // switch wordsz to 4
				wordsz = 4
				mod, _ := decodeHex(str, bytecnt*wordsz)
				addrMod = mod << 4
			case IH_ELAR:
				mod, _ := decodeHex(str, bytecnt*wordsz)
				addrMod = mod << 16
			case IH_SSAR:
				continue // do we care about CS:IP registers?
			case IH_SLAR:
				continue // " " EIP?
			case IH_EOF:
				out <- &Record{
					Address: 0,
					Data:    nil,
				}
				return
			}

		}
	}()

	return &IntelHEX{
		stream: out,
	}
}
Exemple #12
0
// Read a private key (file) string and create a public key. Return the private key.
func (k *RR_DNSKEY) ReadPrivateKey(q io.Reader) (PrivateKey, os.Error) {
	p := new(rsa.PrivateKey)
	p.Primes = []*big.Int{nil, nil}
	var left, right string
	r := line.NewReader(q, 300)
	line, _, err := r.ReadLine()
	for err == nil {
		n, _ := fmt.Sscanf(string(line), "%s %s+\n", &left, &right)
		if n > 0 {
			switch left {
			case "Private-key-format:":
				if right != "v1.3" {
					return nil, ErrPrivKey
				}
			case "Algorithm:":
				a, _ := strconv.Atoi(right)
				if a == 0 {
					return nil, ErrAlg
				}
				k.Algorithm = uint8(a)
			case "Modulus:", "PublicExponent:", "PrivateExponent:", "Prime1:", "Prime2:":
				v, err := packBase64([]byte(right))
				if err != nil {
					return nil, err
				}
				if left == "Modulus:" {
					p.PublicKey.N = big.NewInt(0)
					p.PublicKey.N.SetBytes(v)
				}
				if left == "PublicExponent:" {
					i := big.NewInt(0)
					i.SetBytes(v)
					// Int64 should be large enough
					p.PublicKey.E = int(i.Int64())
				}
				if left == "PrivateExponent:" {
					p.D = big.NewInt(0)
					p.D.SetBytes(v)
				}
				if left == "Prime1:" {
					p.Primes[0] = big.NewInt(0)
					p.Primes[0].SetBytes(v)
				}
				if left == "Prime2:" {
					p.Primes[1] = big.NewInt(0)
					p.Primes[1].SetBytes(v)
				}
			case "Exponent1:", "Exponent2:", "Coefficient:":
				/* not used in Go (yet) */
			case "Created:", "Publish:", "Activate:":
				/* not used in Go (yet) */
			default:
				return nil, ErrKey
			}
		}
		line, _, err = r.ReadLine()
	}
	if !k.setPublicKeyRSA(p.PublicKey.E, p.PublicKey.N) {
		return nil, ErrKey
	}
	return p, nil
}