Example #1
1
//Skip skip
func (t *TTFParser) Skip(fd *bytes.Reader, length int) error {
	_, err := fd.Seek(int64(length), 1)
	if err != nil {
		return err
	}
	return nil
}
Example #2
1
//  degob converts a gob into a new copy of a subtree.
func degob(buf *bytes.Reader) Node {
	var tree Node
	buf.Seek(0, 0)
	dec := gob.NewDecoder(buf)
	CkErr(dec.Decode(&tree))
	return tree
}
Example #3
1
func UnmarshalStream(url string, reader io.Reader) (feed *Feed, err error) {
	// Read the stream into memory (we'll need to parse it twice)
	var contentReader *bytes.Reader
	var content []byte
	if content, err = ioutil.ReadAll(reader); err == nil {
		contentReader = bytes.NewReader(content)

		genericFeed := GenericFeed{}

		decoder := xml.NewDecoder(contentReader)
		decoder.CharsetReader = charset.NewReader

		// First pass - parse the feed as-is
		if err = decoder.Decode(&genericFeed); err != nil {
			// Error - check for invalid entities and correct as appropriate
			if fixed, fixedContent := fixEntities(content); fixed {
				// At least one replacement was made. Retry
				contentReader = bytes.NewReader(fixedContent)
				decoder = xml.NewDecoder(contentReader)
				decoder.CharsetReader = charset.NewReader

				// Try decoding again
				err = decoder.Decode(&genericFeed)
			}
		}

		if err != nil {
			return
		}

		var xmlFeed FeedMarshaler
		if genericFeed.XMLName.Space == "http://www.w3.org/1999/02/22-rdf-syntax-ns#" && genericFeed.XMLName.Local == "RDF" {
			xmlFeed = &rss1Feed{}
		} else if genericFeed.XMLName.Local == "rss" {
			xmlFeed = &rss2Feed{}
		} else if genericFeed.XMLName.Space == "http://www.w3.org/2005/Atom" && genericFeed.XMLName.Local == "feed" {
			xmlFeed = &atomFeed{}
		} else {
			err = errors.New("Unsupported type of feed (" +
				genericFeed.XMLName.Space + ":" + genericFeed.XMLName.Local + ")")
			return
		}

		contentReader.Seek(0, 0)

		decoder = xml.NewDecoder(contentReader)
		decoder.CharsetReader = charset.NewReader

		if err = decoder.Decode(xmlFeed); err == nil {
			if feed, err = xmlFeed.Marshal(); err == nil {
				feed.URL = url
			}
		}
	}

	return
}
Example #4
1
func copyResp(code int, headers http.Header, body *bytes.Reader, newRw http.ResponseWriter) {
	h := newRw.Header()

	for k, v := range headers {
		h[k] = append(h[k], v...)
	}

	newRw.WriteHeader(code)
	body.WriteTo(newRw)
	body.Seek(0, 0)
}
Example #5
1
//Seek seek by tag
func (t *TTFParser) Seek(fd *bytes.Reader, tag string) error {
	table, ok := t.tables[tag]
	if !ok {
		return ErrTableNotFound
	}
	val := table.Offset
	_, err := fd.Seek(int64(val), 0)
	if err != nil {
		return err
	}
	return nil
}
Example #6
1
func processQueries(
	db *cablastp.DB, transQueries *bytes.Reader, searchBuf *bytes.Buffer) error {
	// now we will read from queryBuf!
	// I think we create a NewReader from queryBuf?
	// this now needs to become the replacement for inputFastaQuery
	// so must use a different buffer for that.
	// we need a buffer for the query trans/reduce
	// and a buffer for coarse blast results

	cablastp.Vprintln("\nBlasting query on coarse database...")
	err := blastCoarse(db, transQueries, searchBuf)
	handleFatalError("Error blasting coarse database", err)

	cablastp.Vprintln("Decompressing blast hits...")
	expandedSequences, err := expandBlastHits(db, searchBuf)
	handleFatalError("Error decompressing blast hits", err)
	if len(expandedSequences) == 0 {
		cablastp.Vprintln("No results from coarse search")
	} else {

		// Write the contents of the expanded sequences to a fasta file.
		// It is then indexed using makeblastdb.
		searchBuf.Reset()
		err = writeFasta(expandedSequences, searchBuf)
		handleFatalError("Could not create FASTA input from coarse hits", err)

		// Create the fine blast db in a temporary directory
		cablastp.Vprintln("Building fine BLAST database...")
		tmpDir, err := makeFineBlastDB(db, searchBuf)
		handleFatalError("Could not create fine database to search on", err)

		// retrieve the cluster members for the original representative query seq

		// pass them to blastx on the expanded (fine) db

		// Finally, run the query against the fine fasta database and pass on the
		// stdout and stderr...
		cablastp.Vprintln("Blasting query on fine database...")
		_, err = transQueries.Seek(0, 0) // First 0 is amount to offset, Second 0
		// is code for absolute
		handleFatalError("Could not seek to start of query fasta input", err)

		err = blastFine(db, tmpDir, transQueries)
		handleFatalError("Error blasting fine database", err)
		// Delete the temporary fine database.
		if !flagNoCleanup {
			err := os.RemoveAll(tmpDir)
			handleFatalError("Could not delete fine BLAST database", err)
		}
	}
	return nil
}
Example #7
1
File: feed.go Project: huluwa/grr
func UnmarshalStream(reader io.Reader) (*Feed, string, error) {
	format := ""

	// Read the stream into memory (we'll need to parse it twice)
	var contentReader *bytes.Reader
	if buffer, err := ioutil.ReadAll(reader); err == nil {
		contentReader = bytes.NewReader(buffer)
	} else {
		return nil, format, err
	}

	genericFeed := GenericFeed{}

	decoder := xml.NewDecoder(contentReader)
	decoder.CharsetReader = charsetReader

	if err := decoder.Decode(&genericFeed); err != nil {
		return nil, format, err
	}

	var xmlFeed FeedMarshaler

	if genericFeed.XMLName.Space == "http://www.w3.org/1999/02/22-rdf-syntax-ns#" && genericFeed.XMLName.Local == "RDF" {
		xmlFeed = &rss1Feed{}
		format = "RSS1"
	} else if genericFeed.XMLName.Local == "rss" {
		xmlFeed = &rss2Feed{}
		format = "RSS2"
	} else if genericFeed.XMLName.Space == "http://www.w3.org/2005/Atom" && genericFeed.XMLName.Local == "feed" {
		xmlFeed = &atomFeed{}
		format = "Atom"
	} else {
		return nil, format, errors.New("Unsupported type of feed (" +
			genericFeed.XMLName.Space + ":" + genericFeed.XMLName.Local + ")")
	}

	contentReader.Seek(0, 0)

	decoder = xml.NewDecoder(contentReader)
	decoder.CharsetReader = charsetReader

	if err := decoder.Decode(xmlFeed); err != nil {
		return nil, format, err
	}

	feed, err := xmlFeed.Marshal()
	if err != nil {
		return nil, format, err
	}

	return &feed, format, nil
}
Example #8
0
func testZipArc(f *bytes.Reader) {
	za, err := New(&reader{f}, f.Size())
	if err != nil {
		panic(err)
	}

	compareFile(za, "a.txt", "This is a file.\n")
	compareFile(za, "k/l/m/a.txt", "k-l-m-a!\n")
	compareFile(za, "z.txt", "This is also a file.\n")

	d, err := za.Open("a")
	if err != nil {
		panic(err)
	}

	dfi, err := d.Readdir(0)
	if err != nil {
		panic(err)
	}

	h := sha256.New()
	for _, x := range dfi {
		s := fmt.Sprintf("%v\t%v\t%v\t%v\n", x.Name(), x.IsDir(), x.Mode(), x.Size())
		h.Write([]byte(s))
	}
	hs := hex.EncodeToString(h.Sum(nil))
	if hs != "93d5c5b0a7a5205b5ad687a8726dbec1a355a3937151ca96f7e704675ae1e536" {
		panic("hash mismatch: " + hs)
	}
}
Example #9
0
// HandleLog is the default http log handler
func HandleLog(h HTTP, entries <-chan *log.Entry) {
	var e *log.Entry
	var b []byte
	var reader *bytes.Reader

	formatter := h.FormatFunc()(h)
	remoteHost := h.RemoteHost()
	httpClient := stdhttp.Client{}

	req, _ := stdhttp.NewRequest(h.Method(), remoteHost, nil)
	req.Header = h.Headers()

	for e = range entries {

		b = formatter(e)

		reader = bytes.NewReader(b)
		req.Body = ioutil.NopCloser(reader)
		req.ContentLength = int64(reader.Len())

		resp, err := httpClient.Do(req)
		if err != nil {
			log.Error("Could not post data to %s: %v\n", remoteHost, err)
			goto END
		}

		if resp.StatusCode < 200 || resp.StatusCode >= 299 {
			bt, _ := ioutil.ReadAll(resp.Body)
			log.Error("Received HTTP %d during POST request to %s body: %s\n", resp.StatusCode, remoteHost, string(bt))
		}

	END:
		e.Consumed()
	}
}
Example #10
0
func _readLong3(file *bytes.Reader, offset int64) int64 {
	buf := make([]byte, 4)
	file.ReadAt(buf, offset)
	buf[3] = 0x00

	return int64(binary.LittleEndian.Uint32(buf))
}
Example #11
0
func readRune(r *bytes.Reader) rune {
	n, s, err := r.ReadRune()
	if n == _EOF_ || s == 0 || err != nil {
		return _EOF_
	}
	return n
}
Example #12
0
func installZip(source *bytes.Reader, dest string) error {
	zr, err := zip.NewReader(source, int64(source.Len()))
	if err != nil {
		return err
	}

	for _, f := range zr.File {
		fileCopy, err := os.OpenFile(dest, installFlag, f.Mode())
		if err != nil {
			return err
		}
		defer fileCopy.Close()

		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer rc.Close()

		_, err = io.Copy(fileCopy, rc)
		if err != nil {
			return err
		}
	}

	return nil
}
Example #13
0
func (k *KeenIoMetrics) AuthedRequest(method, path string, body *bytes.Reader) (resp *http.Response, err error) {
	path = fmt.Sprintf("https://api.keen.io/3.0/projects/%s%s", k.ProjectToken, path)
	req, err := http.NewRequest(method, path, body)
	if err != nil {
		return
	}

	req.Header.Add("Authorization", k.ApiKey)

	if body != nil {
		req.Header.Add("Content-Type", "application/json")
		req.ContentLength = int64(body.Len())
	}

	resp, err = k.HttpClient.Do(req)

	if err != nil {
		k.Error("Failed to send metric event to keen.io %v", err)
	} else {
		defer resp.Body.Close()
		if resp.StatusCode != 201 {
			bytes, _ := ioutil.ReadAll(resp.Body)
			k.Error("Got %v response from keen.io: %s", resp.StatusCode, bytes)
		}
	}

	return
}
Example #14
0
func (this *HSPerfData) readEntryValueAsLong(reader *bytes.Reader, StartOfs int64, entry *PerfDataEntry) error {
	reader.Seek(StartOfs+int64(entry.DataOffset), os.SEEK_SET)
	reader.Read(this.globalbuf[:8])
	entry.LongValue = int64(this.byteOrder.Uint64(this.globalbuf[:8]))

	return nil
}
Example #15
0
func readBytes(f *bytes.Reader, len int) ([]byte, error) {
	b := make([]byte, len)
	_, err := f.Read(b)
	if err != nil {
		return nil, err
	}
	return b, nil
}
Example #16
0
func ReadMac(reader *bytes.Reader) uint64 {
	mac_byte := make([]byte, 6)
	reader.Read(mac_byte)
	mac := []byte{0, 0}
	mac = append(mac, mac_byte...)

	return binary.BigEndian.Uint64(mac)
}
Example #17
0
File: text.go Project: h8liu/d8
func UnpackString(in *bytes.Reader, n uint16) (Text, error) {
	buf := make([]byte, n)
	_, e := in.Read(buf)
	if e != nil {
		return "", e
	}
	return Text(string(buf)), nil
}
Example #18
0
File: bytes.go Project: h8liu/d8
func UnpackBytes(in *bytes.Reader, n uint16) (Bytes, error) {
	ret := make([]byte, n)
	if _, e := in.Read([]byte(ret)); e != nil {
		return nil, e
	}

	return Bytes(ret), nil
}
Example #19
0
func (section_header *Section_Header) Verify(datar *bytes.Reader) bool {
	var buf []byte

	datar.Read(buf)
	fmt.Println(section_header.Checksum, len(buf))
	return section_header.Checksum == adler32.Checksum(buf[:72])

}
Example #20
0
func (sitemap *SitemapPage) parseSitemapPageNoGzip(mem_seek *bytes.Reader) (byte_reader *bytes.Reader, err error) {
	// validate uncompressed size
	if mem_seek.Len() > max_sitemap_page_size {
		return mem_seek, max_sitemap_page_size_error
	}

	err = sitemap.determineIteratorFormat(mem_seek)
	return mem_seek, err
}
Example #21
0
func render_line(line *bytes.Reader, h int) {
	m_x, _ := termbox.Size()
	for x := 0; x <= m_x; x++ {
		if r, _, err := line.ReadRune(); err == nil {
			termbox.SetCell(x, h, r, termbox.ColorBlack, termbox.ColorWhite)
		} else {
			termbox.SetCell(x, h, ' ', termbox.ColorBlack, termbox.ColorWhite)
		}
	}
}
Example #22
0
File: question.go Project: h8liu/d8
func (self *Question) unpackFlags(in *bytes.Reader) error {
	buf := make([]byte, 4)
	if _, e := in.Read(buf); e != nil {
		return e
	}

	self.Type = enc.Uint16(buf[0:2])
	self.Class = enc.Uint16(buf[2:4])

	return nil
}
Example #23
0
func ReadString(r *bytes.Reader) string {
	length, _ := binary.ReadUvarint(r)
	if length < 1 {
		return ""
	} else {
		buf := make([]byte, length)
		r.Read(buf)

		return string(buf)
	}
}
Example #24
0
File: rr.go Project: h8liu/d8
func (self *RR) unpackFlags(in *bytes.Reader) error {
	var buf [8]byte
	if _, e := in.Read(buf[:]); e != nil {
		return e
	}
	self.Type = enc.Uint16(buf[0:2])
	self.Class = enc.Uint16(buf[2:4])
	self.TTL = enc.Uint32(buf[4:8])

	return nil
}
Example #25
0
File: ipv6.go Project: h8liu/d8
func UnpackIPv6(in *bytes.Reader, n uint16) (IPv6, error) {
	if n != 16 {
		return nil, fmt.Errorf("IPv6 with %d bytes", n)
	}
	buf := make([]byte, 16)
	_, e := in.Read(buf)
	if e != nil {
		return nil, e
	}

	return IPv6(buf), nil
}
Example #26
0
//Read read
func (t *TTFParser) Read(fd *bytes.Reader, length int) ([]byte, error) {
	buff := make([]byte, length)
	readlength, err := fd.Read(buff)
	if err != nil {
		return nil, err
	}
	if readlength != length {
		return nil, errors.New("file out of length")
	}
	//fmt.Printf("%d,%s\n", readlength, string(buff))
	return buff, nil
}
Example #27
0
func (self *Entity) Store(
	storageObj *data.Storage,
	objectObj *data.Object,
	input *bytes.Reader,
) error {
	ctx := self.Ctx()

	closer := ctx.LogMark("[Entity.Store]")
	defer closer()

	uri := strings.Join([]string{storageObj.Uri, objectObj.InternalName}, "/")
	cl := input.Len()

	ctx.Debugf("Going to store %d bytes in %s", cl, uri)

	req, err := http.NewRequest("PUT", uri, input)
	if err != nil {
		ctx.Debugf("Failed to create request: %s", err)
		return err
	}

	// XXX Need to check if this vanilla http client is ok
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		ctx.Debugf("Failed to send PUT request to %s (storage = %d): %s", uri, storageObj.Id, err)
		return err
	}

	if resp.StatusCode != 201 {
		err = errors.New(
			fmt.Sprintf(
				"Expected response 201 for PUT request, but did not get it: %s",
				resp.Status,
			),
		)
		ctx.Debugf("Failed to store PUT request to %s (storage = %d): %s", uri, storageObj.Id, err)
		return err
	}

	ctx.Debugf("Successfully stored object in %s", uri)

	err = self.Create(
		objectObj.Id,
		storageObj.Id,
	)

	if err != nil {
		return err
	}

	return nil
}
Example #28
0
File: ipv4.go Project: h8liu/d8
func UnpackIPv4(in *bytes.Reader, n uint16) (IPv4, error) {
	if n != 4 {
		return nil, fmt.Errorf("IPv4 with %d bytes", n)
	}

	buf := make([]byte, 4)
	_, e := in.Read(buf)
	if e != nil {
		return nil, e
	}

	return IPv4(net.IPv4(buf[0], buf[1], buf[2], buf[3])), nil
}
Example #29
0
func (p *Pattern) parseTracks(r *bytes.Reader) error {
	for r.Len() > 0 {
		t := &Track{}
		err := t.parse(r)
		if err != nil {
			fmt.Println("parse Track failed:", err)
			return err
		}
		p.Tracks = append(p.Tracks, *t)
	}

	return nil
}
Example #30
0
func (h *HTTPSender) buildBaseRequest(contextPath string, method string, headers map[string][]string, bodyReader *bytes.Reader) *http.Request {
	var req http.Request
	req.Method = h.method
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = false
	req.Header = h.headers
	req.URL = h.parsedContextPath
	req.URL.Host = h.hosts[h.currentHost]
	req.Body = ioutil.NopCloser(bodyReader)
	req.ContentLength = int64(bodyReader.Len())
	return &req
}