func webQuitHandler(diskwriter *csv.Writer, gzipwriter *gzip.Writer, csvfile *os.File) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "flushing to disk and shutting down")
diskwriter.Flush()
if gzipwriter != nil {
gzipwriter.Flush()
gzipwriter.Close()
}
csvfile.Close()
os.Exit(0)
}
}
func getPubmedRecords(urlFetcher *gopubmed.Fetcher, first bool, meshWriter *gzip.Writer, xmlWriter *gzip.Writer, transport *http.Transport, pmids []string) {
preUrlTime := time.Now()
articles, raw, err := urlFetcher.GetArticlesAndRaw(pmids)
if err != nil {
log.Fatal(err)
}
s := string(raw[:len(raw)])
for i := 0; i < len(articles); i++ {
pubmedArticle := articles[i]
if pubmedArticle.MedlineCitation != nil && pubmedArticle.MedlineCitation.MeshHeadingList != nil && pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading != nil {
fmt.Fprint(meshWriter, articles[i].MedlineCitation.PMID.Text)
for j := 0; j < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading); j++ {
fmt.Fprint(meshWriter, "|")
fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Attr_UI)
fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Text)
if len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName) > 0 {
fmt.Fprint(meshWriter, "=")
for q := 0; q < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName); q++ {
if q != 0 {
fmt.Fprint(meshWriter, "&")
}
fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Attr_UI)
fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Text)
}
}
}
fmt.Fprintln(meshWriter, "")
}
}
meshWriter.Flush()
if !first {
s = strings.Replace(s, startXml, "", -1)
s = strings.Replace(s, docType, "", -1)
s = strings.Replace(s, startPubmedArticleSet, "", -1)
}
s = strings.Replace(s, endPubmedArticleSet, "<!-- breakset -->", -1)
xmlWriter.Write([]byte(s))
postUrlTime := time.Now()
log.Println("Total request time:", postUrlTime.Sub(preUrlTime))
}
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
tr := &http.Transport{
ResponseHeaderTimeout: time.Second * 500,
DisableKeepAlives: false,
DisableCompression: false,
}
var wXml *gzip.Writer = nil
var ww *bufio.Writer = nil
var xFile *os.File = nil
meshFile, err2 := os.Create(meshFile)
if err2 != nil {
return
}
defer meshFile.Close()
wwMesh := bufio.NewWriter(meshFile)
wMesh := gzip.NewWriter(wwMesh)
//w := bufio.NewWriter(file)
numIdsPerUrl := findNumIdsPerUrl()
pmids := make([]string, numIdsPerUrl)
urlFetcher := gopubmed.Fetcher{
Ssl: false,
Transport: &http.Transport{
ResponseHeaderTimeout: time.Second * 500,
DisableKeepAlives: false,
DisableCompression: false,
//TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
allCount := 0
count := 0
reader, err := makeReader()
if err != nil {
log.Fatal(err)
}
first := true
chunkCount := 0
for {
thisNumIdsPerUrl := findNumIdsPerUrl()
if numIdsPerUrl != thisNumIdsPerUrl {
numIdsPerUrl = thisNumIdsPerUrl
pmids = make([]string, numIdsPerUrl)
}
line, err := reader.ReadString('\n')
if err != nil {
// You may check here if err == io.EOF
break
}
line = strings.TrimSpace(line)
err = lineChecker(line)
if err != nil {
log.Fatal(err)
}
//log.Println(line)
pmids[count] = line
if wXml == nil {
wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
}
count = count + 1
// Collected enough pmids: get their XML from NIH
if count == numIdsPerUrl {
getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
checkTime()
first = false
count = 0
zeroArray(pmids)
} else {
}
allCount += 1
chunkCount += 1
// Start new xml file: close old one: open new one
if chunkCount > recordsPerFile {
fmt.Fprintln(wXml, endPubmedArticleSet)
wXml.Flush()
wXml.Close()
ww.Flush()
wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
chunkCount = 0
first = true
}
if allCount%500 == 0 {
log.Println(allCount)
}
}
if count != 0 {
getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
}
fmt.Fprintln(wXml, endPubmedArticleSet)
wXml.Flush()
wXml.Close()
ww.Flush()
wMesh.Flush()
wwMesh.Flush()
wMesh.Close()
xFile.Close()
}
func (h *handler) sendContinuousChangesByWebSocket(inChannels base.Set, options db.ChangesOptions) (error, bool) {
forceClose := false
handler := func(conn *websocket.Conn) {
h.logStatus(101, "Upgraded to WebSocket protocol")
defer func() {
conn.Close()
base.LogTo("HTTP+", "#%03d: --> WebSocket closed", h.serialNumber)
}()
// Read changes-feed options from an initial incoming WebSocket message in JSON format:
var compress bool
if msg, err := readWebSocketMessage(conn); err != nil {
return
} else {
var channelNames []string
var err error
if _, options, _, channelNames, _, compress, err = h.readChangesOptionsFromJSON(msg); err != nil {
return
}
if channelNames != nil {
inChannels, _ = channels.SetFromArray(channelNames, channels.ExpandStar)
}
}
// Set up GZip compression
var writer *bytes.Buffer
var zipWriter *gzip.Writer
if compress {
writer = bytes.NewBuffer(nil)
zipWriter = GetGZipWriter(writer)
}
caughtUp := false
_, forceClose = h.generateContinuousChanges(inChannels, options, func(changes []*db.ChangeEntry) error {
var data []byte
if changes != nil {
data, _ = json.Marshal(changes)
} else if !caughtUp {
caughtUp = true
data, _ = json.Marshal([]*db.ChangeEntry{})
} else {
data = []byte{}
}
if compress && len(data) > 8 {
// Compress JSON, using same GZip context, and send as binary msg:
zipWriter.Write(data)
zipWriter.Flush()
data = writer.Bytes()
writer.Reset()
conn.PayloadType = websocket.BinaryFrame
} else {
conn.PayloadType = websocket.TextFrame
}
_, err := conn.Write(data)
return err
})
if zipWriter != nil {
ReturnGZipWriter(zipWriter)
}
}
server := websocket.Server{
Handshake: func(*websocket.Config, *http.Request) error { return nil },
Handler: handler,
}
server.ServeHTTP(h.response, h.rq)
return nil, forceClose
}
func (p Handler) processText(s *Session, w http.ResponseWriter, resp *http.Response) (err error) {
var (
zr *gzip.Reader
zw *gzip.Writer
body []byte
gzipped bool = resp.Header.Get("Content-Encoding") == "gzip"
reqHost string = resp.Request.URL.Host
reqPath string = resp.Request.URL.Path
)
if resp.ContentLength != 0 && resp.Request.Method != "HEAD" {
if gzipped {
zr, err = gzip.NewReader(resp.Body)
if err == nil {
body, err = ioutil.ReadAll(zr)
if !consumeError(&err) {
return dumpError(err)
}
}
} else {
body, err = ioutil.ReadAll(resp.Body)
if !consumeError(&err) {
return dumpError(err)
}
}
}
w.Header().Del("Content-Length")
w.Header().Set("Content-Encoding", "gzip")
w.WriteHeader(resp.StatusCode)
if len(body) <= 0 {
return
}
var (
rules []ReRule
bodyExtraHeader string
)
switch p {
case HD_html:
rules = reRules.Html
case HD_javascript:
rules = reRules.Js
case HD_json:
rules = reRules.Json
case HD_css:
rules = reRules.Css
}
if log.V(5) {
log.Infof("Original entity %s\n%s", reqPath, string(body))
}
if s.abusing {
imgSrc := fmt.Sprintf(`<img src="/!%s/sorry`, reqHost)
body = bytes.Replace(body, []byte(`<img src="/sorry`), []byte(imgSrc), 1)
rules = nil
}
for i, r := range rules {
if r.PathRe != nil && r.PathRe.FindString(reqPath) == NULL {
if log.V(4) {
log.Infof("re.%d=[%s] pathRe=deny", i, r.ContentPattern.Pattern)
}
continue
}
if log.V(4) {
log.Infof("re.%d=[%s] applied", i, r.ContentPattern.Pattern)
}
if r.Scheme&0xff > 0 {
body = r.ContentRe.Replace(body, r.Replacement)
}
if r.Scheme&0xff00 > 0 {
bodyExtraHeader += r.InsertHeader
}
}
zw = gzip.NewWriter(w)
if len(bodyExtraHeader) > 0 {
zw.Write([]byte(bodyExtraHeader))
}
zw.Write(body)
err = zw.Flush()
return
}
func (p *plug) daemon() {
var diskwriter *csv.Writer
var gzipwriter *gzip.Writer
fmt.Println("starting foreground daemon ;-)")
// write csv from disk into the buffer
fmt.Println("loading history (" + p.csvfile + ")")
p.buffer.Write(readcsv(p.csvfile))
// create/append the csvfile on disk
csvfile, err := os.OpenFile(p.csvfile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
log.Fatal("Error:", err)
}
defer csvfile.Close()
// create a bufferwriter (appends to csv already in p.buffer)
bufferwriter := csv.NewWriter(&p.buffer)
// compressed or not
if strings.Contains(p.csvfile, ".gz") {
gzipwriter, _ = gzip.NewWriterLevel(csvfile, gzip.BestCompression)
defer gzipwriter.Close()
// wrap csv around gzipwriter
diskwriter = csv.NewWriter(gzipwriter)
} else {
// create a diskwriter (appends to csv on disk)
diskwriter = csv.NewWriter(csvfile)
}
// connect via telnet to the device and login
conn, err := p.DialTimeout("tcp", p.device, time.Duration(time.Second*30))
if err != nil {
log.Fatal("can't connect")
}
// create http handlers
http.HandleFunc("/quit", webQuitHandler(diskwriter, gzipwriter, csvfile))
http.HandleFunc("/history", webHistoryHandler)
http.HandleFunc("/stream", webStreamHandler)
http.HandleFunc("/read.csv", webReadCsvHandler(p))
http.HandleFunc("/read.json", webReadJsonHandler(p))
// needed for occasionally flushing on a newline
recordcount := 0
// start infinite polling loop
for {
// measure how long it takes
start := time.Now()
// specify correct format for dygraph
record := []string{start.Format("2006/01/02 15:04:05")}
// get statistics from device and cleanup
status := sendln(conn, plugGetInfoStats, '#')
status = strings.Replace(status, plugGetInfoStats+"\r\n", "", 1)
status = strings.Replace(status, "#", "", 1)
// split up the 4 results a newline
results := strings.SplitN(status, "\r\n", 4)
re := regexp.MustCompile("01(I|V|W|E)[0-9]+ 0*([0-9]+)")
// for each GetInfo result, do a regexp match, adjust value and create a CSV record
for i, result := range results {
match := re.FindStringSubmatch(result)
value := "0"
// check if we got the right size of slice
if len(match) == 3 {
value = match[2]
}
temp, _ := strconv.ParseFloat(value, 32)
switch i {
case 0:
// mAmp/10 -> Amp
value = strconv.FormatFloat(temp/10000, 'f', 2, 32)
// centiWatt -> Watt
case 1:
value = strconv.FormatFloat(temp/100, 'f', 2, 32)
// mWatt/h -> Watt/h | mVolt -> Volt
case 2, 3:
value = strconv.FormatFloat(temp/1000, 'f', 2, 32)
}
record = append(record, value)
recordcount += 1
}
// latestentry is needed in JSON for the realtime streaming
p.latestEntry, _ = json.Marshal(record)
// write the record to disk
err := diskwriter.Write(record)
if err != nil {
fmt.Println("Error:", err)
}
// write the record to buffer (in memory)
err = bufferwriter.Write(record)
if err != nil {
fmt.Println("Error:", err)
}
// flush disk every 25 records
if recordcount%100 == 0 {
diskwriter.Flush()
if strings.Contains(p.csvfile, ".gz") {
gzipwriter.Flush()
}
}
// flush memory immediately
bufferwriter.Flush()
if debug {
fmt.Print(record)
fmt.Println(" took", time.Since(start))
}
// sleep the right amount of time
time.Sleep(time.Second*time.Duration(p.delay) - time.Since(start))
}
}
func NewEventHandler(buildsDB BuildsDB, buildID int) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
flusher := w.(http.Flusher)
closed := w.(http.CloseNotifier).CloseNotify()
w.Header().Add("Content-Type", "text/event-stream; charset=utf-8")
w.Header().Add("Cache-Control", "no-cache, no-store, must-revalidate")
w.Header().Add("Connection", "keep-alive")
w.Header().Add(ProtocolVersionHeader, CurrentProtocolVersion)
var start uint = 0
if r.Header.Get("Last-Event-ID") != "" {
_, err := fmt.Sscanf(r.Header.Get("Last-Event-ID"), "%d", &start)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
start++
}
var responseWriter io.Writer = w
var responseFlusher *gzip.Writer
w.Header().Add("Vary", "Accept-Encoding")
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
gz := gzip.NewWriter(w)
defer gz.Close()
responseWriter = gz
responseFlusher = gz
}
events, err := buildsDB.GetBuildEvents(buildID, start)
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
defer events.Close()
es := make(chan atc.Event)
errs := make(chan error, 1)
go func() {
for {
ev, err := events.Next()
if err != nil {
errs <- err
return
} else {
select {
case es <- ev:
case <-closed:
return
}
}
}
}()
for {
select {
case ev := <-es:
payload, err := json.Marshal(event.Message{ev})
if err != nil {
return
}
err = sse.Event{
ID: fmt.Sprintf("%d", start),
Name: "event",
Data: payload,
}.Write(responseWriter)
if err != nil {
return
}
start++
if responseFlusher != nil {
err = responseFlusher.Flush()
if err != nil {
return
}
}
flusher.Flush()
case err := <-errs:
if err == db.ErrEndOfBuildEventStream {
err = sse.Event{Name: "end"}.Write(responseWriter)
if err != nil {
return
}
}
return
case <-closed:
return
}
}
return
})
}
