func addEmails(reason string, emailsReader io.Reader) {
scanner := bufio.NewScanner(emailsReader)
emailsFile, emailsFileErr := os.OpenFile("./emails.txt", os.O_APPEND|os.O_WRONLY, 0600)
if emailsFileErr != nil {
fmt.Println(emailsFileErr.Error())
}
defer emailsFile.Close()
emailsListFile, emailsListFileErr := os.OpenFile("./emailsList.txt", os.O_APPEND|os.O_WRONLY, 0600)
if emailsListFileErr != nil {
fmt.Println(emailsListFileErr.Error())
}
defer emailsListFile.Close()
emailsWritter := bufio.NewWriter(emailsFile)
emailsListWritter := bufio.NewWriter(emailsListFile)
for scanner.Scan() {
// TODO refactor r:reason,e:email
email := scanner.Text()
emailLine := "reason:" + reason + ",email:" + email + "\n"
_, writeEmailsErr := emailsWritter.WriteString(emailLine)
if writeEmailsErr != nil {
fmt.Println(writeEmailsErr.Error())
}
_, writeEmailsListErr := emailsListWritter.WriteString(email + "\n")
if writeEmailsListErr != nil {
fmt.Println(writeEmailsListErr.Error())
}
}
emailsWritter.Flush()
emailsListWritter.Flush()
}
func Trim() {
var inputFile, outputFile string
var mean, window, minLength int
flags := flag.NewFlagSet("trim", flag.ExitOnError)
flags.StringVar(&inputFile, "input", "", "Name of the fastq input file. Otherwise STDIN.")
flags.StringVar(&outputFile, "output", "", "Name of the output file. Otherwise STDOUT.")
flags.IntVar(&mean, "mean", 20, "Mean quality of output sequences.")
flags.IntVar(&window, "window", 50, "Size of trailing window.")
flags.IntVar(&minLength, "length", 50, "Minimum sequence length.")
flags.Parse(os.Args[2:])
input := bufio.NewReader(os.Stdin)
output := bufio.NewWriter(os.Stdout)
if inputFile != "" {
file, err := os.Open(inputFile)
if err != nil {
panic("Cannot open specified input file: " + inputFile)
}
input = bufio.NewReader(file)
}
if outputFile != "" {
file, err := os.Create(outputFile)
if err != nil {
panic("Cannot create specified output file: " + outputFile)
}
output = bufio.NewWriter(file)
}
qtrim.TrimIO(input, output, mean, window, minLength)
}
func sendCommand(command, args string, connector models.Connector) string {
port := 5985
if connector.Port != "" {
port, _ = strconv.Atoi(connector.Port)
}
endpoint := winrm.NewEndpoint(connector.Server, port, false, false, nil, nil, nil, 0)
rmclient, err := winrm.NewClient(endpoint, connector.Login, connector.Pass)
if err != nil {
log.Println("Error connecting to endpoint:", err)
return "Error connecting to endpoint: " + err.Error()
}
var in bytes.Buffer
var out bytes.Buffer
var e bytes.Buffer
stdin := bufio.NewReader(&in)
stdout := bufio.NewWriter(&out)
stderr := bufio.NewWriter(&e)
_, err = rmclient.RunWithInput(command, stdout, stderr, stdin)
if err != nil {
return "Error running command: " + err.Error()
}
if e.String() != "" {
return e.String()
}
return out.String()
}
func RenderDetail(shdarray []*DP, t *template.Template, sh string) error {
var hisName = time.Now().Format("2006-01-02")
dshfilename := homeDir + "/changshi/today/detail_" + sh + ".html"
if _, err := os.Stat(dshfilename); os.IsExist(err) {
os.Remove(dshfilename)
}
dshf, _ := os.Create(dshfilename)
dshfbw := bufio.NewWriter(dshf)
err := t.ExecuteTemplate(dshfbw, "detail.tmpl", shdarray)
if err != nil {
return err
}
dshfbw.Flush()
dshf.Close()
dshhfilename := homeDir + "/changshi/his/d_" + sh + "_" + hisName + ".html"
if _, err := os.Stat(dshhfilename); os.IsExist(err) {
os.Remove(dshhfilename)
}
dshhf, _ := os.Create(dshhfilename)
dshhfbw := bufio.NewWriter(dshhf)
err1 := t.ExecuteTemplate(dshhfbw, "detail.tmpl", shdarray)
if err1 != nil {
return err
}
dshhfbw.Flush()
dshhf.Close()
return nil
}
func writeOutput(data []byte) {
var output *bufio.Writer
// prepare "output"
if *out != "" {
file, err := os.Open(*out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
checkOutputFailure(err)
defer file.Close()
output = bufio.NewWriter(file)
} else {
output = bufio.NewWriter(os.Stdout)
}
// write package clause if any
if *pkg != "" {
_, err := fmt.Fprintf(output, "package %s\n\n", *pkg)
checkOutputFailure(err)
}
// write data
writeData(data, output)
// flush
err := output.Flush()
checkOutputFailure(err)
}
// A unsubscribing client should not be considered for message delivery
func TestQueue_sendMessageAfterUnsubscribe_messageReceivedSuccessfully(t *testing.T) {
// Need gomega for async testing
gomega.RegisterTestingT(t)
testMessagePayload := []byte("Testing!")
expectedMessagePayload := []byte("Testing!\r\n.\r\n")
testMessage := message.NewHeaderlessMessage(&testMessagePayload)
dummyMetricsPipe := make(chan<- *Metric, 10)
dummyClosingPipe := make(chan<- *string)
underTest := newMessageQueue(TEST_QUEUE_NAME, dummyMetricsPipe, dummyClosingPipe)
writerBuffer1 := new(bytes.Buffer)
dummyWriter1 := bufio.NewWriter(writerBuffer1)
closedChannel1 := make(chan bool)
dummyClient1 := Client{Name: "Test1", Writer: dummyWriter1, Closed: &closedChannel1}
writerBuffer2 := new(bytes.Buffer)
dummyWriter2 := bufio.NewWriter(writerBuffer2)
closedChannel2 := make(chan bool)
dummyClient2 := Client{Name: "Test2", Writer: dummyWriter2, Closed: &closedChannel2}
// Add the subscription
underTest.AddSubscriber(&dummyClient1)
underTest.AddSubscriber(&dummyClient2)
// Queue the message
underTest.Publish(testMessage)
// Bit of a hack - only one of the subscribers will get the message,
// and we don't know which one
gomega.Eventually(func() []byte {
if writerBuffer1.String() == "" {
return writerBuffer2.Bytes()
} else {
return writerBuffer1.Bytes()
}
}).Should(gomega.Equal(expectedMessagePayload))
// We'll be reusing these buffers
writerBuffer1.Reset()
writerBuffer2.Reset()
// Close one client
*dummyClient1.Closed <- true
// Should remove the client from the map
gomega.Eventually(func() bool {
return underTest.subscribers[dummyClient1.Name] == nil
}).Should(gomega.BeTrue())
// Now send a message - the remaining client should receive it without issue
underTest.Publish(testMessage)
gomega.Eventually(func() []byte {
return writerBuffer2.Bytes()
}).Should(gomega.Equal(expectedMessagePayload))
}
func main() {
// parse command line arguments and populate op, res, params, and flags
initializeArgs()
// set up file handle for results
if *output != "" {
fo, err := os.Create(*output)
if err != nil {
log.Critical("%s", err)
}
defer func() {
if err := fo.Close(); err != nil {
log.Critical("%s", err)
}
}()
outfile = bufio.NewWriter(fo)
} else {
outfile = bufio.NewWriter(os.Stdout)
}
defer outfile.Flush()
// set up logging and initialize log
initializeLogger()
// parse config file and populate env data structure
initializeConfig()
// bundle up the request into a data structure that can be easily marshaled
createJSONRequest()
// POST the request to the api, receive the results, and process them
postRequest()
}
func main() {
proxy := goproxy.NewProxyHttpServer()
proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*baidu.com$"))).
HandleConnect(goproxy.AlwaysReject)
proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))).
HandleConnect(goproxy.AlwaysMitm)
// enable curl -p for all hosts on port 80
proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))).
HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
defer func() {
if e := recover(); e != nil {
ctx.Logf("error connecting to remote: %v", e)
client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
}
client.Close()
}()
clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
remote, err := net.Dial("tcp", req.URL.Host)
orPanic(err)
remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
for {
req, err := http.ReadRequest(clientBuf.Reader)
orPanic(err)
orPanic(req.Write(remoteBuf))
orPanic(remoteBuf.Flush())
resp, err := http.ReadResponse(remoteBuf.Reader, req)
orPanic(err)
orPanic(resp.Write(clientBuf.Writer))
orPanic(clientBuf.Flush())
}
})
proxy.Verbose = true
log.Fatal(http.ListenAndServe(":8080", proxy))
}
func TestNewWriter(t *testing.T) {
Convey("Testing newWriter()", t, func() {
tests := []struct {
name string
exp io.Writer
expStr string
}{
{
name: "vanilla",
exp: func() io.Writer {
var b bytes.Buffer
return bufio.NewWriter(&b)
}(),
expStr: "Es ist Krieg!",
},
}
for _, tt := range tests {
act := NewWriter()
Convey("Testing "+tt.name, func() {
So(act, ShouldResemble, tt.exp)
logIt(act, tt.expStr)
var b bytes.Buffer
want := bufio.NewWriter(&b)
io.Copy(want, strings.NewReader(tt.expStr))
So(act, ShouldResemble, want)
})
}
})
}
func generateProject(htmlSettings HtmlTemplateSettings, packageSettings PackageTemplateSettings) {
htmlTemplate, _ := Asset("client/index.html")
packageTemplate, _ := Asset("package.json")
htmlTempl := template.Must(template.New("").Parse(string(htmlTemplate)))
packageTempl := template.Must(template.New("").Parse(string(packageTemplate)))
appFolder := packageSettings.AppName
RestoreAssets(appFolder, "")
htmlFile, _ := os.Create(path.Join(appFolder, "client", "index.html"))
htmlFileWriter := bufio.NewWriter(htmlFile)
defer func() {
htmlFileWriter.Flush()
htmlFile.Close()
}()
packageFile, _ := os.Create(path.Join(appFolder, "package.json"))
packageFileWriter := bufio.NewWriter(packageFile)
defer func() {
packageFileWriter.Flush()
packageFile.Close()
}()
htmlTempl.Execute(htmlFileWriter, htmlSettings)
packageTempl.Execute(packageFileWriter, packageSettings)
}
func NewFile(fileName, fileType string, compress bool) (*File, error) {
fullName := fileName + fileType
if _, err := os.Stat(fullName); err == nil {
os.Rename(fullName, fileName+".01"+fileType)
fullName = fileName + ".02" + fileType
} else if _, err := os.Stat(fileName + ".01" + fileType); err == nil {
for fileId := 1; true; fileId++ {
fullName = fileName + fmt.Sprintf(".%02d", fileId) + fileType
if _, err := os.Stat(fullName); err != nil {
break
}
}
}
f, err := os.OpenFile(fullName, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
log := &File{f: f}
if compress {
log.bufio = bufio.NewWriter(log.f)
log.gzip = gzip.NewWriter(log.bufio)
log.json = json.NewEncoder(log.gzip)
} else {
log.bufio = bufio.NewWriter(log.f)
log.json = json.NewEncoder(log.bufio)
}
return log, nil
}
func (DS *MainSearch) initLogs(logdir string) {
// open logs
DS.logDir = logdir
os.Mkdir(DS.logDir, os.ModePerm)
tmpF0, err5 := os.Create(DS.logDir + "main:err.log")
if err5 != nil {
log.Fatal("couldn't create errs log", err5)
}
DS.errLogBuf = bufio.NewWriter(tmpF0)
DS.errLogBuf.Flush()
DS.errLog = log.New(DS.errLogBuf, "", log.LstdFlags)
tmpF1, err1 := os.Create(DS.logDir + "main:main.log")
if err1 != nil {
log.Fatal("couldn't create main log", err1)
}
DS.mainLogBuf = bufio.NewWriter(tmpF1)
DS.mainLogBuf.Flush()
DS.mainLog = log.New(DS.mainLogBuf, "", log.LstdFlags)
tmpF2, err2 := os.Create(DS.logDir + "main:eqns.log")
if err2 != nil {
log.Fatal("couldn't create eqns log", err2)
}
DS.eqnsLogBuf = bufio.NewWriter(tmpF2)
DS.eqnsLogBuf.Flush()
DS.eqnsLog = log.New(DS.eqnsLogBuf, "", log.LstdFlags)
}
// Dial connects to server at address
func (handler *ConnHandler) Dial(addr string) ([]Peer, error) {
var peers []Peer
conn, err := net.Dial("tcp", addr)
if err != nil {
log.Println(err)
return nil, err
}
writer := bufio.NewWriter(conn)
reader := bufio.NewReader(conn)
line, _ := reader.ReadString('\n')
line = strings.Trim(line, "\n")
addrs := strings.Split(line, ";")
info := strings.Split(addrs[0], ",")
writer.WriteString(handler.String() + "\n")
writer.Flush()
peers = append(peers, NewPeer(conn, conn, info[0], info[1]))
for _, a := range addrs[1:] {
info := strings.Split(a, ",")
c, _ := net.Dial("tcp", info[0])
writer = bufio.NewWriter(c)
writer.WriteString(handler.String() + "\n")
writer.Flush()
reader = bufio.NewReader(c)
line, _ = reader.ReadString('\n')
peers = append(peers, NewPeer(c, c, info[0], info[1]))
}
return peers, nil
}
func runShell(command *Command, args []string) bool {
r := bufio.NewReader(os.Stdin)
o := bufio.NewWriter(os.Stdout)
e := bufio.NewWriter(os.Stderr)
prompt := func() {
o.WriteString("> ")
o.Flush()
}
readLine := func() string {
ret, err := r.ReadString('\n')
if err != nil {
fmt.Fprint(e, err)
os.Exit(1)
}
return ret
}
execCmd := func(cmd string) int {
if cmd != "" {
o.WriteString(cmd)
}
return 0
}
cmd := ""
for {
prompt()
cmd = readLine()
execCmd(cmd)
}
return true
}
func main() {
addr, _ := net.ResolveTCPAddr("0.0.0.0:9009")
go func() {
l, _ := net.ListenTCP("tcp", addr)
conn, _ := l.AcceptTCP()
println("accepted")
go func(c io.ReadWriter) {
buf := make([]byte, N)
b := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
if n, e := b.Read(buf); e != nil || n < N {
println("read", n, e.String())
return
}
if n, e := b.Write(buf); n < N || e != nil {
println("write", n, e.String())
}
b.Flush()
time.Sleep(1)
}(conn)
}()
time.Sleep(1e9)
c, _ := net.DialTCP("tcp", nil, addr)
println("connected")
f := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
b := make([]byte, N)
if n, e := f.Write(b); n < N || e != nil {
panic("write failed")
}
f.Flush()
if n, e := f.Read(b); e != nil || n < N {
println("read 2", n, e.String())
}
}
func saveKeydir(root string, kd *keydir, fileId int32) error {
filename := fmt.Sprintf("%s_%d", keydirFilename, fileId)
f, err := os.Create(filepath.Join(root, filename))
if err != nil {
return err
}
defer f.Close()
bw := bufio.NewWriter(f)
defer bw.Flush()
sha1Bytes, err := writeKeydir(bw, kd)
if err != nil {
return err
}
filename = fmt.Sprintf("%s_%d", keydirSha1Filename, fileId)
fsha, err := os.Create(filepath.Join(root, filename))
if err != nil {
return err
}
defer fsha.Close()
bwsha := bufio.NewWriter(fsha)
defer bwsha.Flush()
_, err = bwsha.Write(sha1Bytes)
return err
}
func jwtConfigFromP12KeyFile(pth, email string) (*jwt.Config, error) {
cmd := cmdex.NewCommand("openssl", "pkcs12", "-in", pth, "-passin", "pass:notasecret", "-nodes")
var outBuffer bytes.Buffer
outWriter := bufio.NewWriter(&outBuffer)
cmd.SetStdout(outWriter)
var errBuffer bytes.Buffer
errWriter := bufio.NewWriter(&errBuffer)
cmd.SetStderr(errWriter)
if err := cmd.Run(); err != nil {
if !errorutil.IsExitStatusError(err) {
return nil, err
}
return nil, errors.New(string(errBuffer.Bytes()))
}
return &jwt.Config{
Email: email,
PrivateKey: outBuffer.Bytes(),
TokenURL: google.JWTTokenURL,
Scopes: []string{androidpublisher.AndroidpublisherScope},
}, nil
}
func (c *EtcdAdapter) ExecWithLog(cmd *exec.Cmd, i int) {
stdoutPipe, err := cmd.StdoutPipe()
check(err)
stderrPipe, err := cmd.StderrPipe()
check(err)
outFile, err := os.Create(fmt.Sprintf("/tmp/etcd_%d.out", i))
check(err)
errFile, err := os.Create(fmt.Sprintf("/tmp/etcd_%d.err", i))
check(err)
outWriter := bufio.NewWriter(outFile)
errWriter := bufio.NewWriter(errFile)
defer outWriter.Flush()
defer errWriter.Flush()
// Start the command
err = cmd.Start()
check(err)
go io.Copy(outWriter, stdoutPipe)
go io.Copy(errWriter, stderrPipe)
c.launchProcess <- cmd.Process
cmd.Wait()
}
func outputWriter(outdir string, name string, ext string) (*bufio.Writer, *os.File, string, error) {
sname := "anonymous"
if strings.HasSuffix(outdir, ext) {
name = filepath.Base(outdir)
sname = name[:len(name)-len(ext)]
outdir = filepath.Dir(outdir)
}
if name != "" {
sname = name
}
if outdir == "" {
return bufio.NewWriter(os.Stdout), nil, sname, nil
}
outfile := sname
if !strings.HasSuffix(outfile, ext) {
outfile += ext
}
path := filepath.Join(outdir, outfile)
f, err := os.Create(path)
if err != nil {
return nil, nil, "", err
}
writer := bufio.NewWriter(f)
return writer, f, sname, nil
}
func Push(devices []Transporter, local io.Reader, mode os.FileMode, modtime uint32, remote string) error {
d := make([]io.Writer, 0, len(devices))
for _, t := range devices {
conn, err := GetPushWriter(t, remote, uint32(mode))
if err != nil {
return err
}
d = append(d, io.Writer(conn))
defer conn.VerifyOk()
}
reader := bufio.NewReader(local)
sections := NewSectionedMultiWriter(d...)
writer := bufio.NewWriter(sections)
writer.ReadFrom(reader)
writer.Flush()
sections.Close()
wr := bufio.NewWriter(io.MultiWriter(d...))
wr.WriteString("DONE")
binary.Write(wr, binary.LittleEndian, modtime)
wr.Flush()
return nil
}
func openLogFile(fileName, fileType string, compress bool, rotateMode int, maxSize int64) (*jsonFile, error) {
fullName := fileName + fileType
if _, err := os.Stat(fullName); err == nil {
os.Rename(fullName, fileName+".01"+fileType)
fullName = fileName + ".02" + fileType
} else if _, err := os.Stat(fileName + ".01" + fileType); err == nil {
for fileId := 1; true; fileId++ {
fullName = fileName + fmt.Sprintf(".%02d", fileId) + fileType
if _, err := os.Stat(fullName); err != nil {
break
}
}
}
file, err := os.OpenFile(fullName, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
return nil, err
}
jsonfile := &jsonFile{file: file, curFile: fullName, rotateMode: rotateMode}
if compress {
jsonfile.bufio = bufio.NewWriter(jsonfile.file)
jsonfile.gzip = gzip.NewWriter(jsonfile.bufio)
jsonfile.json = json.NewEncoder(jsonfile.gzip)
} else {
jsonfile.bufio = bufio.NewWriter(jsonfile.file)
jsonfile.json = json.NewEncoder(jsonfile.bufio)
}
if jsonfile.rotateMode == ROTATE_BY_SIZE && maxSize == 0 {
jsonfile.maxSize = 1024 * 1024 * 1024 * 25
} else {
jsonfile.maxSize = maxSize
}
return jsonfile, nil
}
// How to Confirm
// cat hightemp.txt | cut -f1
// cat hightemp.txt | cut -f2
func main() {
filename := "./hightemp.txt"
fp, err := os.Open(filename)
if err != nil {
panic(err)
}
defer fp.Close()
fp1, err := os.Create("col1.txt")
if err != nil {
panic(err)
}
defer fp1.Close()
fw1 := bufio.NewWriter(fp1)
fp2, err := os.Create("col2.txt")
if err != nil {
panic(err)
}
defer fp2.Close()
fw2 := bufio.NewWriter(fp2)
scanner := bufio.NewScanner(fp)
for scanner.Scan() {
splitted := strings.Split(scanner.Text(), "\t")
fmt.Fprint(fw1, splitted[0]+"\n")
fmt.Fprint(fw2, splitted[1]+"\n")
fw1.Flush()
fw2.Flush()
}
}
func (isle *SSetIsland) initLogs() {
os.Mkdir(isle.logDir, os.ModePerm)
tmpF0, err5 := os.Create(isle.logDir + fmt.Sprintf("sisle%d:err.log", isle.id))
if err5 != nil {
log.Fatal("couldn't create errs log")
}
isle.errLogBuf = bufio.NewWriter(tmpF0)
isle.errLog = log.New(isle.errLogBuf, "", log.LstdFlags)
tmpF1, err1 := os.Create(isle.logDir + fmt.Sprintf("sisle%d:main.log", isle.id))
if err1 != nil {
log.Fatal("couldn't create main log")
}
isle.mainLogBuf = bufio.NewWriter(tmpF1)
isle.mainLog = log.New(isle.mainLogBuf, "", log.LstdFlags)
tmpF3, err3 := os.Create(isle.logDir + fmt.Sprintf("sisle%d:sset.log", isle.id))
if err3 != nil {
log.Fatal("couldn't create ssets log")
}
isle.ssetLogBuf = bufio.NewWriter(tmpF3)
isle.ssetLog = log.New(isle.ssetLogBuf, "", log.LstdFlags)
tmpF4, err4 := os.Create(isle.logDir + fmt.Sprintf("sisle%d:eqns.log", isle.id))
if err4 != nil {
log.Fatal("couldn't create eqns log")
}
isle.eqnsLogBuf = bufio.NewWriter(tmpF4)
isle.eqnsLog = log.New(isle.eqnsLogBuf, "", log.LstdFlags)
}
// This test custom adapter just acts as a bridge for uploads/downloads
// in order to demonstrate & test the custom transfer adapter protocols
// All we actually do is relay the requests back to the normal storage URLs
// of our test server for simplicity, but this proves the principle
func main() {
scanner := bufio.NewScanner(os.Stdin)
writer := bufio.NewWriter(os.Stdout)
errWriter := bufio.NewWriter(os.Stderr)
for scanner.Scan() {
line := scanner.Text()
var req request
if err := json.Unmarshal([]byte(line), &req); err != nil {
writeToStderr(fmt.Sprintf("Unable to parse request: %v\n", line), errWriter)
continue
}
switch req.Event {
case "init":
writeToStderr(fmt.Sprintf("Initialised test custom adapter for %s\n", req.Operation), errWriter)
resp := &initResponse{}
sendResponse(resp, writer, errWriter)
case "download":
writeToStderr(fmt.Sprintf("Received download request for %s\n", req.Oid), errWriter)
performDownload(req.Oid, req.Size, req.Action, writer, errWriter)
case "upload":
writeToStderr(fmt.Sprintf("Received upload request for %s\n", req.Oid), errWriter)
performUpload(req.Oid, req.Size, req.Action, req.Path, writer, errWriter)
case "terminate":
writeToStderr("Terminating test custom adapter gracefully.\n", errWriter)
break
}
}
}
// *****************************************************************************
// CREATION OF INDEX FILE
// *****************************************************************************
func (g *YIndex) CreateIndexFiles() (int, error) {
count := 0
// -------------------------------------------------------------------------
// OPEN AN INPUT FILE
// -------------------------------------------------------------------------
fi, err := os.Open(g.InputFileName)
if err != nil {
return count, errors.New(
fmt.Sprintf("Cannot open file <%s>.", g.InputFileName),
)
}
bfi := bufio.NewReaderSize(fi, 1*1024*1024)
defer fi.Close()
// -------------------------------------------------------------------------
// CREATE MAIN INDEXING FILE
// -------------------------------------------------------------------------
fo1, err := os.Create(g.InputFileName + ".yidx1")
if err != nil {
return count, errors.New(
fmt.Sprintf("Cannot create main indexing file <%s.yidx1>.",
g.InputFileName+".yidx1"),
)
}
bfo1 := bufio.NewWriter(fo1)
count, err = g.createIndexFile1(bfi, bfo1)
if err != nil {
return count, err
}
fo1.Close()
// -------------------------------------------------------------------------
// CREATE DOUBLE INDEXING FILE (THISFILE WILL BE LOADED INTO MEMORY)
// -------------------------------------------------------------------------
fi1, err := os.Open(g.InputFileName + ".yidx1")
if err != nil {
return count, errors.New(
fmt.Sprintf("Cannot read main indexing file <%s>.",
g.InputFileName+".yidx1"),
)
}
bfi1 := bufio.NewReaderSize(fi1, 1*1024*1024)
defer fi1.Close()
fo2, err := os.Create(g.InputFileName + ".yidx2")
if err != nil {
return count, errors.New(
fmt.Sprintf("Cannot create double indexing file <%s>.",
g.InputFileName+".yidx2"),
)
}
bfo2 := bufio.NewWriter(fo2)
count, err = g.createIndexFile2(bfi1, bfo2)
if err != nil {
return count, err
}
fo2.Close()
return count, nil
}
// Split bytes of input file into nMap splits, but split only on white space
func (mr *MapReduce) Split(fileName string) {
Logger.Printf("Split file:%s\n", fileName)
fmt.Printf("Split %s\n", fileName)
infile, err := os.Open(fileName)
if err != nil {
log.Fatal("Split: ", err)
}
defer infile.Close()
/*Stat return s the FileInfo structure desc file.
*If there is an error, it will be of type *PathError.
*
*type fileInfo interface {
* Name() string
* Size() int64
* Mode() FileMode
* ModTime() time.Time
* IsDir() bool
* Sys() interface{}
*}
*/
fi, err := infile.Stat()
if err != nil {
log.Fatal("Split: ", err)
}
size := fi.Size()
nchunk := size / int64(mr.nMap)
nchunk += 1
Logger.Printf("----------------------------\n")
Logger.Printf("file size:%d\n", size)
Logger.Printf("Split into %d\n", mr.nMap)
Logger.Printf("chunk size:%d\n", nchunk)
Logger.Printf("----------------------------\n")
outfile, err := os.Create(MapName(fileName, 0))
if err != nil {
log.Fatal("Split: ", err)
}
writer := bufio.NewWriter(outfile)
m := 1
i := 0
scanner := bufio.NewScanner(infile)
for scanner.Scan() { //line by line scan
if int64(i) > nchunk*int64(m) {
writer.Flush()
outfile.Close()
//create a new file and write to the new file.
outfile, err = os.Create(MapName(fileName, m))
writer = bufio.NewWriter(outfile)
m += 1
}
line := scanner.Text() + "\n"
writer.WriteString(line)
i += len(line)
}
writer.Flush()
outfile.Close()
}
func printRules(wordFileName, ruleFileName string, words chan []Word) {
var wordFile *os.File
var ruleFile *os.File
// may not even need to stat it.
// os.create will just overwrite the file
// we remove the old files so we can write the new contents
if _, err := os.Stat(wordFileName); err != nil {
if os.IsExist(err) {
err = os.Remove(wordFileName)
if err != nil {
log.Println(err.Error())
}
}
}
wordFile, err := os.Create(wordFileName)
if err != nil {
log.Println("cannot open file to write to:" + err.Error())
}
if _, err := os.Stat(ruleFileName); err != nil {
if os.IsExist(err) {
err = os.Remove(ruleFileName)
if err != nil {
log.Println(err.Error())
}
}
}
ruleFile, err = os.Create(ruleFileName)
if err != nil {
log.Println("cannot open file to write to:" + err.Error())
}
defer wordFile.Close()
defer ruleFile.Close()
wordbuf := bufio.NewWriter(wordFile)
rulebuf := bufio.NewWriter(ruleFile)
for word := range words {
for _, a := range word {
fmt.Fprintln(wordbuf, a.suggestion)
fmt.Fprintf(rulebuf, "%v", a.hashcatRules)
// try to flush right before the buffer gets filled
if wordbuf.Buffered() >= 4000 {
wordbuf.Flush()
}
if rulebuf.Buffered() >= 4000 {
rulebuf.Flush()
}
}
}
// make sure that everything is flushed
rulebuf.Flush()
wordbuf.Flush()
}
func main() {
var device *string = flag.String("i", "", "interface")
var snaplen *int = flag.Int("s", 65535, "snaplen")
var hexdump *bool = flag.Bool("X", false, "hexdump")
expr := ""
out = bufio.NewWriter(os.Stdout)
errout = bufio.NewWriter(os.Stderr)
flag.Usage = func() {
fmt.Fprintf(errout, "usage: %s [ -i interface ] [ -s snaplen ] [ -X ] [ expression ]\n", os.Args[0])
errout.Flush()
os.Exit(1)
}
flag.Parse()
if len(flag.Args()) > 0 {
expr = flag.Arg(0)
}
if *device == "" {
devs, err := pcap.Findalldevs()
if err != nil {
fmt.Fprintf(errout, "tcpdump: couldn't find any devices: %s\n", err)
}
if 0 == len(devs) {
flag.Usage()
}
*device = devs[0].Name
}
h, err := pcap.Openlive(*device, int32(*snaplen), true, 0)
if h == nil {
fmt.Fprintf(errout, "tcpdump: %s\n", err)
errout.Flush()
return
}
defer h.Close()
if expr != "" {
ferr := h.Setfilter(expr)
if ferr != nil {
fmt.Fprintf(out, "tcpdump: %s\n", ferr)
out.Flush()
}
}
for pkt := h.Next(); pkt != nil; pkt = h.Next() {
pkt.Decode()
fmt.Fprintf(out, "%s\n", pkt.String())
if *hexdump {
Hexdump(pkt)
}
out.Flush()
}
}
func NewGobCodecs(conn io.ReadWriteCloser) (*gobServerCodec, *gobClientCodec) {
sbuf := bufio.NewWriter(conn)
srv := &gobServerCodec{conn, gob.NewDecoder(conn), gob.NewEncoder(sbuf), sbuf}
cbuf := bufio.NewWriter(conn)
cli := &gobClientCodec{conn, gob.NewDecoder(conn), gob.NewEncoder(cbuf), cbuf}
return srv, cli
}
func (lb *LogBuffer) httpDumpHandler(w http.ResponseWriter, req *http.Request) {
parsedQuery := url.ParseQuery(req.URL)
name, ok := parsedQuery.Table["name"]
if !ok {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusBadRequest)
return
}
_, recentFirst := parsedQuery.Flags["recentFirst"]
if name == "latest" {
lbFilename := ""
lb.rwMutex.Lock()
if lb.file != nil {
lbFilename = lb.file.Name()
}
lb.rwMutex.Unlock()
if lbFilename == "" {
writer := bufio.NewWriter(w)
defer writer.Flush()
lb.Dump(writer, "", "", recentFirst)
return
}
name = path.Base(lbFilename)
}
file, err := os.Open(path.Join(lb.logDir, path.Base(path.Clean(name))))
if err != nil {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound)
return
}
defer file.Close()
writer := bufio.NewWriter(w)
defer writer.Flush()
if recentFirst {
scanner := bufio.NewScanner(file)
lines := make([]string, 0)
for scanner.Scan() {
line := scanner.Text()
if len(line) < 1 {
continue
}
lines = append(lines, line)
}
if err = scanner.Err(); err == nil {
reverseStrings(lines)
for _, line := range lines {
fmt.Fprintln(writer, line)
}
}
} else {
_, err = io.Copy(writer, bufio.NewReader(file))
}
if err != nil {
fmt.Fprintln(writer, err)
}
}