Exemple #1
0
func main() {
	sflag.Parse(&opt)
	genutil.CheckFileIsReadableAndNonzeroOrDie(opt.Inbase + "/" + opt.Infile)
	fi, _ := os.Lstat(opt.Inbase + "/" + opt.Infile)
	modtime := fi.ModTime()
	outfiles := make([]string, 0)
	fmt.Println("doing file=", opt.Infile)
	buf := ""
	opt.Tempbase = strings.TrimRight(opt.Tempbase, "/ ")
	opt.Inbase = strings.TrimRight(opt.Inbase, "/ ")
	opt.Outbase = strings.TrimRight(opt.Outbase, "/ ")
	dir := opt.Tempbase
	numrex, err := regexp.Compile("[12][0-9][0-9][0-9][01][0-9][0-3][0-9]") // regexp.MustCompile("\d\d\d\d\d\d\d\d")
	if err != nil {
		panic("could not compile date regexp")
	}
	infile := opt.Inbase + "/" + opt.Infile
	inlen := len(infile)
	if genutil.IsZipFilename(infile) {
		yyyymmdd2 := genutil.GetYyyymmddFromFilenameYymmddFromEndWithSuffixLen(infile, len(".zip"), "19010101") // grab the YYMMDD from filenames of form foo_YYMMDD.zip
		if yyyymmdd2 == "19010101" {
			fmt.Println("skipping unknown zip filename=", infile)
			return
		}
		//tmpdir	:= genutil.MakeDirOrDie(opt.Tempbase, "TEMP")
		tmpdir, tmperr := ioutil.TempDir(opt.Tempbase, "vendor1")
		if tmperr != nil {
			panic("Cannot create temp dir by ioutil.TempDir, err=" + tmperr.Error())
		}

		buf = genutil.BashExecOrDie(false, "/usr/bin/unzip -u -d "+tmpdir+" "+infile, dir)
		for _, ff := range genutil.FileList(tmpdir) {
			fflen := len(ff)
			yyyymmdd := ff[fflen-8:]
			if (fflen < 9) || !numrex.MatchString(ff[fflen-8:]) {
				fmt.Println("substituting non-yyyymmdd fname=", ff, " infile=", infile)
				yyyymmdd = yyyymmdd2
			}
			ffile := tmpdir + "/" + ff
			genutil.EnsureDirOrDie(opt.Outbase, yyyymmdd)
			buf = genutil.BashExecOrDie(false, "/usr/bin/gzip "+ffile, dir)                              // fmt.Println(buf)
			buf = genutil.BashExecOrDie(false, "/bin/mv -f "+ffile+".gz "+opt.Outbase+"/"+yyyymmdd, dir) // fmt.Println(buf)
			os.Chtimes(opt.Outbase+"/"+yyyymmdd+"/"+ff+".gz", modtime, modtime)
			outfiles = append(outfiles, yyyymmdd+"/"+ff+".gz")
		}
		buf = genutil.BashExecOrDie(false, "/bin/rmdir "+tmpdir, dir) // fmt.Println(buf)
		if false {
			fmt.Println(buf)
		}
	} else if (inlen > 9) && numrex.MatchString(infile[inlen-8:]) { // some files are not zipfiles, but end in a date
		yyyymmdd := infile[inlen-8:]
		fmt.Println("nonzip file=", infile, " yyyymmdd=", yyyymmdd)
		outfile := opt.Outbase + "/" + yyyymmdd + "/" + opt.Infile
		outfiles = append(outfiles, yyyymmdd+"/"+opt.Infile+".gz")
		if true {
			genutil.EnsureDirOrDie(opt.Outbase, yyyymmdd)
			buf = genutil.BashExecOrDie(false, "/bin/cp -f "+infile+" "+outfile, dir) // fmt.Println(buf)
			buf = genutil.BashExecOrDie(false, "/usr/bin/gzip -f "+outfile, dir)      // fmt.Println(buf)
		} else {
			buf = genutil.BashExecOrDie(false, "/usr/bin/gzip -f "+infile, dir) // fmt.Println(buf)
			genutil.EnsureDirOrDie(opt.Outbase, yyyymmdd)
			buf = genutil.BashExecOrDie(false, "/bin/mv -f "+infile+".gz "+opt.Outbase+"/"+yyyymmdd, dir) // fmt.Println(buf)
		}
		os.Chtimes(outfile+".gz", modtime, modtime)
	} else {
		fmt.Println("nonzip file=", infile, "with no yyyymmdd pattern, so doing nothing!")
	}
	ccfg := qcfg.NewCfgMem(opt.Metafile)
	ccfg.EditEntry("splitExample", "file", "outdir", opt.Outbase)
	ccfg.EditEntry("splitExample", "file", "outfiles", strings.Join(outfiles, ","))
	ccfg.CfgWrite(opt.Metafile)
}
Exemple #2
0
func manageDB() {
	var chanStruct = ABChans{
		addModRowChan:  make(chan alertbaseutil.ROW),
		getListChan:    make(chan CHANROWS),
		addOpsRowChan:  make(chan alertbaseutil.OPSROW),
		getOpsListChan: make(chan CHANOPSROWS),
	}
	var db = map[int64]*alertbaseutil.ROW{}
	var sortedRows = []*alertbaseutil.ROW{}
	var opsarr = make([]*alertbaseutil.OPSROW, 15) // Remember only last 15 takeovers
	var fp *os.File = nil
	var fpops *os.File = nil
	lg.Println("manageDB : starting up...")
	if _, err := os.Stat(opt.Alertsfile); err != nil {
		fp, err = os.Create(opt.Alertsfile)
		if err != nil {
			lg.Println("manageDB : cannot create alertsfile =", opt.Alertsfile, "err =", err, "Exiting...")
			os.Exit(1)
		}
		fp.WriteString((&alertbaseutil.ROW{}).GetHeader() + "\n")
		fp.Sync()
	} else {
		fp, err = os.OpenFile(opt.Alertsfile, os.O_RDWR|os.O_APPEND, 0755)
		if err != nil {
			lg.Println("manageDB : cannot open alertsfile =", opt.Alertsfile, "err =", err, "Exiting...")
			os.Exit(1)
		}
	}
	lg.Println("manageDB : Opened alertfile =", opt.Alertsfile)
	defer fp.Close()
	if _, err := os.Stat(opt.Opsfile); err != nil {
		fpops, err = os.Create(opt.Opsfile)
		if err != nil {
			lg.Println("manageDB : cannot create opsfile =", opt.Opsfile, "err =", err, "Exiting...")
			os.Exit(1)
		}
	} else {
		fpops, err = os.OpenFile(opt.Opsfile, os.O_RDWR|os.O_APPEND, 0755)
		if err != nil {
			lg.Println("manageDB : cannot open opsfile =", opt.Opsfile, "err =", err, "Exiting...")
			os.Exit(1)
		}
	}
	lg.Println("manageDB : Opened opsfile =", opt.Opsfile)
	defer fpops.Close()
	scanner := bufio.NewScanner(fp)
	lineno := 1
	for scanner.Scan() {
		if lineno == 1 {
			lineno += 1
			continue
		}
		row := new(alertbaseutil.ROW)
		err1 := row.SetFromCSV(scanner.Text())
		if err1 != nil {
			lg.Println("manageDB : Found bad csv line at line number =", lineno, "err =", err1)
			lineno += 1
			continue
		}
		key := row.GetKey()
		if key < int64(0) {
			lg.Println("manageDB : Found alert with bad key at line number =", lineno)
			lineno += 1
			continue
		}
		if row.IsOlderThan(4 * 24 * time.Hour) {
			lineno += 1
			continue
		}
		db[key] = row
		lineno += 1
	}
	if err := scanner.Err(); err != nil {
		lg.Println("manageDB : Error reading alertsfile =", opt.Alertsfile, "err =", err, "Will continue anyway...")
	}

	sortedRows = make([]*alertbaseutil.ROW, len(db))
	ii := 0
	for _, vv := range db {
		sortedRows[ii] = vv
		ii += 1
	}
	sort.Sort(alertbaseutil.ROWS(sortedRows))

	// Load last 15 lines from opsfile
	out := strings.Trim(genutil.BashExecOrDie(false, "/usr/bin/tail -15 "+opt.Opsfile+" | /usr/bin/tac", path.Dir(opt.Opsfile)), " \r\t\n")
	numopsrows := 0
	if out != "" {
		lines := strings.Split(out, "\n")
		idx := 14
		for ii := 0; ii < len(lines); ii++ {
			row := new(alertbaseutil.OPSROW)
			line := lines[len(lines)-ii-1]
			errrow := row.SetFromCSV(strings.Trim(line, " \t\r\n"))
			if errrow != nil {
				continue
			}
			opsarr[idx] = row
			idx -= 1
			numopsrows += 1
		}
	}

	go startCimServer(&chanStruct)
	lg.Println("manageDB : started cim server")
	lg.Println("manageDB : starting event loop")
	for {
		var row = alertbaseutil.ROW{}
		select {
		case row = <-(chanStruct.addModRowChan):
			rowptr := &row
			lg.Println("manageDB : Got an alert from outside row =", row)
			key := rowptr.GetKey()
			if key < int64(0) {
				lg.Println("manageDB : But this alert has bad openat. Doing nothing")
				continue
			}
			if rowptr.IsOlderThan(4 * 24 * time.Hour) {
				lg.Println("manageDB : But this alert is too old. Doing nothing")
			}
			rowInDb, ok := db[key]
			if !ok {
				lg.Println("manageDB : This alert is not present in the db, so adding")
				db[key] = rowptr
				fp.WriteString(rowptr.GetCSV() + "\n")
				fp.Sync()
			} else {
				rowInDb.UpdateWith(rowptr)
				fp.WriteString(rowInDb.GetCSV() + "\n")
				fp.Sync()
			}
			// remove old ones
			for kk, vv := range db {
				if vv.IsOlderThan(4 * 24 * time.Hour) {
					delete(db, kk)
				}
			}
			sortedRows = make([]*alertbaseutil.ROW, len(db))
			ii := 0
			for _, vv := range db {
				sortedRows[ii] = vv
				ii += 1
			}
			sort.Sort(alertbaseutil.ROWS(sortedRows))

		case chanrows := <-(chanStruct.getListChan):
			chanrows.Retchan <- sortedRows
		case opsrowtmp := <-(chanStruct.addOpsRowChan):
			opsrowptr := &opsrowtmp
			lg.Println("manageDB : Got an ops row from outside : row =", opsrowtmp)
			if numopsrows < 15 {
				opsarr[14-numopsrows] = opsrowptr
				numopsrows += 1
			} else {
				numopsrows = 15
				for ii := 13; ii >= 0; ii-- {
					opsarr[ii+1] = opsarr[ii]
				}
				opsarr[0] = opsrowptr
			}
			fpops.WriteString(opsrowptr.GetCSV() + "\n")
			fpops.Sync()
		case chanopsrows := <-(chanStruct.getOpsListChan):
			chanopsrows.Retchan <- opsarr[15-numopsrows : 15]
		}
	}
}