func (bot *SlackBot) SetDebug(debug bool) {
bot.api.SetDebug(debug)
if debug {
logging.SetLevel(logging.INFO, "slackbot")
} else {
logging.SetLevel(logging.ERROR, "slackbot")
}
}
func configureLogging(verbose bool) {
backend := log.NewLogBackend(os.Stdout, "", 0)
f := "%{time:15:04:05.000} %{module} ▶ %{level:.5s} %{id:03x} %{message}"
// f = "%{time:15:04:05.000} %{module} ▶ %{level:.5s} %{id:03x} %{message}"
format := log.MustStringFormatter(f)
formatter := log.NewBackendFormatter(backend, format)
log.SetBackend(formatter)
log.SetLevel(log.ERROR, "lidc")
if verbose {
log.SetLevel(log.DEBUG, "lidc")
}
}
// i thought conn will drop messages because the tE tcp handler can't keep up.
// but looks like that's not true (anymore?), it just works without having to sleep after dispatch
// also note the dummyPackets uses a channel api which probably causes most of the slowdown
func BenchmarkTableDisPatchAndEndpointReceive(b *testing.B) {
logging.SetLevel(logging.WARNING, "carbon-relay-ng") // testendpoint sends a warning because it does something bad with conn at end but it's harmless
tE := NewTestEndpointCounter(b, ":2005")
tE.Start()
table := NewTableOrFatal(b, "", "addRoute sendAllMatch test1 127.0.0.1:2005 flush=10")
tE.WaitAccepts(1, time.Second)
// reminder: go benchmark will invoke this with N = 0, then maybe N = 20, then maybe more
// and the time it prints is function run divided by N, which
// should be of a more or less stable time, which gets printed
metric70 := []byte("abcde_fghij.klmnopqrst.uv_wxyz.1234567890abcdefg 12345.6789 1234567890") // key = 48, val = 10, ts = 10 -> 70
dest, err := table.GetRoute("test1").GetDestination(0)
if err != nil {
panic(err)
}
<-dest.WaitOnline()
b.ResetTimer()
go func() {
for i := 0; i < b.N; i++ {
table.Dispatch(metric70)
}
}()
tE.WaitMetrics(b.N, 5*time.Second)
b.StopTimer()
err = table.Shutdown()
if err != nil {
b.Fatal(err)
}
tE.Close()
}
// i thought conn will drop messages because the tE tcp handler can't keep up.
// but looks like that's not true (anymore?), it just works without having to sleep after dispatch
// also note the dummyPackets uses a channel api which probably causes most of the slowdown
func benchmarkSendAndReceive(b *testing.B, dp *dummyPackets) {
logging.SetLevel(logging.ERROR, "carbon-relay-ng") // testendpoint sends a warning because it does something bad with conn at end but it's harmless
tE := NewTestEndpoint(nil, ":2005")
na := tE.conditionNumAccepts(1)
tE.Start()
table = NewTableOrFatal(b, "", "addRoute sendAllMatch test1 127.0.0.1:2005")
na.Wait()
// reminder: go benchmark will invoke this with N = 0, then maybe N = 20, then maybe more
// and the time it prints is function run divided by N, which
// should be of a more or less stable time, which gets printed
fmt.Println()
for i := 0; i < b.N; i++ {
log.Notice("iteration %d: sending %d metrics", i, dp.amount)
ns := tE.conditionNumSeen(dp.amount * (i + 1))
for m := range dp.All() {
//fmt.Println("dispatching", m)
//fmt.Printf("dispatching '%s'\n", string(m))
table.Dispatch(m)
}
log.Notice("waiting until all %d messages received", dp.amount*(i+1))
ns.Wait()
log.Notice("iteration %d done. received %d metrics (%d total)", i, dp.amount, dp.amount*(i+1))
}
log.Notice("received all %d messages. wrapping up benchmark run", dp.amount*b.N)
err := table.Shutdown()
if err != nil {
b.Fatal(err)
}
tE.Close()
}
func BenchmarkTreeAdd(b *testing.B) {
dropTestTree()
prepareTestTree(b)
logging.SetLevel(logging.ERROR, "metricsearch")
f, err := os.Open("payload.txt")
if err != nil {
b.Error("Please provide metric list in payload.txt file in the current directory")
return
}
defer f.Close()
payload := make([]string, 0, 10000000)
sc := bufio.NewScanner(f)
for sc.Scan() {
payload = append(payload, sc.Text())
}
c := 0
b.ResetTimer()
for i := 0; i < b.N; i++ {
tree.Add(payload[c])
c++
if c == len(payload) {
fmt.Println("Warning: Payload is too short")
c = 0
}
}
}
// just dispatch (coming into table), no matching or sending to route
func BenchmarkTableDispatch(b *testing.B) {
logging.SetLevel(logging.WARNING, "carbon-relay-ng") // don't care about unroutable notices
metric70 := []byte("abcde_fghij.klmnopqrst.uv_wxyz.1234567890abcdefg 12345.6789 1234567890") // key = 48, val = 10, ts = 10 -> 70
table := NewTableOrFatal(b, "", "")
for i := 0; i < b.N; i++ {
table.Dispatch(metric70)
}
}
func init() {
backend := logging.NewLogBackend(os.Stderr, "", 0)
logging.SetBackend(backend)
formatter := logging.MustStringFormatter("[%{time:15:04:05.000}] %{shortfile:18s}: %{color}[%{level:-5s}]%{color:reset} %{message}")
logging.SetFormatter(formatter)
logger = logging.MustGetLogger("orderer/kafka")
logging.SetLevel(logging.INFO, "") // Silence debug-level outputs when testing
}
// just dispatch (coming into table), no matching or sending to route
func BenchmarkTableDispatchMillion(b *testing.B) {
logging.SetLevel(logging.WARNING, "carbon-relay-ng") // don't care about unroutable notices
table = NewTableOrFatal(b, "", "")
for i := 0; i < b.N; i++ {
for j := 0; j < 1000000; j++ {
table.Dispatch(metric70)
}
}
}
func initLogging(level logging.Level, color bool) {
format := logging.MustStringFormatter(logFormat)
logging.SetFormatter(format)
for _, s := range logModules {
logging.SetLevel(level, s)
}
stdout := logging.NewLogBackend(os.Stdout, "", 0)
stdout.Color = color
logging.SetBackend(stdout)
}
func getLogger(name string, logConfig LogConfig) *logger.Logger {
mainLogger.Info("Make logger of %s at file %s", name, logConfig.Filename)
log1 := logger.MustGetLogger(name)
var leveledBackend logger.LeveledBackend
f, err := logger.NewStringFormatter("%{shortfile} %{time:2006-01-02T15:04:05} %{level:.1s} %{id:04d} %{module} %{message}")
if err != nil {
mainLogger.Info("failed to set format: %s", err)
}
logger.SetFormatter(f)
if logConfig.Filename != "" {
logFileName := logConfig.Filename
logFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
backend := logger.NewLogBackend(logFile, "", 0)
if err != nil {
mainLogger.Error("Failed to open log file - " + logFileName)
panic(err)
} else {
leveledBackend = logger.AddModuleLevel(backend)
}
} else {
backend := logger.NewLogBackend(os.Stdout, "", 0)
leveledBackend = logger.AddModuleLevel(backend)
}
switch logConfig.Level {
case "debug":
leveledBackend.SetLevel(logger.NOTICE, name)
case "error":
logger.SetLevel(logger.ERROR, name)
case "info":
logger.SetLevel(logger.INFO, name)
case "warn":
logger.SetLevel(logger.WARNING, name)
}
log1.SetBackend(leveledBackend)
return log1
}
// initialize logging using this LogConfig;
// it panics if l.Format is invalid or l.Level is invalid
func (l *LogConfig) InitLogger() {
l.initLevel()
format := logging.MustStringFormatter(l.Format)
logging.SetFormatter(format)
for _, s := range l.Modules {
logging.SetLevel(l.level, s)
}
stdout := logging.NewLogBackend(os.Stdout, "", 0)
stdout.Color = l.Colors
logging.SetBackend(stdout)
}
func initLog(logConfig map[string]interface{}) {
mainLogger.Info("LogConfig: %v", logConfig)
_, ok := logConfig["level"]
if ok {
switch logConfig["level"].(string) {
case "debug":
logger.SetLevel(logger.NOTICE, moduleName)
case "error":
logger.SetLevel(logger.ERROR, moduleName)
case "info":
logger.SetLevel(logger.INFO, moduleName)
case "warn":
logger.SetLevel(logger.WARNING, moduleName)
}
}
f, err := logger.NewStringFormatter("%{shortfile} %{time:2006-01-02T15:04:05} %{level:.1s} %{id:04d} %{module} %{message}")
if err != nil {
mainLogger.Info("failed to set format: %s", err)
}
logger.SetFormatter(f)
_, ok = logConfig["file"]
if ok {
logFileName := logConfig["file"].(string)
logFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
backend := logger.NewLogBackend(logFile, "", 0)
if err != nil {
mainLogger.Error("Failed to open log file - " + logFileName)
panic(err)
} else {
logger.SetBackend(backend)
}
} else {
backend := logger.NewLogBackend(os.Stdout, "", 0)
logger.SetBackend(backend)
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////
// 2way of run
// - 1st: cmd web
// call from brower: http://localhost:8080/main/core/1418,1419,2502,2694,2932,2933,2695
// - 2nd: cmd core/graphite 1418,1419,2502,2694,2932,2933,2695
//////////////////////////////////////////////////////////////////////////////////////////////////////
func main() {
LOGFILE, err := os.OpenFile(LOGPATH, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
LOG.Fatalf("Log file error: %s %s", LOGPATH, err)
}
defer func() {
LOGFILE.WriteString(fmt.Sprintf("closing %s", time.UnixDate))
LOGFILE.Close()
}()
logback := logging.NewLogBackend(LOGFILE, "", 0)
logformatted := logging.NewBackendFormatter(logback, LOGFORMAT)
loglevel := "DEBUG"
GLOGLEVEL, err := logging.LogLevel(loglevel)
if err != nil {
GLOGLEVEL = logging.DEBUG
}
logging.SetBackend(logformatted)
logging.SetLevel(GLOGLEVEL, "")
programName := os.Args[0:1]
if len(os.Args) < 2 {
HCIDS = append(HCIDS, 1418, 1419, 2502, 2694, 2932, 2933, 2695)
mainExec()
} else {
typeStr := os.Args[1:2]
if len(os.Args) >= 3 {
hcidStr := os.Args[2:3]
allArgs := os.Args[1:]
fmt.Println(programName, typeStr, hcidStr, allArgs)
arry := strings.Split(hcidStr[0], ",")
for i := range arry {
n, _ := strconv.Atoi(arry[i])
HCIDS = append(HCIDS, n)
}
} else {
allArgs := os.Args[1:]
fmt.Println(programName, typeStr, allArgs)
}
if typeStr[0] == "web" {
webserver()
} else {
STYPE = typeStr[0]
mainExec()
}
}
fmt.Println(HCIDS)
}
func init() {
packets0A = NewDummyPackets("0A", 1)
packets1A = NewDummyPackets("1A", 10)
packets1B = NewDummyPackets("1B", 10)
packets1C = NewDummyPackets("1C", 10)
packets3A = NewDummyPackets("3A", 1000)
packets3B = NewDummyPackets("3B", 1000)
packets3C = NewDummyPackets("3C", 1000)
packets4A = NewDummyPackets("4A", 10000)
packets5A = NewDummyPackets("5A", 100000)
packets6A = NewDummyPackets("6A", 1000000)
//packets6B = NewDummyPackets("6B", 1000000)
//packets6C = NewDummyPackets("6C", 1000000)
logging.SetLevel(logging.NOTICE, "carbon-relay-ng")
metric70 = []byte("abcde_fghij.klmnopqrst.uv_wxyz.1234567890abcdefg 12345.6789 1234567890") // key = 48, val = 10, ts = 10 -> 70
}
func configureLogging(colorLogs bool) {
golog.SetFormatter(golog.MustStringFormatter("[0x%{id:x}] [%{level}] [%{module}] %{message}"))
stdoutLogBackend := golog.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lshortfile)
stdoutLogBackend.Color = colorLogs
golog.SetBackend(stdoutLogBackend)
golog.SetLevel(golog.DEBUG, "")
// test logs
jabberwocky, err := ioutil.ReadFile("the_jabberwocky.txt")
if err != nil {
log.Warning("Failed to read \"The Jabberwocky\", it looks like nonsense")
return
}
log.Notice(fmt.Sprintf("\n\n%s", string(jabberwocky)))
}
func main() {
var loglevel string
client := &clientImpl{doneChan: make(chan struct{})}
backend := logging.NewLogBackend(os.Stderr, "", 0)
logging.SetBackend(backend)
formatter := logging.MustStringFormatter("[%{time:15:04:05}] %{shortfile:18s}: %{color}[%{level:-5s}]%{color:reset} %{message}")
logging.SetFormatter(formatter)
logger = logging.MustGetLogger(pkgName)
flag.StringVar(&loglevel, "loglevel", "info",
"The logging level. (Suggested values: info, debug)")
flag.StringVar(&client.config.server, "server",
"127.0.0.1:7050", "The RPC server to connect to.")
flag.StringVar(&client.config.cmd.cmd, "cmd", "new-chain",
"The action that this client is requesting via the config transaction.")
flag.StringVar(&client.config.cmd.args.creationPolicy, "creationPolicy", "AcceptAllPolicy",
"In case of a new-chain command, the chain createion policy this request should be validated against.")
flag.StringVar(&client.config.cmd.args.chainID, "chainID", "NewChainID",
"In case of a new-chain command, the chain ID to create.")
flag.Parse()
client.config.logLevel, _ = logging.LogLevel(strings.ToUpper(loglevel))
logging.SetLevel(client.config.logLevel, logger.Module)
conn, err := grpc.Dial(client.config.server, grpc.WithInsecure())
if err != nil {
logger.Fatalf("Client did not connect to %s: %v", client.config.server, err)
}
defer conn.Close()
client.rpc = ab.NewAtomicBroadcastClient(conn)
switch client.config.cmd.cmd {
case "new-chain":
envelope := newChainRequest(client.config.cmd.args.creationPolicy, client.config.cmd.args.chainID)
logger.Infof("Requesting the creation of chain \"%s\"", client.config.cmd.args.chainID)
client.broadcast(envelope)
default:
panic("Invalid cmd given")
}
}
func Configure() {
golog.SetFormatter(golog.MustStringFormatter("[0x%{id:x}] [%{level}] [%{module}] %{message}"))
stdoutLogBackend := golog.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lshortfile)
stdoutLogBackend.Color = true
golog.SetLevel(defaultLevel, "")
// NOTE these file permissions are restricted by umask, so they probably won't work right.
err := os.MkdirAll("./log", 0775)
if err != nil {
panic(err)
}
logFile, err := os.OpenFile("./log/zendesk-livestats.log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
if err != nil {
panic(err)
}
fileLogBackend := golog.NewLogBackend(logFile, "", stdlog.LstdFlags|stdlog.Lshortfile)
fileLogBackend.Color = false
golog.SetBackend(stdoutLogBackend, fileLogBackend)
}
// Generate a token based on the key file and username
func main() {
var password string
var username string
const required = "REQUIRED"
be := logging.NewLogBackend(os.Stderr, "", 0)
logging.SetBackend(be)
logging.SetLevel(logging.CRITICAL, "")
flag.StringVar(&username, "user", required, "Username to create the token for")
flag.StringVar(&password, "pwd", required, "Password to use")
flag.Parse()
if username == required || password == required {
flag.PrintDefaults()
return
}
util.LoadConfig()
fmt.Printf("%s=%s\n", username, pwd.HashPassword(password))
}
func SetLogging(cfg Config) (err error) {
var file *os.File
file = os.Stdout
if cfg.Logfile != "" {
file, err = os.OpenFile(cfg.Logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
log.Fatal(err)
}
}
logBackend := logging.NewLogBackend(file, "",
stdlog.LstdFlags|stdlog.Lmicroseconds|stdlog.Lshortfile)
logging.SetBackend(logBackend)
logging.SetFormatter(logging.MustStringFormatter("%{level}: %{message}"))
lv, err := logging.LogLevel(cfg.Loglevel)
if err != nil {
panic(err.Error())
}
logging.SetLevel(lv, "")
return
}
func main() {
params := ParseArgs()
if !ValidateParams(params) {
os.Exit(1)
}
logWriter := (io.Writer)(nil)
if params.LogFile != "" {
logWriter = ioextras.NewStaticRotatingWriter(
func(_ interface{}) (string, error) {
path := strftime.Format(params.LogFile, time.Now())
return path, nil
},
func(path string, _ interface{}) (io.Writer, error) {
dir, _ := filepath.Split(path)
err := os.MkdirAll(dir, os.FileMode(0777))
if err != nil {
return nil, err
}
return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.FileMode(0666))
},
nil,
)
} else {
logWriter = os.Stderr
}
logBackend := logging.NewLogBackend(logWriter, "[fluentd-forwarder] ", log.Ldate|log.Ltime|log.Lmicroseconds)
logging.SetBackend(logBackend)
logger := logging.MustGetLogger("fluentd-forwarder")
logging.SetLevel(params.LogLevel, "fluentd-forwarder")
if progVersion != "" {
logger.Info("Version %s starting...", progVersion)
}
workerSet := fluentd_forwarder.NewWorkerSet()
if params.CPUProfileFile != "" {
f, err := os.Create(params.CPUProfileFile)
if err != nil {
Error(err.Error())
os.Exit(1)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
output := (PortWorker)(nil)
err := (error)(nil)
switch params.OutputType {
case "fluent":
output, err = fluentd_forwarder.NewForwardOutput(
logger,
params.ForwardTo,
params.RetryInterval,
params.ConnectionTimeout,
params.WriteTimeout,
params.FlushInterval,
params.JournalGroupPath,
params.MaxJournalChunkSize,
params.Metadata,
)
case "td":
rootCAs := (*x509.CertPool)(nil)
if params.SslCACertBundleFile != "" {
b, err := ioutil.ReadFile(params.SslCACertBundleFile)
if err != nil {
Error("Failed to read CA bundle file: %s", err.Error())
os.Exit(1)
}
rootCAs = x509.NewCertPool()
if !rootCAs.AppendCertsFromPEM(b) {
Error("No valid certificate found in %s", params.SslCACertBundleFile)
os.Exit(1)
}
}
output, err = fluentd_forwarder.NewTDOutput(
logger,
params.ForwardTo,
params.ConnectionTimeout,
params.WriteTimeout,
params.FlushInterval,
params.Parallelism,
params.JournalGroupPath,
params.MaxJournalChunkSize,
params.ApiKey,
params.DatabaseName,
params.TableName,
"",
params.Ssl,
rootCAs,
"", // TODO:http-proxy
params.Metadata,
)
}
if err != nil {
Error("%s", err.Error())
return
}
workerSet.Add(output)
input, err := fluentd_forwarder.NewForwardInput(logger, params.ListenOn, output)
if err != nil {
Error(err.Error())
return
}
workerSet.Add(input)
signalHandler := NewSignalHandler(workerSet)
input.Start()
output.Start()
signalHandler.Start()
for _, worker := range workerSet.Slice() {
worker.WaitForShutdown()
}
logger.Notice("Shutting down...")
}
// SetLevel sets the desired log level for the default logger.
func SetLevel(level oplogging.Level) {
oplogging.SetLevel(level, LOG_MODULE)
}
func main() {
var format = logging.MustStringFormatter("%{level} %{message}")
logging.SetFormatter(format)
logging.SetLevel(logging.INFO, "iloveck101")
usr, _ := user.Current()
baseDir = fmt.Sprintf("%v/Pictures/iloveck101", usr.HomeDir)
var postUrl string
var workerNum int
rootCmd := &cobra.Command{
Use: "iloveck101",
Short: "Download all the images in given post url",
Run: func(cmd *cobra.Command, args []string) {
crawler(postUrl, workerNum)
},
}
rootCmd.Flags().StringVarP(&postUrl, "url", "u", "http://ck101.com/thread-2876990-1-1.html", "Url of post")
rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 25, "Number of workers")
searchCmd := &cobra.Command{
Use: "search",
Short: "Download all the images in given post url",
Run: func(cmd *cobra.Command, args []string) {
page := 0
keyword := args[0]
hrefs := printGoogleResult(keyword, page)
scanner := bufio.NewScanner(os.Stdin)
quit := false
for !quit {
fmt.Print("ck101> ")
if !scanner.Scan() {
break
}
line := scanner.Text()
parts := strings.Split(line, " ")
cmd := parts[0]
args := parts[1:]
switch cmd {
case "quit":
quit = true
case "n":
page = page + 1
hrefs = printGoogleResult(keyword, page)
case "p":
if page > 0 {
page = page - 1
}
hrefs = printGoogleResult(keyword, page)
case "s":
page = 0
hrefs = printGoogleResult(args[0], page)
case "o":
open.Run(baseDir)
case "d":
index, err := strconv.ParseUint(args[0], 0, 0)
if err != nil {
fmt.Println(err)
continue
}
if int(index) >= len(hrefs) {
fmt.Println("Invalid index")
continue
}
// Only support url with format ck101.com/thread-xxx
if threadId.Match([]byte(hrefs[index])) {
crawler(hrefs[index], 25)
fmt.Println("Done!")
} else {
fmt.Println("Unsupport url:", hrefs[index])
}
default:
fmt.Println("Unrecognized command:", cmd, args)
}
}
},
}
rootCmd.AddCommand(searchCmd)
rootCmd.Execute()
}
//////////////////////////////////////////////////////////////////////////////////////////////////////
// 2way of run
// - 1st: graphite_m web
// call from brower: http://localhost:8080/main/core/1418,1419,2502,2694,2932,2933,2695
// - 2nd: graphite_m core/graphite 1418,1419,2502,2694,2932,2933,2695
//////////////////////////////////////////////////////////////////////////////////////////////////////
func main() {
var logfile = LOGPATH
if _, err := os.Stat(LOGPATH); err != nil {
LOGPATH, _ := os.Getwd()
logfile = LOGPATH + "/graphite_m.log"
}
LOGFILE, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
LOG.Fatalf("Log file error: %s %s", logfile, err)
}
defer func() {
LOGFILE.WriteString(fmt.Sprintf("closing %s", time.UnixDate))
LOGFILE.Close()
}()
logback := logging.NewLogBackend(LOGFILE, "", 0)
logformatted := logging.NewBackendFormatter(logback, LOGFORMAT)
loglevel := "DEBUG"
GLOGLEVEL, err := logging.LogLevel(loglevel)
if err != nil {
GLOGLEVEL = logging.DEBUG
}
logging.SetBackend(logformatted)
logging.SetLevel(GLOGLEVEL, "")
// cfg, err := ini.LoadFile(*configfile)
// if err != nil {
// LOG.Fatalf("parse config "+*configfile+" file error: %s", err)
// }
//
// logfile, ok := cfg.Get("core_api_url", "logfile")
// if !ok {
// LOG.Fatalf("'logfile' missing from 'system' section")
// }
DOMAIN = readValConf(CONFIGFILE, "core_api_url")
if DOMAIN == "" {
DOMAIN = "http://core.local.xdn.com"
}
programName := os.Args[0:1]
if len(os.Args) < 2 {
HCIDS = append(HCIDS, 1418, 1419, 2502, 2694, 2932, 2933, 2695)
mainExec()
} else {
typeStr := os.Args[1:2]
if len(os.Args) >= 3 {
hcidStr := os.Args[2:3]
allArgs := os.Args[1:]
fmt.Println(programName, typeStr, hcidStr, allArgs)
arry := strings.Split(hcidStr[0], ",")
for i := range arry {
n, _ := strconv.Atoi(arry[i])
HCIDS = append(HCIDS, n)
}
} else {
allArgs := os.Args[1:]
fmt.Println(programName, typeStr, allArgs)
}
if typeStr[0] == "web" {
webserver()
} else {
STYPE = typeStr[0]
mainExec()
}
}
fmt.Println(HCIDS)
}
func main() {
flag.Usage = usage
flag.Parse()
runtime.SetBlockProfileRate(*blockProfileRate)
runtime.MemProfileRate = *memProfileRate
// validation defaults
config.Validation_level_legacy.Level = m20.MediumLegacy
config.Validation_level_m20.Level = m20.MediumM20
config_file = "/etc/carbon-relay-ng.ini"
if 1 == flag.NArg() {
config_file = flag.Arg(0)
}
if _, err := toml.DecodeFile(config_file, &config); err != nil {
log.Error("Cannot use config file '%s':\n", config_file)
log.Error(err.Error())
usage()
return
}
//runtime.SetBlockProfileRate(1) // to enable block profiling. in my experience, adds 35% overhead.
levels := map[string]logging.Level{
"critical": logging.CRITICAL,
"error": logging.ERROR,
"warning": logging.WARNING,
"notice": logging.NOTICE,
"info": logging.INFO,
"debug": logging.DEBUG,
}
level, ok := levels[config.Log_level]
if !ok {
log.Error("unrecognized log level '%s'\n", config.Log_level)
return
}
logging.SetLevel(level, "carbon-relay-ng")
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if len(config.Instance) == 0 {
log.Error("instance identifier cannot be empty")
os.Exit(1)
}
runtime.GOMAXPROCS(config.Max_procs)
log.Notice("===== carbon-relay-ng instance '%s' starting. =====\n", config.Instance)
stats.New(config.Instance)
if config.Pid_file != "" {
f, err := os.Create(config.Pid_file)
if err != nil {
fmt.Println("error creating pidfile:", err.Error())
os.Exit(1)
}
_, err = f.Write([]byte(strconv.Itoa(os.Getpid())))
if err != nil {
fmt.Println("error writing to pidfile:", err.Error())
os.Exit(1)
}
f.Close()
}
if config.Instrumentation.Graphite_addr != "" {
addr, err := net.ResolveTCPAddr("tcp", config.Instrumentation.Graphite_addr)
if err != nil {
log.Fatal(err)
}
go metrics.Graphite(metrics.DefaultRegistry, time.Duration(config.Instrumentation.Graphite_interval)*time.Millisecond, "", addr)
}
log.Notice("creating routing table...")
maxAge, err := time.ParseDuration(config.Bad_metrics_max_age)
if err != nil {
log.Error("could not parse badMetrics max age")
log.Error(err.Error())
os.Exit(1)
}
badMetrics = badmetrics.New(maxAge)
table = tbl.New(config.Spool_dir)
log.Notice("initializing routing table...")
for i, cmd := range config.Init {
log.Notice("applying: %s", cmd)
err = imperatives.Apply(table, cmd)
if err != nil {
log.Error("could not apply init cmd #%d", i+1)
log.Error(err.Error())
os.Exit(1)
}
}
tablePrinted := table.Print()
log.Notice("===========================")
log.Notice("========== TABLE ==========")
log.Notice("===========================")
for _, line := range strings.Split(tablePrinted, "\n") {
log.Notice(line)
}
if config.Listen_addr != "" {
_, err = input.NewPlain(config, config.Listen_addr, table, badMetrics)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
if config.Pickle_addr != "" {
_, err = input.NewPickle(config, config.Pickle_addr, table, badMetrics)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
if config.Admin_addr != "" {
go func() {
err := telnet.Start(config.Admin_addr, table)
if err != nil {
fmt.Println("Error listening:", err.Error())
os.Exit(1)
}
}()
}
if config.Http_addr != "" {
go web.Start(config.Http_addr, config, table, badMetrics)
}
select {}
}
// SetLogLevel sets the package logging level
func SetLogLevel(level string) {
logLevel, _ := logging.LogLevel(strings.ToUpper(level)) // TODO Validate input
logging.SetLevel(logLevel, logger.Module)
}
func main() {
logBackend := logging.NewLogBackend(os.Stderr, "", 0)
logging.SetBackend(logBackend)
logging.SetFormatter(logging.MustStringFormatter(format))
logging.SetLevel(logging.NOTICE, "streamLog")
homeDir := os.Getenv("HOME")
app := cli.NewApp()
app.Name = "tarzan"
app.Usage = "naive cached automated build implementation"
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "p,port",
Value: "3000",
Usage: "port to serve tarzan on",
},
cli.StringFlag{
Name: "alt-registry",
Value: "",
Usage: "alternative registry to push images to instead of Docker Hub",
},
cli.StringFlag{
Name: "secret",
Value: "",
Usage: "secret to use when receiving webhook payload",
},
cli.StringFlag{
Name: "hub-name",
Value: "",
Usage: "specify a username on Docker Hub which is different than your Github handle",
},
cli.StringFlag{
Name: "docker-binary-name",
Value: "docker",
Usage: "specify the docker binary name (if it is not docker in $PATH)",
},
}
app.Action = func(c *cli.Context) {
dockerBinary := c.String("docker-binary-name")
if err := loadBuildStatus(); err != nil {
fmt.Fprintln(os.Stderr, "Error loading build status (did your repos/ dir get corrupted?):", err)
os.Exit(1)
}
if _, err := os.Stat(fmt.Sprintf("%s/.dockercfg", homeDir)); err != nil {
if os.IsNotExist(err) {
log.Warning("Detected no Docker Hub login. Please log in now.")
cmd := exec.Command(dockerBinary, "login")
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
fmt.Fprintln(os.Stderr, "Error running docker login")
os.Exit(1)
}
}
}
r := render.New(render.Options{})
router := mux.NewRouter()
router.HandleFunc("/build", BuildHookReceiver(c, r, dockerBinary)).Methods("POST")
router.HandleFunc("/buildList", MakeBuildListHandler(r)).Methods("GET")
n := negroni.Classic()
n.Use(negroni.NewStatic(http.Dir("frontend/")))
n.UseHandler(router)
n.Run(fmt.Sprintf(":%s", c.String("port")))
}
app.Run(os.Args)
}
func main() {
flag.Usage = usage
flag.Parse()
runtime.SetBlockProfileRate(*blockProfileRate)
runtime.MemProfileRate = *memProfileRate
// Default to strict validation
config.Legacy_metric_validation.Level = m20.Strict
config_file = "/etc/carbon-relay-ng.ini"
if 1 == flag.NArg() {
config_file = flag.Arg(0)
}
if _, err := toml.DecodeFile(config_file, &config); err != nil {
log.Error("Cannot use config file '%s':\n", config_file)
log.Error(err.Error())
usage()
return
}
//runtime.SetBlockProfileRate(1) // to enable block profiling. in my experience, adds 35% overhead.
levels := map[string]logging.Level{
"critical": logging.CRITICAL,
"error": logging.ERROR,
"warning": logging.WARNING,
"notice": logging.NOTICE,
"info": logging.INFO,
"debug": logging.DEBUG,
}
level, ok := levels[config.Log_level]
if !ok {
log.Error("unrecognized log level '%s'\n", config.Log_level)
return
}
logging.SetLevel(level, "carbon-relay-ng")
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if len(config.Instance) == 0 {
log.Error("instance identifier cannot be empty")
os.Exit(1)
}
runtime.GOMAXPROCS(config.max_procs)
instance = config.Instance
expvar.NewString("instance").Set(instance)
expvar.NewString("service").Set(service)
log.Notice("===== carbon-relay-ng instance '%s' starting. =====\n", instance)
numIn = Counter("unit=Metric.direction=in")
numInvalid = Counter("unit=Err.type=invalid")
if config.Instrumentation.Graphite_addr != "" {
addr, err := net.ResolveTCPAddr("tcp", config.Instrumentation.Graphite_addr)
if err != nil {
log.Fatal(err)
}
go metrics.Graphite(metrics.DefaultRegistry, time.Duration(config.Instrumentation.Graphite_interval)*time.Millisecond, "", addr)
}
log.Notice("creating routing table...")
maxAge, err := time.ParseDuration(config.Bad_metrics_max_age)
if err != nil {
log.Error("could not parse badMetrics max age")
log.Error(err.Error())
os.Exit(1)
}
badMetrics = badmetrics.New(maxAge)
table = NewTable(config.Spool_dir)
log.Notice("initializing routing table...")
for i, cmd := range config.Init {
log.Notice("applying: %s", cmd)
err = applyCommand(table, cmd)
if err != nil {
log.Error("could not apply init cmd #%d", i+1)
log.Error(err.Error())
os.Exit(1)
}
}
tablePrinted := table.Print()
log.Notice("===========================")
log.Notice("========== TABLE ==========")
log.Notice("===========================")
for _, line := range strings.Split(tablePrinted, "\n") {
log.Notice(line)
}
// Follow the goagain protocol, <https://github.com/rcrowley/goagain>.
l, ppid, err := goagain.GetEnvs()
if nil != err {
laddr, err := net.ResolveTCPAddr("tcp", config.Listen_addr)
if nil != err {
log.Error(err.Error())
os.Exit(1)
}
l, err = net.ListenTCP("tcp", laddr)
if nil != err {
log.Error(err.Error())
os.Exit(1)
}
log.Notice("listening on %v/tcp", laddr)
go accept(l.(*net.TCPListener), config)
} else {
log.Notice("resuming listening on %v/tcp", l.Addr())
go accept(l.(*net.TCPListener), config)
if err := goagain.KillParent(ppid); nil != err {
log.Error(err.Error())
os.Exit(1)
}
for {
err := syscall.Kill(ppid, 0)
if err != nil {
break
}
time.Sleep(10 * time.Millisecond)
}
}
udp_addr, err := net.ResolveUDPAddr("udp", config.Listen_addr)
if nil != err {
log.Error(err.Error())
os.Exit(1)
}
udp_conn, err := net.ListenUDP("udp", udp_addr)
if nil != err {
log.Error(err.Error())
os.Exit(1)
}
log.Notice("listening on %v/udp", udp_addr)
go handle(udp_conn, config)
if config.Pid_file != "" {
f, err := os.Create(config.Pid_file)
if err != nil {
fmt.Println("error creating pidfile:", err.Error())
os.Exit(1)
}
_, err = f.Write([]byte(strconv.Itoa(os.Getpid())))
if err != nil {
fmt.Println("error writing to pidfile:", err.Error())
os.Exit(1)
}
f.Close()
}
if config.Admin_addr != "" {
go func() {
err := adminListener(config.Admin_addr)
if err != nil {
fmt.Println("Error listening:", err.Error())
os.Exit(1)
}
}()
}
if config.Http_addr != "" {
go HttpListener(config.Http_addr, table)
}
if err := goagain.AwaitSignals(l); nil != err {
log.Error(err.Error())
os.Exit(1)
}
}
// SetLevel sets the desired log level for the default logger.
func SetLevel(level oplogging.Level) {
oplogging.SetLevel(level, logModule)
}
func main() {
var format = logging.MustStringFormatter("%{level} %{message}")
logging.SetFormatter(format)
logging.SetLevel(logging.INFO, "iloveptt")
usr, _ := user.Current()
baseDir = fmt.Sprintf("%v/Pictures/iloveptt", usr.HomeDir)
var workerNum int
rootCmd := &cobra.Command{
Use: "iloveptt",
Short: "Download all the images in given post url",
Run: func(cmd *cobra.Command, args []string) {
page := 0
hrefs := parsePttBoardIndex(page)
scanner := bufio.NewScanner(os.Stdin)
quit := false
for !quit {
fmt.Print("ptt:> ")
if !scanner.Scan() {
break
}
line := scanner.Text()
parts := strings.Split(line, " ")
cmd := parts[0]
args := parts[1:]
switch cmd {
case "quit":
quit = true
case "n":
page = page + 1
hrefs = parsePttBoardIndex(page)
case "p":
if page > 0 {
page = page - 1
}
hrefs = parsePttBoardIndex(page)
case "s":
page = 0
hrefs = parsePttBoardIndex(page)
case "o":
open.Run(filepath.FromSlash(baseDir))
case "d":
if len(args) == 0 {
fmt.Println("You don't input any article index. Input as 'd 1'")
continue
}
index, err := strconv.ParseUint(args[0], 0, 0)
if err != nil {
fmt.Println(err)
continue
}
if int(index) >= len(hrefs) {
fmt.Println("Invalid index")
continue
}
if threadId.Match([]byte(hrefs[index])) {
crawler(hrefs[index], 25)
fmt.Println("Done!")
} else {
fmt.Println("Unsupport url:", hrefs[index])
}
default:
fmt.Println("Unrecognized command:", cmd, args)
}
}
},
}
rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 25, "Number of workers")
rootCmd.Execute()
}