func main() {
limitFl := flag.Int64("limit", 10240, "Page read size limit")
flag.Parse()
for _, urlStr := range os.Args[1:] {
resp, err := http.Get(urlStr)
if err != nil {
log.Fatal("cannot GET url", "url", urlStr, "error", err.Error())
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(io.LimitReader(resp.Body, *limitFl))
if err != nil {
log.Fatal("cannot read HTTP response", "url", urlStr, "error", err.Error())
}
s := html.UnescapeString(string(b))
doc, err := readability.NewDocument(s)
if err != nil {
log.Fatal("cannot parse page", "error", err.Error())
}
fmt.Println(doc.Content())
}
}
func main() {
conf := struct {
HTTP string
Postgres string
Schema string
}{
HTTP: "localhost:8000",
Postgres: "dbname=postgres user=postgres sslmode=disable",
}
if err := envconf.LoadEnv(&conf); err != nil {
log.Fatal("cannot load configuration", "error", err.Error())
}
ctx, done := context.WithCancel(context.Background())
defer done()
db, err := sql.Open("postgres", conf.Postgres)
if err != nil {
log.Fatal("cannot connect to database", "error", err.Error())
}
defer db.Close()
if err := db.Ping(); err != nil {
log.Error("cannot ping database",
"postgres", conf.Postgres,
"error", err.Error())
}
ctx = pg.WithDB(ctx, db)
if conf.Schema != "" {
if err := pg.LoadSchema(db, conf.Schema); err != nil {
log.Error("cannot load schema",
"schema", conf.Schema,
"error", err.Error())
}
}
app := NewApplication(ctx)
if err := http.ListenAndServe(conf.HTTP, app); err != nil {
log.Error("HTTP server error", "error", err.Error())
}
}
func main() {
ctx := context.Background()
db, err := sql.Open("sqlite3", "gallery.sqlite3")
if err != nil {
log.Fatal("cannot open database", "error", err.Error())
}
if err := db.Ping(); err != nil {
log.Fatal("cannot ping database", "error", err.Error())
}
ctx = sq.WithDB(ctx, db)
ctx = gallery.WithFileStore(ctx, "/tmp/gallery")
app := gallery.NewApplication(ctx)
log.Debug("running HTTP server", "address", ":8000")
if err := http.ListenAndServe(":8000", app); err != nil {
log.Fatal("HTTP server error", "error", err.Error())
}
}
func main() {
redisFl := flag.String("redis", "redis://127.0.0.1:6379/0", "Redis address")
stopwFl := flag.String("stopw", "", "Stopwords list")
httpFl := flag.String("http", "localhost:8000", "HTTP server address")
flag.Parse()
rp := redisPool(*redisFl)
defer rp.Close()
urlc := make(chan string, 100)
stopw := make(map[string]struct{})
if *stopwFl != "" {
stopw = stopwords(*stopwFl)
}
for i := 0; i < 10; i++ {
go crawler(rp, urlc, stopw)
}
for _, u := range flag.Args() {
urlc <- u
}
handleArticles := articleHandler(rp)
handleScrap := scrapHandler(urlc)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
switch r.Method {
case "GET":
handleArticles(w, r)
case "POST":
handleScrap(w, r)
default:
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
}
})
if err := http.ListenAndServe(*httpFl, nil); err != nil {
log.Fatal("HTTP server error", "error", err.Error())
}
}
func main() {
conf := struct {
HTTP string
Postgres string
}{
HTTP: "localhost:8000",
Postgres: "host=localhost port=5432 user=postgres dbname=bb sslmode=disable",
}
envconf.Must(envconf.LoadEnv(&conf))
ctx := context.Background()
ctx = auth.WithOAuth(ctx, map[string]*oauth2.Config{
"google": &oauth2.Config{
ClientID: "352914691292-2h70272sb408r3vibe4jm4egote804ka.apps.googleusercontent.com",
ClientSecret: "L_bgOHLCgNYL-3KG8a5u99mF",
RedirectURL: "http://bb.example.com:8000/login/success",
Scopes: []string{
"https://www.googleapis.com/auth/userinfo.profile",
"https://www.googleapis.com/auth/userinfo.email",
},
Endpoint: oauth2google.Endpoint,
},
})
ctx = cache.WithLocalCache(ctx, 1000)
db, err := sql.Open("postgres", conf.Postgres)
if err != nil {
log.Fatal("cannot open database", "error", err.Error())
}
defer db.Close()
ctx = pg.WithDB(ctx, db)
go func() {
if err := db.Ping(); err != nil {
log.Error("cannot ping database", "error", err.Error())
}
}()
app := bb.NewApp(ctx)
log.Debug("running HTTP server", "address", conf.HTTP)
if err := http.ListenAndServe(conf.HTTP, app); err != nil {
log.Error("HTTP server error", "error", err.Error())
}
}
func main() {
conf := struct {
HTTP string
}{
HTTP: "localhost:8000",
}
if err := envconf.LoadEnv(&conf); err != nil {
log.Fatal("cannot load configuration", "error", err.Error())
}
ctx := context.Background()
km, stop := keyManager()
defer stop()
ctx = keys.WithManager(ctx, km)
app := NewApplication(ctx)
if err := http.ListenAndServe(conf.HTTP, app); err != nil {
log.Error("HTTP server error", "error", err.Error())
}
}