Example #1
0
func main() {
	limitFl := flag.Int64("limit", 10240, "Page read size limit")
	flag.Parse()

	for _, urlStr := range os.Args[1:] {
		resp, err := http.Get(urlStr)
		if err != nil {
			log.Fatal("cannot GET url", "url", urlStr, "error", err.Error())
		}
		defer resp.Body.Close()

		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, *limitFl))
		if err != nil {
			log.Fatal("cannot read HTTP response", "url", urlStr, "error", err.Error())
		}

		s := html.UnescapeString(string(b))
		doc, err := readability.NewDocument(s)
		if err != nil {
			log.Fatal("cannot parse page", "error", err.Error())
		}

		fmt.Println(doc.Content())
	}
}
Example #2
0
File: main.go Project: husio/apps
func main() {
	conf := struct {
		HTTP     string
		Postgres string
		Schema   string
	}{
		HTTP:     "localhost:8000",
		Postgres: "dbname=postgres user=postgres sslmode=disable",
	}
	if err := envconf.LoadEnv(&conf); err != nil {
		log.Fatal("cannot load configuration", "error", err.Error())
	}

	ctx, done := context.WithCancel(context.Background())
	defer done()

	db, err := sql.Open("postgres", conf.Postgres)
	if err != nil {
		log.Fatal("cannot connect to database", "error", err.Error())
	}
	defer db.Close()
	if err := db.Ping(); err != nil {
		log.Error("cannot ping database",
			"postgres", conf.Postgres,
			"error", err.Error())
	}
	ctx = pg.WithDB(ctx, db)

	if conf.Schema != "" {
		if err := pg.LoadSchema(db, conf.Schema); err != nil {
			log.Error("cannot load schema",
				"schema", conf.Schema,
				"error", err.Error())
		}
	}

	app := NewApplication(ctx)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}
Example #3
0
func main() {
	ctx := context.Background()

	db, err := sql.Open("sqlite3", "gallery.sqlite3")
	if err != nil {
		log.Fatal("cannot open database", "error", err.Error())
	}
	if err := db.Ping(); err != nil {
		log.Fatal("cannot ping database", "error", err.Error())
	}
	ctx = sq.WithDB(ctx, db)

	ctx = gallery.WithFileStore(ctx, "/tmp/gallery")

	app := gallery.NewApplication(ctx)

	log.Debug("running HTTP server", "address", ":8000")
	if err := http.ListenAndServe(":8000", app); err != nil {
		log.Fatal("HTTP server error", "error", err.Error())
	}
}
Example #4
0
File: scrap.go Project: husio/apps
func main() {
	redisFl := flag.String("redis", "redis://127.0.0.1:6379/0", "Redis address")
	stopwFl := flag.String("stopw", "", "Stopwords list")
	httpFl := flag.String("http", "localhost:8000", "HTTP server address")
	flag.Parse()

	rp := redisPool(*redisFl)
	defer rp.Close()

	urlc := make(chan string, 100)

	stopw := make(map[string]struct{})
	if *stopwFl != "" {
		stopw = stopwords(*stopwFl)
	}

	for i := 0; i < 10; i++ {
		go crawler(rp, urlc, stopw)
	}

	for _, u := range flag.Args() {
		urlc <- u
	}

	handleArticles := articleHandler(rp)
	handleScrap := scrapHandler(urlc)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		switch r.Method {
		case "GET":
			handleArticles(w, r)
		case "POST":
			handleScrap(w, r)
		default:
			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		}
	})
	if err := http.ListenAndServe(*httpFl, nil); err != nil {
		log.Fatal("HTTP server error", "error", err.Error())
	}
}
Example #5
0
File: bb.go Project: husio/apps
func main() {
	conf := struct {
		HTTP     string
		Postgres string
	}{
		HTTP:     "localhost:8000",
		Postgres: "host=localhost port=5432 user=postgres dbname=bb sslmode=disable",
	}
	envconf.Must(envconf.LoadEnv(&conf))

	ctx := context.Background()

	ctx = auth.WithOAuth(ctx, map[string]*oauth2.Config{
		"google": &oauth2.Config{
			ClientID:     "352914691292-2h70272sb408r3vibe4jm4egote804ka.apps.googleusercontent.com",
			ClientSecret: "L_bgOHLCgNYL-3KG8a5u99mF",
			RedirectURL:  "http://bb.example.com:8000/login/success",
			Scopes: []string{
				"https://www.googleapis.com/auth/userinfo.profile",
				"https://www.googleapis.com/auth/userinfo.email",
			},
			Endpoint: oauth2google.Endpoint,
		},
	})

	ctx = cache.WithLocalCache(ctx, 1000)

	db, err := sql.Open("postgres", conf.Postgres)
	if err != nil {
		log.Fatal("cannot open database", "error", err.Error())
	}
	defer db.Close()
	ctx = pg.WithDB(ctx, db)
	go func() {
		if err := db.Ping(); err != nil {
			log.Error("cannot ping database", "error", err.Error())
		}
	}()

	app := bb.NewApp(ctx)
	log.Debug("running HTTP server", "address", conf.HTTP)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}
Example #6
0
File: main.go Project: husio/apps
func main() {
	conf := struct {
		HTTP string
	}{
		HTTP: "localhost:8000",
	}
	if err := envconf.LoadEnv(&conf); err != nil {
		log.Fatal("cannot load configuration", "error", err.Error())
	}

	ctx := context.Background()

	km, stop := keyManager()
	defer stop()
	ctx = keys.WithManager(ctx, km)

	app := NewApplication(ctx)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}