Beispiel #1
0
func handleImageDetails(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	db := sq.DB(ctx)
	img, err := ImageByID(db, web.Args(ctx).ByIndex(0))
	switch err {
	case nil:
		// all good
	case sq.ErrNotFound:
		web.StdHTMLResp(w, http.StatusNotFound)
		return
	default:
		log.Error("cannot get image",
			"image", web.Args(ctx).ByIndex(0),
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	img.Tags, err = ImageTags(db, img.ImageID)
	if err != nil {
		log.Error("cannot get image",
			"image", web.Args(ctx).ByIndex(0),
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	web.JSONResp(w, img, http.StatusOK)
}
Beispiel #2
0
func fetch(urls []string) Entries {
	var result Entries

	p := gofeed.NewParser()

	for _, url := range urls {
		resp, err := http.Get(url)
		if err != nil {
			log.Error("cannot fetch feed", "url", url, "error", err.Error())
			continue
		}

		feed, err := p.Parse(resp.Body)
		resp.Body.Close()

		if err != nil {
			log.Error("cannot parse feed", "url", url, "error", err.Error())
			continue
		}
		for _, it := range feed.Items {
			result = append(result, &Entry{
				Feed: Feed{
					Title: html.UnescapeString(feed.Title),
					Link:  feed.Link,
				},
				Title:     html.UnescapeString(it.Title),
				Link:      it.Link,
				Published: parseTime(it.Published),
			})
		}
	}
	return result
}
Beispiel #3
0
func handleCreateTopic(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	account, ok := auth.AuthRequired(pg.DB(ctx), w, r)
	if !ok {
		return
	}

	input := struct {
		Title   string
		Tags    []string
		Content string
	}{
		Title:   r.FormValue("title"),
		Tags:    strings.Fields(r.FormValue("tags")),
		Content: r.FormValue("content"),
	}

	var errs []string
	if r.Method == "POST" {
		if input.Title == "" {
			errs = append(errs, `"title" is required`)
		}
		if input.Content == "" {
			errs = append(errs, `"content" is required`)
		}
	}

	if r.Method == "GET" || len(errs) != 0 {
		render(w, "topic_create.tmpl", input)
		return
	}

	db := pg.DB(ctx)
	tx, err := db.Beginx()
	if err != nil {
		log.Error("cannot start transaction", "error", err.Error())
		respond500(w, r)
		return
	}
	defer tx.Rollback()

	topic := Topic{
		AuthorID: int64(account.AccountID),
		Title:    input.Title,
		Tags:     input.Tags,
	}
	t, _, err := CreateTopicWithComment(tx, topic, input.Content)
	if err != nil {
		log.Error("cannot create topic with comment", "error", err.Error())
		respond500(w, r)
		return
	}

	if err := tx.Commit(); err != nil {
		log.Error("cannot commit transaction", "error", err.Error())
		respond500(w, r)
		return
	}

	http.Redirect(w, r, fmt.Sprintf("/t/%d", t.TopicID), http.StatusSeeOther)
}
Beispiel #4
0
func main() {
	go func() {
		sigc := make(chan os.Signal)
		signal.Notify(sigc, syscall.SIGUSR1, syscall.SIGHUP)
		defer signal.Stop(sigc)

		log.Debug("updating feeds", "trigger", "init")
		feedreader.Update()

		for {
			select {
			case now := <-time.After(30 * time.Minute):
				log.Debug("updating feeds", "trigger", "clock", "time", now.String())
			case sig := <-sigc:
				log.Debug("updating feeds", "trigger", "signal", "signal", sig.String())
			}
			feedreader.Update()
		}
	}()

	rt := web.NewRouter(web.Routes{
		{"GET", `/`, feedreader.HandleListEntries},
		{"GET", `/sources`, feedreader.HandleListSources},
		{web.AnyMethod, `.*`, feedreader.Handle404},
	})
	if err := http.ListenAndServe(":8000", rt); err != nil {
		log.Error("HTTP server failed", "error", err.Error())
	}
}
Beispiel #5
0
func handleListBookmarks(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	offset, _ := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
	if offset < 0 {
		offset = 0
	}

	bookmarks := make([]*Bookmark, 0, 100)
	err := pg.DB(ctx).Select(&bookmarks, `
		SELECT b.*
		FROM bookmarks b
		ORDER BY created DESC
		LIMIT $1 OFFSET $2
	`, 500, offset)
	if err != nil {
		log.Error("cannot select bookmarks", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	resp := struct {
		Bookmarks []*Bookmark `json:"bookmarks"`
	}{
		Bookmarks: bookmarks,
	}
	web.JSONResp(w, resp, http.StatusOK)
}
Beispiel #6
0
func words(r io.Reader, stopw map[string]struct{}) map[string]int {
	counts := make(map[string]int)
	scanner := bufio.NewScanner(r)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		w := strings.ToLower(scanner.Text())
		if strings.HasPrefix(w, "<") || strings.HasSuffix(w, ">") {
			continue
		}
		w = strings.TrimRight(w, ",.")

		if len(w) > 40 {
			continue
		}

		if _, ok := stopw[w]; ok {
			continue
		}

		counts[w]++
	}

	if err := scanner.Err(); err != nil {
		log.Error("scanner error", "error", err.Error())
	}
	return counts
}
Beispiel #7
0
func handleAddBookmark(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	var input struct {
		Url string `json:"url"`
	}
	if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
		web.JSONErr(w, err.Error(), http.StatusBadRequest)
		return
	}

	resp, err := ctxhttp.Get(ctx, &crawler, input.Url)
	if err != nil {
		log.Error("cannot crawl",
			"url", input.Url,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	body := make([]byte, 1024*20)
	if n, err := resp.Body.Read(body); err != nil && err != io.EOF {
		log.Error("cannot read crawler response",
			"url", input.Url,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	} else {
		body = body[:n]
	}

	title := pageTitle(body)

	var b Bookmark
	err = pg.DB(ctx).Get(&b, `
		INSERT INTO bookmarks (title, url, created)
		VALUES ($1, $2, $3)
		ON CONFLICT DO NOTHING
		RETURNING *
	`, title, input.Url, time.Now())
	if err != nil {
		log.Error("cannot create bookmark", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	web.JSONResp(w, b, http.StatusCreated)
}
Beispiel #8
0
func imageMeta(r io.ReadSeeker) (*Image, error) {
	conf, err := jpeg.DecodeConfig(r)
	if err != nil {
		return nil, fmt.Errorf("cannot decode JPEG: %s", err)
	}

	// compute image hash from image content
	oid := sha256.New()
	if _, err := io.Copy(oid, r); err != nil {
		return nil, fmt.Errorf("cannot compute SHA: %s", err)
	}
	img := Image{
		ImageID: encode(oid),
		Width:   conf.Width,
		Height:  conf.Height,
	}

	if _, err := r.Seek(0, os.SEEK_SET); err != nil {
		return nil, fmt.Errorf("cannot seek: %s", err)
	}
	if meta, err := exif.Decode(r); err != nil {
		log.Error("cannot extract EXIF metadata", "error", err.Error())
	} else {
		if orientation, err := meta.Get(exif.Orientation); err != nil {
			log.Debug("cannot extract image orientation",
				"decoder", "EXIF",
				"error", err.Error())
		} else {
			if o, err := orientation.Int(0); err != nil {
				log.Debug("cannot format orientation",
					"decoder", "EXIF",
					"error", err.Error())
			} else {
				img.Orientation = o
			}
		}
		if dt, err := meta.Get(exif.DateTimeOriginal); err != nil {
			log.Debug("cannot extract image datetime original",
				"decoder", "EXIF",
				"error", err.Error())
		} else {
			if raw, err := dt.StringVal(); err != nil {
				log.Debug("cannot format datetime original",
					"decoder", "EXIF",
					"error", err.Error())
			} else {
				img.Created, err = time.Parse("2006:01:02 15:04:05", raw)
				if err != nil {
					log.Debug("cannot parse datetime original",
						"decoder", "EXIF",
						"value", raw,
						"error", err.Error())
				}
			}
		}
	}

	return &img, nil
}
Beispiel #9
0
Datei: bb.go Projekt: husio/apps
func main() {
	conf := struct {
		HTTP     string
		Postgres string
	}{
		HTTP:     "localhost:8000",
		Postgres: "host=localhost port=5432 user=postgres dbname=bb sslmode=disable",
	}
	envconf.Must(envconf.LoadEnv(&conf))

	ctx := context.Background()

	ctx = auth.WithOAuth(ctx, map[string]*oauth2.Config{
		"google": &oauth2.Config{
			ClientID:     "352914691292-2h70272sb408r3vibe4jm4egote804ka.apps.googleusercontent.com",
			ClientSecret: "L_bgOHLCgNYL-3KG8a5u99mF",
			RedirectURL:  "http://bb.example.com:8000/login/success",
			Scopes: []string{
				"https://www.googleapis.com/auth/userinfo.profile",
				"https://www.googleapis.com/auth/userinfo.email",
			},
			Endpoint: oauth2google.Endpoint,
		},
	})

	ctx = cache.WithLocalCache(ctx, 1000)

	db, err := sql.Open("postgres", conf.Postgres)
	if err != nil {
		log.Fatal("cannot open database", "error", err.Error())
	}
	defer db.Close()
	ctx = pg.WithDB(ctx, db)
	go func() {
		if err := db.Ping(); err != nil {
			log.Error("cannot ping database", "error", err.Error())
		}
	}()

	app := bb.NewApp(ctx)
	log.Debug("running HTTP server", "address", conf.HTTP)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}
Beispiel #10
0
func render(w io.Writer, templateName string, context interface{}) {
	// XXX cache

	t, err := parseFiles(baseTmplPath, "bb/templates/"+templateName)
	if err != nil {
		log.Error("cannot parse template",
			"template", templateName,
			"error", err.Error())
		return
	}
	t = t.Funcs(baseFuncs)

	if err := t.Execute(w, context); err != nil {
		log.Error("cannot render template",
			"template", templateName,
			"error", err.Error())
		return
	}
}
Beispiel #11
0
func handleTopicDetails(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	db := pg.DB(ctx)

	tid, _ := strconv.ParseInt(web.Args(ctx).ByIndex(0), 10, 64)
	topic, err := TopicByID(db, tid)
	switch err {
	case nil:
		// all good
	case pg.ErrNotFound:
		respond404(w, r)
		return
	default:
		log.Error("cannot get topic by ID",
			"topic", web.Args(ctx).ByIndex(0),
			"error", err.Error())
		respond500(w, r)
		return
	}

	page, _ := strconv.ParseInt(r.URL.Query().Get("page"), 10, 64)
	comments, err := Comments(db, CommentsOpts{
		Offset:  (page - 1) * 200,
		Limit:   200,
		TopicID: topic.TopicID,
	})
	if err != nil {
		log.Error("cannot get comments for topic",
			"topic", fmt.Sprint(topic.TopicID),
			"error", err.Error())
		respond500(w, r)
		return
	}

	context := struct {
		Topic    *Topic
		Comments []*Comment
	}{
		Topic:    topic,
		Comments: comments,
	}
	render(w, "topic_details.tmpl", context)
}
Beispiel #12
0
func main() {
	conf := struct {
		HTTP     string
		Postgres string
		Schema   string
	}{
		HTTP:     "localhost:8000",
		Postgres: "dbname=postgres user=postgres sslmode=disable",
	}
	if err := envconf.LoadEnv(&conf); err != nil {
		log.Fatal("cannot load configuration", "error", err.Error())
	}

	ctx, done := context.WithCancel(context.Background())
	defer done()

	db, err := sql.Open("postgres", conf.Postgres)
	if err != nil {
		log.Fatal("cannot connect to database", "error", err.Error())
	}
	defer db.Close()
	if err := db.Ping(); err != nil {
		log.Error("cannot ping database",
			"postgres", conf.Postgres,
			"error", err.Error())
	}
	ctx = pg.WithDB(ctx, db)

	if conf.Schema != "" {
		if err := pg.LoadSchema(db, conf.Schema); err != nil {
			log.Error("cannot load schema",
				"schema", conf.Schema,
				"error", err.Error())
		}
	}

	app := NewApplication(ctx)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}
Beispiel #13
0
func articleHandler(rp *redis.Pool) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		type Article struct {
			Key   string `redis:"-"`
			Url   string `redis:"url"`
			Title string `redis:"title"`
		}

		rc := rp.Get()
		defer rc.Close()

		var articles []*Article
		for _, w := range r.URL.Query()["word"] {
			keys, err := redis.Strings(rc.Do("ZREVRANGE", "word:"+w, 0, 100))
			if err != nil {
				log.Error("cannot get keys", "error", err.Error())
				continue
			}
			for _, key := range keys {
				raw, err := redis.Values(rc.Do("HGETALL", key))
				if err != nil {
					log.Error("cannot get article",
						"key", key,
						"error", err.Error())
					continue
				}
				var art Article
				if err := redis.ScanStruct(raw, &art); err != nil {
					log.Error("cannot scan article",
						"key", key,
						"error", err.Error())
					continue
				}
				art.Key = key
				articles = append(articles, &art)
			}
		}
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(articles)
	}
}
Beispiel #14
0
func render(w io.Writer, name string, context interface{}) {
	if debugTemplate {
		tmpl = template.Must(template.New("").Funcs(template.FuncMap{
			"favicon":       favicon,
			"hashcolor":     hashcolor,
			"truncatechars": truncatechars,
		}).ParseGlob(getenv("TEMPLATES", "*/*/*.tmpl")))
	}
	if err := tmpl.ExecuteTemplate(w, name, context); err != nil {
		log.Error("cannot render template", "name", name, "error", err.Error())
	}
}
Beispiel #15
0
func main() {
	minRepFl := flag.Int("minrep", 2, "Minimum repetition amount for word to be relevant")
	minWLenFl := flag.Int("minwlen", 3, "Minimum word length")
	stopwFl := flag.String("stopw", "", "Stopwords list")
	flag.Parse()

	stopw := make(map[string]struct{})
	if *stopwFl != "" {
		stopw = stopwords(*stopwFl)
	}

	counts := make(map[string]int)
	scanner := bufio.NewScanner(os.Stdin)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		w := strings.ToLower(scanner.Text())
		if strings.HasPrefix(w, "<") || strings.HasSuffix(w, ">") {
			continue
		}
		w = strings.TrimRight(w, ",.")

		if len(w) > 40 {
			continue
		}

		if len(w) < *minWLenFl {
			continue
		}

		if _, ok := stopw[w]; ok {
			continue
		}

		counts[w]++
	}

	if err := scanner.Err(); err != nil {
		log.Error("scanner error", "error", err.Error())
	}

	var pairs pairs
	for word, count := range counts {
		if count >= *minRepFl {
			pairs = append(pairs, pair{word, count})
		}
	}

	sort.Sort(pairs)

	for _, pair := range pairs {
		fmt.Printf("%s\t%d\n", pair.word, pair.count)
	}
}
Beispiel #16
0
func stopwords(path string) map[string]struct{} {
	stopw := make(map[string]struct{})
	fd, err := os.Open(path)
	if err != nil {
		log.Error("cannot open stopwords file", "error", err.Error())
		return stopw
	}
	defer fd.Close()

	rd := bufio.NewReader(fd)
	for {
		word, err := rd.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				log.Error("cannot read stopwords", "error", err.Error())
			}
			return stopw
		}
		stopw[strings.TrimSpace(word)] = struct{}{}
	}
}
Beispiel #17
0
func Update() {
	urls, err := sources()
	if err != nil {
		log.Error("cannot get sources", "error", err.Error())
		return
	}
	entries := fetch(urls)
	sort.Sort(sort.Reverse(entries))

	feeds.Lock()
	defer feeds.Unlock()

	feeds.entries = entries
}
Beispiel #18
0
func handleListNotes(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	notes, err := ListNotes(ctx)
	if err != nil {
		log.Error("cannot read note", "error", err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	resp := struct {
		Notes []*Note `json:"notes"`
	}{
		Notes: notes,
	}
	web.JSONResp(w, resp, http.StatusOK)
}
Beispiel #19
0
func keyManager() (*keys.KeyManager, func()) {
	var m keys.KeyManager
	t := time.NewTicker(24 * time.Hour)

	go func() {
		for range t.C {
			if id, err := m.GenerateKey(24 * 7 * time.Hour); err != nil {
				log.Error("cannot generate key", "error", err.Error())
			} else {
				log.Debug("new key generated", "id", id)
			}
		}
	}()

	return &m, t.Stop
}
Beispiel #20
0
func handleStorePaste(pid string, w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024))
	if err != nil {
		log.Error("cannot read body", "error", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "internal server error")
		return
	}

	db.mu.Lock()
	db.mem[pid] = b
	defer db.mu.Unlock()

	w.WriteHeader(http.StatusCreated)
	fmt.Fprintln(w, pid)
}
Beispiel #21
0
func scrapHandler(urlc chan string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Error("cannot read body", "error", err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		go func() {
			for _, urlStr := range strings.Fields(string(b)) {
				urlc <- urlStr
			}
		}()

		fmt.Fprintln(w, "ok")
	}
}
Beispiel #22
0
func handleListTopics(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	opts := TopicsOpts{
		Limit: 200,
		Tags:  r.URL.Query()["tag"],
	}

	db := pg.DB(ctx)
	topics, err := Topics(db, opts)
	if err != nil {
		log.Error("cannot list topics", "error", err.Error())
	}

	context := struct {
		Topics []*TopicWithAuthor
	}{
		Topics: topics,
	}
	render(w, "topic_list.tmpl", context)
}
Beispiel #23
0
func main() {
	conf := struct {
		HTTP string
	}{
		HTTP: "localhost:8000",
	}
	if err := envconf.LoadEnv(&conf); err != nil {
		log.Fatal("cannot load configuration", "error", err.Error())
	}

	ctx := context.Background()

	km, stop := keyManager()
	defer stop()
	ctx = keys.WithManager(ctx, km)

	app := NewApplication(ctx)
	if err := http.ListenAndServe(conf.HTTP, app); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}
Beispiel #24
0
func handleCreateComment(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	input := struct {
		Content string
	}{
		Content: r.FormValue("content"),
	}

	var errs []string
	if len(input.Content) == 0 {
		errs = append(errs, `"content" is required`)
	}
	if len(errs) != 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "%v", errs)
		return
	}

	db := pg.DB(ctx)
	tid, _ := strconv.ParseInt(web.Args(ctx).ByIndex(0), 10, 64)
	c, err := CreateComment(db, Comment{
		TopicID:  tid,
		Content:  input.Content,
		AuthorID: 1,
	})
	switch err {
	case nil:
		// ok
	default:
		log.Error("cannot create comment",
			"topic", fmt.Sprint(tid),
			"error", err.Error())
		respond500(w, r)
		return
	}

	http.Redirect(w, r, fmt.Sprintf("/t/%d", c.TopicID), http.StatusSeeOther)
}
Beispiel #25
0
func handleListImages(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	offset, _ := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
	opts := ImagesOpts{
		Offset: offset,
		Limit:  200,
	}

	// narrow to images tagged as specified
	for name, values := range r.URL.Query() {
		if !strings.HasPrefix(name, "tag_") {
			continue
		}
		for _, value := range values {
			opts.Tags = append(opts.Tags, KeyValue{
				Key:   name[4:],
				Value: value,
			})
		}
	}

	imgs, err := Images(sq.DB(ctx), opts)
	if err != nil {
		log.Error("cannot list images", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	if imgs == nil {
		imgs = make([]*Image, 0)
	}

	resp := struct {
		Images []*Image `json:"images"`
	}{
		Images: imgs,
	}
	web.JSONResp(w, resp, http.StatusOK)
}
Beispiel #26
0
func handleTagImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	var input struct {
		Name  string
		Value string
	}
	if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
		web.JSONErr(w, err.Error(), http.StatusBadRequest)
		return
	}

	var errs []string
	if input.Name == "" {
		errs = append(errs, `"name" is required`)
	}
	if input.Value == "" {
		errs = append(errs, `"value" is required`)
	}
	if len(errs) != 0 {
		web.JSONErrs(w, errs, http.StatusBadRequest)
		return
	}

	db := sq.DB(ctx)

	img, err := ImageByID(db, web.Args(ctx).ByIndex(0))
	switch err {
	case nil:
		// all good
	case sq.ErrNotFound:
		web.JSONErr(w, "parent image does not exist", http.StatusBadRequest)
		return
	default:
		log.Error("database error",
			"image", web.Args(ctx).ByIndex(0),
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	tag, err := CreateTag(db, Tag{
		ImageID: img.ImageID,
		Name:    input.Name,
		Value:   input.Value,
	})
	switch err {
	case nil:
		// all good, update storage meta
	case sq.ErrConflict:
		web.JSONResp(w, tag, http.StatusOK)
		return
	default:
		log.Error("cannot create object", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	if img.Tags, err = ImageTags(db, img.ImageID); err != nil {
		log.Error("cannot get image tags",
			"image", img.ImageID,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	fs := FileStore(ctx)
	if err := fs.PutMeta(img); err != nil {
		log.Error("cannot store image metadata",
			"image", img.ImageID,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	web.JSONResp(w, tag, http.StatusCreated)
}
Beispiel #27
0
func handleServeImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	img, err := ImageByID(sq.DB(ctx), web.Args(ctx).ByIndex(0))
	switch err {
	case nil:
		// all good
	case sq.ErrNotFound:
		web.StdJSONResp(w, http.StatusNotFound)
		return
	default:
		log.Error("cannot get object",
			"object", web.Args(ctx).ByIndex(0),
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	if web.CheckLastModified(w, r, img.Created) {
		return
	}

	fs := FileStore(ctx)
	fd, err := fs.Read(img.Created.Year(), img.ImageID)
	if err != nil {
		log.Error("cannot read image file",
			"image", img.ImageID,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	defer fd.Close()

	w.Header().Set("X-Image-ID", img.ImageID)
	w.Header().Set("X-Image-Width", fmt.Sprint(img.Width))
	w.Header().Set("X-Image-Height", fmt.Sprint(img.Height))
	w.Header().Set("X-Image-Created", img.Created.Format(time.RFC3339))
	w.Header().Set("Content-Type", "image/jpeg")

	if r.URL.Query().Get("resize") == "" {
		io.Copy(w, fd)
		return
	}

	image, err := jpeg.Decode(fd)
	if err != nil {
		log.Error("cannot read image file",
			"image", img.ImageID,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	var width, height int
	if _, err := fmt.Sscanf(r.URL.Query().Get("resize"), "%dx%d", &width, &height); err != nil {
		log.Error("cannot resize image",
			"image", img.ImageID,
			"error", err.Error())
	} else {
		switch img.Orientation {
		case 1:
			// all good
		case 3:
			image = imaging.Rotate180(image)
		case 8:
			image = imaging.Rotate90(image)
		case 6:
			image = imaging.Rotate270(image)
		default:
			log.Debug("unknown image orientation",
				"decoder", "EXIF",
				"image", img.ImageID,
				"value", fmt.Sprint(img.Orientation))
		}
		image = imaging.Fill(image, width, height, imaging.Center, imaging.Linear)
	}
	imaging.Encode(w, image, imaging.JPEG)
}
Beispiel #28
0
func handleUploadImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if err := r.ParseMultipartForm(10 * megabyte); err != nil {
		web.JSONResp(w, err.Error(), http.StatusBadRequest)
		return
	}

	var header *multipart.FileHeader
	for _, headers := range r.MultipartForm.File {
		for _, h := range headers {
			log.Debug("uploading file", "name", h.Filename)
			if header != nil {
				web.JSONErr(w, "cannot upload more than one time at once", http.StatusBadRequest)
				return
			}
			header = h
		}
	}
	if header == nil {
		web.JSONErr(w, "image file missing", http.StatusBadRequest)
		return
	}
	if !strings.HasSuffix(strings.ToLower(header.Filename), ".jpg") {
		// XXX this is not the best validation
		web.JSONErr(w, "only JPEG format is allowed", http.StatusBadRequest)
		return
	}

	fd, err := header.Open()
	if err != nil {
		log.Error("cannot open uploaded file",
			"name", header.Filename,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	defer fd.Close()

	image, err := imageMeta(fd)
	if err != nil {
		log.Error("cannot extract image metadata", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	// store image in database
	db := sq.DB(ctx)
	image, err = CreateImage(db, *image)
	switch err {
	case nil:
		// all good
	case sq.ErrConflict:
		// image already exists, nothing more to do here
		web.JSONResp(w, image, http.StatusOK)
		return
	default:
		log.Error("cannot create object", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	if _, err := fd.Seek(0, os.SEEK_SET); err != nil {
		log.Error("cannot seek image", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	fs := FileStore(ctx)
	if err := fs.Put(image, fd); err != nil {
		log.Error("cannot store image", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	log.Debug("image file created", "id", image.ImageID)

	web.JSONResp(w, image, http.StatusCreated)
}
Beispiel #29
0
func crawl(rp *redis.Pool, urlStr string, stopw map[string]struct{}) []string {
	log.Debug("crawling started", "url", urlStr)
	defer log.Debug("crawling done", "url", urlStr)

	resp, err := httpcli.Get(urlStr)
	if err != nil {
		log.Error("cannot GET",
			"url", urlStr,
			"error", err.Error())
		return nil
	}
	defer resp.Body.Close()

	if ct := resp.Header.Get("Content-Type"); !isHtml(ct) {
		log.Debug("non HTML resource",
			"url", urlStr,
			"contentType", ct)
		return nil
	}

	body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 100000))
	if err != nil {
		log.Error("cannot read body",
			"url", urlStr,
			"error", err.Error())
		return nil
	}
	text := htmlToText(bytes.NewReader(body))
	key := fmt.Sprintf("article:%x", sha1.Sum(text))

	rc := rp.Get()
	defer rc.Close()

	if exists, err := redis.Bool(rc.Do("EXISTS", key)); err != nil {
		log.Error("cannot query database",
			"url", urlStr,
			"error", err.Error())
		return nil
	} else if exists {
		log.Debug("article already stored",
			"url", urlStr,
			"key", key)
		return nil
	}

	func() {
		// all redis update is done in single batch
		if err := rc.Send("MULTI"); err != nil {
			log.Error("cannot start MULTI", "error", err.Error())
			return
		}

		err = rc.Send("HMSET", key,
			"url", urlStr,
			"title", pageTitle(body),
			"crated", time.Now().Unix())
		if err != nil {
			log.Error("cannot write article data",
				"key", key,
				"url", urlStr,
				"error", err.Error())
			return
		}

		for w, n := range words(bytes.NewReader(text), stopw) {
			if len(w) < 3 {
				continue
			}
			if err := rc.Send("ZADD", "word:"+w, n, key); err != nil {
				log.Error("cannot write word count",
					"key", key,
					"url", urlStr,
					"word", w,
					"error", err.Error())
				return
			}
		}
		if _, err := rc.Do("EXEC"); err != nil {
			log.Error("cannot flush redis command",
				"key", key,
				"url", urlStr,
				"error", err.Error())
			return
		}
	}()

	return pageUrls(body)
}
Beispiel #30
0
func main() {
	if err := http.ListenAndServe(":8000", rt); err != nil {
		log.Error("HTTP server error", "error", err.Error())
	}
}