Ejemplo n.º 1
0
func createDBCmd(args []string) {
	fs := flag.NewFlagSet("createdb", flag.ExitOnError)
	drop := fs.Bool("drop", false, "drop DB before creating")
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: thesrc createdb [options] 

Creates the necessary DB tables and indexes.

The options are:
`)
		fs.PrintDefaults()
		os.Exit(1)
	}
	fs.Parse(args)

	if fs.NArg() != 0 {
		fs.Usage()
	}

	datastore.Connect()
	if *drop {
		datastore.Drop()
	}
	datastore.Create()
}
Ejemplo n.º 2
0
func serveCmd(args []string) {
	fs := flag.NewFlagSet("serve", flag.ExitOnError)
	httpAddr := fs.String("http", ":5000", "HTTP service address")
	templateDir := fs.String("tmpl-dir", app.TemplateDir, "template directory")
	staticDir := fs.String("static-dir", app.StaticDir, "static assets directory")
	reload := flag.Bool("reload", true, "reload templates on each request (dev mode)")
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: thesrc serve [options] 

Starts the web server that serves the app and API.

The options are:
`)
		fs.PrintDefaults()
		os.Exit(1)
	}
	fs.Parse(args)

	if fs.NArg() != 0 {
		fs.Usage()
	}

	app.StaticDir = *staticDir
	app.TemplateDir = *templateDir
	app.ReloadTemplates = *reload
	app.LoadTemplates()

	datastore.Connect()

	m := http.NewServeMux()
	m.Handle("/api/", http.StripPrefix("/api", api.Handler()))
	m.Handle("/", app.Handler())

	log.Print("Listening on ", *httpAddr)
	err := http.ListenAndServe(*httpAddr, m)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
Ejemplo n.º 3
0
func classifyCmd(args []string) {
	fs := flag.NewFlagSet("classify", flag.ExitOnError)
	concurrency := fs.Int("c", 10, "concurrent classifiers")
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: thesrc classify [options]

Classifies posts.

The options are:
`)
		fs.PrintDefaults()
		os.Exit(1)
	}
	fs.Parse(args)

	if fs.NArg() != 0 {
		fs.Usage()
	}

	var mu sync.Mutex
	summary := map[string]int{}

	workChan := make(chan *thesrc.Post)
	quitChan := make(chan struct{})
	for i := 0; i < *concurrency; i++ {
		go func() {
			for {
				select {
				case post := <-workChan:
					c, err := classifier.Classify(post)
					if err != nil {
						log.Printf("Error classifying %q: %s. (Continuing...)", post.LinkURL, err)
						continue
					}
					changed := firstWord(c) != firstWord(post.Classification)
					if changed {
						post.Classification = c
						// TODO(sqs): add post update endpoint so we can run
						// `thesrc` against the HTTP API
						if _, err := datastore.DBH.Update(post); err != nil {
							log.Fatal(err)
						}
						mu.Lock()
						summary[firstWord(post.Classification)]++
						mu.Unlock()
					}
					fmt.Printf("%v %-20s %s\n", changed, post.Classification, post.LinkURL)
				case <-quitChan:
					return
				}
			}
		}()
	}

	datastore.Connect()
	perPage := 100
	for pg := 1; true; pg++ {
		log.Println("Fetching more posts...")
		posts, err := apiclient.Posts.List(&thesrc.PostListOptions{ListOptions: thesrc.ListOptions{PerPage: perPage, Page: pg}})
		if err != nil {
			log.Fatal(err)
		}
		if len(posts) == 0 {
			break
		}

		for _, post := range posts {
			workChan <- post
		}
	}

	close(quitChan)

	fmt.Fprintf(os.Stderr, "# classified posts: %v\n", summary)
}
Ejemplo n.º 4
0
func importCmd(args []string) {
	fs := flag.NewFlagSet("import", flag.ExitOnError)
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: thesrc import [options]

Imports posts from other sites.

The available sites are:
`)
		for _, f := range importer.Fetchers {
			fmt.Fprintln(os.Stderr, "  ", f.Site())
		}
		fmt.Fprintln(os.Stderr, `

The options are:
`)
		fs.PrintDefaults()
		os.Exit(1)
	}
	fs.Parse(args)

	if fs.NArg() != 0 {
		fs.Usage()
	}

	var numTotal, numCreated int
	var mu sync.Mutex
	importer.Imported = func(site string, post *thesrc.Post, created bool) {
		mu.Lock()
		defer mu.Unlock()
		numTotal++
		if !created {
			return
		}
		fmt.Printf("%-12s  %-50s\n              %-60s\n", site, post.Title, post.LinkURL)
		numCreated++
	}

	datastore.Connect()
	var failed bool
	var wg sync.WaitGroup
	for _, f_ := range importer.Fetchers {
		f := f_
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := importer.Import(f); err != nil {
				log.Printf("Error fetching from %s: %s.", f.Site(), err)
				mu.Lock()
				failed = true
				mu.Unlock()
			}
		}()
	}
	wg.Wait()

	log.Printf("# import: %d new posts, %d already existed", numCreated, numTotal-numCreated)
	if failed {
		os.Exit(1)
	}
}