Ejemplo n.º 1
0
func main() {
	args := os.Args
	if len(args) != 4 {
		fmt.Printf("usage: %s [URL] [DOWNLOAD PATH] [# MBs]\n", path.Base(args[0]))
		os.Exit(1)
	}

	url := args[1]
	download := args[2]
	size, err := strconv.Atoi(args[3])

	if err != nil {
		fmt.Printf("Error: %s\n", err.Error())
		fmt.Printf("usage: %s [URL] [DOWNLOAD PATH] [# MBs]\n", path.Base(args[0]))
	}

	done := make(chan struct{})
	at := types.NewAtomicBool(false)
	feeder := channels.NewInfiniteChannel()

	if !dirExists(download) {
		if err := os.MkdirAll(download, os.ModePerm); err != nil {
			fmt.Printf("Failed to create directory (%v): %v.\n", download, err.Error())
			os.Exit(1)
		}
	}

	feeder.In() <- url

	uFeeder := func(sf types.SetupFunction) {
		urlFeeder(sf, feeder.Out(), at)
	}

	producer := func(sf types.SetupFunction) {
		UrlProducer(sf, "UrlFeeder")
	}

	looper := func(sf types.SetupFunction) {
		urlLooper(sf, feeder.In(), at)
	}

	var cancel types.Canceller

	downloader := func(sf types.SetupFunction) {
		textDownloader(sf, download, size, done, func() {
			if !at.Get() {
				feeder.Close()
				cancel()
			}
			at.Set(true)
		})
	}

	cancel = hydra.NewSetupScaffolding()(uFeeder, looper, downloader, producer, UrlParser, MimeDetector, MimeSplitterHtml, MimeSplitterText)

	<-done
}
Ejemplo n.º 2
0
func main() {
	args := os.Args
	if len(args) < 2 {
		fmt.Printf("usage: %s [Paths...]\n", path.Base(args[0]))
		os.Exit(1)
	}

	validator := func(sf types.SetupFunction) {
		wordCount.PathValidator(sf, "PathProducer")
	}

	producer := func(sf types.SetupFunction) {
		pathProducer(sf, args[1:])
	}

	done := make(chan struct{})
	cp := func(sf types.SetupFunction) {
		wordPrinter(sf, done)
	}

	hydra.NewSetupScaffolding()(producer, validator, wordCount.WordExtractor, cp, wordCount.SymbolRemover, wordCount.WordCounter, wordCount.FinalWordCounter)

	<-done
}