Esempio n. 1
0
func main() {
	providersFile := flag.String("providers_file", "providers.json", "Path to oembed providers json file")
	workerCount := flag.Int64("worker_count", 1000, "Amount of workers to start")
	host := flag.String("host", "localhost", "Host to listen on")
	port := flag.Int("port", 8000, "Port to listen on")
	maxHTMLBytesToRead := flag.Int64("html_bytes_to_read", 50000, "How much data to read from URL if it's an html page")
	maxBinaryBytesToRead := flag.Int64("binary_bytes_to_read", 4096, "How much data to read from URL if it's NOT an html page")
	waitTimeout := flag.Int("wait_timeout", 7, "How much time to wait for/fetch response from remote server")
	whiteListRanges := flag.String("whitelist_ranges", "", "What IP ranges to allow. Example: 178.25.32.1/8")
	blackListRanges := flag.String("blacklist_ranges", "", "What IP ranges to disallow. Example: 178.25.32.1/8")

	flag.Parse()

	buf, err := ioutil.ReadFile(*providersFile)

	if err != nil {
		panic(err)
	}

	var whiteListNetworks []*net.IPNet
	if len(*whiteListRanges) > 0 {
		if whiteListNetworks, err = stringsToNetworks(strings.Split(*whiteListRanges, " ")); err != nil {
			panic(err)
		}
	}

	var blackListNetworks []*net.IPNet
	if len(*blackListRanges) > 0 {
		if blackListNetworks, err = stringsToNetworks(strings.Split(*blackListRanges, " ")); err != nil {
			panic(err)
		}
	}

	oe := oembed.NewOembed()
	oe.ParseProviders(bytes.NewReader(buf))

	workers := make([]tunny.TunnyWorker, *workerCount)
	for i := range workers {
		p := url2oembed.NewParser(oe)
		p.MaxHTMLBodySize = *maxHTMLBytesToRead
		p.MaxBinaryBodySize = *maxBinaryBytesToRead
		p.WaitTimeout = time.Duration(*waitTimeout) * time.Second
		p.BlacklistedIPNetworks = blackListNetworks
		p.WhitelistedIPNetworks = whiteListNetworks
		workers[i] = &(apiWorker{Parser: p})
	}

	pool, err := tunny.CreateCustomPool(workers).Open()

	if err != nil {
		log.Fatal(err)
	}

	defer pool.Close()

	workerPool = pool

	startServer(*host, *port, *waitTimeout)
}
Esempio n. 2
0
func main() {
	providersData, err := ioutil.ReadFile("providers.json")

	if err != nil {
		panic(err)
	}

	oe := oembed.NewOembed()
	oe.ParseProviders(bytes.NewReader(providersData))

	parser := url2oembed.NewParser(oe)
	if parser.BlacklistedIPNetworks, err = StringsToNetworks([]string{"195.59.58.240/32"}); err != nil {
		panic(err)
	}

	data := parser.Parse("http://techcrunch.com/2010/11/02/365-days-10-million-3-rounds-2-companies-all-with-5-magic-slides/")

	fmt.Printf("%s\n", data)
}