Beispiel #1
0
	Last  *Link
}

func (this *Queue) Enqueue(link Link) {
	this.Last.Next = &link
	this.Last = this.Last.Next
}

func (this *Queue) Dequeue() *Link {
	first := this.First
	this.First = this.First.Next
	return first
}

var (
	config         = easyConfig.New(&Configuration{}, "./config.json").(*Configuration)
	db             = database.Get()
	fetchQueue     = Queue{}
	processQueue   = Queue{}
	currentAddress = &config.StartAddress //todo: coalesce last crawled address and startingAddress
)

func Crawl() {

	resp, err := http.Get(*currentAddress)
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
Beispiel #2
0
type DbFile struct {
	Id          int64
	Sha1        [20]byte
	Md5         [16]byte
	Size        int64
	ContentType string
}

type DbPastebinFile struct {
	Id   int64
	File DbFile
	Url  string
}

var (
	dbConfig              = easyConfig.New(&Configuration{}, "../database/config.json").(*Configuration)
	currentDbDefinition   = dbConfig.Dev //TODO: read command line flag
	readConnectionString  = buildConnectionString(dbConfig.ReadUser)
	writeConnectionString = buildConnectionString(dbConfig.WriteUser)
)

func buildConnectionString(user DatabaseUser) string {

	return fmt.Sprintf("%s:%s@%s(%s)/%s?timeout=5s",
		user.Name, user.Password,
		currentDbDefinition.Protocol,
		currentDbDefinition.Address,
		currentDbDefinition.Name)
}

func Get() Database {