Last *Link
}
func (this *Queue) Enqueue(link Link) {
this.Last.Next = &link
this.Last = this.Last.Next
}
func (this *Queue) Dequeue() *Link {
first := this.First
this.First = this.First.Next
return first
}
var (
config = easyConfig.New(&Configuration{}, "./config.json").(*Configuration)
db = database.Get()
fetchQueue = Queue{}
processQueue = Queue{}
currentAddress = &config.StartAddress //todo: coalesce last crawled address and startingAddress
)
func Crawl() {
resp, err := http.Get(*currentAddress)
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
type DbFile struct {
Id int64
Sha1 [20]byte
Md5 [16]byte
Size int64
ContentType string
}
type DbPastebinFile struct {
Id int64
File DbFile
Url string
}
var (
dbConfig = easyConfig.New(&Configuration{}, "../database/config.json").(*Configuration)
currentDbDefinition = dbConfig.Dev //TODO: read command line flag
readConnectionString = buildConnectionString(dbConfig.ReadUser)
writeConnectionString = buildConnectionString(dbConfig.WriteUser)
)
func buildConnectionString(user DatabaseUser) string {
return fmt.Sprintf("%s:%s@%s(%s)/%s?timeout=5s",
user.Name, user.Password,
currentDbDefinition.Protocol,
currentDbDefinition.Address,
currentDbDefinition.Name)
}
func Get() Database {