func checkdata(dbconnection *sql.DB) (int, error) {
hit, err := watcherdatabase.GetLastHit()
if err != nil {
return 0, fmt.Errorf("Cannot read last DBHit: %s", err)
}
var processids []string
if hit == nil {
logger.Println("No checkpoint in database found, getting all datasets")
processids, err = portal.GetAllMetaDataIDs()
} else {
logger.Printf("Getting changed datasets since %s\n", hit)
processids, err = portal.GetChangedPackageIDsSince(*hit, getnumworkers())
}
if err != nil {
return 0, err
}
anzids := len(processids)
if anzids > 0 {
tx, err := dbconnection.Begin()
if err != nil {
return 0, fmt.Errorf("Cannot create database transaction: %s", err)
}
scheduler := schedule.New(getnumworkers())
logger.Printf("Doing %d jobs in parallel\n", scheduler.GetWorkers())
conn := &watcherdb{database.DBConn{DBer: tx, Appid: AppID}}
f := func(slice []interface{}) error {
if err := processmetadataids(conn, ifaceslicetostring(slice)); err != nil {
return err
}
return nil
}
watcherdatabase.LogMessage(fmt.Sprintf("%d Medadaten werden verarbeitet", anzids), database.StateOk, true)
workchannel := scheduler.Schedule(f, stringslicetoiface(processids))
select {
case workreply := <-workchannel:
if err := workreply.Err; err != nil {
return 0, fmt.Errorf("Scheduler didn't return success: %s", err)
} else if workreply.Code == schedule.StateFinish {
tx.Commit()
watcherdatabase.LogMessage("Idle", database.StateOk, true)
logger.Printf("Finished processing %d datasets\n", anzids)
}
}
}
return anzids, nil
}
func checkurls(dbconnection *sql.DB) (int, error) {
urls, err := watcherdatabase.GetDataUrls()
if err != nil {
return 0, err
}
anzurls := len(urls)
if anzurls > 0 {
tx, err := dbconnection.Begin()
if err != nil {
return 0, fmt.Errorf("Cannot create database transaction: %s", err)
}
scheduler := schedule.New(getnumworkers())
logger.Printf("Doing %d jobs in parallel\n", scheduler.GetWorkers())
conn := &watcherdb{database.DBConn{DBer: tx, Appid: AppID}}
f := func(slice []interface{}) error {
if err := processdataseturls(conn, ifaceslicetodataurl(slice)); err != nil {
return err
}
return nil
}
watcherdatabase.LogMessage(fmt.Sprintf("%d Urls werden gecheckt", anzurls), database.StateOk, true)
workchannel := scheduler.Schedule(f, dataurlslicetoiface(urls))
select {
case workreply := <-workchannel:
if err := workreply.Err; err != nil {
return 0, fmt.Errorf("Scheduler didn't return success: %s", err)
} else if workreply.Code == schedule.StateFinish {
tx.Commit()
watcherdatabase.LogMessage("Idle", database.StateOk, true)
logger.Printf("Finished checking %d Urls\n", anzurls)
}
}
}
return anzurls, nil
}
func (p *Portal) GetChangedPackageIDsSince(t time.Time, workers int) ([]string, error) {
revs, err := p.GetRevisionsetSince(t)
if err != nil {
return nil, err
}
scheduler := schedule.New(workers)
conset := newSet()
f := func(slice []interface{}) error {
for _, val := range slice {
revid, ok := val.(string)
if !ok {
panic("Interface value not of string type")
}
rev, err := p.GetRevisionforID(revid)
if err != nil {
conset.deleteAll()
return err
}
for _, packageid := range rev.Packages {
conset.add(packageid)
}
}
return nil
}
<-scheduler.Schedule(f, stringslicetoiface(revs))
changedids := make([]string, len(conset.value))
idx := 0
for key, _ := range conset.value {
changedids[idx] = key
idx++
}
return changedids, nil
}