Example #1
0
func checkdata(dbconnection *sql.DB) (int, error) {

	hit, err := watcherdatabase.GetLastHit()
	if err != nil {
		return 0, fmt.Errorf("Cannot read last DBHit: %s", err)
	}

	var processids []string
	if hit == nil {
		logger.Println("No checkpoint in database found, getting all datasets")
		processids, err = portal.GetAllMetaDataIDs()
	} else {
		logger.Printf("Getting changed datasets since %s\n", hit)
		processids, err = portal.GetChangedPackageIDsSince(*hit, getnumworkers())
	}

	if err != nil {
		return 0, err
	}

	anzids := len(processids)
	if anzids > 0 {

		tx, err := dbconnection.Begin()
		if err != nil {
			return 0, fmt.Errorf("Cannot create database transaction: %s", err)
		}
		scheduler := schedule.New(getnumworkers())
		logger.Printf("Doing %d jobs in parallel\n", scheduler.GetWorkers())
		conn := &watcherdb{database.DBConn{DBer: tx, Appid: AppID}}
		f := func(slice []interface{}) error {
			if err := processmetadataids(conn, ifaceslicetostring(slice)); err != nil {
				return err
			}
			return nil
		}

		watcherdatabase.LogMessage(fmt.Sprintf("%d Medadaten werden verarbeitet", anzids), database.StateOk, true)
		workchannel := scheduler.Schedule(f, stringslicetoiface(processids))
		select {
		case workreply := <-workchannel:
			if err := workreply.Err; err != nil {
				return 0, fmt.Errorf("Scheduler didn't return success: %s", err)
			} else if workreply.Code == schedule.StateFinish {
				tx.Commit()
				watcherdatabase.LogMessage("Idle", database.StateOk, true)
				logger.Printf("Finished processing %d datasets\n", anzids)
			}
		}
	}
	return anzids, nil
}
Example #2
0
func checkurls(dbconnection *sql.DB) (int, error) {

	urls, err := watcherdatabase.GetDataUrls()
	if err != nil {
		return 0, err
	}

	anzurls := len(urls)
	if anzurls > 0 {

		tx, err := dbconnection.Begin()
		if err != nil {
			return 0, fmt.Errorf("Cannot create database transaction: %s", err)
		}

		scheduler := schedule.New(getnumworkers())
		logger.Printf("Doing %d jobs in parallel\n", scheduler.GetWorkers())

		conn := &watcherdb{database.DBConn{DBer: tx, Appid: AppID}}

		f := func(slice []interface{}) error {
			if err := processdataseturls(conn, ifaceslicetodataurl(slice)); err != nil {
				return err
			}
			return nil
		}

		watcherdatabase.LogMessage(fmt.Sprintf("%d Urls werden gecheckt", anzurls), database.StateOk, true)
		workchannel := scheduler.Schedule(f, dataurlslicetoiface(urls))

		select {
		case workreply := <-workchannel:
			if err := workreply.Err; err != nil {
				return 0, fmt.Errorf("Scheduler didn't return success: %s", err)
			} else if workreply.Code == schedule.StateFinish {
				tx.Commit()
				watcherdatabase.LogMessage("Idle", database.StateOk, true)
				logger.Printf("Finished checking %d Urls\n", anzurls)
			}
		}
	}
	return anzurls, nil
}
Example #3
0
File: ckan.go Project: the42/ogdat
func (p *Portal) GetChangedPackageIDsSince(t time.Time, workers int) ([]string, error) {
	revs, err := p.GetRevisionsetSince(t)
	if err != nil {
		return nil, err
	}

	scheduler := schedule.New(workers)
	conset := newSet()

	f := func(slice []interface{}) error {
		for _, val := range slice {
			revid, ok := val.(string)
			if !ok {
				panic("Interface value not of string type")
			}
			rev, err := p.GetRevisionforID(revid)
			if err != nil {
				conset.deleteAll()
				return err
			}
			for _, packageid := range rev.Packages {
				conset.add(packageid)
			}
		}
		return nil
	}

	<-scheduler.Schedule(f, stringslicetoiface(revs))

	changedids := make([]string, len(conset.value))
	idx := 0

	for key, _ := range conset.value {
		changedids[idx] = key
		idx++
	}
	return changedids, nil
}