Example #1
0
func insertCTWorker(entries <-chan ct.LogEntry, db *sqldb.EntriesDatabase, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()
	for ep := range entries {
		err := db.InsertCTEntry(&ep)
		if err != nil {
			log.Printf("Problem inserting certificate: index: %d log: %s error: %s", ep.Index, *config.LogUrl, err)
		}
	}
}
Example #2
0
func insertCensysWorker(entries <-chan censysdata.CensysEntry, db *sqldb.EntriesDatabase, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()
	for ep := range entries {
		if ep.Valid_nss {
			err := db.InsertCensysEntry(&ep)
			if err != nil {
				log.Printf("Problem inserting certificate: index: %d error: %s", ep.Offset, err)
			}
		}
	}
}
Example #3
0
func downloadLog(ctLogUrl *url.URL, ctLog *client.LogClient, db *sqldb.EntriesDatabase) error {
	if *config.OffsetByte > 0 {
		return fmt.Errorf("Cannot set offsetByte for CT log downloads")
	}

	fmt.Printf("Fetching signed tree head... ")
	sth, err := ctLog.GetSTH()
	if err != nil {
		return err
	}

	// Set pointer in DB, now that we've verified the log works
	err = db.SetLog(fmt.Sprintf("%s%s", ctLogUrl.Host, ctLogUrl.Path))
	if err != nil {
		log.Fatalf("unable to set Certificate Log: %s", err)
	}

	var origCount uint64
	// Now we're OK to use the DB
	if *config.Offset > 0 {
		log.Printf("Starting from offset %d", *config.Offset)
		origCount = *config.Offset
	} else {
		log.Printf("Counting existing entries... ")
		origCount, err = db.Count()
		if err != nil {
			err = fmt.Errorf("Failed to read entries file: %s", err)
			return err
		}
	}

	fmt.Printf("%d total entries at %s\n", sth.TreeSize, sqldb.Uint64ToTimestamp(sth.Timestamp).Format(time.ANSIC))
	if origCount == sth.TreeSize {
		fmt.Printf("Nothing to do\n")
		return nil
	}

	endPos := sth.TreeSize
	if *config.Limit > 0 && endPos > origCount+*config.Limit {
		endPos = origCount + *config.Limit
	}

	fmt.Printf("Going from %d to %d\n", origCount, endPos)

	entryChan := make(chan ct.LogEntry, 100)
	statusChan := make(chan OperationStatus, 1)
	wg := new(sync.WaitGroup)

	displayProgress(statusChan, wg)
	numWorkers := *config.NumThreads * runtime.NumCPU()
	for i := 0; i < numWorkers; i++ {
		go insertCTWorker(entryChan, db, wg)
	}
	_, err = downloadCTRangeToChannel(ctLog, entryChan, statusChan, int64(origCount), int64(endPos))
	wg.Wait()

	clearLine()
	if err != nil {
		err = fmt.Errorf("Error while downloading: %s", err)
		return err
	}

	return nil
}