func (o *debian) getChangelogCache(meta *cache.Meta, pack models.PackageInfo) string { cachedPack, found := meta.FindPack(pack.Name) if !found { o.log.Debugf("Not found: %s", pack.Name) return "" } if cachedPack.NewVersion != pack.NewVersion { o.log.Debugf("Expired: %s, cache: %s, new: %s", pack.Name, cachedPack.NewVersion, pack.NewVersion) return "" } changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name) if err != nil { o.log.Warnf("Failed to get changelog. bucket: %s, key:%s, err: %s", meta.Name, pack.Name, err) return "" } if len(changelog) == 0 { o.log.Debugf("Empty string: %s", pack.Name) return "" } o.log.Debugf("Hit: %s, %s, cache: %s, new: %s len: %d, %s...", meta.Name, pack.Name, cachedPack.NewVersion, pack.NewVersion, len(changelog), util.Truncate(changelog, 30)) return changelog }
func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache.Meta) (models.VulnInfos, error) { type strarray []string resChan := make(chan struct { models.PackageInfo strarray }, len(upgradablePacks)) errChan := make(chan error, len(upgradablePacks)) reqChan := make(chan models.PackageInfo, len(upgradablePacks)) defer close(resChan) defer close(errChan) defer close(reqChan) go func() { for _, pack := range upgradablePacks { reqChan <- pack } }() timeout := time.After(30 * 60 * time.Second) concurrency := 10 tasks := util.GenWorkers(concurrency) for range upgradablePacks { tasks <- func() { select { case pack := <-reqChan: func(p models.PackageInfo) { changelog := o.getChangelogCache(meta, p) if 0 < len(changelog) { cveIDs := o.getCveIDFromChangelog(changelog, p.Name, p.Version) resChan <- struct { models.PackageInfo strarray }{p, cveIDs} return } // if the changelog is not in cache or failed to get from local cache, // get the changelog of the package via internet. // After that, store it in the cache. if cveIDs, err := o.scanPackageCveIDs(p); err != nil { errChan <- err } else { resChan <- struct { models.PackageInfo strarray }{p, cveIDs} } }(pack) } } } // { CVE ID: [packageInfo] } cvePackages := make(map[string][]models.PackageInfo) errs := []error{} for i := 0; i < len(upgradablePacks); i++ { select { case pair := <-resChan: pack := pair.PackageInfo cveIDs := pair.strarray for _, cveID := range cveIDs { cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack) } o.log.Infof("(%d/%d) Scanned %s-%s : %s", i+1, len(upgradablePacks), pair.Name, pair.PackageInfo.Version, cveIDs) case err := <-errChan: errs = append(errs, err) case <-timeout: errs = append(errs, fmt.Errorf("Timeout scanPackageCveIDs")) } } if 0 < len(errs) { return nil, fmt.Errorf("%v", errs) } var cveIDs []string for k := range cvePackages { cveIDs = append(cveIDs, k) } o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs) var vinfos models.VulnInfos for k, v := range cvePackages { vinfos = append(vinfos, models.VulnInfo{ CveID: k, Packages: v, }) } // Update meta package information of changelog cache to the latest one. meta.Packs = upgradablePacks if err := cache.DB.RefreshMeta(*meta); err != nil { return nil, err } return vinfos, nil }