func (o *debian) fillCandidateVersion(packs []models.PackageInfo) ([]models.PackageInfo, error) {
reqChan := make(chan models.PackageInfo, len(packs))
resChan := make(chan models.PackageInfo, len(packs))
errChan := make(chan error, len(packs))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
go func() {
for _, pack := range packs {
reqChan <- pack
}
}()
timeout := time.After(5 * 60 * time.Second)
concurrency := 5
tasks := util.GenWorkers(concurrency)
for range packs {
tasks <- func() {
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
cmd := fmt.Sprintf("apt-cache policy %s", p.Name)
r := o.ssh(cmd, sudo)
if !r.isSuccess() {
errChan <- fmt.Errorf(
"Failed to %s. status: %d, stdout: %s, stderr: %s",
cmd, r.ExitStatus, r.Stdout, r.Stderr)
return
}
ver, err := o.parseAptCachePolicy(r.Stdout, p.Name)
if err != nil {
errChan <- fmt.Errorf("Failed to parse %s", err)
}
p.NewVersion = ver.Candidate
resChan <- p
}(pack)
}
}
}
errs := []error{}
result := []models.PackageInfo{}
for i := 0; i < len(packs); i++ {
select {
case pack := <-resChan:
result = append(result, pack)
o.log.Infof("(%d/%d) Upgradable: %s-%s -> %s",
i+1, len(packs), pack.Name, pack.Version, pack.NewVersion)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, fmt.Errorf("Timeout fillCandidateVersion")
}
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
return result, nil
}
// Write results to Azure Blob storage
func (w AzureBlobWriter) Write(scanResults []models.ScanResult) (err error) {
reqChan := make(chan models.ScanResult, len(scanResults))
resChan := make(chan bool)
errChan := make(chan error, len(scanResults))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
timeout := time.After(10 * 60 * time.Second)
concurrency := 10
tasks := util.GenWorkers(concurrency)
go func() {
for _, r := range scanResults {
reqChan <- r
}
}()
for range scanResults {
tasks <- func() {
select {
case sresult := <-reqChan:
func(r models.ScanResult) {
err := w.upload(r)
if err != nil {
errChan <- err
}
resChan <- true
}(sresult)
}
}
}
errs := []error{}
for i := 0; i < len(scanResults); i++ {
select {
case <-resChan:
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
errs = append(errs, fmt.Errorf("Timeout while uploading to azure Blob"))
}
}
if 0 < len(errs) {
return fmt.Errorf("Failed to upload json to Azure Blob: %v", errs)
}
return nil
}
func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache.Meta) (models.VulnInfos, error) {
type strarray []string
resChan := make(chan struct {
models.PackageInfo
strarray
}, len(upgradablePacks))
errChan := make(chan error, len(upgradablePacks))
reqChan := make(chan models.PackageInfo, len(upgradablePacks))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
go func() {
for _, pack := range upgradablePacks {
reqChan <- pack
}
}()
timeout := time.After(30 * 60 * time.Second)
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range upgradablePacks {
tasks <- func() {
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
changelog := o.getChangelogCache(meta, p)
if 0 < len(changelog) {
cveIDs := o.getCveIDFromChangelog(changelog, p.Name, p.Version)
resChan <- struct {
models.PackageInfo
strarray
}{p, cveIDs}
return
}
// if the changelog is not in cache or failed to get from local cache,
// get the changelog of the package via internet.
// After that, store it in the cache.
if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
errChan <- err
} else {
resChan <- struct {
models.PackageInfo
strarray
}{p, cveIDs}
}
}(pack)
}
}
}
// { CVE ID: [packageInfo] }
cvePackages := make(map[string][]models.PackageInfo)
errs := []error{}
for i := 0; i < len(upgradablePacks); i++ {
select {
case pair := <-resChan:
pack := pair.PackageInfo
cveIDs := pair.strarray
for _, cveID := range cveIDs {
cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack)
}
o.log.Infof("(%d/%d) Scanned %s-%s : %s",
i+1, len(upgradablePacks), pair.Name, pair.PackageInfo.Version, cveIDs)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
errs = append(errs, fmt.Errorf("Timeout scanPackageCveIDs"))
}
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
var cveIDs []string
for k := range cvePackages {
cveIDs = append(cveIDs, k)
}
o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
var vinfos models.VulnInfos
for k, v := range cvePackages {
vinfos = append(vinfos, models.VulnInfo{
CveID: k,
Packages: v,
})
}
// Update meta package information of changelog cache to the latest one.
meta.Packs = upgradablePacks
if err := cache.DB.RefreshMeta(*meta); err != nil {
return nil, err
}
return vinfos, nil
}
func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) {
if config.Conf.CveDictionaryURL == "" {
return api.FetchCveDetailsFromCveDB(cveIDs)
}
api.baseURL = config.Conf.CveDictionaryURL
reqChan := make(chan string, len(cveIDs))
resChan := make(chan response, len(cveIDs))
errChan := make(chan error, len(cveIDs))
defer close(reqChan)
defer close(resChan)
defer close(errChan)
go func() {
for _, cveID := range cveIDs {
reqChan <- cveID
}
}()
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range cveIDs {
tasks <- func() {
select {
case cveID := <-reqChan:
url, err := util.URLPathJoin(api.baseURL, "cves", cveID)
if err != nil {
errChan <- err
} else {
log.Debugf("HTTP Request to %s", url)
api.httpGet(cveID, url, resChan, errChan)
}
}
}
}
timeout := time.After(2 * 60 * time.Second)
var errs []error
for range cveIDs {
select {
case res := <-resChan:
if len(res.CveDetail.CveID) == 0 {
cveDetails = append(cveDetails, cve.CveDetail{
CveID: res.Key,
})
} else {
cveDetails = append(cveDetails, res.CveDetail)
}
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return []cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE")
}
}
if len(errs) != 0 {
return []cve.CveDetail{},
fmt.Errorf("Failed to fetch CVE. err: %v", errs)
}
sort.Sort(cveDetails)
return
}
func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) {
// { CVE ID: [packageInfo] }
cvePackages := make(map[string][]models.PackageInfo)
type strarray []string
resChan := make(chan struct {
models.PackageInfo
strarray
}, len(unsecurePacks))
errChan := make(chan error, len(unsecurePacks))
reqChan := make(chan models.PackageInfo, len(unsecurePacks))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
go func() {
for _, pack := range unsecurePacks {
reqChan <- pack
}
}()
timeout := time.After(30 * 60 * time.Second)
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range unsecurePacks {
tasks <- func() {
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
errChan <- err
} else {
resChan <- struct {
models.PackageInfo
strarray
}{p, cveIDs}
}
}(pack)
}
}
}
errs := []error{}
for i := 0; i < len(unsecurePacks); i++ {
select {
case pair := <-resChan:
pack := pair.PackageInfo
cveIDs := pair.strarray
for _, cveID := range cveIDs {
cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack)
}
o.log.Infof("(%d/%d) Scanned %s-%s : %s",
i+1, len(unsecurePacks), pair.Name, pair.PackageInfo.Version, cveIDs)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
//TODO append to errs
return nil, fmt.Errorf("Timeout scanPackageCveIDs")
}
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
var cveIDs []string
for k := range cvePackages {
cveIDs = append(cveIDs, k)
}
o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
o.log.Info("Fetching CVE details...")
cveDetails, err := cveapi.CveClient.FetchCveDetails(cveIDs)
if err != nil {
return nil, err
}
o.log.Info("Done")
for _, detail := range cveDetails {
cvePacksList = append(cvePacksList, CvePacksInfo{
CveID: detail.CveID,
CveDetail: detail,
Packs: cvePackages[detail.CveID],
// CvssScore: cinfo.CvssScore(conf.Lang),
})
}
return
}