func CheckFiles(pso2path string, checkHash bool, patches *download.PatchList) (changes []*download.PatchEntry, err error) { pbar := pb.New(len(patches.Entries)) pbar.SetRefreshRate(time.Second / 30) pbar.Start() for i := range patches.Entries { e := &patches.Entries[i] filepath := path.Join(pso2path, download.RemoveExtension(e.Path)) st, err := os.Stat(filepath) if os.IsNotExist(err) { changes = append(changes, e) } else { if err != nil { return nil, err } if st.Size() != e.Size { changes = append(changes, e) } else if checkHash { f, err := os.Open(filepath) if err != nil { return nil, err } h := md5.New() _, err = io.Copy(h, f) f.Close() if err != nil { return nil, err } if bytes.Compare(h.Sum(nil), e.MD5[:]) != 0 { changes = append(changes, e) } } } pbar.Increment() runtime.Gosched() } pbar.Finish() return }
func DownloadChanges(pso2path string, changes []*download.PatchEntry, parallel int) (errs []error) { if parallel <= 0 { parallel = 1 } changesSize := int64(0) for _, e := range changes { changesSize += e.Size } pbar := pb.New64(changesSize) pbar.SetUnits(pb.U_BYTES) pbar.SetRefreshRate(time.Second / 30) pbar.ShowSpeed = true pbar.Start() queue := make(chan *download.PatchEntry) done := make(chan bool) errlock := sync.Mutex{} complain := func(err error) bool { if err != nil { errlock.Lock() errs = append(errs, err) errlock.Unlock() return true } return false } for i := 0; i < parallel; i++ { go func() { h := md5.New() for { e, ok := <-queue if !ok { break } filepath := path.Join(pso2path, download.RemoveExtension(e.Path)) err := os.MkdirAll(path.Dir(filepath), 0777) if complain(err) { break } pathUrl, err := e.URL() if complain(err) { continue } resp, err := download.Request(pathUrl.String()) if complain(err) { continue } if resp.StatusCode != 200 { complain(errors.New(pathUrl.String() + ": " + resp.Status)) continue } if resp.ContentLength >= 0 && resp.ContentLength != e.Size { resp.Body.Close() complain(errors.New(e.Path + ": invalid file size")) continue } f, err := os.Create(filepath) if complain(err) { resp.Body.Close() continue } h.Reset() n, err := io.Copy(io.MultiWriter(f, h, pbar), resp.Body) resp.Body.Close() f.Close() if !complain(err) { if n != e.Size { complain(errors.New(pathUrl.String() + ": download finished prematurely")) } else if bytes.Compare(h.Sum(nil), e.MD5[:]) != 0 { complain(errors.New(pathUrl.String() + ": download hash mismatch")) } } } done <- true }() } for _, e := range changes { queue <- e } close(queue) for i := 0; i < parallel; i++ { <-done } pbar.Finish() return }