// downloadAndApplyDelta downloads and then applies the delta to the current snap.
func (s *Store) downloadAndApplyDelta(name, targetPath string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) error {
deltaInfo := &downloadInfo.Deltas[0]
deltaPath := fmt.Sprintf("%s.%s-%d-to-%d.partial", targetPath, deltaInfo.Format, deltaInfo.FromRevision, deltaInfo.ToRevision)
deltaName := filepath.Base(deltaPath)
w, err := os.Create(deltaPath)
if err != nil {
return err
}
defer func() {
if cerr := w.Close(); cerr != nil && err == nil {
err = cerr
}
os.Remove(deltaPath)
}()
err = s.downloadDelta(deltaName, downloadInfo, w, pbar, user)
if err != nil {
return err
}
logger.Debugf("Successfully downloaded delta for %q at %s", name, deltaPath)
if err := applyDelta(name, deltaPath, deltaInfo, targetPath, downloadInfo.Sha3_384); err != nil {
return err
}
logger.Debugf("Successfully applied delta for %q at %s, saving %d bytes.", name, deltaPath, downloadInfo.Size-deltaInfo.Size)
return nil
}
// authenticateUser will add the store expected Macaroon Authorization header for user
func authenticateUser(r *http.Request, user *auth.UserState) {
var buf bytes.Buffer
fmt.Fprintf(&buf, `Macaroon root="%s"`, user.StoreMacaroon)
// deserialize root macaroon (we need its signature to do the discharge binding)
root, err := auth.MacaroonDeserialize(user.StoreMacaroon)
if err != nil {
logger.Debugf("cannot deserialize root macaroon: %v", err)
return
}
for _, d := range user.StoreDischarges {
// prepare discharge for request
discharge, err := auth.MacaroonDeserialize(d)
if err != nil {
logger.Debugf("cannot deserialize discharge macaroon: %v", err)
return
}
discharge.Bind(root.Signature())
serializedDischarge, err := auth.MacaroonSerialize(discharge)
if err != nil {
logger.Debugf("cannot re-serialize discharge macaroon: %v", err)
return
}
fmt.Fprintf(&buf, `, discharge="%s"`, serializedDischarge)
}
r.Header.Set("Authorization", buf.String())
}
// retryRequest calls doRequest and decodes the response in a retry loop.
func (s *Store) retryRequest(ctx context.Context, client *http.Client, reqOptions *requestOptions, user *auth.UserState, decode func(ok bool, resp *http.Response) error) (resp *http.Response, err error) {
var attempt *retry.Attempt
startTime := time.Now()
for attempt = retry.Start(defaultRetryStrategy, nil); attempt.Next(); {
if attempt.Count() > 1 {
delta := time.Since(startTime) / time.Millisecond
logger.Debugf("Retyring %s, attempt %d, delta time=%v ms", reqOptions.URL, attempt.Count(), delta)
}
if cancelled(ctx) {
return nil, ctx.Err()
}
resp, err = s.doRequest(ctx, client, reqOptions, user)
if err != nil {
if shouldRetryError(attempt, err) {
continue
}
break
}
if shouldRetryHttpResponse(attempt, resp) {
resp.Body.Close()
continue
} else {
ok := (resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated)
// always decode on success; decode failures only if body is not empty
if !ok && resp.ContentLength == 0 {
resp.Body.Close()
break
}
err = decode(ok, resp)
resp.Body.Close()
if err != nil {
if shouldRetryError(attempt, err) {
continue
} else {
return nil, err
}
}
}
// break out from retry loop
break
}
if attempt.Count() > 1 {
var status string
delta := time.Since(startTime) / time.Millisecond
if err != nil {
status = err.Error()
} else if resp != nil {
status = fmt.Sprintf("%d", resp.StatusCode)
}
logger.Debugf("The retry loop for %s finished after %d retries, delta time=%v ms, status: %s", reqOptions.URL, attempt.Count(), delta, status)
}
return resp, err
}
// ExecInCoreSnap makes sure you're executing the binary that ships in
// the core snap.
func ExecInCoreSnap() {
if !release.OnClassic {
// you're already the real deal, natch
return
}
if os.Getenv(key) != "1" {
return
}
exe, err := os.Readlink("/proc/self/exe")
if err != nil {
return
}
full := filepath.Join(newCore, exe)
if !osutil.FileExists(full) {
if rev, err := os.Readlink(oldCore); err != nil {
return
} else if revno, err := strconv.Atoi(rev); err != nil || revno < minOldRevno {
return
}
full = filepath.Join(oldCore, exe)
if !osutil.FileExists(full) {
return
}
}
logger.Debugf("restarting into %q", full)
env := append(os.Environ(), key+"=0")
panic(syscall.Exec(full, os.Args, env))
}
// Init sets up the Daemon's internal workings.
// Don't call more than once.
func (d *Daemon) Init() error {
t0 := time.Now()
listeners, err := activation.Listeners(false)
if err != nil {
return err
}
listenerMap := make(map[string]net.Listener)
for _, listener := range listeners {
listenerMap[listener.Addr().String()] = listener
}
// The SnapdSocket is required-- without it, die.
if listener, ok := listenerMap[dirs.SnapdSocket]; ok {
d.snapdListener = &ucrednetListener{listener}
} else {
return fmt.Errorf("daemon is missing the listener for %s", dirs.SnapdSocket)
}
// Note that the SnapSocket listener does not use ucrednet. We use the lack
// of remote information as an indication that the request originated with
// this socket. This listener may also be nil if that socket wasn't among
// the listeners, so check it before using it.
d.snapListener = listenerMap[dirs.SnapSocket]
d.addRoutes()
logger.Debugf("init done in %s", time.Now().Sub(t0))
return nil
}
func (r *TaskRunner) clean(t *Task) {
if !t.Change().IsReady() {
// Whole Change is not ready so don't run cleanups yet.
return
}
cleanup, ok := r.cleanups[t.Kind()]
if !ok {
t.SetClean()
return
}
tomb := &tomb.Tomb{}
r.tombs[t.ID()] = tomb
tomb.Go(func() error {
tomb.Kill(cleanup(t, tomb))
// Locks must be acquired in the same order everywhere.
r.mu.Lock()
defer r.mu.Unlock()
r.state.Lock()
defer r.state.Unlock()
delete(r.tombs, t.ID())
if tomb.Err() != nil {
logger.Debugf("Cleaning task %s: %s", t.ID(), tomb.Err())
} else {
t.SetClean()
}
return nil
})
}
// RoundTrip is from the http.RoundTripper interface.
func (tr *LoggedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
flags := tr.getFlags()
if flags.debugRequest() {
buf, _ := httputil.DumpRequestOut(req, tr.body && flags.debugBody())
logger.Debugf("> %q", buf)
}
rsp, err := tr.Transport.RoundTrip(req)
if err == nil && flags.debugResponse() {
buf, _ := httputil.DumpResponse(rsp, tr.body && flags.debugBody())
logger.Debugf("< %q", buf)
}
return rsp, err
}
// build a new http.Request with headers for the store
func (s *Store) newRequest(reqOptions *requestOptions, user *auth.UserState) (*http.Request, error) {
var body io.Reader
if reqOptions.Data != nil {
body = bytes.NewBuffer(reqOptions.Data)
}
req, err := http.NewRequest(reqOptions.Method, reqOptions.URL.String(), body)
if err != nil {
return nil, err
}
if s.authContext != nil {
device, err := s.authContext.Device()
if err != nil {
return nil, err
}
// we don't have a session yet but have a serial, try
// to get a session
if device.SessionMacaroon == "" && device.Serial != "" {
err = s.refreshDeviceSession(device)
if err == auth.ErrNoSerial {
// missing serial assertion, log and continue without device authentication
logger.Debugf("cannot set device session: %v", err)
}
if err != nil && err != auth.ErrNoSerial {
return nil, err
}
}
authenticateDevice(req, device)
}
// only set user authentication if user logged in to the store
if hasStoreAuth(user) {
authenticateUser(req, user)
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("Accept", reqOptions.Accept)
req.Header.Set("X-Ubuntu-Architecture", s.architecture)
req.Header.Set("X-Ubuntu-Series", s.series)
req.Header.Set("X-Ubuntu-Classic", strconv.FormatBool(release.OnClassic))
req.Header.Set("X-Ubuntu-Wire-Protocol", UbuntuCoreWireProtocol)
if reqOptions.ContentType != "" {
req.Header.Set("Content-Type", reqOptions.ContentType)
}
for header, value := range reqOptions.ExtraHeaders {
req.Header.Set(header, value)
}
s.setStoreID(req)
return req, nil
}
func (t *Task) addLog(kind, format string, args []interface{}) {
if len(t.log) > 9 {
copy(t.log, t.log[len(t.log)-9:])
t.log = t.log[:9]
}
tstr := timeNow().Format(time.RFC3339)
msg := fmt.Sprintf(tstr+" "+kind+" "+format, args...)
t.log = append(t.log, msg)
logger.Debugf(msg)
}
func shouldSearchStore(r *http.Request) bool {
// we should jump to the old behaviour iff q is given, or if
// sources is given and either empty or contains the word
// 'store'. Otherwise, local results only.
query := r.URL.Query()
if _, ok := query["q"]; ok {
logger.Debugf("use of obsolete \"q\" parameter: %q", r.URL)
return true
}
if src, ok := query["sources"]; ok {
logger.Debugf("use of obsolete \"sources\" parameter: %q", r.URL)
if len(src) == 0 || strings.Contains(src[0], "store") {
return true
}
}
return false
}
func main() {
if err := logger.SimpleSetup(); err != nil {
fmt.Fprintf(os.Stderr, "failed to activate logging: %v\n", err)
os.Exit(1)
}
logger.Debugf("fakestore starting")
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func logit(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ww := &wrappedWriter{w: w}
t0 := time.Now()
handler.ServeHTTP(ww, r)
t := time.Now().Sub(t0)
url := r.URL.String()
if !strings.Contains(url, "/changes/") {
logger.Debugf("%s %s %s %s %d", r.RemoteAddr, r.Method, r.URL, t, ww.s)
}
})
}
func updateDesktopDatabase(desktopFiles []string) error {
if len(desktopFiles) == 0 {
return nil
}
if _, err := exec.LookPath("update-desktop-database"); err == nil {
if output, err := exec.Command("update-desktop-database", dirs.SnapDesktopFilesDir).CombinedOutput(); err != nil {
return fmt.Errorf("cannot update-desktop-database %q: %s", output, err)
}
logger.Debugf("update-desktop-database successful")
}
return nil
}
func (d *Daemon) addRoutes() {
d.router = mux.NewRouter()
for _, c := range api {
c.d = d
logger.Debugf("adding %s", c.Path)
d.router.Handle(c.Path, c).Name(c.Path)
}
// also maybe add a /favicon.ico handler...
d.router.NotFoundHandler = NotFound("not found")
}
// downloadAndApplyDelta downloads and then applies the delta to the current snap.
func (s *Store) downloadAndApplyDelta(name string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) (path string, err error) {
deltaInfo := &downloadInfo.Deltas[0]
workingDir, err := ioutil.TempDir(dirs.SnapPartialBlobDir, "deltas-"+name)
if err != nil {
return "", err
}
defer os.RemoveAll(workingDir)
deltaPath, err := s.downloadDelta(name, workingDir, downloadInfo, pbar, user)
if err != nil {
return "", err
}
logger.Debugf("Successfully downloaded delta for %q at %s", name, deltaPath)
snapPath, err := applyDelta(name, deltaPath, deltaInfo)
if err != nil {
return "", err
}
logger.Debugf("Successfully applied delta for %q at %s. Returning %s instead of full download and saving %d bytes.", name, deltaPath, snapPath, downloadInfo.Size-deltaInfo.Size)
return snapPath, nil
}
func (s *Store) setStoreID(r *http.Request) {
storeID := s.fallbackStoreID
if s.authContext != nil {
cand, err := s.authContext.StoreID(storeID)
if err != nil {
logger.Debugf("cannot get store ID from state: %v", err)
} else {
storeID = cand
}
}
if storeID != "" {
r.Header.Set("X-Ubuntu-Store", storeID)
}
}
// Download downloads the snap addressed by download info and returns its
// filename.
// The file is saved in temporary storage, and should be removed
// after use to prevent the disk from running out of space.
func (s *Store) Download(ctx context.Context, name string, targetPath string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) error {
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
if useDeltas() {
logger.Debugf("Available deltas returned by store: %v", downloadInfo.Deltas)
}
if useDeltas() && len(downloadInfo.Deltas) == 1 {
err := s.downloadAndApplyDelta(name, targetPath, downloadInfo, pbar, user)
if err == nil {
return nil
}
// We revert to normal downloads if there is any error.
logger.Noticef("Cannot download or apply deltas for %s: %v", name, err)
}
w, err := os.OpenFile(targetPath+".partial", os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return err
}
resume, err := w.Seek(0, os.SEEK_END)
if err != nil {
return err
}
defer func() {
if cerr := w.Close(); cerr != nil && err == nil {
err = cerr
}
if err != nil {
os.Remove(w.Name())
}
}()
url := downloadInfo.AnonDownloadURL
if url == "" || hasStoreAuth(user) {
url = downloadInfo.DownloadURL
}
err = download(ctx, name, downloadInfo.Sha3_384, url, user, s, w, resume, pbar)
if err != nil {
return err
}
if err := os.Rename(w.Name(), targetPath); err != nil {
return err
}
return w.Sync()
}
// patch5:
// - regenerate generated .service files
func patch5(st *state.State) error {
log := log{}
snapStates, err := snapstate.All(st)
if err != nil {
return err
}
for snapName, snapState := range snapStates {
if !snapState.Active {
continue
}
info, err := snapState.CurrentInfo()
if err != nil {
return err
}
if len(info.Apps) == 0 {
logger.Debugf("patch 5: skipping for %q: no apps", snapName)
continue
}
err = wrappers.StopSnapServices(info, log)
if err != nil {
return err
}
err = wrappers.AddSnapServices(info, log)
if err != nil {
return err
}
err = wrappers.StartSnapServices(info, log)
if err != nil {
return err
}
logger.Noticef("patch 5: %q updated", snapName)
}
return nil
}
// Download downloads the snap addressed by download info and returns its
// filename.
// The file is saved in temporary storage, and should be removed
// after use to prevent the disk from running out of space.
func (s *Store) Download(name string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) (path string, err error) {
if err := os.MkdirAll(dirs.SnapPartialBlobDir, 0755); err != nil {
return "", err
}
if useDeltas() {
logger.Debugf("Available deltas returned by store: %v", downloadInfo.Deltas)
}
if useDeltas() && len(downloadInfo.Deltas) == 1 {
snapPath, err := s.downloadAndApplyDelta(name, downloadInfo, pbar, user)
if err == nil {
return snapPath, nil
}
// We revert to normal downloads if there is any error.
logger.Noticef("Cannot download or apply deltas for %s: %v", name, err)
}
w, err := os.Create(filepath.Join(dirs.SnapPartialBlobDir, downloadInfo.Sha3_384+".snap"))
if err != nil {
return "", err
}
defer func() {
if cerr := w.Close(); cerr != nil && err == nil {
err = cerr
}
if err != nil {
os.Remove(w.Name())
path = ""
}
}()
url := downloadInfo.AnonDownloadURL
if url == "" || hasStoreAuth(user) {
url = downloadInfo.DownloadURL
}
if err := download(name, url, user, s, w, pbar); err != nil {
return "", err
}
return w.Name(), w.Sync()
}
// Download downloads the snap addressed by download info and returns its
// filename.
// The file is saved in temporary storage, and should be removed
// after use to prevent the disk from running out of space.
func (s *Store) Download(ctx context.Context, name string, targetPath string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) error {
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
if useDeltas() {
logger.Debugf("Available deltas returned by store: %v", downloadInfo.Deltas)
}
if useDeltas() && len(downloadInfo.Deltas) == 1 {
err := s.downloadAndApplyDelta(name, targetPath, downloadInfo, pbar, user)
if err == nil {
return nil
}
// We revert to normal downloads if there is any error.
logger.Noticef("Cannot download or apply deltas for %s: %v", name, err)
}
partialPath := targetPath + ".partial"
w, err := os.OpenFile(partialPath, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return err
}
resume, err := w.Seek(0, os.SEEK_END)
if err != nil {
return err
}
defer func() {
if cerr := w.Close(); cerr != nil && err == nil {
err = cerr
}
if err != nil {
os.Remove(w.Name())
}
}()
url := downloadInfo.AnonDownloadURL
if url == "" || hasStoreAuth(user) {
url = downloadInfo.DownloadURL
}
err = download(ctx, name, downloadInfo.Sha3_384, url, user, s, w, resume, pbar)
// If sha3 checksum is incorrect and it was a resumed download, retry from scratch.
// Note that we will retry this way only once.
if _, ok := err.(HashError); ok && resume > 0 {
logger.Debugf("Error on resumed download: %v", err.Error())
err = w.Truncate(0)
if err != nil {
return err
}
_, err = w.Seek(0, os.SEEK_SET)
if err != nil {
return err
}
err = download(ctx, name, downloadInfo.Sha3_384, url, user, s, w, 0, pbar)
}
if err != nil {
return err
}
if err := os.Rename(w.Name(), targetPath); err != nil {
return err
}
return w.Sync()
}
// ListRefresh returns the available updates for a list of snap identified by fullname with channel.
func (s *Store) ListRefresh(installed []*RefreshCandidate, user *auth.UserState) (snaps []*snap.Info, err error) {
candidateMap := map[string]*RefreshCandidate{}
currentSnaps := make([]currentSnapJson, 0, len(installed))
for _, cs := range installed {
revision := cs.Revision.N
if !cs.Revision.Store() {
revision = 0
}
// the store gets confused if we send snaps without a snapid
// (like local ones)
if cs.SnapID == "" {
continue
}
confinement := snap.StrictConfinement
if cs.DevMode {
confinement = snap.DevModeConfinement
}
currentSnaps = append(currentSnaps, currentSnapJson{
SnapID: cs.SnapID,
Channel: cs.Channel,
Confinement: confinement,
Epoch: cs.Epoch,
Revision: revision,
})
candidateMap[cs.SnapID] = cs
}
// build input for the updates endpoint
jsonData, err := json.Marshal(metadataWrapper{
Snaps: currentSnaps,
Fields: s.detailFields,
})
if err != nil {
return nil, err
}
reqOptions := &requestOptions{
Method: "POST",
URL: s.bulkURI,
Accept: halJsonContentType,
ContentType: jsonContentType,
Data: jsonData,
}
if useDeltas() {
logger.Debugf("Deltas enabled. Adding header X-Ubuntu-Delta-Formats: %v", s.deltaFormat)
reqOptions.ExtraHeaders = map[string]string{
"X-Ubuntu-Delta-Formats": s.deltaFormat,
}
}
var updateData searchResults
resp, err := s.retryRequestDecodeJSON(context.TODO(), s.client, reqOptions, user, &updateData, nil)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, respToError(resp, "query the store for updates")
}
res := make([]*snap.Info, 0, len(updateData.Payload.Packages))
for _, rsnap := range updateData.Payload.Packages {
rrev := snap.R(rsnap.Revision)
cand := candidateMap[rsnap.SnapID]
// the store also gives us identical revisions, filter those
// out, we are not interested
if rrev == cand.Revision {
continue
}
// do not upgade to a version we rolledback back from
if findRev(rrev, cand.Block) {
continue
}
res = append(res, infoFromRemote(rsnap))
}
s.extractSuggestedCurrency(resp)
return res, nil
}
// Ensure starts new goroutines for all known tasks with no pending
// dependencies.
// Note that Ensure will lock the state.
func (r *TaskRunner) Ensure() {
r.mu.Lock()
defer r.mu.Unlock()
if r.stopped {
// we are stopping, don't run another ensure
return
}
// Locks must be acquired in the same order everywhere.
r.state.Lock()
defer r.state.Unlock()
r.someBlocked = false
running := make([]*Task, 0, len(r.tombs))
for tid := range r.tombs {
t := r.state.Task(tid)
if t != nil {
running = append(running, t)
}
}
ensureTime := timeNow()
nextTaskTime := time.Time{}
for _, t := range r.state.Tasks() {
handlers, ok := r.handlers[t.Kind()]
if !ok {
// Handled by a different runner instance.
continue
}
tb := r.tombs[t.ID()]
if t.Status() == AbortStatus {
if tb != nil {
tb.Kill(nil)
continue
}
r.tryUndo(t)
}
if tb != nil {
// Already being handled.
continue
}
status := t.Status()
if status.Ready() {
if !t.IsClean() {
r.clean(t)
}
continue
}
if status == UndoStatus && handlers.undo == nil {
// Cannot undo. Revert to done status.
t.SetStatus(DoneStatus)
if len(t.WaitTasks()) > 0 {
r.state.EnsureBefore(0)
}
continue
}
if mustWait(t) {
// Dependencies still unhandled.
continue
}
// skip tasks scheduled for later and also track the earliest one
tWhen := t.AtTime()
if !tWhen.IsZero() && ensureTime.Before(tWhen) {
if nextTaskTime.IsZero() || nextTaskTime.After(tWhen) {
nextTaskTime = tWhen
}
continue
}
if r.blocked != nil && r.blocked(t, running) {
r.someBlocked = true
continue
}
logger.Debugf("Running task %s on %s: %s", t.ID(), t.Status(), t.Summary())
r.run(t)
running = append(running, t)
}
// schedule next Ensure no later than the next task time
if !nextTaskTime.IsZero() {
r.state.EnsureBefore(nextTaskTime.Sub(ensureTime))
}
}
