コード例 #1
0
func (s *SyncUserHandler) hBsoGET(w http.ResponseWriter, r *http.Request) {

	if !AcceptHeaderOk(w, r) {
		return
	}

	var (
		bId string
		ok  bool
		cId int
		err error
		bso *syncstorage.BSO
	)

	if bId, ok = extractBsoIdFail(w, r); !ok {
		return
	}

	cId, err = s.getcid(r, false)

	if err != nil {
		if err == syncstorage.ErrNotFound {
			sendRequestProblem(w, r, http.StatusNotFound, errors.Wrap(err, "Collection Not Found"))
		} else {
			InternalError(w, r, err)
		}
		return
	}

	if bso, err = s.db.GetBSO(cId, bId); err == nil {

		if sentNotModified(w, r, bso.Modified) {
			return
		}

		log.WithFields(log.Fields{
			"bso_t": bso.TTL,
			"now":   syncstorage.Now(),
			"diff":  syncstorage.Now() - bso.TTL,
		}).Debug("bso-expired")

		m := syncstorage.ModifiedToString(bso.Modified)
		w.Header().Set("X-Last-Modified", m)
		JsonNewline(w, r, bso)
	} else {
		if err == syncstorage.ErrNotFound {
			sendRequestProblem(w, r, http.StatusNotFound, errors.Wrap(err, "BSO Not Found"))
		} else {
			InternalError(w, r, err)
		}
	}
}
コード例 #2
0
func (s *SyncUserHandler) hCollectionDELETE(w http.ResponseWriter, r *http.Request) {

	cId, err := s.getcid(r, false)

	if err != nil {
		if err == syncstorage.ErrNotFound {
			w.Header().Set("Content-Type", "application/json")
			fmt.Fprintf(w, `{"modified":%s}`, syncstorage.ModifiedToString(syncstorage.Now()))
			return
		} else {
			InternalError(w, r, err)
		}
		return
	}

	cmodified, err := s.db.GetCollectionModified(cId)
	if err != nil {
		InternalError(w, r, err)
		return
	} else if sentNotModified(w, r, cmodified) {
		return
	}

	modified := syncstorage.Now()
	bids, idExists := r.URL.Query()["ids"]
	if idExists {

		bidlist := strings.Split(bids[0], ",")

		if len(bidlist) > s.config.MaxPOSTRecords {
			sendRequestProblem(w, r, http.StatusBadRequest, errors.New("Exceeded max batch size"))
			return
		}

		modified, err = s.db.DeleteBSOs(cId, bidlist...)
		if err != nil {
			InternalError(w, r, err)
			return
		}
	} else {
		err = s.db.DeleteCollection(cId)
		if err != nil {
			InternalError(w, r, err)
			return
		}
	}

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"modified":%s}`, syncstorage.ModifiedToString(modified))
}
コード例 #3
0
func (s *SyncUserHandler) hDeleteEverything(w http.ResponseWriter, r *http.Request) {
	err := s.db.DeleteEverything()
	if err != nil {
		InternalError(w, r, err)
	} else {
		m := syncstorage.ModifiedToString(syncstorage.Now())
		w.Header().Set("Content-Type", "text/plain")
		w.Header().Set("X-Last-Modified", m)
		w.Write([]byte(m))
	}
}
コード例 #4
0
// RequestToPostBSOInput extracts and unmarshals request.Body into a syncstorage.PostBSOInput. It
// returns a PostResults as well since it also validates BSOs
func RequestToPostBSOInput(r *http.Request, maxPayloadSize int) (
	syncstorage.PostBSOInput,
	*syncstorage.PostResults,
	error,
) {

	// bsoToBeProcessed will actually get sent to the DB
	bsoToBeProcessed := syncstorage.PostBSOInput{}
	results := syncstorage.NewPostResults(syncstorage.Now())

	// a list of all the raw json encoded BSOs
	var raw []json.RawMessage

	if ct := getMediaType(r.Header.Get("Content-Type")); ct == "application/json" || ct == "text/plain" {
		decoder := json.NewDecoder(r.Body)
		err := decoder.Decode(&raw)
		if err != nil {
			return nil, nil, errors.Wrap(err, "Could not unmarshal Request body")
		}
	} else { // deal with application/newlines
		raw = ReadNewlineJSON(r.Body)
	}

	for _, rawJSON := range raw {
		var b syncstorage.PutBSOInput
		if parseErr := parseIntoBSO(rawJSON, &b); parseErr == nil {
			if b.Payload != nil && len(*b.Payload) > maxPayloadSize {
				results.AddFailure(b.Id, "Payload too large")
			} else {
				bsoToBeProcessed = append(bsoToBeProcessed, &b)
			}
		} else {
			// couldn't parse a BSO into something real
			// abort immediately
			if parseErr.field == "-" { // json error, not an object
				return nil, nil, errors.Wrap(parseErr, "Could not unmarshal BSO")
			}

			results.AddFailure(parseErr.bId, fmt.Sprintf("invalid %s", parseErr.field))
		}
	}

	// change TTL from seconds (what clients sends)
	// to milliseconds (what the DB uses)
	for _, p := range bsoToBeProcessed {
		if p.TTL != nil {
			tmp := *p.TTL * 1000
			p.TTL = &tmp
		}
	}

	return bsoToBeProcessed, results, nil
}
コード例 #5
0
func testtoken(secret string, uid uint64) token.Token {
	node := "https://syncnode-12345.services.mozilla.com"
	payload := token.TokenPayload{
		Uid:      uid,
		Node:     node,
		Expires:  float64(syncstorage.Now()+60) / 1000,
		Salt:     "pacific",
		FxaUID:   "fxa_" + strconv.FormatUint(uid, 10),
		DeviceId: "device_" + strconv.FormatUint(uid, 10),
	}

	tok, err := token.NewToken([]byte(secret), payload)
	if err != nil {
		panic(err)
	}

	return tok
}
コード例 #6
0
// hCollectionPOSTBatch handles batch=? requests. It is called internally by hCollectionPOST
// to handle batch request logic
func (s *SyncUserHandler) hCollectionPOSTBatch(collectionId int, w http.ResponseWriter, r *http.Request) {

	// CHECK client provided headers to quickly determine if batch exceeds limits
	// this is meant to be a cheap(er) check without actually having to parse the
	// data provided by the user
	for _, headerName := range []string{"X-Weave-Total-Records", "X-Weave-Total-Bytes", "X-Weave-Records", "X-Weave-Bytes"} {
		if strVal := r.Header.Get(headerName); strVal != "" {
			if intVal, err := strconv.Atoi(strVal); err == nil {
				max := 0
				switch headerName {
				case "X-Weave-Total-Records":
					max = s.config.MaxTotalRecords
				case "X-Weave-Total-Bytes":
					max = s.config.MaxTotalBytes
				case "X-Weave-Bytes":
					max = s.config.MaxPOSTBytes
				case "X-Weave-Records":
					max = s.config.MaxPOSTRecords
				}

				if intVal > max {
					WeaveSizeLimitExceeded(w, r,
						errors.Errorf("Limit %s exceed. %d/%d", headerName, intVal, max))
					return
				}
			} else {
				// header value is invalid (not an int)
				sendRequestProblem(w, r, http.StatusBadRequest, errors.Errorf("Invalid integer value for %s", headerName))
				return
			}
		}
	}

	// CHECK the POST size, if possible from client supplied data
	// hopefully shortcut a fail if this exceeds limits
	if r.ContentLength > 0 && r.ContentLength > int64(s.config.MaxPOSTBytes) {
		WeaveSizeLimitExceeded(w, r,
			errors.Errorf("MaxPOSTBytes exceeded in request.ContentLength(%d) > %d",
				r.ContentLength, s.config.MaxPOSTBytes))
		return
	}

	// EXTRACT actual data to check
	bsoToBeProcessed, results, err := RequestToPostBSOInput(r, s.config.MaxRecordPayloadBytes)
	if err != nil {
		WeaveInvalidWBOError(w, r, errors.Wrap(err, "Failed turning POST body into BSO work list"))
		return
	}

	// CHECK actual BSOs sent to see if they exceed limits
	if len(bsoToBeProcessed) > s.config.MaxPOSTRecords {
		sendRequestProblem(w, r, http.StatusRequestEntityTooLarge,
			errors.Errorf("Exceeded %d BSO per request", s.config.MaxPOSTRecords))
		return
	}

	// CHECK BSO decoding validation errors. Don't even start a Batch if there are.
	if len(results.Failed) > 0 {
		modified := syncstorage.Now()
		w.Header().Set("X-Last-Modified", syncstorage.ModifiedToString(modified))
		JsonNewline(w, r, &PostResults{
			Modified: modified,
			Success:  nil,
			Failed:   results.Failed,
		})
		return
	}

	// Get batch id, commit command and internal collection Id
	_, batchId, batchCommit := GetBatchIdAndCommit(r)

	// CHECK batch id is valid for appends. Do this before loading and decoding
	// the body to be more efficient.
	if batchId != "true" {
		id, err := batchIdInt(batchId)
		if err != nil {
			sendRequestProblem(w, r, http.StatusBadRequest, errors.Wrap(err, "Invalid Batch ID Format"))
			return
		}

		if found, err := s.db.BatchExists(id, collectionId); err != nil {
			InternalError(w, r, err)
		} else if !found {
			sendRequestProblem(w, r, http.StatusBadRequest,
				errors.Errorf("Batch id: %s does not exist", batchId))
		}
	}

	filteredBSOs := make([]*syncstorage.PutBSOInput, 0, len(bsoToBeProcessed))
	failures := make(map[string][]string)

	for _, putInput := range bsoToBeProcessed {
		var failId string
		var failReason string

		if !syncstorage.BSOIdOk(putInput.Id) {
			failId = "na"
			failReason = fmt.Sprintf("Invalid BSO id %s", putInput.Id)
		}

		if putInput.SortIndex != nil && !syncstorage.SortIndexOk(*putInput.SortIndex) {
			failId = putInput.Id
			failReason = fmt.Sprintf("Invalid sort index for: %s", putInput.Id)
		}

		if putInput.TTL != nil && !syncstorage.TTLOk(*putInput.TTL) {
			failId = putInput.Id
			failReason = fmt.Sprintf("Invalid TTL for: %s", putInput.Id)
		}

		if failReason != "" {
			if failures[failId] == nil {
				failures[failId] = []string{failReason}
			} else {
				failures[failId] = append(failures[failId], failReason)
			}
			continue
		}

		filteredBSOs = append(filteredBSOs, putInput)
	}

	// JSON Serialize the data for storage in the DB
	buf := new(bytes.Buffer)
	if len(filteredBSOs) > 0 {
		encoder := json.NewEncoder(buf)
		for _, bso := range filteredBSOs {
			if err := encoder.Encode(bso); err != nil { // Note: this writes a newline after each record
				// whoa... presumably should never happen
				InternalError(w, r, errors.Wrap(err, "Failed encoding BSO for payload"))
				return
			}
		}
	}

	// Save either as a new batch or append to an existing batch
	var dbBatchId int
	//   - batchIdInt used to track the internal batchId number in the database after
	//   - the create || append

	appendedOkIds := make([]string, 0, len(filteredBSOs))
	if batchId == "true" {
		newBatchId, err := s.db.BatchCreate(collectionId, buf.String())
		if err != nil {
			InternalError(w, r, errors.Wrap(err, "Failed creating batch"))
			return
		}

		dbBatchId = newBatchId
	} else {
		id, err := batchIdInt(batchId)
		if err != nil {
			sendRequestProblem(w, r, http.StatusBadRequest, errors.Wrap(err, "Invalid Batch ID Format"))
			return
		}

		if len(filteredBSOs) > 0 { // append only if something to do
			if err := s.db.BatchAppend(id, collectionId, buf.String()); err != nil {
				InternalError(w, r, errors.Wrap(err, fmt.Sprintf("Failed append to batch id:%d", dbBatchId)))
				return
			}

		}

		dbBatchId = id
	}

	// The success list only includes the BSO Ids in the
	// POST request. These need to kept to be sent back in
	// the response. Matches the python server's behaviour / passes
	// the tests
	for _, bso := range filteredBSOs {
		appendedOkIds = append(appendedOkIds, bso.Id)
	}

	if batchCommit {
		batchRecord, err := s.db.BatchLoad(dbBatchId, collectionId)
		if err != nil {
			InternalError(w, r, errors.Wrap(err, "Failed Loading Batch to commit"))
			return
		}

		rawJSON := ReadNewlineJSON(bytes.NewBufferString(batchRecord.BSOS))

		// CHECK final data before committing it to the database
		numInBatch := len(rawJSON)
		if numInBatch > s.config.MaxTotalRecords {
			s.db.BatchRemove(dbBatchId)
			WeaveSizeLimitExceeded(w, r,
				errors.Errorf("Too many BSOs (%d) in Batch(%d)", numInBatch, dbBatchId))
			return
		}

		postData := make(syncstorage.PostBSOInput, len(rawJSON), len(rawJSON))
		for i, bsoJSON := range rawJSON {
			var bso syncstorage.PutBSOInput
			if parseErr := parseIntoBSO(bsoJSON, &bso); parseErr != nil {
				// well there is definitely a bug somewhere if this happens
				InternalError(w, r, errors.Wrap(parseErr, "Could not decode batch data"))
				return
			} else {
				postData[i] = &bso
			}
		}

		// CHECK that actual Batch data size
		sum := 0
		for _, bso := range postData {
			if bso.Payload == nil {
				continue
			}

			sum := sum + len(*bso.Payload)
			if sum > s.config.MaxTotalBytes {
				s.db.BatchRemove(dbBatchId)
				WeaveSizeLimitExceeded(w, r,
					errors.Errorf("Batch size(%d) exceeded MaxTotalBytes limit(%d)",
						sum, s.config.MaxTotalBytes))

				return
			}
		}

		postResults, err := s.db.PostBSOs(collectionId, postData)
		if err != nil {
			InternalError(w, r, err)
			return
		}

		// merge failures
		for key, reasons := range postResults.Failed {
			if failures[key] == nil {
				failures[key] = reasons
			} else {
				failures[key] = append(failures[key], reasons...)
			}
		}

		// DELETE the batch from the DB
		s.db.BatchRemove(dbBatchId)

		w.Header().Set("X-Last-Modified", syncstorage.ModifiedToString(postResults.Modified))

		JsonNewline(w, r, &PostResults{
			Modified: postResults.Modified,
			Success:  appendedOkIds,
			Failed:   failures,
		})
	} else {
		modified := syncstorage.Now()
		w.Header().Set("X-Last-Modified", syncstorage.ModifiedToString(modified))
		JsonNewlineStatus(w, r, http.StatusAccepted, &PostResults{
			Batch:    batchIdString(dbBatchId),
			Modified: modified,
			Success:  appendedOkIds,
			Failed:   failures,
		})
	}
}
コード例 #7
0
func TestCacheHandlerInfoCollections(t *testing.T) {
	assert := assert.New(t)
	handler := NewCacheHandler(cacheMockHandler, DefaultCacheHandlerConfig)
	url := syncurl(uniqueUID(), "info/collections?version=1.5") // sometimes we see queries in the requests

	{
		// basic test to make sure data is what's expected and caching works
		resp := request("GET", url, nil, handler)
		assert.Equal(http.StatusOK, resp.Code)
		assert.True(strings.Contains(resp.Body.String(), `"Type":"info/collections"`))
		assert.Equal("application/json", resp.Header().Get("Content-Type"))

		resp2 := request("GET", url, nil, handler)
		assert.Equal(http.StatusOK, resp2.Code)
		assert.Equal(resp.Body.String(), resp2.Body.String(), "Expected cached value")
		assert.Equal(resp.Header().Get("X-Last-Modified"), resp2.Header().Get("X-Last-Modified"))
	}

	// test POST, PUT, DELETE invalidates the cache
	{
		var last *httptest.ResponseRecorder
		for _, method := range []string{"POST", "PUT", "DELETE"} {

			if last == nil {
				last = request("GET", url, nil, handler)
			}

			// ensures things stay the same
			resp := request("GET", url, nil, handler)
			assert.Equal(http.StatusOK, resp.Code)
			assert.Equal(last.Body.String(), resp.Body.String())
			assert.Equal(last.Header().Get("X-Last-Modified"), resp.Header().Get("X-Last-Modified"))

			// swap it
			last = resp

			// this should clear the cache
			// compensate for sync's time format..
			time.Sleep(10 * time.Millisecond)

			request(method, url, nil, handler)

			_ = method

			//// make sure things are changed
			resp = request("GET", url, nil, handler)
			assert.Equal(http.StatusOK, resp.Code)
			assert.NotEqual(last.Body.String(), resp.Body.String(), "Should have changed")
			assert.NotEqual(last.Header().Get("X-Last-Modified"), resp.Header().Get("X-Last-Modified"))

			last = resp
		}
	}

	// test that different uids get different values
	{
		resp1a := request("GET", syncurl("123", "info/collections"), nil, handler)
		resp1b := request("GET", syncurl("123", "info/collections"), nil, handler)

		time.Sleep(10 * time.Millisecond) /// SYNNNC!! .. so awesome timestamps

		resp2b := request("GET", syncurl("456", "info/collections"), nil, handler)
		resp2a := request("GET", syncurl("456", "info/collections"), nil, handler)

		assert.Equal(http.StatusOK, resp1a.Code)
		assert.Equal(http.StatusOK, resp1b.Code)
		assert.Equal(http.StatusOK, resp2a.Code)
		assert.Equal(http.StatusOK, resp2b.Code)

		// cacheing works
		assert.Equal(resp1a.Body.String(), resp1b.Body.String())
		assert.Equal(resp2a.Body.String(), resp2b.Body.String())

		// no conflicts
		assert.NotEqual(resp1a.Body.String(), resp2a.Body.String())
	}

	{ // test with a real SyncUserHandler
		uid := "123456"
		db, _ := syncstorage.NewDB(":memory:", nil)
		userHandler := NewSyncUserHandler(uid, db, nil)
		handler := NewCacheHandler(userHandler, DefaultCacheHandlerConfig)
		collections := []string{
			"clients", "crypto", "forms", "history", "keys", "meta",
			"bookmarks", "prefs", "tabs", "passwords", "addons",
		}
		for _, cName := range collections {
			cId, _ := db.GetCollectionId(cName)
			db.TouchCollection(cId, syncstorage.Now()+cId) // turn the cId into milliseconds
		}

		resp := request("GET", syncurl(uid, "info/collections"), nil, handler)
		resp2 := request("GET", syncurl(uid, "info/collections"), nil, handler)

		assert.Equal(resp.Body.String(), resp2.Body.String())
		assert.Equal(resp.Header().Get("X-Last-Modified"), resp2.Header().Get("X-Last-Modified"))
	}
}
コード例 #8
0
	"strings"
	"testing"
	"time"

	"github.com/mozilla-services/go-syncstorage/syncstorage"
	"github.com/mozilla-services/go-syncstorage/token"
	"github.com/stretchr/testify/assert"
)

// generates a unique response each time so caching can be tested
var cacheMockHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

	resp := struct {
		Type  string
		Value string
	}{"OK", syncstorage.ModifiedToString(syncstorage.Now())}

	w.Header().Set("X-Last-Modified", resp.Value)

	if infoCollectionsRoute.MatchString(req.URL.Path) {
		resp.Type = "info/collections"
		JSON(w, req, http.StatusOK, resp)
	} else if infoConfigurationRoute.MatchString(req.URL.Path) {
		resp.Type = "info/configuration"
		JSON(w, req, http.StatusOK, resp)
	} else {
		JSON(w, req, http.StatusOK, resp)
	}
})

func TestCacheHandlerInfoCollections(t *testing.T) {