func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Host == "gopkgdoc.appspot.com" {
p := r.URL.Path
if strings.HasPrefix(p, "/pkg/") {
p = p[len("/pkg"):]
}
if r.URL.RawQuery != "" {
p = p + "?" + r.URL.RawQuery
}
http.Redirect(w, r, "http://go.pkgdoc.org"+p, 301)
return
}
err := f(w, r)
if err != nil {
appengine.NewContext(r).Errorf("Error %s", err.Error())
if e, ok := err.(doc.GetError); ok {
http.Error(w, "Error getting files from "+e.Host+".", http.StatusInternalServerError)
} else if appengine.IsCapabilityDisabled(err) || appengine.IsOverQuota(err) {
http.Error(w, "Internal error: "+err.Error(), http.StatusInternalServerError)
} else {
http.Error(w, "Internal Error", http.StatusInternalServerError)
}
}
}
func actualRemovePeer(future *removePeerFuture) {
defer close(future.resultChannel)
future.ctx.Debugf("[RemovePeer] Removing peer: (hash: %s) (ID: %s)", future.hash, future.peerID)
err := ds.Delete(future.ctx, getPeerKey(future.ctx, future.hash, future.peerID))
if err == nil {
future.ctx.Debugf("[RemovePeer] Deleted peer from datastore.")
for removalsAttempted := 1; removalsAttempted <= 3; removalsAttempted++ {
future.ctx.Debugf("[RemovePeer] Attempting cache removal #%d", removalsAttempted)
var peers []*Peer
if peersItem, memcacheErr := memcache.Gob.Get(future.ctx, getHashKey(future.ctx, future.hash), &peers); memcacheErr == nil {
for i, peer := range peers {
if peer.ID == future.peerID {
peers = append(peers[:i], peers[i+1:]...)
}
}
peersItem.Object = peers
memcacheErr := memcache.Gob.CompareAndSwap(future.ctx, peersItem)
if memcacheErr == nil {
future.ctx.Debugf("[RemovePeer] Deleted peer from cached peer list.")
break
} else if memcacheErr == memcache.ErrCASConflict {
future.ctx.Debugf("[RemovePeer] Cached peer list conflicted during modification: %s", memcacheErr)
continue
} else if memcacheErr == memcache.ErrNotStored {
future.ctx.Debugf("[RemovePeer] Cached peer list was deleted during modification: %s", memcacheErr)
break
} else {
future.ctx.Errorf("[RemovePeer] Error when attempting to set cached peer list using CAS: %s", memcacheErr)
future.resultChannel <- memcacheErr
return
}
} else if memcacheErr == memcache.ErrCacheMiss {
future.ctx.Debugf("[RemovePeer] Peer list not cached.")
break
} else if appengine.IsCapabilityDisabled(memcacheErr) {
future.ctx.Warningf("[RemovePeer] Memcache capability is disabled.")
break
} else {
future.ctx.Errorf("[RemovePeer] Error when attempting to get cache list for deletion: %s", memcacheErr)
future.resultChannel <- memcacheErr
return
}
}
} else if err == dserrors.ErrNoSuchEntity {
future.ctx.Debugf("[RemovePeer] Peer does not exist in datastore")
return
}
future.ctx.Debugf("[RemovePeer] Successfully removed peer")
if err != nil {
future.resultChannel <- err
}
}
func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Host != "www.gorillatoolkit.org" && !appengine.IsDevAppServer() {
r.URL.Host = "www.gorillatoolkit.org"
http.Redirect(w, r, r.URL.String(), 301)
return
}
err := f(w, r)
if err != nil {
appengine.NewContext(r).Errorf("Error %s", err.Error())
if e, ok := err.(doc.GetError); ok {
http.Error(w, "Error getting files from "+e.Host+".", http.StatusInternalServerError)
} else if appengine.IsCapabilityDisabled(err) || appengine.IsOverQuota(err) {
http.Error(w, "Internal error: "+err.Error(), http.StatusInternalServerError)
} else {
http.Error(w, "Internal Error", http.StatusInternalServerError)
}
}
}
func actualGetPeers(future *getPeersFuture) {
defer close(future.errorChannel)
defer close(future.resultChannel)
future.ctx.Debugf("[GetPeers] Getting peer list (%s)", future.hash)
var res []*Peer
if _, err := memcache.Gob.Get(future.ctx, getHashKey(future.ctx, future.hash), &res); err == nil {
if len(res) == 0 {
future.ctx.Debugf("[GetPeers] Cached peer list is empty")
future.ctx.Debugf("[GetPeers] Successfully retrieved peer list from memcache (empty)")
return
}
future.ctx.Debugf("[GetPeers] Valid list exists in memcache")
if len(future.include) > 0 {
peerToInclude := new(Peer)
if err := ds.Get(future.ctx, getPeerKey(future.ctx, future.hash, future.include), peerToInclude); err != nil {
if err != dserrors.ErrNoSuchEntity {
future.ctx.Errorf("[GetPeers] Failed to include peer: %s", err)
} else {
future.ctx.Debugf("[GetPeers] Failed to include peer because peer doesn't exist")
}
} else {
res = append(res, peerToInclude)
}
}
future.ctx.Debugf("[GetPeers] Filtering peers")
peersFiltered := make([]*Peer, 0, len(res))
currentTime := time.Now()
foundInclude := false
for _, peer := range res {
if !peer.Expires.After(currentTime) {
continue
} else if peer.ID == future.include {
if foundInclude {
continue
} else {
foundInclude = true
}
}
peer.Key = getPeerKey(future.ctx, future.hash, peer.ID)
// We have to copy the peer to this because the "peer" variable from the loop will always be at the same address.
peersFiltered = append(peersFiltered, peer)
}
// This looks so ugly. TODO: Clean this up
var filteredLength int64 = int64(len(peersFiltered))
if filteredLength <= future.numberWanted {
future.ctx.Debugf("[GetPeers] Returning all %d filtered peers: request wants more than we have", filteredLength)
for _, peer := range peersFiltered {
future.resultChannel <- peer
}
return
}
giveNumber := int64(math.Min(float64(filteredLength), float64(future.numberWanted)))
given := make(map[int64]bool)
randomGen := rand.New(rand.NewSource(currentTime.UnixNano()))
future.ctx.Debugf("[GetPeers] Returning %d/%d random filtered peers", giveNumber, filteredLength)
var i int64
for i = 1; i <= giveNumber; i++ {
giveNumber := randomGen.Int63n(filteredLength - 1)
if isGiven, ok := given[giveNumber]; !ok || !isGiven {
future.resultChannel <- peersFiltered[giveNumber]
given[giveNumber] = true
} else {
i--
}
}
future.ctx.Debugf("[GetPeers] Successfully retrieved peer list from memcache")
return
} else if appengine.IsCapabilityDisabled(err) {
future.ctx.Warningf("[GetPeers] Memcache capability disabled: %s", err)
} else if err != memcache.ErrCacheMiss {
future.ctx.Errorf("[GetPeers] Error retrieving cached peer list: %s", err)
future.errorChannel <- err
return
}
future.ctx.Debugf("[GetPeers] No valid list exists in memcache")
peerQuery := datastore.NewQuery("Peer").Filter("Hash =", future.hash).Filter("Expires >", time.Now())
future.ctx.Debugf("[GetPeers] Querying for non-expired peers")
numPeers, err := peerQuery.Count(future.ctx)
if err != nil {
future.ctx.Warningf("[GetPeers] Failed to get count of peers: %s", err)
numPeers = 0
}
peers := make([]*Peer, 0, numPeers)
var count int64 = 0
for iterator := peerQuery.Run(future.ctx); ; {
var thisPeer Peer
key, err := iterator.Next(&thisPeer)
if err == datastore.Done {
break
} else if err != nil {
future.ctx.Errorf("[GetPeers] Error retrieving peers from query: %s", err)
future.errorChannel <- err
return
}
thisPeer.Key = key
peers = append(peers, &thisPeer)
if count < future.numberWanted {
future.resultChannel <- &thisPeer
count++
}
}
future.ctx.Debugf("[GetPeers] Retrieved %d peers", len(peers))
memcacheItem := &memcache.Item{
Key: getHashKey(future.ctx, future.hash),
Object: peers,
}
if err := memcache.Gob.Set(future.ctx, memcacheItem); err == nil {
future.ctx.Debugf("[GetPeers] Stored peer list in memcache")
} else if appengine.IsCapabilityDisabled(err) {
future.ctx.Warningf("[GetPeers] Memcache capability disabled: %s", err)
} else {
future.ctx.Warningf("[GetPeers] Error storing peer list in memcache: %s", err)
}
future.ctx.Debugf("[GetPeers] Successfully retrieved peer list from datastore")
return
}
func actualInsertPeer(future *insertPeerFuture) {
defer close(future.resultChannel)
future.ctx.Debugf("[InsertPeer] Inserting peer: (hash: %s) (ID: %s) (IP: %s) (port: %d)", future.hash, future.peerID, future.ip, future.port)
changed := false
peer := new(Peer)
err := datastore.RunInTransaction(future.ctx, func(tc appengine.Context) error {
peerKey := getPeerKey(tc, future.hash, future.peerID)
if err := ds.Get(tc, peerKey, peer); err == nil {
future.ctx.Debugf("[InsertPeer] Peer already exists, updating...")
if peer.IP != future.ip {
peer.IP = future.ip
changed = true
future.ctx.Debugf("[InsertPeer] Peer's IP has changed")
}
if peer.Port != int32(future.port) {
peer.Port = int32(future.port)
changed = true
future.ctx.Debugf("[InsertPeer] Peer's port has changed")
}
currentTime := time.Now()
if !peer.Expires.After(currentTime) || peer.Expires.Sub(currentTime) <= PeerRefreshTime {
peer.Expires = time.Now().Add(PeerExpireTime).Add(PeerExpireGrace)
changed = true
future.ctx.Debugf("[InsertPeer] Peer's expire time has changed")
}
if changed {
future.ctx.Debugf("[InsertPeer] Putting peer...")
_, err := ds.Put(tc, peerKey, peer)
if err != nil {
future.ctx.Errorf("[InsertPeer] Error when putting updated peer: %s", err)
}
return err
} else {
future.ctx.Debugf("[InsertPeer] No need to update peer.")
return nil
}
} else if err != dserrors.ErrNoSuchEntity {
peer = nil
future.ctx.Errorf("[InsertPeer] Error when retrieving peer from datastore for update: %s", err)
return err
}
future.ctx.Debugf("[InsertPeer] Peer doesn't exist, creating")
peer = &Peer{
Hash: future.hash,
ID: future.peerID,
IP: future.ip,
Port: int32(future.port),
Expires: time.Now().Add(PeerExpireTime).Add(PeerExpireGrace),
}
_, err := ds.Put(tc, peerKey, peer)
if err != nil {
future.ctx.Errorf("[InsertPeer] Error when putting new peer: %s", err)
}
return err
}, nil)
if err == nil && peer != nil {
memcacheKey := getHashKey(future.ctx, future.hash)
for updatesAttempted := 1; updatesAttempted <= 3; updatesAttempted++ {
future.ctx.Debugf("[InsertPeer] Attempting cache update #%d", updatesAttempted)
var peers []*Peer
if peersItem, memcacheErr := memcache.Gob.Get(future.ctx, memcacheKey, &peers); memcacheErr == nil {
exists := false
for _, thisPeer := range peers {
if thisPeer.ID == future.peerID {
exists = true
break
}
}
if !exists {
peers = append(peers, peer)
peersItem.Object = peers
memcacheErr := memcache.Gob.CompareAndSwap(future.ctx, peersItem)
if memcacheErr == nil {
future.ctx.Debugf("[InsertPeer] Added peer to cached peer list.")
break
} else if memcacheErr == memcache.ErrCASConflict {
future.ctx.Debugf("[InsertPeer] Cached peer list conflicted during modification: %s", memcacheErr)
continue
} else if memcacheErr == memcache.ErrNotStored {
future.ctx.Debugf("[InsertPeer] Cached peer list was deleted during modification: %s", memcacheErr)
continue
} else if appengine.IsCapabilityDisabled(memcacheErr) {
future.ctx.Warningf("[InsertPeer] Memcache capability disabled when attempting to add peer to cached peer list")
continue
} else {
future.ctx.Errorf("[InsertPeer] Error when attempting to set cached peer list using CAS: %s", memcacheErr)
future.resultChannel <- memcacheErr
return
}
} else {
future.ctx.Debugf("[InsertPeer] Peer already exists in cached list.")
break
}
} else if memcacheErr == memcache.ErrCacheMiss {
future.ctx.Debugf("[InsertPeer] Peer list not cached. Creating...")
peersItem := &memcache.Item{
Key: memcacheKey,
Object: []*Peer{peer},
}
if memcacheErr := memcache.Gob.CompareAndSwap(future.ctx, peersItem); memcacheErr == nil {
future.ctx.Debugf("[InsertPeer] Peer list created")
break
} else if memcacheErr == memcache.ErrCASConflict {
future.ctx.Debugf("[InsertPeer] Cached peer list conflicted during creation: %s", memcacheErr)
continue
} else if memcacheErr == memcache.ErrNotStored {
future.ctx.Debugf("[InsertPeer] Cached peer list was deleted during creation: %s", memcacheErr)
continue
} else if appengine.IsCapabilityDisabled(memcacheErr) {
future.ctx.Warningf("[InsertPeer] Memcache capability disabled when attempting to add peer to create cached peer list")
continue
} else {
future.ctx.Errorf("[InsertPeer] Error when creating peer list: %s", memcacheErr)
future.resultChannel <- memcacheErr
return
}
} else if appengine.IsCapabilityDisabled(memcacheErr) {
future.ctx.Warningf("[InsertPeer] Memcache capability is disabled.")
break
} else {
future.ctx.Errorf("[InsertPeer] Error when attempting to get cache list for insertion: %s", memcacheErr)
future.resultChannel <- memcacheErr
return
}
}
future.ctx.Debugf("[InsertPeer] Successfully inserted peer")
} else if err != nil {
future.resultChannel <- err
}
}
