Beispiel #1
0
func (self *log) appendRequest(request *protocol.Request, shardId uint32) error {
	bytes, err := request.Encode()

	if err != nil {
		return err
	}
	// every request is preceded with the length, shard id and the request number
	hdr := &entryHeader{
		shardId:       shardId,
		requestNumber: request.GetRequestNumber(),
		length:        uint32(len(bytes)),
	}
	writtenHdrBytes, err := hdr.Write(self.file)
	if err != nil {
		logger.Error("Error while writing header: %s", err)
		return err
	}
	written, err := self.file.Write(bytes)
	if err != nil {
		logger.Error("Error while writing request: %s", err)
		return err
	}
	if written < len(bytes) {
		err = fmt.Errorf("Couldn't write entire request")
		logger.Error("Error while writing request: %s", err)
		return err
	}
	self.fileSize += uint64(writtenHdrBytes + written)
	return nil
}
Beispiel #2
0
func (self *log) appendRequest(request *protocol.Request, shardId uint32) (uint32, error) {
	self.assignSequenceNumbers(shardId, request)
	bytes, err := request.Encode()

	if err != nil {
		return 0, err
	}
	requestNumber := self.state.getNextRequestNumber()
	// every request is preceded with the length, shard id and the request number
	hdr := &entryHeader{
		shardId:       shardId,
		requestNumber: requestNumber,
		length:        uint32(len(bytes)),
	}
	writtenHdrBytes, err := hdr.Write(self.file)
	if err != nil {
		logger.Error("Error while writing header: %s", err)
		return 0, err
	}
	written, err := self.file.Write(bytes)
	if err != nil {
		logger.Error("Error while writing request: %s", err)
		return 0, err
	}
	if written < len(bytes) {
		err = fmt.Errorf("Couldn't write entire request")
		logger.Error("Error while writing request: %s", err)
		return 0, err
	}
	self.fileSize += uint64(writtenHdrBytes + written)
	self.conditionalBookmarkAndIndex()
	return requestNumber, nil
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
	if *request.Type == protocol.Request_WRITE {
		shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
		log.Debug("HANDLE: (%d):%d:%v", self.clusterConfig.LocalServer.Id, request.GetId(), shard)
		err := shard.WriteLocalOnly(request)
		if err != nil {
			log.Error("ProtobufRequestHandler: error writing local shard: ", err)
			return err
		}
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}
		return self.WriteResponse(conn, response)
	} else if *request.Type == protocol.Request_DROP_DATABASE {
		go self.handleDropDatabase(request, conn)
		return nil
	} else if *request.Type == protocol.Request_QUERY {
		go self.handleQuery(request, conn)
	} else if *request.Type == protocol.Request_HEARTBEAT {
		response := &protocol.Response{RequestId: request.Id, Type: &heartbeatResponse}
		return self.WriteResponse(conn, response)
	} else {
		log.Error("unknown request type: %v", request)
		return errors.New("Unknown request type")
	}
	return nil
}
Beispiel #4
0
func (self *CoordinatorImpl) ReplicateWrite(request *protocol.Request) error {
	id := atomic.AddUint32(&self.requestId, uint32(1))
	request.Id = &id
	location := common.RingLocation(request.Database, request.Series.Name, request.Series.Points[0].Timestamp)
	replicas := self.clusterConfiguration.GetServersByRingLocation(request.Database, &location)
	request.Type = &replicateWrite
	self.sendRequestToReplicas(request, replicas)
	return nil
}
Beispiel #5
0
func (self *CoordinatorImpl) ReplicateDelete(request *protocol.Request) error {
	id := atomic.AddUint32(&self.requestId, uint32(1))
	request.Id = &id
	server := self.clusterConfiguration.GetServerById(request.OwnerServerId)
	_, replicas := self.clusterConfiguration.GetReplicas(server, request.Database)
	request.Type = &replicateDelete
	self.sendRequestToReplicas(request, replicas)
	return nil
}
Beispiel #6
0
func (self *CoordinatorImpl) sendRequestToReplicas(request *protocol.Request, replicas []*ClusterServer) {
	for _, server := range replicas {
		if server.Id != self.clusterConfiguration.localServerId {
			err := server.MakeRequest(request, nil)
			if err != nil {
				log.Warn("REPLICATION ERROR: ", request.GetSequenceNumber(), err)
			}
		}
	}
}
Beispiel #7
0
func (self *ClusterServer) Write(request *protocol.Request) error {
	responseChan := make(chan *protocol.Response)
	err := self.connection.MakeRequest(request, responseChan)
	if err != nil {
		return err
	}
	log.Debug("Waiting for response to %d", request.GetRequestNumber())
	response := <-responseChan
	if response.ErrorMessage != nil {
		return errors.New(*response.ErrorMessage)
	}
	return nil
}
Beispiel #8
0
func (self *CoordinatorImpl) proxyWrite(clusterServer *ClusterServer, request *protocol.Request) error {
	originatingServerId := request.OriginatingServerId
	request.OriginatingServerId = nil
	defer func() { request.OriginatingServerId = originatingServerId }()

	responseChan := make(chan *protocol.Response, 1)
	clusterServer.protobufClient.MakeRequest(request, responseChan)
	response := <-responseChan
	if *response.Type == protocol.Response_WRITE_OK {
		return nil
	} else {
		return errors.New(response.GetErrorMessage())
	}
}
func (self *ProtobufRequestHandler) handleWrites(request *protocol.Request, conn net.Conn) {
	shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
	log.Debug("HANDLE: (%d):%d:%v", self.clusterConfig.LocalServer.Id, request.GetId(), shard)
	err := shard.WriteLocalOnly(request)
	var errorMsg *string
	if err != nil {
		log.Error("ProtobufRequestHandler: error writing local shard: %s", err)
		errorMsg = protocol.String(err.Error())
	}
	response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk, ErrorMessage: errorMsg}
	if err := self.WriteResponse(conn, response); err != nil {
		log.Error("ProtobufRequestHandler: error writing local shard: %s", err)
	}
}
Beispiel #10
0
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
	if request.Id == nil {
		id := atomic.AddUint32(&self.lastRequestId, uint32(1))
		request.Id = &id
	}
	if responseStream != nil {
		self.requestBufferLock.Lock()

		// this should actually never happen. The sweeper should clear out dead requests
		// before the uint32 ids roll over.
		if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
			message := "already has a request with this id, must have timed out"
			log.Error(message)
			oldReq.responseChan <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message}
		}
		self.requestBuffer[*request.Id] = &runningRequest{timeMade: time.Now(), responseChan: responseStream, request: request}
		self.requestBufferLock.Unlock()
	}

	data, err := request.Encode()
	if err != nil {
		return err
	}

	conn := self.getConnection()
	if conn == nil {
		conn = self.reconnect()
		if conn == nil {
			return fmt.Errorf("Failed to connect to server %s", self.hostAndPort)
		}
	}

	if self.writeTimeout > 0 {
		conn.SetWriteDeadline(time.Now().Add(self.writeTimeout))
	}
	buff := bytes.NewBuffer(make([]byte, 0, len(data)+8))
	binary.Write(buff, binary.LittleEndian, uint32(len(data)))
	_, err = conn.Write(append(buff.Bytes(), data...))

	if err == nil {
		return nil
	}

	// if we got here it errored out, clear out the request
	self.requestBufferLock.Lock()
	delete(self.requestBuffer, *request.Id)
	self.requestBufferLock.Unlock()
	self.reconnect()
	return err
}
Beispiel #11
0
func (self *ShardData) forwardRequest(request *p.Request) ([]<-chan *p.Response, []uint32, error) {
	ids := []uint32{}
	responses := []<-chan *p.Response{}
	for _, server := range self.clusterServers {
		responseChan := make(chan *p.Response, 1)
		// do this so that a new id will get assigned
		request.Id = nil
		log.Debug("Forwarding request %s to %d", request.GetDescription(), server.Id)
		server.MakeRequest(request, responseChan)
		responses = append(responses, responseChan)
		ids = append(ids, server.Id)
	}
	return responses, ids, nil
}
Beispiel #12
0
// This method never blocks. It'll buffer writes until they fill the buffer then drop the on the
// floor and let the background goroutine replay from the WAL
func (self *WriteBuffer) Write(request *protocol.Request) {
	self.shardLastRequestNumber[request.GetShardId()] = request.GetRequestNumber()
	select {
	case self.writes <- request:
		return
	default:
		select {
		case self.stoppedWrites <- *request.RequestNumber:
			return
		default:
			return
		}
	}
}
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
	// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
	var numberKey []byte
	if request.SequenceNumber == nil {
		sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
		if err != nil {
			return err
		}
		request.SequenceNumber = &sequenceNumber
	} else {
		// this is for a replicated write, ensure that it's the next in line for this owner and server
		name := self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
		numberKey = append(ATOMIC_INCREMENT_PREFIX, []byte(name)...)
		numberBytes, err := self.db.Get(self.readOptions, numberKey)
		if err != nil {
			return err
		}
		previousSequenceNumber := self.bytesToCurrentNumber(numberBytes)
		if previousSequenceNumber+uint64(1) != *request.SequenceNumber {
			return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber}
		}
	}

	self.requestLogLock.RLock()
	requestLog := self.currentRequestLog
	self.requestLogLock.RUnlock()

	// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
	if *request.Type == protocol.Request_PROXY_WRITE {
		request.Type = &replicateWrite
	}

	data, err := request.Encode()
	if err != nil {
		return err
	}

	key := self.requestLogKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
	err = requestLog.db.Put(self.writeOptions, key, data)
	if err != nil {
		return err
	}
	if numberKey != nil {
		currentNumberBuffer := bytes.NewBuffer(make([]byte, 0, 8))
		binary.Write(currentNumberBuffer, binary.BigEndian, *request.SequenceNumber)
		self.db.Put(self.writeOptions, numberKey, currentNumberBuffer.Bytes())
	}
	return nil
}
Beispiel #14
0
func (self *ShardData) Write(request *protocol.Request) error {
	request.ShardId = &self.id
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.store.BufferWrite(request)
	}
	for _, server := range self.clusterServers {
		server.BufferWrite(request)
	}
	return nil
}
Beispiel #15
0
func (self *LevelDbDatastore) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
	// log to this key structure on a different DB sharded by day: <cluster version><owner id><sequence server id><replication sequence>
	updateSequenceNumber := false
	if request.SequenceNumber == nil {
		sequenceNumber, err := self.AtomicIncrement(self.keyForOwnerAndServerSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId), 1)
		if err != nil {
			return err
		}
		request.SequenceNumber = &sequenceNumber
	} else {
		updateSequenceNumber = true
		previousSequenceNumber, err := self.CurrentSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId)
		if err != nil {
			return err
		}
		// Do a less than comparison because it's ok if we're just getting the same write again. As long as we haven't missed one.
		if previousSequenceNumber+uint64(1) < *request.SequenceNumber {
			log.Warn("MISSING REQUESTS: %d, %d", previousSequenceNumber, *request.SequenceNumber)
			return SequenceMissingRequestsError{"Missing requests between last seen and this one.", previousSequenceNumber, *request.SequenceNumber}
		}
	}

	self.requestLogLock.RLock()
	requestLog := self.currentRequestLog
	self.requestLogLock.RUnlock()

	// proxied writes should be logged as replicated ones. That's what is expected if they're replayed later
	if *request.Type == protocol.Request_PROXY_WRITE {
		request.Type = &replicateWrite
	}

	data, err := request.Encode()
	if err != nil {
		return err
	}

	key := NewWALKey(request.ClusterVersion, request.OriginatingServerId, ownerServerId, request.SequenceNumber, replicationFactor)
	err = requestLog.db.Put(self.writeOptions, key, data)
	if err != nil {
		return err
	}
	if updateSequenceNumber {
		return self.updateSequenceNumber(request.ClusterVersion, replicationFactor, ownerServerId, request.OriginatingServerId, *request.SequenceNumber)
	}
	return nil
}
Beispiel #16
0
func (self *ShardData) Write(request *p.Request) error {
	request.ShardId = &self.id
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.store.BufferWrite(request)
	}
	for _, server := range self.clusterServers {
		// we have to create a new reqeust object because the ID gets assigned on each server.
		requestWithoutId := &p.Request{Type: request.Type, Database: request.Database, MultiSeries: request.MultiSeries, ShardId: &self.id, RequestNumber: request.RequestNumber}
		server.BufferWrite(requestWithoutId)
	}
	return nil
}
Beispiel #17
0
// Makes a request to the server. If the responseStream chan is not nil it will expect a response from the server
// with a matching request.Id. The REQUEST_RETRY_ATTEMPTS constant of 3 and the RECONNECT_RETRY_WAIT of 100ms means
// that an attempt to make a request to a downed server will take 300ms to time out.
func (self *ProtobufClient) MakeRequest(request *protocol.Request, responseStream chan *protocol.Response) error {
	if responseStream != nil {
		self.requestBufferLock.Lock()

		// this should actually never happen. The sweeper should clear out dead requests
		// before the uint32 ids roll over.
		if oldReq, alreadyHasRequestById := self.requestBuffer[*request.Id]; alreadyHasRequestById {
			log.Error("already has a request with this id, must have timed out")
			close(oldReq.responseChan)
		}
		self.requestBuffer[*request.Id] = &runningRequest{time.Now(), responseStream}
		self.requestBufferLock.Unlock()
	}

	data, err := request.Encode()
	if err != nil {
		return err
	}

	// retry sending this at least a few times
	for attempts := 0; attempts < REQUEST_RETRY_ATTEMPTS; attempts++ {
		conn := self.getConnection()
		if conn == nil {
			self.reconnect()
			continue
		}

		err = binary.Write(conn, binary.LittleEndian, uint32(len(data)))
		if err == nil {
			_, err = conn.Write(data)
			if err == nil {
				return nil
			}
		}
		log.Error("ProtobufClient: error making request: %s", err)
		// TODO: do something smarter here based on whatever the error is.
		// failed to make the request, reconnect and try again.
		self.reconnect()
	}

	// if we got here it errored out, clear out the request
	self.requestBufferLock.Lock()
	delete(self.requestBuffer, *request.Id)
	self.requestBufferLock.Unlock()
	return err
}
Beispiel #18
0
func (self *ShardData) WriteLocalOnly(request *protocol.Request) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	self.store.BufferWrite(request)
	return nil
}
Beispiel #19
0
func (self *CoordinatorImpl) proxyWrite(clusterServer *ClusterServer, request *protocol.Request) error {
	originatingServerId := request.OriginatingServerId
	request.OriginatingServerId = nil
	defer func() { request.OriginatingServerId = originatingServerId }()

	responseChan := make(chan *protocol.Response, 1)
	err := clusterServer.MakeRequest(request, responseChan)
	if err != nil {
		log.Warn("PROXY WRITE ERROR: ", err)
		return err
	}
	response := <-responseChan
	if *response.Type == protocol.Response_WRITE_OK {
		return nil
	} else {
		return errors.New(response.GetErrorMessage())
	}
}
Beispiel #20
0
func (self *WriteBuffer) write(request *protocol.Request) {
	attempts := 0
	for {
		self.shardIds[*request.ShardId] = true
		requestNumber := *request.RequestNumber
		err := self.writer.Write(request)
		if err == nil {
			self.shardCommitedRequestNumber[request.GetShardId()] = request.GetRequestNumber()
			self.wal.Commit(requestNumber, self.serverId)
			return
		}
		if attempts%100 == 0 {
			log.Error("%s: WriteBuffer: error on write to server %d: %s", self.writerInfo, self.serverId, err)
		}
		attempts += 1
		// backoff happens in the writer, just sleep for a small fixed amount of time before retrying
		time.Sleep(time.Millisecond * 100)
	}
}
Beispiel #21
0
func (self *ShardData) LogAndHandleDestructiveQuery(querySpec *parser.QuerySpec, request *protocol.Request, response chan *protocol.Response, runLocalOnly bool) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	var localResponses chan *protocol.Response
	if self.localShard != nil {
		localResponses = make(chan *protocol.Response, 1)

		// this doesn't really apply at this point since destructive queries don't output anything, but it may later
		maxPointsFromDestructiveQuery := 1000
		processor := engine.NewPassthroughEngine(localResponses, maxPointsFromDestructiveQuery)
		err := self.localShard.Query(querySpec, processor)
		processor.Close()
		if err != nil {
			return err
		}
	}
	if !runLocalOnly {
		responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
		for i, server := range self.clusterServers {
			responseChan := make(chan *protocol.Response, 1)
			responses[i] = responseChan
			// do this so that a new id will get assigned
			request.Id = nil
			server.MakeRequest(request, responseChan)
		}
		for i, responseChan := range responses {
			for {
				res := <-responseChan
				if *res.Type == endStreamResponse {
					self.wal.Commit(requestNumber, self.clusterServers[i].Id)
					break
				}
				response <- res
			}
		}
	}

	if localResponses != nil {
		for {
			res := <-localResponses
			if *res.Type == endStreamResponse {
				self.wal.Commit(requestNumber, self.localServerId)
				break
			}
			response <- res
		}
	}

	response <- &protocol.Response{Type: &endStreamResponse}
	return nil
}
func (self *ProtobufRequestHandler) handleSequenceNumberRequest(request *protocol.Request, conn net.Conn) {
	replicationFactor := uint8(*request.ReplicationFactor)
	var err error
	lastKnownSequenceNumber, err := self.coordinator.GetLastSequenceNumber(replicationFactor, *request.OriginatingServerId, *request.OwnerServerId)
	var response *protocol.Response
	if err != nil {
		response = &protocol.Response{Type: &sequenceNumberResponse, Request: request, RequestId: request.Id, ErrorCode: &internalError}
	} else {
		response = &protocol.Response{Type: &sequenceNumberResponse, Request: request, RequestId: request.Id}
		request.LastKnownSequenceNumber = &lastKnownSequenceNumber
	}
	self.WriteResponse(conn, response)
}
Beispiel #23
0
// This method never blocks. It'll buffer writes until they fill the buffer then drop the on the
// floor and let the background goroutine replay from the WAL
func (self *WriteBuffer) Write(request *protocol.Request) {
	self.shardLastRequestNumber[request.GetShardId()] = request.GetRequestNumber()
	select {
	case self.writes <- request:
		log.Debug("Buffering %d:%d for %s", request.GetRequestNumber(), request.GetShardId(), self.writerInfo)
		return
	default:
		select {
		case self.stoppedWrites <- *request.RequestNumber:
			return
		default:
			return
		}
	}
}
Beispiel #24
0
func (self *ShardData) SyncWrite(request *p.Request) error {
	request.ShardId = &self.id
	for _, server := range self.clusterServers {
		if err := server.Write(request); err != nil {
			return err
		}
	}

	if self.store == nil {
		return nil
	}

	return self.store.Write(request)
}
Beispiel #25
0
func (self *ShardData) HandleDestructiveQuery(querySpec *parser.QuerySpec, request *p.Request, response chan *p.Response, runLocalOnly bool) {
	if !self.IsLocal && runLocalOnly {
		panic("WTF islocal is false and runLocalOnly is true")
	}

	responseChannels := []<-chan *p.Response{}
	serverIds := []uint32{}

	if self.IsLocal {
		channel, err := self.deleteDataLocally(querySpec)
		if err != nil {
			msg := err.Error()
			response <- &p.Response{Type: &endStreamResponse, ErrorMessage: &msg}
			log.Error(msg)
			return
		}
		responseChannels = append(responseChannels, channel)
		serverIds = append(serverIds, self.localServerId)
	}

	log.Debug("request %s, runLocalOnly: %v", request.GetDescription(), runLocalOnly)
	if !runLocalOnly {
		responses, ids, _ := self.forwardRequest(request)
		serverIds = append(serverIds, ids...)
		responseChannels = append(responseChannels, responses...)
	}

	accessDenied := false
	for idx, channel := range responseChannels {
		serverId := serverIds[idx]
		log.Debug("Waiting for response to %s from %d", request.GetDescription(), serverId)
		for {
			res := <-channel
			log.Debug("Received %s response from %d for %s", res.GetType(), serverId, request.GetDescription())
			if *res.Type == endStreamResponse {
				break
			}

			// don't send the access denied response until the end so the readers don't close out before the other responses.
			// See https://github.com/influxdb/influxdb/issues/316 for more info.
			if *res.Type != accessDeniedResponse {
				response <- res
			} else {
				accessDenied = true
			}
		}
	}

	if accessDenied {
		response <- &p.Response{Type: &accessDeniedResponse}
	}
	response <- &p.Response{Type: &endStreamResponse}
}
Beispiel #26
0
func (self *ShardData) SyncWrite(request *p.Request, assignSeqNum bool) error {
	if assignSeqNum {
		self.wal.AssignSequenceNumbers(request)
	}

	request.ShardId = &self.id
	for _, server := range self.clusterServers {
		if err := server.Write(request); err != nil {
			return err
		}
	}

	if self.store == nil {
		return nil
	}

	return self.store.Write(request)
}
func (self *DatastoreMock) LogRequestAndAssignSequenceNumber(request *protocol.Request, replicationFactor *uint8, ownerServerId *uint32) error {
	id := uint64(1)
	request.SequenceNumber = &id
	return nil
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
	if *request.Type == protocol.Request_PROXY_WRITE {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		location := common.RingLocation(request.Database, request.Series.Name, request.Series.Points[0].Timestamp)
		ownerId := self.clusterConfig.GetOwnerIdByLocation(&location)
		request.OriginatingServerId = &self.clusterConfig.localServerId
		// TODO: make request logging and datastore write atomic
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, ownerId)
		if err != nil {
			return err
		}
		err = self.db.WriteSeriesData(*request.Database, request.Series)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		// TODO: add quorum writes?
		self.coordinator.ReplicateWrite(request)
		return err
	} else if *request.Type == protocol.Request_PROXY_DELETE {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		request.OriginatingServerId = &self.clusterConfig.localServerId
		// TODO: make request logging and datastore write atomic
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			return err
		}
		query, _ := parser.ParseQuery(*request.Query)
		err = self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		// TODO: add quorum writes?
		self.coordinator.ReplicateDelete(request)
		return err
	} else if *request.Type == protocol.Request_REPLICATION_WRITE {
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		self.db.WriteSeriesData(*request.Database, request.Series)
		return nil
	} else if *request.Type == protocol.Request_REPLICATION_DELETE {
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		query, _ := parser.ParseQuery(*request.Query)
		return self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
	} else if *request.Type == protocol.Request_QUERY {
		go self.handleQuery(request, conn)
	} else if *request.Type == protocol.Request_REPLICATION_REPLAY {
		self.handleReplay(request, conn)
	} else {
		log.Error("unknown request type: %v", request)
		return errors.New("Unknown request type")
	}
	return nil
}
Beispiel #29
0
func (self *WriteBuffer) replayAndRecover(missedRequest uint32) {
	var req *protocol.Request
	for {
		log.Info("%s: REPLAY: Replaying dropped requests...", self.writerInfo)
		// empty out the buffer before the replay so new writes can buffer while we're replaying
		channelLen := len(self.writes)

		// if req is nil, this is the first run through the replay. Start from the start of the write queue
		if req == nil {
			for i := 0; i < channelLen; i++ {
				r := <-self.writes
				if req == nil {
					req = r
				}
			}
		}
		if req == nil {
			log.Error("%s: REPLAY: emptied channel, but no request set", self.writerInfo)
			return
		}
		log.Debug("%s: REPLAY: Emptied out channel", self.writerInfo)
		shardIds := make([]uint32, 0)
		for shardId, _ := range self.shardIds {
			shardIds = append(shardIds, shardId)
		}

		log.Debug("%s: REPLAY: from request %d. Shards: ", self.writerInfo, req.GetRequestNumber(), shardIds)
		self.wal.RecoverServerFromRequestNumber(*req.RequestNumber, shardIds, func(request *protocol.Request, shardId uint32) error {
			log.Debug("%s: REPLAY: writing request number: %d", self.writerInfo, request.GetRequestNumber())
			req = request
			request.ShardId = &shardId
			self.write(request)
			return nil
		})

		log.Info("%s: REPLAY: Emptying out reqeusts from buffer that we've already replayed", self.writerInfo)
	RequestLoop:
		for {
			select {
			case newReq := <-self.writes:
				if *newReq.RequestNumber == *req.RequestNumber {
					break RequestLoop
				}
			default:
				log.Error("%s: REPLAY: Got to the end of the write buffer without getting to the last written request.", self.writerInfo)
				break RequestLoop
			}
		}

		log.Info("%s: REPLAY: done.", self.writerInfo)

		// now make sure that no new writes were dropped. If so, do the replay again from this place.
		select {
		case <-self.stoppedWrites:
			log.Info("%s: REPLAY: Buffer backed up while replaying, going again.", self.writerInfo)
			continue
		default:
			return
		}
	}
}
func (self *ProtobufRequestHandler) HandleRequest(request *protocol.Request, conn net.Conn) error {
	if *request.Type == protocol.Request_PROXY_WRITE {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		request.OriginatingServerId = &self.clusterConfig.localServerId
		// TODO: make request logging and datastore write atomic
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			return err
		}
		err = self.db.WriteSeriesData(*request.Database, request.Series)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		// TODO: add quorum writes?
		self.coordinator.ReplicateWrite(request)
		return err
	} else if *request.Type == protocol.Request_PROXY_DROP_SERIES {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		request.OriginatingServerId = &self.clusterConfig.localServerId
		replicationFactor := uint8(*request.ReplicationFactor)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			return err
		}
		err = self.db.DropSeries(*request.Database, *request.Series.Name)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		self.coordinator.ReplicateWrite(request)
		return err
	} else if *request.Type == protocol.Request_REPLICATION_DROP_SERIES {
		replicationFactor := uint8(*request.ReplicationFactor)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		return self.db.DropSeries(*request.Database, *request.Series.Name)
	} else if *request.Type == protocol.Request_PROXY_DROP_DATABASE {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		request.OriginatingServerId = &self.clusterConfig.localServerId
		replicationFactor := uint8(*request.ReplicationFactor)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			return err
		}
		err = self.db.DropDatabase(*request.Database)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		self.coordinator.ReplicateWrite(request)
		return err
	} else if *request.Type == protocol.Request_REPLICATION_DROP_DATABASE {
		replicationFactor := uint8(*request.ReplicationFactor)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		return self.db.DropDatabase(*request.Database)
	} else if *request.Type == protocol.Request_PROXY_DELETE {
		response := &protocol.Response{RequestId: request.Id, Type: &self.writeOk}

		request.OriginatingServerId = &self.clusterConfig.localServerId
		// TODO: make request logging and datastore write atomic
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			return err
		}
		query, err := parser.ParseQuery(*request.Query)
		if err != nil {
			return err
		}
		err = self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
		if err != nil {
			return err
		}
		err = self.WriteResponse(conn, response)
		// TODO: add quorum writes?
		self.coordinator.ReplicateDelete(request)
		return err
	} else if *request.Type == protocol.Request_REPLICATION_WRITE {
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				log.Warn("Missing sequence number error: Request SN: %v Last Known SN: %v", request.GetSequenceNumber(), err.LastKnownRequestSequence)
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		self.db.WriteSeriesData(*request.Database, request.Series)
		return nil
	} else if *request.Type == protocol.Request_REPLICATION_DELETE {
		replicationFactor := self.clusterConfig.GetReplicationFactor(request.Database)
		// TODO: make request logging and datastore write atomic
		err := self.db.LogRequestAndAssignSequenceNumber(request, &replicationFactor, request.OwnerServerId)
		if err != nil {
			switch err := err.(type) {
			case datastore.SequenceMissingRequestsError:
				go self.coordinator.ReplayReplication(request, &replicationFactor, request.OwnerServerId, &err.LastKnownRequestSequence)
				return nil
			default:
				return err
			}
		}
		query, err := parser.ParseQuery(*request.Query)
		if err != nil {
			return err
		}
		return self.db.DeleteSeriesData(*request.Database, query[0].DeleteQuery)
	} else if *request.Type == protocol.Request_QUERY {
		go self.handleQuery(request, conn)
	} else if *request.Type == protocol.Request_LIST_SERIES {
		go self.handleListSeries(request, conn)
	} else if *request.Type == protocol.Request_REPLICATION_REPLAY {
		self.handleReplay(request, conn)
	} else if *request.Type == protocol.Request_SEQUENCE_NUMBER {
		self.handleSequenceNumberRequest(request, conn)
	} else {
		log.Error("unknown request type: %v", request)
		return errors.New("Unknown request type")
	}
	return nil
}