Example #1
0
func UnmarshalPQMsgMetaData(sn uint64, buf []byte) *PQMsgMetaData {
	p := PQMsgMetaData{SerialNumber: sn}
	if err := p.Unmarshal(buf); err != nil {
		log.Error("Could not unmarshal message: %d", sn)
		return nil
	}
	return &p
}
Example #2
0
// SaveServiceConfig saves service config into database.
func SaveServiceConfig(serviceId string, conf apis.BinaryMarshaller) error {
	db := db.DatabaseInstance()
	data, _ := conf.Marshal()
	err := db.StoreData(cfgKey(serviceId), data)
	if err != nil {
		log.Error("Failed to save config: %s", err.Error())
		return mpqerr.ServerError("Can not save service data: " + serviceId)
	}
	return nil
}
Example #3
0
func LoadServiceConfig(serviceId string, cfg apis.BinaryMarshaller) error {
	db := db.DatabaseInstance()
	data := db.GetData(cfgKey(serviceId))
	if len(data) == 0 {
		return mpqerr.NotFoundRequest("No service settings found: " + serviceId)
	}

	if err := cfg.Unmarshal(data); err != nil {
		log.Error("Error in '%s' service settings: %s", serviceId, err.Error())
		return mpqerr.ServerError("Service settings error: " + serviceId)
	}
	return nil
}
Example #4
0
func DeleteServiceData(serviceId string) {
	desc := GetServiceDescription(serviceId)
	if desc == nil {
		log.Error("Attempt to delete unknown service id: %s", serviceId)
		return
	}
	desc.ToDelete = true
	SaveServiceDescription(desc)

	db := db.DatabaseInstance()
	db.DeleteDataWithPrefix(serviceId)
	db.DeleteData(cfgKey(serviceId))
	db.DeleteData(descKey(serviceId))
}
Example #5
0
// Close flushes data on disk and closes database.
func (ds *LevelDBStorage) Close() {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	if !ds.closed {
		log.Info("Flushing database cache")
		ds.FlushCache()
		ds.closed = true
		log.Info("Closing the database")
		ds.db.Close()
		log.Info("The database has been closed.")
	} else {
		log.Error("Attempt to close database more than once!")
	}
}
Example #6
0
func getDatabase() *LevelDBStorage {
	var err error
	if database == nil {
		database, err = NewLevelDBStorage(conf.CFG)
		if err != nil {
			log.Error("Cannot initialize FireMPQ database: %s", err)
			os.Exit(255)
		}
	}
	if database.closed {
		database = nil
		return getDatabase()
	}
	return database
}
Example #7
0
func (s *ServiceManager) loadService(desc *queue_info.ServiceDescription) (apis.ISvc, bool) {
	if desc.Disabled {
		log.Error("Service is disabled. Skipping: %s", desc.Name)
		return nil, false
	}
	if desc.ToDelete {
		log.Warning("Service should be deleted: %s", desc.Name)
		queue_info.DeleteServiceData(desc.Name)
		return nil, false
	}
	log.Debug("Loading service data for: %s", desc.Name)

	serviceLoader, ok := GetServiceLoader(desc.SType)
	if !ok {
		log.Error("Unknown service '%s' type: %s", desc.Name, desc.SType)
		return nil, false
	}
	svcInstance, err := serviceLoader(s, desc)
	if err != nil {
		log.Error("Service '%s' was not loaded because of: %s", desc.Name, err)
		return nil, false
	}
	return svcInstance, true
}
Example #8
0
// GetServiceDescriptions Loads all service descriptions prefixed with ServiceDescPrefix
func GetServiceDescriptions() ServiceDescriptionList {
	sdList := make(ServiceDescriptionList, 0, 16)
	db := db.DatabaseInstance()
	descIter := db.IterData(ServiceDescPrefix)
	defer descIter.Close()

	for ; descIter.Valid(); descIter.Next() {
		svcDesc, err := UnmarshalServiceDesc([]byte(descIter.GetValue()))
		if err != nil {
			log.Error("Coudn't read service '%s' description: %s", descIter.GetTrimKey(), err.Error())
			continue
		}
		sdList = append(sdList, svcDesc)
	}
	return sdList
}
Example #9
0
func getDatabase() apis.DataStorage {
	if database == nil {
		var err error
		database, err = ldb.NewLevelDBStorage(conf.CFG)
		if err != nil {
			log.Error("Cannot initialize FireMPQ database: %s", err)
			os.Exit(255)
		}
		return database
	}

	if database.IsClosed() {
		database = nil
		return getDatabase()
	}
	return database
}
Example #10
0
func main() {
	// Initialize logging to a default INFO level to be able to log config error.

	conf.ParseConfigParameters()
	if len(conf.CFG.Profiler) > 0 {
		go func() {
			if err := http.ListenAndServe(conf.CFG.Profiler, nil); err != nil {
				log.Error("Could not initialize profiler: %v", err)
			}
		}()
	}

	// Reinitialize log level according to the config data.
	log.InitLogging()

	srv, err := server.Server(server.SimpleServerType, conf.CFG.FMPQServerInterface)
	if err != nil {
		log.Critical("Error: %s", err.Error())
		return
	}
	srv.Start()
}
Example #11
0
func CheckAvailableQueues(
	svcMgr *qmgr.ServiceManager,
	attr *QueueAttributes,
	sqsQuery *urlutils.SQSQuery) sqs_response.SQSResponse {

	svc, ok := svcMgr.GetService(sqsQuery.QueueName)
	if ok {
		if svc.Info().Type != apis.ServiceTypePriorityQueue {
			return sqserr.QueueAlreadyExistsError("Queue already exists for a different type of service")
		}
		pq, _ := svc.(*pqueue.PQueue)
		pqConfig := pq.Config()
		if !ok {
			log.Error("Unexpected config type from the found service!")
			return sqserr.ServerSideError("Queue config data error")
		}
		if attr.VisibilityTimeout >= 0 && attr.VisibilityTimeout != pqConfig.PopLockTimeout {
			return sqserr.QueueAlreadyExistsError(errQueueExists + AttrVisibilityTimeout)
		}
		if attr.DelaySeconds >= 0 && attr.DelaySeconds != pqConfig.DeliveryDelay {
			return sqserr.QueueAlreadyExistsError(errQueueExists + AttrDelaySeconds)
		}
		if attr.MaximumMessageSize >= 0 && attr.MaximumMessageSize != pqConfig.MaxMsgSize {
			return sqserr.QueueAlreadyExistsError(errQueueExists + AttrMaximumMessageSize)
		}
		if attr.MessageRetentionPeriod >= 0 && attr.MessageRetentionPeriod != pqConfig.MsgTtl {
			return sqserr.QueueAlreadyExistsError(errQueueExists + AttrMessageRetentionPeriod)
		}
		if attr.ReceiveMessageWaitTimeSeconds >= 0 && attr.ReceiveMessageWaitTimeSeconds != pqConfig.PopWaitTimeout {
			return sqserr.QueueAlreadyExistsError(errQueueExists + AttrReceiveMessageWaitTimeSeconds)
		}
		return &CreateQueueResponse{
			QueueUrl:  sqsQuery.Host + "/queue/" + sqsQuery.QueueName,
			RequestId: "1111-2222-3333",
		}
	}
	return nil
}
Example #12
0
func (tm *TopicManager) ListTopics(offset int) ([]string, int) {
	tm.Lock()
	defer tm.Unlock()
	var res []string

	maxLen := offset + ReturnBatchSize
	ret_offset := maxLen

	if maxLen >= len(tm.TopicList) {
		ret_offset = 0
		maxLen = len(tm.TopicList)
	}

	for i := offset; i < maxLen; i++ {
		topicName := tm.TopicList[i]
		if t := tm.Topics[topicName]; t == nil {
			log.Error("Inconsistent data! No data for topic name: %s", topicName)
		} else {
			res = append(res, t.Arn)
		}
	}
	return res, ret_offset
}
Example #13
0
func MakeMessageResponse(iMsg apis.IResponseItem, opts *ReceiveMessageOptions,
	sqsQuery *urlutils.SQSQuery) *MessageResponse {
	msg, ok := iMsg.(*pqueue.MsgResponseItem)
	if !ok {
		return nil
	}
	sqsMsg := &sqsmsg.SQSMessagePayload{}
	payload := msg.Payload()
	if err := sqsMsg.Unmarshal([]byte(payload)); err != nil {
		// Recovering from error. Non SQS messages will be filled with bulk info.
		sqsMsg.Payload = string(msg.Payload())
		sqsMsg.SenderId = "unknown"
		sqsMsg.SentTimestamp = strconv.FormatInt(utils.Uts(), 10)
		sqsMsg.MD5OfMessageAttributes = fmt.Sprintf("%x", md5.Sum(nil))
		sqsMsg.MD5OfMessageBody = fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(sqsMsg.Payload)))
	}

	msgMeta := msg.GetMeta()

	output := &MessageResponse{
		MD5OfMessageAttributes: sqsMsg.MD5OfMessageAttributes,
		MD5OfMessageBody:       sqsMsg.MD5OfMessageBody,
		ReceiptHandle:          msg.Receipt(),
		MessageId:              msg.ID(),
		Body:                   sqsMsg.Payload,
	}

	if opts.AllSysAttributes {
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrSenderId, Value: sqsMsg.SenderId,
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
		})
		output.Attributes = append(output.Attributes, &SysAttribute{
			Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
		})
	} else {
		for _, k := range opts.Attributes {
			switch k {
			case AttrSenderId:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrSenderId, Value: sqsMsg.SenderId,
				})
			case AttrSentTimestamp:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrSentTimestamp, Value: sqsMsg.SentTimestamp,
				})
			case AttrApproximateReceiveCount:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrApproximateReceiveCount, Value: strconv.FormatInt(msgMeta.PopCount, 10),
				})
			case AttrApproximateFirstReceiveTimestamp:
				output.Attributes = append(output.Attributes, &SysAttribute{
					Name: AttrApproximateFirstReceiveTimestamp, Value: strconv.FormatInt(utils.Uts(), 10),
				})
			}
		}
	}

	if opts.AllMessageAttributes {
		for attrName, attrData := range sqsMsg.UserAttributes {
			output.MessageAttributes = append(
				output.MessageAttributes, MakeMessageAttr(attrName, attrData))
		}
	} else {
		for _, attrName := range opts.MessageAttributes {
			if v, ok := sqsMsg.UserAttributes[attrName]; ok {
				output.MessageAttributes = append(
					output.MessageAttributes, MakeMessageAttr(attrName, v))
			}
		}
	}

	if !ok {
		log.Error("Failed to cast response item")
		return nil
	}
	return output
}
Example #14
0
func PushAMessage(pq *pqueue.PQueue, senderId string, paramList []string) sqs_response.SQSResponse {
	out := &MessageParams{
		DelaySeconds: -1,
		MessageBody:  "",
	}
	attrs, err := urlutils.ParseNNotationAttr("MessageAttribute.", paramList, out.Parse, NewReqQueueAttr)
	if err != nil {
		return err
	}
	attrsLen := len(attrs)
	outAttrs := make(map[string]*sqsmsg.UserAttribute)

	for i := 1; i <= attrsLen; i++ {
		a, ok := attrs[i]
		if !ok {
			return sqserr.InvalidParameterValueError("The request must contain non-empty message attribute name.")
		}
		reqMsgAttr, _ := a.(*ReqMsgAttr)

		sqs_err := validation.ValidateMessageAttrName(reqMsgAttr.Name)
		if sqs_err != nil {
			return sqs_err
		}

		sqs_err = validation.ValidateMessageAttrName(reqMsgAttr.DataType)
		if sqs_err != nil {
			return sqs_err
		}

		if reqMsgAttr.BinaryValue != "" && reqMsgAttr.StringValue != "" {
			return sqserr.InvalidParameterValueError(
				"Message attribute name '%s' has multiple values.", reqMsgAttr.Name)
		}

		if _, ok := outAttrs[reqMsgAttr.Name]; ok {
			return sqserr.InvalidParameterValueError(
				"Message attribute name '%s' already exists.", reqMsgAttr.Name)
		}

		if strings.HasPrefix(reqMsgAttr.DataType, "Number") {
			if _, err := strconv.Atoi(reqMsgAttr.StringValue); err != nil {
				return sqserr.InvalidParameterValueError(
					"Could not cast message attribute '%s' value to number.", reqMsgAttr.Name)
			}
		}

		if reqMsgAttr.BinaryValue != "" {
			if reqMsgAttr.DataType != "Binary" {
				return sqserr.InvalidParameterValueError(
					"The message attribute '%s' with type 'Binary' must use field 'Binary'", reqMsgAttr.Name)
			}
			outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
				Type:  reqMsgAttr.DataType,
				Value: reqMsgAttr.BinaryValue,
			}
			continue
		}

		if reqMsgAttr.StringValue != "" {
			if reqMsgAttr.DataType != "String" && reqMsgAttr.DataType != "Number" {
				return sqserr.InvalidParameterValueError(
					"The message attribute '%s' with type 'String' must use field 'String'", reqMsgAttr.Name)
			}
			outAttrs[reqMsgAttr.Name] = &sqsmsg.UserAttribute{
				Type:  reqMsgAttr.DataType,
				Value: reqMsgAttr.StringValue,
			}
		}
	}

	msgId := IdGen.RandId()
	if out.DelaySeconds < 0 {
		out.DelaySeconds = pq.Config().DeliveryDelay
	} else if out.DelaySeconds > conf.CFG_PQ.MaxDeliveryDelay {
		return sqserr.InvalidParameterValueError(
			"Delay secods must be between 0 and %d", conf.CFG_PQ.MaxDeliveryDelay/1000)
	}
	bodyMd5str := fmt.Sprintf("%x", md5.Sum(enc.UnsafeStringToBytes(out.MessageBody)))
	attrMd5 := CalcAttrMd5(outAttrs)

	msgPayload := sqsmsg.SQSMessagePayload{
		UserAttributes:         outAttrs,
		MD5OfMessageBody:       bodyMd5str,
		MD5OfMessageAttributes: attrMd5,
		SenderId:               senderId,
		SentTimestamp:          strconv.FormatInt(utils.Uts(), 10),
		Payload:                out.MessageBody,
	}

	d, marshalErr := msgPayload.Marshal()
	if marshalErr != nil {
		log.Error("Failed to serialize message payload: %v", err)
	}
	payload := enc.UnsafeBytesToString(d)

	resp := pq.Push(msgId, payload, pq.Config().MsgTtl, out.DelaySeconds, 1)
	if resp.IsError() {
		e, _ := resp.(error)
		return sqserr.InvalidParameterValueError(e.Error())
	}

	return &SendMessageResponse{
		MessageId:              msgId,
		MD5OfMessageBody:       bodyMd5str,
		MD5OfMessageAttributes: attrMd5,
		RequestId:              "req",
	}
}