Ejemplo n.º 1
0
// NewEventSubSocketHandler returns a function that handles event notifications
func NewEventSubSocketHandler(eventSubSocket *zmq4.Socket) func(zmq4.State) error {
	var nodeIDMap map[string]uint32
	var err error

	if nodeIDMap, err = centraldb.GetNodeIDMap(); err != nil {
		fog.Critical("centraldb.GetNodeIDMap() failed %s", err)
	}

	return func(_ zmq4.State) error {
		var err error
		var ok bool
		var webWriterStart msg.WebWriterStart
		var timestamp time.Time
		var sourceNodeID uint32

		marshalledMessage, err := eventSubSocket.RecvMessage(0)
		if err != nil {
			return fmt.Errorf("RecvMessage %s", err)
		}

		// the 0th part should be the topic, we skip that

		err = json.Unmarshal([]byte(marshalledMessage[1]), &webWriterStart)
		if err != nil {
			return fmt.Errorf("Unmarshal %s", err)
		}

		if webWriterStart.MessageType != "web-writer-start" {
			return fmt.Errorf("unknown message type '%s'",
				webWriterStart.MessageType)
		}

		timestamp, err = tools.ParseTimestampRepr(webWriterStart.TimestampRepr)
		if err != nil {
			return fmt.Errorf("unable to parse %s %s",
				webWriterStart.TimestampRepr, err)
		}

		sourceNodeID, ok = nodeIDMap[webWriterStart.SourceNodeName]
		if !ok {
			return fmt.Errorf("unknown source_node_name %s",
				webWriterStart.SourceNodeName)
		}

		fog.Debug("cancel-segments-from-node %s", webWriterStart.SourceNodeName)

		// cancel all all segment rows
		//    * from a specifiic source node
		//    * are in active status
		//    * with a timestamp earlier than the specified time.
		// This is triggered by a web server restart
		stmt := nodedb.Stmts["cancel-segments-from-node"]
		if _, err = stmt.Exec(sourceNodeID, timestamp); err != nil {
			return fmt.Errorf("cancel-segments-from-node %s", err)
		}

		return nil
	}
}
Ejemplo n.º 2
0
// FinishConjoinedArchive completes a conjoined archive
func handleFinishConjoinedArchive(state *writerState, request requestFinishConjoinedArchive) {
	conjoined := request.Conjoined
	var err error
	var ok bool
	var handoffNodeID uint32
	var timestamp time.Time

	fog.Debug("%s FinishConjoinedArchive %s", conjoined.UserRequestID, conjoined)

	if timestamp, err = tools.ParseTimestampRepr(conjoined.TimestampRepr); err != nil {
		request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
		return
	}

	if conjoined.HandoffNodeName != "" {
		if handoffNodeID, ok = state.NodeIDMap[conjoined.HandoffNodeName]; !ok {
			request.resultChan <- fmt.Errorf("unknown handoff node %s", conjoined.HandoffNodeName)
		}
		stmt := nodedb.Stmts["finish-conjoined-for-handoff"]
		_, err = stmt.Exec(
			timestamp,
			conjoined.CollectionID,
			conjoined.Key,
			conjoined.UnifiedID,
			handoffNodeID)

		if err != nil {
			request.resultChan <- fmt.Errorf("finish-conjoined-for-handoff %s", err)
			return
		}
	} else {

		stmt := nodedb.Stmts["finish-conjoined"]
		_, err = stmt.Exec(
			timestamp,
			conjoined.CollectionID,
			conjoined.Key,
			conjoined.UnifiedID)

		if err != nil {
			request.resultChan <- fmt.Errorf("finish-conjoined %s", err)
			return
		}

	}

	request.resultChan <- nil
}
Ejemplo n.º 3
0
// DestroyKey makes a key inaccessible
func handleDestroyKey(state *writerState, request requestDestroyKey) {
	destroyKey := request.DestroyKey
	var err error
	var ok bool
	var timestamp time.Time
	var sourceNodeID uint32
	var handoffNodeID uint32

	fog.Debug("DestroyKey (%d)", destroyKey.UnifiedIDToDestroy)

	if sourceNodeID, ok = state.NodeIDMap[destroyKey.SourceNodeName]; !ok {
		request.resultChan <- fmt.Errorf("unknown source node %s", destroyKey.SourceNodeName)
		return
	}

	if timestamp, err = tools.ParseTimestampRepr(destroyKey.TimestampRepr); err != nil {
		request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
		return
	}

	if destroyKey.UnifiedIDToDestroy > 0 {
		if destroyKey.HandoffNodeName != "" {
			if handoffNodeID, ok = state.NodeIDMap[destroyKey.HandoffNodeName]; !ok {
				request.resultChan <- fmt.Errorf("unknown handoff node %s", destroyKey.HandoffNodeName)
				return
			}
			stmt := nodedb.Stmts["new-tombstone-for-unified-id-for-handoff"]
			_, err = stmt.Exec(
				destroyKey.CollectionID,
				destroyKey.Key,
				destroyKey.UnifiedID,
				timestamp,
				destroyKey.SegmentNum,
				destroyKey.UnifiedIDToDestroy,
				sourceNodeID,
				handoffNodeID)

			if err != nil {
				request.resultChan <- fmt.Errorf("new-tombstone-for-unified-id-for-handoff %d %s",
					destroyKey.UnifiedIDToDestroy, err)
				return
			}
		} else {
			stmt := nodedb.Stmts["new-tombstone-for-unified-id"]
			_, err = stmt.Exec(
				destroyKey.CollectionID,
				destroyKey.Key,
				destroyKey.UnifiedID,
				timestamp,
				destroyKey.SegmentNum,
				destroyKey.UnifiedIDToDestroy,
				sourceNodeID)

			if err != nil {
				request.resultChan <- fmt.Errorf("new-tombstone-for-unified-id %d %s",
					destroyKey.UnifiedIDToDestroy, err)
				return
			}
		}

		stmt := nodedb.Stmts["delete-conjoined-for-unified-id"]
		_, err = stmt.Exec(
			timestamp,
			destroyKey.CollectionID,
			destroyKey.Key,
			destroyKey.UnifiedIDToDestroy)

		if err != nil {
			request.resultChan <- fmt.Errorf("delete-conjoined-for-unified-id %d %s",
				destroyKey.UnifiedIDToDestroy, err)
			return
		}
	} else {
		if destroyKey.HandoffNodeName != "" {
			if handoffNodeID, ok = state.NodeIDMap[destroyKey.HandoffNodeName]; !ok {
				request.resultChan <- fmt.Errorf("unknown handoff node %s", destroyKey.HandoffNodeName)
				return
			}
			stmt := nodedb.Stmts["new-tombstone-for-handoff"]
			_, err = stmt.Exec(
				destroyKey.CollectionID,
				destroyKey.Key,
				destroyKey.UnifiedID,
				timestamp,
				destroyKey.SegmentNum,
				sourceNodeID,
				handoffNodeID)

			if err != nil {
				request.resultChan <- fmt.Errorf("new-tombstone-for-handoff %s", err)
				return
			}
		} else {
			stmt := nodedb.Stmts["new-tombstone"]
			_, err = stmt.Exec(
				destroyKey.CollectionID,
				destroyKey.Key,
				destroyKey.UnifiedID,
				timestamp,
				destroyKey.SegmentNum,
				sourceNodeID)

			if err != nil {
				request.resultChan <- fmt.Errorf("new-tombstone %s", err)
				return
			}
		}

		stmt := nodedb.Stmts["delete-conjoined"]
		_, err = stmt.Exec(
			timestamp,
			destroyKey.CollectionID,
			destroyKey.Key,
			destroyKey.UnifiedID)

		if err != nil {
			request.resultChan <- fmt.Errorf("delete-conjoined %s", err)
			return
		}
	}

	request.resultChan <- nil
}
Ejemplo n.º 4
0
// FinishSegment finishes storing the segment
func handleFinishSegment(state *writerState, request requestFinishSegment) {
	userRequestID := request.UserRequestID
	segment := request.Segment
	file := request.File
	metaData := request.MetaData
	var err error
	var md5Digest []byte
	var timestamp time.Time

	fog.Debug("%s FinishSegment", userRequestID)

	key := segmentKey{segment.UnifiedID, segment.ConjoinedPart,
		segment.SegmentNum}
	entry, ok := state.SegmentMap[key]
	if !ok {
		request.resultChan <- fmt.Errorf("FinishSegment unknown segment %s", key)
		return
	}

	delete(state.SegmentMap, key)

	md5Digest, err = base64.StdEncoding.DecodeString(file.EncodedFileMD5Digest)
	if err != nil {
		request.resultChan <- err
		return
	}

	if timestamp, err = tools.ParseTimestampRepr(segment.TimestampRepr); err != nil {
		request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
		return
	}

	stmt := nodedb.Stmts["finish-segment"]
	_, err = stmt.Exec(
		file.FileSize,
		file.FileAdler32,
		md5Digest,
		entry.SegmentID)

	if err != nil {
		request.resultChan <- fmt.Errorf("finish-segment %s", err)
		return
	}

	for _, metaEntry := range metaData {
		stmt := nodedb.Stmts["new-meta-data"]
		_, err = stmt.Exec(
			segment.CollectionID,
			entry.SegmentID,
			metaEntry.Key,
			metaEntry.Value,
			timestamp)

		if err != nil {
			request.resultChan <- fmt.Errorf("new-meta-data %s", err)
			return
		}
	}

	request.resultChan <- nil
}
Ejemplo n.º 5
0
func handleStartSegment(state *writerState, request requestStartSegment) {
	userRequestID := request.UserRequestID
	segment := request.Segment
	nodeNames := request.NodeNames

	var entry segmentMapEntry
	var err error
	var sourceNodeID uint32
	var handoffNodeID uint32
	var ok bool
	var timestamp time.Time

	fog.Debug("%s StartSegment", userRequestID)

	if sourceNodeID, ok = state.NodeIDMap[nodeNames.SourceNodeName]; !ok {
		request.resultChan <- fmt.Errorf("unknown source node %s", nodeNames.SourceNodeName)
		return
	}

	if timestamp, err = tools.ParseTimestampRepr(segment.TimestampRepr); err != nil {
		request.resultChan <- fmt.Errorf("unable to parse timestamp %s", err)
		return
	}

	if nodeNames.HandoffNodeName != "" {
		if handoffNodeID, ok = state.NodeIDMap[nodeNames.HandoffNodeName]; !ok {
			request.resultChan <- fmt.Errorf("unknown handoff node %s", nodeNames.HandoffNodeName)
			return
		}

		stmt := nodedb.Stmts["new-segment-for-handoff"]
		row := stmt.QueryRow(
			segment.CollectionID,
			segment.Key,
			segment.UnifiedID,
			timestamp,
			segment.SegmentNum,
			segment.ConjoinedPart,
			sourceNodeID,
			handoffNodeID)
		if err = row.Scan(&entry.SegmentID); err != nil {
			request.resultChan <- err
			return
		}
	} else {
		stmt := nodedb.Stmts["new-segment"]
		row := stmt.QueryRow(
			segment.CollectionID,
			segment.Key,
			segment.UnifiedID,
			timestamp,
			segment.SegmentNum,
			segment.ConjoinedPart,
			sourceNodeID)
		if err = row.Scan(&entry.SegmentID); err != nil {
			request.resultChan <- err
			return
		}
	}
	entry.LastActionTime = tools.Timestamp()

	key := segmentKey{segment.UnifiedID, segment.ConjoinedPart,
		segment.SegmentNum}

	state.SegmentMap[key] = entry

	request.resultChan <- nil
}