func (self *CoordinatorSuite) TestShouldQuerySequentially(c *C) {
end := time.Now().Truncate(24 * time.Hour)
start := end.Add(-7 * 24 * time.Hour)
shard := cluster.NewShard(1, start, end, "", "", nil)
shards := []*cluster.ShardData{shard}
coordinator := NewCoordinatorImpl(&configuration.Configuration{
ClusterMaxResponseBufferSize: 1000,
}, nil, nil, nil)
queries := map[string]bool{
"list series": false,
"select count(foo) from /.*bar.*/ group by time(1d)": true,
"select count(foo) from bar": true,
"select foo from bar": true,
"select count(foo) from bar group by baz": true,
"select count(foo) from bar group by time(1d)": false,
"select count(foo) from bar group by time(3d)": true,
}
for query, result := range queries {
fmt.Printf("Testing %s\n", query)
parsedQuery, err := parser.ParseQuery(query)
c.Assert(err, IsNil)
c.Assert(parsedQuery, HasLen, 1)
querySpec := parser.NewQuerySpec(nil, "", parsedQuery[0])
c.Assert(coordinator.shouldQuerySequentially(shards, querySpec), Equals, result)
}
}
func (self *ProtobufRequestHandler) handleQuery(request *protocol.Request, conn net.Conn) {
// the query should always parse correctly since it was parsed at the originating server.
queries, err := parser.ParseQuery(*request.Query)
if err != nil || len(queries) < 1 {
log.Error("Error parsing query: ", err)
errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
response := &protocol.Response{
Type: protocol.Response_ERROR.Enum(),
ErrorMessage: &errorMsg,
RequestId: request.Id,
}
self.WriteResponse(conn, response)
return
}
query := queries[0]
var user common.User
if *request.IsDbUser {
user = self.clusterConfig.GetDbUser(*request.Database, *request.UserName)
} else {
user = self.clusterConfig.GetClusterAdmin(*request.UserName)
}
if user == nil {
errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
response := &protocol.Response{
Type: protocol.Response_ERROR.Enum(),
ErrorMessage: &errorMsg,
RequestId: request.Id,
}
self.WriteResponse(conn, response)
return
}
shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
querySpec := parser.NewQuerySpec(user, *request.Database, query)
responseChan := make(chan *protocol.Response)
if querySpec.IsDestructiveQuery() {
go shard.HandleDestructiveQuery(querySpec, request, responseChan, true)
} else {
go shard.Query(querySpec, responseChan)
}
for {
response := <-responseChan
response.RequestId = request.Id
self.WriteResponse(conn, response)
switch rt := response.GetType(); rt {
case protocol.Response_END_STREAM,
protocol.Response_ERROR:
return
case protocol.Response_QUERY:
continue
default:
panic(fmt.Errorf("Unexpected response type: %s", rt))
}
}
}
func (self *HttpServer) configureDatabase(w libhttp.ResponseWriter, r *libhttp.Request) {
self.tryAsClusterAdmin(w, r, func(u User) (int, interface{}) {
databaseConfig := &DatabaseConfig{}
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(databaseConfig)
if err != nil {
return libhttp.StatusInternalServerError, err.Error()
}
database := r.URL.Query().Get(":db")
// validate before creating anything
for _, queryString := range databaseConfig.ContinuousQueries {
q, err := parser.ParseQuery(queryString)
if err != nil {
return libhttp.StatusBadRequest, err.Error()
}
for _, query := range q {
if !query.SelectQuery.IsContinuousQuery() {
return libhttp.StatusBadRequest, fmt.Errorf("This query isn't a continuous query. Use 'into'. %s", query.QueryString)
}
}
}
err = self.coordinator.CreateDatabase(u, database)
if err != nil {
return libhttp.StatusBadRequest, err.Error()
}
for _, space := range databaseConfig.Spaces {
space.Database = database
err = self.raftServer.CreateShardSpace(space)
if err != nil {
return libhttp.StatusInternalServerError, err.Error()
}
}
for _, queryString := range databaseConfig.ContinuousQueries {
q, _ := parser.ParseQuery(queryString)
for _, query := range q {
err := self.coordinator.CreateContinuousQuery(u, database, query.QueryString)
if err != nil {
return libhttp.StatusInternalServerError, err.Error()
}
}
}
return libhttp.StatusCreated, nil
})
}
func (dm *DataMigrator) migrateDatabaseInShard(database string, shard *LevelDbShard) error {
log.Info("Migrating database %s for shard", database)
seriesNames := shard.GetSeriesForDatabase(database)
log.Info("Migrating %d series", len(seriesNames))
admin := dm.clusterConfig.GetClusterAdmin(dm.clusterConfig.GetClusterAdmins()[0])
pointCount := 0
for _, series := range seriesNames {
q, err := parser.ParseQuery(fmt.Sprintf("select * from \"%s\"", series))
if err != nil {
log.Error("Problem migrating series %s", series)
continue
}
query := q[0]
seriesChan := make(chan *protocol.Response)
queryEngine := engine.NewPassthroughEngine(seriesChan, 2000)
querySpec := parser.NewQuerySpec(admin, database, query)
go func() {
err := shard.Query(querySpec, queryEngine)
if err != nil {
log.Error("Error migrating %s", err.Error())
}
queryEngine.Close()
seriesChan <- &protocol.Response{Type: &endStreamResponse}
}()
for {
response := <-seriesChan
if *response.Type == endStreamResponse {
break
}
err := dm.coord.WriteSeriesData(admin, database, []*protocol.Series{response.Series})
if err != nil {
log.Error("Writing Series data: %s", err.Error())
}
pointCount += len(response.Series.Points)
if pointCount > POINT_COUNT_TO_PAUSE {
pointCount = 0
time.Sleep(dm.pauseTime)
}
}
}
log.Info("Done migrating %s for shard", database)
return nil
}
func (self *ProtobufRequestHandler) handleQuery(request *protocol.Request, conn net.Conn) {
// the query should always parse correctly since it was parsed at the originating server.
queries, err := parser.ParseQuery(*request.Query)
if err != nil || len(queries) < 1 {
log.Error("Error parsing query: ", err)
errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
response := &protocol.Response{Type: &endStreamResponse, ErrorMessage: &errorMsg, RequestId: request.Id}
self.WriteResponse(conn, response)
return
}
query := queries[0]
var user common.User
if *request.IsDbUser {
user = self.clusterConfig.GetDbUser(*request.Database, *request.UserName)
} else {
user = self.clusterConfig.GetClusterAdmin(*request.UserName)
}
if user == nil {
errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
response := &protocol.Response{Type: &accessDeniedResponse, ErrorMessage: &errorMsg, RequestId: request.Id}
self.WriteResponse(conn, response)
return
}
shard := self.clusterConfig.GetLocalShardById(*request.ShardId)
querySpec := parser.NewQuerySpec(user, *request.Database, query)
responseChan := make(chan *protocol.Response)
if querySpec.IsDestructiveQuery() {
go shard.HandleDestructiveQuery(querySpec, request, responseChan, true)
} else {
go shard.Query(querySpec, responseChan)
}
for {
response := <-responseChan
response.RequestId = request.Id
self.WriteResponse(conn, response)
if response.GetType() == protocol.Response_END_STREAM || response.GetType() == protocol.Response_ACCESS_DENIED {
return
}
}
}
func (self *Coordinator) RunQuery(user common.User, database string, queryString string, p engine.Processor) (err error) {
log.Info("Start Query: db: %s, u: %s, q: %s", database, user.GetName(), queryString)
defer func(t time.Time) {
log.Debug("End Query: db: %s, u: %s, q: %s, t: %s", database, user.GetName(), queryString, time.Now().Sub(t))
}(time.Now())
// don't let a panic pass beyond RunQuery
defer common.RecoverFunc(database, queryString, nil)
q, err := parser.ParseQuery(queryString)
if err != nil {
return err
}
for _, query := range q {
err := self.runSingleQuery(user, database, query, p)
if err != nil {
return err
}
}
return nil
}
func (self *CoordinatorImpl) RunQuery(user common.User, database string, queryString string, seriesWriter SeriesWriter) (err error) {
log.Info("Start Query: db: %s, u: %s, q: %s", database, user.GetName(), queryString)
defer func(t time.Time) {
log.Debug("End Query: db: %s, u: %s, q: %s, t: %s", database, user.GetName(), queryString, time.Now().Sub(t))
}(time.Now())
// don't let a panic pass beyond RunQuery
defer common.RecoverFunc(database, queryString, nil)
q, err := parser.ParseQuery(queryString)
if err != nil {
return err
}
for _, query := range q {
querySpec := parser.NewQuerySpec(user, database, query)
if query.DeleteQuery != nil {
if err := self.clusterConfiguration.CreateCheckpoint(); err != nil {
return err
}
if err := self.runDeleteQuery(querySpec, seriesWriter); err != nil {
return err
}
continue
}
if query.DropQuery != nil {
if err := self.DeleteContinuousQuery(user, database, uint32(query.DropQuery.Id)); err != nil {
return err
}
continue
}
if query.IsListQuery() {
if query.IsListSeriesQuery() {
self.runListSeriesQuery(querySpec, seriesWriter)
} else if query.IsListContinuousQueriesQuery() {
queries, err := self.ListContinuousQueries(user, database)
if err != nil {
return err
}
for _, q := range queries {
if err := seriesWriter.Write(q); err != nil {
return err
}
}
}
continue
}
if query.DropSeriesQuery != nil {
err := self.runDropSeriesQuery(querySpec, seriesWriter)
if err != nil {
return err
}
continue
}
selectQuery := query.SelectQuery
if selectQuery.IsContinuousQuery() {
return self.CreateContinuousQuery(user, database, queryString)
}
if err := self.checkPermission(user, querySpec); err != nil {
return err
}
return self.runQuery(querySpec, seriesWriter)
}
seriesWriter.Close()
return nil
}