// GetIDsForServerList retrieves the server ID numbers for a given set of hosts,
// from the server database file, in response to a request to build the master
// server detail list or the list of server details in response to a request
// coming in over the API. It sends its results over a map channel consisting of
// a host to id mapping.
func (sdb *SDB) GetIDsForServerList(result chan map[string]int64,
hosts map[string]string) {
m := make(map[string]int64, len(hosts))
for host, game := range hosts {
rows, err := sdb.db.Query(
"SELECT server_id FROM servers WHERE host =? AND game =? LIMIT 1",
host, game)
if err != nil {
logger.LogAppErrorf(
"GetIDsForServerList: Error querying database to retrieve ID for host %s and game %s: %s",
host, game, err)
return
}
defer rows.Close()
var id int64
for rows.Next() {
if err := rows.Scan(&id); err != nil {
logger.LogAppErrorf(
"GetIDsForServerList: Error querying database to retrieve ID for host %s: %s",
host, err)
return
}
}
m[host] = id
}
result <- m
}
// GetIDsAPIQuery Retrieves the server ID numbers, hosts, and game name for a given
// set of hosts (represented by query string values) from the server database
// file in response to a query from the API. Sends the results over a DbServerID
// channel for consumption.
func (sdb *SDB) GetIDsAPIQuery(result chan *models.DbServerID, hosts []string) {
m := &models.DbServerID{}
for _, h := range hosts {
logger.WriteDebug("DB: GetIDsAPIQuery, host: %s", h)
rows, err := sdb.db.Query(
"SELECT server_id, host, game FROM servers WHERE host LIKE ?",
fmt.Sprintf("%%%s%%", h))
if err != nil {
logger.LogAppErrorf(
"GetIDsAPIQuery: Error querying database to retrieve ID for host %s: %s",
h, err)
return
}
defer rows.Close()
var id int64
host, game := "", ""
for rows.Next() {
sid := models.DbServer{}
if err := rows.Scan(&id, &host, &game); err != nil {
logger.LogAppErrorf(
"GetIDsAPIQuery: Error querying database to retrieve ID for host %s: %s",
h, err)
return
}
sid.ID = id
sid.Host = host
sid.Game = game
m.Servers = append(m.Servers, sid)
}
}
m.ServerCount = len(m.Servers)
result <- m
}
func useDumpFileAsMasterList(dumppath string) *models.APIServerList {
f, err := os.Open(dumppath)
if err != nil {
logger.LogAppErrorf("Unable to open test API server dump file: %s", err)
return nil
}
defer f.Close()
r := bufio.NewReader(f)
d := json.NewDecoder(r)
ml := &models.APIServerList{}
if err := d.Decode(ml); err != nil {
logger.LogAppErrorf("Unable to decode test API server dump as json: %s", err)
return nil
}
return ml
}
func extractHosts(hbs []byte) ([]string, int, error) {
var sl []string
pos, total := 0, 0
for i := 0; i < len(hbs); i++ {
if len(sl) > 0 && sl[len(sl)-1] == "0.0.0.0:0" {
logger.LogSteamInfo("0.0.0.0:0 detected. Got %d total hosts.", total-1)
break
}
if pos+6 > len(hbs) {
logger.LogSteamInfo("Got %d total hosts.", total)
break
}
host, err := parseIP(hbs[pos : pos+6])
if err != nil {
logger.LogAppErrorf("Error parsing host: %s", err)
} else {
sl = append(sl, host)
total++
}
// host:port = 6 bytes
pos = pos + 6
}
return sl, total, nil
}
func (sdb *SDB) getHostAndGame(id string) (host, game string, err error) {
rows, err := sdb.db.Query("SELECT host, game FROM servers WHERE server_id =? LIMIT 1",
id)
if err != nil {
return host, game,
logger.LogAppErrorf("getHostAndGame: Error querying database for id %s: %s",
id, err)
}
defer rows.Close()
for rows.Next() {
if err := rows.Scan(&host, &game); err != nil {
return host, game,
logger.LogAppErrorf("getHostAndGame: Error querying database for id %s: %s",
id, err)
}
}
return host, game, nil
}
// AddServersToDB inserts a specified host and port with its game name into the
// server database.
func (sdb *SDB) AddServersToDB(hostsgames map[string]string) {
toInsert := make(map[string]string, len(hostsgames))
for host, game := range hostsgames {
// If direct queries are enabled, don't add 'Unspecified' game to server DB
if game == filters.GameUnspecified.String() {
continue
}
exists, err := sdb.serverExists(host, game)
if err != nil {
continue
}
if exists {
continue
}
toInsert[host] = game
}
tx, err := sdb.db.Begin()
if err != nil {
logger.LogAppErrorf("AddServersToDB error creating tx: %s", err)
return
}
var txexecerr error
for host, game := range toInsert {
_, txexecerr = tx.Exec("INSERT INTO servers (host, game) VALUES ($1, $2)",
host, game)
if txexecerr != nil {
logger.LogAppErrorf(
"AddServersToDB exec error for host %s and game %s: %s", host, game, err)
break
}
}
if txexecerr != nil {
if err = tx.Rollback(); err != nil {
logger.LogAppErrorf("AddServersToDB error rolling back tx: %s", err)
return
}
}
if err = tx.Commit(); err != nil {
logger.LogAppErrorf("AddServersToDB error committing tx: %s", err)
return
}
}
func dumpServersToDisk(gamename string, sl *models.APIServerList) error {
j, err := json.Marshal(sl)
if err != nil {
return logger.LogAppErrorf("Error marshaling json: %s", err)
}
t := time.Now()
if err := util.CreateDirectory(constants.DumpDirectory); err != nil {
return logger.LogAppErrorf("Couldn't create '%s' dir: %s\n",
constants.DumpDirectory, err)
}
// Windows doesn't allow ":" in filename so use '-' separators for time
err = util.CreateByteFile(j, constants.DumpFileFullPath(
fmt.Sprintf("%s-servers-%d-%02d-%02d.%02d-%02d-%02d.json",
gamename, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())),
true)
if err != nil {
return logger.LogAppErrorf("Error creating server dump file: %s", err)
}
return nil
}
// StartMasterRetrieval starts a timed retrieval of servers specified by a given
// filter from the Steam Master server after an initial delay of initialDelay
// seconds. It retrieves the list every timeBetweenQueries seconds thereafter.
// A bool can be sent to the stop channel to cancel all timed retrievals.
func StartMasterRetrieval(stop chan bool, filter filters.Filter,
initialDelay int, timeBetweenQueries int) {
retrticker := time.NewTicker(time.Duration(timeBetweenQueries) * time.Second)
logger.WriteDebug(
"Waiting %d seconds before grabbing %s servers. Will retrieve servers every %d secs afterwards.", initialDelay, filter.Game.Name, timeBetweenQueries)
logger.LogAppInfo(
"Waiting %d seconds before grabbing %s servers from master. Will retrieve every %d secs afterwards.", initialDelay, filter.Game.Name, timeBetweenQueries)
firstretrieval := time.NewTimer(time.Duration(initialDelay) * time.Second)
<-firstretrieval.C
logger.WriteDebug("Starting first retrieval of %s servers from master.",
filter.Game.Name)
sl, err := retrieve(filter)
if err != nil {
logger.LogAppErrorf("Error when performing timed master retrieval: %s", err)
}
models.MasterList = sl
for {
select {
case <-retrticker.C:
go func(filters.Filter) {
logger.WriteDebug("%s: Starting %s master server query", time.Now().Format(
"Mon Jan 2 15:04:05 2006 EST"), filter.Game.Name)
logger.LogAppInfo("%s: Starting %s master server query", time.Now().Format(
"Mon Jan 2 15:04:05 2006 EST"), filter.Game.Name)
sl, err := retrieve(filter)
if err != nil {
logger.LogAppErrorf("Error when performing timed master retrieval: %s",
err)
}
models.MasterList = sl
}(filter)
case <-stop:
retrticker.Stop()
return
}
}
}
func retrieve(filter filters.Filter) (*models.APIServerList, error) {
var mq MasterQuery
var err error
if config.Config.SteamConfig.UseWebServerList {
mq, err = NewMasterWebQuery(filter)
} else {
mq, err = NewMasterQuery(filter)
}
if err != nil {
return nil, logger.LogSteamErrorf("Master server error: %s", err)
}
if filter.Game.IgnoreInfo && filter.Game.IgnorePlayers && filter.Game.IgnoreRules {
return nil, logger.LogAppErrorf("Cannot ignore all three AS2 requests!")
}
data := a2sData{}
hg := make(map[string]filters.Game, len(mq.Servers))
for _, h := range mq.Servers {
hg[h] = filter.Game
}
data.HostsGames = hg
// Order of retrieval is by amount of work that must be done (generally 1, 2, 3)
// 1. rules (request chal #, recv chal #, req rules, recv rules)
// games with multi-packet A2S_RULES replies do the most work; otherwise 1 = 2, 3
// 2. players (request chal #, recv chal #, req players, recv players)
// 3. info: just request info & receive info
// Note: some servers (i.e. new beta games) don't have all 3 of AS2_RULES/PLAYER/INFO
if !filter.Game.IgnoreRules {
data.Rules = batchRuleQuery(mq.Servers)
}
if !filter.Game.IgnorePlayers {
data.Players = batchPlayerQuery(mq.Servers)
}
if !filter.Game.IgnoreInfo {
data.Info = batchInfoQuery(mq.Servers)
}
serverlist, err := buildServerList(data, true)
if err != nil {
return nil, logger.LogAppError(err)
}
if config.Config.DebugConfig.EnableServerDump {
if err := dumpServersToDisk(filter.Game.Name, serverlist); err != nil {
logger.LogAppError(err)
}
}
return serverlist, nil
}
func verifyServerDbPath() error {
if err := util.CreateDirectory(constants.DbDirectory); err != nil {
logger.LogAppError(err)
panic(fmt.Sprintf("Unable to create database directory %s: %s",
constants.DbDirectory, err))
}
if err := createServerDBtable(constants.GetServerDBPath()); err != nil {
logger.LogAppErrorf("Unable to verify database path: %s", err)
panic("Unable to verify database path")
}
return nil
}
func (sdb *SDB) serverExists(host string, game string) (bool, error) {
rows, err := sdb.db.Query(
"SELECT host, game FROM servers WHERE host =? AND GAME =? LIMIT 1",
host, game)
if err != nil {
return false, logger.LogAppErrorf(
"serverExists: Error querying database for host %s and game %s: %s",
host, game, err)
}
defer rows.Close()
h, g := "", ""
for rows.Next() {
if err := rows.Scan(&h, &g); err != nil {
return false, logger.LogAppErrorf(
"serverExists: Error querying database for host %s and game %s: %s",
host, game, err)
}
}
if h != "" && g != "" {
return true, nil
}
return false, nil
}
// GetHostsAndGameFromIDAPIQuery Retrieves the hosts and game names from the
// server database file in response to a user-specified API query for a given
// set of server ID numbers. Sends the results over a channel consisting of a
// host to game name string mapping.
func (sdb *SDB) GetHostsAndGameFromIDAPIQuery(result chan map[string]string,
ids []string) {
hosts := make(map[string]string, len(ids))
for _, id := range ids {
host, game, err := sdb.getHostAndGame(id)
if err != nil {
logger.LogAppErrorf("Error getting host from ID for API query: %s", err)
return
}
if host == "" && game == "" {
continue
}
hosts[host] = game
}
result <- hosts
}
func createServerDBtable(dbfile string) error {
create := `CREATE TABLE servers (
server_id INTEGER NOT NULL,
host TEXT NOT NULL,
game TEXT NOT NULL,
PRIMARY KEY(server_id)
)`
if util.FileExists(dbfile) {
// already exists, so verify integrity
db, err := sql.Open("sqlite3", dbfile)
if err != nil {
return logger.LogAppErrorf(
"Unable to open server DB file for verification: %s", err)
}
defer db.Close()
var name string
err = db.QueryRow(
"SELECT name from sqlite_master where type='table' and name='servers'").Scan(&name)
switch {
case err == sql.ErrNoRows:
if _, err = db.Exec(create); err != nil {
return logger.LogAppErrorf("Unable to create servers table in DB: %s", err)
}
case err != nil:
return logger.LogAppErrorf("Server DB table verification error: %s", err)
}
return nil
}
err := util.CreateEmptyFile(dbfile, true)
if err != nil {
return logger.LogAppErrorf("Unable to create server DB: %s", err)
}
db, err := sql.Open("sqlite3", dbfile)
if err != nil {
return logger.LogAppErrorf(
"Unable to open server DB file for table creation: %s", err)
}
defer db.Close()
_, err = db.Exec(create)
if err != nil {
return logger.LogAppErrorf("Unable to create servers table in servers DB: %s",
err)
}
return nil
}
func buildServerList(data a2sData, addtoServerDB bool) (*models.APIServerList,
error) {
// Cannot ignore all three requests
for _, g := range data.HostsGames {
if g.IgnoreInfo && g.IgnorePlayers && g.IgnoreRules {
return nil, logger.LogAppErrorf("Cannot ignore all three A2S_ requests!")
}
}
successcount := 0
var success bool
srvDBhosts := make(map[string]string, len(data.HostsGames))
sl := &models.APIServerList{
Servers: make([]models.APIServer, 0),
FailedServers: make([]string, 0),
}
for host, game := range data.HostsGames {
info, iok := data.Info[host]
players, pok := data.Players[host]
if players == nil {
// return empty array instead of nil pointers (null) in json
players = make([]models.SteamPlayerInfo, 0)
}
rules, rok := data.Rules[host]
success = iok && rok && pok
if game.IgnoreInfo {
success = pok && rok
}
if game.IgnorePlayers {
success = iok && rok
}
if game.IgnoreRules {
rules = make(map[string]string, 0)
success = iok && pok
}
if game.IgnoreInfo && game.IgnorePlayers {
success = rok
}
if game.IgnoreInfo && game.IgnoreRules {
success = pok
}
if game.IgnorePlayers && game.IgnoreRules {
success = iok
}
if success {
srv := models.APIServer{
Game: game.Name,
Players: players,
FilteredPlayers: removeBuggedPlayers(players),
Rules: rules,
Info: info,
}
// Gametype support: gametype can be found in rules, info, or not
// at all depending on the game (currently just for QuakeLive & Reflex)
srv.Info.GameTypeShort, srv.Info.GameTypeFull = getGameType(game, srv)
ip, port, serr := net.SplitHostPort(host)
if serr == nil {
srv.IP = ip
srv.Host = host
p, perr := strconv.Atoi(port)
if perr == nil {
srv.Port = p
}
if !strings.EqualFold(game.Name, filters.GameUnspecified.String()) {
srvDBhosts[host] = game.Name
}
loc := make(chan models.DbCountry, 1)
go db.CountryDB.GetCountryInfo(loc, ip)
srv.CountryInfo = <-loc
}
sl.Servers = append(sl.Servers, srv)
successcount++
} else {
sl.FailedServers = append(sl.FailedServers, host)
}
}
sl.RetrievedAt = time.Now().Format("Mon Jan 2 15:04:05 2006 EST")
sl.RetrievedTimeStamp = time.Now().Unix()
sl.ServerCount = len(sl.Servers)
sl.FailedCount = len(sl.FailedServers)
if len(srvDBhosts) != 0 {
go db.ServerDB.AddServersToDB(srvDBhosts)
sl.Servers = setServerIDsForList(sl.Servers)
}
logger.LogAppInfo(
"Successfully queried (%d/%d) servers. %d timed out or otherwise failed.",
successcount, len(data.HostsGames), sl.FailedCount)
logger.WriteDebug("Server Queries: Successful: (%d/%d) servers\tFailed: %d servers",
successcount, len(data.HostsGames), sl.FailedCount)
return sl, nil
}
// Close closes the country geolocation database.
func (cdb *CDB) Close() {
err := cdb.db.Close()
if err != nil {
logger.LogAppErrorf("Error closing country database: %s", err)
}
}
// Close closes the server database's underlying connection.
func (sdb *SDB) Close() {
err := sdb.db.Close()
if err != nil {
logger.LogAppErrorf("Error closing server DB: %s", err)
}
}
func getServers(filter filters.Filter) ([]string, error) {
maxHosts := config.Config.SteamConfig.MaximumHostsToReceive
var serverlist []string
var c net.Conn
var err error
retrieved := 0
addr := "0.0.0.0:0"
c, err = net.DialTimeout("udp", masterServerHost,
time.Duration(QueryTimeout)*time.Second)
if err != nil {
logger.LogSteamError(ErrHostConnection(err.Error()))
return nil, ErrHostConnection(err.Error())
}
defer c.Close()
c.SetDeadline(time.Now().Add(time.Duration(QueryTimeout) * time.Second))
for {
s, err := queryMasterServer(c, addr, filter)
if err != nil {
// usually timeout - Valve throttles >30 UDP packets (>6930 servers) per min
logger.WriteDebug("Master query error, likely due to Valve throttle/timeout :%s",
err)
break
}
// get hosts:ports beginning after header (0xFF, 0xFF, 0xFF, 0xFF, 0x66, 0x0A)
ips, total, err := extractHosts(s[6:])
if err != nil {
return nil, logger.LogAppErrorf("Error when extracting addresses: %s",
err)
}
retrieved = retrieved + total
if retrieved >= maxHosts {
logger.LogSteamInfo("Max host limit of %d reached!", maxHosts)
logger.WriteDebug("Max host limit of %d reached!", maxHosts)
break
}
logger.LogSteamInfo("%d hosts retrieved so far from master.", retrieved)
logger.WriteDebug("%d hosts retrieved so far from master.", retrieved)
for _, ip := range ips {
serverlist = append(serverlist, ip)
}
if (serverlist[len(serverlist)-1]) != "0.0.0.0:0" {
logger.LogSteamInfo("More hosts need to be retrieved. Last IP was: %s",
serverlist[len(serverlist)-1])
logger.WriteDebug("More hosts need to be retrieved. Last IP was: %s",
serverlist[len(serverlist)-1])
addr = serverlist[len(serverlist)-1]
} else {
logger.LogSteamInfo("IP retrieval complete!")
logger.WriteDebug("IP retrieval complete!")
break
}
}
// remove 0.0.0.0:0
if len(serverlist) != 0 {
if serverlist[len(serverlist)-1] == "0.0.0.0:0" {
serverlist = serverlist[:len(serverlist)-1]
}
}
return serverlist, nil
}