// outbound setup, check capabilities and set mode
// returns (supports stream, supports reader) + error
func (self nntpConnection) outboundHandshake(conn *textproto.Conn) (stream, reader bool, err error) {
log.Println(self.name, "outbound handshake")
var code int
var line string
for err == nil {
code, line, err = conn.ReadCodeLine(-1)
log.Println(self.name, line)
if err == nil {
if code == 200 {
// send capabilities
log.Println(self.name, "ask for capabilities")
err = conn.PrintfLine("CAPABILITIES")
if err == nil {
// read response
dr := conn.DotReader()
r := bufio.NewReader(dr)
for {
line, err = r.ReadString('\n')
if err == io.EOF {
// we are at the end of the dotreader
// set err back to nil and break out
err = nil
break
} else if err == nil {
// we got a line
if line == "MODE-READER\n" || line == "READER\n" {
log.Println(self.name, "supports READER")
reader = true
} else if line == "STREAMING\n" {
stream = true
log.Println(self.name, "supports STREAMING")
} else if line == "POSTIHAVESTREAMING\n" {
stream = true
reader = false
log.Println(self.name, "is SRNd")
}
} else {
// we got an error
log.Println("error reading capabilities", err)
break
}
}
// return after reading
return
}
} else if code == 201 {
log.Println("feed", self.name, "does not allow posting")
// we don't do auth yet
break
} else {
continue
}
}
}
return
}
// ask for an article from the remote server
// feed it to the daemon if we get it
func (self *nntpConnection) requestArticle(daemon NNTPDaemon, conn *textproto.Conn, msgid string) (err error) {
log.Println(self.name, "asking for", msgid)
// send command
err = conn.PrintfLine("ARTICLE %s", msgid)
// read response
code, line, err := conn.ReadCodeLine(-1)
if code == 220 {
// awwww yeh we got it
var hdr textproto.MIMEHeader
// read header
hdr, err = conn.ReadMIMEHeader()
if err == nil {
// prepare to read body
dr := conn.DotReader()
// check header and decide if we want this
reason, err := self.checkMIMEHeader(daemon, hdr)
if err == nil {
if len(reason) > 0 {
log.Println(self.name, "discarding", msgid, reason)
// we don't want it, discard
io.Copy(ioutil.Discard, dr)
daemon.database.BanArticle(msgid, reason)
} else {
// yeh we want it open up a file to store it in
f := daemon.store.CreateTempFile(msgid)
if f == nil {
// already being loaded elsewhere
} else {
// write header to file
writeMIMEHeader(f, hdr)
// write article body to file
_, _ = io.Copy(f, dr)
// close file
f.Close()
log.Println(msgid, "obtained via reader from", self.name)
// tell daemon to load article via infeed
daemon.infeed_load <- msgid
}
}
} else {
// error happened while processing
log.Println(self.name, "error happend while processing MIME header", err)
}
} else {
// error happened while reading header
log.Println(self.name, "error happened while reading MIME header", err)
}
} else if code == 430 {
// they don't know it D:
log.Println(msgid, "not known by", self.name)
} else {
// invalid response
log.Println(self.name, "invald response to ARTICLE:", code, line)
}
return
}
func newClient(conn *textproto.Conn) (*Conn, error) {
_, msg, err := conn.ReadCodeLine(200)
if err != nil {
return nil, err
}
return &Conn{
conn: conn,
Banner: msg,
}, nil
}
// ask for an article from the remote server
// feed it to the daemon if we get it
func (self *nntpConnection) requestArticle(daemon *NNTPDaemon, conn *textproto.Conn, msgid string) (err error) {
// send command
err = conn.PrintfLine("ARTICLE %s", msgid)
// read response
code, line, err := conn.ReadCodeLine(-1)
if code == 220 {
// awwww yeh we got it
var hdr textproto.MIMEHeader
// read header
hdr, err = readMIMEHeader(conn.R)
if err == nil {
// prepare to read body
dr := conn.DotReader()
// check header and decide if we want this
reason, ban, err := self.checkMIMEHeaderNoAuth(daemon, hdr)
if err == nil {
if len(reason) > 0 {
log.Println(self.name, "discarding", msgid, reason)
// we don't want it, discard
io.Copy(ioutil.Discard, dr)
if ban {
daemon.database.BanArticle(msgid, reason)
}
} else {
// yeh we want it open up a file to store it in
err = self.storeMessage(daemon, hdr, dr)
if err != nil {
log.Println(self.name, "failed to obtain article", err)
// probably an invalid signature or format
daemon.database.BanArticle(msgid, err.Error())
}
}
} else {
// error happened while processing
log.Println(self.name, "error happend while processing MIME header", err)
}
} else {
// error happened while reading header
log.Println(self.name, "error happened while reading MIME header", err)
}
} else if code == 430 {
// they don't know it D:
} else {
// invalid response
log.Println(self.name, "invald response to ARTICLE:", code, line)
}
return
}
// switch modes
func (self *nntpConnection) modeSwitch(mode string, conn *textproto.Conn) (success bool, err error) {
self.access.Lock()
mode = strings.ToUpper(mode)
conn.PrintfLine("MODE %s", mode)
var code int
code, _, err = conn.ReadCodeLine(-1)
if code >= 200 && code < 300 {
// accepted mode change
if len(self.mode) > 0 {
log.Printf(self.name, "mode switch %s -> %s", self.mode, mode)
} else {
log.Println(self.name, "switched to mode", mode)
}
self.mode = mode
success = len(self.mode) > 0
}
self.access.Unlock()
return
}
func (self *nntpConnection) Quit(conn *textproto.Conn) (err error) {
conn.PrintfLine("QUIT")
_, _, err = conn.ReadCodeLine(0)
return
}
// grab every post from the remote server, assumes outbound connection
func (self *nntpConnection) scrapeServer(daemon NNTPDaemon, conn *textproto.Conn) (err error) {
log.Println(self.name, "scrape remote server")
success := true
if success {
// send newsgroups command
err = conn.PrintfLine("NEWSGROUPS %d 000000 GMT", timeNow())
if err == nil {
// read response line
code, _, err := conn.ReadCodeLine(231)
if code == 231 {
var groups []string
// valid response, we expect a multiline
dr := conn.DotReader()
// read lines
sc := bufio.NewScanner(dr)
for sc.Scan() {
line := sc.Text()
idx := strings.Index(line, " ")
if idx > 0 {
groups = append(groups, line[:idx])
} else {
// invalid line? wtf.
log.Println(self.name, "invalid line in newsgroups multiline response:", line)
}
}
err = sc.Err()
if err == nil {
log.Println(self.name, "got list of newsgroups")
// for each group
for _, group := range groups {
var banned bool
// check if the newsgroup is banned
banned, err = daemon.database.NewsgroupBanned(group)
if banned {
// we don't want it
} else if err == nil {
// scrape the group
err = self.scrapeGroup(daemon, conn, group)
if err != nil {
log.Println(self.name, "did not scrape", group, err)
break
}
} else {
// error while checking for ban
log.Println(self.name, "checking for newsgroup ban failed", err)
break
}
}
} else {
// we got a bad multiline block?
log.Println(self.name, "bad multiline response from newsgroups command", err)
}
} else if err == nil {
// invalid response no error
log.Println(self.name, "gave us invalid response to newsgroups command", code)
} else {
// invalid response with error
log.Println(self.name, "error while reading response from newsgroups command", err)
}
} else {
log.Println(self.name, "failed to send newsgroups command", err)
}
} else if err == nil {
// failed to switch mode to reader
log.Println(self.name, "does not do reader mode, bailing scrape")
} else {
// failt to switch mode because of error
log.Println(self.name, "failed to switch to reader mode when scraping", err)
}
return
}
// scrape all posts in a newsgroup
// download ones we do not have
func (self *nntpConnection) scrapeGroup(daemon NNTPDaemon, conn *textproto.Conn, group string) (err error) {
log.Println(self.name, "scrape newsgroup", group)
// send GROUP command
err = conn.PrintfLine("GROUP %s", group)
if err == nil {
// read reply to GROUP command
code := 0
code, _, err = conn.ReadCodeLine(211)
// check code
if code == 211 {
// success
// send XOVER command, dummy parameter for now
err = conn.PrintfLine("XOVER 0")
if err == nil {
// no error sending command, read first line
code, _, err = conn.ReadCodeLine(224)
if code == 224 {
// maps message-id -> references
articles := make(map[string]string)
// successful response, read multiline
dr := conn.DotReader()
sc := bufio.NewScanner(dr)
for sc.Scan() {
line := sc.Text()
parts := strings.Split(line, "\t")
if len(parts) > 5 {
// probably valid line
msgid := parts[4]
// msgid -> reference
articles[msgid] = parts[5]
} else {
// probably not valid line
// ignore
}
}
err = sc.Err()
if err == nil {
// everything went okay when reading multiline
// for each article
for msgid, refid := range articles {
// check the reference
if len(refid) > 0 && ValidMessageID(refid) {
// do we have it?
if daemon.database.HasArticle(refid) {
// we have it don't do anything
} else if daemon.database.ArticleBanned(refid) {
// thread banned
} else {
// we don't got root post and it's not banned, try getting it
err = self.requestArticle(daemon, conn, refid)
if err != nil {
// something bad happened
log.Println(self.name, "failed to obtain root post", refid, err)
return
}
}
}
// check the actual message-id
if len(msgid) > 0 && ValidMessageID(msgid) {
// do we have it?
if daemon.database.HasArticle(msgid) {
// we have it, don't do shit
} else if daemon.database.ArticleBanned(msgid) {
// this article is banned, don't do shit yo
} else {
// we don't have it but we want it
err = self.requestArticle(daemon, conn, msgid)
if err != nil {
// something bad happened
log.Println(self.name, "failed to obtain article", msgid, err)
return
}
}
}
}
} else {
// something bad went down when reading multiline
log.Println(self.name, "failed to read multiline for", group, "XOVER command")
}
}
}
} else if err == nil {
// invalid response code no error
log.Println(self.name, "says they don't have", group, "but they should")
} else {
// error recving response
log.Println(self.name, "error recving response from GROUP command", err)
}
}
return
}
// outbound setup, check capabilities and set mode
// returns (supports stream, supports reader, supports tls) + error
func (self *nntpConnection) outboundHandshake(conn *textproto.Conn, conf *FeedConfig) (stream, reader, tls bool, err error) {
log.Println(self.name, "outbound handshake")
var line string
var code int
for err == nil {
code, line, err = conn.ReadCodeLine(-1)
log.Println(self.name, line)
if err == nil {
if code == 200 {
// send capabilities
log.Println(self.name, "ask for capabilities")
err = conn.PrintfLine("CAPABILITIES")
if err == nil {
// read response
dr := conn.DotReader()
r := bufio.NewReader(dr)
for {
line, err = r.ReadString('\n')
if err == io.EOF {
// we are at the end of the dotreader
// set err back to nil and break out
err = nil
break
} else if err == nil {
if line == "STARTTLS\n" {
log.Println(self.name, "supports STARTTLS")
tls = true
} else if line == "MODE-READER\n" || line == "READER\n" {
log.Println(self.name, "supports READER")
reader = true
} else if line == "STREAMING\n" {
stream = true
log.Println(self.name, "supports STREAMING")
} else if line == "POSTIHAVESTREAMING\n" {
stream = true
reader = false
log.Println(self.name, "is SRNd")
}
} else {
// we got an error
log.Println("error reading capabilities", err)
break
}
}
// return after reading
break
}
} else if code == 201 {
log.Println("feed", self.name, "does not allow posting")
// we don't do auth yet
break
} else {
continue
}
}
}
if conf != nil && len(conf.username) > 0 && len(conf.passwd) > 0 {
log.Println(self.name, "authenticating...")
err = conn.PrintfLine("AUTHINFO USER %s", conf.username)
if err == nil {
var code int
code, line, err = conn.ReadCodeLine(381)
if code == 381 {
err = conn.PrintfLine("AUTHINFO PASS %s", conf.passwd)
if err == nil {
code, line, err = conn.ReadCodeLine(281)
if code == 281 {
log.Println(self.name, "Auth Successful")
} else {
log.Println(self.name, "Auth incorrect", line)
conn.PrintfLine("QUIT")
conn.Close()
return false, false, false, io.EOF
}
}
}
}
}
return
}