/
main.go
105 lines (85 loc) · 1.86 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
package main
import (
"os"
"log"
"bufio"
"fmt"
"sync"
_ "github.com/mattn/go-sqlite3"
"database/sql"
"runtime"
"flag"
"net/http"
)
func requestWorker(domains chan string, wg *sync.WaitGroup, db *sql.DB) {
defer wg.Done()
httpClient := &http.Client{}
for domain := range domains {
response, err := httpClient.Get(domain)
var statusCode int;
if (err == nil) {
statusCode = response.StatusCode
log.Println(fmt.Sprintf("[%s] %d", domain, response.StatusCode))
} else {
statusCode = -1
log.Println(fmt.Sprintf("[%s] HTTP Error %s", domain, err.Error()))
}
go func() {
db.Exec(fmt.Sprintf("INSERT INTO sites(domain, code) values(\"%s\", %d)", domain, statusCode))
}()
}
}
func initDataBase() *sql.DB {
db, err := sql.Open("sqlite3", "./sites.db")
if err != nil {
log.Fatal(err)
}
defer db.Close()
sqlStmt := `
DELETE FROM sites;
DROP TABLE sites;
`
db.Exec(sqlStmt)
if err != nil {
log.Panic("%q: %s\n", err, sqlStmt)
os.Exit(1)
}
sqlStmt = `
CREATE TABLE sites (domain TEXT NOT NULL PRIMARY KEY, code INTEGER);
`
_, err = db.Exec(sqlStmt)
if err != nil {
log.Panic("%q: %s\n", err, sqlStmt)
os.Exit(1)
}
return db
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
var threadsCount int;
flag.IntVar(&threadsCount, "threads", 15, "Count of used threads (goroutines)")
file, err := os.Open("ru_domains_200_ok")
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
taskChannel := make(chan string, 10000)
go func() {
for scanner.Scan() {
domain := fmt.Sprintf("http://%s/wso.php", scanner.Text())
taskChannel <- domain
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
close(taskChannel)
}()
db := initDataBase()
wg := new(sync.WaitGroup)
for i := 0; i < threadsCount; i++ {
wg.Add(1)
go requestWorker(taskChannel, wg, db)
}
wg.Wait()
}