func main() {
client := &http.Client{}
method := flag.String("x", "GET", "a string")
flag.Parse()
url := flag.Args()[0]
req, err := http.NewRequest(*method, url, nil)
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
}
Header("HEADER")
PPKeyVal("URL:", url)
PPKeyVal("Method:", *method)
PPKeyVal("HTTP Status:", res.Status)
PPrintCode("Status Code:", res.StatusCode)
PPKeyVal("Protocol:", res.Proto)
Header("BODY")
resBody, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if IsJSON(string(resBody)) {
PrintJSON(resBody)
} else {
fmt.Println(gohtml.Format(string(resBody)))
}
}
func main() {
flag.BoolVar(&noFormat, "no-format", false, "output HTML without format")
flag.BoolVar(&lineNo, "lineno", false, "output formatted HTML with line numbers")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage:\n %s [options] [base.ace] [inner.ace]\n\nOptions:\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
var (
compiled string
err error
)
baseFile := flag.Arg(0)
if len(baseFile) == 0 {
compiled, err = compileResultFromStdin()
} else {
compiled, err = compileResultFromFile(baseFile, flag.Arg(1))
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if noFormat {
fmt.Println(compiled)
} else {
if lineNo {
fmt.Println(gohtml.FormatWithLineNo(compiled))
} else {
fmt.Println(gohtml.Format(compiled))
}
}
}
func Pretty(c *slurp.C) slurp.Stage {
return func(in <-chan slurp.File, out chan<- slurp.File) {
var wg sync.WaitGroup
defer wg.Wait()
for file := range in {
wg.Add(1)
go func(file slurp.File) {
defer wg.Done()
html := new(bytes.Buffer)
_, err := html.ReadFrom(file.Reader)
if err != nil {
fmt.Println(err.Error())
return
}
pretty := bytes.NewBufferString(gohtml.Format(html.String()))
file.Reader = pretty
file.FileInfo.SetSize(int64(pretty.Len()))
out <- file
}(file)
}
}
}
func main() {
nextFileNum := "2"
f, _ := os.Create("dat" + nextFileNum)
defer f.Close()
dat, err := ioutil.ReadFile("doc" + nextFileNum + ".txt")
check(err)
formattedHTML1 := gohtml.Format(string(dat))
formattedHTML := ""
for _, line := range strings.Split(formattedHTML1, "\n") {
formattedHTML += strings.TrimSpace(line + "\n")
}
// m := make(map[string]int)
// m["tally123"] = 0
strInt, _ := strconv.Atoi(nextFileNum)
m := loadPats("dat" + strconv.Itoa(strInt-1) + ".encoding")
curWord := ""
for _, c := range strings.TrimSpace(formattedHTML) {
strC := string(c)
if len(strings.TrimSpace(strC)) == 0 || strings.Contains("!#$%&'()*+,-.:;=?@[/\\]^_`{|}~><' \n\t\b", strC) || strings.Contains(`""`, strC) {
if len(curWord) > 0 {
word := strings.ToLower(curWord)
fmt.Println("|" + strings.TrimSpace(curWord) + "|")
if _, ok := m[word]; ok {
// has key
} else {
m[word] = m["tally123"]
m["tally123"]++
}
i := byte(0)
if curWord == strings.Title(word) {
i = byte(1)
} else if curWord == strings.ToUpper(word) {
i = byte(2)
}
j := byte(m[word] / 254)
k := byte(math.Mod(float64(m[word]), 254))
fmt.Println(i)
d2 := []byte{62, j, k, 60}
f.Write(d2)
curWord = ""
} else {
d2 := []byte{byte(c)}
f.Write(d2)
}
} else {
curWord += strC
}
}
fmt.Println(curWord)
savePats("dat"+nextFileNum+".encoding", m)
fmt.Println(m)
fmt.Println(m["tally123"])
}
func callURIs() {
select {
case <-run:
inFile, _ := os.Open("canales_list.txt")
defer inFile.Close()
scanner := bufio.NewScanner(inFile)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
firstChannelCallTime = time.Now()
time.Sleep(time.Millisecond * 1500)
// send message for new channel
msgOut <- msgStruct{Type: "C", Content: scanner.Text()}
resp, body, errs := gorequest.New().Get(fmt.Sprintf("%s%s", *remoteService, scanner.Text())).End()
if errs != nil {
log.Fatalf("log failed: %v", errs)
}
encoding := ""
for _, item := range strings.Split(resp.Header.Get("Content-Type"), " ") {
if strings.Contains(item, "charset=") {
encoding = strings.Split(item, "=")[1]
}
}
ct, _, _ := mime.ParseMediaType(http.DetectContentType([]byte(body)))
mime := ""
switch ct {
case "text/html":
mime = "html"
case "application/xml", "text/xml":
mime = "xml"
case "application/xslt+xml":
mime = "xslt"
case "application/json":
mime = "json"
}
if encoding != "" {
msgOut <- msgStruct{Type: "B", Content: fmt.Sprintf("Codificacion: %s", encoding)}
}
if mime != "" {
msgOut <- msgStruct{Type: "B", Content: fmt.Sprintf("Mime: %s", mime)}
}
msgOut <- msgStruct{Type: "O", Content: gohtml.Format(stripCtlAndExtFromUnicode(body))}
}
case <-end:
log.Info("done")
os.Exit(0)
}
}
func (route *ScraperRoute) TestURL(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
var selector scraper.ScrapSelector
err := RequestToJsonObject(r, &selector)
if err != nil {
HandleHttpErrors(w, err)
return
}
// make sure only test one page
selector.PageParam = ""
scr := scraper.NewScrapper()
jobId, itemsc, err := scr.Scrap(selector)
if err != nil {
HandleHttpErrors(w, err)
return
}
var items []model.Item
for it := range itemsc {
err = it.Err
if err != nil {
HandleHttpErrors(w, err)
return
}
items = append(items, it.Item)
}
snippet, err := scraper.SnippetBase(selector)
if err != nil {
HandleHttpErrors(w, err)
return
}
scrapped := &ItemsResponse{
Items: items,
}
result := map[string]interface{}{
"jobId": jobId,
"snippet": gohtml.Format(snippet),
"scrapped": scrapped,
}
Render().JSON(w, http.StatusOK, result)
}
func main() {
flag.Parse()
typ, err := inputType(flag.Arg(0))
if err != nil {
log.Fatal(err)
}
var data string
switch typ {
case FILE:
data, err = openLocal(flag.Arg(0))
case URL:
data, err = openURL(flag.Arg(0))
}
var file *os.File
if err == nil {
switch {
case *overwrite && typ == FILE:
file, err = os.Create(flag.Arg(0))
case *overwrite && typ == URL:
filename := path.Base(flag.Arg(0))
if filename == "/" || filename == "." {
filename = "result.html"
}
if path.Ext(filename) == "" {
filename += ".html"
}
file, err = os.Create(filename)
case *output != "":
file, err = os.Create(*output)
}
}
if err != nil {
log.Fatal(err)
}
result := gohtml.Format(data)
if file != nil {
file.Write([]byte(result))
} else {
os.Stdout.Write([]byte(result))
}
}
func main() {
// formattedHTML := gohtml.Format(`<html>hello<p>I said Hello</p><div>What is going on and hello there</div></html>`)
dat, _ := ioutil.ReadFile("html")
formattedHTML1 := gohtml.Format(string(dat))
formattedHTML := ""
for _, line := range strings.Split(formattedHTML1, "\n") {
formattedHTML += strings.TrimSpace(line + "\n")
}
m := make(map[string]int)
decode := make(map[int]string)
m["tally123"] = 0
// ENCODING
fmt.Println("encoding")
bits := BitString{curBit: 0}
bits.bitSet.SetMaxBitNdx(8)
curWord := ""
for _, c := range strings.TrimSpace(formattedHTML) {
strC := string(c)
if len(strings.TrimSpace(strC)) == 0 || strings.Contains("!#$%&'()*+,-.:;=?@[/\\]^_`{|}~><' \n\t\b", strC) || strings.Contains(`""`, strC) {
if len(curWord) > 2 {
word := strings.ToLower(curWord)
fmt.Println("|" + strings.TrimSpace(curWord) + "|")
if _, ok := m[word]; ok {
// has key
} else {
m[word] = m["tally123"]
decode[m["tally123"]] = word
m["tally123"]++
}
bits.AddBits(0, 1)
if curWord == strings.Title(word) {
bits.AddBits(1, 1)
bits.AddBits(0, 1)
} else if curWord == strings.ToUpper(word) {
bits.AddBits(0, 1)
bits.AddBits(1, 1)
} else {
bits.AddBits(0, 1)
bits.AddBits(0, 1)
}
bits.AddBits(m[word], 9)
curWord = ""
} else if len(curWord) > 0 {
// write the character bytes that aren't quite enough to be a word
for _, d := range curWord {
bits.AddBits(1, 1)
bits.AddBits(int(d), 8)
}
curWord = ""
}
// write the current character byte
if int(c) == 32 {
// space
bits.AddBits(0, 1)
bits.AddBits(1, 1)
bits.AddBits(1, 1)
} else {
bits.AddBits(1, 1)
bits.AddBits(int(c), 8)
}
} else {
curWord += strC
}
}
fmt.Println(m)
fmt.Println("writing bytes")
fmt.Println(len(dat))
fmt.Println(len(bits.String()))
writeBytes(bits.String())
fmt.Println("reading bytes")
bsString := readBytes()
fmt.Println(len(bsString))
i := 0
for {
if bsString[i] == 49 {
i++
nextByte := bitStringSum(bsString[i : i+8])
fmt.Print(string(nextByte))
i += 8
} else {
i++
firstBit := bitStringSum(string(bsString[i]))
i++
secondBit := bitStringSum(string(bsString[i]))
i++
decodedWord := ""
if firstBit == 1 && secondBit == 0 {
decodedWord = decode[bitStringSum(bsString[i:i+9])]
decodedWord = strings.Title(decodedWord)
i += 9
} else if firstBit == 0 && secondBit == 1 {
decodedWord = decode[bitStringSum(bsString[i:i+9])]
decodedWord = strings.ToUpper(decodedWord)
i += 9
} else if firstBit == 1 && secondBit == 1 {
decodedWord = " "
} else {
decodedWord = decode[bitStringSum(bsString[i:i+9])]
i += 9
}
fmt.Print(decodedWord)
}
if i >= len(bsString)-8 {
break
}
}
}