func main() {
str := "\tthe import roles of utf8 text\n"
str = strings.TrimFunc(str, unicode.IsSpace)
fmt.Printf("%s\n", str[0:len(str)/2])
u8 := utf8string.NewString(str)
FirstHalf := u8.Slice(0, u8.RuneCount()/2)
fmt.Printf("%s\n", FirstHalf)
}
func twitterize(msg string) string {
msg = strings.Replace(msg, "\n", " ", -1)
msg = strings.Replace(msg, ".com", "", -1)
if utf8.RuneCountInString(msg) >= 113 {
msg = utf8string.NewString(msg).Slice(0, 113) + "... "
} else if !strings.HasSuffix(msg, " ") {
msg = msg + " "
}
return msg
}
func twitterize(article *Article) string {
msg := "@NYTMinusContext " + html.UnescapeString(article.Headline)
// remove any URLs from headline so our link works
msg = cleanRegex.ReplaceAllString(msg, "")
// leave 24 for space and link!
if utf8.RuneCountInString(msg) > 116 {
msg = utf8string.NewString(msg).Slice(0, 113) + "..."
}
// tack on the URL
msg += " " + article.URL
return msg
}
func ValidateRedirect(redirect string, min, max int) error {
utf8name := utf8string.NewString(redirect)
len := utf8name.RuneCount()
if len < min || len > max {
return e.New(e.ErrInvalidLength)
}
for _, s := range utf8name.Slice(1, len) {
if !uni.IsLetter(s) && !unicode.IsDigit(s) && s != '/' && s != '-' && s != '_' && s != '?' && s != '&' && s != '=' && s != '%' && s != '*' && s != '+' && s != ' ' && s != ',' {
return e.Push(e.New("the character '%v' in redirect is invalid", string([]byte{byte(s)})), e.New("invalid redirect"))
}
}
return nil
}
func (theParser *CommandParser) prepareRule(name, expression string) *RuleStruct {
items := []*RuleItem{}
ruleItems, ruleType := theParser.splitRule(expression)
rs := &RuleStruct{
Name: name,
Items: items,
Type: ruleType,
}
for _, ruleItem := range ruleItems {
item := &RuleItem{
Cardinality: theParser.expressionCardinality(ruleItem),
ExprType: theParser.expressionType(ruleItem),
ExprString: ruleItem,
ParentRule: rs,
Seen: false,
}
if item.Cardinality != CardinalityOne {
// strip out cardinality char
item.ExprString = item.ExprString[0 : len(item.ExprString)-1]
}
switch item.ExprType {
case IdentifierExpr, CharExpr:
// remove quotes
item.ExprString = item.ExprString[1 : len(item.ExprString)-1]
case DataTypeExpr:
// remove exclamation mark
item.ExprString = item.ExprString[1:]
}
// TODO: find a better way to extract runes from a string
if item.ExprType == CharExpr {
s := utf8string.NewString(item.ExprString)
r := s.At(1)
if r == '\'' {
item.ExprString = s.String()
} else {
panic("CharExpr does not contain rune literal!")
}
}
rs.Items = append(rs.Items, item)
}
return rs
}
// Ruby implementation of the Double Metaphone algorithm by Lawrence Philips,
// originally published in the June 2000 issue of C/C++ Users Journal.
//
// Based on double_metaphone.rb from https://github.com/threedaymonk/text
//
// Returns the primary and secondary double metaphone tokens (the secondary
// will be nil if equal to the primary).
func DoubleMetaphone(str string) (string, string) {
primary := make([]string, 0, 5)
secondary := make([]string, 0, 5)
original := utf8string.NewString(strings.ToUpper(str))
current := 0
if mm(ss(original, 0, 2), []string{"GN", "KN", "PN", "WR", "PS"}) {
current += 1
}
if ss(original, 0, 1) == "X" {
primary = append(primary, "S")
secondary = append(secondary, "S")
current += 1
}
for len(primary) < 4 || len(secondary) < 4 {
if current > original.RuneCount() {
break
}
p, s, n := doubleMetaphoneLookup(original, current)
if p != "" {
primary = append(primary, p)
}
if s != "" {
secondary = append(secondary, s)
}
current += n
}
p := strings.Join(primary, "")
if len(p) > 4 {
p = p[0:4]
}
s := strings.Join(secondary, "")
if len(s) > 4 {
s = s[0:4]
}
if p == s {
return p, ""
}
return p, s
}
func (s *Server) EntryHandler(c *gin.Context) {
uuid := c.Params.ByName("uuid")
req := &pb.EntryRequest{uuid}
_, feed, err := s.FetchFeed(c, req)
if RequestError(c, err) {
return
}
entry := feed.Entries[0]
rawBody := utf8string.NewString(entry.RawBody)
title := rawBody.String()
if rawBody.RuneCount() > 42 {
title = rawBody.Slice(0, 42)
}
data := pongo2.Context{
"title": title,
"name": entry.From.Name,
"feed": feed,
"show_paging": false,
}
s.HTML(c, 200, "feed.html", data)
}
func procdirs(outfile *os.File) {
ORG_ROOT := "sanfrancisco.huzzlee.com//"
const filechunk = 8192 // we settle for 8KB
outfile.WriteString("\r\nSTART PROCFILES \r\n\r\n")
db, err := sql.Open("mysql", "fdiscovery:F1tPar0la@/fdiscovery")
if err != nil {
outfile.WriteString(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
}
defer db.Close()
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.dirhash ///
stDirHash, err := db.Prepare("INSERT INTO fdiscovery.dirhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stDirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.rdirhash ///
stRDirHash, err := db.Prepare("INSERT INTO fdiscovery.rdirhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stRDirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.odirhash ///
stODirHash, err := db.Prepare("INSERT INTO fdiscovery.odirhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stODirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.dirstatus ///
stDirStatus, err := db.Prepare("INSERT INTO fdiscovery.dirstatus VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stDirStatus.Close() //
filequery, err := db.Prepare("SELECT RecId, RootId, FileType, FileName, FileExtension, FileDir, FilePath, FileSize, ModTimeLocal FROM fdiscovery.sourcefiles WHERE RecId = ?")
if err != nil {
log.Fatal(err)
}
var (
RecId int64
RootId string
FileType string
FileName string
FileExtension string
Dir string
FilePath string
FileSize string
ModTimeLocal string
//FileNameHash string
//FileDirHash string
//FilePathHash string
//FileHash string
//FileStatus string
//FileSizeNum int64
)
FileStatus := "NEW"
for i := 7430; i < 1438884; i++ {
err = filequery.QueryRow(i).Scan(&RecId, &RootId, &FileType, &FileName, &FileExtension, &Dir, &FilePath, &FileSize, &ModTimeLocal)
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
if FileType == "DIR" {
outfile.WriteString("\r\nFILENO: ")
outfile.WriteString(strconv.Itoa(int(i)))
//outfile.WriteString( "\r\nFILENAME: " )
//outfile.WriteString( FileName )
outfile.WriteString("\r\nFILEPATH: ")
outfile.WriteString(FilePath)
///===============================================================================
DirVec := []byte(Dir)
DirHasher := sha256.New()
DirHasher.Write(DirVec)
DirHash := base64.URLEncoding.EncodeToString(DirHasher.Sum(nil))
DirU8 := utf8string.NewString(Dir)
RDir := DirU8.Slice(3, DirU8.RuneCount())
RDirVec := []byte(RDir)
RDirHasher := sha256.New()
RDirHasher.Write(RDirVec)
RDirHash := base64.URLEncoding.EncodeToString(RDirHasher.Sum(nil))
//outfile.WriteString( "\r\nRDir: " )
//outfile.WriteString( RDir )
ODir := ORG_ROOT + Dir
ODirVec := []byte(ODir)
ODirHasher := sha256.New()
ODirHasher.Write(ODirVec)
ODirHash := base64.URLEncoding.EncodeToString(ODirHasher.Sum(nil))
//outfile.WriteString( "\r\nODir: " )
//outfile.WriteString( ODir )
_, err = stDirHash.Exec(RecId, DirHash, Dir) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stRDirHash.Exec(RecId, RDirHash, RDir) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stODirHash.Exec(RecId, ODirHash, ODir) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
_, err = stDirStatus.Exec(RecId, FileStatus) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
}
///===============================================================================
}
}
func procfiles(outfile *os.File) {
ORG_ROOT := "sanfrancisco.huzzlee.com//"
const filechunk = 8192 // we settle for 8KB
outfile.WriteString("\r\nSTART PROCFILES \r\n\r\n")
db, err := sql.Open("mysql", "fdiscovery:F1tPar0la@/fdiscovery")
if err != nil {
outfile.WriteString(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
}
defer db.Close()
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filenamehash ///
stFileNameHash, err := db.Prepare("INSERT INTO fdiscovery.filenamehash VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileNameHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filedirhash ///
stFileDirHash, err := db.Prepare("INSERT INTO fdiscovery.filedirhash VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileDirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filerdirhash ///
stFileRDirHash, err := db.Prepare("INSERT INTO fdiscovery.filerdirhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileRDirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.fileodirhash ///
stFileODirHash, err := db.Prepare("INSERT INTO fdiscovery.fileodirhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileODirHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filepathhash ///
stFilePathHash, err := db.Prepare("INSERT INTO fdiscovery.filepathhash VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFilePathHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filerpathhash ///
stFileRPathHash, err := db.Prepare("INSERT INTO fdiscovery.filerpathhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileRPathHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.fileopathhash ///
stFileOPathHash, err := db.Prepare("INSERT INTO fdiscovery.fileopathhash VALUES( ?, ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileOPathHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filehash ///
stFileHash, err := db.Prepare("INSERT INTO fdiscovery.filehash VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileHash.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filesize ///
stFileSize, err := db.Prepare("INSERT INTO fdiscovery.filesize VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileSize.Close() //
//----------------------------------------------------------------------
// Prepare statement for inserting data into fdiscovery.filestatus ///
stFileStatus, err := db.Prepare("INSERT INTO fdiscovery.filestatus VALUES( ?, ? )") //
if err != nil {
outfile.WriteString(err.Error()) //
}
defer stFileStatus.Close() //
filequery, err := db.Prepare("SELECT RecId, RootId, FileType, FileName, FileExtension, FileDir, FilePath, FileSize, ModTimeLocal FROM fdiscovery.sourcefiles WHERE RecId = ?")
if err != nil {
log.Fatal(err)
}
var (
RecId int64
RootId string
FileType string
FileName string
FileExtension string
FileDir string
FilePath string
FileSize string
ModTimeLocal string
//FileNameHash string
//FileDirHash string
//FilePathHash string
//FileHash string
//FileStatus string
//FileSizeNum int64
)
FileStatus := "NEW"
for i := 4; i < 1438884; i++ {
err = filequery.QueryRow(i).Scan(&RecId, &RootId, &FileType, &FileName, &FileExtension, &FileDir, &FilePath, &FileSize, &ModTimeLocal)
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
//outfile.WriteString( "\r\nFILENO: " )
//outfile.WriteString( strconv.Itoa( int( i) ) );
//outfile.WriteString( "\r\nFILENAME: " )
//outfile.WriteString( FileName )
//outfile.WriteString( "\r\nFILEPATH: " )
//outfile.WriteString( FilePath )
///===============================================================================
FileNameVec := []byte(FileName)
FileNameHasher := sha256.New()
FileNameHasher.Write(FileNameVec)
FileNameHash := base64.URLEncoding.EncodeToString(FileNameHasher.Sum(nil))
_, err = stFileNameHash.Exec(RecId, FileNameHash) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
FileDirVec := []byte(FileDir)
FileDirHasher := sha256.New()
FileDirHasher.Write(FileDirVec)
FileDirHash := base64.URLEncoding.EncodeToString(FileDirHasher.Sum(nil))
FileDirU8 := utf8string.NewString(FileDir)
FileRDir := FileDirU8.Slice(3, FileDirU8.RuneCount())
FileRDirVec := []byte(FileRDir)
FileRDirHasher := sha256.New()
FileRDirHasher.Write(FileRDirVec)
FileRDirHash := base64.URLEncoding.EncodeToString(FileRDirHasher.Sum(nil))
//outfile.WriteString( "\r\nFileRDir: " )
//outfile.WriteString( FileRDir )
FileODir := ORG_ROOT + FileDir
FileODirVec := []byte(FileODir)
FileODirHasher := sha256.New()
FileODirHasher.Write(FileODirVec)
FileODirHash := base64.URLEncoding.EncodeToString(FileODirHasher.Sum(nil))
//outfile.WriteString( "\r\nFileODir: " )
//outfile.WriteString( FileODir )
_, err = stFileDirHash.Exec(RecId, FileDirHash) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stFileRDirHash.Exec(RecId, FileRDirHash, FileRDir) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stFileODirHash.Exec(RecId, FileODirHash, FileODir) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
FilePathVec := []byte(FilePath)
FilePathHasher := sha256.New()
FilePathHasher.Write(FilePathVec)
FilePathHash := base64.URLEncoding.EncodeToString(FilePathHasher.Sum(nil))
FilePathU8 := utf8string.NewString(FilePath)
FileRPath := FilePathU8.Slice(3, FilePathU8.RuneCount())
FileRPathVec := []byte(FileRPath)
FileRPathHasher := sha256.New()
FileRPathHasher.Write(FileRPathVec)
FileRPathHash := base64.URLEncoding.EncodeToString(FileRPathHasher.Sum(nil))
//outfile.WriteString( "\r\nFileRPath: " )
//outfile.WriteString( FileRPath )
FileOPath := ORG_ROOT + FilePath
FileOPathVec := []byte(FileOPath)
FileOPathHasher := sha256.New()
FileOPathHasher.Write(FileOPathVec)
FileOPathHash := base64.URLEncoding.EncodeToString(FileOPathHasher.Sum(nil))
//outfile.WriteString( "\r\nFileOPath: " )
//outfile.WriteString( FileOPath )
_, err = stFilePathHash.Exec(RecId, FilePathHash) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stFileRPathHash.Exec(RecId, FileRPathHash, FileRPath) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
_, err = stFileOPathHash.Exec(RecId, FileOPathHash, FileOPath) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
FileSizeNum, err := strconv.ParseInt(FileSize, 10, 64)
//FileSizeNum, err := strconv.Atoi( FileSize )
_, err = stFileSize.Exec(RecId, FileSizeNum) // Insert data from fd
if err != nil {
outfile.WriteString("\r\nFileSize: ")
outfile.WriteString(FileSize)
outfile.WriteString("\r\n")
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
_, err = stFileStatus.Exec(RecId, FileStatus) // Insert data from fd
if err != nil {
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
}
///===============================================================================
FileHash := "DIR"
RECY := "D:\\$RECYCLE.BIN\\"
if strings.Contains(FilePath, RECY) {
outfile.WriteString("\r\n PROBLEM: ")
outfile.WriteString(FilePath)
}
/*
if FileType == "FILE" {
if true != strings.Contains(FilePath, RECY) {
outfile.WriteString( "\r\nFILE: " )
outfile.WriteString( FilePath )
File, err1 := os.Open( FilePath )
if err1 != nil {
outfile.WriteString( "\r\nFILE ERROR !!!!\r\n" )
outfile.WriteString(err1.Error()) // proper error handling instead of panic in your app
outfile.WriteString( "\r\n" )
}
defer File.Close()
FileHasher := sha256.New()
// calculate the file size
info, _ := File.Stat()
ifilesize := info.Size()
blocks := uint64(math.Ceil(float64(ifilesize) / float64(filechunk)))
blocksize := int(math.Min(filechunk, float64(ifilesize-int64(i*filechunk))))
buf := make([] byte, blocksize)
for i := uint64(0); i < blocks; i++ {
File.Read(buf)
FileHasher.Write( buf )
}
FileHash = base64.URLEncoding.EncodeToString(FileHasher.Sum(nil))
}
}
*/
_, err = stFileHash.Exec(RecId, FileHash) // Insert data from fd
if err != nil {
outfile.WriteString("\r\nDBERROR\r\n")
outfile.WriteString(err.Error()) // proper error handling instead of panic in your app
outfile.WriteString("\r\n")
}
///===============================================================================
}
}
func MakeFile(pkg, pkgout, src string, debug bool) (*File, error) {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "", src, parser.ParseComments)
if err != nil {
return nil, e.New(err)
}
v := &Visitor{
src: src,
Struct: make(map[string]*Strct),
Cmap: ast.NewCommentMap(fset, f, f.Comments),
StructOrder: make([]*Strct, 0),
Pkgs: NewPkgs(),
}
v.Pkgs.Have("crypto/tls")
v.Pkgs.Have("fmt")
v.Pkgs.Have("log")
v.Pkgs.Have("reflect")
v.Pkgs.Have("tls")
v.Pkgs.Have("time")
v.Pkgs.Have("github.com/fcavani/e")
v.Pkgs.Have("github.com/fcavani/gormethods/auth")
v.Pkgs.Have("github.com/fcavani/gormethods/client")
v.Pkgs.Have("gopkg.in/vmihailenco/msgpack.v2")
ast.Walk(v, f)
file := &File{
Package: f.Name.Name,
Debug: debug,
PkgComplete: f.Name.Name,
Pkgs: v.Pkgs.Packages(),
}
if pkg != "" {
file.PkgComplete = pkg
}
if pkgout != "" {
file.Package = pkgout
}
objs := make(map[string]*Object)
order := make([]string, 0)
for _, f := range v.Functions {
u := utf8string.NewString(f.RecvType)
if u.RuneCount() > 0 && !unicode.IsUpper(u.At(0)) {
continue
}
if f.RecvType == "" {
continue
}
if FuncForbiden(f.Name) {
continue
}
obj, found := objs[f.RecvType]
if !found {
function, err := makeFunction(f)
if err != nil {
return nil, e.Forward(err)
}
t := f.RecvType
if f.PtrRecv {
t = "*" + t
}
strct, found := v.Struct[f.RecvType]
if !found {
return nil, e.New("struct %v not found", f.RecvType)
}
objs[f.RecvType] = &Object{
Name: f.RecvType,
NameClient: f.RecvType + "Client",
RecvVar: f.Recv,
RecvType: t,
Functions: []*Function{function},
Type: makeType(src, strct.Type),
Comments: strings.Replace(strct.Comments, f.RecvType, f.RecvType+"Client", -1),
}
v.Struct[f.RecvType].FunctionCount++
order = append(order, f.RecvType)
} else {
function, err := makeFunction(f)
if err != nil {
return nil, e.Forward(err)
}
obj.Functions = append(obj.Functions, function)
}
}
for _, s := range v.StructOrder {
if s.FunctionCount == 0 {
first, err := rand.Chars(1, rand.Letters, "go")
if err != nil {
return nil, e.Push(err, "can't create a name to the receiver for the struct")
}
recv, err := rand.Chars(2, rand.NumberLetters, "go")
if err != nil {
return nil, e.Push(err, "can't create a name to the receiver for the struct")
}
recv = strings.ToLower(first + recv)
objs[s.Name] = &Object{
Name: s.Name,
NameClient: s.Name + "Client",
RecvVar: recv,
RecvType: s.Name,
Functions: nil,
Type: makeType(src, s.Type),
Comments: strings.Replace(s.Comments, s.Name, s.Name+"Client", -1),
}
}
}
for _, s := range v.StructOrder {
if obj, found := objs[s.Name]; found {
file.Objects = append(file.Objects, obj)
} else {
return nil, e.New("object not found: %v", s.Name)
}
}
return file, nil
}