示例#1
0
// Encoding UTF-8 to Cp1251 and back
func main() {
	var err error

	encoder := charmap.Windows1251.NewEncoder()
	decoder := charmap.Windows1251.NewDecoder()

	inUtf8 := "Ёжики пушистые 好 ἱερογλυφικὰ γράμματ"
	// inUtf8 := "Ёжики пушистые"

	sr := strings.NewReader(inUtf8)
	tr := transform.NewReader(sr, encoder)
	inCp1251, err := ioutil.ReadAll(tr)
	if err != nil {
		fmt.Println("Encoding error: ", err)
	}

	srBack := bytes.NewReader(inCp1251)
	trBack := transform.NewReader(srBack, decoder)
	outUtf8, err := ioutil.ReadAll(trBack)
	if err != nil {
		fmt.Println("Decoding error: ", err)
	}

	fmt.Println("Source UTF8:", inUtf8)
	fmt.Println("CP1251:", inCp1251, string(inCp1251))
	fmt.Println("Result UTF8:", string(outUtf8))

	fmt.Println(strings.Repeat("=", 80))
	fmt.Println("Test https://github.com/fiam/gounidecode")
	fmt.Println("Original: ", inUtf8)
	fmt.Println("Translit: ", unidecode.Unidecode(inUtf8))
}
示例#2
0
func wordToMultiset(word string) (map[rune]int, int) {
	var decoded = unidecode.Unidecode(word)
	decoded = strings.ToLower(decoded)
	var result = make(map[rune]int)
	var l = 0
	for _, r := range decoded {
		result[r]++
		l++
	}
	return result, l
}
示例#3
0
func main() {
	str := "xyz"
	str2 := ""
	bool := true
	// len - количество байт в строке
	fmt.Printf("Количество байт в строках: %d %d\n", len(str), len(str2))

	// количество символов в строке
	count := utf8.RuneCountInString(str)
	fmt.Printf("Количество символов в строке %q: %d\n", str, count)

	fmt.Printf("bool: %t\n", bool)

	p := false
	fmt.Printf("|%T|%v|%#v|\n", p, p, p)

	fmt.Println("Hello World"[1])
	fmt.Println(".png"[1:])

	command := "set a 1"
	fmt.Println(command[4:])

	printf := func(format string, a ...interface{}) {
		fmt.Println(fmt.Sprintf(format, a...))
	}

	printf("test %s, %d, %v", "string", 123, true)

	str = "Donec ante."
	fmt.Println(str[:utf8.RuneCountInString(str)-1])

	for i := 0; i < len(str); i++ {
		fmt.Println(str[i : i+1])
	}

	// transliteration
	str = "Количество байт в строках"
	fmt.Println(strings.Replace(strings.ToLower(unidecode.Unidecode(str)), " ", "_", -1))

	// https://habrastorage.org/files/6b3/ae5/fcb/6b3ae5fcbeaf49c480baca60f88e7d40.jp
	str = "6b3ae5fcbeaf49c480baca60f88e7d40"
	depth := 3
	for i := 0; i < depth; i++ {
		fmt.Println(str[i*3 : (i+1)*3])
	}

}
示例#4
0
func cleanWords(query string) ([]string, bool) {
	query = quotationMarksRegex.ReplaceAllString(query, "")
	query = junkRegex.ReplaceAllString(query, " ")
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))

	var asciiTerm string
	unidecodeExpanded := false
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
		asciiTerm = unidecode.Unidecode(terms[i])
		if asciiTerm != terms[i] {
			terms = append(terms, asciiTerm)
			unidecodeExpanded = true
		}
	}
	return terms, unidecodeExpanded
}
示例#5
0
文件: meowkov.go 项目: lidel/meowkov
func chainTransliterations(seeds [][]string) [][]string {
	var transliterations [][]string

	for _, chain := range seeds {
		var (
			asciiChain []string
			diff       = false
		)
		for _, word := range chain {
			ascii := unidecode.Unidecode(word)
			asciiChain = append(asciiChain, ascii)
			if ascii != word {
				diff = true
			}
		}
		if diff {
			transliterations = append(transliterations, asciiChain)
		}
	}
	return transliterations
}
示例#6
0
	FilterUppercase = "uppercase"
)

type Filter func(input string) string

var filters = map[string]Filter{
	FilterBase64: func(input string) string {
		data := []byte(input)
		return base64.StdEncoding.EncodeToString(data)
	},
	FilterMd5: func(input string) string {
		hash := md5.Sum([]byte(input))
		return hex.EncodeToString(hash[:])
	},
	FilterTranslit: func(input string) string {
		return unidecode.Unidecode(input)
	},
	FilterUppercase: func(input string) string {
		return strings.ToUpper(input)
	},
}

// Creates new Filter instance
func New(id string) Filter {
	if filter, ok := filters[id]; ok {
		return filter
	} else {
		return nil
	}
}
示例#7
0
func (a *HTTPServer) Start() {
	// Create a new token
	m := martini.Classic()

	// GET / - version info
	m.Get("/", func() string {
		// config.VERSION is a const containing the version string
		return "aiw3/np-server " + config.VERSION
	})

	// Remote kick function. Secure before using.
	/*m.Get("/rkick/:name", func(params martini.Params) string {
			var rows []*struct {
				Id int
			}

			err := environment.Env.Database.Query(`
	SELECT
		id
	FROM
		misago_user
	WHERE
		username = ?`, params["name"]).Rows(&rows)

			if err != nil {
				return err.Error()
			}

			if len(rows) != 1 {
				return "User not found"
			}

			npid := structs.IdToNpid(rows[0].Id)

			connection := storage.GetClientConnection(npid)

			if connection == nil {
				return "Not connected"
			}

			connection.IsUnclean = true

			if connection.ServerId != 0 {
				err = utils.KickUser(connection.ServerId, npid, 10001)
				if err != nil {
					return "Cannot kick user"
				}

				return "User marked as unclean and kicked"
			} else {
				// ehmehgehrd spam with packets
				for _, server := range storage.Servers {
					if server != nil && server.Valid {
						utils.KickUser(server.Npid, npid, 10001)
					}
				}
			}

			return "User not connected to a server; marked as unclean"
		})*/

	// GET /authenticate - remauth replacement
	m.Post("/authenticate", func(r *http.Request) string {
		// Read all data from the body into a variable
		// It might be insecure and cause high memory usage if anyone exploits it
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			logger.Error(err)
			return GenerateResponse(false, "Error while reading from POST body", 0, "Anonymous", "*****@*****.**", "0")
		}

		// As specified in the client, "username" (email in our usecase) and password are seperated by &&
		parts := strings.Split(string(body), "&&")

		// There MUST be two parts or the program will panic!
		if len(parts) != 2 {
			return GenerateResponse(false, "Invalid request", 0, "Anonymous", "*****@*****.**", "0")
		}

		// Trim newlines from the email and password
		// In original implementation \0 was also removed, but it seems that golang won't compile a const with \0
		email := strings.Trim(parts[0], trim_cutset)
		password := strings.Trim(parts[1], trim_cutset)

		// Not sure if it's performant, but that's just like the example in Jet documentation
		// This struct is used for query result mapping
		var rows []*struct {
			Id       int
			Username string
			Password string
			Rank     int
			Email    string
			Reason   string
			Expires  time.Time
		}

		// The larger query seems to be faster than two seperate ones
		// Might be because Jet's mapper uses reflection which isn't that fast
		err = environment.Env.Database.Query(`
SELECT
	u.id AS id,
	u.username AS username,
	u.password AS password,
	u.rank_id AS rank,
	u.email AS email,
	b.reason_user AS reason,
	b.expires AS expires
FROM 
	misago_user AS u
LEFT OUTER JOIN
	misago_ban AS b
ON
	(b.test = 0 AND b.ban = u.username OR b.ban = u.email)
OR
	(b.test = 1 AND b.ban = u.username)
OR
	b.ban = u.email	
WHERE
	u.email = ?`, email).Rows(&rows)

		// Probably MySQL went down
		if err != nil {
			logger.Error(err)
			return GenerateResponse(false, "Database error", 0, "Anonymous", "*****@*****.**", "0")
		}

		// There's no email like that!
		if len(rows) != 1 {
			return GenerateResponse(false, "Wrong email or password", 0, "Anonymous", "*****@*****.**", "0")
		}

		// Put it here, because we already know if the user is banned.
		// TODO: Escape #s, either here or in the GenerateResponse function
		if rows[0].Reason != "" && rows[0].Expires.After(time.Now()) {
			return GenerateResponse(
				false,
				"Your account is banned. Reason: "+strings.Trim(rows[0].Reason, "\n\r#"),
				0,
				"Anonymous",
				"*****@*****.**",
				"0",
			)
		}

		// I'm too lazy to add the original Django hash implementation. Some people will have to login onto forums,
		// so Django's auth module can rehash their passwords.
		if rows[0].Password[:7] != "bcrypt$" {
			return GenerateResponse(
				false,
				"Your account is not compatible with the client. Please log in onto forums.",
				0,
				"Anonymous",
				"*****@*****.**",
				"0",
			)
		}

		// I wonder if the bcrypt password checker is timing-attack proof
		err = bcrypt.CompareHashAndPassword([]byte(rows[0].Password[7:]), []byte(password))
		if err != nil {
			return GenerateResponse(false, "Wrong email or password", 0, "Anonymous", "*****@*****.**", "0")
		}

		// Unidecode the username
		username := unidecode.Unidecode(rows[0].Username)

		token := uniuri.NewLen(20)
		existing, err := environment.Env.Redis.Keys("session:" + strconv.Itoa(rows[0].Id) + ":*").Result()
		if err != nil {
			logger.Warning(err)
			return GenerateResponse(false, "Error while accessing session service", 0, "Anonymous", "*****@*****.**", "0")
		}

		if len(existing) > 0 {
			removed, err := environment.Env.Redis.Del(existing...).Result()
			if err != nil {
				logger.Warning(err)
				return GenerateResponse(false, "Error while accessing session service", 0, "Anonymous", "*****@*****.**", "0")
			}

			logger.Debugf("Removed %d old sessions while authenticating %s", removed, username)
		}

		status, err := environment.Env.Redis.Set("session:"+strconv.Itoa(rows[0].Id)+":"+token, structs.StripPort(r.RemoteAddr)+";"+strconv.Itoa(rows[0].Rank)+";"+username).Result()
		if err != nil {
			logger.Warning(err)
			return GenerateResponse(false, "Error while accessing session service", 0, "Anonymous", "*****@*****.**", "0")
		}

		if status != "OK" {
			logger.Warning("Authentication of %s failed. Redis returned %s on set.", username, status)
			return GenerateResponse(false, "Error while accessing session service", 0, "Anonymous", "*****@*****.**", "0")
		}

		// Return a "Success response"
		return GenerateResponse(
			true,
			"Success",
			rows[0].Id,
			strings.Trim(username, "\n\r#"),
			rows[0].Email,
			strconv.Itoa(rows[0].Id)+":"+token,
		)
	})

	// Standard HTTP serving method
	logger.Infof("Serving auth server on %s", environment.Env.Config.HTTP.BindingAddress)
	http.ListenAndServe(environment.Env.Config.HTTP.BindingAddress, m)
}
示例#8
0
//createSlug makes url slug out of string
func createSlug(s string) string {
	s = strings.ToLower(unidecode.Unidecode(s))                     //transliterate if it is not in english
	s = regexp.MustCompile("[^a-z0-9\\s]+").ReplaceAllString(s, "") //spaces
	s = regexp.MustCompile("\\s+").ReplaceAllString(s, "-")         //spaces
	return s
}
示例#9
0
// Replaces non-ASCII characters with an ASCII approximation, or if none
// Transliterate("Ærøskøbing") => "AEroskobing"
// Rails documentation: http://api.rubyonrails.org/classes/ActiveSupport/Inflector.html#method-i-transliterate
func Transliterate(str string) string {
	return unidecode.Unidecode(str)
}