// NewTransientFilesystem returns a new TransientFilesystem. Expects an
// absolute path to the location on disk. Returns an error if the path cannot
// be created, or if a file already exists in that location.
// If create is true, then the directory will be created if it does not yet exist.
func NewTransientFilesystem(absPath string, create bool) (*TransientFilesystem, error) {
if !filepath.IsAbs(absPath) {
return nil, ErrPathNotAbsolute
}
fi, err := os.Stat(absPath)
// short circuit for create
if create && os.IsNotExist(err) {
if err = os.MkdirAll(absPath, 0755); err != nil {
return nil, err
}
fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath}
if err = fs.Sync(); err != nil {
return nil, err
}
return fs, nil
}
if err != nil {
return nil, fmt.Errorf("Unable to stat path %s", absPath)
}
if !fi.IsDir() {
return nil, fmt.Errorf("Specified path %s is not a directory", absPath)
}
fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath}
if err = fs.Sync(); err != nil {
return nil, err
}
return fs, nil
}
// NewLocationMuxer returns a new LocationMuxer for the given Locations
func NewLocationMuxer(locations []*types.Location) (*LocationMuxer, error) {
lm := new(LocationMuxer)
lm.locationTrie = patricia.NewTrie()
for _, location := range locations {
switch {
case isLocationType(location, none):
lm.locationTrie.Set([]byte(location.Name), location)
case isLocationType(location, caseInsensitiveRegular):
if err := lm.addRegexForLocation(fmt.Sprintf(`(?i)%s`, location.Name[len(caseInsensitiveRegular):]), location); err != nil {
return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err)
}
case isLocationType(location, caseSensitiveRegular):
if err := lm.addRegexForLocation(location.Name[len(caseSensitiveRegular):], location); err != nil {
return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err)
}
case isLocationType(location, exact):
lm.locationTrie.Set([]byte(location.Name[len(exact)-1:]), location)
case isLocationType(location, bestNonRegular):
lm.locationTrie.Set([]byte(location.Name[len(bestNonRegular)-1:]), location)
default:
return nil, fmt.Errorf("Location %s is not parsable", location)
}
}
return lm, nil
}
// ValidateProto returns an error if the given proto has problems
// that would cause InitFromProto to fail.
func ValidateProto(config *tableaclpb.Config) (err error) {
t := patricia.NewTrie()
for _, group := range config.TableGroups {
for _, name := range group.TableNamesOrPrefixes {
var prefix patricia.Prefix
if strings.HasSuffix(name, "%") {
prefix = []byte(strings.TrimSuffix(name, "%"))
} else {
prefix = []byte(name + "\000")
}
if bytes.Contains(prefix, []byte("%")) {
return fmt.Errorf("got: %s, '%%' means this entry is a prefix and should not appear in the middle of name or prefix", name)
}
overlapVisitor := func(_ patricia.Prefix, item patricia.Item) error {
return fmt.Errorf("conflicting entries: %q overlaps with %q", name, item)
}
if err := t.VisitSubtree(prefix, overlapVisitor); err != nil {
return err
}
if err := t.VisitPrefixes(prefix, overlapVisitor); err != nil {
return err
}
t.Insert(prefix, name)
}
}
return nil
}
func main() {
key := flag.String("key", "", "key in dot notation")
version := flag.Bool("v", false, "show version and exit")
flag.Parse()
if *version {
fmt.Println(Version)
os.Exit(0)
}
if *key == "" {
log.Fatal("key is required")
}
var reader *bufio.Reader
if flag.NArg() < 1 {
reader = bufio.NewReader(os.Stdin)
} else {
file, err := os.Open(flag.Arg(0))
if err != nil {
log.Fatal(err)
}
defer file.Close()
reader = bufio.NewReader(file)
}
trie := patricia.NewTrie()
for {
line, err := reader.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
doc := make(map[string]interface{})
err = json.Unmarshal([]byte(line), &doc)
if err != nil {
log.Fatal(err)
}
val, err := StringValue(*key, doc)
if err != nil {
log.Fatal(err)
}
if trie.Match(patricia.Prefix(val)) {
log.Println("SKIP")
continue
}
b, err := json.Marshal(doc)
if err != nil {
log.Fatal(err)
}
trie.Insert(patricia.Prefix(val), 1)
fmt.Println(string(b))
}
}
// Constructor function for Exchange, nothing special here.
func New() *Exchange {
mu := new(sync.RWMutex)
return &Exchange{
trie: patricia.NewTrie(),
topicForHandle: make(map[Handle]Topic),
mu: mu,
cond: sync.NewCond(mu),
}
}
func NewTruncIndex(ids []string) (idx *TruncIndex) {
idx = &TruncIndex{
ids: make(map[string]struct{}),
trie: patricia.NewTrie(),
}
for _, id := range ids {
idx.addId(id)
}
return
}
func LoadDictionaries() {
var newDictionaryMap = make(map[string]*patricia.Trie)
var itemMap = ImportDictionaries()
numPrefixes := 0
numSuggestions := 0
numDictionaries := 0
for dictionaryName, suggestItems := range itemMap {
numDictionaries++
log.Print("Loading dictionary " + dictionaryName)
// First see if the trie already exists
var trie, ok = newDictionaryMap[dictionaryName]
if !ok {
trie = patricia.NewTrie()
}
// Great, we have a trie, now let's see if prefixes for the
// suggestItems exist in the trie
for _, suggestItem := range suggestItems {
numSuggestions++
//Tokenize the suggested term by whitespace. Each token will become a prefix in the trie
var tokens = strings.Fields(suggestItem.Term)
tokens = append(tokens, suggestItem.Term)
for _, token := range tokens {
numPrefixes++
//TODO: use ascii folding
lowerToken := strings.ToLower(token)
// The values in the trie are sorted sets of SuggestItems
trieItem := trie.Get([]byte(lowerToken))
if trieItem != nil {
suggestItemSet := trieItem.(treeset.Set)
//If the set already exists, add the new suggestion to the set
suggestItemSet.Add(suggestItem)
} else {
// Otherwise create a new set, add the SuggestItem, and insert it into
// the trie using the lowercase token for the prefix
suggestItemSet := treeset.NewWith(models.SuggestItemComparator)
// log.Printf("Inserting suggestion item %s (%s)", lowerToken, suggestItem.Term)
suggestItemSet.Add(suggestItem)
trie.Insert(patricia.Prefix([]byte(lowerToken)), *suggestItemSet)
}
}
}
newDictionaryMap[dictionaryName] = trie
log.Print("Dictionary " + dictionaryName + " loaded")
}
//Atomic swap
DictionaryMap = newDictionaryMap
log.Printf("All dictionaries updated")
}
// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs.
func NewTruncIndex(ids []string) (idx *TruncIndex) {
idx = &TruncIndex{
ids: make(map[string]struct{}),
// Change patricia max prefix per node length,
// because our len(ID) always 64
trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)),
}
for _, id := range ids {
idx.addID(id)
}
return
}
// NewDefaultRouter creates a very basic WAMP router.
func NewNode() Router {
log.Println("Creating new Rabric Node")
trie := patricia.NewTrie()
r := &Realm{URI: "pd"}
trie.Insert(patricia.Prefix("pd"), r)
return &node{
realms: make(map[URI]Realm),
sessionOpenCallbacks: []func(uint, string){},
sessionCloseCallbacks: []func(uint, string){},
root: trie,
}
}
func (_ Test) Patricia() {
e.InfoLog.Println("\nPatricia:")
printItem := func(prefix patricia.Prefix, item patricia.Item) error {
e.InfoLog.Println(string(prefix), item.(uint32))
return nil
}
trie := patricia.NewTrie()
for i := range ExampleWords {
trie.Insert(patricia.Prefix(ExampleWords[i]), ExampleUint32[i])
}
trie.Set(patricia.Prefix("coca cola"), 188)
e.InfoLog.Println("SubTree:")
trie.VisitSubtree(patricia.Prefix("blost"), printItem)
e.InfoLog.Println("Prefixes:")
trie.VisitPrefixes(patricia.Prefix("borte med blesten mega"), printItem)
trie.Delete(patricia.Prefix("coca cola"))
trie.DeleteSubtree(patricia.Prefix("blost"))
e.InfoLog.Println("What is left:")
trie.Visit(printItem)
}
// Sync brings the in-memory struct and underlying file system into sync.
// Walk the entire tree and recreate it from scratch.
func (fs *TransientFilesystem) Sync() error {
newTrie := trie.NewTrie()
err := filepath.Walk(fs.RootPath(), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
fi, err := os.Stat(path)
if err != nil {
return err
}
if !newTrie.Insert(trie.Prefix(path), fi) {
return fmt.Errorf("Path %s already exists in tree", path)
}
return nil
})
if err != nil {
return err
}
fs.lock.Lock()
defer fs.lock.Unlock()
fs.trie = newTrie
return nil
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstTrie := patricia.NewTrie()
secondTrie := patricia.NewTrie()
wg := new(sync.WaitGroup)
type urlAttr struct {
Size int64
Type os.FileMode
}
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for firstContentCh := range firstClnt.List(true) {
if firstContentCh.Err != nil {
ch <- DiffMessage{
Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
}
}(ch)
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for secondContentCh := range secondClnt.List(true) {
if secondContentCh.Err != nil {
ch <- DiffMessage{
Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
}
return
}
secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
}
}(ch)
doneCh := make(chan struct{})
defer close(doneCh)
go func(doneCh <-chan struct{}) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
wg.Wait()
doneCh <- struct{}{}
if !globalQuietFlag && !globalJSONFlag {
console.Printf("%c[2K\n", 27)
console.Printf("%c[A", 27)
}
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
firstTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
firstURL := firstURLDelimited + matchName
secondURL := secondURLDelimited + matchName
if !secondTrie.Match(patricia.Prefix(matchName)) {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
} else {
firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)
if firstURLAttr.Type.IsRegular() {
if !secondURLAttr.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Type.IsDir() {
if !secondURLAttr.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Size != secondURLAttr.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
}
}
}
func initPrefixes() *gotrie.Trie {
prefixes := gotrie.NewTrie()
prefixes.Insert(gotrie.Prefix("1403"), "CA")
prefixes.Insert(gotrie.Prefix("1587"), "CA")
prefixes.Insert(gotrie.Prefix("1780"), "CA")
prefixes.Insert(gotrie.Prefix("1825"), "CA")
prefixes.Insert(gotrie.Prefix("1236"), "CA")
prefixes.Insert(gotrie.Prefix("1250"), "CA")
prefixes.Insert(gotrie.Prefix("1604"), "CA")
prefixes.Insert(gotrie.Prefix("1672"), "CA")
prefixes.Insert(gotrie.Prefix("1778"), "CA")
prefixes.Insert(gotrie.Prefix("1204"), "CA")
prefixes.Insert(gotrie.Prefix("1431"), "CA")
prefixes.Insert(gotrie.Prefix("1506"), "CA")
prefixes.Insert(gotrie.Prefix("1709"), "CA")
prefixes.Insert(gotrie.Prefix("1782"), "CA")
prefixes.Insert(gotrie.Prefix("1902"), "CA")
prefixes.Insert(gotrie.Prefix("1548"), "CA")
prefixes.Insert(gotrie.Prefix("1249"), "CA")
prefixes.Insert(gotrie.Prefix("1289"), "CA")
prefixes.Insert(gotrie.Prefix("1343"), "CA")
prefixes.Insert(gotrie.Prefix("1365"), "CA")
prefixes.Insert(gotrie.Prefix("1387"), "CA")
prefixes.Insert(gotrie.Prefix("1416"), "CA")
prefixes.Insert(gotrie.Prefix("1437"), "CA")
prefixes.Insert(gotrie.Prefix("1519"), "CA")
prefixes.Insert(gotrie.Prefix("1613"), "CA")
prefixes.Insert(gotrie.Prefix("1647"), "CA")
prefixes.Insert(gotrie.Prefix("1705"), "CA")
prefixes.Insert(gotrie.Prefix("1742"), "CA")
prefixes.Insert(gotrie.Prefix("1807"), "CA")
prefixes.Insert(gotrie.Prefix("1905"), "CA")
prefixes.Insert(gotrie.Prefix("1782"), "CA")
prefixes.Insert(gotrie.Prefix("1902"), "CA")
prefixes.Insert(gotrie.Prefix("1418"), "CA")
prefixes.Insert(gotrie.Prefix("1438"), "CA")
prefixes.Insert(gotrie.Prefix("1450"), "CA")
prefixes.Insert(gotrie.Prefix("1514"), "CA")
prefixes.Insert(gotrie.Prefix("1579"), "CA")
prefixes.Insert(gotrie.Prefix("1581"), "CA")
prefixes.Insert(gotrie.Prefix("1819"), "CA")
prefixes.Insert(gotrie.Prefix("1873"), "CA")
prefixes.Insert(gotrie.Prefix("1306"), "CA")
prefixes.Insert(gotrie.Prefix("1639"), "CA")
prefixes.Insert(gotrie.Prefix("1867"), "CA")
prefixes.Insert(gotrie.Prefix("1"), "US")
prefixes.Insert(gotrie.Prefix("1242"), "BS")
prefixes.Insert(gotrie.Prefix("1246"), "BB")
prefixes.Insert(gotrie.Prefix("1264"), "AI")
prefixes.Insert(gotrie.Prefix("1268"), "AG")
prefixes.Insert(gotrie.Prefix("1284"), "VG")
prefixes.Insert(gotrie.Prefix("1340"), "VI")
prefixes.Insert(gotrie.Prefix("1345"), "KY")
prefixes.Insert(gotrie.Prefix("1441"), "BM")
prefixes.Insert(gotrie.Prefix("1473"), "GD")
prefixes.Insert(gotrie.Prefix("1649"), "TC")
prefixes.Insert(gotrie.Prefix("1664"), "MS")
prefixes.Insert(gotrie.Prefix("1670"), "MP")
prefixes.Insert(gotrie.Prefix("1671"), "GU")
prefixes.Insert(gotrie.Prefix("1684"), "AS")
prefixes.Insert(gotrie.Prefix("1721"), "SX")
prefixes.Insert(gotrie.Prefix("1758"), "LC")
prefixes.Insert(gotrie.Prefix("1767"), "DM")
prefixes.Insert(gotrie.Prefix("1784"), "VC")
prefixes.Insert(gotrie.Prefix("1787"), "PR")
prefixes.Insert(gotrie.Prefix("1809"), "DO")
prefixes.Insert(gotrie.Prefix("1829"), "DO")
prefixes.Insert(gotrie.Prefix("1849"), "DO")
prefixes.Insert(gotrie.Prefix("1868"), "TT")
prefixes.Insert(gotrie.Prefix("1869"), "KN")
prefixes.Insert(gotrie.Prefix("1876"), "JM")
prefixes.Insert(gotrie.Prefix("1939"), "PR")
prefixes.Insert(gotrie.Prefix("20"), "EG")
prefixes.Insert(gotrie.Prefix("211"), "SS")
prefixes.Insert(gotrie.Prefix("212"), "MA")
prefixes.Insert(gotrie.Prefix("213"), "DZ")
prefixes.Insert(gotrie.Prefix("216"), "TN")
prefixes.Insert(gotrie.Prefix("218"), "LY")
prefixes.Insert(gotrie.Prefix("220"), "GM")
prefixes.Insert(gotrie.Prefix("221"), "SN")
prefixes.Insert(gotrie.Prefix("222"), "MR")
prefixes.Insert(gotrie.Prefix("223"), "ML")
prefixes.Insert(gotrie.Prefix("224"), "GN")
prefixes.Insert(gotrie.Prefix("225"), "CI")
prefixes.Insert(gotrie.Prefix("226"), "BF")
prefixes.Insert(gotrie.Prefix("227"), "NE")
prefixes.Insert(gotrie.Prefix("228"), "TG")
prefixes.Insert(gotrie.Prefix("229"), "BJ")
prefixes.Insert(gotrie.Prefix("230"), "MU")
prefixes.Insert(gotrie.Prefix("231"), "LR")
prefixes.Insert(gotrie.Prefix("232"), "SL")
prefixes.Insert(gotrie.Prefix("233"), "GH")
prefixes.Insert(gotrie.Prefix("234"), "NG")
prefixes.Insert(gotrie.Prefix("235"), "TD")
prefixes.Insert(gotrie.Prefix("236"), "CF")
prefixes.Insert(gotrie.Prefix("237"), "CM")
prefixes.Insert(gotrie.Prefix("238"), "CV")
prefixes.Insert(gotrie.Prefix("239"), "ST")
prefixes.Insert(gotrie.Prefix("240"), "GQ")
prefixes.Insert(gotrie.Prefix("241"), "GA")
prefixes.Insert(gotrie.Prefix("242"), "CG")
prefixes.Insert(gotrie.Prefix("243"), "CD")
prefixes.Insert(gotrie.Prefix("244"), "AO")
prefixes.Insert(gotrie.Prefix("245"), "GW")
prefixes.Insert(gotrie.Prefix("246"), "IO")
prefixes.Insert(gotrie.Prefix("247"), "SH")
prefixes.Insert(gotrie.Prefix("248"), "SC")
prefixes.Insert(gotrie.Prefix("249"), "SD")
prefixes.Insert(gotrie.Prefix("250"), "RW")
prefixes.Insert(gotrie.Prefix("251"), "ET")
prefixes.Insert(gotrie.Prefix("252"), "SO")
prefixes.Insert(gotrie.Prefix("253"), "DJ")
prefixes.Insert(gotrie.Prefix("254"), "KE")
prefixes.Insert(gotrie.Prefix("255"), "TZ")
prefixes.Insert(gotrie.Prefix("256"), "UG")
prefixes.Insert(gotrie.Prefix("257"), "BI")
prefixes.Insert(gotrie.Prefix("258"), "MZ")
prefixes.Insert(gotrie.Prefix("260"), "ZM")
prefixes.Insert(gotrie.Prefix("261"), "MG")
prefixes.Insert(gotrie.Prefix("262"), "RE")
prefixes.Insert(gotrie.Prefix("263"), "ZW")
prefixes.Insert(gotrie.Prefix("264"), "NA")
prefixes.Insert(gotrie.Prefix("265"), "MW")
prefixes.Insert(gotrie.Prefix("266"), "LS")
prefixes.Insert(gotrie.Prefix("267"), "BW")
prefixes.Insert(gotrie.Prefix("268"), "SZ")
prefixes.Insert(gotrie.Prefix("269"), "KM")
prefixes.Insert(gotrie.Prefix("27"), "ZA")
prefixes.Insert(gotrie.Prefix("290"), "SH")
prefixes.Insert(gotrie.Prefix("291"), "ER")
prefixes.Insert(gotrie.Prefix("297"), "AW")
prefixes.Insert(gotrie.Prefix("298"), "FO")
prefixes.Insert(gotrie.Prefix("299"), "GL")
prefixes.Insert(gotrie.Prefix("30"), "GR")
prefixes.Insert(gotrie.Prefix("31"), "NL")
prefixes.Insert(gotrie.Prefix("32"), "BE")
prefixes.Insert(gotrie.Prefix("33"), "FR")
prefixes.Insert(gotrie.Prefix("34"), "ES")
prefixes.Insert(gotrie.Prefix("350"), "GI")
prefixes.Insert(gotrie.Prefix("351"), "PT")
prefixes.Insert(gotrie.Prefix("352"), "LU")
prefixes.Insert(gotrie.Prefix("353"), "IE")
prefixes.Insert(gotrie.Prefix("354"), "IS")
prefixes.Insert(gotrie.Prefix("355"), "AL")
prefixes.Insert(gotrie.Prefix("356"), "MT")
prefixes.Insert(gotrie.Prefix("357"), "CY")
prefixes.Insert(gotrie.Prefix("358"), "FI")
prefixes.Insert(gotrie.Prefix("359"), "BG")
prefixes.Insert(gotrie.Prefix("36"), "HU")
prefixes.Insert(gotrie.Prefix("370"), "LT")
prefixes.Insert(gotrie.Prefix("371"), "LV")
prefixes.Insert(gotrie.Prefix("372"), "EE")
prefixes.Insert(gotrie.Prefix("373"), "MD")
prefixes.Insert(gotrie.Prefix("374"), "AM")
prefixes.Insert(gotrie.Prefix("375"), "BY")
prefixes.Insert(gotrie.Prefix("376"), "AD")
prefixes.Insert(gotrie.Prefix("377"), "MC")
prefixes.Insert(gotrie.Prefix("378"), "SM")
prefixes.Insert(gotrie.Prefix("379"), "VA")
prefixes.Insert(gotrie.Prefix("380"), "UA")
prefixes.Insert(gotrie.Prefix("381"), "RS")
prefixes.Insert(gotrie.Prefix("381"), "XK")
prefixes.Insert(gotrie.Prefix("382"), "ME")
prefixes.Insert(gotrie.Prefix("385"), "HR")
prefixes.Insert(gotrie.Prefix("386"), "SI")
prefixes.Insert(gotrie.Prefix("386"), "XK")
prefixes.Insert(gotrie.Prefix("387"), "BA")
prefixes.Insert(gotrie.Prefix("389"), "MK")
prefixes.Insert(gotrie.Prefix("39"), "IT")
prefixes.Insert(gotrie.Prefix("39066"), "VA")
prefixes.Insert(gotrie.Prefix("40"), "RO")
prefixes.Insert(gotrie.Prefix("41"), "CH")
prefixes.Insert(gotrie.Prefix("420"), "CZ")
prefixes.Insert(gotrie.Prefix("421"), "SK")
prefixes.Insert(gotrie.Prefix("423"), "LI")
prefixes.Insert(gotrie.Prefix("43"), "AT")
prefixes.Insert(gotrie.Prefix("44"), "GB")
prefixes.Insert(gotrie.Prefix("45"), "DK")
prefixes.Insert(gotrie.Prefix("46"), "SE")
prefixes.Insert(gotrie.Prefix("47"), "NO")
prefixes.Insert(gotrie.Prefix("4779"), "SJ")
prefixes.Insert(gotrie.Prefix("48"), "PL")
prefixes.Insert(gotrie.Prefix("49"), "DE")
prefixes.Insert(gotrie.Prefix("500"), "FK")
prefixes.Insert(gotrie.Prefix("501"), "BZ")
prefixes.Insert(gotrie.Prefix("502"), "GT")
prefixes.Insert(gotrie.Prefix("503"), "SV")
prefixes.Insert(gotrie.Prefix("504"), "HN")
prefixes.Insert(gotrie.Prefix("505"), "NI")
prefixes.Insert(gotrie.Prefix("506"), "CR")
prefixes.Insert(gotrie.Prefix("507"), "PA")
prefixes.Insert(gotrie.Prefix("508"), "PM")
prefixes.Insert(gotrie.Prefix("509"), "HT")
prefixes.Insert(gotrie.Prefix("51"), "PE")
prefixes.Insert(gotrie.Prefix("52"), "MX")
prefixes.Insert(gotrie.Prefix("53"), "CU")
prefixes.Insert(gotrie.Prefix("54"), "AR")
prefixes.Insert(gotrie.Prefix("55"), "BR")
prefixes.Insert(gotrie.Prefix("56"), "CL")
prefixes.Insert(gotrie.Prefix("57"), "CO")
prefixes.Insert(gotrie.Prefix("58"), "VE")
prefixes.Insert(gotrie.Prefix("590"), "GP")
prefixes.Insert(gotrie.Prefix("591"), "BO")
prefixes.Insert(gotrie.Prefix("592"), "GY")
prefixes.Insert(gotrie.Prefix("593"), "EC")
prefixes.Insert(gotrie.Prefix("594"), "GF")
prefixes.Insert(gotrie.Prefix("595"), "PY")
prefixes.Insert(gotrie.Prefix("596"), "MQ")
prefixes.Insert(gotrie.Prefix("597"), "SR")
prefixes.Insert(gotrie.Prefix("598"), "UY")
prefixes.Insert(gotrie.Prefix("5997"), "BQ")
prefixes.Insert(gotrie.Prefix("5999"), "CW")
prefixes.Insert(gotrie.Prefix("60"), "MY")
prefixes.Insert(gotrie.Prefix("61"), "AU")
prefixes.Insert(gotrie.Prefix("62"), "ID")
prefixes.Insert(gotrie.Prefix("63"), "PH")
prefixes.Insert(gotrie.Prefix("64"), "NZ")
prefixes.Insert(gotrie.Prefix("65"), "SG")
prefixes.Insert(gotrie.Prefix("66"), "TH")
prefixes.Insert(gotrie.Prefix("670"), "TL")
prefixes.Insert(gotrie.Prefix("672"), "NF")
prefixes.Insert(gotrie.Prefix("673"), "BN")
prefixes.Insert(gotrie.Prefix("674"), "NR")
prefixes.Insert(gotrie.Prefix("675"), "PG")
prefixes.Insert(gotrie.Prefix("676"), "TO")
prefixes.Insert(gotrie.Prefix("677"), "SB")
prefixes.Insert(gotrie.Prefix("678"), "VU")
prefixes.Insert(gotrie.Prefix("679"), "FJ")
prefixes.Insert(gotrie.Prefix("680"), "PW")
prefixes.Insert(gotrie.Prefix("681"), "WF")
prefixes.Insert(gotrie.Prefix("682"), "CK")
prefixes.Insert(gotrie.Prefix("683"), "NU")
prefixes.Insert(gotrie.Prefix("685"), "WS")
prefixes.Insert(gotrie.Prefix("686"), "KI")
prefixes.Insert(gotrie.Prefix("687"), "NC")
prefixes.Insert(gotrie.Prefix("688"), "TV")
prefixes.Insert(gotrie.Prefix("689"), "PF")
prefixes.Insert(gotrie.Prefix("690"), "TK")
prefixes.Insert(gotrie.Prefix("691"), "FM")
prefixes.Insert(gotrie.Prefix("692"), "MH")
prefixes.Insert(gotrie.Prefix("7"), "RU")
prefixes.Insert(gotrie.Prefix("76"), "KZ")
prefixes.Insert(gotrie.Prefix("77"), "KZ")
prefixes.Insert(gotrie.Prefix("81"), "JP")
prefixes.Insert(gotrie.Prefix("82"), "KR")
prefixes.Insert(gotrie.Prefix("84"), "VN")
prefixes.Insert(gotrie.Prefix("850"), "KP")
prefixes.Insert(gotrie.Prefix("852"), "HK")
prefixes.Insert(gotrie.Prefix("853"), "MO")
prefixes.Insert(gotrie.Prefix("855"), "KH")
prefixes.Insert(gotrie.Prefix("856"), "LA")
prefixes.Insert(gotrie.Prefix("86"), "CN")
prefixes.Insert(gotrie.Prefix("880"), "BD")
prefixes.Insert(gotrie.Prefix("886"), "TW")
prefixes.Insert(gotrie.Prefix("90"), "TR")
prefixes.Insert(gotrie.Prefix("91"), "IN")
prefixes.Insert(gotrie.Prefix("92"), "PK")
prefixes.Insert(gotrie.Prefix("93"), "AF")
prefixes.Insert(gotrie.Prefix("94"), "LK")
prefixes.Insert(gotrie.Prefix("95"), "MM")
prefixes.Insert(gotrie.Prefix("960"), "MV")
prefixes.Insert(gotrie.Prefix("961"), "LB")
prefixes.Insert(gotrie.Prefix("962"), "JO")
prefixes.Insert(gotrie.Prefix("963"), "SY")
prefixes.Insert(gotrie.Prefix("964"), "IQ")
prefixes.Insert(gotrie.Prefix("965"), "KW")
prefixes.Insert(gotrie.Prefix("966"), "SA")
prefixes.Insert(gotrie.Prefix("967"), "YE")
prefixes.Insert(gotrie.Prefix("968"), "OM")
prefixes.Insert(gotrie.Prefix("970"), "PS")
prefixes.Insert(gotrie.Prefix("971"), "AE")
prefixes.Insert(gotrie.Prefix("972"), "IL")
prefixes.Insert(gotrie.Prefix("973"), "BH")
prefixes.Insert(gotrie.Prefix("974"), "QA")
prefixes.Insert(gotrie.Prefix("975"), "BT")
prefixes.Insert(gotrie.Prefix("976"), "MN")
prefixes.Insert(gotrie.Prefix("977"), "NP")
prefixes.Insert(gotrie.Prefix("98"), "IR")
prefixes.Insert(gotrie.Prefix("992"), "TJ")
prefixes.Insert(gotrie.Prefix("993"), "TM")
prefixes.Insert(gotrie.Prefix("994"), "AZ")
prefixes.Insert(gotrie.Prefix("995"), "GE")
prefixes.Insert(gotrie.Prefix("996"), "KG")
prefixes.Insert(gotrie.Prefix("998"), "UZ")
return prefixes
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
sourceTrie := patricia.NewTrie()
targetTries := make([]*patricia.Trie, len(targetClnts))
wg := new(sync.WaitGroup)
wg.Add(1)
go func() {
defer wg.Done()
for sourceContentCh := range sourceClnt.List(true) {
if sourceContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
return
}
if sourceContentCh.Content.Type.IsRegular() {
sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
}
}
}()
for i, targetClnt := range targetClnts {
wg.Add(1)
go func(i int, targetClnt client.Client) {
defer wg.Done()
targetTrie := patricia.NewTrie()
for targetContentCh := range targetClnt.List(true) {
if targetContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
return
}
if targetContentCh.Content.Type.IsRegular() {
targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
}
}
targetTries[i] = targetTrie
}(i, targetClnt)
}
wg.Wait()
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
sourceTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
sourceContent := new(client.Content)
var targetContents []*client.Content
for i, targetTrie := range targetTries {
if !targetTrie.Match(patricia.Prefix(matchName)) {
sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
string(sourceClnt.URL().Separator))+1]
newTargetURLParse := *targetClnts[i].URL()
newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
sourceContent.Name = sourceURLDelimited + matchName
targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
}
}
mirrorURLsCh <- mirrorURLs{
SourceContent: sourceContent,
TargetContents: targetContents,
}
}
}()
return mirrorURLsCh
}
