// NewTransientFilesystem returns a new TransientFilesystem. Expects an // absolute path to the location on disk. Returns an error if the path cannot // be created, or if a file already exists in that location. // If create is true, then the directory will be created if it does not yet exist. func NewTransientFilesystem(absPath string, create bool) (*TransientFilesystem, error) { if !filepath.IsAbs(absPath) { return nil, ErrPathNotAbsolute } fi, err := os.Stat(absPath) // short circuit for create if create && os.IsNotExist(err) { if err = os.MkdirAll(absPath, 0755); err != nil { return nil, err } fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath} if err = fs.Sync(); err != nil { return nil, err } return fs, nil } if err != nil { return nil, fmt.Errorf("Unable to stat path %s", absPath) } if !fi.IsDir() { return nil, fmt.Errorf("Specified path %s is not a directory", absPath) } fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath} if err = fs.Sync(); err != nil { return nil, err } return fs, nil }
// NewLocationMuxer returns a new LocationMuxer for the given Locations func NewLocationMuxer(locations []*types.Location) (*LocationMuxer, error) { lm := new(LocationMuxer) lm.locationTrie = patricia.NewTrie() for _, location := range locations { switch { case isLocationType(location, none): lm.locationTrie.Set([]byte(location.Name), location) case isLocationType(location, caseInsensitiveRegular): if err := lm.addRegexForLocation(fmt.Sprintf(`(?i)%s`, location.Name[len(caseInsensitiveRegular):]), location); err != nil { return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err) } case isLocationType(location, caseSensitiveRegular): if err := lm.addRegexForLocation(location.Name[len(caseSensitiveRegular):], location); err != nil { return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err) } case isLocationType(location, exact): lm.locationTrie.Set([]byte(location.Name[len(exact)-1:]), location) case isLocationType(location, bestNonRegular): lm.locationTrie.Set([]byte(location.Name[len(bestNonRegular)-1:]), location) default: return nil, fmt.Errorf("Location %s is not parsable", location) } } return lm, nil }
// ValidateProto returns an error if the given proto has problems // that would cause InitFromProto to fail. func ValidateProto(config *tableaclpb.Config) (err error) { t := patricia.NewTrie() for _, group := range config.TableGroups { for _, name := range group.TableNamesOrPrefixes { var prefix patricia.Prefix if strings.HasSuffix(name, "%") { prefix = []byte(strings.TrimSuffix(name, "%")) } else { prefix = []byte(name + "\000") } if bytes.Contains(prefix, []byte("%")) { return fmt.Errorf("got: %s, '%%' means this entry is a prefix and should not appear in the middle of name or prefix", name) } overlapVisitor := func(_ patricia.Prefix, item patricia.Item) error { return fmt.Errorf("conflicting entries: %q overlaps with %q", name, item) } if err := t.VisitSubtree(prefix, overlapVisitor); err != nil { return err } if err := t.VisitPrefixes(prefix, overlapVisitor); err != nil { return err } t.Insert(prefix, name) } } return nil }
func main() { key := flag.String("key", "", "key in dot notation") version := flag.Bool("v", false, "show version and exit") flag.Parse() if *version { fmt.Println(Version) os.Exit(0) } if *key == "" { log.Fatal("key is required") } var reader *bufio.Reader if flag.NArg() < 1 { reader = bufio.NewReader(os.Stdin) } else { file, err := os.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } defer file.Close() reader = bufio.NewReader(file) } trie := patricia.NewTrie() for { line, err := reader.ReadString('\n') if err == io.EOF { break } if err != nil { log.Fatal(err) } doc := make(map[string]interface{}) err = json.Unmarshal([]byte(line), &doc) if err != nil { log.Fatal(err) } val, err := StringValue(*key, doc) if err != nil { log.Fatal(err) } if trie.Match(patricia.Prefix(val)) { log.Println("SKIP") continue } b, err := json.Marshal(doc) if err != nil { log.Fatal(err) } trie.Insert(patricia.Prefix(val), 1) fmt.Println(string(b)) } }
// Constructor function for Exchange, nothing special here. func New() *Exchange { mu := new(sync.RWMutex) return &Exchange{ trie: patricia.NewTrie(), topicForHandle: make(map[Handle]Topic), mu: mu, cond: sync.NewCond(mu), } }
func NewTruncIndex(ids []string) (idx *TruncIndex) { idx = &TruncIndex{ ids: make(map[string]struct{}), trie: patricia.NewTrie(), } for _, id := range ids { idx.addId(id) } return }
func LoadDictionaries() { var newDictionaryMap = make(map[string]*patricia.Trie) var itemMap = ImportDictionaries() numPrefixes := 0 numSuggestions := 0 numDictionaries := 0 for dictionaryName, suggestItems := range itemMap { numDictionaries++ log.Print("Loading dictionary " + dictionaryName) // First see if the trie already exists var trie, ok = newDictionaryMap[dictionaryName] if !ok { trie = patricia.NewTrie() } // Great, we have a trie, now let's see if prefixes for the // suggestItems exist in the trie for _, suggestItem := range suggestItems { numSuggestions++ //Tokenize the suggested term by whitespace. Each token will become a prefix in the trie var tokens = strings.Fields(suggestItem.Term) tokens = append(tokens, suggestItem.Term) for _, token := range tokens { numPrefixes++ //TODO: use ascii folding lowerToken := strings.ToLower(token) // The values in the trie are sorted sets of SuggestItems trieItem := trie.Get([]byte(lowerToken)) if trieItem != nil { suggestItemSet := trieItem.(treeset.Set) //If the set already exists, add the new suggestion to the set suggestItemSet.Add(suggestItem) } else { // Otherwise create a new set, add the SuggestItem, and insert it into // the trie using the lowercase token for the prefix suggestItemSet := treeset.NewWith(models.SuggestItemComparator) // log.Printf("Inserting suggestion item %s (%s)", lowerToken, suggestItem.Term) suggestItemSet.Add(suggestItem) trie.Insert(patricia.Prefix([]byte(lowerToken)), *suggestItemSet) } } } newDictionaryMap[dictionaryName] = trie log.Print("Dictionary " + dictionaryName + " loaded") } //Atomic swap DictionaryMap = newDictionaryMap log.Printf("All dictionaries updated") }
// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs. func NewTruncIndex(ids []string) (idx *TruncIndex) { idx = &TruncIndex{ ids: make(map[string]struct{}), // Change patricia max prefix per node length, // because our len(ID) always 64 trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)), } for _, id := range ids { idx.addID(id) } return }
// NewDefaultRouter creates a very basic WAMP router. func NewNode() Router { log.Println("Creating new Rabric Node") trie := patricia.NewTrie() r := &Realm{URI: "pd"} trie.Insert(patricia.Prefix("pd"), r) return &node{ realms: make(map[URI]Realm), sessionOpenCallbacks: []func(uint, string){}, sessionCloseCallbacks: []func(uint, string){}, root: trie, } }
func (_ Test) Patricia() { e.InfoLog.Println("\nPatricia:") printItem := func(prefix patricia.Prefix, item patricia.Item) error { e.InfoLog.Println(string(prefix), item.(uint32)) return nil } trie := patricia.NewTrie() for i := range ExampleWords { trie.Insert(patricia.Prefix(ExampleWords[i]), ExampleUint32[i]) } trie.Set(patricia.Prefix("coca cola"), 188) e.InfoLog.Println("SubTree:") trie.VisitSubtree(patricia.Prefix("blost"), printItem) e.InfoLog.Println("Prefixes:") trie.VisitPrefixes(patricia.Prefix("borte med blesten mega"), printItem) trie.Delete(patricia.Prefix("coca cola")) trie.DeleteSubtree(patricia.Prefix("blost")) e.InfoLog.Println("What is left:") trie.Visit(printItem) }
// Sync brings the in-memory struct and underlying file system into sync. // Walk the entire tree and recreate it from scratch. func (fs *TransientFilesystem) Sync() error { newTrie := trie.NewTrie() err := filepath.Walk(fs.RootPath(), func(path string, info os.FileInfo, err error) error { if err != nil { return err } fi, err := os.Stat(path) if err != nil { return err } if !newTrie.Insert(trie.Prefix(path), fi) { return fmt.Errorf("Path %s already exists in tree", path) } return nil }) if err != nil { return err } fs.lock.Lock() defer fs.lock.Unlock() fs.trie = newTrie return nil }
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) { firstTrie := patricia.NewTrie() secondTrie := patricia.NewTrie() wg := new(sync.WaitGroup) type urlAttr struct { Size int64 Type os.FileMode } wg.Add(1) go func(ch chan<- DiffMessage) { defer wg.Done() for firstContentCh := range firstClnt.List(true) { if firstContentCh.Err != nil { ch <- DiffMessage{ Error: firstContentCh.Err.Trace(firstClnt.URL().String()), } return } firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type}) } }(ch) wg.Add(1) go func(ch chan<- DiffMessage) { defer wg.Done() for secondContentCh := range secondClnt.List(true) { if secondContentCh.Err != nil { ch <- DiffMessage{ Error: secondContentCh.Err.Trace(secondClnt.URL().String()), } return } secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type}) } }(ch) doneCh := make(chan struct{}) defer close(doneCh) go func(doneCh <-chan struct{}) { cursorCh := cursorAnimate() for { select { case <-time.Tick(100 * time.Millisecond): if !globalQuietFlag && !globalJSONFlag { console.PrintC("\r" + "Scanning.. " + string(<-cursorCh)) } case <-doneCh: return } } }(doneCh) wg.Wait() doneCh <- struct{}{} if !globalQuietFlag && !globalJSONFlag { console.Printf("%c[2K\n", 27) console.Printf("%c[A", 27) } matchNameCh := make(chan string, 10000) go func(matchNameCh chan<- string) { itemFunc := func(prefix patricia.Prefix, item patricia.Item) error { matchNameCh <- string(prefix) return nil } firstTrie.Visit(itemFunc) defer close(matchNameCh) }(matchNameCh) for matchName := range matchNameCh { firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1] secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1] firstURL := firstURLDelimited + matchName secondURL := secondURLDelimited + matchName if !secondTrie.Match(patricia.Prefix(matchName)) { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "only-in-first", } } else { firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr) secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr) if firstURLAttr.Type.IsRegular() { if !secondURLAttr.Type.IsRegular() { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "type", } continue } } if firstURLAttr.Type.IsDir() { if !secondURLAttr.Type.IsDir() { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "type", } continue } } if firstURLAttr.Size != secondURLAttr.Size { ch <- DiffMessage{ FirstURL: firstURL, SecondURL: secondURL, Diff: "size", } } } } }
func initPrefixes() *gotrie.Trie { prefixes := gotrie.NewTrie() prefixes.Insert(gotrie.Prefix("1403"), "CA") prefixes.Insert(gotrie.Prefix("1587"), "CA") prefixes.Insert(gotrie.Prefix("1780"), "CA") prefixes.Insert(gotrie.Prefix("1825"), "CA") prefixes.Insert(gotrie.Prefix("1236"), "CA") prefixes.Insert(gotrie.Prefix("1250"), "CA") prefixes.Insert(gotrie.Prefix("1604"), "CA") prefixes.Insert(gotrie.Prefix("1672"), "CA") prefixes.Insert(gotrie.Prefix("1778"), "CA") prefixes.Insert(gotrie.Prefix("1204"), "CA") prefixes.Insert(gotrie.Prefix("1431"), "CA") prefixes.Insert(gotrie.Prefix("1506"), "CA") prefixes.Insert(gotrie.Prefix("1709"), "CA") prefixes.Insert(gotrie.Prefix("1782"), "CA") prefixes.Insert(gotrie.Prefix("1902"), "CA") prefixes.Insert(gotrie.Prefix("1548"), "CA") prefixes.Insert(gotrie.Prefix("1249"), "CA") prefixes.Insert(gotrie.Prefix("1289"), "CA") prefixes.Insert(gotrie.Prefix("1343"), "CA") prefixes.Insert(gotrie.Prefix("1365"), "CA") prefixes.Insert(gotrie.Prefix("1387"), "CA") prefixes.Insert(gotrie.Prefix("1416"), "CA") prefixes.Insert(gotrie.Prefix("1437"), "CA") prefixes.Insert(gotrie.Prefix("1519"), "CA") prefixes.Insert(gotrie.Prefix("1613"), "CA") prefixes.Insert(gotrie.Prefix("1647"), "CA") prefixes.Insert(gotrie.Prefix("1705"), "CA") prefixes.Insert(gotrie.Prefix("1742"), "CA") prefixes.Insert(gotrie.Prefix("1807"), "CA") prefixes.Insert(gotrie.Prefix("1905"), "CA") prefixes.Insert(gotrie.Prefix("1782"), "CA") prefixes.Insert(gotrie.Prefix("1902"), "CA") prefixes.Insert(gotrie.Prefix("1418"), "CA") prefixes.Insert(gotrie.Prefix("1438"), "CA") prefixes.Insert(gotrie.Prefix("1450"), "CA") prefixes.Insert(gotrie.Prefix("1514"), "CA") prefixes.Insert(gotrie.Prefix("1579"), "CA") prefixes.Insert(gotrie.Prefix("1581"), "CA") prefixes.Insert(gotrie.Prefix("1819"), "CA") prefixes.Insert(gotrie.Prefix("1873"), "CA") prefixes.Insert(gotrie.Prefix("1306"), "CA") prefixes.Insert(gotrie.Prefix("1639"), "CA") prefixes.Insert(gotrie.Prefix("1867"), "CA") prefixes.Insert(gotrie.Prefix("1"), "US") prefixes.Insert(gotrie.Prefix("1242"), "BS") prefixes.Insert(gotrie.Prefix("1246"), "BB") prefixes.Insert(gotrie.Prefix("1264"), "AI") prefixes.Insert(gotrie.Prefix("1268"), "AG") prefixes.Insert(gotrie.Prefix("1284"), "VG") prefixes.Insert(gotrie.Prefix("1340"), "VI") prefixes.Insert(gotrie.Prefix("1345"), "KY") prefixes.Insert(gotrie.Prefix("1441"), "BM") prefixes.Insert(gotrie.Prefix("1473"), "GD") prefixes.Insert(gotrie.Prefix("1649"), "TC") prefixes.Insert(gotrie.Prefix("1664"), "MS") prefixes.Insert(gotrie.Prefix("1670"), "MP") prefixes.Insert(gotrie.Prefix("1671"), "GU") prefixes.Insert(gotrie.Prefix("1684"), "AS") prefixes.Insert(gotrie.Prefix("1721"), "SX") prefixes.Insert(gotrie.Prefix("1758"), "LC") prefixes.Insert(gotrie.Prefix("1767"), "DM") prefixes.Insert(gotrie.Prefix("1784"), "VC") prefixes.Insert(gotrie.Prefix("1787"), "PR") prefixes.Insert(gotrie.Prefix("1809"), "DO") prefixes.Insert(gotrie.Prefix("1829"), "DO") prefixes.Insert(gotrie.Prefix("1849"), "DO") prefixes.Insert(gotrie.Prefix("1868"), "TT") prefixes.Insert(gotrie.Prefix("1869"), "KN") prefixes.Insert(gotrie.Prefix("1876"), "JM") prefixes.Insert(gotrie.Prefix("1939"), "PR") prefixes.Insert(gotrie.Prefix("20"), "EG") prefixes.Insert(gotrie.Prefix("211"), "SS") prefixes.Insert(gotrie.Prefix("212"), "MA") prefixes.Insert(gotrie.Prefix("213"), "DZ") prefixes.Insert(gotrie.Prefix("216"), "TN") prefixes.Insert(gotrie.Prefix("218"), "LY") prefixes.Insert(gotrie.Prefix("220"), "GM") prefixes.Insert(gotrie.Prefix("221"), "SN") prefixes.Insert(gotrie.Prefix("222"), "MR") prefixes.Insert(gotrie.Prefix("223"), "ML") prefixes.Insert(gotrie.Prefix("224"), "GN") prefixes.Insert(gotrie.Prefix("225"), "CI") prefixes.Insert(gotrie.Prefix("226"), "BF") prefixes.Insert(gotrie.Prefix("227"), "NE") prefixes.Insert(gotrie.Prefix("228"), "TG") prefixes.Insert(gotrie.Prefix("229"), "BJ") prefixes.Insert(gotrie.Prefix("230"), "MU") prefixes.Insert(gotrie.Prefix("231"), "LR") prefixes.Insert(gotrie.Prefix("232"), "SL") prefixes.Insert(gotrie.Prefix("233"), "GH") prefixes.Insert(gotrie.Prefix("234"), "NG") prefixes.Insert(gotrie.Prefix("235"), "TD") prefixes.Insert(gotrie.Prefix("236"), "CF") prefixes.Insert(gotrie.Prefix("237"), "CM") prefixes.Insert(gotrie.Prefix("238"), "CV") prefixes.Insert(gotrie.Prefix("239"), "ST") prefixes.Insert(gotrie.Prefix("240"), "GQ") prefixes.Insert(gotrie.Prefix("241"), "GA") prefixes.Insert(gotrie.Prefix("242"), "CG") prefixes.Insert(gotrie.Prefix("243"), "CD") prefixes.Insert(gotrie.Prefix("244"), "AO") prefixes.Insert(gotrie.Prefix("245"), "GW") prefixes.Insert(gotrie.Prefix("246"), "IO") prefixes.Insert(gotrie.Prefix("247"), "SH") prefixes.Insert(gotrie.Prefix("248"), "SC") prefixes.Insert(gotrie.Prefix("249"), "SD") prefixes.Insert(gotrie.Prefix("250"), "RW") prefixes.Insert(gotrie.Prefix("251"), "ET") prefixes.Insert(gotrie.Prefix("252"), "SO") prefixes.Insert(gotrie.Prefix("253"), "DJ") prefixes.Insert(gotrie.Prefix("254"), "KE") prefixes.Insert(gotrie.Prefix("255"), "TZ") prefixes.Insert(gotrie.Prefix("256"), "UG") prefixes.Insert(gotrie.Prefix("257"), "BI") prefixes.Insert(gotrie.Prefix("258"), "MZ") prefixes.Insert(gotrie.Prefix("260"), "ZM") prefixes.Insert(gotrie.Prefix("261"), "MG") prefixes.Insert(gotrie.Prefix("262"), "RE") prefixes.Insert(gotrie.Prefix("263"), "ZW") prefixes.Insert(gotrie.Prefix("264"), "NA") prefixes.Insert(gotrie.Prefix("265"), "MW") prefixes.Insert(gotrie.Prefix("266"), "LS") prefixes.Insert(gotrie.Prefix("267"), "BW") prefixes.Insert(gotrie.Prefix("268"), "SZ") prefixes.Insert(gotrie.Prefix("269"), "KM") prefixes.Insert(gotrie.Prefix("27"), "ZA") prefixes.Insert(gotrie.Prefix("290"), "SH") prefixes.Insert(gotrie.Prefix("291"), "ER") prefixes.Insert(gotrie.Prefix("297"), "AW") prefixes.Insert(gotrie.Prefix("298"), "FO") prefixes.Insert(gotrie.Prefix("299"), "GL") prefixes.Insert(gotrie.Prefix("30"), "GR") prefixes.Insert(gotrie.Prefix("31"), "NL") prefixes.Insert(gotrie.Prefix("32"), "BE") prefixes.Insert(gotrie.Prefix("33"), "FR") prefixes.Insert(gotrie.Prefix("34"), "ES") prefixes.Insert(gotrie.Prefix("350"), "GI") prefixes.Insert(gotrie.Prefix("351"), "PT") prefixes.Insert(gotrie.Prefix("352"), "LU") prefixes.Insert(gotrie.Prefix("353"), "IE") prefixes.Insert(gotrie.Prefix("354"), "IS") prefixes.Insert(gotrie.Prefix("355"), "AL") prefixes.Insert(gotrie.Prefix("356"), "MT") prefixes.Insert(gotrie.Prefix("357"), "CY") prefixes.Insert(gotrie.Prefix("358"), "FI") prefixes.Insert(gotrie.Prefix("359"), "BG") prefixes.Insert(gotrie.Prefix("36"), "HU") prefixes.Insert(gotrie.Prefix("370"), "LT") prefixes.Insert(gotrie.Prefix("371"), "LV") prefixes.Insert(gotrie.Prefix("372"), "EE") prefixes.Insert(gotrie.Prefix("373"), "MD") prefixes.Insert(gotrie.Prefix("374"), "AM") prefixes.Insert(gotrie.Prefix("375"), "BY") prefixes.Insert(gotrie.Prefix("376"), "AD") prefixes.Insert(gotrie.Prefix("377"), "MC") prefixes.Insert(gotrie.Prefix("378"), "SM") prefixes.Insert(gotrie.Prefix("379"), "VA") prefixes.Insert(gotrie.Prefix("380"), "UA") prefixes.Insert(gotrie.Prefix("381"), "RS") prefixes.Insert(gotrie.Prefix("381"), "XK") prefixes.Insert(gotrie.Prefix("382"), "ME") prefixes.Insert(gotrie.Prefix("385"), "HR") prefixes.Insert(gotrie.Prefix("386"), "SI") prefixes.Insert(gotrie.Prefix("386"), "XK") prefixes.Insert(gotrie.Prefix("387"), "BA") prefixes.Insert(gotrie.Prefix("389"), "MK") prefixes.Insert(gotrie.Prefix("39"), "IT") prefixes.Insert(gotrie.Prefix("39066"), "VA") prefixes.Insert(gotrie.Prefix("40"), "RO") prefixes.Insert(gotrie.Prefix("41"), "CH") prefixes.Insert(gotrie.Prefix("420"), "CZ") prefixes.Insert(gotrie.Prefix("421"), "SK") prefixes.Insert(gotrie.Prefix("423"), "LI") prefixes.Insert(gotrie.Prefix("43"), "AT") prefixes.Insert(gotrie.Prefix("44"), "GB") prefixes.Insert(gotrie.Prefix("45"), "DK") prefixes.Insert(gotrie.Prefix("46"), "SE") prefixes.Insert(gotrie.Prefix("47"), "NO") prefixes.Insert(gotrie.Prefix("4779"), "SJ") prefixes.Insert(gotrie.Prefix("48"), "PL") prefixes.Insert(gotrie.Prefix("49"), "DE") prefixes.Insert(gotrie.Prefix("500"), "FK") prefixes.Insert(gotrie.Prefix("501"), "BZ") prefixes.Insert(gotrie.Prefix("502"), "GT") prefixes.Insert(gotrie.Prefix("503"), "SV") prefixes.Insert(gotrie.Prefix("504"), "HN") prefixes.Insert(gotrie.Prefix("505"), "NI") prefixes.Insert(gotrie.Prefix("506"), "CR") prefixes.Insert(gotrie.Prefix("507"), "PA") prefixes.Insert(gotrie.Prefix("508"), "PM") prefixes.Insert(gotrie.Prefix("509"), "HT") prefixes.Insert(gotrie.Prefix("51"), "PE") prefixes.Insert(gotrie.Prefix("52"), "MX") prefixes.Insert(gotrie.Prefix("53"), "CU") prefixes.Insert(gotrie.Prefix("54"), "AR") prefixes.Insert(gotrie.Prefix("55"), "BR") prefixes.Insert(gotrie.Prefix("56"), "CL") prefixes.Insert(gotrie.Prefix("57"), "CO") prefixes.Insert(gotrie.Prefix("58"), "VE") prefixes.Insert(gotrie.Prefix("590"), "GP") prefixes.Insert(gotrie.Prefix("591"), "BO") prefixes.Insert(gotrie.Prefix("592"), "GY") prefixes.Insert(gotrie.Prefix("593"), "EC") prefixes.Insert(gotrie.Prefix("594"), "GF") prefixes.Insert(gotrie.Prefix("595"), "PY") prefixes.Insert(gotrie.Prefix("596"), "MQ") prefixes.Insert(gotrie.Prefix("597"), "SR") prefixes.Insert(gotrie.Prefix("598"), "UY") prefixes.Insert(gotrie.Prefix("5997"), "BQ") prefixes.Insert(gotrie.Prefix("5999"), "CW") prefixes.Insert(gotrie.Prefix("60"), "MY") prefixes.Insert(gotrie.Prefix("61"), "AU") prefixes.Insert(gotrie.Prefix("62"), "ID") prefixes.Insert(gotrie.Prefix("63"), "PH") prefixes.Insert(gotrie.Prefix("64"), "NZ") prefixes.Insert(gotrie.Prefix("65"), "SG") prefixes.Insert(gotrie.Prefix("66"), "TH") prefixes.Insert(gotrie.Prefix("670"), "TL") prefixes.Insert(gotrie.Prefix("672"), "NF") prefixes.Insert(gotrie.Prefix("673"), "BN") prefixes.Insert(gotrie.Prefix("674"), "NR") prefixes.Insert(gotrie.Prefix("675"), "PG") prefixes.Insert(gotrie.Prefix("676"), "TO") prefixes.Insert(gotrie.Prefix("677"), "SB") prefixes.Insert(gotrie.Prefix("678"), "VU") prefixes.Insert(gotrie.Prefix("679"), "FJ") prefixes.Insert(gotrie.Prefix("680"), "PW") prefixes.Insert(gotrie.Prefix("681"), "WF") prefixes.Insert(gotrie.Prefix("682"), "CK") prefixes.Insert(gotrie.Prefix("683"), "NU") prefixes.Insert(gotrie.Prefix("685"), "WS") prefixes.Insert(gotrie.Prefix("686"), "KI") prefixes.Insert(gotrie.Prefix("687"), "NC") prefixes.Insert(gotrie.Prefix("688"), "TV") prefixes.Insert(gotrie.Prefix("689"), "PF") prefixes.Insert(gotrie.Prefix("690"), "TK") prefixes.Insert(gotrie.Prefix("691"), "FM") prefixes.Insert(gotrie.Prefix("692"), "MH") prefixes.Insert(gotrie.Prefix("7"), "RU") prefixes.Insert(gotrie.Prefix("76"), "KZ") prefixes.Insert(gotrie.Prefix("77"), "KZ") prefixes.Insert(gotrie.Prefix("81"), "JP") prefixes.Insert(gotrie.Prefix("82"), "KR") prefixes.Insert(gotrie.Prefix("84"), "VN") prefixes.Insert(gotrie.Prefix("850"), "KP") prefixes.Insert(gotrie.Prefix("852"), "HK") prefixes.Insert(gotrie.Prefix("853"), "MO") prefixes.Insert(gotrie.Prefix("855"), "KH") prefixes.Insert(gotrie.Prefix("856"), "LA") prefixes.Insert(gotrie.Prefix("86"), "CN") prefixes.Insert(gotrie.Prefix("880"), "BD") prefixes.Insert(gotrie.Prefix("886"), "TW") prefixes.Insert(gotrie.Prefix("90"), "TR") prefixes.Insert(gotrie.Prefix("91"), "IN") prefixes.Insert(gotrie.Prefix("92"), "PK") prefixes.Insert(gotrie.Prefix("93"), "AF") prefixes.Insert(gotrie.Prefix("94"), "LK") prefixes.Insert(gotrie.Prefix("95"), "MM") prefixes.Insert(gotrie.Prefix("960"), "MV") prefixes.Insert(gotrie.Prefix("961"), "LB") prefixes.Insert(gotrie.Prefix("962"), "JO") prefixes.Insert(gotrie.Prefix("963"), "SY") prefixes.Insert(gotrie.Prefix("964"), "IQ") prefixes.Insert(gotrie.Prefix("965"), "KW") prefixes.Insert(gotrie.Prefix("966"), "SA") prefixes.Insert(gotrie.Prefix("967"), "YE") prefixes.Insert(gotrie.Prefix("968"), "OM") prefixes.Insert(gotrie.Prefix("970"), "PS") prefixes.Insert(gotrie.Prefix("971"), "AE") prefixes.Insert(gotrie.Prefix("972"), "IL") prefixes.Insert(gotrie.Prefix("973"), "BH") prefixes.Insert(gotrie.Prefix("974"), "QA") prefixes.Insert(gotrie.Prefix("975"), "BT") prefixes.Insert(gotrie.Prefix("976"), "MN") prefixes.Insert(gotrie.Prefix("977"), "NP") prefixes.Insert(gotrie.Prefix("98"), "IR") prefixes.Insert(gotrie.Prefix("992"), "TJ") prefixes.Insert(gotrie.Prefix("993"), "TM") prefixes.Insert(gotrie.Prefix("994"), "AZ") prefixes.Insert(gotrie.Prefix("995"), "GE") prefixes.Insert(gotrie.Prefix("996"), "KG") prefixes.Insert(gotrie.Prefix("998"), "UZ") return prefixes }
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs { mirrorURLsCh := make(chan mirrorURLs) go func() { defer close(mirrorURLsCh) sourceTrie := patricia.NewTrie() targetTries := make([]*patricia.Trie, len(targetClnts)) wg := new(sync.WaitGroup) wg.Add(1) go func() { defer wg.Done() for sourceContentCh := range sourceClnt.List(true) { if sourceContentCh.Err != nil { mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()} return } if sourceContentCh.Content.Type.IsRegular() { sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size) } } }() for i, targetClnt := range targetClnts { wg.Add(1) go func(i int, targetClnt client.Client) { defer wg.Done() targetTrie := patricia.NewTrie() for targetContentCh := range targetClnt.List(true) { if targetContentCh.Err != nil { mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()} return } if targetContentCh.Content.Type.IsRegular() { targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{}) } } targetTries[i] = targetTrie }(i, targetClnt) } wg.Wait() matchNameCh := make(chan string, 10000) go func(matchNameCh chan<- string) { itemFunc := func(prefix patricia.Prefix, item patricia.Item) error { matchNameCh <- string(prefix) return nil } sourceTrie.Visit(itemFunc) defer close(matchNameCh) }(matchNameCh) for matchName := range matchNameCh { sourceContent := new(client.Content) var targetContents []*client.Content for i, targetTrie := range targetTries { if !targetTrie.Match(patricia.Prefix(matchName)) { sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(), string(sourceClnt.URL().Separator))+1] newTargetURLParse := *targetClnts[i].URL() newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName) sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64) sourceContent.Name = sourceURLDelimited + matchName targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()}) } } mirrorURLsCh <- mirrorURLs{ SourceContent: sourceContent, TargetContents: targetContents, } } }() return mirrorURLsCh }