Exemplo n.º 1
0
func SetMirrorState(r *database.Redis, id string, state bool, reason string) error {
	conn := r.Get()
	defer conn.Close()

	key := fmt.Sprintf("MIRROR_%s", id)

	previousState, err := redis.Bool(conn.Do("HGET", key, "up"))
	if err != nil {
		return err
	}

	var args []interface{}
	args = append(args, key, "up", state, "excludeReason", reason)

	if state != previousState {
		args = append(args, "stateSince", time.Now().Unix())
	}

	_, err = conn.Do("HMSET", args...)

	if err == nil && state != previousState {
		// Publish update
		database.Publish(conn, database.MIRROR_UPDATE, id)
	}

	return err
}
Exemplo n.º 2
0
func (c *cluster) clusterLoop() {
	clusterChan := make(chan string, 10)
	announceTicker := time.NewTicker(1 * time.Second)

	hostname := utils.GetHostname()
	nodeID := fmt.Sprintf("%s-%05d", hostname, rand.Intn(32000))

	c.refreshNodeList(nodeID, nodeID)
	c.redis.Pubsub.SubscribeEvent(database.CLUSTER, clusterChan)

	for {
		select {
		case <-c.stop:
			c.wg.Done()
			return
		case <-announceTicker.C:
			r := c.redis.Get()
			database.Publish(r, database.CLUSTER, fmt.Sprintf("%s %s", clusterAnnounce, nodeID))
			r.Close()
		case data := <-clusterChan:
			if !strings.HasPrefix(data, clusterAnnounce+" ") {
				// Garbage
				continue
			}
			c.refreshNodeList(data[len(clusterAnnounce)+1:], nodeID)
		}
	}
}
Exemplo n.º 3
0
func SetMirrorEnabled(r *database.Redis, id string, state bool) error {
	conn := r.Get()
	defer conn.Close()

	key := fmt.Sprintf("MIRROR_%s", id)
	_, err := conn.Do("HMSET", key, "enabled", state)

	// Publish update
	if err == nil {
		database.Publish(conn, database.MIRROR_UPDATE, id)
	}

	return err
}
Exemplo n.º 4
0
func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}
Exemplo n.º 5
0
func (c *cli) CmdEdit(args ...string) error {
	cmd := SubCmd("edit", "[IDENTIFIER]", "Edit a mirror")

	if err := cmd.Parse(args); err != nil {
		return nil
	}
	if cmd.NArg() != 1 {
		cmd.Usage()
		return nil
	}

	// Find the editor to use
	editor := os.Getenv("EDITOR")

	if editor == "" {
		log.Fatal("Environment variable $EDITOR not set")
	}

	// Guess which mirror to use
	list, err := c.matchMirror(cmd.Arg(0))
	if err != nil {
		return err
	}
	if len(list) == 0 {
		fmt.Fprintf(os.Stderr, "No match for %s\n", cmd.Arg(0))
		return nil
	} else if len(list) > 1 {
		for _, e := range list {
			fmt.Fprintf(os.Stderr, "%s\n", e)
		}
		return nil
	}

	id := list[0]

	// Connect to the database
	r := database.NewRedis()
	conn, err := r.Connect()
	if err != nil {
		log.Fatal("Redis: ", err)
	}
	defer conn.Close()

	// Get the mirror information
	key := fmt.Sprintf("MIRROR_%s", id)
	m, err := redis.Values(conn.Do("HGETALL", key))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot fetch mirror details: %s\n", err)
		return err
	}

	var mirror mirrors.Mirror
	err = redis.ScanStruct(m, &mirror)
	if err != nil {
		return err
	}

	// Generate a yaml configuration string from the struct
	out, err := yaml.Marshal(mirror)

	// Open a temporary file
	f, err := ioutil.TempFile(os.TempDir(), "edit")
	if err != nil {
		log.Fatal("Cannot create temporary file:", err)
	}
	defer os.Remove(f.Name())
	f.WriteString("# You can now edit this mirror configuration.\n" +
		"# Just save and quit when you're done.\n\n")
	f.WriteString(string(out))
	f.WriteString(fmt.Sprintf("\n%s\n\n%s\n", commentSeparator, mirror.Comment))
	f.Close()

	// Checksum the original file
	chk, _ := filesystem.HashFile(f.Name())

reopen:
	// Launch the editor with the filename as first parameter
	exe := exec.Command(editor, f.Name())
	exe.Stdin = os.Stdin
	exe.Stdout = os.Stdout
	exe.Stderr = os.Stderr

	err = exe.Run()
	if err != nil {
		log.Fatal(err)
	}

	// Read the file back
	out, err = ioutil.ReadFile(f.Name())
	if err != nil {
		log.Fatal("Cannot read file", f.Name())
	}

	// Checksum the file back and compare
	chk2, _ := filesystem.HashFile(f.Name())
	if chk == chk2 {
		fmt.Println("Aborted")
		return nil
	}

	var (
		yamlstr string = string(out)
		comment string
	)

	commentIndex := strings.Index(yamlstr, commentSeparator)
	if commentIndex > 0 {
		comment = strings.TrimSpace(yamlstr[commentIndex+len(commentSeparator):])
		yamlstr = yamlstr[:commentIndex]
	}

	// Fill the struct from the yaml
	err = yaml.Unmarshal([]byte(yamlstr), &mirror)
	if err != nil {
	eagain:
		fmt.Printf("%s\nRetry? [Y/n]", err.Error())
		reader := bufio.NewReader(os.Stdin)
		s, _ := reader.ReadString('\n')
		switch s[0] {
		case 'y', 'Y', 10:
			goto reopen
		case 'n', 'N':
			fmt.Println("Aborted")
			return nil
		default:
			goto eagain
		}
	}

	// Reformat contry codes
	mirror.CountryCodes = strings.Replace(mirror.CountryCodes, ",", " ", -1)
	ccodes := strings.Fields(mirror.CountryCodes)
	mirror.CountryCodes = ""
	for _, c := range ccodes {
		mirror.CountryCodes += strings.ToUpper(c) + " "
	}
	mirror.CountryCodes = strings.TrimRight(mirror.CountryCodes, " ")

	// Reformat continent code
	//FIXME sanitize
	mirror.ContinentCode = strings.ToUpper(mirror.ContinentCode)

	// Normalize URLs
	if mirror.HttpURL != "" {
		mirror.HttpURL = utils.NormalizeURL(mirror.HttpURL)
	}
	if mirror.RsyncURL != "" {
		mirror.RsyncURL = utils.NormalizeURL(mirror.RsyncURL)
	}
	if mirror.FtpURL != "" {
		mirror.FtpURL = utils.NormalizeURL(mirror.FtpURL)
	}

	mirror.Comment = comment

	// Save the values back into redis
	_, err = conn.Do("HMSET", key,
		"ID", id,
		"http", mirror.HttpURL,
		"rsync", mirror.RsyncURL,
		"ftp", mirror.FtpURL,
		"sponsorName", mirror.SponsorName,
		"sponsorURL", mirror.SponsorURL,
		"sponsorLogo", mirror.SponsorLogoURL,
		"adminName", mirror.AdminName,
		"adminEmail", mirror.AdminEmail,
		"customData", mirror.CustomData,
		"continentOnly", mirror.ContinentOnly,
		"countryOnly", mirror.CountryOnly,
		"asOnly", mirror.ASOnly,
		"score", mirror.Score,
		"latitude", mirror.Latitude,
		"longitude", mirror.Longitude,
		"continentCode", mirror.ContinentCode,
		"countryCodes", mirror.CountryCodes,
		"asnum", mirror.Asnum,
		"comment", mirror.Comment,
		"enabled", mirror.Enabled)

	if err != nil {
		log.Fatal("Couldn't save the configuration into redis:", err)
	}

	// Publish update
	database.Publish(conn, database.MIRROR_UPDATE, id)

	fmt.Println("Mirror edited successfully")

	return nil
}
Exemplo n.º 6
0
func (c *cli) CmdRemove(args ...string) error {
	cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror")

	if err := cmd.Parse(args); err != nil {
		return nil
	}
	if cmd.NArg() != 1 {
		cmd.Usage()
		return nil
	}

	// Guess which mirror to use
	list, err := c.matchMirror(cmd.Arg(0))
	if err != nil {
		return err
	}
	if len(list) == 0 {
		fmt.Fprintf(os.Stderr, "No match for %s\n", cmd.Arg(0))
		return nil
	} else if len(list) > 1 {
		for _, e := range list {
			fmt.Fprintf(os.Stderr, "%s\n", e)
		}
		return nil
	}

	identifier := list[0]

	r := database.NewRedis()
	conn, err := r.Connect()
	if err != nil {
		log.Fatal("Redis: ", err)
	}
	defer conn.Close()

	// First disable the mirror
	mirrors.DisableMirror(r, identifier)

	// Get all files supported by the given mirror
	files, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("MIRROR_%s_FILES", identifier)))
	if err != nil {
		log.Fatal("Error: Cannot fetch file list: ", err)
	}

	conn.Send("MULTI")

	// Remove each FILEINFO / FILEMIRRORS
	for _, file := range files {
		conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, file))
		conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", file), identifier)
		conn.Send("PUBLISH", database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, file))
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		log.Fatal("Error: FILEINFO/FILEMIRRORS keys could not be removed: ", err)
	}

	// Remove all other keys
	_, err = conn.Do("DEL",
		fmt.Sprintf("MIRROR_%s", identifier),
		fmt.Sprintf("MIRROR_%s_FILES", identifier),
		fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier),
		fmt.Sprintf("HANDLEDFILES_%s", identifier),
		fmt.Sprintf("SCANNING_%s", identifier))

	if err != nil {
		log.Fatal("Error: MIRROR keys could not be removed: ", err)
	}

	// Remove the last reference
	_, err = conn.Do("LREM", "MIRRORS", 0, identifier)

	if err != nil {
		log.Fatal("Error: Could not remove the reference from key MIRRORS")
	}

	// Publish update
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	fmt.Println("Mirror removed successfully")
	return nil
}
Exemplo n.º 7
0
func (c *cli) CmdAdd(args ...string) error {
	cmd := SubCmd("add", "[OPTIONS] IDENTIFIER", "Add a new mirror")
	http := cmd.String("http", "", "HTTP base URL")
	rsync := cmd.String("rsync", "", "RSYNC base URL (for scanning only)")
	ftp := cmd.String("ftp", "", "FTP base URL (for scanning only)")
	sponsorName := cmd.String("sponsor-name", "", "Name of the sponsor")
	sponsorURL := cmd.String("sponsor-url", "", "URL of the sponsor")
	sponsorLogo := cmd.String("sponsor-logo", "", "URL of a logo to display for this mirror")
	adminName := cmd.String("admin-name", "", "Admin's name")
	adminEmail := cmd.String("admin-email", "", "Admin's email")
	customData := cmd.String("custom-data", "", "Associated data to return when the mirror is selected (i.e. json document)")
	continentOnly := cmd.Bool("continent-only", false, "The mirror should only handle its continent")
	countryOnly := cmd.Bool("country-only", false, "The mirror should only handle its country")
	asOnly := cmd.Bool("as-only", false, "The mirror should only handle clients in the same AS number")
	score := cmd.Int("score", 0, "Weight to give to the mirror during selection")
	comment := cmd.String("comment", "", "Comment")

	if err := cmd.Parse(args); err != nil {
		return nil
	}
	if cmd.NArg() < 1 {
		cmd.Usage()
		return nil
	}

	if strings.Contains(cmd.Arg(0), " ") {
		fmt.Fprintf(os.Stderr, "The identifier cannot contain a space\n")
		os.Exit(-1)
	}

	if *http == "" {
		fmt.Fprintf(os.Stderr, "You *must* pass at least an HTTP URL\n")
		os.Exit(-1)
	}

	if !strings.HasPrefix(*http, "http://") && !strings.HasPrefix(*http, "https://") {
		*http = "http://" + *http
	}

	u, err := url.Parse(*http)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't parse HTTP url\n")
		os.Exit(-1)
	}

	ip, err := network.LookupMirrorIP(u.Host)
	if err == network.ErrMultipleAddresses {
		fmt.Fprintf(os.Stderr, "Warning: the hostname returned more than one address! This is highly unreliable.\n")
	} else if err != nil {
		log.Fatal("IP lookup failed: ", err.Error())
	}

	geo := network.NewGeoIP()
	if err := geo.LoadGeoIP(); err != nil {
		log.Fatal(err.Error())
	}

	geoRec := geo.GetRecord(ip)

	r := database.NewRedis()
	conn, err := r.Connect()
	if err != nil {
		log.Fatal("Redis: ", err)
	}
	defer conn.Close()

	key := fmt.Sprintf("MIRROR_%s", cmd.Arg(0))
	exists, err := redis.Bool(conn.Do("EXISTS", key))
	if err != nil {
		return err
	}
	if exists {
		fmt.Fprintf(os.Stderr, "Mirror %s already exists!\n", cmd.Arg(0))
		os.Exit(-1)
	}

	// Normalize the URLs
	if http != nil {
		*http = utils.NormalizeURL(*http)
	}
	if rsync != nil {
		*rsync = utils.NormalizeURL(*rsync)
	}
	if ftp != nil {
		*ftp = utils.NormalizeURL(*ftp)
	}

	var latitude, longitude float32
	var continentCode, countryCode string

	if geoRec.GeoIPRecord != nil {
		latitude = geoRec.GeoIPRecord.Latitude
		longitude = geoRec.GeoIPRecord.Longitude
		continentCode = geoRec.GeoIPRecord.ContinentCode
		countryCode = geoRec.GeoIPRecord.CountryCode
	} else {
		fmt.Fprintf(os.Stderr, "Warning: unable to guess the geographic location of %s\n", cmd.Arg(0))
	}

	_, err = conn.Do("HMSET", key,
		"ID", cmd.Arg(0),
		"http", *http,
		"rsync", *rsync,
		"ftp", *ftp,
		"sponsorName", *sponsorName,
		"sponsorURL", *sponsorURL,
		"sponsorLogo", *sponsorLogo,
		"adminName", *adminName,
		"adminEmail", *adminEmail,
		"customData", *customData,
		"continentOnly", *continentOnly,
		"countryOnly", *countryOnly,
		"asOnly", *asOnly,
		"score", *score,
		"latitude", fmt.Sprintf("%f", latitude),
		"longitude", fmt.Sprintf("%f", longitude),
		"continentCode", continentCode,
		"countryCodes", countryCode,
		"asnum", geoRec.ASNum,
		"comment", strings.TrimSpace(*comment),
		"enabled", false,
		"up", false)
	if err != nil {
		goto oops
	}

	_, err = conn.Do("LPUSH", "MIRRORS", cmd.Arg(0))
	if err != nil {
		goto oops
	}

	// Publish update
	database.Publish(conn, database.MIRROR_UPDATE, cmd.Arg(0))

	fmt.Println("Mirror added successfully")
	return nil
oops:
	fmt.Fprintf(os.Stderr, "Oops: %s", err)
	os.Exit(-1)
	return nil
}