// AddFileHandler adds the file path to the database. It should be usually be given to an POST endpoint
// with id as the parameter
// Ex: /file/:id
func AddFileHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
decoder := json.NewDecoder(r.Body)
// TODO: ps is there for checking emptiness. Should be replaced
var js, ps jsonStruct
if err := decoder.Decode(&js); err != nil || js == ps {
w.WriteHeader(400)
return
}
couchServer, err := couchdb.NewClient("http://127.0.0.1:5984", nil)
db, _ := couchServer.CreateDB("files")
userID := memberlist.DefaultWANConfig().Name
_, err = db.Put(p.ByName("id"), file{UUID: uuid.NewV4().String(), Fname: path.Base(js.Path), UserID: userID}, "")
if err != nil {
w.WriteHeader(500)
fmt.Fprint(w, err)
return
}
// TODO: Send 409 for conflict
if err := AddFile(p.ByName("id"), js.Path); err != nil {
w.WriteHeader(500)
fmt.Fprint(w, err)
return
}
w.WriteHeader(201)
}
// DefaultConfig is used to return a sane default configuration
func DefaultConfig() *Config {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
conf := &Config{
Datacenter: DefaultDC,
NodeName: hostname,
RPCAddr: DefaultRPCAddr,
RaftConfig: raft.DefaultConfig(),
SerfLANConfig: serf.DefaultConfig(),
SerfWANConfig: serf.DefaultConfig(),
ReconcileInterval: 60 * time.Second,
ProtocolVersion: ProtocolVersion2Compatible,
ACLTTL: 30 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "extend-cache",
TombstoneTTL: 15 * time.Minute,
TombstoneTTLGranularity: 30 * time.Second,
SessionTTLMin: 10 * time.Second,
DisableCoordinates: false,
// These are tuned to provide a total throughput of 128 updates
// per second. If you update these, you should update the client-
// side SyncCoordinateRateTarget parameter accordingly.
CoordinateUpdatePeriod: 5 * time.Second,
CoordinateUpdateBatchSize: 128,
CoordinateUpdateMaxBatches: 5,
}
// Increase our reap interval to 3 days instead of 24h.
conf.SerfLANConfig.ReconnectTimeout = 3 * 24 * time.Hour
conf.SerfWANConfig.ReconnectTimeout = 3 * 24 * time.Hour
// WAN Serf should use the WAN timing, since we are using it
// to communicate between DC's
conf.SerfWANConfig.MemberlistConfig = memberlist.DefaultWANConfig()
// Turn LAN Serf to run globally
conf.SerfLANConfig.MemberlistConfig = memberlist.DefaultWANConfig()
// Ensure we don't have port conflicts
conf.SerfLANConfig.MemberlistConfig.BindPort = DefaultLANSerfPort
conf.SerfWANConfig.MemberlistConfig.BindPort = DefaultWANSerfPort
// Disable shutdown on removal
conf.RaftConfig.ShutdownOnRemove = false
// Make Raft more WAN friendly
conf.RaftConfig.HeartbeatTimeout = 5000 * time.Millisecond
conf.RaftConfig.ElectionTimeout = 5000 * time.Millisecond
conf.RaftConfig.CommitTimeout = 100 * time.Millisecond
conf.RaftConfig.LeaderLeaseTimeout = 2500 * time.Millisecond
return conf
}
// DefaultConfig is used to return a sane default configuration
func DefaultConfig() *Config {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
conf := &Config{
Datacenter: DefaultDC,
NodeName: hostname,
RPCAddr: DefaultRPCAddr,
RaftConfig: raft.DefaultConfig(),
SerfLANConfig: serf.DefaultConfig(),
SerfWANConfig: serf.DefaultConfig(),
ReconcileInterval: 60 * time.Second,
ProtocolVersion: ProtocolVersionMax,
}
// Increase our reap interval to 3 days instead of 24h.
conf.SerfLANConfig.ReconnectTimeout = 3 * 24 * time.Hour
conf.SerfWANConfig.ReconnectTimeout = 3 * 24 * time.Hour
// WAN Serf should use the WAN timing, since we are using it
// to communicate between DC's
conf.SerfWANConfig.MemberlistConfig = memberlist.DefaultWANConfig()
// Ensure we don't have port conflicts
conf.SerfLANConfig.MemberlistConfig.BindPort = DefaultLANSerfPort
conf.SerfWANConfig.MemberlistConfig.BindPort = DefaultWANSerfPort
// Disable shutdown on removal
conf.RaftConfig.ShutdownOnRemove = false
return conf
}
// setupAgent is used to create the agent we use
func (c *Command) setupAgent(config *Config, logOutput io.Writer) *Agent {
bindIP, bindPort, err := config.AddrParts(config.BindAddr)
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid bind address: %s", err))
return nil
}
var advertiseIP string
var advertisePort int
if config.AdvertiseAddr != "" {
advertiseIP, advertisePort, err = config.AddrParts(config.AdvertiseAddr)
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid advertise address: %s", err))
return nil
}
}
encryptKey, err := config.EncryptBytes()
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
return nil
}
serfConfig := serf.DefaultConfig()
switch config.Profile {
case "lan":
serfConfig.MemberlistConfig = memberlist.DefaultLANConfig()
case "wan":
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
case "local":
serfConfig.MemberlistConfig = memberlist.DefaultLocalConfig()
default:
c.Ui.Error(fmt.Sprintf("Unknown profile: %s", config.Profile))
return nil
}
serfConfig.MemberlistConfig.BindAddr = bindIP
serfConfig.MemberlistConfig.BindPort = bindPort
serfConfig.MemberlistConfig.AdvertiseAddr = advertiseIP
serfConfig.MemberlistConfig.AdvertisePort = advertisePort
serfConfig.MemberlistConfig.SecretKey = encryptKey
serfConfig.NodeName = config.NodeName
serfConfig.Tags = config.Tags
serfConfig.SnapshotPath = config.SnapshotPath
serfConfig.ProtocolVersion = uint8(config.Protocol)
serfConfig.CoalescePeriod = 3 * time.Second
serfConfig.QuiescentPeriod = time.Second
serfConfig.UserCoalescePeriod = 3 * time.Second
serfConfig.UserQuiescentPeriod = time.Second
// Start Serf
c.Ui.Output("Starting Serf agent...")
agent, err := Create(serfConfig, logOutput)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to start the Serf agent: %v", err))
return nil
}
return agent
}
// DefaultConfig returns the default configuration
func DefaultConfig() *Config {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
c := &Config{
Region: DefaultRegion,
Datacenter: DefaultDC,
NodeName: hostname,
ProtocolVersion: ProtocolVersionMax,
RaftConfig: raft.DefaultConfig(),
RaftTimeout: 10 * time.Second,
LogOutput: os.Stderr,
RPCAddr: DefaultRPCAddr,
SerfConfig: serf.DefaultConfig(),
NumSchedulers: 1,
ReconcileInterval: 60 * time.Second,
EvalGCInterval: 5 * time.Minute,
EvalGCThreshold: 1 * time.Hour,
JobGCInterval: 5 * time.Minute,
JobGCThreshold: 4 * time.Hour,
NodeGCInterval: 5 * time.Minute,
NodeGCThreshold: 24 * time.Hour,
EvalNackTimeout: 60 * time.Second,
EvalDeliveryLimit: 3,
MinHeartbeatTTL: 10 * time.Second,
MaxHeartbeatsPerSecond: 50.0,
HeartbeatGrace: 10 * time.Second,
FailoverHeartbeatTTL: 300 * time.Second,
ConsulConfig: config.DefaultConsulConfig(),
VaultConfig: config.DefaultVaultConfig(),
RPCHoldTimeout: 5 * time.Second,
TLSConfig: &config.TLSConfig{},
}
// Enable all known schedulers by default
c.EnabledSchedulers = make([]string, 0, len(scheduler.BuiltinSchedulers))
for name := range scheduler.BuiltinSchedulers {
c.EnabledSchedulers = append(c.EnabledSchedulers, name)
}
c.EnabledSchedulers = append(c.EnabledSchedulers, structs.JobTypeCore)
// Default the number of schedulers to match the coores
c.NumSchedulers = runtime.NumCPU()
// Increase our reap interval to 3 days instead of 24h.
c.SerfConfig.ReconnectTimeout = 3 * 24 * time.Hour
// Serf should use the WAN timing, since we are using it
// to communicate between DC's
c.SerfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
c.SerfConfig.MemberlistConfig.BindPort = DefaultSerfPort
// Disable shutdown on removal
c.RaftConfig.ShutdownOnRemove = false
return c
}
// DefaultConfig is used to return a sane default configuration
func DefaultConfig() *Config {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
conf := &Config{
Datacenter: DefaultDC,
NodeName: hostname,
RPCAddr: DefaultRPCAddr,
RaftConfig: raft.DefaultConfig(),
SerfLANConfig: serf.DefaultConfig(),
SerfWANConfig: serf.DefaultConfig(),
ReconcileInterval: 60 * time.Second,
ProtocolVersion: ProtocolVersion2Compatible,
ACLTTL: 30 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "extend-cache",
ACLReplicationInterval: 30 * time.Second,
ACLReplicationApplyLimit: 100, // ops / sec
TombstoneTTL: 15 * time.Minute,
TombstoneTTLGranularity: 30 * time.Second,
SessionTTLMin: 10 * time.Second,
DisableCoordinates: false,
// These are tuned to provide a total throughput of 128 updates
// per second. If you update these, you should update the client-
// side SyncCoordinateRateTarget parameter accordingly.
CoordinateUpdatePeriod: 5 * time.Second,
CoordinateUpdateBatchSize: 128,
CoordinateUpdateMaxBatches: 5,
// Hold an RPC for up to 5 seconds by default
RPCHoldTimeout: 5 * time.Second,
}
// Increase our reap interval to 3 days instead of 24h.
conf.SerfLANConfig.ReconnectTimeout = 3 * 24 * time.Hour
conf.SerfWANConfig.ReconnectTimeout = 3 * 24 * time.Hour
// WAN Serf should use the WAN timing, since we are using it
// to communicate between DC's
conf.SerfWANConfig.MemberlistConfig = memberlist.DefaultWANConfig()
// Ensure we don't have port conflicts
conf.SerfLANConfig.MemberlistConfig.BindPort = DefaultLANSerfPort
conf.SerfWANConfig.MemberlistConfig.BindPort = DefaultWANSerfPort
// Enable interoperability with unversioned Raft library, and don't
// start using new ID-based features yet.
conf.RaftConfig.ProtocolVersion = 1
// Disable shutdown on removal
conf.RaftConfig.ShutdownOnRemove = false
return conf
}
// setupAgent is used to create the agent we use
func (c *Command) setupAgent(config *Config, logOutput io.Writer) *Agent {
bindIP, bindPort, err := config.AddrParts(config.BindAddr)
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid bind address: %s", err))
return nil
}
// Check if we have an interface
if iface, _ := config.NetworkInterface(); iface != nil {
addrs, err := iface.Addrs()
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to get interface addresses: %s", err))
return nil
}
if len(addrs) == 0 {
c.Ui.Error(fmt.Sprintf("Interface '%s' has no addresses", config.Interface))
return nil
}
// If there is no bind IP, pick an address
if bindIP == "0.0.0.0" {
found := false
for _, a := range addrs {
addr, ok := a.(*net.IPNet)
if !ok {
continue
}
// Skip self-assigned IPs
if addr.IP.IsLinkLocalUnicast() {
continue
}
// Found an IP
found = true
bindIP = addr.IP.String()
c.Ui.Output(fmt.Sprintf("Using interface '%s' address '%s'",
config.Interface, bindIP))
// Update the configuration
bindAddr := &net.TCPAddr{
IP: net.ParseIP(bindIP),
Port: bindPort,
}
config.BindAddr = bindAddr.String()
break
}
if !found {
c.Ui.Error(fmt.Sprintf("Failed to find usable address for interface '%s'", config.Interface))
return nil
}
} else {
// If there is a bind IP, ensure it is available
found := false
for _, a := range addrs {
addr, ok := a.(*net.IPNet)
if !ok {
continue
}
if addr.IP.String() == bindIP {
found = true
break
}
}
if !found {
c.Ui.Error(fmt.Sprintf("Interface '%s' has no '%s' address",
config.Interface, bindIP))
return nil
}
}
}
var advertiseIP string
var advertisePort int
if config.AdvertiseAddr != "" {
advertiseIP, advertisePort, err = config.AddrParts(config.AdvertiseAddr)
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid advertise address: %s", err))
return nil
}
}
encryptKey, err := config.EncryptBytes()
if err != nil {
c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
return nil
}
serfConfig := serf.DefaultConfig()
switch config.Profile {
case "lan":
serfConfig.MemberlistConfig = memberlist.DefaultLANConfig()
case "wan":
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
case "local":
serfConfig.MemberlistConfig = memberlist.DefaultLocalConfig()
default:
c.Ui.Error(fmt.Sprintf("Unknown profile: %s", config.Profile))
return nil
}
serfConfig.MemberlistConfig.BindAddr = bindIP
serfConfig.MemberlistConfig.BindPort = bindPort
serfConfig.MemberlistConfig.AdvertiseAddr = advertiseIP
serfConfig.MemberlistConfig.AdvertisePort = advertisePort
serfConfig.MemberlistConfig.SecretKey = encryptKey
serfConfig.NodeName = config.NodeName
serfConfig.Tags = config.Tags
serfConfig.SnapshotPath = config.SnapshotPath
serfConfig.ProtocolVersion = uint8(config.Protocol)
serfConfig.CoalescePeriod = 3 * time.Second
serfConfig.QuiescentPeriod = time.Second
serfConfig.UserCoalescePeriod = 3 * time.Second
serfConfig.UserQuiescentPeriod = time.Second
if config.ReconnectInterval != 0 {
serfConfig.ReconnectInterval = config.ReconnectInterval
}
if config.ReconnectTimeout != 0 {
serfConfig.ReconnectTimeout = config.ReconnectTimeout
}
if config.TombstoneTimeout != 0 {
serfConfig.TombstoneTimeout = config.TombstoneTimeout
}
serfConfig.EnableNameConflictResolution = !config.DisableNameResolution
if config.KeyringFile != "" {
serfConfig.KeyringFile = config.KeyringFile
}
serfConfig.RejoinAfterLeave = config.RejoinAfterLeave
// Start Serf
c.Ui.Output("Starting Serf agent...")
agent, err := Create(config, serfConfig, logOutput)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to start the Serf agent: %v", err))
return nil
}
return agent
}
// setupAgent is used to create the agent we use
func (a *AgentCommand) setupSerf(config *Config) *serf.Serf {
bindIP, bindPort, err := config.AddrParts(config.BindAddr)
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid bind address: %s", err))
return nil
}
// Check if we have an interface
if iface, _ := config.NetworkInterface(); iface != nil {
addrs, err := iface.Addrs()
if err != nil {
a.Ui.Error(fmt.Sprintf("Failed to get interface addresses: %s", err))
return nil
}
if len(addrs) == 0 {
a.Ui.Error(fmt.Sprintf("Interface '%s' has no addresses", config.Interface))
return nil
}
// If there is no bind IP, pick an address
if bindIP == "0.0.0.0" {
found := false
for _, ad := range addrs {
var addrIP net.IP
if runtime.GOOS == "windows" {
// Waiting for https://github.com/golang/go/issues/5395 to use IPNet only
addr, ok := ad.(*net.IPAddr)
if !ok {
continue
}
addrIP = addr.IP
} else {
addr, ok := ad.(*net.IPNet)
if !ok {
continue
}
addrIP = addr.IP
}
// Skip self-assigned IPs
if addrIP.IsLinkLocalUnicast() {
continue
}
// Found an IP
found = true
bindIP = addrIP.String()
a.Ui.Output(fmt.Sprintf("Using interface '%s' address '%s'",
config.Interface, bindIP))
// Update the configuration
bindAddr := &net.TCPAddr{
IP: net.ParseIP(bindIP),
Port: bindPort,
}
config.BindAddr = bindAddr.String()
break
}
if !found {
a.Ui.Error(fmt.Sprintf("Failed to find usable address for interface '%s'", config.Interface))
return nil
}
} else {
// If there is a bind IP, ensure it is available
found := false
for _, a := range addrs {
addr, ok := a.(*net.IPNet)
if !ok {
continue
}
if addr.IP.String() == bindIP {
found = true
break
}
}
if !found {
a.Ui.Error(fmt.Sprintf("Interface '%s' has no '%s' address",
config.Interface, bindIP))
return nil
}
}
}
var advertiseIP string
var advertisePort int
if config.AdvertiseAddr != "" {
advertiseIP, advertisePort, err = config.AddrParts(config.AdvertiseAddr)
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid advertise address: %s", err))
return nil
}
}
encryptKey, err := config.EncryptBytes()
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
return nil
}
serfConfig := serf.DefaultConfig()
switch config.Profile {
case "lan":
serfConfig.MemberlistConfig = memberlist.DefaultLANConfig()
case "wan":
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
case "local":
serfConfig.MemberlistConfig = memberlist.DefaultLocalConfig()
default:
a.Ui.Error(fmt.Sprintf("Unknown profile: %s", config.Profile))
return nil
}
serfConfig.MemberlistConfig.BindAddr = bindIP
serfConfig.MemberlistConfig.BindPort = bindPort
serfConfig.MemberlistConfig.AdvertiseAddr = advertiseIP
serfConfig.MemberlistConfig.AdvertisePort = advertisePort
serfConfig.MemberlistConfig.SecretKey = encryptKey
serfConfig.NodeName = config.NodeName
serfConfig.Tags = config.Tags
serfConfig.SnapshotPath = config.SnapshotPath
serfConfig.CoalescePeriod = 3 * time.Second
serfConfig.QuiescentPeriod = time.Second
serfConfig.UserCoalescePeriod = 3 * time.Second
serfConfig.UserQuiescentPeriod = time.Second
if config.ReconnectInterval != 0 {
serfConfig.ReconnectInterval = config.ReconnectInterval
}
if config.ReconnectTimeout != 0 {
serfConfig.ReconnectTimeout = config.ReconnectTimeout
}
if config.TombstoneTimeout != 0 {
serfConfig.TombstoneTimeout = config.TombstoneTimeout
}
serfConfig.EnableNameConflictResolution = !config.DisableNameResolution
if config.KeyringFile != "" {
serfConfig.KeyringFile = config.KeyringFile
}
serfConfig.RejoinAfterLeave = config.RejoinAfterLeave
// Create a channel to listen for events from Serf
a.eventCh = make(chan serf.Event, 64)
serfConfig.EventCh = a.eventCh
// Start Serf
a.Ui.Output("Starting Serf agent...")
log.Info("agent: Serf agent starting")
serfConfig.LogOutput = ioutil.Discard
serfConfig.MemberlistConfig.LogOutput = ioutil.Discard
// Create serf first
serf, err := serf.Create(serfConfig)
if err != nil {
a.Ui.Error(err.Error())
log.Error(err)
return nil
}
return serf
}
// DefaultConfig is used to return a sane default configuration
func DefaultConfig() *Config {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
conf := &Config{
Datacenter: DefaultDC,
NodeName: hostname,
RPCAddr: DefaultRPCAddr,
RaftConfig: raft.DefaultConfig(),
SerfLANConfig: serf.DefaultConfig(),
SerfWANConfig: serf.DefaultConfig(),
ReconcileInterval: 60 * time.Second,
ProtocolVersion: ProtocolVersion2Compatible,
ACLTTL: 30 * time.Second,
ACLDefaultPolicy: "allow",
ACLDownPolicy: "extend-cache",
ACLReplicationInterval: 30 * time.Second,
ACLReplicationApplyLimit: 100, // ops / sec
TombstoneTTL: 15 * time.Minute,
TombstoneTTLGranularity: 30 * time.Second,
SessionTTLMin: 10 * time.Second,
DisableCoordinates: false,
// These are tuned to provide a total throughput of 128 updates
// per second. If you update these, you should update the client-
// side SyncCoordinateRateTarget parameter accordingly.
CoordinateUpdatePeriod: 5 * time.Second,
CoordinateUpdateBatchSize: 128,
CoordinateUpdateMaxBatches: 5,
// This holds RPCs during leader elections. For the default Raft
// config the election timeout is 5 seconds, so we set this a
// bit longer to try to cover that period. This should be more
// than enough when running in the high performance mode.
RPCHoldTimeout: 7 * time.Second,
}
// Increase our reap interval to 3 days instead of 24h.
conf.SerfLANConfig.ReconnectTimeout = 3 * 24 * time.Hour
conf.SerfWANConfig.ReconnectTimeout = 3 * 24 * time.Hour
// WAN Serf should use the WAN timing, since we are using it
// to communicate between DC's
conf.SerfWANConfig.MemberlistConfig = memberlist.DefaultWANConfig()
// Ensure we don't have port conflicts
conf.SerfLANConfig.MemberlistConfig.BindPort = DefaultLANSerfPort
conf.SerfWANConfig.MemberlistConfig.BindPort = DefaultWANSerfPort
// Enable interoperability with unversioned Raft library, and don't
// start using new ID-based features yet.
conf.RaftConfig.ProtocolVersion = 1
conf.ScaleRaft(DefaultRaftMultiplier)
// Disable shutdown on removal
conf.RaftConfig.ShutdownOnRemove = false
// Check every 5 seconds to see if there are enough new entries for a snapshot
conf.RaftConfig.SnapshotInterval = 5 * time.Second
return conf
}
