// GetDefaultLocalAddress returns an address at which the local host can
// be reached, or 0.0.0.0 (which should work for locations from the host
// to itself) if the actual default local address cannot be determined.
func GetDefaultLocalAddress() string {
addr := "0.0.0.0"
ip, err := util.DefaultLocalIP4()
if err == nil {
addr = ip.String()
}
return addr
}
// GetDefaultLocalAddress returns an address at which the local host can
// be reached, or 0.0.0.0 (which should work for locations from the host
// to itself) if the actual default local address cannot be determined.
func GetDefaultLocalAddress() string {
addr := "0.0.0.0"
if a := os.Getenv("OPENSHIFT_ROUTER_SERVER_ADDRESS"); len(a) > 0 {
return a
}
ip, err := util.DefaultLocalIP4()
if err == nil {
addr = ip.String()
}
return addr
}
// FindLocalIPForDNS attempts to find an IP that will be reachable from
// inside containers as an IP address. It will try to use the Host values of
// the DNSBindAddr, the MasterAddr, and the MasterPublicAddr, before falling
// back to the local IP. This method will fail if the Master*Addrs point to
// an IP loadbalancer, so this method is at best a heuristic.
func findLocalIPForDNS(m *MasterArgs) (net.IP, error) {
if ip := specifiedIP(m.DNSBindAddr.Host); ip != nil {
return ip, nil
}
if ip := specifiedIP(m.MasterAddr.Host); ip != nil {
return ip, nil
}
if ip := specifiedIP(m.MasterPublicAddr.Host); ip != nil {
return ip, nil
}
return cmdutil.DefaultLocalIP4()
}
func TestMasterAddressDefaultingToBindValues(t *testing.T) {
defaultIP, err := util.DefaultLocalIP4()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expected := "http://" + defaultIP.String() + ":9012"
masterArgs := NewDefaultMasterArgs()
masterArgs.ListenArg.ListenAddr.Set("http://0.0.0.0:9012")
actual, err := masterArgs.GetMasterAddress()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if expected != actual.String() {
t.Fatalf("expected %v, got %v", expected, actual)
}
}
// GetMasterAddress checks for an unset master address and then attempts to use the first
// public IPv4 non-loopback address registered on this host.
// TODO: make me IPv6 safe
func (args MasterArgs) GetMasterAddress() (*url.URL, error) {
if args.MasterAddr.Provided {
return args.MasterAddr.URL, nil
}
// Use the bind port by default
port := args.ListenArg.ListenAddr.Port
// Use the bind scheme by default
scheme := args.ListenArg.ListenAddr.URL.Scheme
addr := ""
if ip, err := cmdutil.DefaultLocalIP4(); err == nil {
addr = ip.String()
} else if err == cmdutil.ErrorNoDefaultIP {
addr = "127.0.0.1"
} else if err != nil {
return nil, fmt.Errorf("Unable to find a public IP address: %v", err)
}
masterAddr := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port))
return url.Parse(masterAddr)
}
func TestDNS(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
masterConfig, clientFile, err := testserver.StartTestMaster()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
localAddr := ""
if ip, err := cmdutil.DefaultLocalIP4(); err == nil {
localAddr = ip.String()
} else if err == cmdutil.ErrorNoDefaultIP {
localAddr = "127.0.0.1"
} else if err != nil {
t.Fatalf("Unable to find a local IP address: %v", err)
}
localIP := net.ParseIP(localAddr)
var masterIP net.IP
// verify service DNS entry is visible
stop := make(chan struct{})
waitutil.Until(func() {
m1 := &dns.Msg{
MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: false},
Question: []dns.Question{{"kubernetes.default.svc.cluster.local.", dns.TypeA, dns.ClassINET}},
}
in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
if err != nil {
t.Logf("unexpected error: %v", err)
return
}
if len(in.Answer) != 1 {
t.Logf("unexpected answer: %#v", in)
return
}
if a, ok := in.Answer[0].(*dns.A); ok {
if a.A == nil {
t.Fatalf("expected an A record with an IP: %#v", a)
}
masterIP = a.A
} else {
t.Fatalf("expected an A record: %#v", in)
}
t.Log(in)
close(stop)
}, 50*time.Millisecond, stop)
client, err := testutil.GetClusterAdminKubeClient(clientFile)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Verify kubernetes service port is 53 and target port is the
// configured masterConfig.DNSConfig.BindAddress port.
_, dnsPortString, err := net.SplitHostPort(masterConfig.DNSConfig.BindAddress)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
dnsPort, err := strconv.Atoi(dnsPortString)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
kubernetesService, err := client.Services(kapi.NamespaceDefault).Get("kubernetes")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
found := false
for _, port := range kubernetesService.Spec.Ports {
if port.Port == 53 && port.TargetPort.IntVal == int32(dnsPort) && port.Protocol == kapi.ProtocolTCP {
found = true
}
}
if !found {
t.Fatalf("did not find DNS port in kubernetes service: %#v", kubernetesService)
}
for {
if _, err := client.Services(kapi.NamespaceDefault).Create(&kapi.Service{
ObjectMeta: kapi.ObjectMeta{
Name: "headless",
},
Spec: kapi.ServiceSpec{
ClusterIP: kapi.ClusterIPNone,
Ports: []kapi.ServicePort{{Port: 443}},
},
}); err != nil {
if errors.IsForbidden(err) {
t.Logf("forbidden, sleeping: %v", err)
time.Sleep(100 * time.Millisecond)
continue
}
t.Fatalf("unexpected error: %v", err)
}
if _, err := client.Endpoints(kapi.NamespaceDefault).Create(&kapi.Endpoints{
ObjectMeta: kapi.ObjectMeta{
Name: "headless",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "172.0.0.1"}},
Ports: []kapi.EndpointPort{
{Port: 2345},
},
}},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
break
}
headlessIP := net.ParseIP("172.0.0.1")
headlessIPHash := getHash(headlessIP.String())
if _, err := client.Services(kapi.NamespaceDefault).Create(&kapi.Service{
ObjectMeta: kapi.ObjectMeta{
Name: "headless2",
},
Spec: kapi.ServiceSpec{
ClusterIP: kapi.ClusterIPNone,
Ports: []kapi.ServicePort{{Port: 443}},
},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := client.Endpoints(kapi.NamespaceDefault).Create(&kapi.Endpoints{
ObjectMeta: kapi.ObjectMeta{
Name: "headless2",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "172.0.0.2"}},
Ports: []kapi.EndpointPort{
{Port: 2345, Name: "other"},
{Port: 2346, Name: "http"},
},
}},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
headless2IP := net.ParseIP("172.0.0.2")
precannedIP := net.ParseIP("10.2.4.50")
headless2IPHash := getHash(headless2IP.String())
tests := []struct {
dnsQuestionName string
recursionExpected bool
retry bool
expect []*net.IP
srv []*dns.SRV
}{
{ // wildcard resolution of a service works
dnsQuestionName: "foo.kubernetes.default.svc.cluster.local.",
expect: []*net.IP{&masterIP},
},
{ // resolving endpoints of a service works
dnsQuestionName: "_endpoints.kubernetes.default.svc.cluster.local.",
expect: []*net.IP{&localIP},
},
{ // openshift override works
dnsQuestionName: "openshift.default.svc.cluster.local.",
expect: []*net.IP{&masterIP},
},
{ // pod by IP
dnsQuestionName: "10-2-4-50.default.pod.cluster.local.",
expect: []*net.IP{&precannedIP},
},
{ // headless service
dnsQuestionName: "headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // specific port of a headless service
dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // SRV record for that service
dnsQuestionName: "headless.default.svc.cluster.local.",
srv: []*dns.SRV{
{
Target: headlessIPHash + "._unknown-port-2345._tcp.headless.default.svc.cluster.local.",
Port: 2345,
},
},
},
{ // the SRV record resolves to the IP
dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // headless 2 service
dnsQuestionName: "headless2.default.svc.cluster.local.",
expect: []*net.IP{&headless2IP},
},
{ // SRV records for that service
dnsQuestionName: "headless2.default.svc.cluster.local.",
srv: []*dns.SRV{
{
Target: headless2IPHash + "._http._tcp.headless2.default.svc.cluster.local.",
Port: 2346,
},
{
Target: headless2IPHash + "._other._tcp.headless2.default.svc.cluster.local.",
Port: 2345,
},
},
},
{ // the SRV record resolves to the IP
dnsQuestionName: "other.e1.headless2.default.svc.cluster.local.",
expect: []*net.IP{&headless2IP},
},
{
dnsQuestionName: "www.google.com.",
recursionExpected: true,
},
}
for i, tc := range tests {
qType := dns.TypeA
if tc.srv != nil {
qType = dns.TypeSRV
}
m1 := &dns.Msg{
MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: tc.recursionExpected},
Question: []dns.Question{{tc.dnsQuestionName, qType, dns.ClassINET}},
}
ch := make(chan struct{})
count := 0
failedLatency := 0
waitutil.Until(func() {
count++
if count > 100 {
t.Errorf("%d: failed after max iterations", i)
close(ch)
return
}
before := time.Now()
in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
if err != nil {
return
}
after := time.Now()
delta := after.Sub(before)
if delta > 500*time.Millisecond {
failedLatency++
if failedLatency > 10 {
t.Errorf("%d: failed after 10 requests took longer than 500ms", i)
close(ch)
}
return
}
switch {
case tc.srv != nil:
if len(in.Answer) != len(tc.srv) {
t.Logf("%d: incorrect number of answers: %#v", i, in)
return
}
case tc.recursionExpected:
if len(in.Answer) == 0 {
t.Errorf("%d: expected forward resolution: %#v", i, in)
}
close(ch)
return
default:
if len(in.Answer) != len(tc.expect) {
t.Logf("%d: did not resolve or unexpected forward resolution: %#v", i, in)
return
}
}
for _, answer := range in.Answer {
switch a := answer.(type) {
case *dns.A:
matches := false
if a.A != nil {
for _, expect := range tc.expect {
if a.A.String() == expect.String() {
matches = true
break
}
}
}
if !matches {
t.Errorf("%d: A record does not match any expected answer for %q: %v", i, tc.dnsQuestionName, a.A)
}
case *dns.SRV:
matches := false
for _, expect := range tc.srv {
if expect.Port == a.Port && expect.Target == a.Target {
matches = true
break
}
}
if !matches {
t.Errorf("%d: SRV record does not match any expected answer %q: %#v", i, tc.dnsQuestionName, a)
}
default:
t.Errorf("%d: expected an A or SRV record %q: %#v", i, tc.dnsQuestionName, in)
}
}
t.Log(in)
close(ch)
}, 50*time.Millisecond, ch)
}
}
func (c *ClientStartConfig) determineIP(out io.Writer) (string, error) {
if ip := net.ParseIP(c.PublicHostname); ip != nil && !ip.IsUnspecified() {
fmt.Fprintf(out, "Using public hostname IP %s as the host IP\n", ip)
return ip.String(), nil
}
// If using port-forwarding, find a local IP that can be used to communicate with the
// Origin container
if c.PortForwarding {
ip4, err := cmdutil.DefaultLocalIP4()
if err != nil {
return "", errors.NewError("cannot determine local IP address").WithCause(err)
}
glog.V(2).Infof("Testing local IP %s", ip4.String())
err = c.OpenShiftHelper().TestForwardedIP(ip4.String())
if err == nil {
return ip4.String(), nil
}
glog.V(2).Infof("Failed to use %s: %v", ip4.String(), err)
otherIPs, err := cmdutil.AllLocalIP4()
if err != nil {
return "", errors.NewError("cannot find local IP addresses to test").WithCause(err)
}
for _, ip := range otherIPs {
if ip.String() == ip4.String() {
continue
}
err = c.OpenShiftHelper().TestForwardedIP(ip.String())
if err == nil {
return ip.String(), nil
}
glog.V(2).Infof("Failed to use %s: %v", ip.String(), err)
}
return "", errors.NewError("could not determine local IP address to use").WithCause(err)
}
if len(c.DockerMachine) > 0 {
glog.V(2).Infof("Using docker machine %q to determine server IP", c.DockerMachine)
ip, err := dockermachine.IP(c.DockerMachine)
if err != nil {
return "", errors.NewError("Could not determine IP address").WithCause(err).WithSolution("Ensure that docker-machine is functional.")
}
fmt.Fprintf(out, "Using docker-machine IP %s as the host IP\n", ip)
return ip, nil
}
// First, try to get the host from the DOCKER_HOST if communicating via tcp
var err error
ip := c.DockerHelper().HostIP()
if ip != "" {
glog.V(2).Infof("Testing Docker host IP (%s)", ip)
if err = c.OpenShiftHelper().TestIP(ip); err == nil {
return ip, nil
}
}
glog.V(2).Infof("Cannot use the Docker host IP(%s): %v", ip, err)
// Next, use the the --print-ip output from openshift
ip, err = c.OpenShiftHelper().ServerIP()
if err == nil {
glog.V(2).Infof("Testing openshift --print-ip (%s)", ip)
if err = c.OpenShiftHelper().TestIP(ip); err == nil {
return ip, nil
}
glog.V(2).Infof("OpenShift server ip test failed: %v", err)
}
glog.V(2).Infof("Cannot use OpenShift IP: %v", err)
// Next, try other IPs on Docker host
ips, err := c.OpenShiftHelper().OtherIPs(ip)
if err != nil {
return "", err
}
for i := range ips {
glog.V(2).Infof("Testing additional IP (%s)", ip)
if err = c.OpenShiftHelper().TestIP(ips[i]); err == nil {
return ip, nil
}
glog.V(2).Infof("OpenShift additional ip test failed: %v", err)
}
return "", errors.NewError("cannot determine an IP to use for your server.")
}
func TestDNS(t *testing.T) {
masterConfig, clientFile, err := testserver.StartTestMaster()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
localAddr := ""
if ip, err := cmdutil.DefaultLocalIP4(); err == nil {
localAddr = ip.String()
} else if err == cmdutil.ErrorNoDefaultIP {
localAddr = "127.0.0.1"
} else if err != nil {
t.Fatalf("Unable to find a local IP address: %v", err)
}
localIP := net.ParseIP(localAddr)
var masterIP net.IP
// verify service DNS entry is visible
stop := make(chan struct{})
util.Until(func() {
m1 := &dns.Msg{
MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: false},
Question: []dns.Question{{"kubernetes.default.svc.cluster.local.", dns.TypeA, dns.ClassINET}},
}
in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
if err != nil {
t.Logf("unexpected error: %v", err)
return
}
if len(in.Answer) != 1 {
t.Logf("unexpected answer: %#v", in)
return
}
if a, ok := in.Answer[0].(*dns.A); ok {
if a.A == nil {
t.Fatalf("expected an A record with an IP: %#v", a)
}
masterIP = a.A
} else {
t.Fatalf("expected an A record: %#v", in)
}
t.Log(in)
close(stop)
}, 50*time.Millisecond, stop)
client, err := testutil.GetClusterAdminKubeClient(clientFile)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for {
if _, err := client.Services(kapi.NamespaceDefault).Create(&kapi.Service{
ObjectMeta: kapi.ObjectMeta{
Name: "headless",
},
Spec: kapi.ServiceSpec{
ClusterIP: kapi.ClusterIPNone,
Ports: []kapi.ServicePort{{Port: 443}},
},
}); err != nil {
if errors.IsForbidden(err) {
t.Logf("forbidden, sleeping: %v", err)
time.Sleep(100 * time.Millisecond)
continue
}
t.Fatalf("unexpected error: %v", err)
}
if _, err := client.Endpoints(kapi.NamespaceDefault).Create(&kapi.Endpoints{
ObjectMeta: kapi.ObjectMeta{
Name: "headless",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "172.0.0.1"}},
Ports: []kapi.EndpointPort{
{Port: 2345},
},
}},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
break
}
headlessIP := net.ParseIP("172.0.0.1")
if _, err := client.Services(kapi.NamespaceDefault).Create(&kapi.Service{
ObjectMeta: kapi.ObjectMeta{
Name: "headless2",
},
Spec: kapi.ServiceSpec{
ClusterIP: kapi.ClusterIPNone,
Ports: []kapi.ServicePort{{Port: 443}},
},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := client.Endpoints(kapi.NamespaceDefault).Create(&kapi.Endpoints{
ObjectMeta: kapi.ObjectMeta{
Name: "headless2",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "172.0.0.2"}},
Ports: []kapi.EndpointPort{
{Port: 2345, Name: "other"},
{Port: 2346, Name: "http"},
},
}},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
headless2IP := net.ParseIP("172.0.0.2")
tests := []struct {
dnsQuestionName string
recursionExpected bool
retry bool
expect []*net.IP
srv []*dns.SRV
}{
{ // wildcard resolution of a service works
dnsQuestionName: "foo.kubernetes.default.svc.cluster.local.",
expect: []*net.IP{&masterIP},
},
{ // resolving endpoints of a service works
dnsQuestionName: "_endpoints.kubernetes.default.svc.cluster.local.",
expect: []*net.IP{&localIP},
},
{ // openshift override works
dnsQuestionName: "openshift.default.svc.cluster.local.",
expect: []*net.IP{&masterIP},
},
{ // headless service
dnsQuestionName: "headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // specific port of a headless service
dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // SRV record for that service
dnsQuestionName: "headless.default.svc.cluster.local.",
srv: []*dns.SRV{
{
Target: "unknown-port-2345.e1.headless.",
Port: 2345,
},
},
},
{ // the SRV record resolves to the IP
dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
expect: []*net.IP{&headlessIP},
},
{ // headless 2 service
dnsQuestionName: "headless2.default.svc.cluster.local.",
expect: []*net.IP{&headless2IP},
},
{ // SRV records for that service
dnsQuestionName: "headless2.default.svc.cluster.local.",
srv: []*dns.SRV{
{
Target: "http.e1.headless2.",
Port: 2346,
},
{
Target: "other.e1.headless2.",
Port: 2345,
},
},
},
{ // the SRV record resolves to the IP
dnsQuestionName: "other.e1.headless2.default.svc.cluster.local.",
expect: []*net.IP{&headless2IP},
},
{
dnsQuestionName: "www.google.com.",
recursionExpected: false,
},
}
for i, tc := range tests {
qType := dns.TypeA
if tc.srv != nil {
qType = dns.TypeSRV
}
m1 := &dns.Msg{
MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: false},
Question: []dns.Question{{tc.dnsQuestionName, qType, dns.ClassINET}},
}
ch := make(chan struct{})
count := 0
util.Until(func() {
count++
if count > 100 {
t.Errorf("%d: failed after max iterations", i)
close(ch)
return
}
in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
if err != nil {
return
}
switch {
case tc.srv != nil:
if len(in.Answer) != len(tc.srv) {
t.Logf("%d: incorrect number of answers: %#v", i, in)
return
}
case tc.recursionExpected:
if len(in.Answer) == 0 {
t.Errorf("%d: expected forward resolution: %#v", i, in)
}
close(ch)
return
default:
if len(in.Answer) != len(tc.expect) {
t.Logf("%d: did not resolve or unexpected forward resolution: %#v", i, in)
return
}
}
for _, answer := range in.Answer {
switch a := answer.(type) {
case *dns.A:
matches := false
if a.A != nil {
for _, expect := range tc.expect {
if a.A.String() == expect.String() {
matches = true
break
}
}
}
if !matches {
t.Errorf("A record does not match any expected answer: %v", a.A)
}
case *dns.SRV:
matches := false
for _, expect := range tc.srv {
if expect.Port == a.Port && expect.Target == a.Target {
matches = true
break
}
}
if !matches {
t.Errorf("SRV record does not match any expected answer: %#v", a)
}
default:
t.Errorf("expected an A or SRV record: %#v", in)
}
}
t.Log(in)
close(ch)
}, 50*time.Millisecond, ch)
}
}