Question: <ins> /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package network import ( "context" "fmt" "net" "strconv" "strings" "time" compute "google.golang.org/api/compute/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" cloudprovider "k8s.io/cloud-provider" "k8s.io/kubernetes/test/e2e/framework" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/framework/providers/gce" e2erc "k8s.io/kubernetes/test/e2e/framework/rc" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" gcecloud "k8s.io/legacy-cloud-providers/gce" "github.com/onsi/ginkgo" "github.com/onsi/gomega" ) var _ = SIGDescribe("LoadBalancers", func() { f := framework.NewDefaultFramework("loadbalancers") var cs clientset.Interface serviceLBNames := []string{} ginkgo.BeforeEach(func() { cs = f.ClientSet }) ginkgo.AfterEach(func() { if ginkgo.CurrentGinkgoTestDescription().Failed { DescribeSvc(f.Namespace.Name) } for _, lb := range serviceLBNames { framework.Logf("cleaning load balancer resource for %s", lb) e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone) } //reset serviceLBNames serviceLBNames = []string{} }) ginkgo.It("should be able to change the type and ports of a TCP service [Slow]", func() { // requires cloud load-balancer support e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws") loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault if framework.ProviderIs("aws") { loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS } loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) // This test is more monolithic than we'd like because LB turnup can be // very slow, so we lumped all the tests into one LB lifecycle. serviceName := "mutability-test" ns1 := f.Namespace.Name // LB1 in ns1 on TCP framework.Logf("namespace for TCP test: %s", ns1) nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1) tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName) tcpService, err := tcpJig.CreateTCPService(nil) framework.ExpectNoError(err) svcPort := int(tcpService.Spec.Ports[0].Port) framework.Logf("service port TCP: %d", svcPort) ginkgo.By("creating a pod to be part of the TCP service " + serviceName) _, err = tcpJig.Run(nil) framework.ExpectNoError(err) // Change the services to NodePort. ginkgo.By("changing the TCP service to type=NodePort") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) // Change the services to LoadBalancer. // Here we test that LoadBalancers can receive static IP addresses.  This isn't // necessary, but is an additional feature this monolithic test checks. requestedIP := "" staticIPName := "" if framework.ProviderIs("gce", "gke") { ginkgo.By("creating a static load balancer IP") staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID) gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region()) defer func() { if staticIPName != "" { // Release GCE static IP - this is not kube-managed and will not be automatically released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Logf("failed to release static IP %s: %v", staticIPName, err) } } }() framework.ExpectNoError(err, "failed to create region address: %s", staticIPName) reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region()) framework.ExpectNoError(err, "failed to get region address: %s", staticIPName) requestedIP = reservedAddr.Address framework.Logf("Allocated static load balancer IP: %s", requestedIP) } ginkgo.By("changing the TCP service to type=LoadBalancer") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService)) ginkgo.By("waiting for the TCP service to have a load balancer") // Wait for the load balancer to be created asynchronously tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort) } if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) framework.Logf("TCP load balancer: %s", tcpIngressIP) if framework.ProviderIs("gce", "gke") { // Do this as early as possible, which overrides the `defer` above. // This is mostly out of fear of leaking the IP in a timeout case // (as of this writing we're not 100% sure where the leaks are // coming from, so this is first-aid rather than surgery). ginkgo.By("demoting the static IP to ephemeral") if staticIPName != "" { gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") // Deleting it after it is attached "demotes" it to an // ephemeral IP, which can be auto-released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Failf("failed to release static IP %s: %v", staticIPName, err) } staticIPName = "" } } ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' node ports. ginkgo.By("changing the TCP service's NodePort") tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort) framework.ExpectNoError(err) tcpNodePortOld := tcpNodePort tcpNodePort = int(tcpService.Spec.Ports[0].NodePort) if tcpNodePort == tcpNodePortOld { framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("hitting the TCP service's new NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' main ports. ginkgo.By("changing the TCP service's port") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) svcPortOld := svcPort svcPort = int(tcpService.Spec.Ports[0].Port) if svcPort == svcPortOld { framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort) } if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("service port TCP: %d", svcPort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) ginkgo.By("Scaling the pods to 0") err = tcpJig.Scale(0) framework.ExpectNoError(err) ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort") testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer") testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) ginkgo.By("Scaling the pods to 1") err = tcpJig.Scale(1) framework.ExpectNoError(err) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) // Change the services back to ClusterIP. ginkgo.By("changing TCP service back to type=ClusterIP") tcpReadback, err := tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if tcpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("TCP Spec.Ports[0].NodePort was not cleared") } // Wait for the load balancer to be destroyed asynchronously _, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) ginkgo.By("checking the TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the TCP LoadBalancer is closed") testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) }) ginkgo.It("should be able to change the type and ports of a UDP service [Slow]", func() { // requires cloud load-balancer support e2eskipper.SkipUnlessProviderIs("gce", "gke") loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) // This test is more monolithic than we'd like because LB turnup can be // very slow, so we lumped all the tests into one LB lifecycle. serviceName := "mutability-test" ns2 := f.Namespace.Name // LB1 in ns2 on TCP framework.Logf("namespace for TCP test: %s", ns2) nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2) udpJig := e2eservice.NewTestJig(cs, ns2, serviceName) udpService, err := udpJig.CreateUDPService(nil) framework.ExpectNoError(err) svcPort := int(udpService.Spec.Ports[0].Port) framework.Logf("service port UDP: %d", svcPort) ginkgo.By("creating a pod to be part of the UDP service " + serviceName) _, err = udpJig.Run(nil) framework.ExpectNoError(err) // Change the services to NodePort. ginkgo.By("changing the UDP service to type=NodePort") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) udpNodePort := int(udpService.Spec.Ports[0].NodePort) framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) // Change the services to LoadBalancer. // Here we test that LoadBalancers can receive static IP addresses.  This isn't // necessary, but is an additional feature this monolithic test checks. requestedIP := "" staticIPName := "" ginkgo.By("creating a static load balancer IP") staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID) gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region()) defer func() { if staticIPName != "" { // Release GCE static IP - this is not kube-managed and will not be automatically released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Logf("failed to release static IP %s: %v", staticIPName, err) } } }() framework.ExpectNoError(err, "failed to create region address: %s", staticIPName) reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region()) framework.ExpectNoError(err, "failed to get region address: %s", staticIPName) requestedIP = reservedAddr.Address framework.Logf("Allocated static load balancer IP: %s", requestedIP) ginkgo.By("changing the UDP service to type=LoadBalancer") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService)) // Do this as early as possible, which overrides the `defer` above. // This is mostly out of fear of leaking the IP in a timeout case // (as of this writing we're not 100% sure where the leaks are // coming from, so this is first-aid rather than surgery). ginkgo.By("demoting the static IP to ephemeral") if staticIPName != "" { gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") // Deleting it after it is attached "demotes" it to an // ephemeral IP, which can be auto-released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Failf("failed to release static IP %s: %v", staticIPName, err) } staticIPName = "" } var udpIngressIP string ginkgo.By("waiting for the UDP service to have a load balancer") // 2nd one should be faster since they ran in parallel. udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort) } udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) framework.Logf("UDP load balancer: %s", udpIngressIP) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' node ports. ginkgo.By("changing the UDP service's NodePort") udpService, err = udpJig.ChangeServiceNodePort(udpNodePort) framework.ExpectNoError(err) udpNodePortOld := udpNodePort udpNodePort = int(udpService.Spec.Ports[0].NodePort) if udpNodePort == udpNodePortOld { framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort) } if e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the UDP service's new NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' main ports. ginkgo.By("changing the UDP service's port") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].Port) != svcPort { framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port) } if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort) } if e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("service port UDP: %d", svcPort) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) ginkgo.By("Scaling the pods to 0") err = udpJig.Scale(0) framework.ExpectNoError(err) ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort") testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer") testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) ginkgo.By("Scaling the pods to 1") err = udpJig.Scale(1) framework.ExpectNoError(err) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) // Change the services back to ClusterIP. ginkgo.By("changing UDP service back to type=ClusterIP") udpReadback, err := udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if udpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("UDP Spec.Ports[0].NodePort was not cleared") } // Wait for the load balancer to be destroyed asynchronously _, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) ginkgo.By("checking the UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the UDP LoadBalancer is closed") testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) }) ginkgo.It("should only allow access from service loadbalancer source ranges [Slow]", func() { // this feature currently supported only on GCE/GKE/AWS e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws") loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) namespace := f.Namespace.Name serviceName := "lb-sourcerange" jig := e2eservice.NewTestJig(cs, namespace, serviceName) ginkgo.By("Prepare allow source ips") // prepare the exec pods // acceptPod are allowed to access the loadbalancer acceptPod := e2epod.CreateExecPodOrFail(cs, namespace, "execpod-accept", nil) dropPod := e2epod.CreateExecPodOrFail(cs, namespace, "execpod-drop", nil) ginkgo.By("creating a pod to be part of the service " + serviceName) // This container is an nginx container listening on port 80 // See kubernetes/contrib/ingress/echoheaders/nginx.conf for content of response _, err := jig.Run(nil) framework.ExpectNoError(err) // Make sure acceptPod is running. There are certain chances that pod might be teminated due to unexpected reasons. acceptPod, err = cs.CoreV1().Pods(namespace).Get(context.TODO(), acceptPod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get pod %s", acceptPod.Name) framework.ExpectEqual(acceptPod.Status.Phase, v1.PodRunning) framework.ExpectNotEqual(acceptPod.Status.PodIP, "") // Create loadbalancer service with source range from node[0] and podAccept svc, err := jig.CreateTCPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.LoadBalancerSourceRanges = []string{acceptPod.Status.PodIP + "/32"} }) framework.ExpectNoError(err) defer func() { ginkgo.By("Clean up loadbalancer service") e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) }() svc, err = jig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) ginkgo.By("check reachability from different sources") svcIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) // We should wait until service changes are actually propagated in the cloud-provider, // as this may take significant amount of time, especially in large clusters. // However, the information whether it was already programmed isn't achievable. // So we're resolving it by using loadBalancerCreateTimeout that takes cluster size into account. checkReachabilityFromPod(true, loadBalancerCreateTimeout, namespace, acceptPod.Name, svcIP) checkReachabilityFromPod(false, loadBalancerCreateTimeout, namespace, dropPod.Name, svcIP) // Make sure dropPod is running. There are certain chances that the pod might be teminated due to unexpected reasons. dropPod, err = cs.CoreV1().Pods(namespace).Get(context.TODO(), dropPod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get pod %s", dropPod.Name) framework.ExpectEqual(acceptPod.Status.Phase, v1.PodRunning) framework.ExpectNotEqual(acceptPod.Status.PodIP, "") ginkgo.By("Update service LoadBalancerSourceRange and check reachability") _, err = jig.UpdateService(func(svc *v1.Service) { // only allow access from dropPod svc.Spec.LoadBalancerSourceRanges = []string{dropPod.Status.PodIP + "/32"} }) framework.ExpectNoError(err) // We should wait until service changes are actually propagates, as this may take // significant amount of time, especially in large clusters. // However, the information whether it was already programmed isn't achievable. // So we're resolving it by using loadBalancerCreateTimeout that takes cluster size into account. checkReachabilityFromPod(false, loadBalancerCreateTimeout, namespace, acceptPod.Name, svcIP) checkReachabilityFromPod(true, loadBalancerCreateTimeout, namespace, dropPod.Name, svcIP) ginkgo.By("Delete LoadBalancerSourceRange field and check reachability") _, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.LoadBalancerSourceRanges = nil }) framework.ExpectNoError(err) // We should wait until service changes are actually propagates, as this may take // significant amount of time, especially in large clusters. // However, the information whether it was already programmed isn't achievable. // So we're resolving it by using loadBalancerCreateTimeout that takes cluster size into account. checkReachabilityFromPod(true, loadBalancerCreateTimeout, namespace, acceptPod.Name, svcIP) checkReachabilityFromPod(true, loadBalancerCreateTimeout, namespace, dropPod.Name, svcIP) }) ginkgo.It("should be able to create an internal type load balancer [Slow]", func() { e2eskipper.SkipUnlessProviderIs("azure", "gke", "gce") createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) pollInterval := framework.Poll * 10 namespace := f.Namespace.Name serviceName := "lb-internal" jig := e2eservice.NewTestJig(cs, namespace, serviceName) ginkgo.By("creating pod to be part of service " + serviceName) _, err := jig.Run(nil) framework.ExpectNoError(err) enableILB, disableILB := enableAndDisableInternalLB() isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool { ingressEndpoint := e2eservice.GetIngressPoint(lbIngress) // Needs update for providers using hostname as endpoint. return strings.HasPrefix(ingressEndpoint, "10.") } ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled") svc, err := jig.CreateTCPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer enableILB(svc) }) framework.ExpectNoError(err) defer func() { ginkgo.By("Clean up loadbalancer service") e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) }() svc, err = jig.WaitForLoadBalancer(createTimeout) framework.ExpectNoError(err) lbIngress := &svc.Status.LoadBalancer.Ingress[0] svcPort := int(svc.Spec.Ports[0].Port) // should have an internal IP. framework.ExpectEqual(isInternalEndpoint(lbIngress), true) // ILBs are not accessible from the test orchestrator, so it's necessary to use //  a pod to test the service. ginkgo.By("hitting the internal load balancer from pod") framework.Logf("creating pod with host network") hostExec := launchHostExecPod(f.ClientSet, f.Namespace.Name, "ilb-host-exec") framework.Logf("Waiting up to %v for service %q's internal LB to respond to requests", createTimeout, serviceName) tcpIngressIP := e2eservice.GetIngressPoint(lbIngress) if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) { cmd := fmt.Sprintf(`curl -m 5 'http://%v:%v/echo?msg=hello'`, tcpIngressIP, svcPort) stdout, err := framework.RunHostCmd(hostExec.Namespace, hostExec.Name, cmd) if err != nil { framework.Logf("error curling; stdout: %v. err: %v", stdout, err) return false, nil } if !strings.Contains(stdout, "hello") { framework.Logf("Expected output to contain 'hello', got %q; retrying...", stdout) return false, nil } framework.Logf("Successful curl; stdout: %v", stdout) return true, nil }); pollErr != nil { framework.Failf("ginkgo.Failed to hit ILB IP, err: %v", pollErr) } ginkgo.By("switching to external type LoadBalancer") svc, err = jig.UpdateService(func(svc *v1.Service) { disableILB(svc) }) framework.ExpectNoError(err) framework.Logf("Waiting up to %v for service %q to have an external LoadBalancer", createTimeout, serviceName) if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) { svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err != nil { return false, err } lbIngress = &svc.Status.LoadBalancer.Ingress[0] return !isInternalEndpoint(lbIngress), nil }); pollErr != nil { framework.Failf("Loadbalancer IP not changed to external.") } // should have an external IP. gomega.Expect(isInternalEndpoint(lbIngress)).To(gomega.BeFalse()) ginkgo.By("hitting the external load balancer") framework.Logf("Waiting up to %v for service %q's external LB to respond to requests", createTimeout, serviceName) tcpIngressIP = e2eservice.GetIngressPoint(lbIngress) e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, e2eservice.LoadBalancerLagTimeoutDefault) // GCE cannot test a specific IP because the test may not own it. This cloud specific condition // will be removed when GCP supports similar functionality. if framework.ProviderIs("azure") { ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.") internalStaticIP := "10.240.11.11" svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.LoadBalancerIP = internalStaticIP enableILB(svc) }) framework.ExpectNoError(err) framework.Logf("Waiting up to %v for service %q to have an internal LoadBalancer", createTimeout, serviceName) if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) { svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err != nil { return false, err } lbIngress = &svc.Status.LoadBalancer.Ingress[0] return isInternalEndpoint(lbIngress), nil }); pollErr != nil { framework.Failf("Loadbalancer IP not changed to internal.") } // should have the given static internal IP. framework.ExpectEqual(e2eservice.GetIngressPoint(lbIngress), internalStaticIP) } }) // This test creates a load balancer, make sure its health check interval // equals to gceHcCheckIntervalSeconds. Then the interval is manipulated // to be something else, see if the interval will be reconciled. ginkgo.It("should reconcile LB health check interval [Slow][Serial]", func() { const gceHcCheckIntervalSeconds = int64(8) // This test is for clusters on GCE. // (It restarts kube-controller-manager, which we don't support on GKE) e2eskipper.SkipUnlessProviderIs("gce") e2eskipper.SkipUnlessSSHKeyPresent() clusterID, err := gce.GetClusterID(cs) if err != nil { framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err) } gceCloud, err := gce.GetGCECloud() if err != nil { framework.Failf("framework.GetGCECloud() = _, %v; want nil", err) } namespace := f.Namespace.Name serviceName := "lb-hc-int" jig := e2eservice.NewTestJig(cs, namespace, serviceName) ginkgo.By("create load balancer service") // Create loadbalancer service with source range from node[0] and podAccept svc, err := jig.CreateTCPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) defer func() { ginkgo.By("Clean up loadbalancer service") e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) }() svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs)) framework.ExpectNoError(err) hcName := gcecloud.MakeNodesHealthCheckName(clusterID) hc, err := gceCloud.GetHTTPHealthCheck(hcName) if err != nil { framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err) } framework.ExpectEqual(hc.CheckIntervalSec, gceHcCheckIntervalSeconds) ginkgo.By("modify the health check interval") hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1 if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil { framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err) } ginkgo.By("restart kube-controller-manager") if err := e2ekubesystem.RestartControllerManager(); err != nil { framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err) } if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil { framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err) } ginkgo.By("health check should be reconciled") pollInterval := framework.Poll * 10 loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs) if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) { hc, err := gceCloud.GetHTTPHealthCheck(hcName) if err != nil { framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err) return false, err } framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec) return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil }); pollErr != nil { framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds) } }) // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed. // [LinuxOnly]: Windows does not support session affinity. ginkgo.It("should have session affinity work for LoadBalancer service with ESIPP on [Slow] [DisabledForLargeClusters] [LinuxOnly]", func() { // L4 load balancer affinity `ClientIP` is not supported on AWS ELB. e2eskipper.SkipIfProviderIs("aws") svc := getServeHostnameService("affinity-lb-esipp") svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal execAffinityTestForLBService(f, cs, svc) }) // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed. // [LinuxOnly]: Windows does not support session affinity. ginkgo.It("should be able to switch session affinity for LoadBalancer service with ESIPP on [Slow] [DisabledForLargeClusters] [LinuxOnly]", func() { // L4 load balancer affinity `ClientIP` is not supported on AWS ELB. e2eskipper.SkipIfProviderIs("aws") svc := getServeHostnameService("affinity-lb-esipp-transition") svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal execAffinityTestForLBServiceWithTransition(f, cs, svc) }) // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed. // [LinuxOnly]: Windows does not support session affinity. ginkgo.It("should have session affinity work for LoadBalancer service with ESIPP off [Slow] [DisabledForLargeClusters] [LinuxOnly]", func() { // L4 load balancer affinity `ClientIP` is not supported on AWS ELB. e2eskipper.SkipIfProviderIs("aws") svc := getServeHostnameService("affinity-lb") svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster execAffinityTestForLBService(f, cs, svc) }) // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed. // [LinuxOnly]: Windows does not support session affinity. ginkgo.It("should be able to switch session affinity for LoadBalancer service with ESIPP off [Slow] [DisabledForLargeClusters] [LinuxOnly]", func() { // L4 load balancer affinity `ClientIP` is not supported on AWS ELB. e2eskipper.SkipIfProviderIs("aws") svc := getServeHostnameService("affinity-lb-transition") svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster execAffinityTestForLBServiceWithTransition(f, cs, svc) }) // This test verifies if service load balancer cleanup finalizer is properly // handled during service lifecycle. // 1. Create service with type=LoadBalancer. Finalizer should be added. // 2. Update service to type=ClusterIP. Finalizer should be removed. // 3. Update service to type=LoadBalancer. Finalizer should be added. // 4. Delete service with type=LoadBalancer. Finalizer should be removed. ginkgo.It("should handle load balancer cleanup finalizer for service [Slow]", func() { jig := e2eservice.NewTestJig(cs, f.Namespace.Name, "lb-finalizer") ginkgo.By("Create load balancer service") svc, err := jig.CreateTCPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) defer func() { ginkgo.By("Check that service can be deleted with finalizer") e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) }() ginkgo.By("Wait for load balancer to serve traffic") svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs)) framework.ExpectNoError(err) ginkgo.By("Check if finalizer presents on service with type=LoadBalancer") e2eservice.WaitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true) ginkgo.By("Check if finalizer is removed on service after changed to type=ClusterIP") err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, e2eservice.GetServiceLoadBalancerCreationTimeout(cs)) framework.ExpectNoError(err) e2eservice.WaitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, false) ginkgo.By("Check if finalizer is added back to service after changed to type=LoadBalancer") err = jig.ChangeServiceType(v1.ServiceTypeLoadBalancer, e2eservice.GetServiceLoadBalancerCreationTimeout(cs)) framework.ExpectNoError(err) e2eservice.WaitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true) }) }) var _ = SIGDescribe("ESIPP [Slow]", func() { f := framework.NewDefaultFramework("esipp") var loadBalancerCreateTimeout time.Duration var cs clientset.Interface serviceLBNames := []string{} ginkgo.BeforeEach(func() { // requires cloud load-balancer support - this feature currently supported only on GCE/GKE e2eskipper.SkipUnlessProviderIs("gce", "gke") cs = f.ClientSet loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs) }) ginkgo.AfterEach(func() { if ginkgo.CurrentGinkgoTestDescription().Failed { DescribeSvc(f.Namespace.Name) } for _, lb := range serviceLBNames { framework.Logf("cleaning load balancer resource for %s", lb) e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone) } //reset serviceLBNames serviceLBNames = []string{} }) ginkgo.It("should work for type=LoadBalancer", func() { namespace := f.Namespace.Name serviceName := "external-local-lb" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) if healthCheckNodePort == 0 { framework.Failf("Service HealthCheck NodePort was not allocated") } defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) // Make sure we didn't leak the health check node port. const threshold = 2 nodes, err := getEndpointNodesWithInternalIP(jig) framework.ExpectNoError(err) config := e2enetwork.NewNetworkingTestConfig(f) for _, internalIP := range nodes { err := testHTTPHealthCheckNodePortFromTestContainer( config, internalIP, healthCheckNodePort, e2eservice.KubeProxyLagTimeout, false, threshold) framework.ExpectNoError(err) } err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() svcTCPPort := int(svc.Spec.Ports[0].Port) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) ginkgo.By("reading clientIP using the TCP service's service port via its external VIP") clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip") framework.ExpectNoError(err) framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP) ginkgo.By("checking if Source IP is preserved") if strings.HasPrefix(clientIP, "10.") { framework.Failf("Source IP was NOT preserved") } }) ginkgo.It("should work for type=NodePort", func() { namespace := f.Namespace.Name serviceName := "external-local-nodeport" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalNodePortService(true) framework.ExpectNoError(err) defer func() { err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() tcpNodePort := int(svc.Spec.Ports[0].NodePort) endpointsNodeMap, err := getEndpointNodesWithInternalIP(jig) framework.ExpectNoError(err) dialCmd := "clientip" config := e2enetwork.NewNetworkingTestConfig(f) for nodeName, nodeIP := range endpointsNodeMap { ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v:%v/%v", nodeName, nodeIP, tcpNodePort, dialCmd)) clientIP, err := GetHTTPContentFromTestContainer(config, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, dialCmd) framework.ExpectNoError(err) framework.Logf("ClientIP detected by target pod using NodePort is %s, the ip of test container is %s", clientIP, config.TestContainerPod.Status.PodIP) // the clientIP returned by agnhost contains port if !strings.HasPrefix(clientIP, config.TestContainerPod.Status.PodIP) { framework.Failf("Source IP was NOT preserved") } } }) ginkgo.It("should only target nodes with endpoints", func() { namespace := f.Namespace.Name serviceName := "external-local-nodes" jig := e2eservice.NewTestJig(cs, namespace, serviceName) nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests) framework.ExpectNoError(err) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false, func(svc *v1.Service) { // Change service port to avoid collision with opened hostPorts // in other tests that run in parallel. if len(svc.Spec.Ports) != 0 { svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port)) svc.Spec.Ports[0].Port = 8081 } }) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) if healthCheckNodePort == 0 { framework.Failf("Service HealthCheck NodePort was not allocated") } ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) svcTCPPort := int(svc.Spec.Ports[0].Port) const threshold = 2 config := e2enetwork.NewNetworkingTestConfig(f) for i := 0; i < len(nodes.Items); i++ { endpointNodeName := nodes.Items[i].Name ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName) _, err = jig.Run(func(rc *v1.ReplicationController) { rc.Name = serviceName if endpointNodeName != "" { rc.Spec.Template.Spec.NodeName = endpointNodeName } }) framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName)) err = jig.WaitForEndpointOnNode(endpointNodeName) framework.ExpectNoError(err) // HealthCheck should pass only on the node where num(endpoints) > 0 // All other nodes should fail the healthcheck on the service healthCheckNodePort for n, internalIP := range ips { // Make sure the loadbalancer picked up the health check change. // Confirm traffic can reach backend through LB before checking healthcheck nodeport. e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout) expectedSuccess := nodes.Items[n].Name == endpointNodeName port := strconv.Itoa(healthCheckNodePort) ipPort := net.JoinHostPort(internalIP, port) framework.Logf("Health checking %s, http://%s/healthz, expectedSuccess %v", nodes.Items[n].Name, ipPort, expectedSuccess) err := testHTTPHealthCheckNodePortFromTestContainer( config, internalIP, healthCheckNodePort, e2eservice.KubeProxyEndpointLagTimeout, expectedSuccess, threshold) framework.ExpectNoError(err) } framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName)) } }) ginkgo.It("should work from pods", func() { var err error namespace := f.Namespace.Name serviceName := "external-local-pods" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) port := strconv.Itoa(int(svc.Spec.Ports[0].Port)) ipPort := net.JoinHostPort(ingressIP, port) path := fmt.Sprintf("%s/clientip", ipPort) ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state") deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1) framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment") defer func() { framework.Logf("Deleting deployment") err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name) }() deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Error in retrieving pause pod deployment") labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment") pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()}) framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments") pausePod := pausePods.Items[0] framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path) var srcIP string loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs) ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName)) if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) { stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd) if err != nil { framework.Logf("got err: %v, retry until timeout", err) return false, nil } srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0]) return srcIP == pausePod.Status.PodIP, nil }); pollErr != nil { framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP) } }) ginkgo.It("should handle updates to ExternalTrafficPolicy field", func() { namespace := f.Namespace.Name serviceName := "external-local-update" jig := e2eservice.NewTestJig(cs, namespace, serviceName) nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests) framework.ExpectNoError(err) if len(nodes.Items) < 2 { framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint") } svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() // save the health check node port because it disappears when ESIPP is turned off. healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) ginkgo.By("turning ESIPP off") svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster }) framework.ExpectNoError(err) if svc.Spec.HealthCheckNodePort > 0 { framework.Failf("Service HealthCheck NodePort still present") } epNodes, err := jig.ListNodesWithEndpoint() framework.ExpectNoError(err) // map from name of nodes with endpoint to internal ip // it is assumed that there is only a single node with the endpoint endpointNodeMap := make(map[string]string) // map from name of nodes without endpoint to internal ip noEndpointNodeMap := make(map[string]string) for _, node := range epNodes { ips := e2enode.GetAddresses(&node, v1.NodeInternalIP) if len(ips) < 1 { framework.Failf("No internal ip found for node %s", node.Name) } endpointNodeMap[node.Name] = ips[0] } for _, n := range nodes.Items { ips := e2enode.GetAddresses(&n, v1.NodeInternalIP) if len(ips) < 1 { framework.Failf("No internal ip found for node %s", n.Name) } if _, ok := endpointNodeMap[n.Name]; !ok { noEndpointNodeMap[n.Name] = ips[0] } } framework.ExpectNotEqual(len(endpointNodeMap), 0) framework.ExpectNotEqual(len(noEndpointNodeMap), 0) svcTCPPort := int(svc.Spec.Ports[0].Port) svcNodePort := int(svc.Spec.Ports[0].NodePort) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) path := "/clientip" dialCmd := "clientip" config := e2enetwork.NewNetworkingTestConfig(f) ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap)) for nodeName, nodeIP := range noEndpointNodeMap { ginkgo.By(fmt.Sprintf("Checking %v (%v:%v/%v) proxies to endpoints on another node", nodeName, nodeIP[0], svcNodePort, dialCmd)) _, err := GetHTTPContentFromTestContainer(config, nodeIP, svcNodePort, e2eservice.KubeProxyLagTimeout, dialCmd) framework.ExpectNoError(err, "Could not reach HTTP service through %v:%v/%v after %v", nodeIP, svcNodePort, dialCmd, e2eservice.KubeProxyLagTimeout) } for nodeName, nodeIP := range endpointNodeMap { ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIP)) var body string pollFn := func() (bool, error) { // we expect connection failure here, but not other errors resp, err := config.GetResponseFromTestContainer( "http", "healthz", nodeIP, healthCheckNodePort) if err != nil { return false, nil } if len(resp.Errors) > 0 { return true, nil } if len(resp.Responses) > 0 { body = resp.Responses[0] } return false, nil } if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollFn); pollErr != nil { framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s", nodeName, healthCheckNodePort, body) } } // Poll till kube-proxy re-adds the MASQUERADE rule on the node. ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP)) var clientIP string pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) { clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip") if err != nil { return false, nil } if strings.HasPrefix(clientIP, "10.") { return true, nil } return false, nil }) if pollErr != nil { framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP) } // TODO: We need to attempt to create another service with the previously // allocated healthcheck nodePort. If the health check nodePort has been // freed, the new service creation will succeed, upon which we cleanup. // If the health check nodePort has NOT been freed, the new service // creation will fail. ginkgo.By("setting ExternalTraffic field back to OnlyLocal") svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal // Request the same healthCheckNodePort as before, to test the user-requested allocation path svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort) }) framework.ExpectNoError(err) pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) { clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path) if err != nil { return false, nil } ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP)) if !strings.HasPrefix(clientIP, "10.") { return true, nil } return false, nil }) if pollErr != nil { framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP) } }) }) </ins>

Which jobs are flaking: gce-master-scale-correctness Which test(s) are flaking: Services should be able to change the type and ports of a service [Slow] Testgrid link: https://k8s- Reason for failure: Test timeouts in ~half of the runs since it was enabled here: /sig scalability /sig network
/cc /cc
The context is - the problems are strictly related to scale - 5k-node jobs are timing out. I very briefly skimmed through the test and didn't see anything obviously suspicious (apart from the fact there are gazillions long timeouts there, which alltogether sum up long hours). That said - it seems to me that there are effectively two separate tests inside it: one for TCP one for UDP They don't seem to share anything with each other. - would it make sense to split this test two effectively two separate tests: one for TCP and one for UDP. (I guess it would also allow us to share some code there).
[And potentially it could help with debugging the problem if it is strictly related to only one of those cases...]
I wanted to focus on this past week but I didn´t have time, this is my list of more expensive times and approx average duration, all of the sig-network ones use LoadBalancers. I wanted to time the test to understand if there are places where can save time ... I don´t have much context on those tests, maybe Tim can help us more, but this comment makes me think that the use of both protocols is intended
/triage accepted
/triage accepted It's been a very long time since I touched this, and I recall holding my nose on it in the first place. Splitting TCP from UDP seems reasonable if we can run them in parallel. we'll want to share the flow, and parameterize some shared funcs. I don't know that this will prevent test timeouts, though.
We could use a volunteer to go poke at this, and see how hard it will really be.
You mean "forcing them to run in parallel"? Or "allowing them to run in parallel"? The later doesn't require any additiona work. The former I think mean that it will still have to be a single test (just the code will be organized better) and that won't really help with debugging (nor preventing timeouts).

this was discussed yesterday in sig-net meeting and my understanding was to split current test in 2 different so the test can run in parallel, if you want to do so and one protocol doesn´t have to wait for the other. I think that is what you meant here
As long as we CAN parallelize. Creating LBs can be sloooooow. :P Antonio Ojea <:
Reopening - I don't think just splitting will address the problem. It may help us with debugging though...
With the split we moved from 60 mins to run the test, to run 2 tests in [30-40] mins each https://k8s- Seems a win :smile:
Awesome - that's something I was silently counting for TBH :) [I doubt it will fix flakiness, but maybe it will make it easier to debug...]
Actually - I think we're getting close thanks to the two changes ( + this one). Here is the run: This particular one filed on UDP (not sure if it's just UDP-related, or just manifested with UDP now). And we have a clear failure now: In my PR I decreased the timeout from 2h to 45m, as it seemed to me that in the past it should be enough. But it failed on 45m timeout here. So I think the next question now is: is it that slow - that we need more than 45m or something isn't working as expected (either in the k8s itself or in the test) But I think we have a clear single place to look at now.
Actually - the full stacktrace: So it's this line that failed: What is interesting is that something was changing during those 45 minutes:
Note that it's 4th time we actually wait for LB in this test. The failure happens after the pod were scaled down to 0 and then to 1. So that makes me think, that's it's actually not LB-related, but rather something internally in k8s, becuase L4 LB doesn't depend on pods - it targets nodes...
If it is always UDP, I start to see ghosts between GCE and UDP ... we only have KIND to compare against, or do we have some other non gce enviroment?

I mean, there are other tests failing in UDP , and show much more failures in GCE than in kind
- this does not look "normal". Can we give some prio to helping track this flake?
hmm interesting. we will investigate.
/assign
- any progress on this?
I haven't had a chance to look into this yet. I will prioritize it this week.
Digging through the logs I believe this was a bug in the test code that was fixed by . In the test, the service port changes around 12:33. However due to the bug, the test continues to use the old service port as seen in the test output: The port should be 81. I believe the following sequence occurs from this point: 1) The reachability test fails in the beginning because the service has changed the port and the load balancer has not been updated yet resulting in the error. 1) Around 12:38, the service controller is syncing the service and notices that the service port has changed and starts to delete the forwarding rule causing the error to change to . 1) Once the forwarding rule is recreated, the error goes back to being because the forwarding rule exists but the wrong port is used. Looking at , the test runs after the one linked have all passed. Is there anything else that needs to be investigated as part of this issue?
OK - the test was quite flaky in the past but indeed it didn't flake for the last 3 weeks (neither this nor TCP one). I'm fine with closing this one - we can open a new one if this happens again. - thanks for debugging. - it seems that the splitting of the test actually made things waay better. Which is great.


Question: return service, err } <ins> func (sr *ServiceRegistryStorage) deleteExternalLoadBalancer(service *api.Service) error { if !service.CreateExternalLoadBalancer || sr.cloud == nil { return nil } zones, ok := sr.cloud.Zones() if !ok { // We failed to get zone enumerator. // As this should have failed when we tried in "create" too, // assume external load balancer was never created. return nil } balancer, ok := sr.cloud.TCPLoadBalancer() if !ok { // See comment above. return nil } zone, err := zones.GetZone() if err != nil { return err } if err := balancer.DeleteTCPLoadBalancer(service.JSONBase.ID, zone); err != nil { return err } return nil } </ins> func (sr *ServiceRegistryStorage) Delete(id string) (<-chan interface{}, error) { service, err := sr.registry.GetService(id) if err != nil { return nil, err } return apiserver.MakeAsync(func() (interface{}, error) { <del> if service.CreateExternalLoadBalancer { var balancer cloudprovider.TCPLoadBalancer var ok bool if sr.cloud != nil { balancer, ok = sr.cloud.TCPLoadBalancer() } if ok && balancer != nil { err = balancer.DeleteTCPLoadBalancer(id, "us-central1") if err != nil { return nil, err } } } </del> <ins> sr.deleteExternalLoadBalancer(service) </ins> return api.Status{Status: api.StatusSuccess}, sr.registry.DeleteService(id) }), nil }

pull-kubernetes-e2e-azure-disk-windows pull-kubernetes-e2e-azure-disk-windows It seems caused by connection failed: from kubelet logs: This repeats until timeout
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig windows
/area providers/azure
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /area providers/azure Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Last run (): The run before (): no code changes in between
Now my PR passed the lane without any code changes (just yet another retest):
note that is not a presubmit for merge, or else it would show: <img width="89" alt="Screen Shot 2021-03-08 at 11 30 57 PM" src="https://user-"non-required presubmits can be bypassed with , but are generally worth looking at first
/assign Can you take a look?
There's something odd going on here. As the test description indicates, it deletes the pod for the StatefulSet and expects it to be recreated. When the kubelet attempts to recreate the pod, however, the mount point for the volume is no longer there. From the log above, we can see the pod was deleted succesfully. It was then rescheduled but failed because the volume mount point does not exist. In the kubelet logs, we see the successful mount for the initial pod creation: We then see the delete of the pod and the failed attempt to recreate it: Finally during cleanup, we see a failure during unmount the volume because the mount point does not exist: None of the other collected logs show what happened to the original mount point.


Question: go_test( name = "go_default_test", <del> srcs = select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ "proxier_test.go", ], "//conditions:default": [], }), </del> <ins> srcs = ["proxier_test.go"], </ins> library = ":go_default_library", <del> deps = select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ "//pkg/api:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/util:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//pkg/util/ipvs:go_default_library", "//pkg/util/ipvs/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], "//conditions:default": [], }), </del> <ins> deps = [ "//pkg/api:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/util:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//pkg/util/ipvs:go_default_library", "//pkg/util/ipvs/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], </ins> ) go_library(

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Currently, IPVS proxier only runs in linux platfrom. What you expected to happen: Run IPVS proxier UTs everywhere, including !linux platfrom, which will help a lot when developing in windows platfrom. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration**: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig network
/area kube-proxy


Question: impersonationUserExtras: map[string][]string{"scopes": {"scope-a", "scope-b"}}, expectedUser: &user.DefaultInfo{ Name:   "system:admin", <del> Groups: []string{}, </del> <ins> Groups: []string{"system:authenticated"}, </ins> Extra:  map[string][]string{"scopes": {"scope-a", "scope-b"}}, }, expectedCode: http.StatusOK,

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): impersonate system:authenticated Is this a BUG REPORT or FEATURE REQUEST? (choose one): FEATURE REQUEST <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): Environment: Cloud provider or hardware configuration: baremetal OS (e.g. from /etc/os-release): Centos7 Kernel (e.g. ): Install tools: custom Others: Using RBAC What happened: Impersonating a user is not setting group which causes impersonate to fail What you expected to happen: If a user is in the tokens file they and they are impersonated the group should be for authz. How to reproduce it (as minimally and precisely as possible): sure group has cluster role a test user via token file a user with impersonate access submit a command via kubectl That request will fail with 403 response even though the user is in the group. Anything else we need to know: cc
We can probably be smart and put the authenticated group adder around the result of the impersonation filter. you see any issues with that?
We provide the group field to provide this kind of information. I'm not sure I would expect my request to the server about a particular user/group tuple to be evaluated for a different user/group tuple.
for impersonation, we already add groups when known (service accounts). doing so for the system:authenticated/system:unauthenticated groups makes sense to me
Only when the groups aren't specified, right? If memory serves, that's how we did it downstream where we can make group determination.


Question: if len(kp.Path) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("path"), "")) } <del> allErrs = append(allErrs, validateLocalNonReservedPath(kp.Path, fldPath.Child("path"))...) </del> <ins> allErrs = append(allErrs, ValidateLocalNonReservedPath(kp.Path, fldPath.Child("path"))...) </ins> if kp.Mode != nil && (*kp.Mode > 0777 || *kp.Mode < 0) { allErrs = append(allErrs, field.Invalid(fldPath.Child("mode"), *kp.Mode, fileModeErrorMsg)) }

Originally posted by in
Hello, I'm new here. Can I try solving this? it seems like a good first issue
sure, context is the change is needed here: We have to tolerate existing pods / pod specs that fail this validation, so we should only do this additional validation on create or if the existing object passes this validation. To do this: an field to , initialize that field to false, and then conditionally override it to if is non-nil and has a projected token volume path that fails (will have to export the function to check this) , if is false, call the validation on the path:
Hi, I'd like to work on this if there is no one is proceeding on. /assign


Question: // excludeCIDRs is a list of CIDR's which the ipvs proxier should not touch // when cleaning up ipvs services. ExcludeCIDRs []string <ins> // strict ARP configure arp_ignore and arp_announce to avoid answering ARP queries // from kube-ipvs0 interface StrictARP bool </ins> } // KubeProxyConntrackConfiguration contains conntrack settings for

/kind bug Kubernetes version: 1.13.1. kube-proxy mode: IPVS. SDN: flannel. We are used services like this: where IP 192.168.10.201 is from our internal /24 network (in particular, our Kube nodes have IPs 192.168.10.1, 192.168.10.2 etc). Note that there is no conflicts between real IPs and virtual external ones. Before updating from 1.12.3 to 1.13.1, it worked correctly. In partucular these IPs was accessible from other non-Kubernetes hosts from 192.168.10.0/24 network. After update kube-proxy to 1.13.1, this feature stops working: non-Kubernetes hosts cannot resolve service IPs via ARP: On Kubernetes hosts it still working, because not requires ARP resolving. If we reset sysctl parameter arpignore on Kube hosts, all works normally even from non-Kube hosts. Quck debugging shows that problem is in the commit . Please make this behaviour opt-out. (Now we resets arpignore to 0 via cron every 1 min.)
/sig network
Having the same problem on Kubernetes 1.13.2 btw Thanks for a workaround!
Same problem, fix this please!!!
This is similar to We've had a discussion there on how to best implement a solution but haven't decided yet I'm in favor of flag which defaults to the previous behavior to avoid breaking existing setups (alternatives are cleaner but involve creating an additional arptable rule per service which I think we should avoid) what do you think?
OK

What happened: After PR: , we noticed communications issue in our pods. We use the Lyft CNI plugin () which relies on two interfaces in pod: an IPVLAN interface connecting to the AWS VPC which has a VPC IP an Unnumbered point-to-point interface (veth) connecting to the host and that does not have an IP (it "borrows" the one from the IPVLAN interface) On the host side, there is a veth interface which also doesn't have an IP (it borrows one from the main interface). With the latest versions of kube-proxy communication between host and pods break What you expected to happen: Communication still works How to reproduce it (as minimally and precisely as possible): Use the Lyft CNI plugin Deploy kube-proxy Try to ping the host from a pod (or a pod from the host) Anything else we need to know?: The new sysctls create 2 issues: containers started after kube-proxy inherit so they will not answer ARP queries on the veth interface the veth on the host side also does not have an IP so it will no answer ARP queries from the pods I know that our setup is pretty uncommon but assuming that the veth on the host side will have an IP seems dangerous. Some discussions on the PTP plugin for instance suggest not using IP for the host interface: containernetworking/plugins Possible fixes: create a flag to enable ARP filtering (I think to avoid breaking existing setups, we should disable it by default but I'm fine with enabling it by default as long as we document the solution) use arptables to filter ARP queries for clusterIPs (): a bit more work because we need to do it for all clusterIPs (there may be a more efficient way to do it but I could not make it work by filtering on the kube-ipvs interface) and this requires the arptables binary. We could test for the existence of the binary and log a warning if it is missing (similar to behavior with conntrack) Environment: Kubernetes version (use ): kube-proxy OS (e.g. from /etc/os-release): Kernel (e.g. ): /area ipvs /sig network /kind bug /assign ping
I can say only that solution with arptables working fine with MetalLB.
Oh, really? Glad you confirm that.

But I would like to make sure that we won't create too many arptables rules. Can ipset works with arptables?
It doesn't look like arptables works with ipset
Unfortunately doesn't support ipsets, but new do. (not fully implemented yet) Eg we can create ipset for arp: But current implementation is not allowing to use it for arp rules for now: Will return: I guess because there is still no type specified for (destination ip) and (source ip) BTW: nft does not allow to create set of integer components: But there is a way for specify multiple IPs in one rule, eg: Where is IPv4 addresses in decimal format More info here:
Another option would be to use the and filter all queries but this flag is not in kube-proxy today
Bad idea, it will handle but not - which can be really differ
can you explain more detail why Lyft CNI can not assign IPs to the right interfaces for avoid this kind of issues?
Yes you are right. bad idea It could definitely add IPs to these interfaces but it's not the choice that was made. More details here: ("Unnumbered point-to-point interface"). I could probably PR a fix there. I'm a bit worried that some other CNI plugins could break here though
We are about to deploy some 1.13 clusters and can't use kube-proxy 1.13 as is We could start with a custom build but I assume others will be impacted The simplest way forward for now is probably to use a flag to control this behavior. I think the safest is probably to disable this by default to avoid the issue for people upgrading but if you think it is really important we can enable it by default. I know we'd like to avoid adding more flags but other solutions seem a bit more complicated and will most likely result in one iptables rule per service. What do you think ?
Just FYI, we have another CNI plugin that is affected by this.
We addressed the issue by assigning a link-local address 169.254.1.1/32 to the host-side veth in . So this issue is not critical at least for our case.
Good news everyone, new nft supports setting ip address for arp filters : that's makes possible for using sets


Question: if err != nil { return err } <del> fmt.Fprintf(o.Out, string(jsonBuffer.Bytes())) </del> <ins> fmt.Fprint(o.Out, string(jsonBuffer.Bytes())) </ins> case "yaml": yamlOutput, err := yaml.JSONToYAML([]byte(str)) if err != nil { return err } <del> fmt.Fprintf(o.Out, string(yamlOutput)) </del> <ins> fmt.Fprint(o.Out, string(yamlOutput)) </ins> } }

Is this a request for help?: No What keywords did you search in Kubernetes issues before filing this one?: save-config, view-last-applied Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version: Environment: Cloud provider or hardware configuration: GKE OS (e.g. from /etc/os-release): cos What happened: Create this: Then we print it: Notice that is printed as . But the saved config is correct, so it's clearly a YAML-printing error: This seems to be caused by being interpreted as a Go format code, because I also get this for other occurrences of : What you expected to happen: It should print the exact saved data. How to reproduce it (as minimally and precisely as possible): See above.
/sig cli /kind bug
/assign
/unassign Unassign since already has a PR


Question: pathdir, _ := filepath.Split(dir) // break when we reach root if pathdir == "/" { <ins> mount, found = i.mounts["/"] </ins> break } // trim "/" from the new parent path otherwise the next possible

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Kubelet v1.19.0 doesn't start on nodes with CRI-O 1.17.2 (storagedriver="btrfs") with BTRFS as root filesystem. Related/previous issues: This error shows up in the journal: The is referring to the btrfs root filesystem, mounted at the time. : : What you expected to happen: Kubelet starts and detects the filesystem backing , even if it is BTRFS. How to reproduce it (as minimally and precisely as possible): Install kubeadm/kubectl 1.19.0 Install CRI-O, I used 1.17.2 from: Configure CRI-O to use Perform "kubeadm init" creating a new cluster Observe journal (check for kubelet issues) Anything else we need to know?: A possible workaround is to make sure a bind mount exists which allows kubelet's logic to find the backing fileystem. Eg. add the following fstab entry and then perform : The resulting mount should look somewhat like this: The workaround has been tested and seems to make kubelet work as expected. Environment: Kubernetes version: 1.19.0 OS: Debian Bullseye (Testing) Kernel: Linux 5.8.5 Others: The package has been installed
/sig node
cc xref
I am encounter the same issue. thank you for your workaround. It seems to work.
I got the same issue with latest stable Docker on Debian (btrfs) and k8s v1.19.2.
I confirm the same issue shows up with on btrfs (on LVM on LUKS, Debian buster), running k3s. No btrfs subvols. The workaround works for me; for k3s I also need to bind mount . I did not experience this issue with 1.18.
This also reproduces with Docker runtime. Workaround with bind mount helps, however automatically unmounts it so don't get confused. is on root that is mounted like this: , and docker is configured to use .

Reproduced with k8s 1.19.3 on Gentoo. The rootfs is btrfs, is on rootfs, docker is configured with storage drigver . automatically mount and unmount it on . Also confirmed bind mount workaround helps.
/assign
I think almost anyone using BTRFS as the root filesystem has encountered this issue. If is merged and applied to the kubernetes, this can be fixed. Please take a look at this ?
The has been merged to google/cadvisor. I think if Kubernetes v1.21 includes the fix by using google/cadvisor v0.39, the issue will be resolved.
Just a note that I too have stumbled on this issue after reconfiguring CRI-O storage to use btrfs driver pointing with graphroot directly to a BTRFS subvolume (the root filesystem is not BTRFS). After reading this issue I tried mounting the BTRFS subvolume explicitly on some mountpoint and pointed with graphroot to this explicit subvolume mountpoint which solved the problem. The problem seems to occur only when pointing to a subvolume within some parent subvolume as this is not a real mountpoint.
Ok, so I just want to make sure I understand. Currently when i try to run kubeadm to create a new cluster while using btrfs, it gives me With Kubernetes v1.21, that will no longer be the case?
In my experience (with CRI-O 1.19 and k8s 1.19), the btrfs driver (in ) is supported, but the combination does not work correctly when graphroot is pointed to a btrfs subvolume within some parent subvolume (I don't have experience with graphroot poited to some subdirectory of a default subvolume and/or when default subvolume is also a root filesystem, just when pointed directly to a subvolume within some parent subvolume which is not a root filesystem). For example, when btrfs filesystem is mounted like the following (): when I then create a subvolume: then use it as a graphroot (): But when I explicitly mount the subvolume to some mountpoint (): then use the explicitly mounted path as the graphroot (): it works correctly without any problems.

I guess some logic (either in CRI-O/containers or in kubelet) tries to find the "mountpoint" of the filesystem used for graphroot or something related and can't do that correctly when graphroot is located in a btrfs subvolume that is not explicitly mounted somewhere.
Ah, I see what you meant by saying that is not supported in . I haven't checked that since I created k8s clusted with default (overlay) driver and later migrated to . It is possible that disallows using because of problems like the one described above. But in my experience, when a subvolume is mounted explicitly, the btrfs works as expected - at least I didn't have problems with it.
Can the fix be backported to 1.19? only supports updating one version at a time: () So this leaves people on btrfs with with no upgrade path to which is not really nice.
We've discussed this issue at Regarding Maybe, there are quite a lot of people using btrfs than I expected. What do you think about this? Thanks!
I am running up against this issue as well. I am stuck with using k8s on 1.18
It sounds like for full fix we'll need to cherrypick to be included in k8s 1.19 and 1.20. To do that the following is needed: to cAdvisor v0.37 and v0.38 branches k8s 1.19 uses cAdvisor v0.37.3 - k8s 1.20 uses cAdvisor v0.38.6 - new cAdvisor patch release for v0.37 and v0.38 against k8s on 1.19 and 1.20 bumping to newly released cAdvisor (v0.37 / v0.38) with the fix I'm happy to cut the new cAdvisor release from my side if you want to proceed and think this is critical enough bug that is worth cherrypicking.
New cAdvisor patch versions have been cut with the BTRFS fix: v0.37.4 - v0.38.7 - /cc


Question: import ( "encoding/json" <ins> "errors" "net/http" </ins> "strconv" "strings" "sync"

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Currently, when AAD login is configured, refresh token timeout would make kubectl print an error, not able to re-login without manually remove the tokens. Expect kubectl to promote user to login again. Why is this needed: Configure refresh-token timeout from Azure is a way to force re-login after inactivity. kubectl doesn't promote re-login makes users harder to apply timeout policy.
/sig azure
The label(s) cannot be appled. These labels are supported: <detailsIn response to : /sig azure Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The label(s) cannot be appled. These labels are supported: <detailsIn response to : /sig platform/azure Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/area provider/azure
the root cause of this issue is in this first gets the cached token (result of device code flow) at it's trying to refresh an expired token at and failed. An error is returned. Since the expired token is still in the cache, it will always hit this path until user removes the token from kubeconfig. the fix should be: the token from cache a retry to re-initiate the device code flow in
The label(s) cannot be appled. These labels are supported: <detailsIn response to : /sig platform/azure Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cli These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry.
I'm working on it.
are you working on it? I will grab it if I dont hear it from you :)
have the code ready will try PR this weekend Best, Ping
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.

If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Closing as merged


Question: $ kubectl proxy --port=8011 --www=./local/www/ // Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api <del> // This makes e.g. the pods api available at localhost:8011/k8s-api/v1beta1/pods/ $ kubectl proxy --api-prefix=k8s-api </del> <ins> // This makes e.g. the pods api available at localhost:8011/k8s-api/v1beta3/pods/ $ kubectl proxy --api-prefix=/k8s-api </ins> ``` ### Options

Without , works just fine, With '--api-prefix=xxx-api', it stops work: My version: could you take a look? Thanks.
--api-prefix was in . could you take a look? This appears to be completely broken, so we should fix it or rip it out.
Reading the code and I find the way to fix it. You need to use , note the slashes. I will fix the man page.
What's an example command line that works?
then
Ah, the default flag value has leading and trailing slashes: The example in the documentation is wrong: Re-opening for doc fix.
The trailing slash is not necessary. and duplicate each other.


Question: warn: false, }, { <del> name: "Docker version 19.01.0 is not in the list of validated versions", info: dockerInfo{Driver: "driver_2", ServerVersion: "19.01.0"}, </del> <ins> name: "valid Docker version 19.03.1-ce", info: dockerInfo{Driver: "driver_2", ServerVersion: "19.03.1-ce"}, err:  false, warn: false, }, { name: "Docker version 19.06.0 is not in the list of validated versions", info: dockerInfo{Driver: "driver_2", ServerVersion: "19.06.0"}, </ins> err:  false, warn: true, },

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Support for newer docker versions: 19.x Why is this needed: New users trying to use kubernetes install the latest docker version. It's inconvenient see the following warning and have to uninstall docker and then install an older version. Also, it's not clear what could happen if someone uses a newer version of docker. Lastly, I believe that kubernetes should be able to keep up with docker. Is there any roadmap for this, that I have missed?
/sig usability
you could work with sig-testing and sig-node to set up CI jobs when new versions of Docker shows up. Believe me, it's more inconvenient to debug failures in a new docker version as a end use than the downgrade. We have to have some CI coverage of a version before we ok its use by the community.
Hi I'm new to this, so I'm not sure how to apply your suggestion: Is there any documentation that I could use?
SIGs are teams working on specific areas of kubernetes All our test infrastructure for CI jobs ("Continuous Integration") is in: Out test-grid is here (please type docker in the search box on top to see the current CI jobs):
Since docker-ce 18.09 has reached EOL and there are no new versions planned (), it would be great to have official support for a newer version of docker-ce.
please see:
Hi folks, David from Docker here, we have run and validated Docker Engine 19.03 for Kubernetes 1.14 back in June, so Docker 19.03 is indeed supported for Kubernetes 1.14 and above: Let me know how else we can help.
thanks for confirming PR is here
Thank you Appreciate the PR, we typically test engines for major versions of Engine that are released like 19.03 with the latest version of Kubernetes that is available at the time. On our next engine, we will re-run the test and submit the results to testgrid. Let me know how I can better coordinate this.
Also let me know if there is anything else that is needed to close this issue out. Thanks.
hi, the PR should close this issue. i guess, one option is just creating a similar issue once the test job is available in test-grid and running green for a while? or me can then proceed to update the validators.

Ok thank you, that sounds like a good process we can follow moving forward.
Hi Any chance this can make its way into 1.16 at least? This way it may get picker up by Kops and not have to rely on some old and unpatched Docker version. At the moment Docker CE 18.09 is no longer actively supported
hi, we cannot backport this as it's not a critical fix. at least in the case of kubeadm the 19. versions of docker would only print a warning.
but doesn't the existence of a vulnerability make this critical enough?
the usage of Docker 19. is not blocked in any way in 1.16. you can ignore the and just use it. if something was blocking the usage of 19., then this is backport worthy.
I saw this as the first step before adding 19.03 as a validated external release. Without being documented as such Kops will not pick it up as default and will leave people with old and vulnerable Docker versions. If this I understand correctly the above comments, 19.03 has been validated for some time, just not documented.
i don't think kops uses the kubeadm validators that reside in k/k/cmd/kubeadm/app/... this was resolved in this PR the k8s docs for CRI are still out of date: so if someone wants to send a PR for that page please go ahead: (but AFAIK not all distros have moved to 19. yet)
I couldn't find any repos missing docker 19.03.4. Any chance to do the change in the changelog?


Question: import ( "os" <ins> "path/filepath" </ins> "runtime" ) <del> // HomeDir returns the home directory for the current user </del> <ins> // HomeDir returns the home directory for the current user. // On Windows: // 1. the first of %HOME%, %HOMEDRIVE%%HOMEPATH%, %USERPROFILE% containing a `.kubeconfig` file is returned. // 2. if none of those locations contain a `.kubeconfig` file, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that exists and is writeable is returned. // 3. if none of those locations are writeable, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that exists is returned. // 4. if none of those locations exists, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that is set is returned. </ins> func HomeDir() string { if runtime.GOOS == "windows" { <ins> home := os.Getenv("HOME") homeDriveHomePath := "" if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 { homeDriveHomePath = homeDrive + homePath } userProfile := os.Getenv("USERPROFILE") </ins> <del> // First prefer the HOME environmental variable if home := os.Getenv("HOME"); len(home) > 0 { if _, err := os.Stat(home); err == nil { return home </del> <ins> // Return first of %HOME%, %HOMEDRIVE%/%HOMEPATH%, %USERPROFILE% that contains a `.kubeconfig` file. // %HOMEDRIVE%/%HOMEPATH% is preferred over %USERPROFILE% for backwards-compatibility. for _, p := range []string{home, homeDriveHomePath, userProfile} { if len(p) == 0 { continue </ins> } <del> } if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 { homeDir := homeDrive + homePath if _, err := os.Stat(homeDir); err == nil { return homeDir </del> <ins> if _, err := os.Stat(filepath.Join(p, ".kube", "config")); err != nil { continue </ins> } <ins> return p </ins> } <del> if userProfile := os.Getenv("USERPROFILE"); len(userProfile) > 0 { if _, err := os.Stat(userProfile); err == nil { return userProfile </del> <ins> firstSetPath := "" firstExistingPath := "" // Prefer %USERPROFILE% over %HOMEDRIVE%/%HOMEPATH% for compatibility with other auth-writing tools for _, p := range []string{home, userProfile, homeDriveHomePath} { if len(p) == 0 { continue } if len(firstSetPath) == 0 { // remember the first path that is set firstSetPath = p } info, err := os.Stat(p) if err != nil { continue </ins> } <ins> if len(firstExistingPath) == 0 { // remember the first path that exists firstExistingPath = p } if info.IsDir() && info.Mode().Perm()&(1<<(uint(7))) != 0 { // return first path that is writeable return p } } // If none are writeable, return first location that exists if len(firstExistingPath) > 0 { return firstExistingPath </ins> } <ins> // If none exist, return first location that is set if len(firstSetPath) > 0 { return firstSetPath } // We've got nothing return "" </ins> } return os.Getenv("HOME") }

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: In , which is used by many clients, homedir favors a concatenation of %HOMEDRIVE% and %HOMEPATH% over %USERPROFILE% What you expected to happen: HOME should be favored if it is set, then fallback to USERPROFILE, then maybe HOMEDRIVE and HOMEPATH combination. Anything else we need to know?: The current implementation of resolves a homedir in this order: concatenation of HOMEDRIVE and HOMEPATH I believe that this is wrong, notably the HOMEDRIVE and HOMEPATH construct. In the corporate environment where I work, these variables are hard set to network drives which are not writable. This results in many problems for dependant projects like minikube, kubectl kubernetes/kubectl (which would benefit from using client-go code). There are almost no mention of using HOMEDRIVE and HOMEPATH, but authoritative information on the subject is hard to find. Relevant documentation: Environment: Windows
cc
sounds reasonable. but I'd like to have sig/windows folks to chime in.
Additional profiles directory information: (v=vs.85).aspx Microsoft makes it difficult to reason about these environment variables as they are not readily mentioned/searchable in the online documentation. Even in the previously mentioned doc on profiles, no mention of the %USERPROFILE% environment variable is made. Python already does this resolution in the order suggested: HOME, USERPROFILE, and HOMEDRIVE/ HOMEPATH: There could be another way to implement this: If it is possible, call the Windows API .aspx) to know the path, regardless of env variables.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale still relevant
This continues to be an issue in 1.11.0.

Basically, the HOMEDRIVE/HOMEPATH is a holdover from NT4 days with roaming profiles. This is not a standard env variable anymore. The existence of HOMEDRIVE / HOMEPATH should be ignored if USERPROFILE is set. Neither HOMEDRIVE nor HOMEPATH are standard environment variables: They are not use in any known folder locations: (v=vs.85).aspx The existence of a HOMEDRIVE / HOMEPATH will cause to look in the wrong home folder for the . A use-case here is with AKS - the Azure CLI (written by Microsoft) will correctly place credentials into , however will look in . Because of this, will think it should connect to localhost until the HOMEDRIVE and HOMEPATH variables are unset or removed in the session.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Since has proposed a different approach, I wanted to discuss it in the original issue for clarity. I do not believe we should not favor the use of at all as is proposed with here: Based on the use-case raised by and experienced by myself as well, this will cause more problems if we continue to prefer . I understand moving existing configs from a prior path to be more aligned valid environment variables (which is not) will cause some breakage without migrating old configs. To illustrate the issue, some tools on Windows like the Azure CLI do not honor and thus, when authenticating users to Microsoft's own managed Kubernetes offerings, it writes the configuration to . When the config is written to , but is looking in , the client does not get the correct configuration.

With the PR proposed, I'm afraid we may further fracture things with clients writing to non-standard environment variables and where some outlier cases would end up with a folder in both and . The , then , then would follow what I think most tools and languages use. I believe this is in wide adoption and the evaluation of environment variables should be in that order. For example, python's ordering:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale


Question: } // IsCriticalPodBasedOnPriority checks if the given pod is a critical pod based on priority resolved from pod Spec. <del> func IsCriticalPodBasedOnPriority(ns string, priority int32) bool { // Critical pods are restricted to "kube-system" namespace as of now. if ns != kubeapi.NamespaceSystem { return false } </del> <ins> func IsCriticalPodBasedOnPriority(priority int32) bool { </ins> if priority >= scheduling.SystemCriticalPriority { return true }

When we used annotations to mark critical pods, we considered pods with such an annotation critical only when they were created in "kube-system" namespace. Going to the new era of using priority to indicate critical pods, we would like to remove the restriction of "kube-system" namespace and consider pods with critical priority as critical no matter what namespace they are in. ref/ /kind bug /sig scheduling /assign cc/
While I am fine with the approach, it seems the only way to ensure that those critical pods are not coming from some random namespace is whitelisting them using priority admission controller or using quota. Atleast previously we have one more level of security that critical pod has to be created in kube-system namespace.
The requirement to limit the critical pod annotation meaning to the kube-system namespace was a workaround for the lack of access control and quota around the annotation. We actively work to prevent special handling of the kube-system namespace wherever possible.
Yes, which is why those aspects are essential to a functional priority system that works for a variety of cluster configurations
Agree with Ravi and Jordan on what is needed for final solution.
There is no doubt that we need to have quota for priority. It is needed not only for critical priorities, but also for any other high priority classes that users may define. That effort is going forward and we will have it soon.
, I think we also need to update Priority admission controller for that.
see later PR re-restricting this until the quota controls graduate to at least beta state:
Got that, thanks :) That makes sense to me.
If this is closed and there even was a for this type of functionality, what is the current status? And is in sync with current status?
It seems that namespace is still , so I guess the current status is that system critical pods are still limited to namespace. Based on that I think this issue should not be closed.
can we reopen this issue please because as pointed out by the issue still exists. Thanks.
Waiting for this to be available myself. No reason my Node Exporter dataset has to run in the namespace instead of my Grafana namespace.
Confirmed code + behavior. /open
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository.

</details
I too am confused based on that documentation. Can we get an update here with an answer and fix the documentation if it's wrong?
This is resolved in 1.17 See