Question: framework.ExpectNoError(err) }) <del> ginkgo.It("should be able to change the type and ports of a service [Slow]", func() { // requires cloud load-balancer support e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws") loadBalancerSupportsUDP := !framework.ProviderIs("aws") loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault if framework.ProviderIs("aws") { loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS } loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) // This test is more monolithic than we'd like because LB turnup can be // very slow, so we lumped all the tests into one LB lifecycle. serviceName := "mutability-test" ns1 := f.Namespace.Name // LB1 in ns1 on TCP framework.Logf("namespace for TCP test: %s", ns1) ginkgo.By("creating a second namespace") namespacePtr, err := f.CreateNamespace("services", nil) framework.ExpectNoError(err, "failed to create namespace") ns2 := namespacePtr.Name // LB2 in ns2 on UDP framework.Logf("namespace for UDP test: %s", ns2) nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) // Test TCP and UDP Services.  Services with the same name in different // namespaces should get different node ports and load balancers. ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1) tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName) tcpService, err := tcpJig.CreateTCPService(nil) framework.ExpectNoError(err) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2) udpJig := e2eservice.NewTestJig(cs, ns2, serviceName) udpService, err := udpJig.CreateUDPService(nil) framework.ExpectNoError(err) ginkgo.By("verifying that TCP and UDP use the same port") if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port { framework.Failf("expected to use the same port for TCP and UDP") } svcPort := int(tcpService.Spec.Ports[0].Port) framework.Logf("service port (TCP and UDP): %d", svcPort) ginkgo.By("creating a pod to be part of the TCP service " + serviceName) _, err = tcpJig.Run(nil) framework.ExpectNoError(err) ginkgo.By("creating a pod to be part of the UDP service " + serviceName) _, err = udpJig.Run(nil) framework.ExpectNoError(err) // Change the services to NodePort. ginkgo.By("changing the TCP service to type=NodePort") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("changing the UDP service to type=NodePort") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) udpNodePort := int(udpService.Spec.Ports[0].NodePort) framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) // Change the services to LoadBalancer. // Here we test that LoadBalancers can receive static IP addresses.  This isn't // necessary, but is an additional feature this monolithic test checks. requestedIP := "" staticIPName := "" if framework.ProviderIs("gce", "gke") { ginkgo.By("creating a static load balancer IP") staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID) gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region()) defer func() { if staticIPName != "" { // Release GCE static IP - this is not kube-managed and will not be automatically released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Logf("failed to release static IP %s: %v", staticIPName, err) } } }() framework.ExpectNoError(err, "failed to create region address: %s", staticIPName) reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region()) framework.ExpectNoError(err, "failed to get region address: %s", staticIPName) requestedIP = reservedAddr.Address framework.Logf("Allocated static load balancer IP: %s", requestedIP) } ginkgo.By("changing the TCP service to type=LoadBalancer") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) if loadBalancerSupportsUDP { ginkgo.By("changing the UDP service to type=LoadBalancer") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) } serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService)) if loadBalancerSupportsUDP { serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService)) } ginkgo.By("waiting for the TCP service to have a load balancer") // Wait for the load balancer to be created asynchronously tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort) } if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) framework.Logf("TCP load balancer: %s", tcpIngressIP) if framework.ProviderIs("gce", "gke") { // Do this as early as possible, which overrides the `defer` above. // This is mostly out of fear of leaking the IP in a timeout case // (as of this writing we're not 100% sure where the leaks are // coming from, so this is first-aid rather than surgery). ginkgo.By("demoting the static IP to ephemeral") if staticIPName != "" { gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") // Deleting it after it is attached "demotes" it to an // ephemeral IP, which can be auto-released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Failf("failed to release static IP %s: %v", staticIPName, err) } staticIPName = "" } } var udpIngressIP string if loadBalancerSupportsUDP { ginkgo.By("waiting for the UDP service to have a load balancer") // 2nd one should be faster since they ran in parallel. udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort) } udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) framework.Logf("UDP load balancer: %s", udpIngressIP) ginkgo.By("verifying that TCP and UDP use different load balancers") if tcpIngressIP == udpIngressIP { framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } } ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } // Change the services' node ports. ginkgo.By("changing the TCP service's NodePort") tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort) framework.ExpectNoError(err) tcpNodePortOld := tcpNodePort tcpNodePort = int(tcpService.Spec.Ports[0].NodePort) if tcpNodePort == tcpNodePortOld { framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("changing the UDP service's NodePort") udpService, err = udpJig.ChangeServiceNodePort(udpNodePort) framework.ExpectNoError(err) udpNodePortOld := udpNodePort udpNodePort = int(udpService.Spec.Ports[0].NodePort) if udpNodePort == udpNodePortOld { framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort) } if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the TCP service's new NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's new NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } // Change the services' main ports. ginkgo.By("changing the TCP service's port") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) svcPortOld := svcPort svcPort = int(tcpService.Spec.Ports[0].Port) if svcPort == svcPortOld { framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort) } if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } ginkgo.By("changing the UDP service's port") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].Port) != svcPort { framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port) } if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort) } if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("service port (TCP and UDP): %d", svcPort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } ginkgo.By("Scaling the pods to 0") err = tcpJig.Scale(0) framework.ExpectNoError(err) err = udpJig.Scale(0) framework.ExpectNoError(err) ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort") testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort") testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer") testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer") testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } ginkgo.By("Scaling the pods to 1") err = tcpJig.Scale(1) framework.ExpectNoError(err) err = udpJig.Scale(1) framework.ExpectNoError(err) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } // Change the services back to ClusterIP. ginkgo.By("changing TCP service back to type=ClusterIP") tcpReadback, err := tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if tcpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("TCP Spec.Ports[0].NodePort was not cleared") } // Wait for the load balancer to be destroyed asynchronously _, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) ginkgo.By("changing UDP service back to type=ClusterIP") udpReadback, err := udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if udpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("UDP Spec.Ports[0].NodePort was not cleared") } if loadBalancerSupportsUDP { // Wait for the load balancer to be destroyed asynchronously _, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) } ginkgo.By("checking the TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the TCP LoadBalancer is closed") testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("checking the UDP LoadBalancer is closed") testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } }) </del> /* Testname: Service, update NodePort, same port different protocol Description: Create a service to accept TCP requests. By default, created service MUST be of type ClusterIP and an ClusterIP MUST be assigned to the service.

Which jobs are flaking: gce-master-scale-correctness Which test(s) are flaking: Services should be able to change the type and ports of a service [Slow] Testgrid link: https://k8s- Reason for failure: Test timeouts in ~half of the runs since it was enabled here: /sig scalability /sig network
/cc /cc
The context is - the problems are strictly related to scale - 5k-node jobs are timing out. I very briefly skimmed through the test and didn't see anything obviously suspicious (apart from the fact there are gazillions long timeouts there, which alltogether sum up long hours). That said - it seems to me that there are effectively two separate tests inside it: one for TCP one for UDP They don't seem to share anything with each other. - would it make sense to split this test two effectively two separate tests: one for TCP and one for UDP. (I guess it would also allow us to share some code there).
[And potentially it could help with debugging the problem if it is strictly related to only one of those cases...]
I wanted to focus on this past week but I didn´t have time, this is my list of more expensive times and approx average duration, all of the sig-network ones use LoadBalancers. I wanted to time the test to understand if there are places where can save time ... I don´t have much context on those tests, maybe Tim can help us more, but this comment makes me think that the use of both protocols is intended
/triage accepted
/triage accepted It's been a very long time since I touched this, and I recall holding my nose on it in the first place. Splitting TCP from UDP seems reasonable if we can run them in parallel. we'll want to share the flow, and parameterize some shared funcs. I don't know that this will prevent test timeouts, though.
We could use a volunteer to go poke at this, and see how hard it will really be.
You mean "forcing them to run in parallel"? Or "allowing them to run in parallel"? The later doesn't require any additiona work. The former I think mean that it will still have to be a single test (just the code will be organized better) and that won't really help with debugging (nor preventing timeouts).

this was discussed yesterday in sig-net meeting and my understanding was to split current test in 2 different so the test can run in parallel, if you want to do so and one protocol doesn´t have to wait for the other. I think that is what you meant here
As long as we CAN parallelize. Creating LBs can be sloooooow. :P Antonio Ojea <:
Reopening - I don't think just splitting will address the problem. It may help us with debugging though...
With the split we moved from 60 mins to run the test, to run 2 tests in [30-40] mins each https://k8s- Seems a win :smile:
Awesome - that's something I was silently counting for TBH :) [I doubt it will fix flakiness, but maybe it will make it easier to debug...]
Actually - I think we're getting close thanks to the two changes ( + this one). Here is the run: This particular one filed on UDP (not sure if it's just UDP-related, or just manifested with UDP now). And we have a clear failure now: In my PR I decreased the timeout from 2h to 45m, as it seemed to me that in the past it should be enough. But it failed on 45m timeout here. So I think the next question now is: is it that slow - that we need more than 45m or something isn't working as expected (either in the k8s itself or in the test) But I think we have a clear single place to look at now.
Actually - the full stacktrace: So it's this line that failed: What is interesting is that something was changing during those 45 minutes:
Note that it's 4th time we actually wait for LB in this test. The failure happens after the pod were scaled down to 0 and then to 1. So that makes me think, that's it's actually not LB-related, but rather something internally in k8s, becuase L4 LB doesn't depend on pods - it targets nodes...
If it is always UDP, I start to see ghosts between GCE and UDP ... we only have KIND to compare against, or do we have some other non gce enviroment?

I mean, there are other tests failing in UDP , and show much more failures in GCE than in kind
- this does not look "normal". Can we give some prio to helping track this flake?
hmm interesting. we will investigate.
/assign
- any progress on this?
I haven't had a chance to look into this yet. I will prioritize it this week.
Digging through the logs I believe this was a bug in the test code that was fixed by . In the test, the service port changes around 12:33. However due to the bug, the test continues to use the old service port as seen in the test output: The port should be 81. I believe the following sequence occurs from this point: 1) The reachability test fails in the beginning because the service has changed the port and the load balancer has not been updated yet resulting in the error. 1) Around 12:38, the service controller is syncing the service and notices that the service port has changed and starts to delete the forwarding rule causing the error to change to . 1) Once the forwarding rule is recreated, the error goes back to being because the forwarding rule exists but the wrong port is used. Looking at , the test runs after the one linked have all passed. Is there anything else that needs to be investigated as part of this issue?
OK - the test was quite flaky in the past but indeed it didn't flake for the last 3 weeks (neither this nor TCP one). I'm fine with closing this one - we can open a new one if this happens again. - thanks for debugging. - it seems that the splitting of the test actually made things waay better. Which is great.


Question: for _, addr := range unbindIPAddr.UnsortedList() { err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice) // Ignore no such address error when try to unbind address <del> if err != nil && err != unix.ENXIO { </del> <ins> if err != nil { </ins> glog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) } }

https://k8s- https://k8s- cc: /priority failing-test
Sorry, but I will fix it ASAP.
/assign
PR is already sent. Wating for CI report...


Question: } } <del> // TODO(random-liu): Add unit test for convertStatusToAPIStatus (issue #20478) </del> <ins> func TestGenerateAPIPodStatusWithSortedContainers(t *testing.T) { testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet numContainers := 10 expectedOrder := []string{} cStatuses := []*kubecontainer.ContainerStatus{} specContainerList := []api.Container{} for i := 0; i < numContainers; i++ { id := fmt.Sprintf("%v", i) containerName := fmt.Sprintf("%vcontainer", id) expectedOrder = append(expectedOrder, containerName) cStatus := &kubecontainer.ContainerStatus{ ID:   kubecontainer.BuildContainerID("test", id), Name: containerName, } // Rearrange container statuses if i%2 == 0 { cStatuses = append(cStatuses, cStatus) } else { cStatuses = append([]*kubecontainer.ContainerStatus{cStatus}, cStatuses...) } specContainerList = append(specContainerList, api.Container{Name: containerName}) } pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ UID:       types.UID("uid1"), Name:      "foo", Namespace: "test", }, Spec: api.PodSpec{ Containers: specContainerList, }, } status := &kubecontainer.PodStatus{ ID:                pod.UID, Name:              pod.Name, Namespace:         pod.Namespace, ContainerStatuses: cStatuses, } for i := 0; i < 5; i++ { apiStatus := kubelet.generateAPIPodStatus(pod, status) for i, c := range apiStatus.ContainerStatuses { if expectedOrder[i] != c.Name { t.Fatalf("Container status not sorted, expected %v at index %d, but found %v", expectedOrder[i], i, c.Name) } } } } func TestGenerateAPIPodStatusWithReasonCache(t *testing.T) { // The following waiting reason and message  are generated in convertStatusToAPIStatus() startWaitingReason := "ContainerCreating" testTimestamp := time.Unix(123456789, 987654321) testErrorReason := fmt.Errorf("test-error") emptyContainerID := (&kubecontainer.ContainerID{}).String() verifyStatus := func(t *testing.T, c int, podStatus api.PodStatus, state, lastTerminationState map[string]api.ContainerState) { statuses := podStatus.ContainerStatuses for _, s := range statuses { if !reflect.DeepEqual(s.State, state[s.Name]) { t.Errorf("case #%d, unexpected state: %s", c, diff.ObjectDiff(state[s.Name], s.State)) } if !reflect.DeepEqual(s.LastTerminationState, lastTerminationState[s.Name]) { t.Errorf("case #%d, unexpected last termination state %s", c, diff.ObjectDiff( lastTerminationState[s.Name], s.LastTerminationState)) } } } testKubelet := newTestKubelet(t) kubelet := testKubelet.kubelet pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ UID:       "12345678", Name:      "foo", Namespace: "new", }, Spec: api.PodSpec{RestartPolicy: api.RestartPolicyOnFailure}, } podStatus := &kubecontainer.PodStatus{ ID:        pod.UID, Name:      pod.Name, Namespace: pod.Namespace, } tests := []struct { containers                   []api.Container statuses                     []*kubecontainer.ContainerStatus reasons                      map[string]error oldStatuses                  []api.ContainerStatus expectedState                map[string]api.ContainerState expectedLastTerminationState map[string]api.ContainerState }{ // For container with no historical record, State should be Waiting, LastTerminationState should be retrieved from // old status from apiserver. { []api.Container{{Name: "without-old-record"}, {Name: "with-old-record"}}, []*kubecontainer.ContainerStatus{}, map[string]error{}, []api.ContainerStatus{{ Name:                 "with-old-record", LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{}}, }}, map[string]api.ContainerState{ "without-old-record": {Waiting: &api.ContainerStateWaiting{ Reason: startWaitingReason, }}, "with-old-record": {Waiting: &api.ContainerStateWaiting{ Reason: startWaitingReason, }}, }, map[string]api.ContainerState{ "with-old-record": {Terminated: &api.ContainerStateTerminated{}}, }, }, // For running container, State should be Running, LastTerminationState should be retrieved from latest terminated status. { []api.Container{{Name: "running"}}, []*kubecontainer.ContainerStatus{ { Name:      "running", State:     kubecontainer.ContainerStateRunning, StartedAt: testTimestamp, }, { Name:     "running", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, }, map[string]error{}, []api.ContainerStatus{}, map[string]api.ContainerState{ "running": {Running: &api.ContainerStateRunning{ StartedAt: unversioned.NewTime(testTimestamp), }}, }, map[string]api.ContainerState{ "running": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, }, }, // For terminated container: // * If there is no recent start error record, State should be Terminated, LastTerminationState should be retrieved from // second latest terminated status; // * If there is recent start error record, State should be Waiting, LastTerminationState should be retrieved from latest // terminated status; // * If ExitCode = 0, restart policy is RestartPolicyOnFailure, the container shouldn't be restarted. No matter there is // recent start error or not, State should be Terminated, LastTerminationState should be retrieved from second latest // terminated status. { []api.Container{{Name: "without-reason"}, {Name: "with-reason"}}, []*kubecontainer.ContainerStatus{ { Name:     "without-reason", State:    kubecontainer.ContainerStateExited, ExitCode: 1, }, { Name:     "with-reason", State:    kubecontainer.ContainerStateExited, ExitCode: 2, }, { Name:     "without-reason", State:    kubecontainer.ContainerStateExited, ExitCode: 3, }, { Name:     "with-reason", State:    kubecontainer.ContainerStateExited, ExitCode: 4, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 0, }, { Name:     "succeed", State:    kubecontainer.ContainerStateExited, ExitCode: 5, }, }, map[string]error{"with-reason": testErrorReason, "succeed": testErrorReason}, []api.ContainerStatus{}, map[string]api.ContainerState{ "without-reason": {Terminated: &api.ContainerStateTerminated{ ExitCode:    1, ContainerID: emptyContainerID, }}, "with-reason": {Waiting: &api.ContainerStateWaiting{Reason: testErrorReason.Error()}}, "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    0, ContainerID: emptyContainerID, }}, }, map[string]api.ContainerState{ "without-reason": {Terminated: &api.ContainerStateTerminated{ ExitCode:    3, ContainerID: emptyContainerID, }}, "with-reason": {Terminated: &api.ContainerStateTerminated{ ExitCode:    2, ContainerID: emptyContainerID, }}, "succeed": {Terminated: &api.ContainerStateTerminated{ ExitCode:    5, ContainerID: emptyContainerID, }}, }, }, } for i, test := range tests { kubelet.reasonCache = NewReasonCache() for n, e := range test.reasons { kubelet.reasonCache.add(pod.UID, n, e, "") } pod.Spec.Containers = test.containers pod.Status.ContainerStatuses = test.oldStatuses podStatus.ContainerStatuses = test.statuses apiStatus := kubelet.generateAPIPodStatus(pod, podStatus) verifyStatus(t, i, apiStatus, test.expectedState, test.expectedLastTerminationState) } } </ins>

We moved reason cache and status convert function from to by , , and . After this, we should also migrate related old unit test to kubelet level. For we should add test for: [x] Different image pull policy. [x] Pod creation failure. [x] Pod image failure. For we should add test for: [x] ContainerStatuses order. [x] Last termination state. [x] How reason cache will affect the PodPhase: [x] Pull image failure reason [x] Pod creation failure reason
Time to close this issue? :)
Not yet, The last 3 tests are in . :)


Question: // We do not start listening on SCTP ports, according to our agreement in the // SCTP support KEP } else if svcInfo.Protocol() != v1.ProtocolSCTP { <del> socket, err := proxier.portMapper.OpenLocalPort(&lp) </del> <ins> socket, err := proxier.portMapper.OpenLocalPort(&lp, isIPv6) </ins> if err != nil { klog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } if lp.Protocol == "udp" { <del> isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP()) </del> conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, v1.ProtocolUDP) } replacementPortsMap[lp] = socket

/kind bug What happened: When creating a node port application (in this case the example guestbook) on a host server that is dual stacked and has AAAA and A DNS entries, the node port on the v6 address is open to kube-proxy and hangs. What you expected to happen: The v6 address would refuse the connection and the client would fall back onto the v4 address which would connect correctly. How to reproduce it (as minimally and precisely as possible): kubeadm init with the above, and join in some workers. Then install the guestbook application Obtain the node port with and telnet to one of the servers on that port from a dual stacked client. The client will connect and refuse to respond to 'GET /' telnet directly to the IPv4 address of the server on the port and all works as expected. Anything else we need to know?: kube-proxy is noted as listening on the port. Environment: Kubernetes version (use ): ubuntu kubectl version Client Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-07-17T18:53:20Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-07-17T18:43:26Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: Brightbox OS (e.g. from /etc/os-release): Ubuntu 18.04.1 LTS Kernel (e.g. ): Linux srv-jidtj 4.15.0-29-generic -Ubuntu SMP Tue Jul 17 15:39:52 UTC 2018 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm 1.11 Others:
/sig network
Looks like kube-proxy in ipvs mode listens on the nodeport ports to ensure they're not in use by something else. It just binds and listens, it doesn't serve requests.

It binds on both ipv4 and ipv6 but never expects to receive connections because the iptables/ipvs rules intercept them, but on ipv4 clusters there is nothing stopping ipv6 connections coming in. The comment there does admit that only a bind is necessary to do this, but doing a full listen just made e2e tests easier. /cc
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen This now causes conformance failure on e2e tests due to the promotion of the Nodeport test in
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Still an issue on 1.16 when running in standard IPv4 mode
As I understand this thread, I have the same problem. I thought that it is a node configuration or some kubernetes flag that I didn't set but it seems it ain't.

I ask already on SO: Environment Kubernetes version (use kubectl version): Client Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-09-18T14:36:53Z", GoVersion:"go1.12.9", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-10-02T16:51:36Z", GoVersion:"go1.12.10", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: On premise, on virtual machines using hyper-v OS (e.g. from /etc/os-release): NAME="Ubuntu" VERSION="18.04.3 LTS (Bionic Beaver)" Kernel (e.g. uname -a): Linux kubernetes 5.0.0-29-generic # 31~18.04.1-Ubuntu SMP Thu Sep 12 18:29:21 UTC 2019 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm version: &version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-09-18T14:34:01Z", GoVersion:"go1.12.9", Compiler:"gc", Platform:"linux/amd64"}
/lifecycle frozen
/assign bowei
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
Looking through the comments, this looks like a bug that could have a targeted fix. /remove-triage unresolved /good-first-issue /help-wanted
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : Looking through the comments, this looks like a bug that could have a targeted fix.

/good-first-issue /help-wanted </details
I will attempt to solve this issue.
any success or progress? If you left this issue I would like to give it a go as a first time contributor. Thanks, Yuval
I'm still working on this as my first issue, and planning to produce a solution by 20th.
I wasn't able to tackle the issue and will not continue trying, feel free to take it.
/assign yuvalsteuer
Still, open -- anyone actively working on this? I would like to attempt if not. :) /assign rkc007
OK, again still open, doesn't look like anyone is working on this. I'd like to try it. The fix that occurs to me is simply to only listen to IPv6 addresses if the cluster is IPv6. If the e2e tests fail in this case (from looking at I don't think they will) then maybe they need to be updated. Does that make sense? What I'm not sure about is if we should do nothing with IPv6 ports or bind (to claim the port) but not listen.
/assign michaelbeaumont


Question: echo "Running GKE internal configuration script" . "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" gke-internal-master-start <ins> elif [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then echo "setting up local admin kubeconfig" create-kubeconfig "local-admin" "${KUBE_BEARER_TOKEN}" echo "export KUBECONFIG=/etc/srv/kubernetes/local-admin/kubeconfig" > /etc/profile.d/kubeconfig.sh </ins> fi }

What happened: Used kube- to bring up a GCE cluster and then logged on to the master. Run kubectl on the master, is hangs for a while (30 seconds) and then returns an error. $ kubectl get nodes Error from server (InternalError): an error on the server ("") has prevented the request from succeeding What you expected to happen: I expected to get the node list. If I use one of the system account's kubeconfig files it works. $ sudo kubectl --kubeconfig=/mnt/disks/master-pd/srv/kubernetes/kube-bootstrap/kubeconfig get nodes NAME STATUS ROLES AGE VERSION kubernetes-master Ready,SchedulingDisabled <none26m v1.18.0-alpha.2.55+-dirty kubernetes-minion-group-jfk6 Ready <none26m v1.18.0-alpha.2.55+-dirty kubernetes-minion-group-mqnp Ready <none26m v1.18.0-alpha.2.55+-dirty kubernetes-minion-group-x540 Ready <none26m v1.18.0-alpha.2.55+-dirty How to reproduce it (as minimally and precisely as possible): Bring up a GCE cluster from head, ssh to master and run "kubectl get nodes" Anything else we need to know?: Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.2.55+-dirty", GitCommit:"", GitTreeState:"dirty", BuildDate:"2020-01-23T00:38:39Z", GoVersion:"go1.13.6", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.2.55+-dirty", GitCommit:"", GitTreeState:"dirty", BuildDate:"2020-01-23T00:36:29Z", GoVersion:"go1.13.6", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: GCE OS (e.g: ): COS m_77 Kernel (e.g.

): 4.19.76 Install tools: cluster/kube- Network plugin and version (if this is a network-related bug): Others: I believe this used to work based on talking to KAS on the insecure port using no authentication. The insecure port is now turned off and we require authentication materials. We should probably setup an admin account specifically for the master.
/sig cli
/assign
I also would not have expected a hang, but an immediate error, and I would have expected an authentication error, not an internal error
I believe the internal error is due to it defaulting to port 8080. Nothing is listening on that port. This used to work but I would guess it stopped working when we stopped listening on the insecure port.
changed behavior of client-go on connection refused errors, which changed a short wait and an "invalid host/port" error into a long retry wait and a confusing "internal server" error /assign
if kubectl is intended to work automatically against the local API server, then a kubeconfig file should be set up, but the 30 second delay and incorrect error are a bug that should be fixed in client-go /sig api-machinery
/remove-sig api-machinery /unassign actually, moved the client-go issue to will keep this one for the GCE $KUBECONFIG resolution
I agree that seems like the right fix.
I'm not sure this belong to sig-cli... looks like made this change /sig cluster-lifecycle /cc
Re: set up a kubeconfig That's what we do for GKE: create an authenticated kubeconfig and set the env var. E.g. if you have an authenticated kubeconfig at :
Yes, GKE sets up an admin system account for the master(s) as part of the internal-configure-helper. I believe we should do the same thing for GCE.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale


Question: } } } else { <del> if isInternal { for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] if az.serviceOwnsFrontendIP(config, service) && !strings.EqualFold(*config.Name, lbFrontendIPConfigName) { glog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, *config.Name) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true } </del> <ins> for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] isFipChanged, err := az.isFrontendIPChanged(clusterName, config, service, lbFrontendIPConfigName) if err != nil { return nil, err } if isFipChanged { glog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, *config.Name) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true </ins> } } foundConfig := false

/kind bug What happened: ILB Frontend IP configuration does not change while updating service What you expected to happen: LB Frontend IP configuration should get updated How to reproduce it (as minimally and precisely as possible): Source issue : AKS service is created using .yaml manifest. ILB is described as it is shown in the documentation: annotations: "true" spec: ports: port: 80 type: LoadBalancer loadBalancerIP: 10.240.200.2 ILB instance with the "kubernetes-internal" name is created with a correct private IP address pointing to the service described. If I change the "loadBalancerIP" to another internal IP address and re-deploy the service, the last is deployed with the correct new IP address, but "kubernetes-internal" ILB still has the old Frontend IP configuration. Environment: Cloud provider or hardware configuration: Azure
/sig azure
This also not works for public LB. /cc Could you fix the bug and cherry pick to old versions?
I got this bug today, I changed the external LoadBalancerIP on a LoadBalancer service. The service event log showed that it immediately recognised the change ("oldIP =newIP") but I had to delete/recreate the service to see the change in the Azure loadbalancer.


Question: allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling")) } <ins> rawTopologies := make([]map[string]sets.String, len(topologies)) </ins> for i, term := range topologies { <del> allErrs = append(allErrs, apivalidation.ValidateTopologySelectorTerm(term, fldPath.Index(i))...) </del> <ins> idxPath := fldPath.Index(i) exprMap, termErrs := apivalidation.ValidateTopologySelectorTerm(term, fldPath.Index(i)) allErrs = append(allErrs, termErrs...) // TODO (verult) consider improving runtime for _, t := range rawTopologies { if helper.Semantic.DeepEqual(exprMap, t) { allErrs = append(allErrs, field.Duplicate(idxPath.Child("matchLabelExpressions"), "")) } } rawTopologies = append(rawTopologies, exprMap) </ins> } return allErrs

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --/kind feature The field essentially has the type , but currently there is no validation for set and map semantics, i.e. unique elements / keys. Having this validation at API level removes logic to validate at the volume plugin level. edit: this check will not be in cases where this makes it difficult to add other fields and make this field optional. /sig api-machinery /sig storage
/assign
Thanks for catching this. Would you be able to implement this for this release? We want to try to go beta if we can, and we won't be able to strengthen validation once that happens.
Yes will do!


Question: k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211105084753-ee342a809c29 h1:SgxutK76kGA2O/LIjRjoJ2ABggpGJlaJOiLyOdCjEsU= k8s.io/kube-openapi v0.0.0-20211105084753-ee342a809c29/go.mod h1:X90lRFlqk35/w9FG4WIvZqMPfG3WrZGzdlSaL6uh7rc= <del> k8s.io/system-validators v1.5.0 h1:gGgluCTkpKc/zUszjamp4LFfMVM0wuYG2qjIFL4MMeQ= k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= </del> <ins> k8s.io/system-validators v1.6.0 h1:21qaPNdZ+mQrm4qc5shU0T5Eh49t/miFqZsn4sW8Hr0= k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= </ins> k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=

every k8s release, if changes are to we release a new version of the library and vendor it in this repository. TODO: [ ] wait for changes to accumulate in [ ] add new release in before the k8s 1.23 release [ ] vendor the new release in k/k (example PR: )
/sig cluster-lifecycle node /milestone v1.23 /assign
/triage accepted
/cc


Question: diskName:         fakeGetDiskFailed, oldSize:          *resource.NewQuantity(2*(1024*1024*1024), resource.BinarySI), newSize:          *resource.NewQuantity(3*(1024*1024*1024), resource.BinarySI), <del> existedDisk:      compute.Disk{Name: to.StringPtr(fakeGetDiskFailed), DiskProperties: &compute.DiskProperties{DiskSizeGB: &diskSizeGB}}, </del> <ins> existedDisk:      compute.Disk{Name: to.StringPtr(fakeGetDiskFailed), DiskProperties: &compute.DiskProperties{DiskSizeGB: &diskSizeGB, DiskState: compute.Unattached}}, </ins> expectedQuantity: *resource.NewQuantity(2*(1024*1024*1024), resource.BinarySI), expectedErr:      true, expectedErrMsg:   fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: Get Disk failed"),

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: resize Azure disk issue when it's in attached state: What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: /priority important-soon /sig cloud-provider /area provider/azure /triage accepted /kind bug /assign


Question: <ins> /* Copyright 2014 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package healthz import ( "net/http" "net/http/httptest" "testing" ) func TestInstallHandler(t *testing.T) { mux := http.NewServeMux() InstallHandler(mux) req, err := http.NewRequest("GET", "http://example.com/healthz", nil) if err != nil { t.Errorf("Unexpected error: %v", err) } w := httptest.NewRecorder() mux.ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("Expected %v, got %v", http.StatusOK, w.Code) } if w.Body.String() != "ok" { t.Errorf("Expected %v, got %v", "ok", w.Body.String()) } } </ins>

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --aks-engine-azure-windows-master-staging ServiceAccounts ServiceAccountIssuerDiscovery should support OIDC discovery of service account issuer [Conformance] /sig windows /cc
cc
/assign
/assign Let me know if you want to pair on this
We took a look at this for docker case and we believe it is happening due to Windows DNS networking limitations. DNS can be available sometime after the container starts due to the way networking is set up with docker. In containerd the setup steps are different so the network will be available when the container starts. In this case the test image does a lookup before the DNS is finished coming online and fails.
it fails in linux as well
From the link posted, it looks like the test cluster issuer might be misconfigured (--service-account-issuer is missing https, I'm not sure why this even passed kube-apiserver validation, double-checking that also): Try setting on kube-apiserver.
reminded me we couldn't actually add that validation, because we weren't previously validating schemes and it would've broken existing clusters with bad issuers set... So as far as fixing the tests make sure issuers have schemes, at least.
I guess the config in this case is managed behind AKS service, and AKS has been setting without https scheme?
Thanks! The failures on the Linux tests are a separate issue from the Windows failures on docker. I opened kubernetes-sigs/cloud-provider-azure to resolve the Linux failures (we have already done this for Windows: )
/reopen This is still an issue for windows docker. fixes it
Reopened this issue. <detailsIn response to : /reopen This is still an issue for windows docker. fixes it Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I'm glad this was fixed for open-source test configs, but I assume since the test said "aks" in the name this could be an issue for managed Azure clusters?

Clients should reject issuers without https instead of attempting to validate them, so we'd want all clusters to be using https scheme in the issuer URL, but at the same time, changing the issuer URL on an existing cluster invalidates previously-issued tokens, because kube-apiserver would no longer recognize the old issuer. I just want to confirm that what was happening with tests isn't foreshadowing a bigger issue for existing Azure clusters. can you/someone at Microsoft confirm? FYI
We use aks-engine (self-managed cluster instead of managed AKS cluster) to run the tests. For aks-engine, I opened a PR () to fix the service account issuer. For AKS, I can confirm that the service account issuer does not any scheme in the URL at all. Tagging who works on aks-engine. Tagging who works on AKS.
We confirmed that is being fixed in AKS.
/traige accepted /milestone v1.21 /priority important-soon
/triage accepted
Our ipv6 test is still failing: The service issuer has been set to .
that seems to be a different issue than Windows. could you open a new issue and tag and me?


Question: e.podLister = podInformer.Lister() e.podsSynced = podInformer.Informer().HasSynced <ins> endpointsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ DeleteFunc: e.onEndpointsDelete, }) </ins> e.endpointsLister = endpointsInformer.Lister() e.endpointsSynced = endpointsInformer.Informer().HasSynced

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? no. What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): service endpoints missing Is this a BUG REPORT or FEATURE REQUEST? (choose one): Bug Report <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): 1.6.2 Environment: Cloud provider or hardware configuration: AWS OS (e.g. from /etc/os-release):Kernel (e.g. ): Install tools: manual Others: What happened: Service in namespace sees no endpoints that are available What you expected to happen: For it to see available endpoints How to reproduce it (as minimally and precisely as possible): Not sure I can. It appeared on the latest remove/replace of worker nodes, but was fine before Anything else we need to know: I have a traefik Ingress controller defined and a service for it: Selectors match, the pods are running just fine, yet the endpoints do not show up. I suspect if I delete the service and recreate it, it probably will pick it up, but I shouldn't have to, and am concerned it might occur again; rather actually debug it. How can I check what kube is doing to affiliate it? How often does it check? Where?
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
The system is asking for a SIG, but it isn't clear which SIG is appropriate for this. Taking best guess, apologies if I am wrong.

/sig sig/apps /sig sig/scheduling
/sig network
can you provide the following output?
I wish I could... in the week since, the cluster has been deleted and recreated several times. AWS instances may be cheap, but still aren't free. Client didn't want them sticking around too long for the cost.
Should we close this issue pending more information or keep it open for someone to test out?
we are killing and recreating clusters with some regularity over these weeks. I am happy if you want to close it since I cannot recreate it yet. Just let me reopen it if I manage to recreate and get more info. Just tell me, please, if it does come up again, what additional data should I get? There isn't much data in the docs on how the service resolves what endpoints it should use and how to figure out why it is misbehaving when it does. Happy to take pointers.
Please reopen when it happens again. Please record related endpoints objects and pod objects.
I'm facing same issue with Nginx by trying to expose the nodePort for Nginx-ingress-controller pod
Try .
Your service is using while your pods are using .
Sorry, didn't notice that
OS: ubuntu 16.04 kubernetes version: v1.8.4 hi, I have the same problem now. $ kc get endpoints -njenkins This situation has been going on for a long time and no solution has been found. Help out. Thanks
Same the problem, svc lost the endpoint after a few time. If deleting the pod, then the svc endpoint is show up again. How to debug this problem?
Im having the same problem on Windows Nodes, after some hours all svc on one node loses their endpoints. Kubernetes V1.15
May be fixed by this?

same issue for a custom helm that I created: kubectl version Client Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.10", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-08-19T10:52:43Z", GoVersion:"go1.11.13", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.5", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-04-03T04:12:37Z", GoVersion:"go1.11.5", Compiler:"gc", Platform:"linux/amd64"} kubectl describe service/vnfdb-ui Name: vnfdb-ui Namespace: test-helm Labels: Annotations: <noneSelector: , Type: LoadBalancer IP: 10.105.168.158 Port: http 80/TCP TargetPort: http/TCP NodePort: http /TCP Endpoints: <noneSession Affinity: None External Traffic Policy: Cluster Events: <none$ kubectl get pods -l , NAME READY STATUS RESTARTS AGE vnfdb-ui--ttrwm 1/1 Running 0 9m $ kubectl get endpoints vnfdb-ui NAME ENDPOINTS AGE vnfdb-ui <none9m33s $ kubectl get endpoints vnfdb-ui -o yaml apiVersion: v1 kind: Endpoints metadata: creationTimestamp: "2019-08-24T21:28:58Z" labels: vnfdb Tiller ui "1.0" ui-0.1.0 name: vnfdb-ui namespace: test-helm resourceVersion: "" selfLink: /api/v1/namespaces/test-helm/endpoints/vnfdb-ui uid: -c6b6-11e9-a0d5- Any clue what is wrong here and how can I fix it?
Got it, apparently realized that its not only the selector that has to match labels. The port names should be the same in the pod definition and services definition as well. The latter was the culprit in my case.
Same the problem.My service and pod is running. But the service do not has endpoint,and this situation lasts for 10 hours. It is OK after I restart controller manager.

My last operation was to delete the service and create the service again. I have not done any other operations since then. Kubernetes V1.11.5
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Good catch!
Happens to us once in a while: We have a process creating a service, then a test deleting that same service, which ends up being automatically recreated. Once in a while (every ~100 test), the re-created service never gets any endpoints, even after waiting for several hours. I'm fairly confident recreating the service again fixes it, but that's not a solution. It just happened on a GKE cluster in version , for which I don't have controller-manager logs available unfortunately. I'll try to reach GCP support, they may be able to troubleshoot more from the controller-manager logs. Edit: the endpoint magically appeared after 22 hours.
Thanks for the detailed bug report! I'll try to recreate this scenario with some new tests. /assign
A quick follow up to let you know we haven't forgotten about this issue. From what we can tell, this is being caused by an Endpoints informer cache that can potentially be out of date when syncService() runs. Essentially syncService() would be called twice - once for the delete and once for the create event.

There's a chance that when the second run of syncService() happens the Endpoints cache may not have been updated with the latest run. If that happens, syncService() will not make any changes because it sees an endpoints resource that has all the right ports and addresses set. is working on a fix here that will mirror what we've done for the EndpointSlice controller with the . This will involve watching Endpoints changes and calling syncService() if the change has not already been accounted for by the controller. As far as I can tell, this should be a relatively rare bug. It should only happen if the cache is out of date, the recreated Service is identical, and the Pods have not changed at all. Given the relative complexity of the fix here, I'm not sure how far we'll be able to backport this. At the very least, we're hoping to have a fix in for 1.19.
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I think is almost ready, sorry for not catching the lifecycle close earlier.
Dude, you saved my life.


Question: v1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" <ins> apierrors "k8s.io/apimachinery/pkg/api/errors" </ins> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime"

What happened: Scheduler decided that it wants to schedule Pod P on Node N after preempting some pods from N. It set the P.nominatedNodeName to N. After a while (before scheduling actually happened) node N was removed from the cluster. Scheduler did not reset P.nominatedNodeName to empty value. What you expected to happen: I would expect scheduler to unset P.nominatedNodeName when the node is no longer valid. The trivial reason for node not being valid any more is it is removed from the cluster. But there are more cases to cover. Out of the top of my head I may think of: extra Pod was scheduled to node and there is not enough resources any more either scheduled by scheduler or directly as static pod/daemonset node became unschedulable How to reproduce it (as minimally and precisely as possible): We did not try to reproduce issue manually. Yet following setup should work. create cluster with nodes A and B. ensure all system pods are running on A. Fill up A so it does not have spare resources. add a low priority Pod L with long termination, with requests matching allocatable of B. It should be scheduled to B. add a high priority Pod H. It should get B as nominatedNodeName kill node B the H.nominatedNodeName will still point to B which is no longer in the cluster Anything else we need to know?: Current behavior does not play well with Cluster Autoscaler which assumes that Pod with nominatedNodeName will be scheduled in a while. Therefore such pods do not trigger scale-up. If we can have P.nominatedNodeName set incorrectly for unbounded period of time CA will never provision node for P to run. Environment: Kubernetes version (use ): We observed the behavior during presubmit tests which run k8s built from HEAD of master on 27.11.2019.

Versions logged in test log: I1125 18:55:11.691] Client Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.0.1183+", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-11-25T17:04:18Z", GoVersion:"go1.13.4", Compiler:"gc", Platform:"linux/amd64"} I1125 18:55:11.692] Server Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.0.1183+", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-11-25T17:04:18Z", GoVersion:"go1.13.4", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: GCE /sig scheduling
/assign
/cc
Another interesting case would be creating a new pod with pod antiaffinity blocking the pod with nominated node name from scheduling: up a cluster with pods, including a long-terminating low-priority pod A. high-priority pod B that has the same requests as A. pod C with negligible resource requests and zonal topology pod antiaffinity matching pod B (and no other pods in the cluster). What I'd expect: Pod C keeps pending, nominatedNodeName is cleared. Pod A is restarted on the same node it was running before. I haven't actually tested this scenario, so I'm not sure what will actually happen. If report is correct, this may be a more convenient way of reproducing the issue (no need to add / remove nodes, it's not important which pod will go to which node).
I'm not sure if we need to clear nominatedNodeName on Pod when node is no longer valid immediately. And we have other logic to clear nominatedNodeName. Reuse your example. create cluster with nodes A and B. ensure all system pods are running on A. Fill up A so it does not have spare resources.

add a low priority Pod L with long termination, with requests matching allocatable of B. It should be scheduled to B. add a high priority Pod H. It should get B as nominatedNodeName kill node B the H.nominatedNodeName will still point to B which is no longer in the cluster After this, The H will be scheduled again. Obviously it's going to fail. H will enter the preemption process and the is empty. It will return itself as preemption process will clean nominatedNodeName on the Pod H.
when I have some time I will check if reproduction steps I describe actually work. But we surely observed pods with lingering nominatedNodeName which pointed to inexistent node, during our E2E tests. Quick thought. The method has many return points. If it returned at the place you linked it seems should be cleared. But there are many return points below, which set the list of pods to be cleared to nil. Is it possible that for a pod to consistently exit from with one of those return points?
I think you are right. Actually there are many return points to return nil. I think maybe we need to clear the of the pod after the filter failed again and before the new preemption process to avoid potential risks. WDYT?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or .

/lifecycle stale
/remove-lifecycle stale
I believe it's a bug as in old logic, e.g., v1.18.3, if the Pod (with nominatedNodeName set) cannot be scheduled and also preemption doesn't help ( nil potentialNodes is a corner case, it's more common the pod can't fit on all potentialNodes), it won't hit L413: then in old recordingFailure logic, nominatedNodeName remains as is. In the latest codebase, introduced a fix to combine setting nominatedNodeName as well update the pod's unschedulable condition. However, this doesn't help in this issue either as it ignores the case when the pod's nominatedNodeName is nil:
/reopen as regresses - we need to come up with a complete fix.
Reopened this issue. <detailsIn response to : /reopen as regresses - we need to come up with a complete fix. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle rotten
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.

If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle rotten
what are the implications of not clearing the nominated node name?
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: if len(gotList) != len(tc.expectList) { t.Errorf("got a list of length %d, want %d", len(gotList), len(tc.expectList)) } <del> for i := range gotList { // Make timestamps the same before comparing the two lists. gotList[i].LastProbeTime = tc.expectList[i].LastProbeTime gotList[i].LastTransitionTime = tc.expectList[i].LastTransitionTime } if diff := cmp.Diff(tc.expectList, gotList); diff != "" { </del> <ins> if diff := cmp.Diff(tc.expectList, gotList, cmpopts.IgnoreFields(batch.JobCondition{}, "LastProbeTime", "LastTransitionTime")); diff != "" { </ins> t.Errorf("Unexpected JobCondition list: (-want,+got):n%s", diff) } })

If the user update , duplicate condition: ! Only one Failed condition Create Job and wait it marked as Failed Update the Failed condition to False No response <details</details<details</details<details</details<details</details<details</details<details</details
It's not a bug. The is a list, so it can contain many conditions with different probe timestamps. Also, we should not update the fields.
But should we avoid the duplicate Type's conditionIn my opinionthis will confuse the user. According to kubelet, it will rewrite the 'Ready' condition in pod, and take is as managed only by kubelet. Should we did this for job Failed condition? Refer:
/sig apps
It is a bug. We should only have one condition for each type. /triage accepted
/assign
/assign
/assign
I have reproduced the issue locally and intend work on the fix
/priority backlog Since the user is manually modifying the status, this is not an inherent bug in the controller. Still worth avoiding duplicates.


Question: // apply tags if volumeOptions.Tags != nil { <del> tags := []*ec2.Tag{} for k, v := range *volumeOptions.Tags { tag := &ec2.Tag{} tag.Key = aws.String(k) tag.Value = aws.String(v) tags = append(tags, tag) } tagRequest := &ec2.CreateTagsInput{} tagRequest.Resources = []*string{&awsID} tagRequest.Tags = tags if _, err := s.createTags(tagRequest); err != nil { </del> <ins> if err := s.createTags(awsID, *volumeOptions.Tags); err != nil { </ins> // delete the volume and hope it succeeds _, delerr := s.DeleteDisk(volumeName) if delerr != nil {

After setting up a few services, I noticed the following output in : I assume the service tried to configure an ELB in the past (not sure when or how or what details it tried to use) and has kept trying and now can't even use the AWS API. How can I either: 1) tell it to please stop so that I can use my AWS account again 2) fix it so that it does configure the ELB /cc
Sorry about the problem. Looks like we need a sleep / backoff policy here! To get you up and running again, if you redeploy the nginx service without Type=LoadBalancer, we will (should) stop trying to create a load balancer, and you should get out of the AWS API penalty-box. Sorry about this, and thanks for the report. Looks like this is pretty important to fix. Do you happen to know why you can't create an ELB? My guess is that you have hit your quota?
I'll remove the service, wait for the penalty-box to expire, then build it again so that we can see what the real error was.
Adding the LoadBalancer service back in, I'm seeing:
I took a few different steps and seem to have gotten it both working in different ways: First I deleted the service because it was erroring, but I noticed the count was still going up. So I deleted the security group it said was a duplicate and then it finished setting the ELB up! That meant I needed to manually clean up the ELB resources and security groups because k8 thought it was already deleted. After cleaning up all the resources in AWS again, I the service again and it seemed to work fine! I'm not sure how/why it initially confused itself with the duplicate SG, but it doesn't necessarily seem to happen every time. The code could likely just, in the event that it sees the SG already exists, continue on with the setup. I could send you a PR to handle that if you'd like!
I have a PR that I think covers this: . I thought it was a one-off error, but it sounds like it wasn't. Thanks so much for pausing & retrying also - that was super-helpful. Even if we think covers this, let's leave this issue open to track the need for a backoff to avoid overloading the AWS API.

(And presumably this aggressive retrying happens on other clouds also, though maybe they have overload a sleep built in).
Certainly looks like should fix it. I'm still wondering why I'd see it failing literally for hours in a loop though. If it was a problem with AWS being eventually consistent, then after an hour I'd think that first check, looking to see if the securityGroup existed, would start to return the group! I don't know the k8 code well yet, though so perhaps it was looping in a different place that wasn't running that code to see if the security group already existed.
you're right; my explanation wasn't entirely correct... So security group names are unique per VPC. We also have a per-cluster name, and we filter all our AWS API interactions to only select resources tagged with this name. If we had a security group name collision across two k8s clusters in the same VPC, that would be bad. Thankfully the security group name we're creating includes a UUID, so this "won't happen". However, the problem is that we created the security group, and then failed to tag it. But we were filtering based on our cluster. So our "describe security groups" operation would never see the SG, because it wasn't appropriately tagged. We have a few workarounds to make this much less likely, and I have opened the "big picture" issue . And I'm going to think about whether we should in fact be filtering when we DescribeSecurityGroups - I think the answer is no in the case where the name has a UUID in it.
Ah! That totally makes sense. The race between resource creation and resource tagging can created "invisible" resources if the process of tagging fails. Maybe an anti-entropy function that can look at all the resources and normalize them. In the case of invisible resources, you can just delete them and hope that recreating them won't fail the tagging process.
I see this as well when frequently deploying/testing. Unfortunately, it happens far too often (running v1.0.0).
+1
I've been having a similar issue to this, Kubernetes can't create a load balancer as it says the security group already exists. The workaround I've found is to manually delete the security group and wait for Kubernetes to successfully create it again.

I've generally had the problem when quickly deleting a service with a and then creating it again (I see now I should use ). I wonder if it is something to do with AWS reassigning the same load balancer back to you if you haven't finished deleting it coupled with the tagging issue mentioned earlier.
+1
cc
I had an email discussion with on this and he said that what's remaining shouldn't be considered a blocker for 1.1, so I'll move it to 1.2-candidate.
Can you summarize what's remaining to do for this issue? Should we just close it in favor of ?
I see you the v1.2 milestone -- will you be able to look into this in the next few days?


Question: return cmdutil.NewCachedDiscoveryClient(discoveryClient, cacheDir, time.Duration(10*time.Minute)), nil } <ins> func (f *fakeAPIFactory) CategoryExpander() resource.CategoryExpander { if f.tf.CategoryExpander != nil { return f.tf.CategoryExpander } return f.Factory.CategoryExpander() } </ins> func (f *fakeAPIFactory) ClientSetForVersion(requiredVersion *schema.GroupVersion) (internalclientset.Interface, error) { return f.ClientSet() }

Is this a BUG REPORT or FEATURE REQUEST? (choose one): FEATURE REQUEST Kubernetes version (use ): HEAD Now that RBAC is going to GA, using ServiceAccounts is getting more and more important. Still a lot of users run their apps/addons/whatever in the SA. In order for the admin to be able to change ServiceAccount easily after an application has been installed (which supposedly is crashlooping due to insufficient privs), a command could exist. I know I can do this: but for new users to Kubernetes, it's non-obvious. Proposed command: Tracking issue: WDYT? This would be a good way for a new contributor to get aquainted with the code by contributing this feature.
Is anyone working on this? If not I would like to take this up.
Nobody is working on this. I would be super happy if you could contribute this feature. Be aware of that there isn't consensus around this proposal yet (I proposed it yesterday, saturday, so people haven't had time to catch up on it) However, I do think that this is in line with , so feel free to start working on this if you want to ;) Also, don't hesitate to ask for help if you don't know how to get started and we'll help out! Thanks!
I'm in favor of this command. ?
Thanks, I will start working on this.
Is the SA on a Pod mutable or immutable? What about DaemonSet and StatefulSet? Do those have SA as well? Is there a way to do this so that it dynamically supports anything with a PodTemplate? Maybe this should be done as a subresource that can be dynamically discovered by kubectl using the discovery service.
Maybe helpful:
Pod is immutable. All the other objects (with the exception of stateful set) are mutable.
That pattern is the same for all the things that set aspects of a pod spec (except image, which is also mutable in pods)
Statefulsets are going to loosen their validation soon I expect.
This sounds good to me. just go ahead, I will help review this. Thanks ~
The outout of this command kubectl get rc/nginx-controller -o=yaml , has service acccount at the following path '' , and the corresponding pod has service account at 'spec.serviceAccountName'.

Both refer to the 'serviceAccountName' field of PodSpec struct but can be made to be different by patching the ReplicationController resource. How does ReplicationController and Pod use serviceAccountName differently? Shouldn't the ReplicationControllerSpec have a serviceAccountName field?
ptal, I think you are more familiar with ReplicationController implementation than me.
ReplicationController contains a pod spec object, which contains a serviceAccountName.
Hi , how to go about writing e2e test cases for this. My development is mostly complete.
I wouldn't mind if you posted the PR now and then we'd see what e2e test makes sense to have. In order to move forward...
what are the next steps?


Question: } }) <del> ginkgo.It("should run through the lifecycle of a PV and a PVC", func(ctx context.Context) { </del> <ins> /* Release: v1.29 Testname: PersistentVolumes(Claims), lifecycle Description: Creating PV and PVC MUST succeed. Listing PVs with a labelSelector MUST succeed. Listing PVCs in a namespace MUST succeed. Patching a PV MUST succeed with its new label found. Patching a PVC MUST succeed with its new label found. Reading a PV and PVC MUST succeed with required UID retrieved. Deleting a PVC and PV MUST succeed and it MUST be confirmed. Replacement PV and PVC MUST be created. Updating a PV MUST succeed with its new label found. Updating a PVC MUST succeed with its new label found. Deleting the PVC and PV via deleteCollection MUST succeed and it MUST be confirmed. */ framework.ConformanceIt("should run through the lifecycle of a PV and a PVC", func(ctx context.Context) { </ins> pvClient := c.CoreV1().PersistentVolumes() pvcClient := c.CoreV1().PersistentVolumeClaims(ns)

[X] test approval issue : [X] test pr : 26 May 2023 [x] two weeks soak end date : 9 June 2023 [x] test promotion pr : [x] remove endpoint from [x] End of release 1.28 Release: Remove duplicate Conformance test the do not cover Patch, Replace, DeleteCollection. According to following APIsnoop query, there are still 3 CSIDriver endpoints that are not tested for Conformance. Using a number of existing e2e test practices a new has been created to address these 3 endpoints. The e2e logs for this test are listed below. This query shows the following csiDriver endpoints are hit within a short period of running this e2e test. If a test with these calls gets merged, test coverage will go up by 3 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen For end of release 1.28 Release: Remove duplicate Conformance test the do not cover Patch, Replace, DeleteCollection.
Reopened this issue. <detailsIn response to : /reopen For end of release 1.28 Release: Remove duplicate Conformance test the do not cover Patch, Replace, DeleteCollection. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: // the same network interface couldn't be added to more than one load balancer of // the same type. Omit those nodes (e.g. masters) so Azure ARM won't complain // about this. <ins> newBackendPoolsIDs := make([]string, 0, len(newBackendPools)) </ins> for _, pool := range newBackendPools { <del> backendPool := *pool.ID matches := backendPoolIDRE.FindStringSubmatch(backendPool) if len(matches) == 2 { lbName := matches[1] if strings.HasSuffix(lbName, InternalLoadBalancerNameSuffix) == isInternal { klog.V(4).Infof("vmss %q has already been added to LB %q, omit adding it to a new one", vmSetName, lbName) return nil } </del> <ins> if pool.ID != nil { newBackendPoolsIDs = append(newBackendPoolsIDs, *pool.ID) </ins> } } <ins> isSameLB, oldLBName, err := isBackendPoolOnSameLB(backendPoolID, newBackendPoolsIDs) if err != nil { return err } if !isSameLB { klog.V(4).Infof("VMSS %q has already been added to LB %q, omit adding it to a new one", vmSetName, oldLBName) return nil } </ins> } newBackendPools = append(newBackendPools,

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: SLB backend pools are empty if outbound rules are defined in a separate backend pool. This is because, although we the check for not adding the backend pools from different LBs, the lbName is not checked: The new backend pool's lbName should be also checked in above codes. What you expected to happen: SLB backend pools should be updated correctly when outbound rules are defined in a separate backend pool. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Others: /sig azure /kind bug /assign


Question: expectedArgFunc func(dir string) []string }{ { <del> name: "caKeyPresent-false", </del> <ins> name: "caKeyPresent-false for v1.12.0-beta.2", </ins> cfg: &kubeadmapi.InitConfiguration{ APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ <del> KubernetesVersion: "v1.7.0", </del> <ins> KubernetesVersion: "v1.12.0-beta.2", </ins> Networking:        kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, }, },

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When I use kubeadm to create an old version k8s cluster, the controller manager pod can not work well. The command like this : "kubeadm init --kubernetes-version=v1.11.2" The error of kube controller manager pod like this : What you expected to happen: The controller manager pod can work well. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.12.0 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): CentOS 7 Kernel (e.g. ): Install tools: kubeadm Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-sig contributor-experience /area kubeadm
Please, use when filing kubeadm issues. /sig cluster-lifecycle
OK, thank you for reminding me.


Question: "description": "Selects the pods to which this NetworkPolicy object applies.  The array of ingress rules is applied to any pods selected by this field. Multiple network policies can select the same set of pods.  In this case, the ingress rules for each are combined additively. This field is NOT optional and follows standard label selector semantics. An empty podSelector matches all pods in this namespace." }, "policyTypes": { <del> "description": "List of rule types that the NetworkPolicy relates to. Valid options are Ingress, Egress, or Ingress,Egress. If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include "Egress" (since such a policy would not include an Egress section and would otherwise default to just [ "Ingress" ]). This field is beta-level in 1.8", </del> <ins> "description": "List of rule types that the NetworkPolicy relates to. Valid options are "Ingress", "Egress", or "Ingress,Egress". If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include "Egress" (since such a policy would not include an Egress section and would otherwise default to just [ "Ingress" ]). This field is beta-level in 1.8", </ins> "items": { "type": "string" },

Is this a BUG REPORT or FEATURE REQUEST?: bug report /kind bug What happened: I was reading the documentation for as defined What you expected to happen: To have clear documentation How to reproduce it (as minimally and precisely as possible): Read the following, as seen in the 1.12 docs: Anything else we need to know?: Nope Environment: Kubernetes version (use ): N/A Cloud provider or hardware configuration: N/A OS (e.g. from /etc/os-release): N/A Kernel (e.g. ): N/A Install tools: N/A Others: N/A
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Is it a case whereby only can be specified as , or only , or both (note the ) ? If that's the case, do you think the following would make it clearer?
It is confusing to have commas used for English grammar and array element delineation in the same sentence. What about simply this?
sounds like a great fix! Putting out a quick pr with this change now :)


Question: apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v17 </del> <ins> name: kube-dns-v17.1 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: replicas: $DNS_REPLICAS selector: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: containers:

Observed weird memory spikes for both the dns healthz and kube-dns pods in production. Kube-dns didn't oom because there's enough headroom, but healthz did. I can't explain the spikes yet. I tried a bunch of different repros: no , bad nslookup command, sharing the node with cpu hogs etc, but was unable to trigger it. The memory grows, but stays under 10Mi. As a short term mitigation, we stole some ram from kube-dns, because it was always comfortably within limits. Related b/
can be closed -- healthz has been removed
-- close issue?


Question: rbac.NewRule("list", "watch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), rbac.NewRule("list", "watch", "create", "delete", "patch").Groups(legacyGroup).Resources("pods").RuleOrDie(), rbac.NewRule("create").Groups(legacyGroup).Resources("pods/binding").RuleOrDie(), <del> rbac.NewRule("list", "watch", "create", "delete", "update", "patch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(), </del> <ins> rbac.NewRule("get", "list", "watch", "create", "delete", "update", "patch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(), </ins> eventsRule(), }, })

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /sig auth What happened: Testing a daemon-set spec, I deleted/recreated it multiple times. At one of the recreations, the daemon-set was not scheduled anymore, and the number of desired pods went to 0 (on a 3 nodes cluster): The kube-apiserver stderr logs shows: The kube-controller stderr logs shows: The (default) permissions for the cluster role are: Seems to be missing a permission for the resource. I edited manually the cluster-role adding the permission and then the daemonset was scheduled again. What you expected to happen: The cluster role default permissions should allow getting the resource. How to reproduce it (as minimally and precisely as possible): I tried to reproduce it again, but I'm unable to find the exact pattern to reproduce it. Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release):Kernel (e.g. ): Install tools: Others:
Same here (5 node cluster, k8s version 1.9.3, Ubuntu 16.04 LTS, 4.4.0-109-generic). Deamonset we were trying to create/delete is part of official elastic yaml for metricbeat installation. Problem is specific to daemonset part, other parts of manifests behave correctly on deletion/creation. (). On 1st create - all 5 nodes were populated (kubectl create -f metricbeat-). On 1st subsequent delete followed by create we were stuck with desired 0 and error in scheduler log (E0405 14:09:46. 1 ] kube-system/elk-metricbeat failed with : failed to construct revisions of DaemonSet: "elk-metricbeat-" is forbidden: User "system:serviceaccount:kube-system:daemon-set-controller" cannot get in the namespace "kube-system"). However if we changed name of deamonset (metricbeat-v1 for example) it would bring pods on 4 nodes (master missing), setting desired to 4 (instead of 5).

Again if deleted, neither that name or any previously used name would work (desired 0), while new name would spawn 4 pods (desired 4 instead 5 master missing). No stucked pods or jobs, all other not-affected deamonsets (wave-net, kube-proxy, weave-scope) all work fine.
why did this not happen in all of the recreations and only once did it fail with RBAC issue ?
maybe this is related: "However if we changed name of deamonset (metricbeat-v1 for example) it would bring pods". New name of deamonset would spaws pods, but any subsequent retry with any of previous names would fail (until we change to new name again). Once we get to we were able to spawn pods with previously given names as well, so it might be related...
DaemonSet controller only calls GET ControllerRevisions when there's ControllerRevision name collision when DaemonSet tries to create a new ControllerRevision. This is only called when the ControllerRevision cache store is stale or due to hash collisions.


Question: if changed != &r { t.Errorf("returned request should point to the same object") } <del> if !api.Semantic.DeepDerivative(changed, &original) { </del> <ins> if !reflect.DeepEqual(changed, &original) { </ins> t.Errorf("expected %#v, got %#v", &original, changed) } }

This tree has a commit that demonstrates two obviously different objects that should not pass the semantic DeepEqual but do.
What shall we do with this issue for triage? Labels? Priority?
P2 for when he returns , Satnam Singh wrote:
OK I am now sufficiently worried about this. Fixes incoming.


Question: return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") } <del> if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { </del> <ins> if dualStack { </ins> klog.V(0).Info("creating dualStackProxier for iptables.") <del> // Create iptables handlers for both families, one is already created // Always ordered as IPv4, IPv6 var ipt [2]utiliptables.Interface if iptInterface.IsIPv6() { ipt[1] = iptInterface ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4) } else { ipt[0] = iptInterface ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6) } </del> // Always ordered to match []ipt var localDetectors [2]proxyutiliptables.LocalTrafficDetector localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo)

(update: likely this is benign, but it seems really weird when your reading kube proxy logs) in healthy, IPTABLES jobs, the various ip6tables-restore commands are happy....MEANWHILE in ipvs jobs (healthy, or non healthy, we see that some of the iptables dont exist which makes sense... why is ipvs kube proxy tyrying to do iptables-restore operations on the table ? why is it trying to do iptables stuff at all ? It seems like we get error messages when running ipv6. not sure why though? in particular i found these errors when looking at the test failures for You wouldnt see error messages for ip6tables-restore when running ipvs tests. This may be my own ignorance however, i.e. maybe this is normal. I guess every job has this (wether or not the job passes or fails) ? Here is an example from a passing test.... Similar logs from a failing tests that i originally saw this message ... has this failure, it is reproduced via the corresponding ci job ... the logs in the kube-proxy for one of the nodes look like this: ill add more links as i collect more data. Kubernetes version (use ): prow (main branch as of feb 12 2021)
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
So, this happens in HEALTHY jobs to, so i guess its not a driver for an actual bug.... but its still weird to me :) ... updating original issue...
any idea on why those ipv6 errors are printed in ipvs code ?
The proxier is configured in dual-stack mode in the CI Since we enabled the dual-stack gate recently, this is the expected behaviour
I don't get similar printouts myself so I can't say anything for sure. By first check the old re-occuring (and very annoying) "KUBE-MARK-DROP" problem.

This chain is usually created by "kubeadm" but so far kubeadm is unaware of dual-stack and has only created this chain for ipv4 (in iptables). Then it needs to be created for IPv6 before kube-proxy is started; In my test cluster I always do this in a prep-stage before starting K8s.
The ci nodes doesn't have the kernel ip6tables nat modules loaded, hence they fail all the operation related
hmmm ok so is the issue here that we need to load that module in the CI jobs?
signing here because I want to at least find why ip6tables restore is called on kube-proxy ipvs things :D
ipvs does uses iptables too I think that kube-proxy should implement something similar to what implemented in kubelet The nat table for IPv6 was in 3.7 and it seems that it may be not present in some systems. /cc
/assign
I think it's mostly just about whether IPv6 is enabled, not about the kernel version.
IIRC this specific error in the CI is about the nat module only Anyway, you fixed it :clap: /unassign /assign