Question: } } <ins> // 1. connect to https server with http1.1 using a TCP proxy // 2. the connection has keepalive enabled so it will be reused // 3. break the TCP connection stopping the proxy // 4. close the idle connection to force creating a new connection // 5. count that there are 2 connection to the server (we didn't reuse the original connection) func TestReconnectBrokenTCP_HTTP1(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %s", r.Proto) })) ts.EnableHTTP2 = false ts.StartTLS() defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("failed to parse URL from %q: %v", ts.URL, err) } lb := newLB(t, u.Host) defer lb.ln.Close() stopCh := make(chan struct{}) go lb.serve(stopCh) transport, ok := ts.Client().Transport.(*http.Transport) if !ok { t.Fatal("failed to assert *http.Transport") } config := &Config{ Host:      "https://" + lb.ln.Addr().String(), Transport: utilnet.SetTransportDefaults(transport), // large timeout, otherwise the broken connection will be cleaned by it Timeout: wait.ForeverTestTimeout, // These fields are required to create a REST client. ContentConfig: ContentConfig{ GroupVersion:         &schema.GroupVersion{}, NegotiatedSerializer: &serializer.CodecFactory{}, }, } config.TLSClientConfig.NextProtos = []string{"http/1.1"} client, err := RESTClientFor(config) if err != nil { t.Fatalf("failed to create REST client: %v", err) } data, err := client.Get().AbsPath("/").DoRaw(context.TODO()) if err != nil { t.Fatalf("unexpected err: %s: %v", data, err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } // Deliberately let the LB stop proxying traffic for the current // connection. This mimics a broken TCP connection that's not properly // closed. close(stopCh) stopCh = make(chan struct{}) go lb.serve(stopCh) // Close the idle connections utilnet.CloseIdleConnectionsFor(client.Client.Transport) // If the client didn't close the idle connections, the broken connection // would still be in the connection pool, the following request would // then reuse the broken connection instead of creating a new one, and // thus would fail. data, err = client.Get().AbsPath("/").DoRaw(context.TODO()) if err != nil { t.Fatalf("unexpected err: %v", err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } dials := atomic.LoadInt32(&lb.dials) if dials != 2 { t.Fatalf("expected %d dials, got %d", 2, dials) } } // 1. connect to https server with http1.1 using a TCP proxy making the connection to timeout // 2. the connection has keepalive enabled so it will be reused // 3. close the in-flight connection to force creating a new connection // 4. count that there are 2 connection on the LB but only one succeeds func TestReconnectBrokenTCPInFlight_HTTP1(t *testing.T) { done := make(chan struct{}) defer close(done) received := make(chan struct{}) ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/hang" { conn, _, _ := w.(http.Hijacker).Hijack() close(received) <-done conn.Close() } fmt.Fprintf(w, "Hello, %s", r.Proto) })) ts.EnableHTTP2 = false ts.StartTLS() defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("failed to parse URL from %q: %v", ts.URL, err) } lb := newLB(t, u.Host) defer lb.ln.Close() stopCh := make(chan struct{}) go lb.serve(stopCh) transport, ok := ts.Client().Transport.(*http.Transport) if !ok { t.Fatal("failed to assert *http.Transport") } config := &Config{ Host:      "https://" + lb.ln.Addr().String(), Transport: utilnet.SetTransportDefaults(transport), // Use something extraordinary large to not hit the timeout Timeout: wait.ForeverTestTimeout, // These fields are required to create a REST client. ContentConfig: ContentConfig{ GroupVersion:         &schema.GroupVersion{}, NegotiatedSerializer: &serializer.CodecFactory{}, }, } config.TLSClientConfig.NextProtos = []string{"http/1.1"} client, err := RESTClientFor(config) if err != nil { t.Fatalf("failed to create REST client: %v", err) } // The request will connect, hang and eventually time out // but we can use a context to close once the test is done // we are only interested in have an inflight connection ctx, cancel := context.WithCancel(context.Background()) reqErrCh := make(chan error, 1) defer close(reqErrCh) go func() { _, err = client.Get().AbsPath("/hang").DoRaw(ctx) reqErrCh <- err }() // wait until it connect to the server select { case <-received: case <-time.After(wait.ForeverTestTimeout): t.Fatal("Test timed out waiting for first request to fail") } // Deliberately let the LB stop proxying traffic for the current // connection. This mimics a broken TCP connection that's not properly // closed. close(stopCh) stopCh = make(chan struct{}) go lb.serve(stopCh) // New request will fail if tries to reuse the connection data, err := client.Get().AbsPath("/").DoRaw(context.Background()) if err != nil { t.Fatalf("unexpected err: %v", err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } dials := atomic.LoadInt32(&lb.dials) if dials != 2 { t.Fatalf("expected %d dials, got %d", 2, dials) } // cancel the in-flight connection cancel() select { case <-reqErrCh: if err == nil { t.Fatal("Connection succeeded but was expected to timeout") } case <-time.After(10 * time.Second): t.Fatal("Test timed out waiting for the request to fail") } } </ins> func TestRestClientTimeout(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(2 * time.Second)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --In , client-go don't share transport when c.Dial is not nil, but kubelet custorm Dial by default, After PR is integrated, the connections between kubelet and kube-apiserver in our cluster with 4000 nodes increases by five times. kubelet share transport by default, one kubelet only keep one connection to kube-apiserver. start kubelet with rotate-certificates=false Kubernetes version (use ): v1.19.4 Cloud provider or hardware configuration: OS (e.g: ):EulerOS 2.9 Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig api-machinery /sig node
/triage accepted /assign
how many connections between kubelet and kube apiserver on one node after PR in your case? it's mentioned it increases by five times, does that mean before the PR, there is one connection, after the PR, there are 5 or 6 connections?
before the PR, there is one connection, after the PR, there 3 connections, there are some customization capabilities in our product, and we'll have another two connections. In our kubelet startup parameters, rotate-certificates=false
Some increase was expected, since multiple REST clients constructed from a config with a custom dialer cannot safely share a transport, and client-go constructs a REST client for each API group/version accessed. Something like would be required to rework how client-go constructs clients to start sharing REST clients above the transport level between API groups/versions. I think this is working as intended until client-go client construction is reworked.
do you know where in the code the 5 connections are created? are they all safe to share the transport? also are the 5 connections all active all the time? if not, IdleConnTimeout should cause the connection to be closed.
I test the code on master branch, if I change rotateCertificates to false, there are 6 connections between kubelet and kube-apiserver ! !

transport will share when config.Transport not nil if rotateCertificates is true, config.Transport will been set in kubeletcertificate.UpdateTransport but if rotateCertificates is false, config.Transport will not been set.
/area kubelet Any update on this issue?
I will work on this issue
Hi if rotate-certificates=false kubelet customizes the dialer to provide a closeAllConns to close the connection when the connection is dead but not been closed. In , there is a discuss about another solution after client-go has HTTP 2.0 health check, so we don't need this change. So we can solve the current problem by reverting . I have submitted to revert , and this change requires your final confirmation. Thanks you.
are we guaranteed kubelets always speak to kube-apiserver over http/2? if they use a load-balancer that is not http/2 capable, isn't the "close all connections" fix still required as a backstop?
I thought http1.1 wasn't affected because it will always try to connect a new connection
I'm not confident in that. The original issue dated back to before client-go defaulted to http/2, and we definitely saw issues with stuck TCP connections then.
yeah, I could reproduce it with http1.1, however, it can be solved using the transport CloseIdleConnections method please take a look , the last commit needs more work, but I think that the 2 first commits exposeing the CloseIdleConnections() are useful
ok, this seems to work , but it will be nice to have more feedback would you be able to test and see if this solve your issue?
Hi aojea, I have test on v1.21.1 before merge , there are 6 connections to kube-apiserver from kubelet ! after merge , there is only 1 connection to kube-apiserver from kubelet ! so, can solve my issue. thank you very much.


Question: KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. <del> KUBE_COVER=" " KUBE_RACE=" " "${KUBE_ROOT}/hack/test-go.sh" "" -test.run="^X" -benchtime=1s -bench=. -benchmem </del> <ins> KUBE_COVER="" KUBE_RACE=" " "${KUBE_ROOT}/hack/test-go.sh" "" -test.run="^X" -benchtime=1s -bench=. -benchmem </ins>

Right now, we execute unit tests in Travis with , and you can see the results on each line: but we don't have an easy place to go browse our code and figure out what's covered and what isn't. Is there some awesome way to do this? At the very least, can we get the actual line-by-line coverage results integrated into our CI build process so we can poke at these somehow?
cc
I'm willing to take a look into this to at least see if something is possible.
Looking into using Coveralls; there are a few other teams at Google already using this. Integration with Travis and Go are supported with the goveralls package:
Unit test coverage can now be viewed at - both master push builds and PRs will be shown there; additionally, Coveralls will update the status and post comments on PRs with coverage stats. (If we find these too spammy, we can disable either or both.) The top-level also has a badge/link to the coverage dashbaord.


Question: "//test/e2e/framework/providers/azure:all-srcs", "//test/e2e/framework/providers/gce:all-srcs", "//test/e2e/framework/providers/kubemark:all-srcs", <ins> "//test/e2e/framework/providers/openstack:all-srcs", </ins> "//test/e2e/framework/testfiles:all-srcs", "//test/e2e/framework/timer:all-srcs", "//test/e2e/framework/viperconfig:all-srcs",

What happened: When specifying "openstack" as cloud-provider of e2e test, the test is failed like What you expected to happen: We can run e2e test with openstack cloud provider. How to reproduce it (as minimally and precisely as possible): Just kick the following command: ` Anything else we need to know?: This issue doesn't happen on v1.13, so this issue seems to happen during this dev cycle for v1.14. Environment: Kubernetes version (use ): master for v1.14 Cloud provider or hardware configuration: OpenStack OS (e.g: ): Ubuntu 16.04.5 LTS Kernel (e.g. ): 4.4.0-142-generic Install tools: kubeadm Others:
/sig openstack /sig testing
The exception raises at
The reason of this issue comes from
Ideally it is nice to put a new directory for registering the cloud provider.
/assign
I was wondering we could remove "openstack" provider specific tests and conditions from e2e tests, but there are storage tests which are related to OpenStack in e2e tests. So it is nice to add registration for OpenStack at this time.


Question: readonly KUBE_CONTAINER_RSYNC_PORT=8730 # These are the default versions (image tags) for their respective base images. <del> readonly __default_distroless_iptables_version=v0.6.1 readonly __default_go_runner_version=v2.3.1-go1.23rc2-bookworm.0 </del> <ins> readonly __default_distroless_iptables_version=v0.5.6 readonly __default_go_runner_version=v2.3.1-go1.22.5-bookworm.0 </ins> readonly __default_setcap_version=bookworm-v1.0.3 # These are the base images for the Docker-wrapped binaries.

sgtm, add an issue labeled as release-blocking and in the milestone to either promote to 1.23.0 or revert this to 1.22.x before our 1.31 release Originally posted by in Kubernetes is being built and tested with go1.23rc2 at HEAD () to let it bake in the CI and help iron out issues, if any, with this version of Go. However, for the 1.31 release, the version of Go should be a non-rc, released version. The release timeline of go1.23.0 is ~same as the release timeline of K8s 1.31.0 (see ). If go1.23.0 is not out by then, we should revert to go1.22.x before cutting K8s 1.31.0. /sig release architecture /area code-organization /cc
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted /cc
/milestone v1.31
cc
xposting relevant slack thread:
Also unlike in the past, missing shipping 1.23.0 in 1.31.0 is not a big deal, we have much more room to switch to 1.23.0 in a patch release after all the compat changes with the go team. So we should not fear reverting if the timelines don't line up.
(though go1.23 adds the thing that makes it workable to default to old go version compat settings, so we'll have to eke out another 5 months on 1.21/1.22 for until we can adopt 1.23 on release branches in ~Jan if we miss 1.31, but that's how it looks like it will have to happen)


Question: <ins> load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = ["openstack.go"], importpath = "k8s.io/kubernetes/test/e2e/framework/providers/openstack", visibility = ["//visibility:public"], deps = ["//test/e2e/framework:go_default_library"], ) filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], visibility = ["//visibility:public"], ) </ins>

What happened: When specifying "openstack" as cloud-provider of e2e test, the test is failed like What you expected to happen: We can run e2e test with openstack cloud provider. How to reproduce it (as minimally and precisely as possible): Just kick the following command: ` Anything else we need to know?: This issue doesn't happen on v1.13, so this issue seems to happen during this dev cycle for v1.14. Environment: Kubernetes version (use ): master for v1.14 Cloud provider or hardware configuration: OpenStack OS (e.g: ): Ubuntu 16.04.5 LTS Kernel (e.g. ): 4.4.0-142-generic Install tools: kubeadm Others:
/sig openstack /sig testing
The exception raises at
The reason of this issue comes from
Ideally it is nice to put a new directory for registering the cloud provider.
/assign
I was wondering we could remove "openstack" provider specific tests and conditions from e2e tests, but there are storage tests which are related to OpenStack in e2e tests. So it is nice to add registration for OpenStack at this time.


Question: } func Test_v1_NodeSpec_to_core_NodeSpec(t *testing.T) { <del> failInputs := []v1.NodeSpec{ { // fail PodCIDRs[0] != PodCIDR PodCIDR:  "10.0.0.0/24", PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, </del> <ins> asymmetricInputs := []struct { name string in   v1.NodeSpec out  core.NodeSpec }{ { name: "mismatched podCIDR", in: v1.NodeSpec{ PodCIDR:  "10.0.0.0/24", PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, }, out: core.NodeSpec{ PodCIDRs: []string{"10.0.0.0/24"}, }, }, { name: "unset podCIDR", in: v1.NodeSpec{ PodCIDR:  "", PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, }, out: core.NodeSpec{ PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, }, }, { name: "matching podCIDR", in: v1.NodeSpec{ PodCIDR:  "10.0.1.0/24", PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, }, out: core.NodeSpec{ PodCIDRs: []string{"10.0.1.0/24", "ace:cab:deca::/8"}, }, </ins> }, }

What happened: The cluster master in a Kubernetes v1.14.10 cluster was upgraded to V1.16.7. The process of upgrading nodes is under way and has been spread over a week so far. The cluster owner reported seeing a high percentage of pod status PATCH failures. The apiserver logs show errors like: The errors are all from the v1.14 nodes and have all been for pods associated with DaemonSets. We don't yet know why pods' IPs are changing, but it appears that pod status patches that include a changed result in an object where contains the old IP, and has the new IP. We believe this is because was introduced in Kubernetes v1.15 and the patch request from a v1.14 node results in both and being present in the result. If you aren't monitoring for request failures it may not obvious that any error is occurring: The pods continue to show their last successfully reported state. Deleting such pods, however, results in the pod being stuck in a state. We are only seeing this for DaemonSet pods - I suspect because whatever resulted in pod IPs changing also causes non-DaemonSet pods to be deleted and recreated. We have a number of cluster still on v1.14 that we expect to upgrade to v1.15 or v1.16. What you expected to happen: Kubernetes v1.14 nodes should be able to update pod status during the transition from v1.14 to v1.16 even if pod IPs change during that time. How to reproduce it (as minimally and precisely as possible): We have recreated this in the IBM Cloud Kubernetes Service by "reloading" nodes. The reload process leaves the Kubernetes node object in place, and essentially reloads the image with the proper kubeconfig to rejoin the cluster. The cluster I was debugging does not appear to have had nodes "reloaded" - something else is triggering the pod IP changes. Create a v1.14 cluster Deploy a DaemonSet (anything with pod networking should suffice) Upgrade cluster master to v1.16 Reload a v1.14 node. One the node is back up, shows the pod's network IP has changed. Kubelet and apiserver logs show the errors describe earlier. Anything else we need to know?

: Environment: Kubernetes version (use ): v1.14.x (nodes) and v1.16.7 (master) Cloud provider or hardware configuration: IBM Cloud Kubernetes Service OS (e.g: ): 18.04.4 LTS Kernel (e.g. ): 4.15.0-76-generic Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig api-machinery
/sig network
cc
/priority critical-urgent if it's possible for podIPs to change and the server defaults to , then the API server has to treat the singular pod IP as authoritative if there's a mismatch or the patch case breaks
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-sig api-machinery


Question: Log(ep, "Endpoint Cleanup", 3) if ep.refCount != nil { *ep.refCount-- <del> } // Remove the remote hns endpoint, if no service is referring it // Never delete a Local Endpoint. Local Endpoints are already created by other entities. // Remove only remote endpoints created by this service if (ep.refCount == nil || *ep.refCount <= 0) && !ep.GetIsLocal() { klog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) err := ep.hns.deleteEndpoint(ep.hnsID) if err == nil { ep.hnsID = "" } else { klog.Errorf("Endpoint deletion failed for %v: %v", ep.IP(), err) </del> <ins> // Remove the remote hns endpoint, if no service is referring it // Never delete a Local Endpoint. Local Endpoints are already created by other entities. // Remove only remote endpoints created by this service if *ep.refCount <= 0 && !ep.GetIsLocal() { klog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) err := ep.hns.deleteEndpoint(ep.hnsID) if err == nil { ep.hnsID = "" } else { klog.Errorf("Endpoint deletion failed for %v: %v", ep.IP(), err) } </ins> } <ins> ep.refCount = nil </ins> } }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Two services referring the same endpoint. One of the services was deleted. This brought down the connectivity of the other service. What you expected to happen: The connectivity to service should continue to work How to reproduce it (as minimally and precisely as possible): two services referencing same endpoint one of the services connectivity is broken to the other service Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig network These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry.
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
Do you mean a service referencing the same pod, or literally the same Endpoints object?
I meant service referencing the same pod.
/assign
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
Is this still an issue? If so, it would be helpful to get an answer to Jordan's question above. If this Service is referencing the same Endpoints object that would be a very different issue than if it was referencing the same Pod. /triage needs-information
This issue happens when multiple Services are referencing a same POD. I mean the same backend IP address but different ports. i.e. the POD is hosting two workloads on different ports.
/sig windows
Is this only observed in windows environments?
Yes this happens only on Windows environments
Thanks can you provide more information about the version of Kubernetes you're using? Do you know what proxier mode kube-proxy is running with? Maybe winkernel? I'm not as familiar with the Windows implementation here, adding to help debug this.


Question: # startup, otherwise containers won't land in flannel's network... Requires=docker-cache.service flanneld.service After=docker-cache.service flanneld.service <del> [Service] Environment=DOCKER_OPTS='--registry-mirror=http://$private_ipv4:5000' </del> - name: kube-apiserver.service command: start content: | [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes <del> Requires=etcd2.service After=etcd2.service </del> <ins> Requires=etcd2.service setup-network-environment.service After=etcd2.service setup-network-environment.service </ins> [Service] <ins> EnvironmentFile=/etc/network-environment </ins> ExecStartPre=-/usr/bin/mkdir -p /opt/bin ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-apiserver ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver

Minor issues with 1) The examples rely on cloud vendor specific variables ($private_ipv4 ) 2) The wget command for kube-register always re-downloads


Question: SystemUUID string `json:"systemUUID"` // BootID is the boot-id reported by the node BootID string `json:"bootID"` <ins> // Kernel version reported by the node KernelVersion string `json:"kernelVersion""` // OS image used reported by the node OsImage string `json:"osImage"` // Container runtime version reported by the node ContainerRuntimeVersion string `json:"containerRuntimeVersion"` // Kubelet version reported by the node KubeletVersion string `json:"kubeletVersion"` // Kube-proxy version reported by the node KubeProxyVersion string `json:"kubeProxyVersion"` </ins> } // NodeStatus is information about the current status of a node.

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: According to the two changes are required to enable it: Add to apiserver and scheduler. Add to apiserver. So I modified the apiserver manifest: And the scheduler manifest: confirms that both are running with the required changes. However, I cannot create PodPriorities by following the example in the documentation: also doesn't think the API is enabled: The only relevant message in the apiserver log is: What you expected to happen: I expected to be able to create podpriority resources by following the example in the documentation. How to reproduce it (as minimally and precisely as possible): See description. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration**: GCE OS (e.g. from /etc/os-release): CentOS Linux 7.4 (1708) Kernel (e.g. ): Linux master-head-1 4.13.4- SMP Wed Sep 27 13:32:23 EDT 2017 x8664 x8664 x8664 GNU/Linux Install tools: Custom scripts Others:
/sig scheduling
The example yaml of priorityclass is wrong, the right version is
please try the recommendation from and close this issue?
unfortunately that version change to the YAML does not fix the problem for me either: To me it looks like the error message points me to where I don't see defined?
Yes, you are right, I have commit a pr to fix this.
Not sure but after using the updated config I am able to create Priority Class successfully but unable to use the feature while creating POD. Here are the errors observed Getting following error while creating pod: The PodPriority is enabled in both API server and Scheduler
I'm able to create PriorityClass and schedule pod with specified PriorityClass on Minikube 0.26.0. ` !
This issue was sorted - on kubeadm production setup, this flag needs to be enabled for kubelet as well - updated doc
I've created the PriorityClass and deployed the pod with the specified PriorityClass. The pod is getting created. I'm wondering is there any way to check the PriorityClass of the pod either in UI or CLI? The steps you've shown is giving me the same result. Can you please help me ? Thanks in advance.


Question: return false } <ins> func (tc *testCase) workloadNamesUnique() error { workloadUniqueNames := map[string]bool{} for _, w := range tc.Workloads { if workloadUniqueNames[w.Name] { return fmt.Errorf("%s: workload name %s is not unique", tc.Name, w.Name) } workloadUniqueNames[w.Name] = true } return nil } </ins> // workload is a subtest under a testCase that tests the scheduler performance // for a certain ordering of ops. The set of nodes created and pods scheduled // in a workload may be heterogeneous.

Note: this issue should be viewed in the context of . What would you like to be : When parsing the performance- file, make sure each testCase and workload has a unique name. This can also be implemented by changing the yaml file's structure into maps instead of arrays (as suggested by but this will result in a loss of test/workload order (i.e. a workload may run before a workload). Why is this needed: The name of each testCase and workload is used in the report generated by the framework and should therefore be unique in order to identify each metric. /sig scheduling /sig testing /assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
/unassign /good-first-issue
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /unassign /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hi this is my first time contributing to kubernetes. can you provide me some guidance on how to to get started?
I recommend starting with the and reading from there.
Hi are you suggesting that name of workloads should be different for each test case . eg:-
kinda, but I'm more interested in making sure that each test has a unique name identifier so that I can run it with
Are you working on this?

you can go ahead if you want
I would like to work on this issue if someone has not started already
if no one is currently working on this, I'd like to take a crack at it.
/assign
Of course! Could you explain what you did so noobs like me could learn?
Sure! For the record I am a noob to this project as well, but my process for trying to solve issues on open source projects generally goes like this: Read the issue and figure out what files are going to be relevant. In this case I know the relevant file is going to be in test/integration/schedulerperf because of the issue title and I know a performance- file is going to be important as it's mentioned in the issue. Look at those files and figure out what they do. In this case I'm really only interested in the bits where the performance- is parsed and validated. I Ctrl-F'ed to see if it was reading that performance- file and figured out that it was being checked in the validateTestCases function. Add the changes the issue is asking about. In this case, the issue wants to check that the names of test cases and workloads from the yaml file are unique. So I used a map to track which names had already come up and returned an error if they weren't unique. Hopefully this is helpful. I don't know if my PR will be accepted or not but I find it's usually best to just try and implement the changes needed with the limited information you have and let the reviewers tell you if you're on track or not.


Question: t.Fatalf("cannot assert plugin to be type csiPlugin") } <del> // Wait until the informer in CSI volume plugin has all CSIDrivers. wait.PollImmediate(TestInformerSyncPeriod, TestInformerSyncTimeout, func() (bool, error) { return csiDriverInformer.Informer().HasSynced(), nil }) if volumeAttachmentInformer != nil { wait.PollImmediate(TestInformerSyncPeriod, TestInformerSyncTimeout, func() (bool, error) { return volumeAttachmentInformer.Informer().HasSynced(), nil }) } </del> return csiPlug, fakeWatcher, tmpDir, fakeClient }

Which test(s) are flaking: TestAttacherWithCSIDriver Reason for failure: This is flaking rarely enough that it is not caught by our CI jobs, which currently tolerate up to 2 unit test failures per run (!). With that toleration removed in , this flake has been seen (). Reproducible with the following steps: a test binary for the affected package: the affected test using the tool (setting the timeout to a generous timeout relative to the normal test runtime of .02 seconds): failures are seen, look at the logs to see the details about the error: /sig storage cc
/assign can you help investigate this flake?
I have found the root cause. /assign
filed to address this issue.


Question: sleep 5 ALL_RUNNING=1 for id in $POD_ID_LIST; do <del> CURRENT_STATUS=$(remove-quotes $($CLOUDCFG -json get "pods/$(remove-quotes ${id})" | jq '.currentState.info["mynginx"].State.Running and .currentState.info["net"].State.Running')) </del> <ins> CURRENT_STATUS=$($CLOUDCFG -template '{{and .CurrentState.Info.mynginx.State.Running .CurrentState.Info.net.State.Running}}' get pods/$id) </ins> if [ "$CURRENT_STATUS" != "true" ]; then ALL_RUNNING=0 fi

We have an e2e test that runs the guestbook example. We should also have one that runs through the other example.


Question: import ( "context" <ins> "crypto/subtle" </ins> "encoding/csv" "fmt" "io"

This issue was reported in the Description The kube-apiserver includes multiple authentication backends for client request processing. These range in strength from client certificate authentication to simple HTTP Basic Authentication. When using a password (Basic Authentication), the kube-apiserver does not perform a secure comparison of secret values. In theory, this could allow an attacker to perform a timing attack on the comparison. For example, the AuthenticatePassword function simply performs string comparison to authenticate a user’s password (Figure 9.1). Figure 9.1: Username and password authentication handling in Exploit Scenario Alice runs a Kubernetes cluster in production. In order to support multiple organizational “customers,” she configures the cluster with HTTP Basic Authentication. Eve has a large number of username and password pairs for the organization, and uses the side-channel information from string comparison to tune her credential-stuffing attacks. Recommendation Short term, use a safe, constant-time comparison function such as . The comparison function should take the same amount of time regardless of matching prefix data within the password. Long term, deprecate Basic Authentication in favor of more robust and secure options. Add documentation noting that any Basic Authentication is for use only in development scenarios, and not appropriate for production deployments. This will help users create a robust and secure default stance for all deployments. Anything else we need to know?: See for current status of all issues created from these findings. The vendor gave this issue an ID of ATR-K8S-002 and it was finding 18 of the report. The vendor considers this issue Medium Severity. To view the original finding, begin on page 55 of the Environment: Kubernetes version: 1.13.4 Fixed in v1.16.0 by
/sig auth
/area security /kind bug /remove-kind feature


Question: k8sNodeRouteTag = "k8s-node-route" // AffinityTypeNone - no session affinity. <del> gceAffinityTypeNone = "None" </del> <ins> gceAffinityTypeNone = "NONE" </ins> // AffinityTypeClientIP - affinity based on Client IP. gceAffinityTypeClientIP = "CLIENT_IP" // AffinityTypeClientIPProto - affinity based on Client IP and port.

Hi, I have a problem with my GCE cluster about LoadBalancer type Services It's creating a lot of unused static external IP on the google network of my project the cluster is running kubernetes v1.1.2. Tried restarting nodes, tried destroy/recreate of the entire cluster.
Seeing exactly the same issue at GKE since a few hours. First on a v1.0.7 cluster which I later upgraded to v1.1.2 to fix the problem but the issue is still there. Double checked my yaml config files but no changes from our side. Existing loadbalancers do work as before.
I dealt with this last night and a workaround was to add to the spec
Thx :+1:
We'll get this cherry picked into the two release branches once it's been verified at HEAD.


Question: ss = sst.WaitForStatus(ss) By("getting scale subresource") <del> scale := framework.NewStatefulSetScale(ss) scaleResult := &appsv1beta2.Scale{} err = c.AppsV1().RESTClient().Get().AbsPath("/apis/apps/v1").Namespace(ns).Resource("statefulsets").Name(ssName).SubResource("scale").Do().Into(scale) </del> <ins> scale, err := c.AppsV1().StatefulSets(ns).GetScale(ssName, metav1.GetOptions{}) </ins> if err != nil { framework.Failf("Failed to get scale subresource: %v", err) }

What would you like to be : e2e test suites for: GET /apis/apps/v1/namespaces/{namespace}/statefulsets/{name}/scale PUT /apis/apps/v1/namespaces/{namespace}/statefulsets/{name}/scale Why is this needed: PR has already been merged, but the following tests are still commented out. Therefore, scale sub-resource of statefulsets is not tested and test should be . /sig testing


Question: return service.Spec.LoadBalancerIP, nil } <del> lbStatus, existsLb, err := az.GetLoadBalancer(ctx, clusterName, service) </del> <ins> _, lbStatus, existsLb, err := az.getServiceLoadBalancer(service, clusterName, nil, false) </ins> if err != nil { return "", err }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: The issue has been observed on Azure but should also apply to other providers as well. On Azure we observed that there are orphaned Public IPs that don't belong to any Kubernetes LoadBalancer service but from looking at the tags and the name it is clear that Kubernetes created them. In the Azure Activity Log, you can see that the Public IP was deleted when the K8s service was deleted but recreated a few minutes later. That re-creation is also visible in the K8s controller manager logs as a reconcile of the service. In our case, we add additional worker nodes and delete services at the same time. I think this leads to the re-creation here Because the service list is retrieved at the beginning of the reconcile loop, services that are deleted while the loop is reconciling a recreated by that loop and afterwards orphaned. What you expected to happen: No orphaned public IPs How to reproduce it (as minimally and precisely as possible): Create a few Kubernetes Load Balancer services (we have around 20). Then scale your worker nodes and delete one service at the same time. Anything else we need to know?: Environment: Kubernetes version (use ): 1.15.0 Cloud provider or hardware configuration: Azure OS (e.g: ): Core OS Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig azure
/sig cloud-provider
Thanks for reporting the issue. I think this should be fixed during v1.16 release cycle. /milestone v1.16
In the nodeSyncLoop -updateLoadBalancerHosts -lockedUpdateLoadBalancerHosts call chain, I don't see where the mutex lock is obtained.
Hello! I'm the bug triage lead for the 1.16 release cycle and considering this issue is tagged for 1.16, I'd like to check its status. The code freeze is starting on August 29th EOD PST (about 4 days from now) and once the code freeze starts, only release-blocking changes will be considered to be merged.

Is this issue still planned to be fixed for the 1.16 release?
The issue would probably not able to be fixed in v1.16. AFAIK, no one is working on it now.
Thank you for the update! I'll go ahead and remove the issue from the milestone. If anything changes in the meanwhile, please let us know. /milestone clear
/assign
/assign
GitHub didn't allow me to assign the following users: chewong. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Doesn't LB Finalizer support fix this? We enabled this by default starting v1.16
Initial PR: Bump to beta in v1.16:


Question: pp.timeoutPermit = false pp.waitAndRejectPermit = false pp.waitAndAllowPermit = false <del> pp.allowPermit = false </del> pp.cancelled = false <ins> pp.waitingPod = "" pp.allowingPod = "" pp.rejectingPod = "" </ins> } // newPermitPlugin returns a factory for permit plugin with specified PermitPlugin.

Which jobs are flaking: pull-kubernetes-integration Which test(s) are flaking: TestCoSchedulingWithPermitPlugin Testgrid link: Reason for failure: /sig scheduling
Could be related
/help
This request has been marked as needing help from a contributor. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /help Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Another run
Hmmm I think test plugin is no longer correct: After my change is executed at the end of the scheduling cycle, this means there can be only one happening at any given moment. If there is a race where "signaling-pod" is scheduled before "waiting-pod", then "signaling-pod" will wait 30s in in for the "waiting-pod" to become waiting, but this cannot happen. Basically "signaling-pod" is allowing "waiting-pod" before "waiting-pod" is being scheduled. I think test and test plugin could be changed, so there are no longer two pods of different kind. Instead first pod that arrives to the is considered "waiting-pod" and second pod is always "signaling-pod" that will either reject or allow first pod. This way we do not have to care about order in which scheduler will process these pods. I think it is also consistent with how actual co-scheduling will work.
Can you first confirm that this is what is happening? you could increase the log level to 3 to see the log lines in and . I feel like if that's the case, we have a separate bug with the event handlers or sorting. Other than that, I agree that the implementation of the test's shouldn't have waits. cc
My hope would be that the pod is temporarily unschedulable due to nodes still being synced.
Where you able to confirm the reason of the flake?
not yet, what is the easiest way to run this test multiple times? Should I try to reproduce this locally or create some temporary PR just for this purpose?

I wrote PR for the test fix to remove sleep from the PermitPlugin, it had to be done sooner than later regardless of the cause of flakiness.
Understanding the reason of the flake is important to make sure we are fixing the right thing and that we won't have future flakes. Please try to run the tests locally about 10 times.
It should be something like: If that doesn't work, run first
Flake point:
From the logs provided by you can tell that the problem is a race between "signalling-pod" and "waiting-pod" and waiting inside the Permit(), as I mentioned in this issue. BLUF: In the first case of the TestCoSchedulingWithPermitPlugin "waiting-pod" and "signalling-pod" pods are being retried many times, most of the time "signalling-pod" is rejecting already waiting "waiting-pod", but sometimes race can happen and "signalling-pod" will be processed by the scheduler before rejected "waiting-pod" (there is no guarantees whether rejecting pod or rejected pod will end up in the queue first). When this happens "signalling-pod" waits 30s inside the Permit() for "waiting-pod" to become waiting and blocks the scheduler, meanwhile second test case starts, "signalling-pod" eventually stops waiting for "waiting-pod" and gets bound, after that "waiting-pod" cannot be allowed, which leads to a failure of the second test case. My PR should fix this issue.
I feel like we should avoid sharing the scheduler & permitPlugin instances among the 2 subtests, which can then solve the flake?
I think this flake (in different variations) can happen as long as test depends on the order in which signalling and waiting pod enter the permit plugin.
I believe the PR might fix the issue. But since the reason for the flake is that one subtest is corrupting the other, it's possible that we might surface other flakes. That's why it's important that you run the test several times to check for other flakes. It several tests pass, let's proceed with this PR and leave a TODO to remove the dependency between the subtests.
I ran 10+ times on my branch with the fix and had no issues.


Question: } By("creating the pod with failed condition") <del> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) Expect(err).ToNot(HaveOccurred(), "while creating pod") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) Expect(err).To(HaveOccurred(), "while waiting for pod to be running") </del> var podClient *framework.PodClient podClient = f.PodClient() <ins> pod = podClient.Create(pod) err := framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) Expect(err).To(HaveOccurred(), "while waiting for pod to be running") </ins> By("updating the pod") podClient.Update(podName, func(pod *v1.Pod) {

Which jobs are failing: All new tests introduced for api change Which test(s) are failing: New alpha subpathExpr tests [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion] Existing tests are working fine and the whole suite works correctly in the gce suite https://k8s- Since when has it been failing: 21st February 2019 (since merge of PR) Testgrid link: https://k8s- Reason for failure: Investigation shows that node e2e tests use a different framework pod creation pattern Doing some comparisons of tests that work and those that don't reveals differences in the standard client set as and the existing ones which do work go through the intermediate which uses The version has an additional function specifically for node tests
/sig storage
/sig node /sig test
/assign
/cc
/sig test


Question: } // Columns: Name, ParentRef, Age expected := []metav1.TableRow{ <del> {Cells: []interface{}{"192.168.2.2", "myresource.mygroup/mynamespace/myname", "10y"}}, {Cells: []interface{}{"2001:db8::2", "myresource2.mygroup2/mynamespace2/myname2", "5y1d"}}, </del> <ins> {Cells: []interface{}{"192.168.2.2", "myresource.mygroup/mynamespace/myname", "<unknown>"}}, {Cells: []interface{}{"2001:db8::2", "myresource2.mygroup2/mynamespace2/myname2", "<unknown>"}}, </ins> } rows, err := printIPAddressList(&ipList, printers.GenerateOptions{})

and also v1.27-1.29 TestPrintIPAddressList After 03-01 the leap day caused the failure /sig cli network
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: API_PORT=${API_PORT:-8080} API_HOST=${API_HOST:-127.0.0.1} API_HOST_IP=${API_HOST_IP:-${API_HOST}} <del> API_BIND_ADDR=${API_HOST_IP:-"0.0.0.0"} </del> <ins> API_BIND_ADDR=${API_BIND_ADDR:-"0.0.0.0"} </ins> KUBELET_HOST=${KUBELET_HOST:-"127.0.0.1"} # By default only allow CORS for requests on localhost API_CORS_ALLOWED_ORIGINS=${API_CORS_ALLOWED_ORIGINS:-"/127.0.0.1(:[0-9]+)?$,/localhost(:[0-9]+)?$"}

The script doesn't support a user-value for the Kube-apiserver option. This is the IPs on which the Kube API will listen for secure API requests. The local-cluster-up script is set in such a way that it will only bind to the same IP as the one defined in : The default value of will never be used as is always set on the line before. This means that today getting to the secure Kube API through the kube service IP doesn't work. (By default the secure API only binds to on the host). Proposal is to have a default of for secure API binding, or whatever value is provided by the user.


Question: // GetActivePods returns non-terminal pods func (kl *Kubelet) GetActivePods() []*v1.Pod { <del> allPods := kl.podManager.GetPods() </del> <ins> allPods, mirrorPods := kl.podManager.GetPodsAndMirrorPods() mirrorPodSet := make(map[string]*v1.Pod) for _, p := range mirrorPods { mirrorPodSet[kubecontainer.GetPodFullName(p)] = p } for i := range allPods { podFullName := kubecontainer.GetPodFullName(allPods[i]) // replace static pod with mirror pod as some info (e.g. spec.Priority) // is needed to make further decisions (e.g. eviction) if mirrorPod, ok := mirrorPodSet[podFullName]; ok { allPods[i] = mirrorPod } } </ins> activePods := kl.filterOutTerminatedPods(allPods) return activePods }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Eviction manager evicts the below static and critical pod when the node observed MemoryPressure. The k8s server version is v1.13.2, and the pod priority is enabled. As far as I can see, the eviction manager does not evict such pods. What you expected to happen: Static pods with are not deleted as well as those with a critical pod annotation. How to reproduce it (as minimally and precisely as possible): pod with system-node-critical priority using by kubelet option memory load to the target node until eviction manager observes memory pressure. critical and static pod is evicted. Anything else we need to know?: When I set critical-pod annotation (and enables ) insted of pod priority, the eviction manager DOES NOT evict the static and critical pod. Environment: Kubernetes version (use ): v1.13.2 Cloud provider or hardware configuration: baremetal
/sig node
from enable priority is , more than I can not understand why it has happened
Right. It seems the expected behavior and the actual behavior are different. will remove next k8s v1.14 (it's written in the document), but I'd like to wait for it until the critical priority can prevent eviction.
I see this as well. Start from the bottom and read up. It ranks ibm-master-proxy-static which is a static pod that has priority system-node-critical. It actually is always ranked first as well.
It can be replicated by causing disk pressure (using ) And then viewing the eviction manager logs. `journalctl -u kubelet | grep eviction There's also the piece that when the static pod is evicted kubelet never ends up starting it back up again.
Enabling ExperimentalCriticalPodAnnotation=true makes it not fail and detect properly
Setting on the static pod also works. It appears that a static pod doesn't get set based on . If this is expected then it should be documented. Although it seems like the priority class name should be honored for static pods.
I've confirmed that the problem occurs when a static pod has set to but does not set accordingly.

doesn't see the priority as being set and thus thinks the pod isn't critical which ultimately causes the eviction. I would have expected to translate to .
The code fix is available in master branch right now, which means it will be auto included in the upcoming 1.14 release. For lower releases, you need to wait for a while before cherrypick PRs get approved and merged.
Cherrypick PRs have been created and waiting for approval. 1.13 - 1.12 - 1.11 -
Your patch does not work for me. The case in that you completely replaces static pods with the mirror pods. But static pods contains important annotations such as . For static pods is , for mirror - . As a result, eviction manager supposes that pod is not static.
I doubt it. A mirror pod should be annotated with : Otherwise the PR won't pass the e2e test.
When I get static pod through I see . But when I try to debug eviction manager, I see that pod has annotation
Thanks for the info. Then we are sort of in a dilemma - on one hand annotation is needed to check if it's a static pod, on the other hand we also need the corresponding mirror pod to get the priority value. I will provide a supplemental fix later.
can we please hold on those cherry-picks until the supplemental fix is also available?
sure.
Here is a detailed explanation on why previous PR passed the e2e test: As , the critical static pod failed on IsStaticPod() check (because of mirror pod is tagged as " api") And it's reasonable that it's being marked as "to be deleted": However, as , there is some weird log showing: It's because the pod we passed onto () is a mirror pod, and it's deleted (or probably a dummy delete) by container runtime without any error. To take an analogy, it's sort of like a "controllee" (mirror pod) gets deleted, but "controller" (static pod) is still alive, so it causes data inconsistency, and the above error msg popped up. And in user's perspective, the static pod is "not" evicted, then it "looks" like previous PR works.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available .

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Status of cherry pick PRs: 1.13 - - merged 1.12 - - merged 1.11 - - merged
/close As all PRs have been cherrypicked.
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
When will this be released for 1.11.x? We are getting this same issue with and believe it's related: The api log:
Maybe in v1.11.10 (but I didn't see it in v1.11.10-beta.0 yet)
Thanks, I assume this is pretty high priority if it's causing to be evicted?
Yes. Prior to get it in 1.11.x, you can work it around by manually setting the priority value (not only setting priority name) to the control plane pods (apiserver, controller-manager and scheduler).
I think the priority value is correct:
Did you get that from ? If so, that's not sufficient. You should work around by hardcoding it into your static pod yaml file.
I did get it from , will try hardcoding. Thanks!
Were you successful ? I'm running into this same problem (unfortunately on a 1.10 cluster, which doesn't have this patch) and received the error: Looks like it comes from
I see this as well but the actual disk is not even full.


Question: } // CheckAffinity function tests whether the service affinity works as expected. <del> // If affinity is expected and transitionState is true, the test will // return true once affinityConfirmCount number of same response observed in a // row. If affinity is not expected, the test will keep observe until different // responses observed. The function will return false only when no expected // responses observed before timeout. If transitionState is false, the test will // fail once different host is given if shouldHold is true. func CheckAffinity(jig *ServiceTestJig, execPod *v1.Pod, targetIp string, targetPort int, shouldHold, transitionState bool) bool { </del> <ins> // If affinity is expected, the test will return true once affinityConfirmCount // number of same response observed in a row. If affinity is not expected, the // test will keep observe until different responses observed. The function will // return false only in case of unexpected errors. func CheckAffinity(jig *ServiceTestJig, execPod *v1.Pod, targetIp string, targetPort int, shouldHold bool) bool { </ins> targetIpPort := net.JoinHostPort(targetIp, strconv.Itoa(targetPort)) cmd := fmt.Sprintf(`wget -qO- http://%s/ -T 2`, targetIpPort) timeout := ServiceTestTimeout

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: gce-cos-master-slow Which test(s) are failing: [sig-network] Services should have session affinity work for LoadBalancer service with ESIPP on [Slow] [DisabledForLargeClusters] Since when has it been failing: This test started to be flaky on 11/18 Testgrid link: https://k8s- Reason for failure: Looks to be a pretty straightfowards test not passing: If we look at triage, this test has always been flaky, but it got significantly more flaky (2X failures) on 10/27: Anything else we need to know: This job is non-blocking for 1.13, except if this represents an actual bug in Affinity the 1.13 team needs to know ASAP. This failure may or may not be related to . It looks like this test was disabled for large clusters, but maybe the problem wasn't the cluster size. <!-- DO NOT EDIT BELOW THIS LINE --/kind flake /sig network /priority important-soon attn:
FWIW this seems to be a long standing issue, atleast as far bck as 1.12 timeframe
Sure. But the failures have become 3X as frequent, so I still want a sign-off by sig-network.
cc
Ack. Will work on this.
we plan to lift code freeze for 1.13 tomorrow,11/28 5pm PST. If there is a new regression in 1.13 thats increased the flakes and needs addressing we need to get the fix in asap. Can you please let us know if this is a blocker for 1.13 by tomorrow, 11/28, morning 10am PST ?
Great observation. I checked on the commits around that date but wasn't able to find any change that may cause regression. Hasn't been able to reproduce the same failure locally as well.
I just noticed that the same test is flaking a lot more on 1.12 branches as well https://k8s-
Good catch on that, so the recent increase of flakiness likely is not caused by 1.13 changes so we shouldn't block release on this either. cc
But we will be looking deeper into the root cause.
Several observations: It only happens when is false.

When it's true, the test does more stuff before actually check service session affinity. It doesn't happen when executing in pod, probably due to the same reason above, the test start test pod before actually testing service. I wrote a script which does the same thing in my cluster of version 1.11.3-gke.18 in us-central1-a. cannot reproduce it. The hypothesis is that the service session affinity spec takes some time to reconcile after the service is created. It could happen either on the node or on the GCLB. Since I haven't been able to reproduce it's hard to tell. But it's more likely on GCLB side, even it could be regional issue. Looks like a sleep of seconds will deflake it. but it's just hiding issues. I would rather open another issue to track the real issue.
The most recent failure is here: Basically what happened is GCLB doesn't honor session affinity and still sends the request to different VMs. The log shows that 3 requests are routed to 3 different VMs. Dec 19 07:40:58.094: INFO: [service-fsn65 service-bf8nc] Dec 19 07:40:58.094: INFO: Affinity should hold but didn't. Dec 19 07:40:58.196: INFO: [pod,node] pairs: [{Pod:service-bf8nc Node:bootstrap-e2e-minion-group-f3nb} {Pod:service-fsn65 Node:bootstrap-e2e-minion-group-9sp7} {Pod:service-rnrcw Node:bootstrap-e2e-minion-group-m5rj}]; err: <nil
More recent failures: Logs:
Hello, could this issue be prioritized to help with stabilizing the release blocking upgrade tests? They're currently flaking ~23% of the time in () ~30% of the time in ()
/assign can you take a look
GitHub didn't allow me to assign the following users: take, a, look, can, you. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time.

For more information please see <detailsIn response to : /assign can you take a look Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I am preparing a PR, basically allow Affinity some time to be configured, instead of failing out instantly.
Thanks! Will keep an eye out for it.
The test is no longer flaky: https://k8s- Thanks everyone!
Thanks! Will keep an eye on it for a few days, on the following dashboards:
It's stabler now, but still flaking once/twice a day, e.g. in . - Is this the same issue as you looked at? Is there more to be done here? /reopen
Reopened this issue. <detailsIn response to : It's stabler now, but still flaking once/twice a day, e.g. in . Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
we should probably cherrypick into 1.13 in case of deflaking upgrade/downgrade job, because those jobs would run 1.13 tests against 1.14 (upgraded) cluster or 1.13 (downgraded) cluster. FYI we've hit similar situation for DNS test (ref ).
Good point. Cherrypick PR prepared.
14: <img width="1110" alt="Screen Shot 2019-03-16 at 12 11 07 PM" src="https://user-"13: <img width="1091" alt="Screen Shot 2019-03-16 at 12 11 22 PM" src="https://user-"Do we want to cherry-pick this back further?
Thanks for the nice graph. Look like the fix works. I'll create cherrypick PRs for 1.11 and 1.12 right away.
AFAICT this has been significantly improved: ! Closing this and thanks to /close
Closing this issue. <detailsIn response to : AFAICT this has been significantly improved: ! /close Instructions for interacting with me using PR comments are available .


Question: l.migrationCheck.validateMigrationVolumeOpCounts() } <del> ginkgo.It("multiple pods should access different volumes repeatedly [Slow] [Serial]", func() { l := init() defer func() { cleanup(l) }() for i := 0; i < l.testOptions.NumPods; i++ { framework.Logf("Creating resources for pod %v/%v", i, l.testOptions.NumPods-1) r := CreateVolumeResource(driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange) l.resources = append(l.resources, r) podConfig := e2epod.Config{ NS:           f.Namespace.Name, PVCs:         []*v1.PersistentVolumeClaim{r.Pvc}, SeLinuxLabel: e2epv.SELinuxLabel, } pod, err := e2epod.MakeSecPod(&podConfig) framework.ExpectNoError(err) </del> <ins> ginkgo.BeforeEach(func() { init() createPodsAndVolumes() }) </ins> <del> l.pods = append(l.pods, pod) } </del> <ins> f.AddAfterEach("cleanup", func(f *framework.Framework, failed bool) { cleanup() }) ginkgo.It("multiple pods should access different volumes repeatedly [Slow] [Serial]", func() { </ins> // Restart pod repeatedly for i := 0; i < l.testOptions.NumPods; i++ { podIndex := i

/sig storage What happened: When you interrupt the e2e storage stress test, namespaced objects are cleaned up, but non-namespaced objects like StorageClasses are not cleaned up. What you expected to happen: Non-namespaced objects like StorageClasses are cleaned up so long as the user doesn't interrupt a second time. How to reproduce it (as minimally and precisely as possible):
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: if err != nil { t.Errorf("Test %q failed: GetPV %q returned error: %v", name, pv.Name, err) } <del> if !reflect.DeepEqual(cachedPV, pv) { </del> <ins> // Cache may be overridden by API object with higher version, compare but ignore resource version. newCachedPV := cachedPV.DeepCopy() newCachedPV.ResourceVersion = pv.ResourceVersion if !reflect.DeepEqual(newCachedPV, pv) { </ins> t.Errorf("Test %q failed: cached PV check failed [A-expected, B-got]:n%s", name, diff.ObjectDiff(pv, cachedPV)) } }

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: pull-kubernetes-bazel-test Which test(s) are failing: Since when has it been failing: Testgrid link: Reason for failure: Anything else we need to know:
the test failure seems related to the PR that is being tested no other flakes have been observed:
I've run it against latest master locally and it may fail. Not sure why it has low chance to fail in prow job.
- i can recreate it with
/sig testing


Question: } func (g *GCERunner) machineType(machine string) string { <ins> var ret string </ins> if machine == "" && *instanceType != "" { <del> machine = *instanceType </del> <ins> ret = *instanceType } else if machine != "" { ret = machine </ins> } else { <del> machine = defaultGCEMachine </del> <ins> ret = defaultGCEMachine </ins> } <del> return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine) </del> <ins> return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, ret) </ins> } func (g *GCERunner) rebootInstance(instance *compute.Instance) error { // wait until the instance will not response to SSH

seems like an infra issue. No tests run 7/21 No response No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
/cc
ssh connection timeouts: Which weird given that it ran on the default network and there is a firewall rule in there that allows ssh from anywhere.
Got a fix, another regression from I really need to get these kubetest2 jobs ran as presubmits by default on k/k ASAP.


Question: "k8s.io/kubernetes/test/integration/framework" ) <ins> const ( resourceQuotaTimeout = 10 * time.Second ) </ins> // 1.2 code gets: // 	quota_test.go:95: Took 4.218619579s to scale up without quota // 	quota_test.go:199: unexpected error: timed out waiting for the condition, ended with 342 pods (1 minute)

<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --integration test jobs TestQuotaLimitService which was recently in One example run: I suspect this is because TestQuotaLimitService runs through multiple test cases where each test case relies on state created from the previous case. However, resource quota changes take some time to get propagated on the system and the test is trying to validate acceptance of new Services before the resource quota controller applies the necessary changes. links to appreciated links to specific failures in spyglass appreciated <!-- Please see the deflaking doc () for more guidance! --
cc
/sig network /sig apimachinery
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /sig network /sig apimachinery Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
Opened a fix here
/triage accepted


Question: return fmt.Sprintf("%s%s%s", scaleSetName, vmssNameSeparator, instanceID) } <del> func (ss *scaleSet) extractVmssVMName(name string) (string, string, error) { </del> <ins> func extractVmssVMName(name string) (string, string, error) { </ins> ret := strings.Split(name, vmssNameSeparator) if len(ret) != 2 { glog.Errorf("Failed to extract vmssVMName %q", name)

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: When useInstanceMetadata is set to true for vmss nodes, their providerID is wrong. What you expected to happen: Their providerID should be in format azure:///subscriptions/<subscription-id/resourceGroups/<rg-name/providers/Microsoft.Compute/virtualMachineScaleSets/<scaleset-name/virtualMachines/<instance-id How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/sig azure
adding priority important-soon so that this issue can stay in 1.10. Please update if incorrect. /priority important-soon


Question: }, wantFinishTime: defaultTestTime, }, <ins> // In this case, init container is stopped after the regular containers. // This is because with the sidecar (restartable init) containers, // sidecar containers will always finish later than regular containers. "Pod with sidecar container and all containers terminated": { pod: v1.Pod{ Spec: v1.PodSpec{ InitContainers: []v1.Container{ { Name:          "sidecar", RestartPolicy: &containerRestartPolicyAlways, }, }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime.Add(-1 * time.Second))}, }, }, }, InitContainerStatuses: []v1.ContainerStatus{ { Name: "sidecar", State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime)}, }, }, }, }, }, wantFinishTime: defaultTestTime, }, </ins> } for name, tc := range testCases {

Today the function () only account for the regular containers finish time, presumably assuming that init containers have completed before before. However, with the sidecar (restartable init) containers, sidecar containers will always finish later than regular containers. And those needs to be accounted for the calculation. /kind bug The sidecar finish time will be accounted when calculating the job's finish time. Sidecar container with prolonged termination logic on a job will result in incorrect finish time reporting. /sig apps /priority backlog KEP:
/assign
/triage accepted


Question: } func (b *errorOnce) Send(errIn <-chan error) ErrorOnce { <del> go b.forward(errIn) </del> <ins> if errIn == nil { // don't execute this in a goroutine; save resources AND the caller // likely wants this executed ASAP because some of some operation // ordering semantics. forward() will not block here on a nil input // so this is safe to do. b.forward(nil) } else { go b.forward(errIn) } </ins> return b } func (b *errorOnce) forward(errIn <-chan error) { if errIn == nil { <ins> // important: nil never blocks; Report(nil) is guaranteed to be a // non-blocking operation. </ins> b.Report(nil) return }

Can we get this fixed ASAP? Thanks!
yep, thanks for reporting , Brendan Burns wrote:
how reproduceable is this? I have some changes that I think will resolve it but I couldn't reproduce locally even when all my cpu's were pretty tied up and using a high GOMAXPROCS.
flakey label process?
the labelling is due to me failing @ github's label interface. Bulk editing labels removes them if any of the selected things have the label sigh Thanks for the quick fix.


Question: } } <del> func TestZoneSpreadPriority(t *testing.T) { labels1 := map[string]string{ "foo": "bar", "baz": "blah", } labels2 := map[string]string{ "bar": "foo", "baz": "blah", } zone1 := map[string]string{ "zone": "zone1", } zone2 := map[string]string{ "zone": "zone2", } nozone := map[string]string{ "name": "value", } zone0Spec := v1.PodSpec{ NodeName: "machine01", } zone1Spec := v1.PodSpec{ NodeName: "machine11", } zone2Spec := v1.PodSpec{ NodeName: "machine21", } labeledNodes := map[string]map[string]string{ "machine01": nozone, "machine02": nozone, "machine11": zone1, "machine12": zone1, "machine21": zone2, "machine22": zone2, } tests := []struct { pod          *v1.Pod pods         []*v1.Pod nodes        map[string]map[string]string services     []*v1.Service expectedList framework.NodeScoreList name         string }{ { pod:   new(v1.Pod), nodes: labeledNodes, expectedList: []framework.NodeScore{{Name: "machine11", Score: framework.MaxNodeScore}, {Name: "machine12", Score: framework.MaxNodeScore}, {Name: "machine21", Score: framework.MaxNodeScore}, {Name: "machine22", Score: framework.MaxNodeScore}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "nothing scheduled", }, { pod:   &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods:  []*v1.Pod{{Spec: zone1Spec}}, nodes: labeledNodes, expectedList: []framework.NodeScore{{Name: "machine11", Score: framework.MaxNodeScore}, {Name: "machine12", Score: framework.MaxNodeScore}, {Name: "machine21", Score: framework.MaxNodeScore}, {Name: "machine22", Score: framework.MaxNodeScore}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "no services", }, { pod:      &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods:     []*v1.Pod{{Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}}, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"key": "value"}}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: framework.MaxNodeScore}, {Name: "machine12", Score: framework.MaxNodeScore}, {Name: "machine21", Score: framework.MaxNodeScore}, {Name: "machine22", Score: framework.MaxNodeScore}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "different services", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods: []*v1.Pod{ {Spec: zone0Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: labels1}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: framework.MaxNodeScore}, {Name: "machine12", Score: framework.MaxNodeScore}, {Name: "machine21", Score: 0}, {Name: "machine22", Score: 0}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "three pods, one service pod", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods: []*v1.Pod{ {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: labels1}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: 50}, {Name: "machine12", Score: 50}, {Name: "machine21", Score: 50}, {Name: "machine22", Score: 50}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "three pods, two service pods on different machines", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1, Namespace: metav1.NamespaceDefault}}, pods: []*v1.Pod{ {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1, Namespace: metav1.NamespaceDefault}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1, Namespace: "ns1"}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: labels1}, ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceDefault}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: 0}, {Name: "machine12", Score: 0}, {Name: "machine21", Score: framework.MaxNodeScore}, {Name: "machine22", Score: framework.MaxNodeScore}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "three service label match pods in different namespaces", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods: []*v1.Pod{ {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: labels1}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: 66}, {Name: "machine12", Score: 66}, {Name: "machine21", Score: 33}, {Name: "machine22", Score: 33}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "four pods, three service pods", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods: []*v1.Pod{ {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"baz": "blah"}}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: 33}, {Name: "machine12", Score: 33}, {Name: "machine21", Score: 66}, {Name: "machine22", Score: 66}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "service with partial pod label matches", }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, pods: []*v1.Pod{ {Spec: zone0Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, {Spec: zone2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}}, }, nodes:    labeledNodes, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: labels1}}}, expectedList: []framework.NodeScore{{Name: "machine11", Score: 75}, {Name: "machine12", Score: 75}, {Name: "machine21", Score: 50}, {Name: "machine22", Score: 50}, {Name: "machine01", Score: 0}, {Name: "machine02", Score: 0}}, name: "service pod on non-zoned node", }, } // these local variables just make sure controllerListerreplicaSetListerstatefulSetLister not nil // when construct metaDataProducer sss := []*apps.StatefulSet{{Spec: apps.StatefulSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}} rcs := []*v1.ReplicationController{{Spec: v1.ReplicationControllerSpec{Selector: map[string]string{"foo": "bar"}}}} rss := []*apps.ReplicaSet{{Spec: apps.ReplicaSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}} for _, test := range tests { t.Run(test.name, func(t *testing.T) { nodes := makeLabeledNodeList(labeledNodes) snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes)) zoneSpread := ServiceAntiAffinity{podLister: snapshot.Pods(), serviceLister: fakelisters.ServiceLister(test.services), labels: []string{"zone"}} metaDataProducer := NewMetadataFactory( fakelisters.ServiceLister(test.services), fakelisters.ControllerLister(rcs), fakelisters.ReplicaSetLister(rss), fakelisters.StatefulSetLister(sss), 1, ) metaData := metaDataProducer(test.pod, nodes, snapshot) list, err := runMapReducePriority(zoneSpread.CalculateAntiAffinityPriorityMap, zoneSpread.CalculateAntiAffinityPriorityReduce, metaData, test.pod, snapshot, makeLabeledNodeList(test.nodes)) if err != nil { t.Errorf("unexpected error: %v", err) } // sort the two lists to avoid failures on account of different ordering sortNodeScoreList(test.expectedList) sortNodeScoreList(list) if !reflect.DeepEqual(test.expectedList, list) { t.Errorf("expected %#v, got %#v", test.expectedList, list) } }) } } </del> func makeLabeledNodeList(nodeMap map[string]map[string]string) []*v1.Node { nodes := make([]*v1.Node, 0, len(nodeMap)) for nodeName, labels := range nodeMap {

/sig scheduling /priority important-soon See as an example. /assign
/unassign /help
/assign
/remove-help


Question: <ins> #!/usr/bin/env bash # Copyright 2024 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${KUBE_ROOT}" source hack/lib/init.sh # NOTE: Please do NOT add any to this list!! # # We are aiming to consolidate on: registry.k8s.io/e2e-test-images/agnhost # The sources for which are in test/images/agnhost. # If agnhost is missing functionality for your tests, please reach out to SIG Testing. kube::util::read-array PERMITTED_IMAGES < <(sed '/^#/d' ./test/images/.permitted-images) # get current list of images, ignoring tags echo "Getting e2e image list ..." make WHAT=test/e2e/e2e.test e2e_test="$(kube::util::find-binary e2e.test)" kube::util::read-array IMAGES < <("${e2e_test}" --list-images | sed -E 's/^(.+):[^:]+$/1/' | LC_ALL=C sort -u) # diff versus known permitted images ret=0 >&2 echo "Diffing e2e image list ..." diff -Naupr <(printf '%sn' "${IMAGES[@]}") <(printf '%sn' "${PERMITTED_IMAGES[@]}") || ret=$? if [[ $ret -eq 0 ]]; then >&2 echo "PASS: e2e images used are OK." else >&2 echo "FAIL: e2e images do not match the approved list!" >&2 echo "" >&2 echo "Please use registry.k8s.io/e2e-test-images/agnhost wherever possible, we are consolidating test images." >&2 echo "See: test/images/agnhost/README.md" >&2 echo "" >&2 echo "You can reach out to https://git.k8s.io/community/sig-testing for help." exit 1 fi </ins>

We should check the images configured in test/e2e to ensure we're not adding new images: In almost all cases, should be extended to support any missing functionality and used as the sole image for new tests. This has been the policy in SIG Testing for at least a few years now. To prevent regressing on this, we should check the list of images and permit tag updates but NOT adding any more names to the list. We can do this with an allowlist against /sig testing /kind test Others are welcome to work on this, but I'll submit something soon if nobody beats me to it. /assign Follow up to , we should aim to ship something this release (1.30).
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : We should check the images configured in test/e2e to ensure we're not adding new images: /kind test /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hello! Release Signal shadow here. This issue has not been updated for a long time, so I'd like to check what's the status. The code freeze is starting (less than a month from now), and while there is still plenty of time, we want to ensure that each PR has a chance to be merged on time. As this issue is tagged for 1.30, is it still planned for this release?
We still intend to do this and it's a test-only change so we're aiming for test freeze.
Hello! Release Signal shadow here. I'd like to check what's the status. The code freeze is starting in a week, and we want to ensure that each PR has a chance to be merged on time. Is it still planned for this release?
As has no update recently, /milestone clear we may make it in v1.31.
Yeah I think we have to revisit this later.


Question: }, "service": { "$ref": "v1beta1.ServiceReference", <del> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnIf there is only one port open for the service, that port will be used. If there are multiple ports open, port 443 will be used if it is open, otherwise it is an error." </del> <ins> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnPort 443 will be used if it is open, otherwise it is an error." </ins> }, "caBundle": { "type": "string",

What happened: Deployed a webhook backend and service as follows: Please note that there is only 8443 port on the service. Register the webhook as follows: Got the following error when creating pods: What you expected to happen: Based on , should be called. Not the 443 port shown in the error message. How to reproduce it (as minimally and precisely as possible): Just deploy the same setup and register webhook. Anything else we need to know?: Environment: Kubernetes version (use ): HEAD Client Version: version.Info{Major:"1", Minor:"10+", GitVersion:"v1.10.0-alpha.1.97+-dirty", GitCommit:"", GitTreeState:"dirty", BuildDate:"2018-01-22T20:00:24Z", GoVersion:"go1.9.1", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"11+", GitVersion:"v1.11.0-alpha.0.470+", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-03-12T19:36:42Z", GoVersion:"go1.9.3", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: GCE OS (e.g. from /etc/os-release): COS Kernel (e.g. ): COS
thanks for the detailed bug report!
I actually think that the API specification (or documentation) must change. Not all API servers will have access to look up the Service object and see what ports it defines. If we want to support ports other than 443, the port information must be specified in the webhook object, not discovered by direct Service object lookup
/assign
Looks like it will always use 443 no matter what ports are available, right? Should our documentation reflect this or should it behave differently? (also just sets 443 in the custom dial wrapper thing )
As a first step, I would adjust the documentation. As noted in , if we want to support ports other than 443, that information needs to be to the webhook config object


Question: package kubectl import ( <del> scheduling "k8s.io/api/scheduling/v1alpha1" </del> <ins> scheduling "k8s.io/api/scheduling/v1beta1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "reflect"

/kind bug What happened: When I try to create a priorityclass object like: It fails with And when I try create it with yaml file like: It works. What you expected to happen: Create a priorityclass object with kubectl.
/sig cli
Do you have --runtime- set in kube-apiserver?
No, if set this, it works.
Agree this is a bug. v1beta1 should be preferred.
/sig scheduling


Question: return err } <ins> // (9) if err = w.createUserVisibleFiles(cleanPayload); err != nil { klog.Errorf("%s: error creating visible symlinks in %s: %v", w.logContext, w.targetDir, err) return err } </ins> // (10) if err = w.removeUserVisiblePaths(pathsToRemove); err != nil { klog.Errorf("%s: error removing old visible symlinks: %v", w.logContext, err)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Creating a pod with a nested subpath volume will cause the pod to fail to start on Windows containerd nodes. What you expected to happen: The pods should start normally. How to reproduce it (as minimally and precisely as possible): Create the following pod on a Windows node with containerd: apiVersion: v1 kind: Pod metadata: annotations: e2e-test-privileged-psp name: pod-subpath-test-downwardapi spec: containers: args: mounttest --filecontentinloop=/test-volume --retrytime=20 image: claudiubelu/agnhost:2.21 name: test-container-subpath-downwardapi volumeMounts: mountPath: /test-volume name: test-volume subPath: downward/podname volumes: downwardAPI: defaultMode: 420 items: fieldRef: apiVersion: v1 fieldPath: path: downward/podname name: test-volume Anything else we need to know?: This issue is causing the following tests to fail on Windows Containerd nodes: [sig-storage] Subpath Atomic writer volumes should support subpaths with downward pod [LinuxOnly] [Conformance] [sig-storage] Subpath Atomic writer volumes should support subpaths with projected pod [LinuxOnly] [Conformance] The tests have the ` tag because we did not support Windows Containerd nodes at the time, and the tests still do not work for Windows Docker nodes. After a fix for this bug is merged, these tests will have to be renamed to enable them for Windows Containerd environments but not for Windows Docker environments. /sig windows Environment: Kubernetes version (use ): v1.21.0-alpha.0.493+ Cloud provider or hardware configuration: N/A OS (e.g: ): Windows Server 1809, Containerd Kernel (e.g. ): 10.0. N/A Build Install tools: N/A Network plugin and version (if this is a network-related bug): N/A Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance.

The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale


Question: addr := net.ParseIP(node.Name) if addr != nil { address := api.NodeAddress{Type: api.NodeLegacyHostIP, Address: addr.String()} <del> node.Status.Addresses = append(node.Status.Addresses, address) </del> <ins> api.AddToNodeAddresses(&node.Status.Addresses, address) </ins> } else { addrs, err := lookupIP(node.Name) if err != nil {

Since we append hostips to the list on every node controller run invocation which happens at 10s intervals, this list grows pretty large pretty soon: Maybe: would still satisfy the requirements?
Fix is here I think we generally use list instead of map
This could have been P1, but I just merged a fix so I'll leave at P2. Remaining work to do: Add to validation something that limits the number of IP addresses. (and maybe checks dups?)
Yeah, I'll add validation later.


Question: "k8s.io/kubernetes/pkg/scheduler/internal/heap" "k8s.io/kubernetes/pkg/scheduler/metrics" "k8s.io/kubernetes/pkg/scheduler/util" <ins> "k8s.io/utils/clock" </ins> ) const (

We should use /kind cleanup
/remove-kind feature /good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /remove-kind feature /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling /triage accepted
/assign


Question: return ok } <ins> func isPodStatusCacheTerminal(status *kubecontainer.PodStatus) bool { runningContainers := 0 runningSandboxes := 0 for _, container := range status.ContainerStatuses { if container.State == kubecontainer.ContainerStateRunning { runningContainers++ } } for _, sb := range status.SandboxStatuses { if sb.State == runtimeapi.PodSandboxState_SANDBOX_READY { runningSandboxes++ } } return runningContainers == 0 && runningSandboxes == 0 } </ins> // UpdatePod carries a configuration change or termination state to a pod. A pod is either runnable, // terminating, or terminated, and will transition to terminating if deleted on the apiserver, it is // discovered to have a terminal phase (Succeeded or Failed), or if it is evicted by the kubelet.

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --Restart: Always pods went into OutOfCpu error state after kubelet restart. Pods should remain running as node had sufficient capacity for them. Completed, RestartNever pod should not revert from Completed to OutOfCpu. Launch kubelet with: Run the following script against the local cluster: <details</detailsObserve pod status: Restart the kubelet: Observe pods: xref /sig node Kubernetes version (use ): v1.23.0-alpha.3.183+ - Cloud provider or hardware configuration: Local machine OS (e.g: ) & Kernel (e.g. ):Install tools: local-up- Network plugin and version (if this is a network-related bug): N/A Others:
/triage accepted /priority critical-urgent Reproducible on 1.22
From the reproducer: yields However, in the kubelet logs, it sees 7100m cpu allocated?
Looks like the topology admit handler thinks that the completed pods are using CPU which is why when it gets to the 8th it fails it with OutOfCpu. Must be a bug in there somewhere.
/assign
Unclear if could be related but linking for posterity
Confirmed this only affects 1.22, repro on 1.21 has expected behaviour: /sig scheduling I'll start a bisect.
is
Added some logging With this we see I think that the terminated pods are getting filtered, but for whatever reason they are still getting their resource requests to NodeInfo?
points out that my logging language was a bit backwards and that the completed pods should not have been returned, because they already successfully completed and exited. This code is suspect: The logs show that the completed pods are never filtered, so they are failing one of these checks. I am going to try to write a test case that hits this issue so we can test fixes against it.

The completed pods definitely show up as complete, but for some reason they do not get filtered (note I have changed my patch above for the log lines to make sense now): So I guess isPodTerminationRequested=true on already exited pods, but I would only expect that to be true if the pod was actively in the process of termination and not already exited. cc Log patch: <details</details
I tried changing to but apparently it thinks that's true :( Edit: doesn't work either.
The bug is that the only component in the kubelet that is allowed to authoritatively state a container is terminated (pod worker) takes non-zero time to do so during startup (because we have to call kill container to guarantee it's stopped). This races with the pod config loop and admission, which sends all the containers at once. In order for the race to be eliminated, some equivalent of a sync has to be completed during startup on terminal but potentially non-terminated pod workers BEFORE other adds are processed. That could be running HandlePodCleanups once all sources are synced but before HandlePod* methods are called, but HandlePodAdditions() cannot be invoked until after the pod worker has had a chance to verify all terminal pods are actually terminal. EDIT: Another potential approach is for the pod worker to hold starting pods until SyncKnownPods observes all terminal pods are accounted for at least once, then allowing starts.
Also, we appear to be missing a kubelet restart test that verifies admission with completed pods (which I thought was on our list of "missing e2e tests that would have caught these regressions" already). Such a test would have to: pods that consume resources for admission until they are running some of them with failed and` successful states the kubelet some of the running pods the kubelet none of the terminal pods are restarted, all running pods are restarted, and that only non-terminal pods are accounted to admission


Question: func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) { start := kl.clock.Now() sort.Sort(sliceutils.PodsByCreationTime(pods)) <del> // Responsible for checking limits in resolv.conf // The limits do not have anything to do with individual pods if kl.dnsConfigurer != nil && kl.dnsConfigurer.ResolverConfig != "" { kl.dnsConfigurer.CheckLimitsForResolvConf() } </del> for _, pod := range pods { existingPods := kl.podManager.GetPods() // Always add the pod to the pod manager. Kubelet relies on the pod

What happened: Ref , we noticed that there are some unnecessary check around limit in kubelet, specifically at: and is moving this check to outside of the pods loop make it better, but we should be looking at whether this check can exist in an even higher layer, as it doesn't make much sense in either or . cc
/sig network
/assign Pending outcome of the discussion here, I would be happy to present a PR
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved
poke :-)
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
How about lifting the check to the beginning of Kubelet#syncLoop (outside the for loop) ? All the events emitted by CheckLimitsForResolvConf are at EventTypeWarning level (not error). The CheckLimitsForResolvConf call in HandlePodUpdates would be removed as well.


Question: case *rsa.PrivateKey: key = k default: <del> return nil, errors.Wrapf(err, "the private key file %s isn't in RSA format", privateKeyPath) </del> <ins> return nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath) </ins> } return key, nil

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: When trying to deploy Kubernetes using kubeadm v1.12.1, kubeadm failed to generate certificates, causing it to fail at a later step. After passing to get all the logs I could I found that it was expecting all the certificates to have been pre-made, even though I only the certificate and key for the certificate authority. So I dug into the code, and in I found this: It turns out all the PKI I've been generating for things like etcd has been ECDSA. So I found the issue I was having, but I still couldn't figure out why kubeadm wasn't failing right away. I took a look at the caller of that function: It looks like that original error is swallowed. What you expected to happen: Immediate failure when keys are in the wrong format. How to reproduce it (as minimally and precisely as possible): Use an ECDSA cert/key pair and run . Anything else we need to know?: Environment: Kubernetes version (use ): v1.12.1 Cloud provider or hardware configuration: OpenStack OS (e.g. from /etc/os-release): Ubuntu 18.04 Kernel (e.g. ): 4.15.0-34-generic -Ubuntu SMP Mon Aug 27 15:21:48 UTC 2018 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
Please create a issue on
/area kubeadm /sig cluster-lifecycle
Sure.