Question: return checkMirrorPodDisappear(ctx, f.ClientSet, pod.Name, pod.Namespace) }, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.BeNil()) }) <ins> // Regression test for https://issues.k8s.io/118472 ginkgo.It("should force-delete non-admissible pods created and deleted during kubelet restart", func(ctx context.Context) { podName := "rejected-deleted-pod" + string(uuid.NewUUID()) gracePeriod := int64(30) nodeName := getNodeName(ctx, f) podSpec := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      podName, Namespace: f.Namespace.Name, }, Spec: v1.PodSpec{ NodeName: nodeName, NodeSelector: map[string]string{ "this-label": "does-not-exist-on-any-nodes", }, TerminationGracePeriodSeconds: &gracePeriod, RestartPolicy:                 v1.RestartPolicyNever, Containers: []v1.Container{ { Name:  podName, Image: imageutils.GetPauseImageName(), }, }, }, }) ginkgo.By("Stopping the kubelet") startKubelet := stopKubelet() // wait until the kubelet health check will fail gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) // Create the pod bound to the node. It will remain in the Pending // phase as Kubelet is down. ginkgo.By(fmt.Sprintf("Creating a pod (%v/%v)", f.Namespace.Name, podName)) pod := e2epod.NewPodClient(f).Create(ctx, podSpec) ginkgo.By(fmt.Sprintf("Deleting the pod (%v/%v) to set a deletion timestamp", pod.Namespace, pod.Name)) err := e2epod.NewPodClient(f).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) framework.ExpectNoError(err, "Failed to delete the pod: %q", pod.Name) // Restart Kubelet so that it proceeds with deletion ginkgo.By("Starting the kubelet") startKubelet() // wait until the kubelet health check will succeed gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) // Wait for the Kubelet to be ready. gomega.Eventually(ctx, func(ctx context.Context) bool { nodes, err := e2enode.TotalReady(ctx, f.ClientSet) framework.ExpectNoError(err) return nodes == 1 }, time.Minute, f.Timeouts.Poll).Should(gomega.BeTrue()) ginkgo.By(fmt.Sprintf("After the kubelet is restarted, verify the pod (%v/%v) is deleted by kubelet", pod.Namespace, pod.Name)) gomega.Eventually(ctx, func(ctx context.Context) error { return checkMirrorPodDisappear(ctx, f.ClientSet, pod.Name, pod.Namespace) }, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.BeNil()) }) // Regression test for an extended scenario for https://issues.k8s.io/118472 ginkgo.It("should force-delete non-admissible pods that was admitted and running before kubelet restart", func(ctx context.Context) { nodeLabelKey := "custom-label-key-required" nodeLabelValueRequired := "custom-label-value-required-for-admission" podName := "rejected-deleted-run" + string(uuid.NewUUID()) gracePeriod := int64(30) nodeName := getNodeName(ctx, f) pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      podName, Namespace: f.Namespace.Name, }, Spec: v1.PodSpec{ NodeSelector: map[string]string{ nodeLabelKey: nodeLabelValueRequired, }, NodeName:                      nodeName, TerminationGracePeriodSeconds: &gracePeriod, RestartPolicy:                 v1.RestartPolicyNever, Containers: []v1.Container{ { Name:  podName, Image: imageutils.GetPauseImageName(), }, }, }, }) ginkgo.By(fmt.Sprintf("Adding node label for node (%v) to allow admission of pod (%v/%v)", nodeName, f.Namespace.Name, podName)) e2enode.AddOrUpdateLabelOnNode(f.ClientSet, nodeName, nodeLabelKey, nodeLabelValueRequired) ginkgo.DeferCleanup(func() { e2enode.RemoveLabelOffNode(f.ClientSet, nodeName, nodeLabelKey) }) // Create the pod bound to the node. It will start, but will be rejected after kubelet restart. ginkgo.By(fmt.Sprintf("Creating a pod (%v/%v)", f.Namespace.Name, podName)) pod = e2epod.NewPodClient(f).Create(ctx, pod) ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, pod.Name)) err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, pod.Name) ginkgo.By("Stopping the kubelet") startKubelet := stopKubelet() // wait until the kubelet health check will fail gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) ginkgo.By(fmt.Sprintf("Deleting the pod (%v/%v) to set a deletion timestamp", pod.Namespace, pod.Name)) err = e2epod.NewPodClient(f).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) framework.ExpectNoError(err, "Failed to delete the pod: %q", pod.Name) ginkgo.By(fmt.Sprintf("Removing node label for node (%v) to ensure the pod (%v/%v) is rejected after kubelet restart", nodeName, f.Namespace.Name, podName)) e2enode.RemoveLabelOffNode(f.ClientSet, nodeName, nodeLabelKey) // Restart Kubelet so that it proceeds with deletion ginkgo.By("Starting the kubelet") startKubelet() // wait until the kubelet health check will succeed gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) // Wait for the Kubelet to be ready. gomega.Eventually(ctx, func(ctx context.Context) bool { nodes, err := e2enode.TotalReady(ctx, f.ClientSet) framework.ExpectNoError(err) return nodes == 1 }, time.Minute, f.Timeouts.Poll).Should(gomega.BeTrue()) ginkgo.By(fmt.Sprintf("Once Kubelet is restarted, verify the pod (%v/%v) is deleted by kubelet", pod.Namespace, pod.Name)) gomega.Eventually(ctx, func(ctx context.Context) error { return checkMirrorPodDisappear(ctx, f.ClientSet, pod.Name, pod.Namespace) }, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.BeNil()) }) </ins> }) })

We observed the following series of events: 1) Pod is created with some attributes that later result in the pod being rejected at kubelet admission time 2) The pod is scheduled to a node 3) The pod is gracefully deleted by a controller, i.e. a deletion timestamp is set on the pod 4) The pod is rejected at kubelet admission time 5) The pod never gets deleted and is stuck: despite having the deletion timestamp set. It seems that kubelet does not issue the final force delete for the pod. This is problematic in many scenarios, but one specific case it was hit was when the pod is backed by a daemonset controller. This issue manifested in way where since the pod never got deleted in step 5 above, the daemonset controller did not create a new pod to run the on the node and thus node was was not running a replica of the daemonset controller pod. This seems to a be regression in k8s 1.27, I was not able to repro this behavior in 1.26. It is expected at during the series of events above, after the pod is failed at kubelet admission time, since the pod has a deletion timestamp set, the kubelet should forcefully delete the pod. The pod should not hang in deletion forever. Create a 1.27 kind cluster: Stop kubelet Create a pod bound to the node which will trigger a admission error. The below node selector label does not exist, so we expect that the pod should be rejected in admission. While kubelet is down issue a graceful deletion to set a deletion timestamp on the pod Verify deletion timestamp is set (pod is pending) Start kubelet Observe pod status, the pod is now in failed phase (as expected). However, the hangs, the pod is never force deleted by kubelet. No response 27 <detailsn/a </details<details</details<details</details<details</details<details</details
/sig node
cc
The kubelet logs have the following in the above repro: and there is no log about the pod being deleted by the status manager. Full kubelet logs - The log seems unexpected :) - the pod is not being deleted because the phase is "non-terminal", but the phase is failed which is a contradiction.

It points that in the code something is wrong because
/assign I've checked that this is regression caused by this (the scenario works when this PR is reverted on 1.27.2). One relevant difference is that the function now returns false, because the . However, even if we make it return , still the deletion will not happen because . Wondering about the preferred solution. I guess ideally if we could trigger the . OTOH it wasn't triggered before and a pod rejected due to admission does not really need termination of containers, so maybe a shortcut will work. FYI
Opened PR:
Sigh... then it sounds like the daemonset controller is not handling correctly pods with a terminal phase. It looks like its waiting for them to disappear. Can you open another issue for that?
The pod that revealed the kubelet regression was indeed created by DaemonSet. However, this issue is reproducible without DS. Moreover, I cannot reproduce it with Kubelet fixed.
But this is the same problem as The DS controller should react to a Pod in a terminal phase and create a replacement Pod. I agree, if we fix kubelet, it's a non-issue, but the DS controller is not behaving correctly.
In this scenario the pod already has a , so DS ignores it waiting for it to complete (which would normally happen if Kubelet works ok): Not sure what is the expected behavior of DS here. I guess DS could look at what is the expected , and if exceeded substantially (like a minute or two) then force-delete the pod itself.
DS should consider the Pod terminal and, as such, create a replacement. Note that I'm referring to a state of the Pod where it has a terminal phase, regardless of whether it has a deletionTimestamp.
I see, going to open a dedicated DS Issue to move the discussion on priority and the shape of the fix there. Let's focus here on the Kubelet.
/cc
/triage accepted /priority important-soon
/cc
A pod that has been rejected due to admission should be continually readded to the pod worker by HandlePodCleanups until it succeeds at being terminated. That's because the pod is not in the desired set. Is HandlePodCleanups not adding it it in "Kill" state to the pod worker?
I took a closer look at this and is not handling this case.

This is because the pod has been rejected at kubelet admission time, so a pod worker has not started for it. Additionally, the pod is not visible in the runtime, so it's not an orphan pod either. The main reason is because the pod has never started, so there is no pod worker driving it. The pod is returned from the desired state, pod manager (as part of . However, the pod is filtered out as part of because there was never a pod worker driving it, so there no for it. And since the pod is not part of and it's not an orphan pod (since it doesn't exist in runtime because we haven't started it), we end up not do anything with it in .
Should it be? I think right now our answer is no - but if we moved admission closer to the pod worker, we'd say yes? Pros and cons? We start pod workers for terminating a pod - is rejecting a pod a type of rejection? For the short term, why wouldn't HandlePodCleanups manage this case? It's the "resync" action for the kubelet as controller, and so it's nominally responsible for forcing all other components (pod worker, status manager, runtime) to return to state. Should it dispatch a TerminatePod call for these rejected pods that are not yet deleted, not known to the pod worker, and not known to the status manager?
Long term this might be the best option - having every path go through pod worker and the regular pod termination (i.e. instead of special casing special situations like admission rejection, may be simpler to reason about. Adding this to is definitely one option and I think we could expand it to cover this case. The only issue I see handling it there, is there could potentially be a large number of pods that are in terminal phase, not have a deletion timestamp set, and unknown pod worker and pod runtime. For each of these pods, we would have to call in every iteration of until someone eventually set the deletion timestamp and the pod would be deleted. Looking at the code, it should short-circuit , but we would need to confirm. Me and were also considering two other options: during kubelet rejection.

I think it should be fine overall, but has a risk that after a kubelet restart, if a previous pod was still in runtime and it's running, could lead to the pod being deleted by status manager before the orphaned pod is terminated. by pod worker during kubelet rejection. It should go through pod worker and check that all containers are terminated, so it should avoid issue of the first approach. It sounds like option 2 or adding this HandlePodCleanups is the optimal paths forward. Curious to get your thoughts on the options here.
/retitle Pod graceful deletion hangs if the kubelet already rejected that pod at admission time There are 2 admission phases for Pods (API server admission, kubelet admission). This issue is about the second phase only.
Hi, are there any workarounds in 1.27?
The latest version of GKE contains a mitigation for this issue when it's related to Spot VM preemptions. It's still rolling out in some regions/zones. Please ensure you are in the latest patch version available in GKE.
The fix was cherry-picked for the 1.27 branch in , so it should be included in 1.27.4.
; we're on the latest patch () and we see this.
Can you clarify what you observe? We fixed a few different things that were related. Some of the fixes should be available in 1.27.3 and the last one in 1.27.4.
Hey no problem, we upgraded our cluster from to today, a new node came up and 37 pods were scheduled to it. One of those failed with: We've never seen this before in the 3+ years we've used GKE and upgraded in this manner, so appears to be related to the 1.27 release. I assume you work for Google? If it would help you I can raise a support case and give you the case number so you can inspect our logs.
That sounds unrelated to this issue. Pod failures (even if uncommon) are expected in a distributed system such as kubernetes. What's important is that the system should recover. This usually means that a replacement pod is created. In your case, was a replacement pod not created? (for Pods backed by Daemonset, StatefulSet, Job, Deployment).

Nevertheless, just the error message you describe sounds more like , not this issue.
I am getting at 1.27.4-gke.900 running at GCP.


Question: pod.Labels = jig.Labels _, err = cs.CoreV1().Pods(ns).Create(pod) framework.ExpectNoError(err) <del> framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) </del> <ins> framework.ExpectNoError(f.WaitForPodReady(pod.Name)) </ins> defer func() { framework.Logf("Cleaning up the echo server pod") err := cs.CoreV1().Pods(ns).Delete(serverPodName, nil)

Per PR the test fails to error properly. One line of particular interest from that PR: If you wonder why it does not fail on docker, that's because the test does not report the error correctly but fails the same way. This issue is intended to track the discussion and resolution of making this test error properly so that it is useful to CI Signal /sig networking /cc
The label(s) cannot be appled. These labels are supported: <detailsIn response to : Per PR the test fails to error properly. One line of particular interest from that PR: If you wonder why it does not fail on docker, that's because the test does not report the error correctly but fails the same way. ` /cc </details
I'm following up on this in
/sig network


Question: if err != nil { glog.Fatalf("Error creating storage factory: %v", err) } <del> endpointConfig, err := c.ExtraConfig.StorageFactory.NewConfig(kapi.Resource(DefaultStorageEndpoint)) </del> <ins> endpointConfig, err := c.ExtraConfig.StorageFactory.NewConfig(kapi.Resource("endpoints")) </ins> if err != nil { glog.Fatalf("Error getting storage config: %v", err) }

/kind bug The lease reconciler creates the kubernetes endpoint with a prefix of kube-apiserver-endpoint, however that is not the prefix that the rest of kube uses. The end result is that the kubernetes service doesn't have an endpoint, and kubectl won't list the kubernetes endpoint. The lease reconciler also creates lease files with a really long lease time instead of the default 15 seconds.
/cc
cc xref
kube-apiserver-endpoint is used for the lease itself, from what I remember. I don't think that's an issue by itself.
Oh but it does look like the endpointRegistry for the lease reconciler is misconfigured to use that
i'll look into it
what is the name that should be used?
services/endpoints afaik -
Yeah, making that change locally fixed it for me
But I'm not seeing the masterleases/apiServerIPInfo data in etcd, or I'm issuing the wrong command to find it
I had that same problem. The lease timeout is set way high, but for some reason that resulted in the lease files being created for only a very short time. I'm not sure why. Playing with the ttl amount got the lease files to stay around....forever. I've got a PR I'm about to push to fix both issues.
FYI has a PR for this now
cc


Question: t.Fatalf("expected '%#v', got '%#v'", e, a) } } <ins> func TestRedirectOnMissingTrailingSlash(t *testing.T) { table := []struct { // The requested path path string // The path requested on the proxy server. proxyServerPath string }{ {"/trailing/slash/", "/trailing/slash/"}, {"/", "/"}, // "/" should be added at the end. {"", "/"}, // "/" should not be added at a non-root path. {"/some/path", "/some/path"}, } for _, item := range table { proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.URL.Path != item.proxyServerPath { t.Errorf("Unexpected request on path: %s, expected path: %s, item: %v", req.URL.Path, item.proxyServerPath, item) } })) defer proxyServer.Close() serverURL, _ := url.Parse(proxyServer.URL) simpleStorage := &SimpleRESTStorage{ errors:                    map[string]error{}, resourceLocation:          serverURL, expectedResourceNamespace: "ns", } handler := handleNamespaced(map[string]rest.Storage{"foo": simpleStorage}) server := httptest.NewServer(handler) defer server.Close() proxyTestPattern := "/api/version/proxy/namespaces/ns/foo/id" + item.path req, err := http.NewRequest( "GET", server.URL+proxyTestPattern, strings.NewReader(""), ) if err != nil { t.Errorf("unexpected error %v", err) continue } // Note: We are using a default client here, that follows redirects. resp, err := http.DefaultClient.Do(req) if err != nil { t.Errorf("unexpected error %v", err) continue } if resp.StatusCode != 200 { t.Errorf("Unexpected errorCode: %v, expected: 200. Response: %v, item: %v", resp.StatusCode, resp, item) } } } </ins>

This works: but this does not: We should somehow emphasize the need for the trailing slash for now, and then work out what is going wrong (client side scripting?) to fix this (our fault in the proxy or Grafana's?)
Likewise for Kibana.
For the Kibana case, I observe a sequence of correctly proxied GETs e.g. to: but then I see: which fails because I assume what was really needed is: and a curl against that gives a pile of plausible Javascript.
After a little thought, I think what's going on here is that web clients can't tell that is a directory without the trailing slash. One thing we could potentially do to help is to redirect (301) from to , since I don't think the former is useful for anyone.
cc
How are we generating that URL? Can we append a "/" at the end while generating the URL? Doing a redirect or auto-adding "/" in the proxy for all requests could break other requests.
The URL is not generated anywhere. It is assumed that '/proxy/service/<service name' would work. , Nikhil Jindal wrote:
I discussed this with and To clarify, proxy server does not require a trailing slash. It works as expected (gives the same response with or without a trailing slash) The client code, on the other hand, expects it to be rendered on a path with a trailing slash and breaks when that is not the case. Ideally, we should fix the client code. For now, we decided to update proxy to send a 301 redirect from a path without a trailing slash to a path with a trailing slash. I will send a PR for this.
By "client code" do you mean things like Elasticsearch and Kibana?
Yes. The code being rendered on those paths.
I agree with what said, I just want to clarify that it's not necessarily a "broken" client, just one with different expectations. Adding the 301 redirect seems like an easy way to satisfy such clients and reduce user pain.
Not fixed because of
On GCE, the redirection is still not happening because the request is upgraded before we get a chance to redirect On localhost, the redirection works fine but we are unable to load the css and js files because of
Is proxy still supported in v1beta3? I vaguely remember that we wanted to remove it.

I tried this: but got NotFound error.
Access to cluster-level logging components depends on the proxy so avoid making ELBs. Elasticsearch and Kibana are accessed via the v1beta3 service proxy.
Although the trailing slashes are no longer required.


Question: //  * If the status code and body don't make sense together: *UnexpectedStatusError //  * http.Client.Do errors are returned directly. func (r *Request) Do() Result { <ins> start := time.Now() defer func() { metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start)) }() </ins> body, err := r.DoRaw() if err != nil { return Result{err: err}

Currently we do not measure latency for REST operations. This is important for any component that performs non-watch operations (scheduler, controllers, kubelet). This is also prerequisite to actually make sure that all API calls take less than 1 sec ( ).
The apiserver is already , including for REST operations. You can scrape the metrics by hitting the /metrics HTTP endpoint.
Sorry, I was not clear in the description. I'm more interested on the client side and with more details so that I can measure things like: how long it takes to write bindings in scheduler how long it takes to update status in kubelet I'd think about it as a counterpart for apiserver stats.


Question: <del> /* Copyright 2014 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package app // This file exists to force the desired plugin implementations to be linked. // This should probably be part of some configuration fed into the build for a // given binary target. import ( // Cloud providers _ "k8s.io/kubernetes/pkg/cloudprovider/providers" // Admission policies "k8s.io/apiserver/pkg/admission" "k8s.io/kubernetes/plugin/pkg/admission/admit" "k8s.io/kubernetes/plugin/pkg/admission/alwayspullimages" "k8s.io/kubernetes/plugin/pkg/admission/antiaffinity" "k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds" "k8s.io/kubernetes/plugin/pkg/admission/deny" "k8s.io/kubernetes/plugin/pkg/admission/exec" "k8s.io/kubernetes/plugin/pkg/admission/gc" "k8s.io/kubernetes/plugin/pkg/admission/imagepolicy" "k8s.io/kubernetes/plugin/pkg/admission/initialization" "k8s.io/kubernetes/plugin/pkg/admission/initialresources" "k8s.io/kubernetes/plugin/pkg/admission/limitranger" "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision" "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists" "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label" "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector" "k8s.io/kubernetes/plugin/pkg/admission/podpreset" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority" "k8s.io/kubernetes/plugin/pkg/admission/resourcequota" "k8s.io/kubernetes/plugin/pkg/admission/security/podsecuritypolicy" "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny" "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" "k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault" "k8s.io/kubernetes/plugin/pkg/admission/webhook" ) // RegisterAllAdmissionPlugins registers all admission plugins func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { admit.Register(plugins) alwayspullimages.Register(plugins) antiaffinity.Register(plugins) defaulttolerationseconds.Register(plugins) deny.Register(plugins) exec.Register(plugins) gc.Register(plugins) imagepolicy.Register(plugins) initialization.Register(plugins) initialresources.Register(plugins) limitranger.Register(plugins) autoprovision.Register(plugins) exists.Register(plugins) noderestriction.Register(plugins) label.Register(plugins) podnodeselector.Register(plugins) podpreset.Register(plugins) podtolerationrestriction.Register(plugins) resourcequota.Register(plugins) podsecuritypolicy.Register(plugins) podpriority.Register(plugins) scdeny.Register(plugins) serviceaccount.Register(plugins) setdefault.Register(plugins) webhook.Register(plugins) } </del>

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: The current help message from kube-apiserver is wrong. It shows only the 'NamespaceLifecycle' plug-in is available, and the default value is . Users cannot know what are the available plug-ins, and they get confused where comes from. What you expected to happen: The correct output should be this: How to reproduce it (as minimally and precisely as possible): Run using master or at least 1.7.2. Anything else we need to know?: This is a regression introduced when refactoring code into generic server code. Environment: Kubernetes version (use ): 1.7.2+ ? Cloud provider or hardware configuration**: Any OS (e.g. from /etc/os-release): Any Kernel (e.g. ): Any Install tools: Any Others: None
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Yes, it is really exist. If no guys works on this, i will pick it up.
Thanks for confirming this. I'm already working on it. Will submit pr in minutes.
cool. :)


Question: // Test handleConnection, using go-routine because it needs to be able to write to unbuffered pf.errorChan pf.handleConnection(localConnection, ForwardedPort{Local: 1111, Remote: 2222}) <ins> assert.Equal(t, 0, remoteConnection.streamCount, "stream count should be zero") </ins> assert.Equal(t, "", remoteDataReceived.String()) assert.Equal(t, "", localConnection.receiveBuffer.String()) assert.Equal(t, "Handling connection for 1111n", out.String())

The port forwarding client leaks memory when a port forwarding session is reused over time. The port forwarder client should not leak memory. forwarding a port with kubectl. and close many connections to the forwarded port. The amount of memory leaked for each connection is quite small, so many connections are needed to observe the leak. This is related to , which was reported for the kubelet and fixed in . However, the same problem happens to the client. The client needs to call on the , otherwise the streams created by will be kept in memory until the port forwarder itself is closed, even if the streams themselves closed long before. <details</details<detailsN/A </details<details</details<details</details<details</details<details</details
/sig api-machinery
/cc /triage accepted
/cc


Question: By("Cleaning up cluster role binding") deleteClusterRoleBinding(config) <del> By("Removing the test directory") </del> <ins> By("Removing the test discovery directory") </ins> removeCmd := fmt.Sprintf("rm -r %s", path.Join(hostBase, discoveryDir)) err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, config.node0) Expect(err).NotTo(HaveOccurred())

The local PV provisioner is being updated to discover only real mount points (and not directories), that resolves some other related issues. The related e2e tests using the local PV provisioner need to be updated accordingly.
/assign
/sig storage
FYI:


Question: t.Fatal(err) } if !exists { <del> t.Errorf("No expectations found for DaemonSet %q", oldDSKey) </del> <ins> t.Fatalf("No expectations found for DaemonSet %q", oldDSKey) </ins> } if dsExp.Fulfilled() { t.Errorf("There should be unfulfiled expectation for creating new pods for DaemonSet %q", oldDSKey) } <del> if dsc.queue.Len() != 0 { t.Fatal("Unexpected item in the queue") </del> <ins> // process updates DS, update adds to queue waitForQueueLength(1, "updated DS") ok = dsc.processNextWorkItem() if !ok { t.Fatal("queue is shutting down") </ins> } <ins> // process does not re-update the DS expectStableQueueLength(0) </ins> err = client.AppsV1().DaemonSets(oldDS.Namespace).Delete(context.Background(), oldDS.Name, metav1.DeleteOptions{}) if err != nil { t.Fatal(err) } <del> err = wait.PollImmediate(100*time.Millisecond, informerSyncTimeout, func() (bool, error) { klog.V(8).Infof("Waiting for queue to have 1 item, currently has: %d", dsc.queue.Len()) return dsc.queue.Len() == 1, nil }) if err != nil { t.Fatalf("Deleting DS didn't result in new item in the queue: %v", err) } </del> <ins> waitForQueueLength(1, "deleted DS") </ins> _, exists, err = dsc.expectations.GetExpectations(oldDSKey) if err != nil {

Which jobs are failing: TestExpectationsOnRecreate unit test, in along with expectations bugfix during 1.19 and cherry-picked to 1.18/1.17 Which test(s) are failing: Reason for failure: Anything else we need to know: These failures are masked in CI because bazel test runs with flake retries (), so the test can fail 2/3 times and the job can still pass (cc
adding to the milestone for triage, in case this is a real bug and not just a test issue /milestone v1.19
cc
/assign
looks like the panic is due to the test continuing in the !exists case (rather than exiting with a Fatal): however, that still means the test is failing in three distinct places:
it looks like the same failure existed in 1.18, so this probably isn't release blocking, but it does need to be resolved in order for us to stop ignoring unit test flakes in CI runs
it actually looks like this test was during the 1.19 dev cycle and cherry-picked back to 1.18/1.17, which is why release-1.18 is failing :-/
figured it out... the controller sync is updating the DS, which is triggering an update event and adding an extra enqueue which the test did not account for good news is that it appears to be a test-only issue
possible test deflake in


Question: } } return nil } func (t *tcShaper) deleteInterface(class string) error {

What happened: I tried to set up bandwidth limiting by booting a cluster with , which by default sets up kubenet. I used the annotations xM xM and then used to test the ingress and egress bandwidths, with no effect. Does anyone know if this feature is still supported in kubenet? And if so what are the steps I need to take to boot a cluster that can use pod annotations to do tc shaping. What you expected to happen: Per traffic shaping should be implemented on Kubenet as of 4/12/2018. Anything else we need to know?: There are a lot of issues regarding traffic shaping not working. None have a concrete answer of how to get it to work on kubenet: Relevant: I believe this is where it was first implemented. Environment: Kubernetes version (use ): 1.11.8 Cloud provider or hardware configuration: AWS CC:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
cc
/triage unresolved I am a bot run by
can we put this on sig-network agenda? the discussion item would be: what is the current status of this feature (alpha, beta, GA?) what should we do next with this API?
I have this to the agenda, will grab some info.
AFAIK traffic shaping is still supported in kubenet. Does a pod with these annotations come up successfully but has no traffic shaping applied? Or do things fail earlier? If it comes up successfully, could you run ?
On GKE we do not officially support this, our default image even don't have tc installed. I tried with ubuntu, the kubenet is failed to parse the tc output. kubenet expect. // filter parent 1: protocol ip pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 tc return.// filter parent 1: protocol ip pref 1 u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 notinhw
If this issue has been triaged, please comment .

Meta: /athenabot mark-triage-reminder I am a bot run by
Depending on which CNI plugin you use you need to install the bandwidth plugin, see:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
since it is in progress /priority critical-urgent /milestone v1.17
/lifecycle frozen
/assign
Bug triage for release 1.17 here. Since this issue has been tagged for the milestone, we would like to check it's status. Code freeze is on Nov. 14. Will this issue be resolved before then? If not, we should move it out of the milestone.
The fix is still pending in review.
Will it be merged by this Thursday for v1.17 code freeze?
/milestone v1.18 (v1.17 code freeze)
matching assigned priority on PR
Hello I'm one of the Bug Triage shadows for the 1.18 release. Since this issue is tagged for 1.18, I'd like to check its status. Is this issue intended for milestone 1.18? Friendly reminder: Code freeze is scheduled for March 5th.
Hello Checking in again. Do you think this will make it in for 1.18 release?
Hey Checking in again as this issue is marked as . Are you close to completing this for v1.18 release? Another reminder that code freeze is just around the corner! (March 5th).
Friendly reminder that code freeze is in 2 days and this issue is marked as
The fix got approved by ,should be able to catch the freeze.


Question: } } } <ins> func TestInterpretContainerStatus(t *testing.T) { tests := []struct { status          *api.ContainerStatus expectedState   string expectedMessage string expectErr       bool name            string }{ { status: &api.ContainerStatus{ State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, Ready: true, }, expectedState:   "Running", expectedMessage: "", name:            "basic", }, { status: &api.ContainerStatus{ State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, Ready: false, }, expectedState:   "Running *not ready*", expectedMessage: "", name:            "basic not ready", }, { status: &api.ContainerStatus{ State: api.ContainerState{ Waiting: &api.ContainerStateWaiting{}, }, Ready: false, }, expectedState:   "Waiting", expectedMessage: "", name:            "waiting not ready", }, { status: &api.ContainerStatus{ State: api.ContainerState{ Waiting: &api.ContainerStateWaiting{}, }, Ready: true, }, expectedState:   "Waiting", expectedMessage: "", name:            "waiting", }, { status: &api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: 3, }, }, Ready: false, }, expectedState:   "Terminated", expectedMessage: "exit code 3", name:            "terminated not ready", }, { status: &api.ContainerStatus{ State: api.ContainerState{ Termination: &api.ContainerStateTerminated{ ExitCode: 5, Reason:   "test reason", }, }, Ready: true, }, expectedState:   "Terminated", expectedMessage: "exit code 5, reason: test reason", name:            "terminated", }, } for _, test := range tests { // TODO: test timestamp printing. state, _, msg, err := interpretContainerStatus(test.status) if test.expectErr && err == nil { t.Errorf("unexpected non-error (%s)", test.name) continue } if !test.expectErr && err != nil { t.Errorf("unexpected error: %v (%s)", err, test.name) continue } if state != test.expectedState { t.Errorf("expected: %s, got: %s", test.expectedState, state) } if msg != test.expectedMessage { t.Errorf("expected: %s, got: %s", test.expectedMessage, msg) } } } </ins>

When debugging , I found the output of 'kubectl get pods' was misleading, The status of the pod was reported as , but a further suggested the pod was not ready: shall warn the user about the fact that the pod is not ready.


Question: } } } <ins> func TestSyncRolloutStatus(t *testing.T) { pds := int32(60) testTime := metav1.Date(2017, 2, 15, 18, 49, 00, 00, time.UTC) failedTimedOut := extensions.DeploymentCondition{ Type:   extensions.DeploymentProgressing, Status: v1.ConditionFalse, Reason: util.TimedOutReason, } newRSAvailable := extensions.DeploymentCondition{ Type:               extensions.DeploymentProgressing, Status:             v1.ConditionTrue, Reason:             util.NewRSAvailableReason, LastUpdateTime:     testTime, LastTransitionTime: testTime, } replicaSetUpdated := extensions.DeploymentCondition{ Type:               extensions.DeploymentProgressing, Status:             v1.ConditionTrue, Reason:             util.ReplicaSetUpdatedReason, LastUpdateTime:     testTime, LastTransitionTime: testTime, } tests := []struct { name            string d               *extensions.Deployment allRSs          []*extensions.ReplicaSet newRS           *extensions.ReplicaSet conditionType   extensions.DeploymentConditionType conditionStatus v1.ConditionStatus conditionReason string lastUpdate      metav1.Time lastTransition  metav1.Time }{ { name:   "General: remove Progressing condition and do not estimate progress if deployment has no Progress Deadline", d:      currentDeployment(nil, 3, 2, 2, 2, []extensions.DeploymentCondition{replicaSetUpdated}), allRSs: []*extensions.ReplicaSet{newRSWithAvailable("bar", 0, 1, 1)}, newRS:  newRSWithAvailable("foo", 3, 2, 2), }, { name:            "General: do not estimate progress of deployment with only one active ReplicaSet", d:               currentDeployment(&pds, 3, 3, 3, 3, []extensions.DeploymentCondition{newRSAvailable}), allRSs:          []*extensions.ReplicaSet{newRSWithAvailable("bar", 3, 3, 3)}, conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.NewRSAvailableReason, lastUpdate:      testTime, lastTransition:  testTime, }, { name:            "DeploymentProgressing: dont update lastTransitionTime if deployment already has Progressing=True", d:               currentDeployment(&pds, 3, 2, 2, 2, []extensions.DeploymentCondition{replicaSetUpdated}), allRSs:          []*extensions.ReplicaSet{newRSWithAvailable("bar", 0, 1, 1)}, newRS:           newRSWithAvailable("foo", 3, 2, 2), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.ReplicaSetUpdatedReason, lastTransition:  testTime, }, { name:            "DeploymentProgressing: update everything if deployment has Progressing=False", d:               currentDeployment(&pds, 3, 2, 2, 2, []extensions.DeploymentCondition{failedTimedOut}), allRSs:          []*extensions.ReplicaSet{newRSWithAvailable("bar", 0, 1, 1)}, newRS:           newRSWithAvailable("foo", 3, 2, 2), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.ReplicaSetUpdatedReason, }, { name:            "DeploymentProgressing: create Progressing condition if it does not exist", d:               currentDeployment(&pds, 3, 2, 2, 2, []extensions.DeploymentCondition{}), allRSs:          []*extensions.ReplicaSet{newRSWithAvailable("bar", 0, 1, 1)}, newRS:           newRSWithAvailable("foo", 3, 2, 2), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.ReplicaSetUpdatedReason, }, { name:            "DeploymentComplete: dont update lastTransitionTime if deployment already has Progressing=True", d:               currentDeployment(&pds, 3, 3, 3, 3, []extensions.DeploymentCondition{replicaSetUpdated}), allRSs:          []*extensions.ReplicaSet{}, newRS:           newRSWithAvailable("foo", 3, 3, 3), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.NewRSAvailableReason, lastTransition:  testTime, }, { name:            "DeploymentComplete: update everything if deployment has Progressing=False", d:               currentDeployment(&pds, 3, 3, 3, 3, []extensions.DeploymentCondition{failedTimedOut}), allRSs:          []*extensions.ReplicaSet{}, newRS:           newRSWithAvailable("foo", 3, 3, 3), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.NewRSAvailableReason, }, { name:            "DeploymentComplete: create Progressing condition if it does not exist", d:               currentDeployment(&pds, 3, 3, 3, 3, []extensions.DeploymentCondition{}), allRSs:          []*extensions.ReplicaSet{}, newRS:           newRSWithAvailable("foo", 3, 3, 3), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.NewRSAvailableReason, }, { name:            "DeploymentComplete: defend against NPE when newRS=nil", d:               currentDeployment(&pds, 0, 3, 3, 3, []extensions.DeploymentCondition{replicaSetUpdated}), allRSs:          []*extensions.ReplicaSet{newRSWithAvailable("foo", 0, 0, 0)}, conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionTrue, conditionReason: util.NewRSAvailableReason, }, { name:            "DeploymentTimedOut: update status if rollout exceeds Progress Deadline", d:               currentDeployment(&pds, 3, 2, 2, 2, []extensions.DeploymentCondition{replicaSetUpdated}), allRSs:          []*extensions.ReplicaSet{}, newRS:           newRSWithAvailable("foo", 3, 2, 2), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionFalse, conditionReason: util.TimedOutReason, }, { name:            "DeploymentTimedOut: do not update status if deployment has existing timedOut condition", d:               currentDeployment(&pds, 3, 2, 2, 2, []extensions.DeploymentCondition{failedTimedOut}), allRSs:          []*extensions.ReplicaSet{}, newRS:           newRSWithAvailable("foo", 3, 2, 2), conditionType:   extensions.DeploymentProgressing, conditionStatus: v1.ConditionFalse, conditionReason: util.TimedOutReason, lastUpdate:      testTime, lastTransition:  testTime, }, } for _, test := range tests { fake := fake.Clientset{} dc := &DeploymentController{ client: &fake, } if test.newRS != nil { test.allRSs = append(test.allRSs, test.newRS) } err := dc.syncRolloutStatus(test.allRSs, test.newRS, test.d) if err != nil { t.Error(err) } newCond := util.GetDeploymentCondition(test.d.Status, test.conditionType) switch { case newCond == nil: if test.d.Spec.ProgressDeadlineSeconds != nil { t.Errorf("%s: expected deployment condition: %s", test.name, test.conditionType) } case newCond.Status != test.conditionStatus || newCond.Reason != test.conditionReason: t.Errorf("%s: DeploymentProgressing has status %s with reason %s. Expected %s with %s.", test.name, newCond.Status, newCond.Reason, test.conditionStatus, test.conditionReason) case !test.lastUpdate.IsZero() && test.lastUpdate != testTime: t.Errorf("%s: LastUpdateTime was changed to %s but expected %s;", test.name, test.lastUpdate, testTime) case !test.lastTransition.IsZero() && test.lastTransition != testTime: t.Errorf("%s: LastTransitionTime was changed to %s but expected %s;", test.name, test.lastTransition, testTime) } } } </ins>

Please see There are no unit tests currently for that method. This would be a good started PR for someone interested in helping with Kubernetes.
/sig apps
/cc
/for-new-contributors
/assign
I want this done by the end of this week so we can cherry-pick both the unit test and back to 1.7 and potentially 1.6.
if nobody else is working on it, i'll look into this.
Hey Please do so and cc me on your PR, I haven't had any time to start on it.
/priority important-soon


Question: return &os, nil } <ins> // NewFakeOpenStackCloud creates and returns an instance of Openstack cloudprovider. // Mainly for use in tests that require instantiating Openstack without having // to go through cloudprovider interface. func NewFakeOpenStackCloud(cfg Config) (*OpenStack, error) { provider, err := openstack.NewClient(cfg.Global.AuthURL) if err != nil { return nil, err } emptyDuration := MyDuration{} if cfg.Metadata.RequestTimeout == emptyDuration { cfg.Metadata.RequestTimeout.Duration = time.Duration(defaultTimeOut) } provider.HTTPClient.Timeout = cfg.Metadata.RequestTimeout.Duration os := OpenStack{ provider:     provider, region:       cfg.Global.Region, lbOpts:       cfg.LoadBalancer, bsOpts:       cfg.BlockStorage, routeOpts:    cfg.Route, metadataOpts: cfg.Metadata, } return &os, nil } </ins> // Initialize passes a Kubernetes clientBuilder interface to the cloud provider func (os *OpenStack) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { }

We merged to allow configurable volume limits for Cinder. But proposed adding such a limit to as well, among other things this will enable us to support node specific limits more easily - /sig openstack /sig storage /sig scheduling
/assign
cc
/kind feature
This will require updating in-tree Openstack cloudprovider code. are we onboard with that?
What would be the extent of the modification? It must not break current compatibility, and the in-tree provider is scheduled for removal this year. I'm ok with it as long as existing users aren't broken by it.
it should be a change into which would add a new configurable field. It would be backward compatible but would allow admins to configure different limits on a node if they must.
/milestone v1.14 We're entering and this looks relevant to the 1.14 release. Please if I am incorrect


Question: } // Overwrite the default for storage data format. s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" <ins> // register all admission plugins RegisterAllAdmissionPlugins(s.Admission.Plugins) </ins> // Set the default for admission plugins names s.Admission.PluginNames = []string{"AlwaysAdmit"} return &s

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: The current help message from kube-apiserver is wrong. It shows only the 'NamespaceLifecycle' plug-in is available, and the default value is . Users cannot know what are the available plug-ins, and they get confused where comes from. What you expected to happen: The correct output should be this: How to reproduce it (as minimally and precisely as possible): Run using master or at least 1.7.2. Anything else we need to know?: This is a regression introduced when refactoring code into generic server code. Environment: Kubernetes version (use ): 1.7.2+ ? Cloud provider or hardware configuration**: Any OS (e.g. from /etc/os-release): Any Kernel (e.g. ): Any Install tools: Any Others: None
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Yes, it is really exist. If no guys works on this, i will pick it up.
Thanks for confirming this. I'm already working on it. Will submit pr in minutes.
cool. :)


Question: err := l.volume.CleanupResource(ctx) errs = append(errs, err) <ins> if l.priorityClass != nil { framework.Logf("Deleting PriorityClass %v", l.priorityClass.Name) err := l.cs.SchedulingV1().PriorityClasses().Delete(ctx, l.priorityClass.Name, metav1.DeleteOptions{}) errs = append(errs, err) } </ins> framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource") l.migrationCheck.validateMigrationVolumeOpCounts(ctx) }

For 1.27 we should graduate the ReadWriteOncePod feature to beta. This is needed to rollout the feature to GA. In beta this feature will support preemption of pods using PVCs. See:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig storage /sig scheduling


Question: # GCE Load-Balancer Controller (GLBC) Cluster Addon This cluster addon is composed of: <del> * A [Google L7 LoadBalancer Controller](https://github.com/kubernetes/contrib/tree/master/Ingress/controllers/gce) </del> <ins> * A [Google L7 LoadBalancer Controller](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce) </ins> * A [404 default backend](https://github.com/kubernetes/contrib/tree/master/404-server) Service + RC It relies on the [Ingress resource](../../../../docs/user-guide/ingress.md) only available in Kubernetes version 1.1 and beyond.

The has to the Google L7 load balancer controller.
+1 on merging that PR, looks good.
LGTM'd the pr


Question: ginkgo.Context("With kubeconfig updated to static CPU Manager policy run the Topology Manager tests", func() { runTopologyManagerTests(f) }) })

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: ci-kubernetes-node-kubelet-serial node-kubelet-serial job Which test(s) are failing: Hard to tell, but everything at one point or another. Since when has it been failing: A month or two, by recollection. would indicate several years... ci-kubernetes-node-kubelet-serial | failing_days | 992 Testgrid link: Reason for failure: Hard to tell, some is timeout, but I think timeouts due to test failures. Anything else we need to know: Nothing consistent about the failures that I can tell besides timeouts. This is the only place these serial tests are running.
/sig node
/cc
I will take a look at what happens with the serial lane.
So the kubelet failed to restart because of I think it can be related to the topology manager scope PR, I will dig into the code tomorrow. FYI
It actually looks like it was introduced by: With the same issue reported here: And Alexey has a fix for it here (I haven't reviewed it):
Thanks for the update.
Some of these failures are definitely due to , which is causing tests to fail since pods aren't terminating in time. See e.g.
/assign
Under the latest run, I saw that the kubelet failed to restart because of The problem that one of the topology manager tests failed in the middle and did not remove the state file, I can not guarantee that it a single problem, but let's start from it.


Question: type LRUExpireCache struct { cache *lru.Cache <del> lock  sync.RWMutex </del> <ins> lock  sync.Mutex </ins> } func NewLRUExpireCache(maxSize int) *LRUExpireCache {

Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): "LRUExpireCache" Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT What happened: requires write lock since needs to manipulate its list to track recently used item. Currently it uses read lock so it may introduce race condition. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else do we need to know:
This was fixed in


Question: capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] <del> zonesWithNodes, err := cloud.GetCandidateZonesForDynamicVolume() </del> <ins> zonesWithNodes, err := getCandidateZones(cloud, node) </ins> if err != nil { <del> return "", 0, nil, "", fmt.Errorf("error querying for all zones: %v", err) </del> <ins> return "", 0, nil, "", fmt.Errorf("error finding candidate zone for pvc: %v", err) </ins> } volumeOptions, err := populateVolumeOptions(c.plugin.GetPluginName(), c.options.PVC.Name, capacity, tags, c.options.Parameters, node, allowedTopologies, zonesWithNodes)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We have pretty large clusters in aws and in production, we found an inefficiency in provisioning dynamic EBS volumes, where aws-ebs volume provisioner tries to look for available zone from aws directly. With scale, this can trigger massive from ec2, and affect cluster stability. This is also related to What you expected to happen: To simply get zones with available instances, we can get information from API server for efficiency. There might be a delay between node gets shutting down and such change is reflected in api server, but in case node is not ready or gets terminated, there will be higher level reconciliation triggered so we are safe. How to reproduce it (as minimally and precisely as possible): We only expect this to be hit when the cluster is huge (or aws account is huge) so not easy to reproduce. /assign /sig aws /sig cloud-provider
/assign


Question: // if m.Current() == nil, then we try again and get a valid // client server.backdate = 0 <del> server.expectUserAgent = "FirstClient" </del> <ins> server.SetExpectUserAgent("FirstClient") </ins> if ok, err := r.RotateCerts(); !ok || err != nil { t.Fatalf("unexpected rotation err: %t %v", ok, err) }

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: pull-kubernetes-bazel-test Which test(s) are failing: Test_buildClientCertificateManager Testgrid link: Reason for failure: Anything else we need to know:
/remove-kind failing-test /kind flake /sig scheduling
/sig node


Question: Metadata     map[string]string } <ins> // implements PVLabeler. var _ cloudprovider.PVLabeler = (*OpenStack)(nil) </ins> const ( VolumeAvailableStatus = "available" VolumeInUseStatus     = "in-use"

Resizing a persistent volume is partially broken on openstack (cinder) with k8s v1.11.1. Partially in the sense that static resizing (requiring a pod restart) works, but the newly introduced online resizing (not requiring a pod restart) fails. Upon resizing a pvc it remains in the Resizing condition forever (see the warning event) until I delete the pod that has this pvc mounted: After pod deletion it notifies me (correctly) that it's now waiting for the pod to come up to finish resizing: So far the warning event does not break the ExpandPersistentVolumes feature, as this one anyhow requires a pod restart. However it breaks the newly introduced ExpandInUsePersistentVolumes feature (alpha with k8s v1.11; doesn't require a pod restart for resizing to finish), as the pvc will remain in "Resizing" condition forever (or until the pod is restarted but this defeats the purpose of ExpandInUsePersistentVolumes). I'm not sure whether this is fully related but upon digging around I found and a consecutive fix with which was merged on Jan 23. However a few days later this fix seems to be (probably accidentally) overwritten with this commit: /kind bug /sig storage /sig openstack
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
It is already fixed. So I will close it.
/close
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I am trying cinder online pvc resizing on kubernetes1.12.2. I noticed this kubernetes version has include your fix, but I still failed to resize pvc online.

The error log has no change: Warning VolumeResizeFailed 45s (x2 over 66s) volumeexpand Error expanding volume "default/myclaim-expend" of plugin : volume status is not available The error log is reported below code line "volume.Status != volumeAvailableStatus" // ExpandVolume expands the size of specific cinder volume (in GiB) func (os *OpenStack) ExpandVolume(volumeID string, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { volume, err := os.getVolume(volumeID) if err != nil { return oldSize, err } if volume.Status != volumeAvailableStatus { // cinder volume can not be expanded if its status is not available return oldSize, fmt.Errorf("volume status is not available") } One point confused me, once cinder volume is used by pod, it's status is "in-use", how the online pvc resizing pass the code check "volume.Status != volumeAvailableStatus"? There's where the online pvc resizing fails.
It looks like Cinder can not support online volume expansion - which is what is happening here. Even if online expansion is enabled, I think to actually perform resizing of Cinder volumes - you must bring the pod down.


Question: podVolumes := make(container.VolumeMap) for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) { podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{ <del> Mounter:           mountedVolume.Mounter, BlockVolumeMapper: mountedVolume.BlockVolumeMapper, ReadOnly:          mountedVolume.VolumeSpec.ReadOnly, </del> <ins> Mounter:             mountedVolume.Mounter, BlockVolumeMapper:   mountedVolume.BlockVolumeMapper, ReadOnly:            mountedVolume.VolumeSpec.ReadOnly, InnerVolumeSpecName: mountedVolume.InnerVolumeSpecName, </ins> } } return podVolumes

Fix status: [x] fixed in master in [x] fixed in release-1.10 branch in , will be in 1.10.0 [x] fixed in release-1.9 branch in , will be in 1.9.7 [x] fixed in release-1.8 branch in , will be in 1.8.11 [x] fixed in release-1.7 branch in , will be in 1.7.16 In SubPath fixes, PR adds bind mount for subpaths. It uses as the path name. The volumename in this path is the container's mount volume name which is also called OuterVolumeSpecName in . This becomes an issue for volume reconstruction when kubelet restarts. Volume scans the volume directories and recover the InnerVolumeSpecName (which is used to construct pod volume paths). Volume reconstruction has not enough information to recover the OuterVolumeSpecName and figure out which volume it maps to. In case kubelet needs to clean up mounts (it happens when volume spec cannot be reconstructed), it will fail to clean up the subpaths because of lacking the OuterVolumeSpecName. We think it is better to use InnerVolumeSpecName for volume subPath instead of OuterVolumeSpecName so that volume clean up can work. The issue we might have is that some users might already deploy a cluster with subpath might already . If the users perform an in-place upgrade, since new version will use InnerVolumeSpecName which might have conflicts with some other older version's OuterVolumeSpecName among different volumes. This change requires clear documentation and communication with the users.
cc
This impacts reconstruction of PVCs, where the inner and outer names are different.
/assign
While testing the e2es against a cluster without the fix, I noticed that subsequent pods using volumes failed to start. My theory is that the unmount volume failed because the subpath was still mounted, and that blocked the volume manager from processing other pods. But I thought that volume manager is supposed to run the unmount volume operation in a separate thread per volume, so this shouldn't happen. Need to inspect the code more carefully to confirm the theory. could the volume reconstruction refactor have a unmount volume call in the main volume manager thread?
runs in reconciler's goroutine - so if that function is blocked, it will not be able to process any more volumes.

Was the problem that failed to unmount volume but got hung/stuck ?
I haven't confirmed for sure, but my guess is that unmount volume failed because the subpath was still mounted
moving into the milestone for discussion /milestone v1.10
/priority critical-urgent /status in-progress
/status approved-for-milestone per discussion in sig-release meeting today
[MILESTONENOTIFIER] Milestone Issue: Up-to-date for process <details open<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
Re-opening to track fixes in release branches
keeping in the v1.10 milestone until merges
since this is now fixed in 1.10, I'm taking it out of the milestone so that the release team doesn't keep checking it.
Can this issue be closed now?


Question: "os" "path" "strconv" <del> "strings" </del> "time" "github.com/golang/glog"

Forked from To make debian-based node compatible with docker 1.11.X, I updated both /proc/sys/kernel/keys/rootmaxkey and /proc/sys/kernel/keys/rootmaxbytes. Also other kubernetes components do the similar thing. For example, kube_proxy: We should clean up those configurations into one common place.
cc/
Is this looking for an owner? It seems the ask is to generalize these types of settings in a module.
Yeah. We update quite a few kernel settings in kubelet itself. We should move the ones mentioned in this issue to kubelet if it is applicable to other distros too. want to own this?
Sure, I'll start poking.
Test results so far for a few distributions' default settings Operating SystemValue --- --- Debian Jessie 8.7.5200 --- --- --- Ubuntu 14. --- --- --- Ubuntu 16. --- --- --- CentOS --- --- --- Fedora --- --- --- Redhat Atomic --- --- --- CoreOS stable1000000 --- --- undefined --- undefined


Question: }, "NodeMonitorGracePeriod": { SchemaProps: spec.SchemaProps{ <del> Description: "nodeMontiorGracePeriod is the amount of time which we allow a running node to be unresponsive before marking it unhealthy. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status.", </del> <ins> Description: "nodeMontiorGracePeriod is the amount of time which we allow a running node to be unresponsive before marking it unhealthy. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status. This value should also be greater than the sum of HTTP2_PING_TIMEOUT_SECONDS and HTTP2_READ_IDLE_TIMEOUT_SECONDS.", </ins> Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, },

The controller-manager node-lifecycle-controller watches node lease and update probeTimestamp , and it check node probeTimestamp is before . The node-monitor-grace-period default value is 40s. When the client connection lost, the lease watcher cannot receive new event. With the http2 health check, HTTP2PINGTIMEOUTSECONDS default is 30s, HTTP2READIDLETIMEOUTSECONDS default is 15s, it will delay 45s to kill http2 connection. During this period, node-lifecycle-controller will mark all nodes NotReady. Here are some logs When the http2 connection lost, the watch should stop as soon as possible. The reflector should relist resource, and node-lifecycle-controller should not mark all node NotReady. HTTP2PINGTIMEOUTSECONDS+HTTP2READIDLETIMEOUTSECONDS should be less than 40s. the controller-manager node-lifecycle-controller watch client srcIP:srcPort. Drop all tcp package send into srcIP:srcPort. for more than 40 seconds, all node will be marked NotReady node will be Ready after some seconds. No response <details</details<details</details<details</details<details</details<details</details<details</details
How about double-check node lease before marking node NotReady? The node-lifecycle-controller can get reference lease from apiserver with
/sig node
/sig api-machinery
/cc We have discussed this timeouts values before and myself, I also think got some metrics with lower timeouts for the HTTP2 connections. In this situation is clear that the defaults of the node watcher and the timeouts are related and having good defaults will be ideal, but the question is which values do we change ?
I think in internal network, the ping response almost return in 1 second, the HTTP2PINGTIMEOUT_SECONDS default value should be 5 seconds. And the total timeout will 35 seconds. WDYT?
/triage accepted any pointers in other places where those timeouts were changed for http2? Any notes from those discussions?
/priority important-longterm since it is not a regression I will keep it longterm priority and not higher. But we can reevaluate if we hear more reports about it.

/assign
I'm investigating problems similar to this, but I'm looking for a way to confirm this is what is happening - and if there are tips for how to recreate such a scenario, that would be very helpful. We get infrequent reports of "worker goes NotReady for a short time" incidents that appear to be triggered by events like failing over a LB sitting front of the kube-apiservers. We've done some tests with a simple client-go app and have recreated a scenario in which the app logs messages after 45 secs of "context exceeded" errors. The failures reported to us from customers don't have http2 errors in the kubelet logs, and our attempts to recreate with kubelet have never shown more more than a single error which works on subsequent retry.
Hi folks, we have a simulated recreate of the problem. This recreate involves injecting a 40 second delay in "connectivity" between the Kubernetes API server and etcd (we a 40 second delay to etcd defrag and defragged all etcd members at the same time). The connectivity between these two components is obviously critical to nodes successfully updating their leases in order to stay ready. We use 40 seconds because it matches . The test will usually cause one or more of the cluster nodes will flap their readiness state. Setting on the Kubernetes controller manager prevents the flap. Given this, I think that one or more of the Kubernetes default configurations needs to change in order to tolerate such "connectivity" problems in order to maintain healthy node state.
What's the proposal then? Do we update --node-monitor-grace-period to 50s? It seems the less risky change as the client-go defaults are more sensitive and impact the whole ecosystem, beyond kubernetes itself are you open to increase this default? It seems worth it to improve the reliability of cluster
I think we should update .
can someone send a PR to see how it will look like?
I have been following this discussion as we had encountered the bug as well. I have created a PR - Please let me know if it works
I'll add another data point from our testing. Namely, the simulated recreate noted depends on the Kubernetes controller manager (KCM) leader election configuration. Using the following leader election configuration for the KCM yields a KCM that has no leader after the delayed etcd defrag.

As a result, nodes won't be marked not ready unless the KCM leadership is restored before the nodes renew their lease. Extending the KCM leader election configuration to the following, makes the problem easier to recreate.
/assign Reassigning the issue to me


Question: command := exec.Command("mount", mountArgs...) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </del> <ins> return fmt.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </ins> err, source, target, fstype, options, string(output)) } return err

We have to spend times on figuring out what are the mount error code means from issues and . In , the error message is and is great for diagnosing mount problem. The console message better be returned rather than just error code.


Question: return filepath.Join(kl.getPodsDir(), string(podUID)) } <ins> // getPodVolumesSubpathsDir returns the full path to the per-pod subpaths directory under // which subpath volumes are created for the specified pod.  This directory may not // exist if the pod does not exist or subpaths are not specified. func (kl *Kubelet) getPodVolumeSubpathsDir(podUID types.UID) string { return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletVolumeSubpathsDirName) } </ins> // getPodVolumesDir returns the full path to the per-pod data directory under // which volumes are created for the specified pod.  This directory may not // exist if the pod does not exist.

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Network for all nodes of kubernetes cluster (29 hosts) was turned off for a day. Once the network was brought back, we've scaled down a statefulset that mounts a from local volume which on a root drive and then scaled it back up to previous number of replicas. kubelet on 26 out of 29 hosts proceeded with removing all data from subpaths of the pod. In fact, it was ongoing, such that when we'd create a file on a mounted subpath within the container, the file would be cleaned up within a couple of seconds. The issue would continue until volume-subpath was manually unmounted (with command). Ended up removing 3rd of all of our data (we have 3 drives per host and one was wiped) for the whole cluster. Equivalent definition. Note that host path of local volume has to be on root drive: Kubelet logs (filtered out logs from other pods) during the issue: What you expected to happen: Expected no data loss for peristent local volumes. How to reproduce it (as minimally and precisely as possible): Unfortunately was not able to reproduce. But this bug hit 26 out of 29 nodes simultaneously. Need some way to create a case where a pod becomes orphan and "volumes" folder gets somehow removed (similar to issue ). Then there is a code path that can lead to data loss: We are in orphan pod cleanup and directory is missing (not sure why it's printed twice): No error returned and empty volumes list, so we go into this case which proceeds removing pod directory and along with it even thought they might still be mounted (not sure why): checks if device is the same, but we use root drive for local volume, so the device is the same, so it's not a mount point so ok for removing (this is a bug IMHO). Anything else we need to know?: k8s version upgrades went from 1.9.6 -1.11.1 -1.12.1 -1.12.2 -1.13.0 without restarting the pods.

Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OVH, baremetal servers OS (e.g. from /etc/os-release):Kernel (e.g. ):Install tools: Using Ansible tasks that downloads kubelet from and all other required tools. Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
Thank you for digging into the code. I think there are a number of things that can be done here: why returned no volumes. why base volume was successfully unmounted. Unmounting the volume should have failed because the subpath bind mount would have kept a reference on it. to also look for the volume-subpaths directory to detect bind mounts.
I am working on a fix for 3 and 4
, do you mind to share us the full kubelet log? You can email me jinxu at Thanks!
why podVolumesExist returned no volumes. From the available log, we can see that Unmount volume and Unmount device succeeded. That's why podVolumeExist will return no volume.
/priority critical-urgent
That's unexpected. The subpath bind mount should have prevented the local volume bind mount from successfully unmounting.
Maybe the fact that the local volume mount is itself a bind mount could be the reason
about to why podVolumesExist returned no volumes. I think we do have an issue here podVolumesExist checks pod volume paths and whether they are mounted. Suppose it should stop cleaning up if there are paths still mounted. However, it only checks normal pod volumes /pod-id/volumes, it does not check subPath /pod-id/volume-subpath directories. We should fix this.
I'm adding a check for volume-subpath directories to address 3)
Jing found that the subpath volume directory uses the outer volume spec name, which is wrong. We made a fix for this in 1.11.0, 1.10.0, 1.9.7, 1.8.11, 1.7.16, which changed the on-disk directories to use the inner volume spec name.

had an that in-place node updates were not supported and you would have to drain your pods first. This explains why the volume cleanup proceeded without waiting for the subpath mount cleanup.
We haven't run a version below 1.9.6 on this node.
I've sent the entire kubelet log to your email
to clarify, 1.9.6 uses the old on-disk directory name, so upgrading to anything at least 1.11.0, 1.10.0, 1.9.7, 1.8.11, 1.7.16 requires pod draining first.
Right, I remember stumbling upon when debugging this. However, the release notes say: Shouldn't it specify exact version range where the OuterVolumeSpecName was used (1.9.4-1.9.6)? Unless I'm misreading release notes. I think it would be useful to clarify that.
ah you're right. The release notes should say from at least versions 1.7.14, 1.8.9, and 1.9.4
(will update it)
regardless, I still intend to add more safety checks to this orphaned pod cleanup function
From the log provided by the customer, I think we can confirm the reason of this issue is what mentioned By adding the safety checks during pod cleanup will help avoid this issue.


Question: go_test( name = "go_default_test", srcs = [ <ins> "cacher_whitebox_test.go", </ins> "selection_predicate_test.go", "time_budget_test.go", "util_test.go",

We noticed many goroutines stalled at in a large cluster (~200 nodes, ~1100 pods). It's because if the WatchServer timeout when the channel is full, will be blocked on the write to the even though is , because WatchServer stops consuming the channel after the timeout.
Great finding - thanks for debugging


Question: GO111MODULE=on go install k8s.io/klog/hack/tools/logcheck popd >/dev/null <del> # We store packaged that are migrated in .structured_logging </del> <ins> # We store packages that are migrated in .structured_logging </ins> migrated_packages_file="${KUBE_ROOT}/hack/.structured_logging" # Ensure that file is sorted.

For structured logging migration we have developed a tool to klog invocations For a given directory logcheck validates two things: directory is fully migrated - check if there are non-structured logging calls like usage of and is correct - check if key values passed are paired. We run logcheck as part of Kubernetes verify scripts on directories listed in Directories listed in are marked as migrated to avoid regressions from organized migration, however there are also other code locations that already use new functions that are not yet ready to be marked as migrated (components that are not part of organized migrations are done as best effort). We should split checking if directory was migrated from correctness of new functions and scan whole codebase for correctness. Plan: Add a flag to logcheck for requiring that directory was migrated. Add to verify-strucured- running logcheck of whole k/k codebase to check correctness This will ensure higher quality of migration PRs by validating correctness of directories that were not yet in .structuredlogging /wg structured-logging /cc
I'd like to pick this up. Do I need to wait till it is triage/accepted?
/triage accepted
Feel free to pick it up :P Usually I don't want to self triage, but this issue is purely Structured Logging WG related.
/assign
[Case 2: Correct usage] If the directory is not marked as migrated, we check that it's usage of is correct and don't care about usage of . By default, CI will test for correctness of whole k/k codebase instead of testing for complete migration (which is the current behavior). And to test for migration, a new flag must be passed explicitly. is this the expectation?
To clarify, let me rephrase. Current state is What we want: [Case 2: Correct usage] tested on whole repository
If this is the case, I can start implementing it.
SG, thanks for the help.
/milestone v1.22 Part of work go graduate Structured Logging KEP to beta.
I don't think work here has finished. We still need to implement those changes in k/k repo: Cut new klog Bump klog in k/k Update the script ./hack/verify-structured- /reopen
Reopened this issue. <detailsIn response to : I don't think work here has finished.

We still need to implement those changes in k/k repo: Update the script /reopen </details
Why do we need to cut a new klog and bump in k/k? Can't we make latest logcheck available to k/k without it? I don't know the process.
Hmm, We could try without updating to update. Please send PR that will update ./hack/verify-structured- If updating klog is nesesery feel free to contact for help
If we do both, we'd have to run logcheck twice (with and without flag) which means test time will increase.
This depends on implementation, increasing running time is ok as long it's reasonable, for example x1.5, but not x10.
Due to v1.22 code freeze, I'm moving this issue to the next release. /milestone v1.23
We would like to have static analysis setup before we start migrating logs in 1.23. will you be able to work on it this week? Do you need any help?
Yes, I can. PR kubernetes/kubernetes/pull/ is up for enabling static analysis. It has some unresolved conversations. It'd be helpful if I could get another round of review on that PR. Before we merge that we'd need to fix existing logging issues which are listed here: If I could get help fixing those it'd be great. Else I can fix it too.
I see two issues that need to be addressed: Fix handling of in klog code, this should be done by creating an issue to klog () Fix remaining errors Upgrade klog version to pick up fix for Exit


Question: newInitializers: &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init.k8s.io"}}}, err:             "field is immutable once initialization has completed", }, <ins> { name:            "empty initializer list is treated as nil initializer", oldInitializers: nil, newInitializers: &metav1.Initializers{}, verifyUpdatedObj: func(obj runtime.Object) (bool, string) { accessor, err := meta.Accessor(obj) if err != nil { return false, "cannot get accessor" } if accessor.GetInitializers() != nil { return false, "expect nil initializers" } return true, "" }, err: "", }, </ins> } plugin := initializer{

(SEE UPDATE at ) Currently, if you're writing an initializer controller, you're most likely using it to initialize a and deploying the controller with . Therefore you're likely using the following trick to avoid the chicken and egg problem: This bypasses the initializers for the pods and lets your controller to be deployed even though your InitializerConfiguration is configured to capture all pods. Problem occurs when you apply an update to this Deployment with the same manifest containing empty pending list. You get error: I think this is a somewhat major UX issue. It requires users to deploy the initializer controller with the but later on when they need to update the controller with , they can't use the same manifest containing the empty list. /area api /area extensibility /sig api-machinery /kind bug /assign
/cc
We silently ignore some metadata updates, which is one option. Potentially we shouldn't require empty initializers pending, but simply empty initializers (which means - init directly), and only ignore that. Are there any examples of use of SMP for similar? I'm not sure this meets the bar for 1.8 but if we can find an example of something consistent already in the API I'd be willing to review.
I don't have any concrete examples, there's an example initializer at making use the trick, I use it myself, and I suspect many more initializers to come will use this, too. I'm not sure if this would work, but does setting provide a way to skip initialization (if possible at all)? If not, we might actually need a separate field to signal skipping initialization.
/cc
Currently the initializer plugin to on UPDATE, but that happens after it updating already initialized objects. We can simply change their order.
I'd like to amend the issue, it's more serious than I thought. (thanks for bringing it to my attention If you're deploying your Initializer with and if you have an initializerconfiguration for objects, it's impossible to deploy your initializer in the first place without deleting InitializerConfiguration first: is not inherited by the created on is not honored (this field is not propagated to the created Pod). I think (2) should be fixed somehow, otherwise there will be no way of writing an initializer for Pods and deploying the initializer itself using Deployment/ReplicaSet/RC without deleting the InitializerConfiguration first.

Maybe what we need is an exclusion/exemption selector (or namespace(s)) as discussed in to allow deployment of Initializers themselves easier.
For 1, see I don't want to pursue that though, because i) we'll need to fix workload controllers one by one, and ii) we need to carefully consider permission issues. I think we should focus on adding the namespace selector.


Question: scale MUST be at two replicas. release: v1.16, v1.21 file: test/e2e/apps/statefulset.go <ins> - testname: StatefulSet, list, patch and delete a collection of StatefulSets codename: '[sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets [Conformance]' description: When a StatefulSet is created it MUST succeed. It MUST succeed when listing StatefulSets via a label selector. It MUST succeed when patching a StatefulSet. It MUST succeed when deleting the StatefulSet via deleteCollection. release: v1.22 file: test/e2e/apps/statefulset.go </ins> - testname: StatefulSet, Rolling Update with Partition codename: '[sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should perform canary updates and phased rolling updates of template modifications

[X] test pr : kuberenetes/kubernetes 17 June 2021 [x] two weeks soak end date : 1 July 2021 [X] test promotion pr : kubernetes/kubernetes According to this APIsnoop query, there are still some remaining StatafulSet endpoints that are untested. with this query, you can filter untested endpoints by their category and eligibility for conformance. e.g below shows a query to find all conformance eligible untested, stable, core endpoints Create a StatefulSet Patch the StatefulSet List all StatefulSet in the namespace Delete the collection if StatefulSets in the namespace Verify that all StatefulSet was deleted Due to the complexity of setting up the resources for APIService we have used the current e2e test as a template. It has been extended in a for review.

example endpoint | useragent ---------------------------------------------|
createAppsV1NamespacedStatefulSet | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets deleteAppsV1CollectionNamespacedStatefulSet | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets listAppsV1NamespacedStatefulSet | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets listAppsV1StatefulSetForAllNamespaces | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets patchAppsV1NamespacedStatefulSet | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets readAppsV1NamespacedStatefulSet | (linux/amd64) kubernetes/$Format -- [sig-apps] StatefulSet Basic StatefulSet functionality [StatefulSetBasic] should list, patch and delete a collection of StatefulSets replaceAppsV1NamespacedStatefulSetStatus | kube-controller-manager/v1.20.4 (linux/amd64) kubernetes//system:serviceaccount:kube-system:statefulset-controller (7 rows) If a test with these calls gets merged, test coverage will go up by 3 points This test is also created with the goal of conformance promotion. /sig apps /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return nil } <ins> func waitForMultiPathToExist(devicePaths []string, maxRetries int, deviceUtil volumeutil.DeviceUtil) string { if 0 == len(devicePaths) { return "" } for i := 0; i < maxRetries; i++ { for _, path := range devicePaths { // There shouldnt be any empty device paths. However adding this check // for safer side to avoid the possibility of an empty entry. if path == "" { continue } // check if the dev is using mpio and if so mount it via the dm-XX device if mappedDevicePath := deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" { return mappedDevicePath } } if i == maxRetries-1 { break } time.Sleep(time.Second) } return "" } </ins> // AttachDisk returns devicePath of volume if attach succeeded otherwise returns error func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { var devicePath string

/kind bug /sig storage What happened: When mounting iSCSI PV kubelet doesn't give multipathd enough time to create multipath device and fallbacks to mounting single SCSI device What you expected to happen: it uses multipath device to mount How to reproduce it (as minimally and precisely as possible): create ~5 pods on a single node, each with dynamically provisioned PV (we use Trident for that), observe how some mounts are /dev/sd and not /dev/mapper/ Environment: Kubelet 1.9.3 Baremetal + NetApp ONTAP ove iSCSI + Trident dynamic volume provisioner CoreOS (kernel 4.14.24, multipath-tools 0.6.4)
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
I can reproduce this issue fairly often (although not reliably). The problem is in in the AttachDisk() function. The basic problem is that the method that finds the multipath device looks at each device path one time, and if none of them have an associated multipath device, it just falls back to formatting and mounting the first device path. There is no wait loop for the multipath device to appear. It would be easy to add a wait loop, but that would slow down the non-multipath use case. A possible optimization would be to only wait when 2 or more paths are known, but that still opens up the possibility of incorrect behavior when attaching while all but one paths are down, or when only one path is discovered (for whatever reason).
After thinking about this over lunch, I think a possible fix would be: add a 10 second wait loop for the multipath device if and only if the PV are 2 or more target portals specified. This puts control in the hands of the provisioner without modifying the API at all, and not breaking backwards compatibility. The one corner case I can see is what would happen the PV has 2 portals, but the system is only able to log in to one.

This would occur when multipath was correctly configured on the storage controller and in the provisioner, but one of the paths was down (for whatever reason) at attach time. I'm inclined to make the attachment FAIL if the multipath device can't be found within 10 seconds, which would be a change to current behavior, but this can easily be worked around by the cluster administrator setting up multipathd to create devices for single path disks. Provisioners would want to document the recommended way to do that (i.e. set find_multipaths = no, and configure the blacklist appropriately).
is vaguely described as "target portal ips for high availability", without describing what the "high availability" means. Should we assume that it means multipath? Then Kubernetes can always wait for multipath device to appear when more that one is specified. Would it break anything? Does anyone run multiple portals for the same volume without multipath? Is it something we should support? any opinion?
yes, the iscsi portals were made possible by for HA/mpio use case (see ). The lack of concrete description makes it less obvious though.
was it working perfectly with older kube versions and the issue started to pop up recently or from ? Why I am asking is, multipath support was in kube () and atleast I havent seen this issue reported till this report or very recently. Gluster team were also using multipathing feature for quite some time where we always got mpath device mounted reliably, but noticed this issue very recently or with latest OCP builds. iic, There are good number of changes happened in between especially on device discovery/attach ( ex: ) disk paths. I am not concretely saying its a regression or a behavioral change or side effect of some other patch, but trying to figure out if thats the case. any thoughts here ? my understanding was/is that with more target portal IPs means ( atleast) in ISCSI context, but we could have mention as well in the description. Suppose the backup portal IPs are filled by provisioner/admin but initiator havent enabled ( dm_mulitpath is not loaded or deamon service is not running) software in the kubelet/node, what will happen ? Is it going to be infinite wait for failure ?

my understanding is that, if multipath is enabled, even if it failed ( for some reason) for backup paths, a mpath device will be created for single path in the system. If thats the case above shouldnt be a concern and device mapper layer will take care of the path discovery later when path becomes stable/available. I dont think so.
we started with 1.9.x on kubelets, so can't tell much about mounting behaviour of previous versions. it is going to fail, but it is fine, nodes should be correctly configured. Lets say you specify fsType, but given FS module is not loaded by kernel, or now mkfs.<FSbinary is available to the kubelet, mount is going to fail and thats OK IMHO. Kubelet shouldn't try to fallback to anything less than was asked, mounting via single portal when multiple portals are specified is similar to trying "default FS" if required fsType can't be used. Yes, default configuration for multipathd is option (note, that default in Linux distributions might be different), which creates multipath device for every non-blacklisted block device.
I'm fairly confident that adding a wait loop for the multipath device in the case when there are 2 or more portals in the PV spec won't cause any regressions. The thing that gives me pause is that there are cases when there's a single portal in the PV spec, but the device still supports multipath and the existing sendtargets command is the mechanism for finding those other paths. I'm wondering if it's possible to detect that case (reliably) and use a wait loop for the multipath device in those cases too.
is the default on RHEL 7, see
Did you mean Multiple Connection Per Session (MC/S)? I don't think this is supported in open-iscsi. Open-iSCSI only supports MPIO (multiple portals)
I don't understand the move by RedHat to default to findmultipaths = no. It basically guarantees bad behavior in cloud-oriented use cases when there's a temporary path failure. My reading of the linked document is that the main motivation for changing the default is to avoid the need to setup a correct blacklist. It seems to me that correct behavior is more important that avoiding some work up front when configuring multipath. No, I'm referring to the .

If a storage backend has multiple target portals, but the dynamic provisioner only puts one portal in the PV spec, then as long as that one portal is accessible at attach time, open-iscsi will discover the other portals and log into them. The only weakness to this approach is what happens when that one path is inaccessible at attach time due to a network switch/network port/NIC failure, while the other ports are still available.
, AFAIK multipathd (binary) in RHEL 7 defaults to , but configuration utility they ship writes config with , that utility needs to be called explicitly , IMHO that should be direct scsi mount unconditionally, without multipath involved
The thing that gives me pause is that there are cases when there's a single portal in the PV spec I would be inclined to agree, except that someone the sendtargets logic to the iscsi code. This code exists for no reason other than to allow finding multiple path given a single path. In fact the existing NetApp dynamic provisioner (Trident) relies on that behavior currently to get multipath support (this is something I've fixed and will upstream shortly). If we decide that PVs with exactly one portal should not support multipath, then we should remove that sendtargets logic from the iscsi module in kubelet.


Question: "k8s.io/kubernetes/test/e2e/framework" e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" <del> e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem" </del> e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"

master-informing: ! 2023-09-07 03:13:59 +0000 UTC No response /sig network
Checking this failure This is running ingress-gce version, logs in failure is It seems the healthceck interval is not updated correctly, by reading the test tagging the people with more expertise in this area /cc
/cc
it seems the test restarts the kube-controller-manager but we landed recently the switch from intree to outtree cloud providers, so this logic now runs in the cloud-controller-manager, can this be the problem, that we have to restart the cloud-controller-manager now instead of the kcm?
/triage accepted This flake started after and kept failing after 09-07.
yeah, the logic that handles the loadbalancer shifted to that cloud controller manager with that PR


<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Mistake use obj as prevObj prevEndpointSlice := obj.(discovery.EndpointSlice) What you expected to happen: prevEndpointSlice := prevObj.(discovery.EndpointSlice) How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: cc
/sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved PR for fixing it Thanks


Question: wantCfg := token2Cfg(token) persistedCfg := persiter.Cache() <ins> wantCfgLen := len(wantCfg) persistedCfgLen := len(persistedCfg) if wantCfgLen != persistedCfgLen { t.Errorf("wantCfgLen and persistedCfgLen do not match, wantCfgLen=%v, persistedCfgLen=%v", wantCfgLen, persistedCfgLen) } </ins> for k, v := range persistedCfg { if strings.Compare(v, wantCfg[k]) != 0 { t.Errorf("Token() persisted cfg %s: got %v, want %v", k, v, wantCfg[k])

What happened: AAD support in kubectl does not work for sovereign cloud The 'environment' parameter was introduced to support sovereign cloud, but it works in first round. The 'environment' value is not persisted, so follow-up requests will fail. What you expected to happen: 'environment' parameter persisted in kubectl config file. How to reproduce it (as minimally and precisely as possible): Use kubectl to connect to Azure sovereign cloud, make the token expire, and do auth. Anything else we need to know?: Environment: Kubernetes version (use ): 1.11 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details