Question: <ins> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package apps import ( "context" "fmt" "time" "github.com/onsi/ginkgo" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/retry" extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/controller" labelsutil "k8s.io/kubernetes/pkg/util/labels" "k8s.io/kubernetes/test/e2e/framework" e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2eresource "k8s.io/kubernetes/test/e2e/framework/resource" admissionapi "k8s.io/pod-security-admission/api" ) const ( controllerRevisionRetryPeriod  = 1 * time.Second controllerRevisionRetryTimeout = 1 * time.Minute ) // This test must be run in serial because it assumes the Daemon Set pods will // always get scheduled.  If we run other tests in parallel, this may not // happen.  In the future, running in parallel may work if we have an eviction // model which lets the DS controller kick out other pods to make room. // See http://issues.k8s.io/21767 for more details var _ = SIGDescribe("ControllerRevision [Serial]", func() { var f *framework.Framework ginkgo.AfterEach(func() { // Clean up daemonsets, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err, "unable to dump DaemonSets") if daemonsets != nil && len(daemonsets.Items) > 0 { for _, ds := range daemonsets.Items { ginkgo.By(fmt.Sprintf("Deleting DaemonSet %q", ds.Name)) framework.ExpectNoError(e2eresource.DeleteResourceAndWaitForGC(f.ClientSet, extensionsinternal.Kind("DaemonSet"), f.Namespace.Name, ds.Name)) err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, &ds)) framework.ExpectNoError(err, "error waiting for daemon pod to be reaped") } } if daemonsets, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}); err == nil { framework.Logf("daemonset: %s", runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...), daemonsets)) } else { framework.Logf("unable to dump daemonsets: %v", err) } if pods, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}); err == nil { framework.Logf("pods: %s", runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...), pods)) } else { framework.Logf("unable to dump pods: %v", err) } err = clearDaemonSetNodeLabels(f.ClientSet) framework.ExpectNoError(err) }) f = framework.NewDefaultFramework("controllerrevisions") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline image := WebserverImage dsName := "e2e-" + utilrand.String(5) + "-daemon-set" var ns string var c clientset.Interface ginkgo.BeforeEach(func() { ns = f.Namespace.Name c = f.ClientSet updatedNS, err := patchNamespaceAnnotations(c, ns) framework.ExpectNoError(err) ns = updatedNS.Name err = clearDaemonSetNodeLabels(c) framework.ExpectNoError(err) }) ginkgo.It("should manage the lifecycle of a ControllerRevision", func() { csAppsV1 := f.ClientSet.AppsV1() dsLabel := map[string]string{"daemonset-name": dsName} dsLabelSelector := labels.SelectorFromSet(dsLabel).String() ginkgo.By(fmt.Sprintf("Creating DaemonSet %q", dsName)) testDaemonset, err := csAppsV1.DaemonSets(ns).Create(context.TODO(), newDaemonSetWithLabel(dsName, image, dsLabel), metav1.CreateOptions{}) framework.ExpectNoError(err) ginkgo.By("Check that daemon pods launch on every node of the cluster.") err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, testDaemonset)) framework.ExpectNoError(err, "error waiting for daemon pod to start") err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("Confirm DaemonSet %q successfully created with %q label", dsName, dsLabelSelector)) dsList, err := csAppsV1.DaemonSets("").List(context.TODO(), metav1.ListOptions{LabelSelector: dsLabelSelector}) framework.ExpectNoError(err, "failed to list Daemon Sets") framework.ExpectEqual(len(dsList.Items), 1, "filtered list wasn't found") ds, err := c.AppsV1().DaemonSets(ns).Get(context.TODO(), dsName, metav1.GetOptions{}) framework.ExpectNoError(err) // Listing across all namespaces to verify api endpoint: listAppsV1ControllerRevisionForAllNamespaces ginkgo.By(fmt.Sprintf("Listing all ControllerRevisions with label %q", dsLabelSelector)) revs, err := csAppsV1.ControllerRevisions("").List(context.TODO(), metav1.ListOptions{LabelSelector: dsLabelSelector}) framework.ExpectNoError(err, "Failed to list ControllerRevision: %v", err) framework.ExpectEqual(len(revs.Items), 1, "Failed to find any controllerRevisions") // Locate the current ControllerRevision from the list var initialRevision *appsv1.ControllerRevision rev := revs.Items[0] oref := rev.OwnerReferences[0] if oref.Kind == "DaemonSet" && oref.UID == ds.UID { framework.Logf("Located ControllerRevision: %q", rev.Name) initialRevision, err = csAppsV1.ControllerRevisions(ns).Get(context.TODO(), rev.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to lookup ControllerRevision: %v", err) framework.ExpectNotEqual(initialRevision, nil, "failed to lookup ControllerRevision: %v", initialRevision) } ginkgo.By(fmt.Sprintf("Patching ControllerRevision %q", initialRevision.Name)) payload := "{"metadata":{"labels":{"" + initialRevision.Name + "":"patched"}}}" patchedControllerRevision, err := csAppsV1.ControllerRevisions(ns).Patch(context.TODO(), initialRevision.Name, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch ControllerRevision %s in namespace %s", initialRevision.Name, ns) framework.ExpectEqual(patchedControllerRevision.Labels[initialRevision.Name], "patched", "Did not find 'patched' label for this ControllerRevision. Current labels: %v", patchedControllerRevision.Labels) framework.Logf("%s has been patched", patchedControllerRevision.Name) ginkgo.By("Create a new ControllerRevision") newHash, newName := hashAndNameForDaemonSet(ds) newRevision := &appsv1.ControllerRevision{ ObjectMeta: metav1.ObjectMeta{ Name:            newName, Namespace:       ds.Namespace, Labels:          labelsutil.CloneAndAddLabel(ds.Spec.Template.Labels, appsv1.DefaultDaemonSetUniqueLabelKey, newHash), Annotations:     ds.Annotations, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ds, appsv1.SchemeGroupVersion.WithKind("DaemonSet"))}, }, Data:     initialRevision.Data, Revision: initialRevision.Revision + 1, } newControllerRevision, err := csAppsV1.ControllerRevisions(ns).Create(context.TODO(), newRevision, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create ControllerRevision: %v", err) framework.Logf("Created ControllerRevision: %s", newControllerRevision.Name) ginkgo.By("Confirm that there are two ControllerRevisions") err = wait.PollImmediate(controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, checkControllerRevisionListQuantity(f, dsLabelSelector, 2)) framework.ExpectNoError(err, "failed to count required ControllerRevisions") ginkgo.By(fmt.Sprintf("Deleting ControllerRevision %q", initialRevision.Name)) err = csAppsV1.ControllerRevisions(ns).Delete(context.TODO(), initialRevision.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "Failed to delete ControllerRevision: %v", err) ginkgo.By("Confirm that there is only one ControllerRevision") err = wait.PollImmediate(controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, checkControllerRevisionListQuantity(f, dsLabelSelector, 1)) framework.ExpectNoError(err, "failed to count required ControllerRevisions") listControllerRevisions, err := csAppsV1.ControllerRevisions(ns).List(context.TODO(), metav1.ListOptions{}) currentControllerRevision := listControllerRevisions.Items[0] ginkgo.By(fmt.Sprintf("Updating ControllerRevision %q", currentControllerRevision.Name)) var updatedControllerRevision *appsv1.ControllerRevision err = retry.RetryOnConflict(retry.DefaultRetry, func() error { updatedControllerRevision, err = csAppsV1.ControllerRevisions(ns).Get(context.TODO(), currentControllerRevision.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get ControllerRevision %s", currentControllerRevision.Name) updatedControllerRevision.Labels[currentControllerRevision.Name] = "updated" updatedControllerRevision, err = csAppsV1.ControllerRevisions(ns).Update(context.TODO(), updatedControllerRevision, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "failed to update ControllerRevision in namespace: %s", ns) framework.ExpectEqual(updatedControllerRevision.Labels[currentControllerRevision.Name], "updated", "Did not find 'updated' label for this ControllerRevision. Current labels: %v", currentControllerRevision.Labels) framework.Logf("%s has been updated", updatedControllerRevision.Name) ginkgo.By("Generate another ControllerRevision by patching the Daemonset") patch := fmt.Sprintf(`{"spec":{"template":{"spec":{"terminationGracePeriodSeconds": %d}}},"updateStrategy":{"type":"RollingUpdate"}}`, 1) _, err = c.AppsV1().DaemonSets(ns).Patch(context.TODO(), dsName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) framework.ExpectNoError(err, "error patching daemon set") ginkgo.By("Confirm that there are two ControllerRevisions") err = wait.PollImmediate(controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, checkControllerRevisionListQuantity(f, dsLabelSelector, 2)) framework.ExpectNoError(err, "failed to count required ControllerRevisions") updatedLabel := map[string]string{updatedControllerRevision.Name: "updated"} updatedLabelSelector := labels.SelectorFromSet(updatedLabel).String() ginkgo.By(fmt.Sprintf("Removing a ControllerRevision via 'DeleteCollection' with labelSelector: %q", updatedLabelSelector)) err = csAppsV1.ControllerRevisions(ns).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: updatedLabelSelector}) framework.ExpectNoError(err, "Failed to delete ControllerRevision: %v", err) ginkgo.By("Confirm that there is only one ControllerRevision") err = wait.PollImmediate(controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, checkControllerRevisionListQuantity(f, dsLabelSelector, 1)) framework.ExpectNoError(err, "failed to count required ControllerRevisions") list, err := csAppsV1.ControllerRevisions(ns).List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err, "failed to list ControllerRevision") framework.ExpectEqual(list.Items[0].Revision, int64(3), "failed to find the expected revision for the Controller") framework.Logf("ControllerRevision %q has revision %d", list.Items[0].Name, list.Items[0].Revision) }) }) func checkControllerRevisionListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) { return func() (bool, error) { var err error framework.Logf("Requesting list of ControllerRevisions to confirm quantity") list, err := f.ClientSet.AppsV1().ControllerRevisions(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{ LabelSelector: label}) if err != nil { return false, err } if len(list.Items) != quantity { return false, nil } framework.Logf("Found %d ControllerRevisions", quantity) return true, nil } } func hashAndNameForDaemonSet(ds *appsv1.DaemonSet) (string, string) { hash := fmt.Sprint(controller.ComputeHash(&ds.Spec.Template, ds.Status.CollisionCount)) name := ds.Name + "-" + hash return hash, name } </ins>

[X] Test approval issue: [X] Test PR: 8 Jul 2022 [x] Two weeks soak end date: 22 Jul 2022 [x] Test promotion PR: According to this APIsnoop query, there are still some remaining Controller Revision endpoints which are untested. listAppsV1ControllerRevisionForAllNamespacesreadAppsV1NamespacedControllerRevisionpatchAppsV1NamespacedControllerRevisioncreateAppsV1NamespacedControllerRevisiondeleteAppsV1NamespacedControllerRevisionreplaceAppsV1NamespacedControllerRevisiondeleteAppsV1CollectionNamespacedControllerRevision Using a number of existing e2e test practices a new has been created for 7 ControllerRevision endpoints. The e2e logs for this test are listed below. This query shows the following endpoints are hit within a short period of running this e2e test If a test with these calls gets merged, test coverage will go up by 7 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: // Make a placeholder channel to use when there are no items in our list never := make(<-chan time.Time) <ins> // Make a timer that expires when the item at the head of the waiting queue is ready var nextReadyAtTimer clock.Timer </ins> waitingForQueue := &waitForPriorityQueue{} heap.Init(waitingForQueue)

Hi, We are experiencing a memory leak in our operator and it seems to be caused by some Timers: ! See here the original issue: After some investigations I found out that the can create a lot of timers (at least ) if the queue is not empty : The problem here is that if the event in the queue is to expire in a "far" future the previous timers will pile up and the memory will be reclaimed too late. Could someone confirm this scenario ? A fix would be to create some with and close them when they are not needed (e.g. is called) or superseded by a most recent one. Happy to discuss the issue and contribute if needed.
thanks for the report, moved to the main repo for better visibility /sig api-machinery
yes, using NewTimer would be better here
thanks for your feedback, could you assign me to this issue ? I will try to provide a fix.
Thanks, I have opened a PR:


Question: { Name:  "webserver", Image: "kubernetes/nettest:1.1", <del> Command: []string{ </del> <ins> Args: []string{ </ins> "-service=" + name, fmt.Sprintf("-peers=%d", peers), "-namespace=" + ns},

Forked from downward, this test has been failing since go/k8s-test/kubernetes-e2e-gce/4042, which lines up well with . Assigning to to fix, though a revert may be in order soon (we've had too much noise).
The issue is that I didn't change the networking e2e to set 'Args' instead of 'Command'.
Clients shouldn't be using the internal representation.
hmmmm ... is this really fixed? not working for me atm.
What behaviors are you seeing?
This is a recent run of our Jenkins server:
Re-closing (that was a success I pasted).
This might be the test failure you mentioned over hangout. The detail information can be found at: The issue is fixed this morning. Thanks!


Question: nodes := makeLabeledNodeList(test.nodes) snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes)) serviceLister := fakelisters.ServiceLister(test.services) <del> priorityMapFunction, priorityReduceFunction := priorities.NewServiceAntiAffinityPriority(snapshot.Pods(), serviceLister, test.labels) </del> p := &ServiceAffinity{ <del> sharedLister:           snapshot, serviceLister:          serviceLister, priorityMapFunction:    priorityMapFunction, priorityReduceFunction: priorityReduceFunction, </del> <ins> sharedLister:  snapshot, serviceLister: serviceLister, args: Args{ AntiAffinityLabelsPreference: test.labels, }, </ins> } metaDataProducer := priorities.NewMetadataFactory( fakelisters.ServiceLister(test.services),

/sig scheduling /priority important-soon See as an example. /assign
/unassign /help
/assign
/remove-help


Question: framework.DeletePodWithWait(f, f.ClientSet, pod) }() By("Checking for subpath error in container status") <del> err = waitForPodSubpathError(f, pod) </del> <ins> err = waitForPodSubpathError(f, pod, allowContainerTerminationError) </ins> Expect(err).NotTo(HaveOccurred(), "while waiting for subpath failure") }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: The read only sig-storage tests validate that Pods can come up when readOnly is set. But it doesn't actually validate that the container cannot write to the volume. What you expected to happen: Tests should validate that the container cannot write to readonly volumes. <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/help
This request has been marked as needing help from a contributor. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /help Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
can this issue be closed after your last PR or is there more to work on? If yes I'd like to know what's left because I'm looking for something to work on
I think we should actually reinstate the test case I removed and enhance it to handle the two scenarios I mentioned. It would also be good to see if we have other readOnly test cases that could be enhanced as well
Thanks ! I have a few questions: 1) how do I run the tests in locally? and 2) how can I assert writing to volume has failed? Thank you in advance!!


Question: import ( "k8s.io/kubernetes/pkg/apis/extensions" <ins> unversionedclient "k8s.io/kubernetes/pkg/client/unversioned" </ins> "k8s.io/kubernetes/pkg/controller" <ins> rsutil "k8s.io/kubernetes/pkg/util/replicaset" "k8s.io/kubernetes/pkg/util/wait" </ins> ) // rolloutRecreate implements the logic for recreating a replica set.

The scenario is a single replica with an attached EBS. The create seems to happen right after the delete not waiting for the pod to be terminated. This results in the two kubelets fighting over the EBS which generally results in AWS becoming unhappy meaning the EBS has to be force detached. From the docs I would have thought deployments would wait until the pod is actually killed. Is this a bug or am i misinterpreting the docs?
As a work around, what happens if you use a rolling update with MaxSurge=0 and MaxUnavailable=1?
Just assigned for triage, not to fix the issue. Can clear your assignment once this is triaged.
Deployments with size of 1 should be the same (both in that it's really a Recreate and it has the same problem even though it only matters for Recreate), larger deployments should be rolling.
It's because Deployment scales old RS down to 0 (=0, and RS controller will observe this change later and then kill its pods) and then scales up the new one. So it's possible there's a race here.
Opened that partially fixes this.
Why it's closed? It's not fixed: Recreate still doesn't wait for termination.
what version are you using?
1.6.2-00 from deb package. I'm changing the Deployment to use different configmap. K8s recreates pod, but the new one can't listen on ports that are probably used by the old one (using ). Both are appearing in at the same time (old one is 'Terminating').
This regressed in 1.6 but was fixed again in I don't think it was cherry-picked back to 1.6 though and the cherry-pick is not clean. are you interested in backporting it to the release-1.6 branch?
the cherry-pick isn't clean mostly because of comments. Don't have time to actually build and test the PR though.
I think this is still an issue:
I cannot reproduce it anymore, can you provide a reproducer?
Not easily at the moment -- we ended up changing our pods to handle multiple running simultaneously gracefully.
Hi All, I'm still seeing this on the following version: I am using recreate and can see my current pod go into a terminating state and my new pod come up.

It is my understanding that the new pod should not come up until the old pod is terminated completely.
Seeing this on 1.14.3 too - AWS hardware went bad and stuff was in terminating (but still running) and our stateful singleton was brought back up alongside
We see the same problem with 1.14.3 where the old pod is Terminating and the new pod is in ContainerCreating even if we set the Got problem with Volume is already used by pod(s)...
The easiest way I've found to reproduce this is by using bash interactively - it swallows SIGTERM giving a bigger window to observe the behavior. One thing to note, the behavior is different with an explicit delete and one caused by a changing annotation. When changing an annotation, the new pod won't be scheduled until the old one is completely gone, but an explicit delete will create the new pod right away. That said, I don't know how it will behave with other events, say a node going away or being drained. Here is a way to reproduce it. Delete the pod () and watch the new one spin up right away.
recreate deployments wait for the termination of an old version only during a deployment upgrade. The deployment api has been designed with availability in mind and cannot ensure consistency during node upgrades (use PDBs), unexpected node shutdowns, or any other reason that could cause a pod to be recreated (apart from deployment upgrades). If your application has a hard requirement on consistency (eg. database), use . can you please provide a reproducer? Thanks.
Same issue here with version v1.17.6: Deployment with "Recreate" strategy: If I try to delete a pod, the old pod is still Terminating, the new pod has already get initialized and Running:
/reopen
You can't reopen an issue/PR unless you authored it or you are a collaborator. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
see the comment right above you "recreate deployments wait for the termination of an old version only during a deployment upgrade.

The deployment api has been designed with availability in mind and cannot ensure consistency during node upgrades (use PDBs), unexpected node shutdowns, or any other reason that could cause a pod to be recreated (apart from deployment upgrades). If your application has a hard requirement on consistency (eg. database), use statefulsets."


Question: "strings" "testing" <ins> "github.com/google/go-cmp/cmp" </ins> "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" <ins> v1 "k8s.io/api/core/v1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --my mutating webhook calculates a hash of the , allows reinvocation; no side effects my validating webhook validates the hash of the and fails because the has changed webhook auditing indicates that my webhooks are the only ones processing the upon serializing the spec to JSON for both the mutator and validator, the only change is related to the "kube-api-access-xyz" projected volume, managed by a built-in k8s admission plugin the aforementioned plugin does not appear to mutate the reinvocation context, even through it may mutate a pod (robbing other mutators of the chance to observe the change): it appears that both my hooks see the injected volume, but the validation hook sees the injected SA volume with a "defaultMode" field set, and my mutation hook does NOT see this field set my validation hook should calculate the same hash mutating, validating webhooks are implemented atop controller-runtime v0.8.3, pod objects are being decoded into targets Kubernetes version (use ): 1.21.3 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g.

): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig api-machinery
cc
interesting... defaults are applied when going from versioned → internal server decodes v1 Pod request → applies defaults → converts to internal Pod -in admission control operates on the internal Pod service account admission injects the kube-api-access volume admission is called internal Pod → converts to v1 Pod → serializes → sends to mutating webhook mutating webhook returns patch patch is applied to v1 Pod JSON resulting v1 Pod JSON is decoded → defaults applied (defaultMode is set on the api-token volume here) → converted to internal Pod webhook admission is called internal Pod → converts to v1 Pod → serializes → sends to validating webhook That means if any built-in admission plugin sets up data that would be modified by applying defaulting to the pod, the data sent to mutating webhooks will be missing those applied defaults until the first mutating webhook that returns a patch that changes the object
Possible solutions: defaults are applied to the internal → versioned object before invoking webhooks built-in admission always populates data matching default values for unset fields
As an aside, I'm not sure what you're using the hashing for, but be aware that hashing is fragile in the presence of defaults and new fields over time. A hash you add to a pod spec in one release might not match a hash computed in a different release, or with different admission in place.
/triage accepted /assign /priority backlog


Question: flags.DurationVar(&nodeKiller.SimulatedDowntime, "node-killer-simulated-downtime", 10*time.Minute, "A delay between node death and recreation") } <del> // RegisterNodeFlags registers flags specific to the node e2e test suite. func RegisterNodeFlags(flags *flag.FlagSet) { // Mark the test as node e2e when node flags are api.Registry. TestContext.NodeE2E = true flags.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on.") // TODO(random-liu): Move kubelet start logic out of the test. // TODO(random-liu): Move log fetch logic out of the test. // There are different ways to start kubelet (systemd, initd, docker, manually started etc.) // and manage logs (journald, upstart etc.). // For different situation we need to mount different things into the container, run different commands. // It is hard and unnecessary to deal with the complexity inside the test suite. flags.BoolVar(&TestContext.NodeConformance, "conformance", false, "If true, the test suite will not start kubelet, and fetch system log (kernel, docker, kubelet log etc.) to the report directory.") flags.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") flags.StringVar(&TestContext.ImageDescription, "image-description", "", "The description of the image which the test will be running on.") flags.StringVar(&TestContext.SystemSpecName, "system-spec-name", "", "The name of the system spec (e.g., gke) that's used in the node e2e test. The system specs are in test/e2e_node/system/specs/. This is used by the test framework to determine which tests to run for validating the system requirements.") flags.Var(cliflag.NewMapStringString(&TestContext.ExtraEnvs), "extra-envs", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") } </del> func createKubeConfig(clientCfg *restclient.Config) *clientcmdapi.Config { clusterNick := "cluster" userNick := "user"

What happened: During the work for reducing invalid dependencies from the e2e core framework to subpackages of the core framework, we found a lot of particular test specific functions. That means these each function is used at a single e2e test module only. These functions are unnecessary to be shared as e2e framework function, so we've just moved these functions to the particular test module like This cleanup work will help us to maintain the e2e core framework in long term, so it is helpful to share this work in volunteers as a community. This issue is for tracking this working items. How to do that?: volunteer selects a single file which is empty (no PR, no assignee) from the following e2e core framework all functions in the file all callers of each function finding the function is called at a single e2e test package only, let's move the function to the module with a PR Implementation rules: If a function is called from 2+ test packages, don't move the function from the core package. If moving of the function requires invalid code dependencies, don't move the function. (Dependency from e2e core framework to other repository) List of files of e2e core framework: [x] ( ) [x] () [x] ( [x] () [x] [x] () [x] get-kubemark-resource- [x] ( ) [x] (Done) [x] ( ) [x] () [x] ( ) [x] ( ) [x] ( ) [x] ( ) [x] ( ) [x] ( ) [x] () [x] ( ) [x] () [x] () References: This working item was under discussion as
/kind cleanup /remove-kind bug /area e2e-test-framework /sig testing /priority important-longterm
Thanks for taking care of this quickly :-)
The proposed policy to follow for what methods are to be kept within the framework: What do you think ? does that sound good?
Thanks for bring it up here. As I said on the slack, I agree with his idea.
I looked through files , and , and found some of the functions are used in a single module. Then fixed them.

i will pick up
I will pick
I will pick up
/assign I will pick
i will pick and
I will pick up ,,,
The PR for is already created by . Please check what other people picked already before posting a PR.
is already picked/posted by
That is a nice point. I wrote point as one of the above Implementation Rules, and another rule comes from which adds another invalid dependency.
I didn't noticed that already picked by , sorry about that. After looking around, there is noting need to do with: ,,,,(not picked). I will pick the left: . Found that is also no need to move, is the same.
Thanks for your investigation, I've updated the above list based on your input :-)
I think is does not need to be modified, Cloud you please help me confirm it?
and
There is noting need to do with , and I have filed a PR for
Thanks for your help :-) I've updated the above list based on your input.
Thank you also, I approved the PR and updated the above list from your work.
and do not need to be modified. Could you help me confirm it? THX.
I think there is noting need to do with
Thanks for your work, I've updated the above list based on your input.
I checked but there is no func which is called from less than 2 test packages.
Thanks for your investigation, I've updated the above list based on your input.
Nice!
Yes, thank you so much everyone!! :-)


Question: * [kubectl config](kubectl_config.md)	 - config modifies kubeconfig files <del> ###### Auto generated by spf13/cobra on 8-Dec-2015 </del> <ins> ###### Auto generated by spf13/cobra on 8-Jan-2016 </ins> <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_set-credentials.md?pixel)]()

It acts like a boolean flag but requires an argument, unlike boolean flag is pretty much every environment. We need to find a better way to express the things that this is expressing. Maybe a add a tristate to the pflag lib that behaves the same way but doesn't demand an argument?
cc
The handles "true", "false", "unset" and has a default value. I seem to recall you were looking at changing it, but we needed to preserve the value. The complain here is that you have to do instead of just . /cc
fixing is really easy....


Question: // Duplicated, should be ignored. port("foo", v1.ProtocolUDP, 888, 8888, ""), // Duplicated, should be ignored. <del> port("", v1.ProtocolTCP, 80, 8888, ""), </del> <ins> port("", v1.ProtocolTCP, 80, 8888, "127.0.0.1"), // Duplicated with different address family, shouldn't be ignored port("", v1.ProtocolTCP, 80, 8080, "::"), // No address family specified port("", v1.ProtocolTCP, 1234, 5678, ""), </ins> }, }, []PortMapping{ <del> portMapping("fooContainer-TCP:80", v1.ProtocolTCP, 80, 8080, "127.0.0.1"), portMapping("fooContainer-TCP:443", v1.ProtocolTCP, 443, 4343, "192.168.0.1"), </del> <ins> portMapping("fooContainer-v4-TCP:80", v1.ProtocolTCP, 80, 8080, "127.0.0.1"), portMapping("fooContainer-v4-TCP:443", v1.ProtocolTCP, 443, 4343, "192.168.0.1"), </ins> portMapping("fooContainer-foo", v1.ProtocolUDP, 555, 5555, ""), <ins> portMapping("fooContainer-v6-TCP:80", v1.ProtocolTCP, 80, 8080, "::"), portMapping("fooContainer-any-TCP:1234", v1.ProtocolTCP, 1234, 5678, ""), </ins> }, }, }

What happened: I tried to create a deployment with duplicate containerPorts with hostIPs of two different address families. Only the first mapping was provided as input into my CNI plugin chain. Also, a log message from kubelet warning about a duplicate containerPort was created. What you expected to happen: Both mappings to be provided as input into my CNI plugins. No log messages should be created. How to reproduce it (as minimally and precisely as possible): Create this deployment: Then grab kubelet logs on the host e.g: Anything else we need to know?: This issue here provides some context as to how I ran into this issue: Environment: Kubernetes version (use ): v1.15.1 Cloud provider or hardware configuration: DigitalOcean, custom setup OS (e.g: ):Kernel (e.g. ):Install tools: None Network plugin and version (if this is a network-related bug): Cilium v1.6.0 and Portmap v0.8.2 Others: N/A I've already identified the problem and will submit a PR which fixes this shortly. /sig network
is this dual-stack only? if so, does the fix for this need to target 1.16?
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
This applies to dual-stack only yes. I don't know if it should target 1.16, personally I'd like to get this out so I don't have to run a forked version of kubelet.
/assign
I would agree that this should hit v1.16 if possible.
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.


Question: ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against a pod with different priority class (ScopeSelectorOpNotIn).", func() { _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(context.TODO(), &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass7"}, Value: int32(1000)}, metav1.CreateOptions{}) <del> framework.ExpectEqual(err == nil || apierrors.IsAlreadyExists(err), true) </del> <ins> if err != nil && !apierrors.IsAlreadyExists(err) { framework.Failf("unexpected error while creating priority class: %v", err) } </ins> hard := v1.ResourceList{} hard[v1.ResourcePods] = resource.MustParse("1")

The tests are full of assertions which compare a boolean against true or false, often without any additional explanation. When those assertions fail, the error message is useless for understanding what went wrong, often just saying: Including an explanation is better, but then we still have the distracting noise about the comparison: All of those assertions should be replace with . Such locations can be found by grepping: UPDATE (2024-08-26): in cases where such an if check is not possible (typically in combination with or ), should be used instead of . They emit only the explanation given to them, without including the noise above. The Ginkgo linter (enabled in golangci-lint) points that out. Some progress has been made in the meantime: v1.23.0: 250 v1.24.0-beta.0: 177 However, we were down to 175 at some point, so the number is going back up. Example: Some other places compare values. Those should be replaced with a comparison assertion. Example: It makes sense to split this work into individual PRs, one per OWNERS file under test/e2e. Here's a list of directories: [ ] test/e2e/apimachinery [x] test/e2e/apps: [x] test/e2e/auth [x] test/e2e/autoscaling: [x] test/e2e/cloud/gcp [x] test/e2e/cloud [x] test/e2e/common/network [x] test/e2e/common/node: [x] test/e2e/common/storage: [x] test/e2e/framework: [x] test/e2e/framework/providers/vsphere [x] test/e2e/framework/volume [x] test/e2e/instrumentation/logging [x] test/e2e/instrumentation: [x] test/e2e/kubectl [x] test/e2e/lifecycle [x] test/e2e/node: [x] test/e2e/scheduling: [x] test/e2e/storage: (prior PR: ) [x] test/e2e/testing-manifests/storage-csi [x] test/e2e/upgrades/apps: [x] test/e2e/windows: , When creating a PR, use as title. Use one commit per PR, squash when addressing feedback.

Unless noted otherwise, will work on PRs. Help may be welcome. /sig testing /help Better output for developers when tests fail.
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : ### What would you like to be ? Example: Some other places compare values. Those should be replaced with a comparison assertion. Example: /sig testing /help </details
/good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /good-first-issue </details
/triage accepted
I would like to take this up as my first issue
/assign
I would like to take up this issue as my first issue
Hello I am already working on this issue , can you please take some other issue ?
Okay sure ,how do I un assign myself from this issue?
I don't think you are assigned to this issue.
Oh okay
Hey as discussed in pr I am assuming that you are only working for tests under area.

also finished the work on , see
and only one PR is to be made for the correction under test/e2e/storage/, so how can we distribute the task and also make only single PR. Or can we make multiple PR's
I have a TODO list to the issue description. I'll try to keep that up-to-date with what has been done and who is working on it.
you can continue with in
what is to be changed in this case !
can you inform on which directory you are working.
Currently working on apimachinery
Nothing, just leave it as-is.
But that isn't under - before starting somewhere else, please coordinate with
test/e2e/instrumentation/logging don't have any cases like "Expect.Equal([^,], *(true|false)", so I think it should be removed from tasks to do.
I am thinking that the output should be if !ok { framework.Failf(uexec.ExitError) } in this case, am I thinking reasonable here.
isn't valid Go. The first parameter must be a format string. The code checks whether err is a : Therefore the change should be:
Why don't we just add extra explanation into calls like for easy investigation? I might miss some point.
You have to consider the complete output when that case fails. Gomega will write print the actual value and the expected value, then in addition you get the same information in the custom message. Printing just the custom failure message is shorter and more readable, in particular for a comparison of booleans. For a comparison of integers I see two solutions: ExpectEqual with a fixed message that just explains what is being compared. An if check with Failf where the message includes the actual value. The later seems more useful when the expected value is zero because that value doesn't need to be printed when its obvious.
I am getting difficulty in figuring out what should be output here, what should the appropriate output here. Also what steps you took in knowing so, it would help to improve my understanding.
The "ok" value is the result of checking for a specific key in a map. What a developer therefore wants to know when that check fails is: what was the key what's in the map Both can be included in:
can I work for the section :- test/e2e/scheduling ?

And there was only one changes needed for test/e2e/node which I did but my PR was closed as it only contains trivial edits. That was my first PR, so can you please guide me further?
/assign I'll try to coordinate the work, but won't create PRs myself. I ticked of line items which had no remaining work. feel free to start working on .
I replied there and also suggested an improvement. For those following this issue: try to include more information than before in the error message when the assertion fails. Then it is more obvious to the reviewer that the change is useful without having to read this issue. Examples:
Another tip: don't just reference this issue as explanation for the PR. Reviewers are busy, they won't necessarily invest the time and follow the link to understand what the PR is about. Instead, take the first sentence from this issue and adapt it as needed to the specific changes in the code.
can I start working on test/e2e/common/storage ?
Okay.
for test/e2e/network/netpol , there are already message associated with all of the cases, for example :- So should I just change all of them to ? And same for all the cases in test/e2e/network except these two cases : framework.ExpectEqual(netutils.I sIPv4String(internalIPs[0]) != netutils.IsIPv4String(internalIPs[1]), true) framework.ExpectEqual(netutils.IsIPv4String(p.Status.PodIPs[0].IP) != netutils.IsIPv4String(p.Status.PodIPs[1].IP), true )
Yes.
Okay then I will be working on and
It might make sense to combine them in a single PR, depending on how many changes there are - just use common sense.
IMO : this will never fail because whether err will be nil or it will exist. So, we don't need to change this code. What I had done in this case was :
Note that that is redundant (= not wrong, just not needed). I later clarified that in
/reopen
Reopened this issue. <detailsIn response to : /reopen </details
Hi! I've created a PR to address the tests in e2e/apps It is my first PR so hopefully, everything is ok. Thanks a lot!
Hi can I take test/e2e/common/node?
Yes, please do.

Have we considered / helpers that can skip the comparison output? - is noisy and makes it harder to skim tests, especially given that it's a pretty commonly required pattern.
No, we haven't. Do you find it noisy because it's two lines instead of one? I don't have a problem with that. It seems to be a common approach in unit tests and I find it easy to read, partly because it matches how production code would be written.
It becomes three, plus extra indention - but more importantly stands out in a series of expectations. This draws extra scrutiny to those lines. many unit tests using testify would use to express this because of that.
Isn't that a good thing? is bad without any or no useful extra explanation. The if check makes it obvious that a good failure message is needed. Does print less useless boilerplate code than Gomega? We could use that with Ginkgo (), but then we still need to ensure that the optional explanation is useful.
xref
Hi can I take test/e2e/auth?
for test/e2e/auth, for example : Should I just change it to ?
feel free to work on test/e2e/auth. This one particular example is a bit different: here we are comparing string values, and for those it makes sense to print the actual value because the failure message then adds information. I would leave that one as it is.
OK. test/e2e/auth doesn't need replacing. I : <img width="731" alt="Screen Shot 2022-05-30 at 9 12 59 PM" src="https://user-"There are a lot of positions that need to be modified in . I'll take test/e2e/apps.
That looks like a good candidate. Please go for it. When you need to make up new failure messages, try to think about what information may be useful to have and add that. Then the resulting PR is clearly better than the status quo.
Hi can you please assign me a directory. I want to give it a try. Thanks,
Hi I already posted a PR for e2e/apps some days ago in this issue: If it is fine I'd love to proceed with that. Just to avoid us repeating work.
sorry for not noticing the overlap sooner. Let's go with the older PR.

you can take over test/e2e/storage from There is an incomplete PR for it in You can check out that branch, fix it and submit a new PR, or start from scratch.
it looks like is not active anymore. I've remove him from the todo list and you instead for test/e2e/common/storage.
OK. Thank you. Is there an incomplete PR for .
I in and it returns 0. What should I do?
needs modifying. <img width="1440" alt="Screen Shot 2022-05-31 at 2 40 29 PM" src="https://user-"Can I work on it?
Indeed, did test/e2e/common/storage in You can work on test/e2e/windows.
OK. I'm working on it currently.
Sure, I will look into it. Thanks,
: do you want to continue with test/e2e/apimachinery?
Yes. There's an older PR:
Hi is there anything I could take over?
Hey I am not sure how updated the list is, but I am available to help. Is there any part I could start or continue?
sorry for the late reply. I had (and still have) some big changes for the E2E tests going and had put this on the backseat for a while. If you are interested, how about doing test/e2e/apimachinery and test/e2e/autoscaling? Some work was done on the latter, but there are still several calls (again?).
Sure, I can take that!
Hey I'd also like to contribute on this issue. Is there anything I could take over?
feel free to take my part:
Perfect, then I will take part
Hi :smile: , I would like to contribute to this issue, there is anything that I could do?
I was reluctant to start new work on this because I had one big PR pending that touched a lot of test code (PR ), but that has been merged. you are taking test/e2e/autoscaling, right? test/e2e/apimachinery? you can do test/e2e/apps. That leave test/e2e_node in case someone wants to continue.
yes e2e/autoscaling. I had some trouble to get the tests going initially, but seems fine now!
Hi, Can I do something to contribute to this issue?

you could work on test/e2e_node.
Ok, I will start work on test/e2e_node thanks!
Hi , I would like to contribute. Is there anything I could take over?
thanks for your interest, but right now everything seems to be covered.
Hi Thanks. I am searching for "good-first-issue", feel free to assign one to me. I am exploring api-machinery, but happy to contribute in other sig as well.
maybe you can take a look at if you have some time :)
Hi , There is anything that I could do to contribute with this issue?
Hi , I was looking and found the test/e2e/apimachinery without doing it and only then did I see that it was already assigned if this is a problem just refuse my pr and if you have something else to do let me know.
The remaining work will probably finished in , so there isn't anything else to do. If someone wants to investigate writing a linter for this, then this would be a cool science project - not easy, though. I would start with looking at for this. It has the advantage that it is supported indirectly in golangci-lint ().
Hello I would like to try to do a liter for it. It will be a cool project whit a lot of opportunities to learn. There will be a separate issue for the linter or we continue the discussion here?
Let's continue here.
I ended up creating a separate issue for linter ideas: let's continue there.
Ok!
New tests have been again with this anti-pattern since the last time that I checked. It's clear that without a linter, we'll not get rid of it. So let's focus on that first.
is this issue still open
Yes, this is still open. Current status as of v1.28.0-rc.0: Two calls are in , the rest are in . We have such a linter now, with ginkgolinter in golangci-lint and (soon) the ability to prevent usage of , so it makes sense to continue with removing those calls. do you want to prepare one PR for all remaining calls? I think one PR is enough, no need to split it up.
also has it, which was intentional. I'm undecided whether that should be replaced.

Several of those calls are being touched in , let's wait for that PR to be merged before doing any further work.
no longer finds anything. We might be done... :tada: But not quite. What we have left now is checking a boolean via or , which has the same problem. In some cases, there is an extra message (which is still noisy, see ), in other cases there isn't even that (which makes the failure message fairly useless). We can and should replace some of those calls with the pattern.
is almost resolved. My work-in-progress branch forbids the usage of in favor of the new or plain Go. In some cases, moving the Go code that returns true/false into a matcher might be the best solution because that allows dumping the full actual value that was being checked. Here are the remaining issues: <details</detailsDoes anybody want to work on those?
I would love to!
: please go ahead. Assign PRs to me when you create them. Let's start with a few of those calls initially, do a round of reviews, then continue. /assign
You will have to start with my branch, current master doesn't have gomega with thew new API yet.
I'm starting new to kubernetes. Is there anything easy that I could do ?
Hey Is there anything left I can help you with in this issue ?
Thanks for your interest. There's not much to be done at the moment, though.
Should this be closed?
Oh wow! Looks like we are indeed done. Thanks everyone. /close
Closing this issue. <detailsIn response to : Oh wow! Looks like we are indeed done. Thanks everyone. </details


Question: <ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package runtime import ( "context" v1 "k8s.io/api/core/v1" compbasemetrics "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/scheduler/framework" ) type instrumentedFilterPlugin struct { framework.FilterPlugin metric compbasemetrics.CounterMetric } var _ framework.FilterPlugin = &instrumentedFilterPlugin{} func (p *instrumentedFilterPlugin) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { p.metric.Inc() return p.FilterPlugin.Filter(ctx, state, pod, nodeInfo) } type instrumentedPreFilterPlugin struct { framework.PreFilterPlugin metric compbasemetrics.CounterMetric } var _ framework.PreFilterPlugin = &instrumentedPreFilterPlugin{} func (p *instrumentedPreFilterPlugin) PreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod) (*framework.PreFilterResult, *framework.Status) { result, status := p.PreFilterPlugin.PreFilter(ctx, state, pod) if !status.IsSkip() { p.metric.Inc() } return result, status } </ins>

We observed degradation in scheduling throughput, putting it under 100 pods/s. Respective pprof of kube-scheduler points it to ! /sig scheduling /kind regression 100 pods/node n/a No response <details</details<details</details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
/sig scalability
Thanks for noticing it /assign
just to confirm this bug is present only in 1.27 ( introduced with pluginevaluationtotal metric) , isn't it ?
Exactly, 1.27+ (so the current master as well) to be specific.


Question: { "name": "varetcd", "mountPath": "/var/etcd", "readOnly": false} <ins> ], "env": [ { "name": "GOMAXPROCS", "value": "{{num_cpus}}"} </ins> ] } ],

After recent performance fixes we are now not bound by CPU or disk IOPS. Apiserver during density test uses around 80% of CPU. Still we observe high request latency for pods, events, endpoints and sometimes other resources. I was able to narrow this down to slow etcd. Etcd uses up to 80% of CPU. For some reason we see high latency of requests to etcd. Increasing disk (to increase IOPS) or number of cores doesn't help. In order to meet criteria defined in 1.0 roadmap () we have to make it much faster. Do you have some suggestions how to debug this? Any ideas what could be the reason?
a related question: from the etcd code it seems that etcd is exposing some metrics under /metrics handler: However, when accessing this handler I'm getting 404 error (both with 2.0.9 and 2.0.10 etcd versions). Do you know what could be the reason?
You can try with etcd 2.1.0-alpha0, which gives you a better performance, less CPU usage and the /metrics endpoint (though it is still expr).
Thanks! We will give it a try. Does etcd export pprof data so that we can profile it for lock contention?
Thanks - we will try it. So far I downloaded it locally and indeed it has /metrics endpoint. Thanks!
Not really. If you have time, I think the first thing you can do is to switch the default json encoder to ffjson. Do you know how much performance do you actually need in terms of write/read?
currently we are focusing more on latency. The issue that we observed is that when reading from etcd the whole directory (which is ~3000 entries, each entry of size ~2kB), it can take more than 2 seconds (which seems to be a lot).
Thanks for the suggestion. I will take a look into it.
Yea... That is probably caused by slow json encoding... :(
If that's parsing json shouldn't it mean that there is high CPU usage?
This is what you told me. :) So I highly suspect it is caused by encoding after you gave me more context.
So you're saying that this json encoding happens under lock so that we cannot use more CPU because other requests are blocked. Is my understanding correct?

Can you point us where this encoding happens?
To clarify - by I meant . Sorry for not being clear
The encoding happens without holding any lock: You give etcd multiple cores, and it only uses 80% of one? Is this what you are saying? I still do not quite follow actually.
ffjson seems like a great improvement if the 2x benchmark comparisons vs golang json encoding materialize. Thanks! However I can't help noticing that that even ffjson is more than 30x slower than some protobuf implementations (according to the benchmarks published at ) Have you considered adding a protobuf-based API to etcd to boost performance? Obviously in the short term we might be able to simply "throw cores at the problem", but over time I think that this will become expensive, and a protobuf interface might make sense.
Take a look at:
Beautiful :-)
Let me start from the beginning :) In my test setup master is n1-standard-8 (i.e. 8 cores) machine with 2 PD-SSD disks each 200GB. Under high load generated by density tests total cpu utilization is about 50% (i.e. we use on average 4 cores out of 8). There are two biggest CPU consumers: apiserver -usually up to 3 cores etcd -usually up to 0.8 core (never above 1) With this setup we observe high latency in requests to etcd that list directories with 3000 entries each ~2kB. For some reason etcd does not use all the cpu it could. I tried to look if etcd changes GOMAXPROCS. It seems it doesn't: Maybe this is the reason? As a result etcd cannot consume more than 1 core?
BTW, I would be careful not to conflate JSON and HTTP vs Protobuf and RPC. e.g. One can do Protobuf over HTTP without RPC, and still get most of the performance gains. I'll comment further in the etcd RFC.
Yes. You have to give GOMAXPROCS more cores, then etcd can use more.
We plan to use gRPC, which is protobuf over http2 basically. But we might choose protobuf over plain http with a /rpc endpoint in the end. Not sure yet.
cc
Is it just a regular env variable?

I've only changed this using
Yes. It is.
Perfect! I'll give it a try tomorrow.
This improved stats a lot! The only offending ones (high percentiles above 1s) are listing pods and events: I'll look into this today.
(Just to be clear - this is still on 8 core master with fast drives)
I guess 2 core is enough. Also the read latency has very little thing to do with seek latency. Everything in etcd is in memory for the moment.
Regarding disk - out experiments confirm that seek latency doesn't really matter here. I'm not sure what the difference will be between setting GOMAXPROCS e.g. to 2 and to 4 (we should check it).
so for clarity, did you: export GOMAXPROCS= etcd --myparams ? /cc
Yes. For testing I've just hardcoded this. I'll send PR tomorrow that will automatically set this based on the number of cores. One thing to mention that we still don't use all 8 cores. According to some further debugging now the bottle neck is on apiserver side. I'll debug more tomorrow.


Question: gcePDDetachPollTime = 10 * time.Second nodeStatusTimeout   = 1 * time.Minute nodeStatusPollTime  = 1 * time.Second <ins> gcePDRetryTimeout   = 5 * time.Minute gcePDRetryPollTime  = 5 * time.Second </ins> ) var _ = framework.KubeDescribe("Pod Disks", func() {

For: "Pod Disks should schedule a pod w/ a RW PD, remove it, then schedule it on another host" "Pod Disks should schedule a pod w/ a readonly PD on two hosts, then remove both." (this is not happening on v1.3 master with v1.1 nodes, nor with v1.2 tests against a v1.3 cluster) View logs and artifacts at
Mind taking a look? You should be able to repro by turning up a v1.3 cluster and running v1.1 tests against it.
for your queue
I found the cause of the test failure. The following are the main steps in the test host0Pod, and wait for it running a file to the host0Pod, and then delete it host1Pod, and waiting for it running the file from host1Pod, and then delete it disk with deletePDWithRetry() function, which has 3 minutes timeout defer() function, clean up test environment, and also run deletePDWithRetry() again just in case The problem arises from step 5. In the above test, deletePDWithRetry tried to delete PD every 5 second, but failed in 3 minutes. After it exits retry loop, it failed for the expectNoError(err, "Error deleting PD") in this function. Although it deleted the PD successfully right after this during the clean up phase (step 6). So the main issue for this test failure is the timeout is a little bit short. To fix it issue, we can either increase the timeout, or not fail when calling deletePDWithRetry for the first time. We can fail it if it cannot delete it after calling deletePDWIthRetry second time in step 6. The following is the retry loop and error code in the deletePDWIthRetry(). In current 1.3 version, we only call deletePDWithRetry once in defer() function during clean up. I think to be safe we should increase timeout to 5 minutes, and set it in the constance instead of coding it directly inside of the function.
So two options to fix this: port test fixes from main branch to the 1.1 branch to make the test more robust against delays. for to be merged, which will decrease the time it takes for a volume to be detached and therefore how long it takes to be deleted.


Question: <del> /* Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ /* This soak tests places a specified number of pods on each node and then repeatedly sends queries to a service running on these pods via a service. */ package main import ( "flag" "fmt" "io/ioutil" "net/http" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/klog" "k8s.io/kubernetes/test/e2e/framework" ) var ( queriesAverage = flag.Int("queries", 100, "Number of hostname queries to make in each iteration per pod on average") podsPerNode    = flag.Int("pods_per_node", 1, "Number of serve_hostname pods per node") upTo           = flag.Int("up_to", 1, "Number of iterations or -1 for no limit") maxPar         = flag.Int("max_in_flight", 100, "Maximum number of queries in flight") ) const ( deleteTimeout          = 2 * time.Minute endpointTimeout        = 5 * time.Minute nodeListTimeout        = 2 * time.Minute podCreateTimeout       = 2 * time.Minute podStartTimeout        = 30 * time.Minute serviceCreateTimeout   = 2 * time.Minute namespaceDeleteTimeout = 5 * time.Minute ) func main() { flag.Parse() klog.Infof("Starting cauldron soak test with queries=%d podsPerNode=%d upTo=%d maxPar=%d", *queriesAverage, *podsPerNode, *upTo, *maxPar) cc, err := restclient.InClusterConfig() if err != nil { klog.Fatalf("Failed to make client: %v", err) } client, err := clientset.NewForConfig(cc) if err != nil { klog.Fatalf("Failed to make client: %v", err) } var nodes *corev1.NodeList for start := time.Now(); time.Since(start) < nodeListTimeout; time.Sleep(2 * time.Second) { nodes, err = client.CoreV1().Nodes().List(metav1.ListOptions{}) if err == nil { break } klog.Warningf("Failed to list nodes: %v", err) } if err != nil { klog.Fatalf("Giving up trying to list nodes: %v", err) } if len(nodes.Items) == 0 { klog.Fatalf("Failed to find any nodes.") } klog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) for i, node := range nodes.Items { klog.Infof("%d: %s", i, node.Name) } queries := *queriesAverage * len(nodes.Items) * *podsPerNode // Create a uniquely named namespace. got, err := client.CoreV1().Namespaces().Create(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "serve-hostnames-"}}) if err != nil { klog.Fatalf("Failed to create namespace: %v", err) } ns := got.Name defer func(ns string) { if err := client.CoreV1().Namespaces().Delete(ns, nil); err != nil { klog.Warningf("Failed to delete namespace %s: %v", ns, err) } else { // wait until the namespace disappears for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ { if _, err := client.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil { if errors.IsNotFound(err) { return } } time.Sleep(time.Second) } } }(ns) klog.Infof("Created namespace %s", ns) // Create a service for these pods. klog.Infof("Creating service %s/serve-hostnames", ns) // Make several attempts to create a service. var svc *corev1.Service for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) { t := time.Now() svc, err = client.CoreV1().Services(ns).Create(&corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "serve-hostnames", Labels: map[string]string{ "name": "serve-hostname", }, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{{ Protocol:   "TCP", Port:       9376, TargetPort: intstr.FromInt(9376), }}, Selector: map[string]string{ "name": "serve-hostname", }, }, }) klog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) if err == nil { break } klog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) } if err != nil { klog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) return } // Clean up service defer func() { klog.Infof("Cleaning up service %s/serve-hostnames", ns) // Make several attempts to delete the service. for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err := client.CoreV1().Services(ns).Delete(svc.Name, nil); err == nil { return } klog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) } }() // Put serve-hostname pods on each node. podNames := []string{} for i, node := range nodes.Items { for j := 0; j < *podsPerNode; j++ { podName := fmt.Sprintf("serve-hostname-%d-%d", i, j) podNames = append(podNames, podName) // Make several attempts for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) { klog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) t := time.Now() _, err = client.CoreV1().Pods(ns).Create(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: map[string]string{ "name": "serve-hostname", }, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name:  "serve-hostname", Image: framework.ServeHostnameImage, Ports: []corev1.ContainerPort{{ContainerPort: 9376}}, }, }, NodeName: node.Name, }, }) klog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) if err == nil { break } klog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) } if err != nil { klog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) return } } } // Clean up the pods defer func() { klog.Info("Cleaning up pods") // Make several attempts to delete the pods. for _, podName := range podNames { for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err = client.CoreV1().Pods(ns).Delete(podName, nil); err == nil { break } klog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) } } }() klog.Info("Waiting for the serve-hostname pods to be ready") for _, podName := range podNames { var pod *corev1.Pod for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { pod, err = client.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) if err != nil { klog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) continue } if pod.Status.Phase == corev1.PodRunning { break } } if pod.Status.Phase != corev1.PodRunning { klog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) } else { klog.Infof("%s/%s is running", ns, podName) } } // Wait for the endpoints to propagate. for start := time.Now(); time.Since(start) < endpointTimeout; time.Sleep(10 * time.Second) { _, err = http.Get(fmt.Sprintf("http://serve-hostnames.%s:9376", ns)) if err == nil { break } klog.Infof("After %v while making a request got error %v", time.Since(start), err) } if err != nil { klog.Errorf("Failed to get a response from service: %v", err) } // Repeatedly make requests. for iteration := 0; iteration != *upTo; iteration++ { responseChan := make(chan string, queries) // Use a channel of size *maxPar to throttle the number // of in-flight requests to avoid overloading the service. inFlight := make(chan struct{}, *maxPar) start := time.Now() for q := 0; q < queries; q++ { go func(i int, query int) { inFlight <- struct{}{} t := time.Now() resp, err := http.Get(fmt.Sprintf("http://serve-hostnames.%s:9376", ns)) klog.V(4).Infof("Call to serve-hostnames in namespace %s took %v", ns, time.Since(t)) if err != nil { klog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) // If the query failed return a string which starts with a character // that can't be part of a hostname. responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err) } else { defer resp.Body.Close() hostname, err := ioutil.ReadAll(resp.Body) if err != nil { responseChan <- fmt.Sprintf("!failed in iteration %d to read body of response: %v", i, err) } else { responseChan <- string(hostname) } } <-inFlight }(iteration, q) } responses := make(map[string]int, *podsPerNode*len(nodes.Items)) missing := 0 for q := 0; q < queries; q++ { r := <-responseChan klog.V(4).Infof("Got response from %s", r) responses[r]++ // If the returned hostname starts with '!' then it indicates // an error response. if len(r) > 0 && r[0] == '!' { klog.V(3).Infof("Got response %s", r) missing++ } } if missing > 0 { klog.Warningf("Missing %d responses out of %d", missing, queries) } // Report any nodes that did not respond. for n, node := range nodes.Items { for i := 0; i < *podsPerNode; i++ { name := fmt.Sprintf("serve-hostname-%d-%d", n, i) if _, ok := responses[name]; !ok { klog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) } } } klog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing) } } </del>

We've deleted all repos from Docker Hub and will be retiring the org/user (ref: ) Apparently the repo is referenced for docker push and pull in It's unclear to me whether we have broken anything, or if this is dead code. I would use to find out if this is referenced anywhere (aside from sundry vendor dirs) If dead code: remove it If used: move to test/e2e/images and follow the same pattern there used to push to /sig testing /area test /wg k8s-infra /priority important-soon /help /good-first-issue I believe this is a pretty straightforward issue and am happy to review whatever PR's are made for it
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : We've deleted all repos from Docker Hub and will be retiring the org/user (ref: ) /area test /wg k8s-infra /priority important-soon /good-first-issue I believe this is a pretty straightforward issue and am happy to review whatever PR's are made for it Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: #                 collisions here? "${GCLOUD}" compute firewall-rules create  "${MINION_TAG}-${USER}-http-alt"  <del> --allow tcp:80 tcp:8080  </del> <ins> --allow tcp:80,tcp:8080  </ins> --project "${PROJECT}"  --target-tags "${MINION_TAG}"  --network="${NETWORK}"

This happens over and over. Scary messages. WARNING: Lists should be separated by commas, try "--allow=tcp,udp,icmp,esp,ah,sctp". If you intend to use udp, icmp, esp, ah, sctp for positional arguments, put them before the flag. WARNING: Lists should be separated by commas, try "--scopes=storage-ro,compute-rw". If you intend to use compute-rw for positional arguments, put them before the flag. WARNING: Lists should be separated by commas, try "--metadata-from-file=startup-script=/usr/local/google/home/thockin/src/kubernetes/hack/e2e-internal/../../cluster/../cluster/gce/../../cluster/../cluster/gce/configure-,kube-env=/tmp/kubernetes.MURmzX/master-kube-". If you intend to use kube-env=/tmp/kubernetes.MURmzX/master-kube- for positional arguments, put them before the flag. WARNING: Lists should be separated by commas, try "--disk=name=e2e-test-thockin-master-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no". If you intend to use device-name=master-pd, mode=rw, boot=no, auto-delete=no for positional arguments, put them before the flag.
That's a new one. gcloud started getting a little more anal here. Exciting. cc
They changed list argument syntax a while back, but the warnings are new. Time to update startup scripts I guess.


Question: } // SetRef stores a reference to a pod's container, associating it with the given container ID. <ins> // TODO: move this to client-go v1.ObjectReference </ins> func (c *RefManager) SetRef(id ContainerID, ref *v1.ObjectReference) { c.Lock() defer c.Unlock()

Is this a BUG REPORT or FEATURE REQUEST? BUG REPORT Kubernetes version (use ): kubernetes/master using CRI with docker What happened: Create a pod that uses a readiness probe that does an exec to command that doesnt exist See the following in logs: What you expected to happen: No error in log, expected the event to properly be thrown. How to reproduce it (as minimally and precisely as possible): See above. Anything else we need to know: Nope.
this is a regression.
debugging, will post a fix once found.
Other reports: ,
-- thanks for the other reports. at least now we have a reliable reproduction.
The problem appears to be the following: has a that correlates containers to a . It uses the type . event recorder library we use in the prober works with . If it doesn't get one of those, it expects the passed in object to implement List. As a result, we are passing a v1.ObjectReference the recorder was not expecting.
The bug may have been introduced in , which was there since v1.6.0.
Opened for a fix.
Keep this open until 1.6 is patched.
/remove-priority P0 /priority critical-urgent (I'm not actually sure this is the right priority, just trying to remove the old priority/PN labels)
Closing since has been merged.

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): Is this a BUG REPORT or FEATURE REQUEST? (choose one): <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): 6 tree at Environment: Cloud provider or hardware configuration: bare metal OS (e.g. from /etc/os-release): Ubuntu xenial Kernel (e.g. ): Linux fab7-compute-2 4.4.0-70-generic -Ubuntu SMP Wed Mar 22 12:47:43 UTC 2017 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm What happened: This gets logged ] Could not construct reference to: '&v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"kube-dns--72wvm", UID:"-1edc-11e7-ac00-", APIVersion:"v1", ResourceVersion:"554", FieldPath:"spec.containers{kubedns}"}' due to: 'object does not implement the List interfaces'. Will not report event: 'Warning' 'Unhealthy' 'Readiness probe failed: Get http://10.1.0.130:8081/readiness: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' What you expected to happen: Should log correctly How to reproduce it (as minimally and precisely as possible): Create pod that fails readiness probe
cc not sure if this is related to your changes
I don't think so. I didn't change the identification of or which are being used there.

I always get the same message. What does this exactly mean?
We're also seeing this on 1.6.1
Seeing this in as well
How we can come to know which release this is fixed? I am using 1.6.6 still I can see this issue. Thanks in Advance.


Question: } for _, pv := range allPVs.Items { if pv.Spec.StorageClassName == scName { <ins> pv := pv </ins> remainingPVs = append(remainingPVs, &pv) } }

At , within the loop , variable is shared by all the loop iterations, as described in the Go specification (). All instances of &pv within the loop may refer to the last tem updated by the loop. As a consequence, may hold multiple instances of the same item. Later on these remaining PVs are deleted, so it is possible that the same is deleted multiple times: More information on the bug can be found at A fix is to avoid reusing the shared variable node: Found by #deepgo.
/triage accepted /area e2e-test-framework
Thanks for pointing out, makes complete sense. I'm happy to fix it. /assign
/sig testing

At , within the loop , variable is shared by all the loop iterations, as described in the Go specification (). All instances of &node within the loop may refer to the last node item updated by the loop. As a consequence, may hold multiple instances of the same node. Note that the following unit test checks only the number of nodes rather than the nodes themselves (otherwise the bug may be revealed): More information on the bug can be found at A fixe is to avoid reusing the shared variable : Found by #deepgo.
/triage accepted /area e2e-test-framework /sig testing
Thanks for pointing out, makes complete sense. I'm happy to fix it. /assign


Question: return fmt.Errorf("failed to create client pod %q: %v", clientPod.Name, err) } defer func() { <del> // note the test dir will be removed when the kubelet unmounts it </del> <ins> deleteFile(clientPod, ddInput) </ins> By(fmt.Sprintf("deleting client pod %q...", clientPod.Name)) e := framework.DeletePodWithWait(f, cs, clientPod) if e != nil {

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: Which test(s) are failing: [sig-storage] In-tree Volumes [Driver: hostPath] [Testpattern: Inline-volume (default fs)] volumeIO should write files of various sizes, verify size, validate content [Slow] Since when has it been failing: 11/13 Testgrid link: https://k8s- Reason for failure: Anything else we need to know: This is consistently failing in 1.13 branch and is release blocking. <!-- DO NOT EDIT BELOW THIS LINE --/kind failing-test /sig storage /priority critical-urgent /milestone v1.13 /assign /cc
The test case is failing because it wrote 2MB instead of the expected 1MB. It's also very odd that the test is failing in soak, but passing in regular CI suites.
There's a few problems: .GetVolumeSource returns "/tmp" for everything. This means all tests will share the same directory. This should be probably use some unique directory instead volume IO test generates a unique mountPath directory, however that doesn't mean a unique directory is actually created on the host. Ie, a mountPath of "/a/b/c" on a hostpath volume at "/tmp" doesn't mean directory "/tmp/a/b/c" is created. io test file name is not unique seems like the IO test is not cleaning up the files it creates, so the files are getting reused on subsequent tests due to the non-unique naming issues.
The other thing I can't explain is why it's failing only now
Fix is here:
As per discussion in 1.13 burndown this morning, this is a test job issue and its not release blocking at this point. So we can take this fix in 1.14 as opposed now during code freeze let us know if you feel otherwise
/milestone v1.14 /priority important-soon
/remove-priority critical-urgent


Question: accessibleTopology map[string]string, err error) { klog.V(4).Info(log("calling NodeGetInfo rpc")) <del> if c.nodeV1ClientCreator != nil { return c.nodeGetInfoV1(ctx) } else if c.nodeV0ClientCreator != nil { return c.nodeGetInfoV0(ctx) } err = fmt.Errorf("failed to call NodeGetInfo. Both nodeV1ClientCreator and nodeV0ClientCreator are nil") </del> <ins> // TODO retries should happen at a lower layer (issue #73371) backoff := wait.Backoff{Duration: initialDuration, Factor: factor, Steps: steps} err = wait.ExponentialBackoff(backoff, func() (bool, error) { var getNodeInfoError error if c.nodeV1ClientCreator != nil { nodeID, maxVolumePerNode, accessibleTopology, getNodeInfoError = c.nodeGetInfoV1(ctx) } else if c.nodeV0ClientCreator != nil { nodeID, maxVolumePerNode, accessibleTopology, getNodeInfoError = c.nodeGetInfoV0(ctx) } if nodeID != "" { return true, nil } // kubelet plugin registration service not implemented is a terminal error, no need to retry if strings.Contains(getNodeInfoError.Error(), "no handler registered for plugin type") { return false, getNodeInfoError } // Continue with exponential backoff return false, nil }) </ins> return nodeID, maxVolumePerNode, accessibleTopology, err }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: If kubelet plugin registration fails during the registration process, the kubelet never attempts to register the socket again (assuming the error was transient), unless the socket for the driver is deleted & recreated to trigger a new registration. What you expected to happen: Maybe retry with exponential backoff when kubelet plugin registration fails instead of just logging and waiting for another filesystem event like it does today. Of course, if a socket returns a terminal error (does not implement the kubelet plugin registration service, for example), we should not retry. Follow up on /milestone v1.14 /sig storage How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
I'd like to work on this issue
/assign
Re opening. was a temporary work around for this. is the permanent fix.
Hello! We are starting Bug Freeze for 1.15. Is this issue still planned on being incorporated into 1.15?
the dependent PR is not yet merged. can we get this in for 1.15?
said that is the "permanent fix." and that has merged. So let's close this out /close
Closing this issue. <detailsIn response to : said that is the "permanent fix." and that has merged. So let's close this out Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: Image: framework.GetPauseImageName(f.Client), }, }, <ins> NodeSelector: map[string]string{k: v}, // only launch on our two nodes </ins> }, }) framework.ExpectNoError(err)

https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
Looking into this...
trying to track this flake down. It looks like the pod is launched despite AntiAffinity in . In my local setup I saw the same, but noticed that the new admission controller was not activate.

I have activate it and now get this:
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
Should be fixed by


Question: }) /* <del> Release: v1.16 </del> <ins> Release: v1.16, v1.21 </ins> Testname: StatefulSet resource Replica scaling Description: Create a StatefulSet resource. Newly created StatefulSet resource MUST have a scale of one.

[ ] two weeks soak start date : testgrid-link [ ] two weeks soak end date : [ ] test promotion pr : kubernetes/kubernetes#? According to this APIsnoop query, there are still some remaining StatefulsetScale endpoints that are untested. with this query, you can filter untested endpoints by their category and eligibility for conformance. e.g below shows a query to find all conformance eligible untested, stable, apps endpoints sql-mode select distinct endpoint, right(useragent,73) AS useragent from testing.auditevent -- where useragent ilike '%subresource%' where endpoint ilike '%AppsV1NamespacedStatefulSetScale%' and releasedate::BIGINT round(((EXTRACT(EPOCH FROM NOW()))::numeric)1000,0) - and useragent like 'e2e%' order by endpoint limit 30; example endpoint | useragent -----------------------------------------|--------------------------------------------------------------------------- patchAppsV1NamespacedStatefulSetScale | [StatefulSetBasic] should have a working scale subresource [Conformance] readAppsV1NamespacedStatefulSetScale | [StatefulSetBasic] should have a working scale subresource [Conformance] replaceAppsV1NamespacedStatefulSetScale | [StatefulSetBasic] should have a working scale subresource [Conformance] (3 rows) If a test with these calls gets merged, test coverage will go up by 1 point* This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return filepath.Join(kl.getRootDir(), config.DefaultKubeletPluginsDirName) } <ins> // getPluginsRegistrationDir returns the full path to the directory under which // plugins socket should be placed to be registered. // More information is available about plugin registration in the pluginwatcher // module func (kl *Kubelet) getPluginsRegistrationDir() string { return filepath.Join(kl.getRootDir(), config.DefaultKubeletPluginsRegistrationDirName) } </ins> // getPluginDir returns a data directory name for a given plugin name. // Plugins can use these directories to store data that they need to persist. // For per-pod plugin data, see getPodPluginDir.

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug This is more of an enhancement. What happened: The Plugin Watcher is setup to trigger on file activities when they occur in the subdirectory that it scans. During startup, the Watcher is passed (from within the Kubelet) the Kubelet's root plugin directory as a location to scan for newly plugins - Because the kubelet's plugins subdirectory is also used by existing in-tree drivers (including csi) as a location to store plugins artifacts, the Plugin Watcher is getting triggered by directory/file activities occurring in that subdirectory that has nothing to do with plugin registration as shown in the output below: What you expected to happen: The Plugin Watcher should not be getting triggered by file/directory activities not related to plugin registration/unregistration. One possibility is to have the Watcher scan one directory below plugins like or something similar How to reproduce it (as minimally and precisely as possible): Run plugin watcher with a CSI driver. cc
/sig node
I think you should only place the plugin socket file in the plugin registration directory but not the plugin data.
agree with we can only place the registration file in the specified path instead of placing all data into it. On the other hand, you can place data into the /var/lib/kubelet/plugins/registration even if registration created, that's a no-end loop.
As I mentioned in issue, existing in-tree volume plugins were already using that directory as a location to store storage operation artifacts (not just csi). Here is a shortened list of driver locations that use /var/lib/kubelet/plugins in their code already: And several more locations. The fix would be to have plugin watcher scan a subdirectory of plugins (say ) instead of plugins directly as it is doing today. The watcher functions as intended. Except now it picks up all directory activities whether related or not to plugin registration.
Well given the fact that the pluginwatcher is beta this is a bit of an issue. My understanding of the "beta" guarantees is that we want to be backwards compatible for all plugins published with the beta API. Meaning that even if we move the registration directory to a new location, we'll need to keep scanning the old directory (to pick up v1beta plugins).

From a 10 000 feet point of view it seems easier to me if we move the in-tree volume plugin's data. Though it might be a lot of work. Another less painful option, but a bit grey is the following: Enforce plugins to end with this suffix is recommended for device plugins (though not enforced) and I believe is a convention on linux. This is a bit grey because we would still be breaking v1beta users but since we're moving a convention from "recommended" to "enforced" it seems "less" breaking to me (and I haven't seen device plugins not using that extension). What do you think ?
I like the suffix idea. It can be a recommendation (which csi is already doing).
Hmm, I've just deployed a new k8s cluster on 1.12.0 and noticed this repeating in the logs on a few of our pods. Just for my own sanity, is this something I should be worried about or can I ignore it until a fix is released?
you can ignore these errors. if it's a recommendation then we'll keep spamming the logs with all the data files. Another thing to keep in mind, it seems like CSI plugins puts a lot of data (and creates a lot of directories). It might be worth considering limiting the recursion to one level. I'm concerned we might exhaust the amount of fds by tracking the plugins data.
yeah CSI keeps track of volume operation metadata in the plugins dir. This probably will not change. I understand this is beta. However, besides CSI and Device plugin, are there any other implementers out there for this feature ? Would it be possible to propose a new location (i.e. <pluginsDir/reg/?
let me know if you open a PR for this as discussed.
thanks for open the issue with detail logs, i can see two problems by looking at logs and code. 1) plugin watcher trying to dial non-socket files. 2) too many logs generated (CREATE and DELETE). I a suggestion to fix of this issue we can fix those problems by: 1) check file mode when handle CREATE event, only attempt to dial file when file mode is socket.

2) increase log verbose level for both 'handleDeleteEvent()' and 'handleCreateEvent()' plugin watcher still has activity when any file create/delete under the directory, but they become light weighted (CREATE file will only trigger file mode check, DELETE file event will trigger plugin searching) i am not sure we want to go further to limit directory depth where plugin can live. it has two drawbacks IMO: 1) it is hard to enforce, not sure everyone will read code/readme. 2) it provides less flexibility to plugins: if want to organize many plugin socket files with directory.
These are definitely good suggestions! It seems like there might be some confusion as to why we are thinking of doing this: Most plugins use the plugin directory to store data (actually a lot of files / dirs) Watching all the directories and files created inside the plugin directory puts more burden on the Kubelet in terms of cpu / memory consumption Watching all the directories and files created inside the plugin directory quickly raises the number of fd consumed By limiting the search of the plugins to one directory, we would be severely limiting the ability of the module to unintentionally harm the overall Kubelet. It seems actually pretty easy to enforce, as it's a simple condition. As for not everyone reading code / readme, we can add the documentation to the gRPC interface if this is a concern. In terms of flexibility, is there a real need to create a plugin in a lower directory? Is there a plugin that has so many sockets that a single directory underneath the plugin directory isn't enough? Naively it seems to me that is enough for the different plugins we enable.
ping on the above :)
Naively it seems to me that $ is enough for the different plugins we enable. i can not speak from plugin developer, but both below looks reasonable to me , or as long as we can communicate to plugin developers clearly and effectively (gRPC interface documentation is needed), the directory depth restriction is reasonable to me.
I like . However, why not put plugintype one level up as . For instance
Is the suggestion to hard code the number of levels that the recursion algorithm will search to 3? Or just expect a socket only at a hard coded level (e.g. socket must be nested 3 deep)? Copying my comment from for consistency: Another option I can think of: recursively scanning but before scanning a directory check if it is mounted ().

This will be backwards compatible and will prevent scanning of volume data Thoughts?
i like this idea better if it is always the case that: volume data is mounted, we can modify the watcher code to not watch the mounted directory in the first place.
Skipping mounted dir along with restructuring search path to known (shallow) dir sounds like a good plan. Though with the new suggested paths, sockets will be placed at a location where mounts should not exist.
I think that would be a safe assumption. The plugin directory is only used as a target path for global mounts.
Ephemeral volume types may not need to mount, it could just be a symlink or plain directory. Can we assume any ephemeral volume CSI driver won't implement global mount?
Yes, in-tree ephemeral volumes today don't do a global mount so the mounts directory is which is not the plugin dir. So no issue there. For CSI the will be the global mount path (this plugin dir). Emphermal volumes shouldn't implement call -- but nothing prevents them from doing so. Maybe let's start with just checking for mount and ignoring mount directories. If it is still an issue we can consider adding a configurable black list of paths. Thoughts?
That sounds good. Also, putting the socket files in a specific location that does not clash with mounts will also help (i.e. )
/milestone v1.13 /priority important-soon
I just want to point out that one of the issue we have with recursively watching all directories is that it will significantly increase the amount of fd consumed. Given that plugins actually store data there and in a dynamic way my fear is that it will exhaust the available fds for no real good reasons. Limiting the depth sounds like a reasonable solution to that issue.
btw, because of this verbosity in my cluster writes 5G logs every day and filled up my rootfs (which is small). Is there any way to decrease this verbosity?
We can do both. Don't search mounted dirs, and max depth of search. Who is working on this?
That's pretty bad, can you open another issue to track that.
we have a PR in flight discussing this issue (recursive scanning) here: Thinking more about this, the fact that too much logs is generated is clearly linked to this specific issue and how we process plugins.

In any case I'll be fixing that issues too
how confident are we in getting this fixed by Code freeze tomorrow? 5pm PST
I expect things to wind up today, changes in the above PR aren't very contentious and don't involve significant changes