Question: attemptsExpected: 0, errExpected:      context.DeadlineExceeded, }, <ins> { name:      "context canceled before the second execution and immediate", immediate: true, context: func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Second) }, callback: func(attempts int) (bool, error) { return false, nil }, attemptsExpected: 1, errExpected:      context.DeadlineExceeded, timer:            Backoff{Duration: 2 * time.Second}.Timer(), }, { name:      "immediate and long duration of condition and sliding false", immediate: true, sliding:   false, context: func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Second) }, callback: func(attempts int) (bool, error) { if attempts >= 4 { return true, nil } time.Sleep(time.Second / 5) return false, nil }, attemptsExpected: 4, timer:            Backoff{Duration: time.Second / 5, Jitter: 0.001}.Timer(), }, { name:      "immediate and long duration of condition and sliding true", immediate: true, sliding:   true, context: func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Second) }, callback: func(attempts int) (bool, error) { if attempts >= 4 { return true, nil } time.Sleep(time.Second / 5) return false, nil }, errExpected:      context.DeadlineExceeded, attemptsExpected: 3, timer:            Backoff{Duration: time.Second / 5, Jitter: 0.001}.Timer(), }, </ins> } for _, test := range tests { <del> for _, immediate := range []bool{true, false} { t.Run(fmt.Sprintf("immediate=%t", immediate), func(t *testing.T) { for _, sliding := range []bool{true, false} { t.Run(fmt.Sprintf("sliding=%t", sliding), func(t *testing.T) { t.Run(test.name, func(t *testing.T) { contextFn := test.context if contextFn == nil { contextFn = defaultContext } ctx, cancel := contextFn() defer cancel() timer := Backoff{Duration: time.Microsecond}.Timer() attempts := 0 err := loopConditionUntilContext(ctx, timer, test.immediate, test.sliding, func(_ context.Context) (bool, error) { attempts++ defer func() { if test.cancelContextAfter > 0 && test.cancelContextAfter == attempts { cancel() } }() return test.callback(attempts) }) if test.errExpected != err { t.Errorf("expected error: %v but got: %v", test.errExpected, err) } if test.attemptsExpected != attempts { t.Errorf("expected attempts count: %d but got: %d", test.attemptsExpected, attempts) } }) }) } </del> <ins> t.Run(test.name, func(t *testing.T) { contextFn := test.context if contextFn == nil { contextFn = defaultContext } ctx, cancel := contextFn() defer cancel() timer := test.timer if timer == nil { timer = Backoff{Duration: time.Microsecond}.Timer() } attempts := 0 err := loopConditionUntilContext(ctx, timer, test.immediate, test.sliding, func(_ context.Context) (bool, error) { attempts++ defer func() { if test.cancelContextAfter > 0 && test.cancelContextAfter == attempts { cancel() } }() return test.callback(attempts) </ins> }) <del> } </del> <ins> if test.errExpected != err { t.Errorf("expected error: %v but got: %v", test.errExpected, err) } if test.attemptsExpected != attempts { t.Errorf("expected attempts count: %d but got: %d", test.attemptsExpected, attempts) } }) </ins> } }

While trying to update to use latest version of package and to migrate away from deprecated , function, our tests fail as it seems new function immediately executes given condition twice, which is not mentioned anywhere. Result of executed test specified below: Since has been deprecated and it recommends using , I expected it to have the equivalent semantics to allow easy migration from deprecated functions, as no changelog or documentation mentions new semantics. Playground: Local: file with the following content: as . No response_ <details</details<details</details<details</details<details</details<details</details<details</details
/sig api-machinery
CC since you were involved in
/assign /triage accepted
I'd like to work on this one if it's allowed I think this is a simple fix with few lines removed: i.e.: instead of only wait a loop if immediate is false, we wait a time loop after special execution also if the immediate is true.
/assign
/assign


Question: } } <ins> // Container information is used in adjusting OOM scores, adding ndots and getting the logPath. containerInfo, err := dm.client.InspectContainer(id.ID) if err != nil { return kubecontainer.ContainerID{}, fmt.Errorf("InspectContainer: %v", err) } </ins> // Create a symbolic link to the Docker container log file using a name which captures the // full pod name, the container name and the Docker container ID. Cluster level logging will // capture these symbolic filenames which can be used for search terms in Elasticsearch or for // labels for Cloud Logging. <del> containerLogFile := path.Join(dm.dockerRoot, "containers", id.ID, fmt.Sprintf("%s-json.log", id.ID)) </del> <ins> containerLogFile := containerInfo.LogPath </ins> symlinkFile := LogSymlink(dm.containerLogsDir, kubecontainer.GetPodFullName(pod), container.Name, id.ID) if err = dm.os.Symlink(containerLogFile, symlinkFile); err != nil { glog.Errorf("Failed to create symbolic link to the log file of pod %q container %q: %v", format.Pod(pod), container.Name, err) } <del> // Container information is used in adjusting OOM scores and adding ndots. containerInfo, err := dm.client.InspectContainer(id.ID) if err != nil { return kubecontainer.ContainerID{}, fmt.Errorf("InspectContainer: %v", err) } </del> // Check if current docker version is higher than 1.10. Otherwise, we have to apply OOMScoreAdj instead of using docker API. // TODO: Remove this logic after we stop supporting docker version < 1.10. if err = dm.applyOOMScoreAdjIfNeeded(pod, container, containerInfo); err != nil {

After starting a container, Kubernetes places a symlink to the container's log file in . Unfortunately, rather than querying Docker for the , . This means that any change upstream in the path or filename that Docker uses to store its logs (e.g. to so that a file name transition would be a workable solution for docker/docker) would result in broken symlinks being put in place. It would be a better idea to query for the , potentially by doing something like this: This would allow Docker to put the log files anywhere and have symlinks put in the right place.
I will start work on this one, PR coming soon.


Question: return nil, 0, fmt.Errorf("error creating volume %v", err) } glog.V(1).Infof("glusterfs: volume with size: %d and name: %s created", volume.Size, volume.Name) <del> dynamicHostIps, err := getClusterNodes(cli, volume.Cluster) </del> <ins> dynamicHostIps, err := p.GetClusterNodes(cli, volume.Cluster) </ins> if err != nil { glog.Errorf("glusterfs: error [%v] when getting cluster nodes for volume %s", err, volume) return nil, 0, fmt.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume)

several operations to the glusterfs mount operation that nodes are not permitted to perform: looks up arbitrary secrets based on storage class parameters. this will not succeed, as the node authorizer will only allow nodes access to secrets referenced by pods or persistent volumes used by the node attempts to create a service and endpoint using the node's API client. nodes do not have permission to create arbitrary endpoints and services. For both of those reasons, that recreate operation will fail when attempted from the node. The recreate operation should probably move to one of the storage-related controllers (provisioner or attach/detach) cc
Marking this with 1.7 milestone until we have a better idea of whether this is blocking or not for the 1.7 release.
CC Is the endpoint originally created by the user or by the volume plugin creating/deleting Endpoint objects? In addition to the concerns raised, having a volume create new API objects seems like a bad idea (abstraction layer violation?).
From on Slack: Unless there is justification otherwise, I will roll back the PR on Monday.
cc/
revert button doesn't work, have to do it manually
Revert PR is approved and awaiting merge (). Once that merges this issue can be closed.
Opened an E2E pr to catch this kind of behavior in the future. WIP, still exploring a method to reproduce and catch the behavior.


Question: github.com/gogo/protobuf v1.3.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/mock v1.4.4 <del> github.com/google/cadvisor v0.38.6 </del> <ins> github.com/google/cadvisor v0.38.7 </ins> github.com/google/go-cmp v0.5.2 github.com/google/gofuzz v1.1.0 github.com/google/uuid v1.1.2

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Kubelet v1.19.0 doesn't start on nodes with CRI-O 1.17.2 (storagedriver="btrfs") with BTRFS as root filesystem. Related/previous issues: This error shows up in the journal: The is referring to the btrfs root filesystem, mounted at the time. : : What you expected to happen: Kubelet starts and detects the filesystem backing , even if it is BTRFS. How to reproduce it (as minimally and precisely as possible): Install kubeadm/kubectl 1.19.0 Install CRI-O, I used 1.17.2 from: Configure CRI-O to use Perform "kubeadm init" creating a new cluster Observe journal (check for kubelet issues) Anything else we need to know?: A possible workaround is to make sure a bind mount exists which allows kubelet's logic to find the backing fileystem. Eg. add the following fstab entry and then perform : The resulting mount should look somewhat like this: The workaround has been tested and seems to make kubelet work as expected. Environment: Kubernetes version: 1.19.0 OS: Debian Bullseye (Testing) Kernel: Linux 5.8.5 Others: The package has been installed
/sig node
cc xref
I am encounter the same issue. thank you for your workaround. It seems to work.
I got the same issue with latest stable Docker on Debian (btrfs) and k8s v1.19.2.
I confirm the same issue shows up with on btrfs (on LVM on LUKS, Debian buster), running k3s. No btrfs subvols. The workaround works for me; for k3s I also need to bind mount . I did not experience this issue with 1.18.
This also reproduces with Docker runtime. Workaround with bind mount helps, however automatically unmounts it so don't get confused. is on root that is mounted like this: , and docker is configured to use .

Reproduced with k8s 1.19.3 on Gentoo. The rootfs is btrfs, is on rootfs, docker is configured with storage drigver . automatically mount and unmount it on . Also confirmed bind mount workaround helps.
/assign
I think almost anyone using BTRFS as the root filesystem has encountered this issue. If is merged and applied to the kubernetes, this can be fixed. Please take a look at this ?
The has been merged to google/cadvisor. I think if Kubernetes v1.21 includes the fix by using google/cadvisor v0.39, the issue will be resolved.
Just a note that I too have stumbled on this issue after reconfiguring CRI-O storage to use btrfs driver pointing with graphroot directly to a BTRFS subvolume (the root filesystem is not BTRFS). After reading this issue I tried mounting the BTRFS subvolume explicitly on some mountpoint and pointed with graphroot to this explicit subvolume mountpoint which solved the problem. The problem seems to occur only when pointing to a subvolume within some parent subvolume as this is not a real mountpoint.
Ok, so I just want to make sure I understand. Currently when i try to run kubeadm to create a new cluster while using btrfs, it gives me With Kubernetes v1.21, that will no longer be the case?
In my experience (with CRI-O 1.19 and k8s 1.19), the btrfs driver (in ) is supported, but the combination does not work correctly when graphroot is pointed to a btrfs subvolume within some parent subvolume (I don't have experience with graphroot poited to some subdirectory of a default subvolume and/or when default subvolume is also a root filesystem, just when pointed directly to a subvolume within some parent subvolume which is not a root filesystem). For example, when btrfs filesystem is mounted like the following (): when I then create a subvolume: then use it as a graphroot (): But when I explicitly mount the subvolume to some mountpoint (): then use the explicitly mounted path as the graphroot (): it works correctly without any problems.

I guess some logic (either in CRI-O/containers or in kubelet) tries to find the "mountpoint" of the filesystem used for graphroot or something related and can't do that correctly when graphroot is located in a btrfs subvolume that is not explicitly mounted somewhere.
Ah, I see what you meant by saying that is not supported in . I haven't checked that since I created k8s clusted with default (overlay) driver and later migrated to . It is possible that disallows using because of problems like the one described above. But in my experience, when a subvolume is mounted explicitly, the btrfs works as expected - at least I didn't have problems with it.
Can the fix be backported to 1.19? only supports updating one version at a time: () So this leaves people on btrfs with with no upgrade path to which is not really nice.
We've discussed this issue at Regarding Maybe, there are quite a lot of people using btrfs than I expected. What do you think about this? Thanks!
I am running up against this issue as well. I am stuck with using k8s on 1.18
It sounds like for full fix we'll need to cherrypick to be included in k8s 1.19 and 1.20. To do that the following is needed: to cAdvisor v0.37 and v0.38 branches k8s 1.19 uses cAdvisor v0.37.3 - k8s 1.20 uses cAdvisor v0.38.6 - new cAdvisor patch release for v0.37 and v0.38 against k8s on 1.19 and 1.20 bumping to newly released cAdvisor (v0.37 / v0.38) with the fix I'm happy to cut the new cAdvisor release from my side if you want to proceed and think this is critical enough bug that is worth cherrypicking.
New cAdvisor patch versions have been cut with the BTRFS fix: v0.37.4 - v0.38.7 - /cc


Question: } func isNotFound(err error) bool { <del> e, ok := err.(*gophercloud.ErrUnexpectedResponseCode) return ok && e.Actual == http.StatusNotFound </del> <ins> if _, ok := err.(gophercloud.ErrDefault404); ok { return true } if errCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok { if errCode.Actual == http.StatusNotFound { return true } } return false </ins> } // Zones indicates that we support zones

Is this a BUG REPORT or FEATURE REQUEST?: BUG /kind bug What happened: when using cloud controller manager, function is not working correctly. Common function is not return correct value if instance is deleted from cloudprovider. What you expected to happen: I except that code should end to row if node is deleted from cloudprovider. How to reproduce it (as minimally and precisely as possible): kubernetes cluster to openstack cloudprovider using cloud controller manager debug print after this row cluster is up and running, delete one node from openstack and see the cloud controller log file result is If this works correctly it should return nil as error, from this row But now the code is not going there because is not working correctly. I debugged this function, and when instance is deleted from openstack this row variable e is and ok is . So it is not working as should. Anything else we need to know?: I am thinking that is this working correcly in any other place currently in openstack cloudprovider. Environment: Kubernetes version (use ): master Cloud provider or hardware configuration: openstack mitaka cc as you are author of this function. I am thinking could we just fix that using dirty way where ErrResNotFound is constant
/sig openstack
is this now fixed with the changes referenced?
yes all fixed this can be closed


Question: t.setConfigMap(&v1.ConfigMap{Data: map[string]string{ "Corefile": fmt.Sprintf(`.:53 { health <ins> ready </ins> kubernetes %v in-addr.arpa ip6.arpa { pods insecure upstream

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: ci-kubernetes-e2e-gci-gce-serial Which test(s) are failing: [sig-network] DNS configMap nameserver [IPv4] Change stubDomain should be able to change stubDomain configuration [Slow][Serial] [sig-network] DNS configMap nameserver [IPv4] Forward external name lookup should forward externalname lookup to upstream nameserver [sig-network] DNS configMap nameserver [IPv4] Forward PTR lookup should forward PTR records lookup to upstream nameserver [Slow][Serial] Since when has it been failing: These tests have been failing since 5/18. Testgrid link: Reason for failure: All of these have this failure message in common: The failures coincide with for which the possible fix is . However, we are filing these failures separate from to see if there are any additional steps that need to be done for these series of tests. /cc /priority critical-urgent /kind failing test /milestone v1.15 /cc
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/assign
/remove-triage unresolved
Tests have been green since 6/1


Question: . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" <del> "time" </del> ) // These tests exercise the Kubernetes expansion syntax $(VAR).

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Looks like the newly tests are failing: https://k8s-
/assign Also random question to why is the same test run 4 times?
GitHub didn't allow me to assign the following users: kevtaylor. Note that only and repo collaborators can be assigned. <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
cc
can anyone shed some light on why events are not being returned from a pod in the alpha suite? This has a 1 minute pod timeout and everything works fine locally
Reiterating the mentions to trigger a notification: <detailsIn response to : can anyone shed some light on why events are not being returned from a pod in the alpha suite? This has a 1 minute pod timeout and everything works fine locally Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I suggest unless anyone can define why the events do not appear, we disable that part of the test and just check for error. If you are okay with that please reopen the bug and I'll put in a new fix
it looks like node alpha suite is having some other issues right now that are preventing the tests from being run. So it's not clear right now if the fix helped or not.
it failed again. I did a comparison of a locally running one and the alpha suite. In the log file https://k8s- there are these references But in the locally running tests I don't see that step This running And the events running locally
/reopen


Question: github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= <del> github.com/google/cadvisor v0.38.6 h1:5vu8NaOqsBKF6wOxBLeDPD7hcmxfg/4I5NZGYXK7gIo= github.com/google/cadvisor v0.38.6/go.mod h1:1OFB9sOOMkBdUBGCO/1SArawTnDscgMzTodacVDe8mA= </del> <ins> github.com/google/cadvisor v0.38.7 h1:ZWyUz+23k1PRmEA+yrnDGtEC6IuU4Vc6439x2NQLHnA= github.com/google/cadvisor v0.38.7/go.mod h1:1OFB9sOOMkBdUBGCO/1SArawTnDscgMzTodacVDe8mA= </ins> github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Kubelet v1.19.0 doesn't start on nodes with CRI-O 1.17.2 (storagedriver="btrfs") with BTRFS as root filesystem. Related/previous issues: This error shows up in the journal: The is referring to the btrfs root filesystem, mounted at the time. : : What you expected to happen: Kubelet starts and detects the filesystem backing , even if it is BTRFS. How to reproduce it (as minimally and precisely as possible): Install kubeadm/kubectl 1.19.0 Install CRI-O, I used 1.17.2 from: Configure CRI-O to use Perform "kubeadm init" creating a new cluster Observe journal (check for kubelet issues) Anything else we need to know?: A possible workaround is to make sure a bind mount exists which allows kubelet's logic to find the backing fileystem. Eg. add the following fstab entry and then perform : The resulting mount should look somewhat like this: The workaround has been tested and seems to make kubelet work as expected. Environment: Kubernetes version: 1.19.0 OS: Debian Bullseye (Testing) Kernel: Linux 5.8.5 Others: The package has been installed
/sig node
cc xref
I am encounter the same issue. thank you for your workaround. It seems to work.
I got the same issue with latest stable Docker on Debian (btrfs) and k8s v1.19.2.
I confirm the same issue shows up with on btrfs (on LVM on LUKS, Debian buster), running k3s. No btrfs subvols. The workaround works for me; for k3s I also need to bind mount . I did not experience this issue with 1.18.
This also reproduces with Docker runtime. Workaround with bind mount helps, however automatically unmounts it so don't get confused. is on root that is mounted like this: , and docker is configured to use .

Reproduced with k8s 1.19.3 on Gentoo. The rootfs is btrfs, is on rootfs, docker is configured with storage drigver . automatically mount and unmount it on . Also confirmed bind mount workaround helps.
/assign
I think almost anyone using BTRFS as the root filesystem has encountered this issue. If is merged and applied to the kubernetes, this can be fixed. Please take a look at this ?
The has been merged to google/cadvisor. I think if Kubernetes v1.21 includes the fix by using google/cadvisor v0.39, the issue will be resolved.
Just a note that I too have stumbled on this issue after reconfiguring CRI-O storage to use btrfs driver pointing with graphroot directly to a BTRFS subvolume (the root filesystem is not BTRFS). After reading this issue I tried mounting the BTRFS subvolume explicitly on some mountpoint and pointed with graphroot to this explicit subvolume mountpoint which solved the problem. The problem seems to occur only when pointing to a subvolume within some parent subvolume as this is not a real mountpoint.
Ok, so I just want to make sure I understand. Currently when i try to run kubeadm to create a new cluster while using btrfs, it gives me With Kubernetes v1.21, that will no longer be the case?
In my experience (with CRI-O 1.19 and k8s 1.19), the btrfs driver (in ) is supported, but the combination does not work correctly when graphroot is pointed to a btrfs subvolume within some parent subvolume (I don't have experience with graphroot poited to some subdirectory of a default subvolume and/or when default subvolume is also a root filesystem, just when pointed directly to a subvolume within some parent subvolume which is not a root filesystem). For example, when btrfs filesystem is mounted like the following (): when I then create a subvolume: then use it as a graphroot (): But when I explicitly mount the subvolume to some mountpoint (): then use the explicitly mounted path as the graphroot (): it works correctly without any problems.

I guess some logic (either in CRI-O/containers or in kubelet) tries to find the "mountpoint" of the filesystem used for graphroot or something related and can't do that correctly when graphroot is located in a btrfs subvolume that is not explicitly mounted somewhere.
Ah, I see what you meant by saying that is not supported in . I haven't checked that since I created k8s clusted with default (overlay) driver and later migrated to . It is possible that disallows using because of problems like the one described above. But in my experience, when a subvolume is mounted explicitly, the btrfs works as expected - at least I didn't have problems with it.
Can the fix be backported to 1.19? only supports updating one version at a time: () So this leaves people on btrfs with with no upgrade path to which is not really nice.
We've discussed this issue at Regarding Maybe, there are quite a lot of people using btrfs than I expected. What do you think about this? Thanks!
I am running up against this issue as well. I am stuck with using k8s on 1.18
It sounds like for full fix we'll need to cherrypick to be included in k8s 1.19 and 1.20. To do that the following is needed: to cAdvisor v0.37 and v0.38 branches k8s 1.19 uses cAdvisor v0.37.3 - k8s 1.20 uses cAdvisor v0.38.6 - new cAdvisor patch release for v0.37 and v0.38 against k8s on 1.19 and 1.20 bumping to newly released cAdvisor (v0.37 / v0.38) with the fix I'm happy to cut the new cAdvisor release from my side if you want to proceed and think this is critical enough bug that is worth cherrypicking.
New cAdvisor patch versions have been cut with the BTRFS fix: v0.37.4 - v0.38.7 - /cc


Question: {lesser: podWithRank{highPodDeletionCost, 2}, greater: podWithRank{lowPodDeletionCost, 1}, disablePodDeletioncost: true}, {lesser: podWithRank{ready2Hours, 1}, greater: podWithRank{ready5Hours, 1}}, } for i, test := range inequalityTests { <del> t.Run(fmt.Sprintf("test%d", i), func(t *testing.T) { </del> <ins> t.Run(fmt.Sprintf("Inequality tests %d", i), func(t *testing.T) { </ins> defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDeletionCost, !test.disablePodDeletioncost)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LogarithmicScaleDown, !test.disableLogarithmicScaleDown)()

During implementation of KEP 3393 (PodReplacementPolicy for Jobs) we found that the unit tests of do not follow common patterns established in k/k. 1) The tests are not written using table driven design (). 2) The tests use rather than . This probably should be done as two separate PRs but wanted to highlight some areas that could use some cleanup. Not using means that we don't see diffs in the unit tests that makes this difficult. And not using table driven tests makes it difficult to add new test cases as it usually is brought up during review that we should use a table-driven test but the entire file does not obey this format.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig apps /kind cleanup /help /good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /sig apps /kind cleanup /help /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I'd like to work on it /assign
I'd also like to make some contributions. /assign
Looks like this need two PRs. tests are not written using table driven design (). tests use Differences rather than Cmp.Diff. I think I can work on the 2nd one and leave the 1st one to you.

Sounds good?
By looking through the , I found the main function to comapre the results have already used cmp.Diff. Can you elaborate more on which part you expect to use cmp.Diff? func verifyDeletedAndPatchedPods(t testing.T, client fake.Clientset, wantDeletedPodNames, wantPatchedPodNames sets.String) { t.Helper() deletedPodNames := getDeletedPodNames(client) if diff := cmp.Diff(wantDeletedPodNames, deletedPodNames); diff != "" { t.Errorf("Deleted pod names (-want,+got):n%s", diff) } patchedPodNames := getPatchedPodNames(client) if diff := cmp.Diff(wantPatchedPodNames, patchedPodNames); diff != "" { t.Errorf("Patched pod names (-want,+got):n%s", diff) } }
I am really sorry about this. Too much multitasking!! I meant not
Hey ! Sorry, I didn't see this before. No problem at all, but consider that one change depends on the other, and since my task implies a big refactor, there might be conflicts when merging your PR... That's why I see two possible scenarios happening: Your PR gets merged before I raise mine: Then fine, I'll rebase your changes and adapt my code Mine is ready before yours is merged: In this case, I propose you commit your changes to my PR. Both of us will appear as coauthors of the change Getting a PR merged in the Kubernetes project takes quite some time (trust me, I talk by experience and can confirm ) WDYT?
Let's see how it goes. We can always rebase the code change to adapt new.
btw, did you already begin to work on task 1? If not, I may just go ahead and work on it right away since I have already dig into it.
yes, I already started And If you ask me, better to keep it this way so we both have an opportunity to have a contribution
cool. Sounds good.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hi if this issue needs to be worked on any further, please assign me for the same.
has a PR up that will solve the second part of this issue.


Question: pod:          getPod("pod1", "container1", requirementsGuaranteed), topologyHint: &topologymanager.TopologyHint{Preferred: true}, }, <ins> { description: "should validate NUMA node can not have both single and cross NUMA node memory allocations", assignments: state.ContainerMemoryAssignments{ "pod1": map[string][]state.Block{ "container1": { { NUMAAffinity: []int{0}, Type:         v1.ResourceMemory, Size:         1024 * mb, }, }, }, }, expectedAssignments: state.ContainerMemoryAssignments{ "pod1": map[string][]state.Block{ "container1": { { NUMAAffinity: []int{0}, Type:         v1.ResourceMemory, Size:         1024 * mb, }, }, }, }, machineState: state.NUMANodeMap{ 0: &state.NUMANodeState{ MemoryMap: map[v1.ResourceName]*state.MemoryTable{ v1.ResourceMemory: { Allocatable:    1536 * mb, Free:           512 * mb, Reserved:       1024 * mb, SystemReserved: 512 * mb, TotalMemSize:   2176 * mb, }, hugepages1Gi: { Allocatable:    gb, Free:           gb, Reserved:       0, SystemReserved: 0, TotalMemSize:   gb, }, }, Cells:               []int{0}, NumberOfAssignments: 1, }, 1: &state.NUMANodeState{ MemoryMap: map[v1.ResourceName]*state.MemoryTable{ v1.ResourceMemory: { Allocatable:    512 * mb, Free:           512 * mb, Reserved:       0, SystemReserved: 512 * mb, TotalMemSize:   2176 * mb, }, hugepages1Gi: { Allocatable:    gb, Free:           gb, Reserved:       0, SystemReserved: 0, TotalMemSize:   gb, }, }, Cells:               []int{1}, NumberOfAssignments: 0, }, }, expectedMachineState: state.NUMANodeMap{ 0: &state.NUMANodeState{ MemoryMap: map[v1.ResourceName]*state.MemoryTable{ v1.ResourceMemory: { Allocatable:    1536 * mb, Free:           512 * mb, Reserved:       1024 * mb, SystemReserved: 512 * mb, TotalMemSize:   2176 * mb, }, hugepages1Gi: { Allocatable:    gb, Free:           gb, Reserved:       0, SystemReserved: 0, TotalMemSize:   gb, }, }, Cells:               []int{0}, NumberOfAssignments: 1, }, 1: &state.NUMANodeState{ MemoryMap: map[v1.ResourceName]*state.MemoryTable{ v1.ResourceMemory: { Allocatable:    512 * mb, Free:           512 * mb, Reserved:       0, SystemReserved: 512 * mb, TotalMemSize:   2176 * mb, }, hugepages1Gi: { Allocatable:    gb, Free:           gb, Reserved:       0, SystemReserved: 0, TotalMemSize:   gb, }, }, Cells:               []int{1}, NumberOfAssignments: 0, }, }, systemReserved: systemReservedMemory{ 0: map[v1.ResourceName]uint64{ v1.ResourceMemory: 512 * mb, }, 1: map[v1.ResourceName]uint64{ v1.ResourceMemory: 512 * mb, }, }, pod:           getPod("pod2", "container1", requirementsGuaranteed), topologyHint:  &topologymanager.TopologyHint{NUMANodeAffinity: newNUMAAffinity(0, 1), Preferred: true}, expectedError: fmt.Errorf("[memorymanager] preferred hint violates NUMA node allocation"), }, </ins> } for _, testCase := range testCases {

Dual socket server with 96 threads total (2242), ~192G of RAM, cpu & memory manager static policy, topologyManagerPolicy best-effort, 10Gi of RAM reserved on NUMA node 0, 1 core (2 threads) reserved on NUMA node 0 kubeadm config: If I try to allocate 2 Guaranteed pod with 85Gi of RAM each, one pod is admitted, and the second one fails with even if it would fit using memory of both NUMA nodes. As I'm using deployments a new pod is recreated right away and you end up with tons of failed pod in state. Error in the logs is: Either the pod becomes pending, or memory from both NUMA nodes is used Have some memory reserved on NUMA 0, launch 2 identical pods with memory limits really close to the max so 1 pod fit on NUMA 1 but the second doesn't fit on NUMA 0 (the sum fits on the server) Here the If I launch 1 pod with 46 CPUs / 170Gi RAM, I get CPUs from 1 NUMA nodes, and memory from both NUMA nodes <details</detailsIf I launch a pod with 46 CPUs / 85Gi RAM, it's properly placed on NUMA 1 <details</detailsIf I try to launch 2 pods with 46 CPUs / 85Gi RAM, one fails with If I try to launch 2 pods with 46 CPUs / 80Gi RAM, everything works 26.8 NONE / bare-metal Alma 8.8 base + rpm-ostree kubeadm containerd 1.7.5 No response
/sig node
/cc Hi! Just to clarify, the memory manager can't keep the pod pending, this is a scheduler feature and builtin kube scheduler (= any of its plugin) is not NUMA-aware atm. So in this case memory manager should extend the pod to both NUMA zones, and failing to do so is likely the issue
Hello Is it possible that you are hitting the exact scenario describe at the KEP: ? Please check the first two steps.
I was going to say I'm hitting the expected unexpected error, but I'm not sure the KEP doc 100% match the current memoryManager behavior (at least when using ).

test 1: the "reject pod2 from the KEP" actually works: start pod 60 cpus / 120Gi RAM wait start pod 20 cpus / 20Gi RAM <details</detailstest 2 (start a 1 NUMA node container then one than spans 2 NUMA node) also works: start pod 20 cpus / 20Gi RAM wait start pod 60 cpus / 120Gi RAM <details</detailsWhat doesn't work is when you get too close to the limits it seems
Interesting those pods are guaranteed I presume? And what about the total node memory capacity? When you check the node status do you have enough memory left to fulfill the second pod request?
click on details you will have the content of yes they are guaranteed else they would not show up in memorymanagerstate it's less then the total node capacity else pod would be pending
/assign
assigned to me for bookkeeping reason but is doing the real work here
Hello so far I didn't manage to reproduce the issue you're describing. In my case I manage to ran both pods, or end up with the second one . However, I managed to find something similar (different affinity error) but with topology manager policy and not as you described which I'm going to open a separate issue for. Could you please share the journal logs while you're reproducing the issue again? Basically on the node where the pod is about to land, share the output
One more thing please share output of the node before and after you reproduce the issue
/triage accepted /priority important-longterm
off until Tuesday, I'll try to reproduce with 1.28.2 then and share the logs next week
NP. I won't be here next week either, so no rush
Hello did you manage to reproduce the issue?
yes, reproduced on my dev machine in 1.28.2, tell me what else you need / if you want me to rebuild kubelet with some debug that's also possible <details</details
Hi I did mange to reproduce the issue, but following the code it seems exactly like I mentioned in my previous I'll try to elaborate: When the first pod gets admitted it placed on NUMA 0 (in my case).

When the second pod gets admitted, there is not enough memory on each NUMA individually (NUMA 0 or NUMA1) to satisfy the memory request, so memory manager tries to span the memory allocation through NUMA 0 and NUMA 1. But it failed short in this line: because NUMA node 0 is now used for single-NUMA memory allocation which align with the KEP . In such case, the pod cannot move to because is done at the scheduler level, while from the scheduler's POV there is enough memory on the node to satisfy the request. Now, about your comment here: wait start pod 20 cpus / 20Gi RAM Make sense because both NUMA 0 and NUMA 1 are already serves mulit-NUMA allocation, since the first pod spans across the two NUMAs, the second pod can squeezed into one of them with no problems. wait start pod 60 cpus / 120Gi RAM This case is more interesting and I didn't manage to reproduce it - I always got which is align with the KEP. Could you please increase Kubelet logs verbosity, reproduce test 2 and attached the following information: logs file before 1 manifest file after pod1 2 manifest file after pod2 Thanks
Hi this is a different server with a bit less CPUs, but test 2 still works for me for verbosity I've used v=99 instead of v=2 Here part of my KubeletConfiguration The logs: And the commands: <details</details
I think I pinpointed a bit more the issue For my test2 pod the cpu manager generated a hint with both NUMA nodes. For test2 pod, if CPU request span both NUMA nodes, then the memory allocation will work whether it requires to use 1 or 2 NUMA nodes If CPU request only requires 1 NUMA node, then it'll fail if memory allocation need to extend to 2 NUMA node.
Hello , After carefully going through the code and the KEP I think that the behavior you're expecting: "Either the pod becomes pending, or memory from both NUMA nodes is used" is not aligned with the architecture.

Let's break it down: "The pod becomes pending" - From the default scheduler POV, the node have enough resources (because the scheduler is not NUMA aware) so the fact that the pod scheduled on this node is align with the expected behavior. Sidenote: if you want to enable NUMA aware scheduling you can take a look at "memory from both NUMA nodes is used" - Because NUMA 0 is already serves as single NUMA memory allocation, it cannot serves as multi NUMA memory allocation as well, otherwise the memory pinning is not guaranteed since linux kernel doesn't support multi-NUMA allocation (pinning). What it can do is grouping NUMA 0 with NUMA 1 into a single group but again this is against the original design: Please check and look for "No spare capacity, reject Pod 7" (last example) And you were right, when the hint generated by cpu manager, spans for both NUMAs the pod gets admitted, and that's exactly where actual bug is - the pod should never get admitted. IOW you were suppose get the error: So the fix here should be preventing the pod admission even when the CPU manager hint spans across both NUMAs. Now, in your case, if you still want and need both pods to get admitted you can change the from to because as I mentioned, the memory pinning in your scenario is not guaranteed anyway.
Hello I still think the bug is that memoryManager doesn't respect topologyManagerPolicy: best-effort. For now I'm exploring using a simple NRI plugin to set the to the nodes of
, The best-effort means that even if the affinity is not preferred, the pod would get admit anyway. But here we're talking about respecting the NUMA node grouping done by memory manager. This is an architectural design we should refer to the original creator memory manager or at least to topology maintainers folks. do you know who can answer on such questions?
some additional info about the limitation i mentioned above


Question: errs = append(errs, err) } <ins> // Create/update RBAC rules that makes the 1.8.0+ nodes to rotate certificates and get their CSRs approved automatically if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client, k8sVersion); err != nil { errs = append(errs, err) } </ins> // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade // Create the cluster-info ConfigMap with the associated RBAC rules // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {

Reported in Scenario (CentOS 7): install 1.7.x packages for kubelet on nodes. install cluster with kubeadm init stable-1.7 use kubeadm to upgrade control plane to 1.8.0-rc.1 on nodes which still run 1.7.x kubelet, install new rpms for where certificate-rotation is enabled. systemctl daemon-reload, systemctl restart kubelet. result: kubelet is in wait loop, waiting for csr to be approved. node is in NotReady state. The kubelet blocks on startup, despite having a valid kubeconfig file. It should start up, attempt to rotate in the background, but not block.
I think this is because of the bootstrap kubeconfig flag. If you only provide the cert rotation flag it should only bootstrap in the background. That's what we where doing for bootkube where we need to start static pods while we're doing our initial cert rotation
a bit of debug shown that returns on the last line . So, kubelet has valid cert, but still wants to rotate it. That in theory should be a valid scenario. Question is: why it's not automatically approved then ? Even if cert will be in proper place, does that mean that automatic rotation wouldn't work for expired certs ? Or I'm missing something here.
no, it's because when a file is provided containing only cert/key data, the cert manager blocks on Start() until a CSR is approved
proposed fix for unblocking Start() in , which runs rotation in the background if you have a starting certificate
/cc
what does set up for auto-approval of node self certificate requests during an upgrade to 1.8?
for cert manager hang for kubeadm auto-approval setup issues upgrading to 1.8
looks like kubeadm upgrade instructions indicate approval of CSRs for upgrades nodes is expected: http://kubernetes-io-vnext- I'd expect that only to be required once certs expired, not automatically required as part of upgrade
Yeah, I that step to the kubeadm upgrade instructions last-minute today when the problem was discovered and we didn't think we would have a solution in time for 1.8.0. Manually approving the CSRs seems to work fine, but is a less-than-stellar upgrade experience.

lowering priority a notch, since the upgrade procedure is at least documented, and does not default to true for all installations (specific deployment methods like kubeadm enable it)
In case it helps, kubeadm 1.8 code for starting the controller manager is , and the RBAC rules for auto-approval are .
Also, I'm not sure if we have any other docs that need to be updated regarding upgrading kubelet, since this isn't a kubeadm-specific problem.
the combination of enabling rotation and not setting up auto-approval is what bit kubeadm here is a deployment-specific choice, and is not true by default. is the only deployment I see that enables it on upgrade to 1.8 the auto-approval rules at do not enable nodes to rotate their client certificates, that only sets up auto-approval for the initial TLS bootstrapping token I agree the kubelet should behave better in this scenario if it already has a valid credential (like ), but the configured setup by kubeadm doesn't seem correct
Ah, understood. It wasn't until I was looking at the RBAC rules myself just now that I realized we must have missed that. Definitely needs to get fixed in kubeadm.
I've created a PR for the documentation fix for kubeadm upgrades: We still need a kubeadm code fix to add the RBAC rules, hopefully to get into 1.8.1.
Yeah, not creating the ClusterRoleBinding between and was an oversight, but isn't that critical in any way. We'll make that change in v1.8.1, and document things meanwhile. Upgrading your kubelets is a manual process anyway right now, so it's reasonable to add one extra step to create this binding. Is it expected that a kubelet that already has credentials should renew them on restart if is set?
Not in a blocking way, only in the background if needed because of expiration, see
Reopening. Blocking on start is still an issue if auto approval is not enabled.
Fix in
Should this have the v1.8 milestone attached to it?
is everything here now fixed or are we waiting for
ping on the status of
- needs rebase and the mentioned in there is merged. What else needs to be done?

[MILESTONENOTIFIER] Milestone Removed From Issue Important: Code freeze is in effect and only issues with may remain in the v1.9 milestone. <details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
I think this issue is now solved right?


Question: ownerReferences: []metav1.OwnerReference{ { APIVersion: "customresourceVersion", <del> Kind:       "customresourceKind", </del> <ins> Kind:       "customresourceKind1", </ins> Name:       "name", UID:        "1", Controller: &falseVar, }, { APIVersion: "customresourceVersion", <del> Kind:       "customresourceKind", </del> <ins> Kind:       "customresourceKind2", </ins> Name:       "name", UID:        "2", Controller: &trueVar, }, { APIVersion: "customresourceVersion", <del> Kind:       "customresourceKind", </del> <ins> Kind:       "customresourceKind3", </ins> Name:       "name", UID:        "3", Controller: &trueVar, }, { APIVersion: "customresourceVersion", <del> Kind:       "customresourceKind", </del> <ins> Kind:       "customresourceKind4", </ins> Name:       "name", UID:        "4", }, }, expectError:          true, <del> expectedErrorMessage: "Only one reference can have Controller set to true", </del> <ins> expectedErrorMessage: "Only one reference can have Controller set to true. Found "true" in references for customresourceKind2/name and customresourceKind3/name", </ins> }, }

When there are more than one controller set to true within ownerRef, the validation would return err msg like this: It is confusing to see two same 'foo' controller name. To make the err msg more accurately, like this: Set an object with two , and both of the Controller set true, then apply to the apiserver will got this error.
/sig api-machinery
/triage accepted


Question: <del> amd64=gcr.io/google-containers/ubuntu-slim-amd64:0.12 arm=gcr.io/google-containers/ubuntu-slim-arm:0.12 arm64=gcr.io/google-containers/ubuntu-slim-arm64:0.12 ppc64le=gcr.io/google-containers/ubuntu-slim-ppc64le:0.12 </del> <ins> amd64=gcr.io/google-containers/debian-base-amd64:0.1 arm=gcr.io/google-containers/debian-base-arm:0.1 arm64=gcr.io/google-containers/debian-base-arm64:0.1 ppc64le=gcr.io/google-containers/debian-base-ppc64le:0.1 </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: BASEIMAGE for resource-consumer was ubuntu before and there is a ask for migrating to debian via - What you expected to happen: Use debian-base as BASEIMAGE for resource-consumer test image How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration**: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig testing


Question: expectedNumSlices:     1, expectedClientActions: 0, }, { <ins> testName: "Endpoints with 1 subset, port, and address and existing slice with same fields but different OwnerReferences", subsets: []corev1.EndpointSubset{{ Ports: []corev1.EndpointPort{{ Name:     "http", Port:     80, Protocol: corev1.ProtocolTCP, }}, Addresses: []corev1.EndpointAddress{{ IP:       "10.0.0.1", Hostname: "pod-1", }}, }}, uid: "d8f2c1f6-5285-4b3c-b3c1-9b89f9e7ed7a", existingEndpointSlices: []*discovery.EndpointSlice{{ ObjectMeta: metav1.ObjectMeta{ Name:      "test-ep-1", Namespace: "test", OwnerReferences: []metav1.OwnerReference{ { APIVersion: "v1", Kind:       "Endpoints", Name:       "test-ep", UID:        "fb91e798-1875-4857-b5eb-e2c878157b4d", }, }, }, AddressType: discovery.AddressTypeIPv4, Ports: []discovery.EndpointPort{{ Name:     pointer.String("http"), Port:     pointer.Int32(80), Protocol: &protoTCP, }}, Endpoints: []discovery.Endpoint{{ Addresses:  []string{"10.0.0.1"}, Hostname:   pointer.String("pod-1"), Conditions: discovery.EndpointConditions{Ready: pointer.Bool(true)}, }}, }}, expectedNumSlices:     1, expectedClientActions: 1, }, { </ins> testName: "Endpoints with 1 subset, port, and address and existing slice with an additional annotation", subsets: []corev1.EndpointSubset{{ Ports: []corev1.EndpointPort{{

create a service without selector, and manual create endpoints for this service endpointslicemirroring controller will create an endpointslice for this service. after endpoint slice created, and kube-controller-manager restart. during the kube-controller-manager restart, the endpoint was deleted and recreated after kube-controller-manager restart, garbagecollector controller delete this endpointslice and endpointslice not recreate until endpoint/service update event or kube-controller-manager restart endpointslice will be recreated create a service without selector, and manual create endpoints for this service endpointslicemirroring controller will create an endpointslice for this service. after endpoint slice created, and kube-controller-manager restart. during the kube-controller-manager restart, the endpoint was deleted and recreated No response <details</details<details</details<details</details<details</details<details</details<details</details
/sig network
/assign
Trying to understand this, mostly just restating what you've said above to make sure I'm getting it: during the kube-controller-manager restart, the endpoint was deleted and recreated after kube-controller-manager restart, garbagecollector controller delete this endpointslice and endpointslice not recreate until endpoint/service update event or kube-controller-manager restart So an Endpoints resource is being deleted while kube-controller-manager is not running. The EndpointSlice mirroring controller to point to the Endpoints they're mirroring. When kube-controller-manager is restarted, the garbage collector will delete the mirrored EndpointSlice because it's owned by an Endpoints resource that was previously deleted. That deletion by garbage collection would then be , but if the EndpointSliceTracker doesn't think it should have that resource, it will ignore the deletion and not reconcile, resulting in the mirrored EndpointSlice not being recreated until something updates the Service or Endpoints resource.
/triage accepted


Question: <ins> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package builder import ( "reflect" "testing" "k8s.io/kube-openapi/pkg/spec3" "k8s.io/kube-openapi/pkg/validation/spec" ) func TestMergeSpecV3(t *testing.T) { tests := []struct { name     string specs    []*spec3.OpenAPI expected *spec3.OpenAPI }{ { name: "oneCRD", specs: []*spec3.OpenAPI{{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, }, }, }, }, expected: &spec3.OpenAPI{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, }, }, }, }, { name: "two CRDs same gv", specs: []*spec3.OpenAPI{{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "com.example.stable.v1.CRD1": {}, "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, }, }, }, { Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd2": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "com.example.stable.v1.CRD2":                      {}, "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, }, }, }, }, expected: &spec3.OpenAPI{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, "/apis/stable.example.com/v1/crd2": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, "com.example.stable.v1.CRD1":                      {}, "com.example.stable.v1.CRD2":                      {}, }, }, }, }, { name: "two CRDs with scale", specs: []*spec3.OpenAPI{{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "com.example.stable.v1.CRD1": {}, "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, "io.k8s.api.autoscaling.v1.Scale":                 {}, }, }, }, { Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd2": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "com.example.stable.v1.CRD2":                      {}, "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, "io.k8s.api.autoscaling.v1.Scale":                 {}, }, }, }, }, expected: &spec3.OpenAPI{ Paths: &spec3.Paths{ Paths: map[string]*spec3.Path{ "/apis/stable.example.com/v1/crd1": {}, "/apis/stable.example.com/v1/crd2": {}, }, }, Components: &spec3.Components{ Schemas: map[string]*spec.Schema{ "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {}, "com.example.stable.v1.CRD1":                      {}, "com.example.stable.v1.CRD2":                      {}, "io.k8s.api.autoscaling.v1.Scale":                 {}, }, }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { merged, err := MergeSpecsV3(tt.specs...) if err != nil { t.Error(err) } if !reflect.DeepEqual(merged, tt.expected) { t.Error("Merged spec is different from expected spec") } }) } } </ins>

Related to Seeing the following errors in kube-apiserver logs when CRDs are applied: No errors in logs Apply any CRD manifest No response <details</details<details</details<details</details<detailsRancher RKE2 </details<detailscontainerd 1.6.4 </details<detailscanal </details
/sig api-machinery
/assign
Did this cause any specific trouble to your cluster I'm curious what the impact/severity of this is.
Everything seems to work fine, but there's a bunch of log spew every time a CRD manifest is loaded. In K3s we mix all the control-plane component logs together in a single pair of stdout/stderr streams so it's quite noisy, compared to previous releases. The fact that these come out at WARN level with a scary prefix makes them look pretty serious, and I expect we will get a fair bit of users asking about it once we release.
/triage accepted


Question: "text/tabwriter" "time" <ins> "k8s.io/client-go/tools/cache" </ins> "github.com/onsi/ginkgo" "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1"

[x] test approval issue : [X] test pr : 26 May [x] two weeks soak end date : 9 June [x] test promotion pr : kubernetes/kubernetes According to this APIsnoop query, there are still some remaining DaemonSet endpoints which are untested. Create a watch to track daemon set events that DaemonSet with a static label Create a daemon set with a static label. Confirm that the pods are running. Get the daemon set status. Parse the response and confirm that the daemon set status conditions can be listed. Update the daemon set status. Confirm via the watch that the status has been updated. Patch the daemon set status. Confirm via the watch that the status has been patched. Using an existing as a template for a new which validates that three new endpoints are hit. This query shows the endpoints hit within a short period of running the e2e test If a test with these calls gets merged, test coverage will go up by 3 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: } }) <del> Context("detach from a disrupted node [Slow] [Disruptive]", func() { </del> <ins> Context("detach in a disrupted environment [Slow] [Disruptive]", func() { </ins> const ( deleteNode    = 1 // delete physical node deleteNodeObj = 2 // delete node's api object only <ins> evictPod      = 3 // evict host0Pod on node0 </ins> ) type testT struct { <del> descr  string // It description nodeOp int    // disruptive operation performed on target node </del> <ins> descr     string // It description disruptOp int    // disruptive operation performed on target node </ins> } tests := []testT{ { <del> descr:  "node is deleted", nodeOp: deleteNode, </del> <ins> descr:     "node is deleted", disruptOp: deleteNode, </ins> }, { <del> descr:  "node's API object is deleted", nodeOp: deleteNodeObj, </del> <ins> descr:     "node's API object is deleted", disruptOp: deleteNodeObj, }, { descr:     "pod is evicted", disruptOp: evictPod, </ins> }, } for _, t := range tests { <del> nodeOp := t.nodeOp </del> <ins> disruptOp := t.disruptOp </ins> It(fmt.Sprintf("when %s", t.descr), func() { framework.SkipUnlessProviderIs("gce") origNodeCnt := len(nodes.Items) // healhy nodes running kublet

When pod is evicted, pod is terminated. After this change , attach/detach controller should be able to detach the pod correctly so that new pod which uses the same volume will start up. We need to have an e2e test for this scenario.
what environment is good for this e2e test? GCE or GKE or AWS? I see we have which has func, Can you comment on why is insufficient to cover this issue?
does the pod need to be evicted? Or would be ok to delete the pod and then check that the PD is detached. Then create a new pod using the same PD and test that it attaches?
[MILESTONENOTIFIER] Milestone Removed Important: This issue was missing labels required for the v1.9 milestone for more than 3 days: kind: Must specify exactly one of , or . <details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
ping ^^
sorry for the delay. Here we want to test the detach behavior of evicted pods. So delete a pod directly is different from it. When Pod is evicted, the containers will be killed and the pod status is terminated. But the pod api object is not deleted from API server. Before PR , such evicted pods will not trigger detach operation because the pod api object is not deleted from api server. So the purpose of the test is to make sure detach operation is triggered for such evicted pods.
Thanks Do you have a suggestion for how to force a pod to be evicted?
You could have the pod do some memory intensive application
ah, like my use of in the io-streaming pr :-)
- Here's another option to cause eviction: .


Question: glog.Errorf("Photon Controller attacher: Attach failed to get volume source") return "", err } <ins> attached, err := attacher.photonDisks.DiskIsAttached(volumeSource.PdID, nodeName) </ins> <del> glog.V(4).Infof("Photon Controller: Attach disk called for host %s", hostName) // TODO: if disk is already attached? err = attacher.photonDisks.AttachDisk(volumeSource.PdID, nodeName) </del> if err != nil { <del> glog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.PdID, nodeName, err) return "", err </del> <ins> glog.Warningf("Photon Controller: couldn't check if disk is Attached for host %s, will try attach disk: %+v", hostName, err) attached = false } if !attached { glog.V(4).Infof("Photon Controller: Attach disk called for host %s", hostName) err = attacher.photonDisks.AttachDisk(volumeSource.PdID, nodeName) if err != nil { glog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.PdID, nodeName, err) return "", err } </ins> } PdidWithNoHypens := strings.Replace(volumeSource.PdID, "-", "", -1)

Currently we don't check if disk is already attached : from what i remember it can cause some problems, but not sure, sorry for the mass tag
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig storage /assign


Question: // CustomResourceDefinition represents a resource that should be exposed on the API server.  Its name MUST be in the format // <.spec.name>.<.spec.group>. <del> // Deprecated in v1.16, planned for removal in v1.19. Use apiextensions.k8s.io/v1 CustomResourceDefinition instead. </del> <ins> // Deprecated in v1.16, planned for removal in v1.22. Use apiextensions.k8s.io/v1 CustomResourceDefinition instead. </ins> message CustomResourceDefinition { optional k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta metadata = 1;

The following code snippet is very confusing: Is v1beta1 deprecated in 1.16 or 1.19? Is it to be removed in 1.19 or 1.22? Please clarify the exact versions so the many users of CRD v1beta1 APIs know what to do.
/sig api-machinery
/kind deprecation
/unkind feature
Deprecated in 1.16. Planned to be removed in 1.22. We'll update the docs to be consistent before 1.19 is released.
/assign


Question: api_v1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/testing/core" "k8s.io/kubernetes/pkg/runtime" <ins> "k8s.io/kubernetes/pkg/util/wait" </ins> "k8s.io/kubernetes/pkg/watch" "github.com/golang/glog"

https://k8s- Failed: TestNamespaceController {namespace} Previous issues for this test:
GetObjectFromChan() is timing out (20s) and returning nil causing the panic. Happens in TestSecretController (and probably TestIngressController) controller as well. https://k8s-
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
I have a fix for this in
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
/cc OWNERS of :
I think this is also related to Issue .
fix is pending


Question: // Set PodSpec subdomain field to generate FQDN for pod pod.Spec.Subdomain = subdomain // Expected Pod FQDN <del> hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", pod.ObjectMeta.Name, subdomain, f.Namespace.Name, dnsDomain) </del> <ins> hostFQDN := fmt.Sprintf("%s.%s.%s.svc.%s", pod.ObjectMeta.Name, subdomain, f.Namespace.Name, framework.TestContext.ClusterDNSDomain) </ins> output := []string{fmt.Sprintf("%s;%s;", pod.ObjectMeta.Name, hostFQDN)} // Create Pod f.TestContainerOutput("shortname and fqdn", pod, 0, output)

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: / Which test(s) are failing: - Since when has it been failing: Since was merged Testgrid link: Reason for failure: Example: Looks like the at L122 and L144 doesn't actually get printed? Anything else we need to know: /sig node /cc /priority important-soon
Hi sorry about that, I will work on it asap. Are you aware of any difference between the CI that executes NodeFeature and NodeAlphaFeature? FYI this other PR merged at the same time
/assign javidiaz
Somehow, the /etc/hosts of the pod does not have subdomain # Kubernetes-managed hosts file. 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet fe00::0 ip6-mcastprefix fe00::1 ip6-allnodes fe00::2 ip6-allrouters 172.17.0.2 hostfqdn-.t.hostfqdn- hostfqdn- I think the issue is that framework.TestContext.ClusterDNSDomain is empty String.
Hi any idea of why cluster-subdomain is empty String in e2e node tests? These tests are failing because I made the wrong assumption that cluster-subdomain should be by default. So I set as the expected subdomain when framework.TestContext.ClusterDNSDomain is empty.
/triage accepted Thank you for looking into this, and for opening the issue :pray: Given i think you found the issue. node e2e tests have theirwn entrypoint so i guess we forgot to completely account for all the settings the e2e framework sets and to use those to configure the kubelet. I think in general we need to refactor the node test suite to prevent this sort of issue. Small note: full diff for the timeframe around which these tests began to fail (betwen 11:53 EST and 12:54 EST on 11-05).


Question: } func (h healthzHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { <del> var lastQueued, lastUpdated time.Time if val := h.hs.lastQueued.Load(); val != nil { lastQueued = val.(time.Time) </del> <ins> var oldestPendingQueued, lastUpdated time.Time if val := h.hs.oldestPendingQueued.Load(); val != nil { oldestPendingQueued = val.(time.Time) </ins> } if val := h.hs.lastUpdated.Load(); val != nil { lastUpdated = val.(time.Time)

We upgraded iptables on one of our hosts, and kube-proxy began logging errors about being unable to acquire the . However, we didn't notice this, because kube-proxy's health check () was still passing (and that's generally what we look at as our sign that something is amiss.) As a result, kube-proxy ran for days without being able to update iptables rules, eventually causing an outage for one of our services. We expected that kube-proxy's health check would fail if it was not able to interface with iptables. Our system was running Debian stretch, running kube-proxy in iptables mode with systemd, with iptables version 1.6.0. Given that starting configuration, the repro case is: iptables to 1.6.2. that kube-proxy begins emitting logs saying that it cannot update iptables because another process has the . that kube-proxy's health check endpoint () does not return an error. Example health check output: <details</details<details</details<detailsNone </details<details</details<detailsThe hard way </details<detailsDocker version 19.03.8, build </details<detailsCalico v3.18.4 </details
/sig network
you upgraded iptables just one node ? what is the version of iptables on other nodes ?
does it means that is not able to never get the lock or is just that sometimes is able to get it?
Yeah we upgraded on just one node at first. Like my original bug report says, we started at 1.6.0, so all the other nodes had 1.6.0. it never gets the lock. You have to restart kube-proxy so it can go through the iptables version detection routine again and change its behavior to the 1.6.2 logic branch. That's fine, it's whatever, but the health check doesn't fail even though kube-proxy can't do its job, which is a bug in my opinion. Also yes, we checked to see if some other process had the lock. It didn't. It's something to do with changes in how iptables actually works after the upgrade to 1.6.2, specifically. I believe this launchpad issue is relevant:
the logic of the healthcheck should fail if is not able to process the updates, right?

I this to the description as well, but here is an example response during this failure mode: And here are what the logs look like: Running gives no output.
This is interesting the and values are the same. This is the healthz logic, it will be unhealthy if the difference between both is greater than the health timeout. is it possible to share the entire kube-proxy log ? I want to discard something
to save me some time, could you please tell me what sort of messages you are looking for? I would rather go search splunk for them than to set up a repro case again.
just the first 1000 lines since kube-proxy starts should be enough
The JSON is kind of meaningless these days; if the proxier thinks it's healthy, it will just write out the current time as both and . It's only when it thinks its unhealthy that it writes out the real time. See I'm not sure how this could happen...
I'm afraid I'm not at liberty to give you 1000 lines of logs without some more targeted context. Company policy and whatnot. What I will tell you is I put the proxy up to and reproduced the issue and pulled those logs, and if you tell me the kind of messages you are looking for, I can pull the (redacted) context around them. What I can give you now, even without being directed, is the more verbose version of what I posted before, i.e., what happens around the time that the error gets logged: <detailsFollowed by the whole list of iptables rules. Then </detailsI can also give you the startup arguments: <details</detailsAnd finally, I can give you these lines from the iptables code, the health check getting registered, and a couple other startup items: <details</detailsAny more than this, though, and I'll have to ask you to tell me something to grep for.
First thing to check is that the healthcheck actually works EVER. Given the same timestamp (we should fix that?) it thinks it is healthy.
/assign
/triage accepted
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs.

This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
A situation: when there is a service or endpoint is constantly updated (which will update the ): so there is a high probability that the difference between the and the is less than the , so this will be considered healthy If you reduce ( iptables-sync-period * 2, the defaults value of flag is 30s), you may see response code is 503, but in fact, it is not easy to estimate how much it should be reduced. so As for case 3, I think there are something that can be optimized.
Oh! That's wrong. We don't care about the time of the most-recently queued update, we care about the time of the oldest queued update that was after the last . So rather than having and updating it every time gets called, we should have a variable something like , and can set it to 0, and should only set it to if it was previously 0. In fact, we don't even need . Kube-proxy would be healthy if or .
Thanks I see, I'd fix it, can I take this one?
sure /assign


Question: metricsList = []metrics.Registerable{ scheduleAttempts, e2eSchedulingLatency, <ins> schedulingLatency, </ins> SchedulingAlgorithmLatency, PreemptionVictims, PreemptionAttempts,

I would like to propose the graduation of the following scheduler metrics from alpha to stable: These metrics have been stable for at least 15 months.
/sig scheduling /sig instrumentation
/cc
IIRC, the last addition to these metrics was the profile name label. I don't foresee need for new labels. I don't see many precedents for graduation in core components. are there any defined criteria for graduation?
As requested, the following is how long it has been since a change has been made to each of the requested metrics: - 2 years; last change was to migrate to the metrics stability framework - 17 months; last change was a renaming of the metric - 15 months; last change was a renaming of the metric - 17 months; last change was to add a label - 17 months; last change was to add a label
It's up to the component owners to commit to graduating these metrics, since the component owners own the metrics and therefore need to agree to support them in accordance to stability requirements. If you guys are okay with promoting these metrics, I don't see a reason, from an instrumentation side, why we would object to promotion.
+1 to graduation then. These metrics are very mature. Although, I'm not sure if all metrics already adhere to naming conventions. and don't have a unit suffix, but they are unit-less. Is that fine
is a gauge so it's fine, is a histogram so it's also fine.
I'm recently working with metrics, I'm glad to provide some help if no one has interest about it.
There is already an open PR for this.
Oh sorry, didn't notice about that. Forget it.
These graduations sound good to me, but I would be inclined to rename . I feel that the part of the metric is unnecessary. Having instead and keeping for the algorithm latency is already descriptive enough to know the difference between both. If that's not the case, then the text of the metric can be used to guide the users.
I agree that is rather confusing, so +1 to removing the prefix. The metric doesn't include retries, for example. We have a separate metric that includes retries: We need to rename that one eventually to e2e, but it can only be done after the existing one has been deprecated and deleted for a few releases. /triage accepted
/assign
/cc
The PR for this is ready to be reviewed:


Question: } func (c *Cacher) startCaching(stopChannel <-chan struct{}) { <del> // Whenever we enter startCaching method, usable mutex is held. // We explicitly do NOT Unlock it in this method, because we do // not want to allow any Watch/List methods not explicitly redirected // to the underlying storage when the cache is being initialized. // Once the underlying cache is propagated, onReplace handler will // be called, which will do the usable.Unlock() as configured in // NewCacher(). // Note: the same behavior is also triggered every time we fall out of // backend storage watch event window. defer c.usable.Lock() </del> <ins> // The 'usable' lock is always 'RLock'able when it is safe to use the cache. // It is safe to use the cache after a successful list until a disconnection. // We start with usable (write) locked. The below OnReplace function will // unlock it after a successful list. The below defer will then re-lock // it when this function exits (always due to disconnection), only if // we actually got a successful list. This cycle will repeat as needed. successfulList := false c.watchCache.SetOnReplace(func() { successfulList = true c.usable.Unlock() }) defer func() { if successfulList { c.usable.Lock() } }() </ins> c.terminateAllWatchers() // Note that since onReplace may be not called due to errors, we explicitly

cc
This is still hapening. Assigning Isaac for triage.
Three times in the last three days, BeforeSuite failed (filed ). Otherwise, just Namespaces [Serial] should delete fast enough (90 percent of 100 namespaces in 150 seconds) is just totally borked (filed ). what prevented you from triaging this yourself? It'd be good to know so I'm less in the critical path.
Sorry, I read this wrong. I was looking at the gke-serial suite. Triaging now.
It looks like massive failures are correlated extremely strongly with failure of Etcd failure [Disruptive] should recover from SIGKILL Every time that test passes, everything else (except ) passes; every time it fails, the whole suite goes south. Reassigning to for triage.
sorry for delegating the work to you, but I completely didn't have time to do it and nobody else hasn't volunteered to do it for a while. it seems you are the author of the test, so assigning you.
No problem Just wanted to make sure there wasn't some information missing.
I'm looking at this today
It looks like the token controller is not recovering from etcd failing. The test namespace is also stuck in a terminating phase. I'll continue to investigate
Actually lot's of watch related things are broken. I can create/delete objects in the api but watch is broken. Trying to establish a watch in this state causes: Restarting the apiserver fixes the problem. I can start a watch again: cc
Not particularly useful apiserver logs Also none of the components are trying to reestablish watches so I'm assuming that they still have watches open but are not receiving anything.
startCaching is trying to lock here, but something is holding the lock. I'm looking through a stack trace right now for the offender. This is exactly where I'd expect a problem to be from the description of "bringing etcd down and up repeatedly".
Ah I think what happened is, when etcd stays down, a List never successfully finishes, and therefore it never calls the OnReplace function, and therefore the lock is never unlocked, and therefore startCaching can't exit.

There's lots in apiserver log to support this: Additionally I manually looked through the stack trace and confirmed that all of the stuck calls to Watch are stuck waiting to RLock and not some point afterwards.
This is pretty bad, etcd going down for just a bit could freeze up apiserver until it is rebooted.
I think I can make a fix.
Thanks for looking into this.
Yeah, I repro'd in a UT. Fix coming shortly.
I assigned to but if anyone wants it merged sooner than monday, feel free to review.
Many thanks to for handing me a broken server to look at, it made this super easy to find.


Question: if !os.IsNotExist(err) { return err } <ins> // Emit an update with an empty PodList to allow FileSource to be marked as seen s.updates <- kubelet.PodUpdate{[]api.BoundPod{}, kubelet.SET, kubelet.FileSource} </ins> return fmt.Errorf("path does not exist, ignoring") }

To reproduce: Start a cluster via hack/local-up- Start some pods Kill the cluster "docker ps -a" should show the pod containers running still Start the cluster again "etcdctl ls /registry" should show that there is no boundpods directory -- only minions and services "docker ps -a" should show the old pod containers still running Start a new pod After, say, 10s, "docker ps -a" should show the old containers dead /cc
I suspect the cause of this might be : which guarantees that the pod sync logic is never called if is nil. Need to get a repro to confirm.


Question: go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect <del> go.opentelemetry.io/otel/metric v1.28.0 // indirect </del> go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect

Since updating from Kubernetes 1.26. to 1.29. we experiences OOM-Kills on control plane nodes. After some investigation we found that adding to the fixed the issue. (Found this via pprof/heap of one apiserver: heap dump could be if needed) Before the update to 1.29.* the control planes were running with 4GB of memory with the same worklow in the cluster without issues. After the workarround using the feature gate the memory consumption of the whole control plane is not at ~2GB. In case there is an issue with other components, memory consumption of the apiserver should not be that excessive. We just had the K8s cluster running and saw a permant increase in memory usage of the control plane nodes. And after some time (can not give a specific duration) the OS of the control plane node started to OOM kill processes. Let us know if you need additional information. <details</detailson-premise; control-planes are running with 4 CPU cores, 8GB of memory and and NVME discs <details</details<details</details<details</detailsNo response
/sig api-machinery
Please provide the pprof, and what is the size of your cluster.
As we were not sure if sharing the pprof is secure, we decided to only share screenshots first: ! ! From this information we found the this which pointed us to the mentioned feature flag. We sometimes see etcd connection timeouts in the apiserver logs, maybe this could be related but should not cause this behaviour. 5 control planes ~100 worker nodes ~1700 Pods
Can you provide the pprof without , so could compare the two?
Sure, the current pprof with set: ! !
cc
I believe this is Sounds like we may need to backport some of the dependency bumps
/assign
/sig instrumentation
/triage accepted (as it's now assigned to
/triage accepted
We need to apply the workaround in It should be simple, but needs to be backported to 1.27
all versions starting 1.27 right ?
Thanks for taking care of this so quickly
yes. All versions starting 1.27.
Actually, this only needs to be backported to v1.29.

At 1.28, we were using v0.35.0 of otelgrpc, which did not include metrics at the time.
Cherry-picks:


Question: for i, tc := range cases { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ <del> func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, </del> func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil }, <ins> func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, </ins> func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil }, }, }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--kube-proxy cannot use ipvs mode on Linux 4.19 due to a check for the module, which was changed to (). uses to check for : It should instead accept on Linux 4.19 or above. What happened: kube-proxy fell back from ipvs mode to iptables mode. What you expected to happen: kube-proxy using ipvs. How to reproduce it (as minimally and precisely as possible): Launch a cluster on an image using Linux 4.19 or above with kube-proxy using ipvs mode (see ). Environment: Kubernetes version (use ): All Kernel (e.g. ): 4.19 <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/sig network
/area ipvs /assign /cc
Okay, so should check {nfconntrackipv4 | nf_conntrack} and expect load at least one module?
I think so
Please just go ahead :)


Question: eps.Insert(trimmed) } } <del> return eps, nil </del> <ins> // Return immediately when we successfully fetch endpoints if len(eps) > 0 { return eps, nil } // TODO: get rid of this delay #36281 time.Sleep(hitEndpointRetryDelay) </ins> } } <del> return nil, fmt.Errorf("Failed to get endpoints:nTries %dnCommand %vn", minTries, cmd) </del> <ins> return nil, fmt.Errorf("error getting endpoints:nTries %dnCommand %vn", tries, cmd) </ins> } // DialFromNode executes a tcp or udp request based on protocol via kubectl exec

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /priority failing-test /sig network What happened: From https://k8s-, seems to be broken right after they were by Example error message: and Anything else we need to know?: Is it as expected?
Oct 11 16:37:49.106: test session affinity, cost time: 1m31.742990485s This is a mistake. Sorry for my debug lines sneak in. I was just wondering why this test case doesn't run in upstream CI, so I this debug line. Now, I see because this test case only runs in . Oct 11 16:39:42.406: Expect endpoints: map[], got: map[netserver-1:{}] This check failed unexpectedly, let me try to find out why. I am working on it and will let you know soon when I have some updates.
Thanks. These two tests don't run on pull jobs as they are marked as , though https://k8s-, https://k8s- (and other CIs that include ) would have them running.
Do you mean when I raise a PR, these test cases will not be triggered? My question is how can I make sure my follow-up PR can fix this bug? Where can I access the log message when I raise a new PR?
Can I see the updated test results via directly visiting https://k8s-, https://k8s-
Yep, that is the current behavior. A hacky way to make it run on PR --- add a temporary commit to remove the word from test name so some pull jobs ( I think?) will run it. And you could view logs by clicking the button.
Nope, this CI only includes changes that are merged. It is basically a cronjob that runs tests against head.


Question: ], tags = ["automanaged"], deps = [ <ins> "//cmd/kubeadm/app/apis/kubeadm:go_default_library", </ins> "//federation/apis/federation:go_default_library", "//federation/apis/federation/v1beta1:go_default_library", "//federation/client/clientset_generated/federation_clientset:go_default_library",

used to show masters, but as of it no longer does. This is not in accordance with our deprecation policy I believe. cc
That commit/PR is not relevant as there are no supported users/deployment methods that are using label. kubeadm v1.4 or v1.5 was the last. What you're really looking for is who/what removed printing things based on the label...
No blame on you The regression is that we have stopped printing masters in . There may be more to it that a simple revert of , but we should probably start there.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
This has been fixed in recent versions


Question: volPath := pd.GetPath() if err := os.MkdirAll(volPath, 0750); err != nil { <ins> // TODO: we should really eject the attach/detach out into its own control loop. detachDiskLogError(pd) </ins> return err }

Repro: Create a pod that attempts to mount a GCE PD that is mis-formatted Delete the pod Expected: GCE PD should be detached Observed: GCE PD is not detached.
Chatted with offline and understand the problem with more detail: The basic problem is when filesystem on PD is unformatted, the mount is going to fail. Kubernetes doesn't do cleanup by detaching the PD. Two potential solutions were proposed: 1) use built-in utility: safeformatand_mount in GCE image to format and mount PD to avoid extra step to format the PD disk. 2) Handle mount failure properly. I think we might need handle both anyway. Brendan mentioned that you are looking at a potential new controller for PD, whose responsible is attaching and detaching PD disks. Do you have any latest status on this topic?
No latest status. I haven't done any actual work on it yet. The idea would be to scan the pods assigned to each node, and attaches (or detaches) any PDs that the node needs (or no longer needs). I'm still coming up to speed on a lot of the architectural concepts, so I'd love some guidance.
Now that Tim's volume re-factor has gone in, I'm going to re-visit this issue. Assigning to me.


Question: return b } <del> // willObjectBeDeletedLocked returns true only if the last delta for the // given object is Delete. Caller must lock first. func (f *DeltaFIFO) willObjectBeDeletedLocked(id string) bool { deltas := f.items[id] return len(deltas) > 0 && deltas[len(deltas)-1].Type == Deleted } </del> // queueActionLocked appends to the delta list for the object. // Caller must lock first. func (f *DeltaFIFO) queueActionLocked(actionType DeltaType, obj interface{}) error {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: I did a review of and found out a case where a SharedIndexInformer can miss a notification. Follows an example scenario where this could happen. 1) A SharedIndexInformer is running, and its reflector is WATCHing. 2) A delete notification for an API object X with namespaced name N1 and UID U1 which was in the informer's cache is received. 3) The reflector on X; the relevant store is a DeltaFIFO and its Delete method is . 4) The DeltaFIFO's Delete method invokes , which . Notice that the store of the reflector, i.e., the DeltaFIFO, is NOT the SharedIndexInformer's store/cache: . Assume that BEFORE X's deletion is popped from the DeltaFIFO and applied to the informer's cache, the following sequence of events take place: 5) The WATCH is interrupted because of a network partition. 6) While the reflector is not watching, a new API object Y with namespaced name N1 (same as X) and UID U2 is created. 7) The partition disappears and the reflector does a relist against the API server; there's a "hole" between the last watch notification and the list results; assume Y is in the LIST results. 8) The reflector will load the LIST results in the store/DeltaFIFO with which in turn calls the DeltaFIFO's method. The DeltaFIFO's replace method attempts to add to the DeltaFIFO a delta of type for every object in the LIST results: when processing Y it will which should . But won't enqueue Y's delta, because it enqueues deltas of type only if there's not already a delta of type for the given object ID as the last delta for that object ID (see and ), to avoid recreating a deleted object during a periodic resync triggered by the SharedIndexInformer resync period. Object IDs are namespaced names, and remember that the last delta for Y's namespaced name in the DeltaFIFO is X's deletion.

The consequence is that Y's creation delta is not to the DeltaFIFO: Y is not to the SharedIndexInformer's cache and its creation notification handler does not execute. The SharedIndexInformer missed Y's creation. What you expected to happen: I expected SharedIndexInformers to get all notifications from the API server. Anything else we need to know? Missed notifications should be rare: for them to occur a delta must stay in the DeltaFIFO without being processed for long enough for a new List against the API server to be completed. Also, touches the reflector code that handles list and watches, and it might introduce changes relevant to this bug. Proposed solution: There's a straightforward fix: get rid of the check that skips enqueuing of a delta for object ID X if the last delta to the DeltaFIFO (and still in it) for X is of type ; is the code to remove. Notice that such check applies to deltas of type only. deltas are triggered only by relists and periodic resyncs. In the case of relists the check is harmful as it's the cause of this bug. In the case of periodic resyncs, that check is useless because it will either evaluate to or not execute; in fact, during a period resync, and this condition is checked before the check that would be removed, OTOH, if this condition is true and processing goes on the check that would be removed can only evaluate to (hence the delta for X is enqueued). If this solution has consensus I'd be happy to open a PR.
/assign
/sig api-machinery
cc
cc
xref ,
cc
/cc
This looks similar to failures we were seeing in kube-proxy while doing a rolling master upgrade. It was repeatable enough that someone hacked a workaround that I later spotted - this is close enough to the description that I’m suspicious that it was real /priority critical-urgent Since any missed informer event can cause all sorts of safety violations.
Note that rolling master upgrades break watches en masse, so a backlogged informer (contention / other locking / busy machine) would be more likely so see it the more caches it has.
A great first step would be to craft a test that demonstrates the issue repeatably, to help verify potential fixes resolve it.

IYO what would be the most appropriate kind of test, a unit test for the DeltaFIFO or something higher-level like integration/e2e tests?
The lower-level the test will be the better. So ideally unit test.
the lowest level that demonstrates the problematic behavior. if the issue is with the interaction between reflector and shared informer, then ideally something at the shared informer level
When the reflector calls it is expected that for each object O with ID X in a delta is to the DeltaFIFO, but that is (erroneously) not the case if the DeltaFIFO already stores a delta for ID X . So it's the DeltaFIFO which causes troubles and should be tested. But that alone does not show the user-visible problem; for a test to fail because a SharedIndexInformer missed a notification, the test would have to be at the SharedIndexInformer level.
That seems reasonable. I'd expect a DeltaFIFO test that sets up the described scenario and exercises both the Relist/Replace and Resync cases described here:
Sounds good, I'm at work.
is the testcase literally just adding this to But for some reason this hangs in the Pop(f).
Yes, the test looks very similar to what you posted, but there's a mistake: the expected result must be a single with the two that in your example are the only elements of the two 1-element Deltas that make up , respectively.
(that would be caught and fail the test immediately by changing to )
Aha, you're totally right :) Had to run off to lunch before spending more time on it.
DeltaFIFO only de-dupes deletions, but your example would do effectively extend that to compressing Delete+Sync=Sync. You used to be able to build smarter compression off DeltaFIFO (eg ) but nobody actually used the DeltaCompressor stuff so we removed it. And nobody above DeltaFIFO expects compressed events now :( In your example I'd be a bit worried about the implications of not deleting the object, since the Sync-ed one probably has a different UUID and thus technically not be the same object as the deleted one. Might not be a big issue, but this is critical path code so I'd personally rather be a bit conservative.
I agree about not compressing.

It's a little hard to reason about what this sync event should be when we know that relists can currently time-travel backwards. Edit: it looks like shared informer code will if it follows a delete event.
I was this morning in the "sig-api-machinery" channel of the "Kubernetes" Slack team. My reading of the code had me expecting that I'd get a delete followed by an add.
This issue is specifically focused on how a queued delete event (typically via directly observing a delete watch event or via absence in a relist/replace) followed by an observed add (via presence in a relist) is handled by DeltaFIFO. your question is slightly broader/different, and has multiple facets: UID changes considered when handling reflector object add/delete/update events? They are not; if an object is deleted and recreated between the reflector dropping a watch and relisting, the reflector will enqueue an Sync event with the object with the new uid (which the shared informer will turn into an Update event) delete/add events is actively compressed into an update event? I don't think so, but it's been a while since I've traipsed through reflector/fifo code, so I'm not positive (and since the Update with a new uid can occur from scenario 1, informer clients have to be prepared for uid to change on update anyway)
Listeners need to handle Sync too. And the only two in-tree users of NewDeltaFIFO are and and they look like they correctly handle Sync.
Edit: yep you're right, I was looking at slightly older code.
if this happens, I think it should happen on top of the DeltaFIFO, not in the DF. Let's keep DF simpler since it's (a) complicated enough and (b) critical path code. probably the right place for compression, if any happens. IMHO it's correct that Deletes don't get compressed, eg that Delete + Add is not compressed into Update or something like that.
I agree, though that specific example can happen naturally if it occurs in a window while the reflector isn't getting watch events.

Just to make sure I understand this properly, does this scenario involve an object in the cache with name "x" and UID "u1," then the reflector loses its watch, reestablishes it and lists all the relevant objects in the cluster, finds one named "x" with UID "u2," and since it already has an "x" in its cache, translates this discovery into a call?
exactly (both namespace and name are part of the cache key, uid is not)
does this mean we couldn't use the argument to DeltaFIFO's Replace() to work around whatever is talking about (if that problem still exists)?
clients can only use resourceVersion to check equality, not older-than/newer-than
As noted in Slack, there is now careful commentary in about what is promised. There is no promise that a delete followed by an add is delivered as two notifications; it is allowed to be delivered as one update notification.