Question: } } <ins> func TestURLTemplate(t *testing.T) { uri, _ := url.Parse("http://localhost/some/base/url/path") uriSingleSlash, _ := url.Parse("http://localhost/") testCases := []struct { Request          *Request ExpectedFullURL  string ExpectedFinalURL string }{ { // non dynamic client Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("POST"). Prefix("api", "v1").Resource("r1").Namespace("ns").Name("nm").Param("p0", "v0"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1/nm?p0=v0", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D?p0=%7Bvalue%7D", }, { // non dynamic client with wrong api group Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("POST"). Prefix("pre1", "v1").Resource("r1").Namespace("ns").Name("nm").Param("p0", "v0"), ExpectedFullURL:  "http://localhost/some/base/url/path/pre1/v1/namespaces/ns/r1/nm?p0=v0", ExpectedFinalURL: "http://localhost/%7Bprefix%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/namespaces/ns/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/namespaces/ns/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D", }, { // dynamic client with core group + namespace + resourceResource (with NO name) // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1", }, { // dynamic client with named group + namespace + resourceResource (with NO name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/namespaces/ns/r1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/namespaces/ns/r1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/namespaces/%7Bnamespace%7D/r1", }, { // dynamic client with core group + resourceResource (with name) // /api/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/r1/%7Bname%7D", }, { // dynamic client with named group + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/r1/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME/$SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/%7Bname%7D/finalize", }, { // dynamic client with named group + namespace + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with NO name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/finalize", }, { // dynamic client with named group + namespace + resourceResource (with NO name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/status"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/status", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/status", }, { // dynamic client with named group + namespace + resourceResource (with no name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces", }, { // dynamic client with named group + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D/finalize", }, { // dynamic client with named group + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/status"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/status", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D/status", }, { // dynamic client with named group + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D", }, { // dynamic client with named group + resourceResource (with no name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces", }, { // dynamic client with wrong api group + namespace + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME/$SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/pre1/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/pre1/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/%7Bprefix%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) where baseURL is a single / // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r2/name1"), ExpectedFullURL:  "http://localhost/api/v1/namespaces/ns/r2/name1", ExpectedFinalURL: "http://localhost/api/v1/namespaces/%7Bnamespace%7D/r2/%7Bname%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) where baseURL is 'some/base/url/path' // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r3/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r3/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r3/%7Bname%7D", }, { // dynamic client where baseURL is a single / // / Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/"), ExpectedFullURL:  "http://localhost/", ExpectedFinalURL: "http://localhost/", }, { // dynamic client where baseURL is a single / // /version Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/version"), ExpectedFullURL:  "http://localhost/version", ExpectedFinalURL: "http://localhost/version", }, } for i, testCase := range testCases { r := testCase.Request full := r.URL() if full.String() != testCase.ExpectedFullURL { t.Errorf("%d: unexpected initial URL: %s %s", i, full, testCase.ExpectedFullURL) } actualURL := r.finalURLTemplate() actual := actualURL.String() if actual != testCase.ExpectedFinalURL { t.Errorf("%d: unexpected URL template: %s %s", i, actual, testCase.ExpectedFinalURL) } if r.URL().String() != full.String() { t.Errorf("%d, creating URL template changed request: %s -> %s", i, full.String(), r.URL().String()) } } } </ins> func TestTransformResponse(t *testing.T) { invalid := []byte("aaaaa") uri, _ := url.Parse("http://localhost")

Hello, we have been using client-go to measure the latency for all client-go traffic from Cilium to kube-apiserver. We have that since v1.24.0 the URL path is no longer aggregated in a way that this cardinality would no longer happen as presented in the following image: <img width="330" alt="Screen Shot 2022-08-08 at 10 18 24 AM" src="https://user-"It seems this functionality was removed as a part of as it seems that PR only took into account that users of the would only be using it to measure latency to a particular host and not for a particular URL path as we are doing in . Would it be reasonable to revert commit in order to fix this regression? Thanks cc
/sig api-machinery /sig instrumentation
But reverting will bring back the cardinality problem, will not? The latency of each URL path is available in the apiserver metrics ...
We have never any cardinality problem with the previous code since we were not using the "host" field which is what I assume the original issue about cardinality was trying to fix. We only used the "path" of an URL, which was aggregated in a way that would not be facing any cardinality, like the figure attached in this GH description. Now, starting with = 1.24.0, each and is actually a different value which does cause the cardinality problem to appear.
ah, I see the disconnection, I wasn't understanding the issue sorry. The Interface uses the url as parameter to keep backwards compatibility, but the method that gathers the metrics only uses the host to avoid the cardinality problem. You have to update the method that gathers the metrics to only use the host part of the url
Filed
should fix the code in kubernetes ? however, that will break consumer of the Interface that still wants to process the path, but we know using the path will cause issues with cardinality :shrug:
I would be fine with adding a new label to mitigate that issue since originally the cardinality explosion that was reported was caused by the query string and not the path: When trimming down the whole URL to only the host we even had a discussion to keep the path, but we decided not to since we didn't have any use case for this data.

If you have a use case where you need the path in order to get more granularity on the latency of your network then I would be fine with adding the label to Kubernetes, but another path forward could be to also remove the path information from cilium if deemed unnecessary. In the most common scenarios, you will only need the host to get the network latency based on different proxy routes.
yeah we can't really do much about it, it is either we leak the unsanitized paths or we prevent the users from using the path completely. Since we want to not break the interface and keep the URL parameter, we might just have to keep sanitizing the paths for our consumers even though we are not using the path anymore.
IIUIC the RESTClient is used in the dynamic client-go too, for these users with 1900 CRDs, the path is really going to explode for them
Then it seems that the best option to not break users would be to revert , but at the same time I don't really like the idea of spending CPU time sanitizing something that will never be used in the RESTClient
A suggestion would be to create a new interface similar to the that would have the following method: Thus, those users with 1900 CRDs would use the instead of the .
after digging a little bit the underlying issue it looks that the problem was initially caused by a kubelet bug as pointed out . The dependents of client-go don't have access to all the fields of the struct which uses heavily to generate the aggregated URL.
well, it is clear now that current status is wrong, we pass the whole URL to the interface and rely on the to use only the Host part of the URL :smile: I'm fine with whatever you decide, to restore or just completely strip the Path Sorry for the unfortunate consequence, at that time I didn't consider that the Interface/Method drift and thought that function was orphan
/triage accepted /assign


Question: return noRestraintRequest{}, false } <ins> func (noRestraint) UpdateObservations() { } </ins> func (noRestraint) Dump(bool) debug.QueueSetDump { return debug.QueueSetDump{} }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Not observed yet, but deduced from analyzing the code. If a k/apiserver/pkg/util/flowcontrol/fairqueuing.QueueSet, whose metrics.TimedObserverPair is a pair of k/apiserver/pkg/util/flowcontrol/metrics.sampleAndWaterMarkHistograms, gets zero activity for a long while and then gets some activity, then the loop at will go through a lot of iterations, and do so with a lock held. What you expected to happen: No big hiccups in any scenario. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): branch Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/assign

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: In reviewing I noticed an incorrect fake implementation of Interface. I looked at the comment on the Handle method, and noticed that it does not say when/how the is called. What you expected to happen: I expected the parameter to be as well documented as the others. How to reproduce it (as minimally and precisely as possible): Look at and see how it does not say when/whether/with-what-parameter-values is called. Anything else we need to know?: Environment: Kubernetes version (use ): current Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
Any chance we could write a test for the interface which verifies correct plumbing?
/cc
: that should be do-able


Question: m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI) return m } <ins> func TestGetContainerUsageNanoCores(t *testing.T) { var value0 uint64 var value1 uint64 = 10000000000 tests := []struct { desc          string cpuUsageCache map[string]*runtimeapi.CpuUsage stats         *runtimeapi.ContainerStats expected      *uint64 }{ { desc:          "should return nil if stats is nil", cpuUsageCache: map[string]*runtimeapi.CpuUsage{}, }, { desc:          "should return nil if cpu stats is nil", cpuUsageCache: map[string]*runtimeapi.CpuUsage{}, stats: &runtimeapi.ContainerStats{ Attributes: &runtimeapi.ContainerAttributes{ Id: "1", }, Cpu: nil, }, }, { desc:          "should return nil if usageCoreNanoSeconds is nil", cpuUsageCache: map[string]*runtimeapi.CpuUsage{}, stats: &runtimeapi.ContainerStats{ Attributes: &runtimeapi.ContainerAttributes{ Id: "1", }, Cpu: &runtimeapi.CpuUsage{ Timestamp:            1, UsageCoreNanoSeconds: nil, }, }, }, { desc:          "should return nil if cpu stats is not cached yet", cpuUsageCache: map[string]*runtimeapi.CpuUsage{}, stats: &runtimeapi.ContainerStats{ Attributes: &runtimeapi.ContainerAttributes{ Id: "1", }, Cpu: &runtimeapi.CpuUsage{ Timestamp: 1, UsageCoreNanoSeconds: &runtimeapi.UInt64Value{ Value: 10000000000, }, }, }, }, { desc: "should return zero value if cached cpu stats is equal to current value", stats: &runtimeapi.ContainerStats{ Attributes: &runtimeapi.ContainerAttributes{ Id: "1", }, Cpu: &runtimeapi.CpuUsage{ Timestamp: 1, UsageCoreNanoSeconds: &runtimeapi.UInt64Value{ Value: 10000000000, }, }, }, cpuUsageCache: map[string]*runtimeapi.CpuUsage{ "1": { Timestamp: 0, UsageCoreNanoSeconds: &runtimeapi.UInt64Value{ Value: 10000000000, }, }, }, expected: &value0, }, { desc: "should return correct value if cached cpu stats is not equal to current value", stats: &runtimeapi.ContainerStats{ Attributes: &runtimeapi.ContainerAttributes{ Id: "1", }, Cpu: &runtimeapi.CpuUsage{ Timestamp: int64(time.Second / time.Nanosecond), UsageCoreNanoSeconds: &runtimeapi.UInt64Value{ Value: 20000000000, }, }, }, cpuUsageCache: map[string]*runtimeapi.CpuUsage{ "1": { Timestamp: 0, UsageCoreNanoSeconds: &runtimeapi.UInt64Value{ Value: 10000000000, }, }, }, expected: &value1, }, } for _, test := range tests { provider := &criStatsProvider{cpuUsageCache: test.cpuUsageCache} real := provider.getContainerUsageNanoCores(test.stats) assert.Equal(t, test.expected, real, test.desc) } } </ins>

Related: On Windows, kubelet reports only the cumulative for cpu usage. On Linux, kubelet reports both and the instantaneous . Although In the future, most components/monitoring pipelines will switch to using the cumulative metrics, a few features still depend on the instantaneous metrics today. Examples are: , HPA/VPA. Kubelet should report this metrics on Windows to support the existing features. /cc who explained to me what features depend on the cpu metrics. /cc /cc
- is this something that's clear enough in the CRI layer that you can implement it, or do we need to fix that first?
/assign
do you have any notes or a doc you can point to that better describes what features depend on which metrics? It would be good for us to share and see if anything other than is broken
The horizontal and vertical pod autoscalers also depend on the
Documentation on metrics API and metrics server (i.e., the aggregator): The metrics-server pod keeps spewing errors like this: Running : I haven't verified whether pod autoscaling works in my cluster, but I assume they won't since metrics server seems to discard the metrics from the windows pods. I believe the implementation is in kubelet (for the docker integration), so this should not need CRI changes.
I think this is generally an issue for non-Docker CRIs (not necessarily Windows-exclusive). The kubelet source code suggests that these metrics will be gathered from the CRI (if the CRI is not Docker). The struct that kubelet expects returned from the ContainerStats RPC does not include usageNanoCores. Does this mean that, until this is resolved, custom CRIs are unable to satisfactorily report metrics to the 'metrics-server'?
- are you able to work on this?
that is a good point. For CRI runtimes, the kubelet should calculate the average over 10s to produce this metric.
Good point. We designed CRI around core metrics API, but some CRI runtimes have integrated with cadvisor directly to fill the gap while we wait for core metrics API. I think it's reasonable for kubelet to do this on behalf of all CRI runtimes. I spoke to offline, and this would probably require no more than 2 days of work. If sig-windows (e.g., can pick this up, that'd be great.

has agreed to review the changes (thanks!) If not, we should clearly document that this is not supported in Windows GA for 1.14.
thanks for the update. we will wait to hear back from
Currently, is got from docker's stats API on Windows. From docker stats API , only is available, hence is not set on Windows: Do you know any way to get instantaneous for Windows containers?
I have no idea. any idea?
I think this is broader than just a Windows issue. Officially supported tools such as the kubernetes metrics-server expect the field, but custom CRIs that conform to the interfaces defined in the package are not given the liberty to return this value.
suggested previously to cache the stats temporarily and compute the stats using the diff. David, could you chime in on how this could be done? Yes, this was mentioned in the previous comments too. There are plans to transition those components off this field, but it's WIP. We are looking for a workaround in kubelet for now.
my biggest worry is that if we say this impacts the horizonal POD autoscaler, how is that passing in our end2end and conformance tests?
Do you test HPA? It's not part of conformance, and I doubt this is included in windows testing.
As to how this could be done, in and we could just add a in the struct, which we assign the previous CpuUsage metric for each container. Then, when we need to calculate the rate, we pass the previous CpuUsage along with the current one to , and calculate a rate based on the timestamp and values.
ListPodCPUAndMemoryStats() we could just add a map[string]*runtimeapi.CpuUsage in the criStatsProvider struct I've done some tinkering on this myself and I originally thought that it would be as simple as that. It gets much more complicated when you have to work in between the existing cAdvisor-based logic. I assume the intent is to cut it out entirely eventually, but the information passed through the CRI doesn't even come close to being so comprehensive. So far I've found that this approach fixes the problem for the "/stats/summary" api, but not necessarily for the other apis. In particular, "/stats/container" seems to be completely based on cAdvisor, being unimpacted by the changes we made to those two functions.

Some monitoring tools and metrics scrapers rely on the "/stats/container" api for their metrics, which means it is insufficient to update one api without doing the other.
We don't need to fill in everything in the summary API, just "usageNanoCores". The "/stats/summary" api is the one used by the metrics server, which is why we care about it in this case. The "/stats/container" isn't something we need to implement to support this.
we a couple of tickets to validate if HPA for Windows containers works. My guess is that it works for memory and not for CPU metrics because of this issue. If this is complicated to fix, i wonder if waiting for the CRI containerD changes in windows would be the best option for supporting this.
containerd and cri-o bypass this issue by leveraging cadvisor as a temporary solution. I don't think that'd work for windows.
Sorry the for late response. HPA is actually working for Windows containers. e.g. in my cluster (v1.12.5), cpu-based HPA is working as expected: Which version of kubernetes are you referring for HPA issues?
are you running heapster, or the metrics server to serve HPA? Heapster uses the cumulative metric, but the metrics-server uses the 10 average rate metric.
metrics-server is used in my cluster.
See .
/milestone v1.14


Question: fake_kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/fake" "k8s.io/kubernetes/pkg/client/testing/core" "k8s.io/kubernetes/pkg/runtime" <ins> "k8s.io/kubernetes/pkg/util/wait" </ins> "github.com/stretchr/testify/assert" )

https://k8s- Failed: TestNamespaceController {namespace} Previous issues for this test:
GetObjectFromChan() is timing out (20s) and returning nil causing the panic. Happens in TestSecretController (and probably TestIngressController) controller as well. https://k8s-
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
I have a fix for this in
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
https://k8s- Failed: TestNamespaceController {namespace}
/cc OWNERS of :
I think this is also related to Issue .
fix is pending


Question: if sliding { t.Next() } <del> if timeCh == nil { timeCh = t.C() } // NOTE: b/c there is no priority selection in golang // it is possible for this to race, meaning we could // trigger t.C and doneCh, and t.C select falls through. // In order to mitigate we re-check doneCh at the beginning // of every loop to guarantee at-most one extra execution // of condition. select { case <-doneCh: return ctx.Err() case <-timeCh: } </del> } }

While trying to update to use latest version of package and to migrate away from deprecated , function, our tests fail as it seems new function immediately executes given condition twice, which is not mentioned anywhere. Result of executed test specified below: Since has been deprecated and it recommends using , I expected it to have the equivalent semantics to allow easy migration from deprecated functions, as no changelog or documentation mentions new semantics. Playground: Local: file with the following content: as . No response_ <details</details<details</details<details</details<details</details<details</details<details</details
/sig api-machinery
CC since you were involved in
/assign /triage accepted
I'd like to work on this one if it's allowed I think this is a simple fix with few lines removed: i.e.: instead of only wait a loop if immediate is false, we wait a time loop after special execution also if the immediate is true.
/assign
/assign


Question: if err := yaml.Unmarshal(action.GetPatch(), &patchObj.Object); err != nil { return nil, err } <ins> patchObj.SetName(action.GetName()) </ins> err := o.tracker.Apply(gvr, patchObj, ns, action.PatchOptions) if err != nil { return nil, err

Trying out the (excellent!) new Apply with the fake client, I found a different from the fake vs real client My patch: Here we lose the name. We extract the namespace from the action, but use which is empty. This means we have no name, the fails, and we attempt to a nameless object. This doesn't apply to other patch types, which get the live object and apply the patch to them before this point. This could be fixed, perhaps, by inserting before we call ? cc /sig api-machinery
Good catch and thank for kicking the tires on the new fake client functionality. look right?
/triage accepted /assign /kind bug


Question: return false, -1 } <ins> opIndex := -1 </ins> for previousOpIndex, previousOp := range grm.operations { volumeNameMatch := previousOp.key.volumeName == key.volumeName

on my scenescreate same cronjob use same sfs(the nfs storage on Huawei Cloud) type storage by pvc to do same training work. At the same time, a node will schedule many pods using the same pvc, and there will be many mount and umount tasks about this pvc executed at the same time. And we find all pods on a node at container creating state, because of waiting pvc to mount. check log, we can't find any we can't find any operationExecutor.MountVolume started or failed log like "operationExecutor.MountVolume started for" and also we check volumemanagertotalvolumes metrics, volume is add to dsw. analyze code, if isExpectedError, kubelet not log any thing and operationExecutor will generate operation, add call pendingOperations Run, pendingOperations Run will check operation slice, to find whether has same operation exists, and if operation exist, will check operation's operationPending status, if equal true, means same operation now is excuting, and return AlreadyExists. operationKey composed of volumeName, podName, nodeName so MountVolume operationKey {volumeName, EmptyUniquePodName,EmptyNodeName}, use EmptyUniquePodName to avoid executing mount/map from multiple pods referencing the same volume in parallel soUnmountVolume operationKey {volumeName, podUID,EmptyNodeName}, because All volume plugins can execute unmount/unmap for multiple pods referencing the same volume in parallel now see isOperationExists func, if previousOp.podName equals to EmptyUniquePodName or current operation podName equals to EmptyUniquePodName, will return podNameMatch, mount operation can find umount operation or umount operation can find mount operation because of isOperationExists, this mean if a volume now is excuting mount, a umount can't add to operations, and if a umount now is excuting, a mount operation can't add to operations, but multiple pods referencing the same volume can add multiple umount operation into operation, may lead operation leak.

t0 two pod do umount volume operation t1 pod1 umount failed, update pending to false t2 pod3 add, operation excuter add mount operation, will use index 0, override umount operation t3 mount operation failed , update pending to false t4 pod2 umount operation add, will find index 0, add override opertation, now ,we have two same umount operation, now we have two goroutines to do pod2 umount operations t5 first goroutine pod2 umount failed, will update index 0 operation, update pending to false t6 second goroutine pod2 umount success, will delete index 0 operation, finally lead a umount opertion in cache, and this operation will lead all mount opeartion return AlreadyExistsError, and not cause any csi call. and i think there exists some other scene can trigger this leak. and also, when operation success, deleteOperation, will use tail element to replace, and may increase the probalility of occurrence. because isOperationExists will always pick first match one. and also override mechanism may lead exponentialBackOff invalid for example mount and umount take turns to excute. pv can mount , pod can running reproduce step: create same cronjob that use same pvc(use csi sfs) and csi driver restart periodic No response_ <details</details23 <detailsHuaweiCloud </details<details</details<details</details<details</details<details</detailsCSI
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig storage
/sig node
+1 meet the same problem
I don't know much about volumeManager. It seems that there is a problem with the override mechanism. Can mount and unmount of the same volume be executed at the same time? If it can, mount should not override any unmount operation. Otherwise, the mount operation should override all matched operations, not just the first one.

also don't know much about design history background, but comments in volume manager seems to conflict with each other among UmountDevice and UnmountVolume. need sameone who familar with it to explain Otherwise, the mount operation should override all matched operations, not just the first one. yes, must discuss rules for the existence of an operation
cc
Suppose there are the following cases: volume's "volumemount" should block the next same "volumemount" or "umountdevice" and all related pods's "volumeumount" exp1: , then could not be , then could be volume's "volumeumount" should block the next same "volumeumount" and it's "volumemount" or "umountdevice", but it should not block it's related other pods's "volumeumount" exp2: , then could not be , then could not be , then could not be , then could be volume's "umountdevice" should block the next same "umountdevice" or "volumemount" and all related pods's "volumeumount" exp3: , then could not be , then could be So we should improve the isOperationExists to satify case 2, that is, the intermediate state should not occur. Maybe since this pr: , this logic remains the same. /cc
/remove-sig node for sig storage to triage
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale


Question:
QueryParameter

timeoutSeconds

Timeout for the list/watch call.

Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.

false

integer (int32)

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Watches done with the field of set to some value T end after T regardless of whether there has been any activity in the meantime. What you expected to happen: I expected the watch to end when no events have been delivered in the last T. In ordinary usage, a "timeout" is something that happens if there has been no activity for the specified length of time. How to reproduce it (as minimally and precisely as possible): Create a configmap named in namespace . Then in one shell thrash it periodically like this: Observe that it proceeds to update the configmap every 7 seconds or so. In another shell do a watch with a timeout; observe that it ends even though the thrashing continues. Anything else we need to know?: Environment: Kubernetes version (use ): Server Version: version.Info{Major:"1", Minor:"8+", GitVersion:"v1.8.4+coreos.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-11-21T17:22:25Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}Cloud provider or hardware configuration: local hardware OS (e.g. from /etc/os-release): Ubuntu 16.04.3 LTS Kernel (e.g. ): Linux n17s2ctrl 4.4.0-34-generic -Ubuntu SMP Wed Jul 27 16:06:39 UTC 2016 x8664 x8664 x86_64 GNU/Linux Install tools: local Others:
I am told the observed behavior is the intended behavior, the problem here is that the field name and comment are misleading.
/sig api-machinery
It's a timeout on the request lifetime, not an inactivity timeout
The inactivity condition is implied in ordinary usage of the word "timeout". A word like "lifetime", "duration", or "time limit" would be a better description of the existing behavior. I do not propose changing the field name for this, that would be more work than it is worth. I propose updating the commentary on that field to make its meaning clear. BTW, what exactly is this field's meaning to a list operation?


Question: ServiceTopology:                                {Default: false, PreRelease: featuregate.Alpha}, ServiceAppProtocol:                             {Default: true, PreRelease: featuregate.Beta}, ImmutableEphemeralVolumes:                      {Default: true, PreRelease: featuregate.Beta}, <del> DefaultIngressClass:                            {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 </del> HugePageStorageMediumSize:                      {Default: true, PreRelease: featuregate.Beta}, ExternalPolicyForExternalIP:                    {Default: true, PreRelease: featuregate.GA}, // remove in 1.20 AnyVolumeDataSource:                            {Default: false, PreRelease: featuregate.Alpha},

When should it be removed? /sig network /kind cleanup
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
According to , it need to remove in 1.20. /assign
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
/remove-triage unresolved


Question: d.mu.Unlock() } <ins> // Start tracking the connection d.mu.Lock() d.conns[closable] = struct{}{} d.mu.Unlock() </ins> return closable, nil }

What happened: my kubelet restart unexpectedly,i find log info has more "Failed to list" info and has fatal error: go of nil func value,and then kubelet restart Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] object-"logging"/"elasticsearch-token-99wc2": Failed to list v1.Secret: Get https://vip- Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] failed 5 attempts to update node lease, will retry after 10s Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] object-"logging"/"default-token-6wh7n": Failed to list v1.Secret: Get https://vip- Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] object-"monitoring"/"prometheus-rules": Failed to list v1.ConfigMap: Get https://vip- Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] object-"logging"/"service-registry": Failed to list v1.Secret: Get https://vip- Feb 02 17:46:25 mgt15 kubelet[6914]: E0202 17:46:25. 6914 ] object-"monitoring"/"prometheus-center": Failed to list v1.ConfigMap: Get https://vip- Feb 02 17:46:30 mgt15 kubelet[6914]: E0202 17:46:30. 6914 ] Failed to list v1beta1.RuntimeClass: Get https://vip- Feb 02 17:46:31 mgt15 kubelet[6914]: E0202 17:46:30. 6914 ] object-"logging"/"telegraf-elasticsearch": Failed to list v1.ConfigMap: Get https://vip- Feb 02 17:46:31 mgt15 kubelet[6914]: E0202 17:46:30.

6914 ] object-"monitoring"/"prometheus-center": Failed to list v1.ConfigMap: Get https://vip- Feb 02 17:46:32 mgt15 kubelet[6914]: fatal error: go of nil func value Feb 02 17:46:32 mgt15 kubelet[6914]: goroutine 357 [running]: Feb 02 17:46:32 mgt15 kubelet[6914]: (closableConn).Close(0x4002193f00, 0x0, 0x0) Feb 02 17:46:32 mgt15 kubelet[6914]: /workspace/anago-v1.14.3- Feb 02 17:46:32 mgt15 kubelet[6914]: (Dialer).CloseAll(0x4000bbaea0) Feb 02 17:46:32 mgt15 kubelet[6914]: /workspace/anago-v1.14.3- Feb 02 17:46:32 mgt15 kubelet[6914]: (Kubelet).updateNodeStatus(0x40010ce000, 0x54d0978, 0x40010ce648) Feb 02 17:46:32 mgt15 kubelet[6914]: /workspace/anago-v1.14.3- +0xf0 Feb 02 17:46:32 mgt15 kubelet[6914]: (Kubelet).syncNodeStatus(0x40010ce000) Feb 02 17:46:32 mgt15 kubelet[6914]: /workspace/anago-v1.14.3- What you expected to happen: kubelet do not restart How to reproduce it (as minimally and precisely as possible): when have more info of "Failed to list" Environment: Kubernetes version (use ):v1.14.3 OS (e.g: ):Ubuntu 18.04.3 LTS Kernel (e.g.

):4.15.0-55-generic Network plugin and version (if this is a network-related bug):calico3.8.2
/sig node /wg node /committee node
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig node /wg node /committee node Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
fix is in
/sig auth
/assign
thankswhat version is this bug been fixed?
The fix is not yet merged, but I plan to backport the fix to 1.15.x, 1.16.x, 1.17.x (the exact patch versions depend on when the backports are merged)


Question: }, }) } <ins> // AutoApproveNodeCertificateRotation creates RBAC rules in a way that makes Node certificate rotation CSR auto-approved by the csrapprover controller func AutoApproveNodeCertificateRotation(client clientset.Interface, k8sVersion *version.Version) error { // Create autorotation cluster role binding only if we deploying or upgrading to version that supports it. if k8sVersion.AtLeast(constants.MinimumCSRAutoApprovalClusterRolesVersion) { fmt.Println("[bootstraptoken] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster") return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: NodeAutoApproveCertificateRotationClusterRoleBinding, }, RoleRef: rbac.RoleRef{ APIGroup: rbac.GroupName, Kind:     "ClusterRole", Name:     NodeSelfCSRAutoApprovalClusterRoleName, }, Subjects: []rbac.Subject{ { Kind: "Group", Name: constants.NodesGroup, }, }, }) } return nil } </ins>

Reported in Scenario (CentOS 7): install 1.7.x packages for kubelet on nodes. install cluster with kubeadm init stable-1.7 use kubeadm to upgrade control plane to 1.8.0-rc.1 on nodes which still run 1.7.x kubelet, install new rpms for where certificate-rotation is enabled. systemctl daemon-reload, systemctl restart kubelet. result: kubelet is in wait loop, waiting for csr to be approved. node is in NotReady state. The kubelet blocks on startup, despite having a valid kubeconfig file. It should start up, attempt to rotate in the background, but not block.
I think this is because of the bootstrap kubeconfig flag. If you only provide the cert rotation flag it should only bootstrap in the background. That's what we where doing for bootkube where we need to start static pods while we're doing our initial cert rotation
a bit of debug shown that returns on the last line . So, kubelet has valid cert, but still wants to rotate it. That in theory should be a valid scenario. Question is: why it's not automatically approved then ? Even if cert will be in proper place, does that mean that automatic rotation wouldn't work for expired certs ? Or I'm missing something here.
no, it's because when a file is provided containing only cert/key data, the cert manager blocks on Start() until a CSR is approved
proposed fix for unblocking Start() in , which runs rotation in the background if you have a starting certificate
/cc
what does set up for auto-approval of node self certificate requests during an upgrade to 1.8?
for cert manager hang for kubeadm auto-approval setup issues upgrading to 1.8
looks like kubeadm upgrade instructions indicate approval of CSRs for upgrades nodes is expected: http://kubernetes-io-vnext- I'd expect that only to be required once certs expired, not automatically required as part of upgrade
Yeah, I that step to the kubeadm upgrade instructions last-minute today when the problem was discovered and we didn't think we would have a solution in time for 1.8.0. Manually approving the CSRs seems to work fine, but is a less-than-stellar upgrade experience.

lowering priority a notch, since the upgrade procedure is at least documented, and does not default to true for all installations (specific deployment methods like kubeadm enable it)
In case it helps, kubeadm 1.8 code for starting the controller manager is , and the RBAC rules for auto-approval are .
Also, I'm not sure if we have any other docs that need to be updated regarding upgrading kubelet, since this isn't a kubeadm-specific problem.
the combination of enabling rotation and not setting up auto-approval is what bit kubeadm here is a deployment-specific choice, and is not true by default. is the only deployment I see that enables it on upgrade to 1.8 the auto-approval rules at do not enable nodes to rotate their client certificates, that only sets up auto-approval for the initial TLS bootstrapping token I agree the kubelet should behave better in this scenario if it already has a valid credential (like ), but the configured setup by kubeadm doesn't seem correct
Ah, understood. It wasn't until I was looking at the RBAC rules myself just now that I realized we must have missed that. Definitely needs to get fixed in kubeadm.
I've created a PR for the documentation fix for kubeadm upgrades: We still need a kubeadm code fix to add the RBAC rules, hopefully to get into 1.8.1.
Yeah, not creating the ClusterRoleBinding between and was an oversight, but isn't that critical in any way. We'll make that change in v1.8.1, and document things meanwhile. Upgrading your kubelets is a manual process anyway right now, so it's reasonable to add one extra step to create this binding. Is it expected that a kubelet that already has credentials should renew them on restart if is set?
Not in a blocking way, only in the background if needed because of expiration, see
Reopening. Blocking on start is still an issue if auto approval is not enabled.
Fix in
Should this have the v1.8 milestone attached to it?
is everything here now fixed or are we waiting for
ping on the status of
- needs rebase and the mentioned in there is merged. What else needs to be done?

[MILESTONENOTIFIER] Milestone Removed From Issue Important: Code freeze is in effect and only issues with may remain in the v1.9 milestone. <details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
I think this issue is now solved right?


Question: "strings" "sync" "sync/atomic" <ins> "syscall" </ins> "time" "github.com/golang/glog"

There's some practical limit on the number of concurrent access tokens that can be used in a Kubernetes cluster before it begins to spend all of its time in the authn stack verifying them. Absent fixes to it was < 500; with the fixes linked there it's still not infinitely scalable-- I still hit (what I believe to be) with ~ 5k unique tokens @ ~700QPS. This request is to this dimension an official name & definition, and list it with others, e.g. here: it it have a useful value (I think it should be 10k) /kind bug /sig scalability
/cc
oh and 4. add tests which verify it is as high as believed.
cc since projected tokens will increase the number of unique tokens
/sig auth
LRUs are not good at scaling and sharding has some major pitfalls as pointed out. I would suggest that we try to drop the LRU cache for successful authentication results in the token cache. The main benefit of using an LRU here is bound cache size. Entries in the token cache have a fast expiration independent of the LRU policy. The records probably take up O(2kb) (guesstimate) of memory. Given that these are successful attempts with presumably valid tokens, I don't think we are getting any benefit from the LRU. If we don't have a constraint on fixed cache size, we can back the token cache with a sync.Map and run a single background goroutine to periodically walk the map and remove expired entries. This would fix the problematic locking and eviction due to nonuniform distribution of tokens over shards. The failure cache is less important and we don't cache failures by default. If we consider this the "slowpath", we can probably leave it as. If that's not good enough, we have a couple options (with various tradeoffs) to improve it.
If off by default is not enough, more reasoning why the failed cache is not super useful: it might help with an accidental DoS but it doesn't prevent anything actively malicious DoS. I think we can focus our improvements on the success cache.
/assign He is already working on adding an explicit test for it.
has taken it over and is actively working on this. Unfortunately, I cannot assign this issue to him as he's not a k8s member yet
It's not closed - all testes must be submitted and appropriate documentation must be
Thanks everyone!!


Question: can then aim traffic at the `Service` port on that `Node` and it will be proxied to the backends. <ins> ## Choosing your own PortalIP address A user can specify their own ```PortalIP``` address as part of a service creation request.  For example, if they already have an existing DNS entry that they wish to replace, or legacy systems that are configured for a specific IP address and difficult to re-configure.  The ```PortalIP``` address that a user chooses must be a valid IP address and within the portal net CIDR range that is specified by flag to the API server.  If the PortalIP value is invalid, the apiserver returns a ```422``` HTTP status code to indicate that the value is invalid. </ins> ## Shortcomings We expect that using iptables and userspace proxies for portals will work at

According to this page: "PortalIP is usually assigned by the master. If specified by the user // we will try to respect it or else fail the request. This field can // not be changed by updates." PortalIP string to clarify the above - it means that user is allowed to assign it during creation of service, but not during update service - correct? why update is not supported as well? are the considerations for "we will try to respect it" - what validations are made (Except for addr already in use) ? exact error is returned when it can't be assigned?
1) We did not support update because it is a bit more complicated, but mostly because the semantic we want around services is that the IP never changes. 2) We ensure that the IP is within the "portal net" CIDR and not used already. 3) I am 99% sure it returns an HTTP 422 error , abonas wrote:
is there anywhere (or at least planned) the documentation of various errors returned from REST api? I mean beyond the standard 404, 500, etc. For instance - for duplicate entities, validation errors on entity content, things like the above. Is there a distinction between wrong json (which should always fail) and the above which is more of a runtime error? I know some apis (not kubernetes) will actually return 400 in this case and not 422, so declaration of errors will be a great help for consumers of REST api. (one can always experiment, test and register all those errors, but consider how time consuming that can be)
We do plan to document and test (and, in some cases, fix) status code behavior: .
Brian - you 1.0 milestone to this issue, but I'm not sure what is the action item here. Can you please explain what should be done?
The action here is/was to verify the behavior and ensure it is documented correctly. Thanks to we now have general status code behavior documented: It doesn't say what happens in this specific case, though I don't know that it's possible to document every specific case, at least not in a way that's kept up to date. I can confirm that 422 is returned in this case: It would be useful to document that users/clients can specify PortalIP in the services doc:
I'll take the update to and then close this.


Question: terminationGracePeriodSeconds: 600 hostNetwork: true containers: <del> - image: gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:v1.11.0 </del> <ins> - image: gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:v1.12.0 </ins> livenessProbe: httpGet: path: /healthz

found this in while finding beta API dependencies for 1.22. This will block from running at all in 1.22 I think this is for this deployment: , and in ingress-gce I found
/sig network /triage accepted I pinged some GCE sig-network folks in the PR
given that uses extensions/v1beta1 Deployments, that particular manifest seems unused
once we land the fix in , anyone know how to update the image we use in e2e tests?
/assign The update to use v1 APIs is merged to in What version should be tagged we can pick up in Does need to be picked back to one or more release branches?
It would be helpful to get a version containing the fix tagged, so the manifest can be updated in to get e2e signal even before opens for 1.22.
Ack. We are planning to cut a release soon. Once the release is cut, I will update the k/k to use that
what's the ETA for cutting ingress-gce release? I think it might be related to failing pull-kubernetes-e2e-gce-100-performance run in


Question: testSlowMultiplier = 10 // How long to wait until CSINode gets attach limit from installed CSI driver. <del> csiNodeInfoTimeout = 1 * time.Minute </del> <ins> csiNodeInfoTimeout = 2 * time.Minute </ins> ) var _ storageframework.TestSuite = &volumeLimitsTestSuite{}

/kind failing-test <!-- If this is a flake, please add: /kind flake --/sig storage /kind flake
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: Currently, local-up scripts, GCE scripts and kubeadm support switching IPVS proxy mode via exporting environment variables or specifying flags. ### Prerequisite <del> Ensure IPVS required kernel modules </del> <ins> Ensure IPVS required kernel modules (**Notes**: use `nf_conntrack` instead of `nf_conntrack_ipv4` for Linux kernel 4.19 and later) </ins> ```shell ip_vs ip_vs_rr

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--kube-proxy cannot use ipvs mode on Linux 4.19 due to a check for the module, which was changed to (). uses to check for : It should instead accept on Linux 4.19 or above. What happened: kube-proxy fell back from ipvs mode to iptables mode. What you expected to happen: kube-proxy using ipvs. How to reproduce it (as minimally and precisely as possible): Launch a cluster on an image using Linux 4.19 or above with kube-proxy using ipvs mode (see ). Environment: Kubernetes version (use ): All Kernel (e.g. ): 4.19 <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/sig network
/area ipvs /assign /cc
Okay, so should check {nfconntrackipv4 | nf_conntrack} and expect load at least one module?
I think so
Please just go ahead :)


Question: // the list automatically by PodDisruptionBudget controller after some time. // If everything goes smooth this map should be empty for the most of the time. // Large number of entries in the map may indicate problems with pod deletions. <del> DisruptedPods map[string]metav1.Time `json:"disruptedPods" protobuf:"bytes,2,rep,name=disruptedPods"` </del> <ins> // +optional DisruptedPods map[string]metav1.Time `json:"disruptedPods,omitempty" protobuf:"bytes,2,rep,name=disruptedPods"` </ins> // Number of pod disruptions that are currently allowed. PodDisruptionsAllowed int32 `json:"disruptionsAllowed" protobuf:"varint,3,opt,name=disruptionsAllowed"`

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: PodDisruptionBudget provides inappropriate message and cannot be updated as is required filed. What you expected to happen: works fine for editing labels under . Also, produce correct error message when edit field. How to reproduce it (as minimally and precisely as possible): Then, we will get following error after step-2. When edit , it should provide correct message as .When edit especially labels under metadata, they should be updated without error. Anything else we need to know?: According to the code comment, is valid. Environment: Kubernetes version (use ): build on
/sig cli
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale there is a PR open


Question: } } } <ins> func TestValidatePodDisruptionBudgetUpdate(t *testing.T) { c1 := intstr.FromString("10%") c2 := intstr.FromInt(1) c3 := intstr.FromInt(2) oldPdb := &policy.PodDisruptionBudget{} pdb := &policy.PodDisruptionBudget{} testCases := []struct { generations []int64 name        string specs       []policy.PodDisruptionBudgetSpec status      []policy.PodDisruptionBudgetStatus ok          bool }{ { name:        "only update status", generations: []int64{int64(2), int64(3)}, specs: []policy.PodDisruptionBudgetSpec{ { MinAvailable:   &c1, MaxUnavailable: &c2, }, { MinAvailable:   &c1, MaxUnavailable: &c2, }, }, status: []policy.PodDisruptionBudgetStatus{ { PodDisruptionsAllowed: 10, CurrentHealthy:        5, ExpectedPods:          2, }, { PodDisruptionsAllowed: 8, CurrentHealthy:        5, DesiredHealthy:        3, }, }, ok: true, }, { name:        "only update pdb spec", generations: []int64{int64(2), int64(3)}, specs: []policy.PodDisruptionBudgetSpec{ { MaxUnavailable: &c2, }, { MinAvailable:   &c1, MaxUnavailable: &c3, }, }, status: []policy.PodDisruptionBudgetStatus{ { PodDisruptionsAllowed: 10, }, { PodDisruptionsAllowed: 10, }, }, ok: false, }, { name:        "update spec and status", generations: []int64{int64(2), int64(3)}, specs: []policy.PodDisruptionBudgetSpec{ { MaxUnavailable: &c2, }, { MinAvailable:   &c1, MaxUnavailable: &c3, }, }, status: []policy.PodDisruptionBudgetStatus{ { PodDisruptionsAllowed: 10, CurrentHealthy:        5, ExpectedPods:          2, }, { PodDisruptionsAllowed: 8, CurrentHealthy:        5, DesiredHealthy:        3, }, }, ok: false, }, } for i, tc := range testCases { oldPdb.Spec = tc.specs[0] oldPdb.Generation = tc.generations[0] oldPdb.Status = tc.status[0] pdb.Spec = tc.specs[1] pdb.Generation = tc.generations[1] oldPdb.Status = tc.status[1] errs := ValidatePodDisruptionBudgetUpdate(oldPdb, pdb) if tc.ok && len(errs) > 0 { t.Errorf("[%d:%s] unexpected errors: %v", i, tc.name, errs) } else if !tc.ok && len(errs) == 0 { t.Errorf("[%d:%s] expected errors: %v", i, tc.name, errs) } } } </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When trying to forcefully update using command segmentatio voliation happens. What you expected to happen: podDistributionBudget to be replaced How to reproduce it (as minimally and precisely as possible): create : Create : Forcefull update (replace) : Anything else we need to know?: The same error happens on kubectl version 1.6 or 1.7. Manually deleting and recreating works. Environment: Kubernetes version (use ): 1.7 Cloud provider or hardware configuration**: gke OS (e.g. from /etc/os-release): Debian GNU/Linux 7 (wheezy) Kernel (e.g. ): 3.16.0-4-amd64
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <bre.g., for API Machinery<br(2) specifying the label manually: <bre.g., for sig/scalability<br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
/sig cli
This issue not only applies to PBP's but NetworkPolicy (and perhaps others).


Question: e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" ) <del> const etcdImage = "3.5.3-0" </del> <ins> const etcdImage = "3.5.4-0" </ins> // EtcdUpgrade upgrades etcd on GCE. func EtcdUpgrade(targetStorage, targetVersion string) error {

Please publish 3.5.4-0 image and update to to benefit from fix for , while still keeping v3.5.3 critical correctness fixes (, ). For recent PRs for update to etcd v3.5.3 see and
/sig release /sig api-machinery /kind bug
/cc /triage accepted
Hey, Can I work on this ? I'm a beginner, but all the good first issues are mostly assigned. Can you guide me on doing this ?
I am working on this. /assign
Hi I would like to work on Make sense to backport in v1.22, v1.23 & v1.24? Thanks!!
hi, it makes sense to me, but requires approval from dependency maintainers. i can only approve the kubeadm changes.
was this backported ?


Question: event := &v1.Event{ ObjectMeta: v1.ObjectMeta{ <del> Name: FederationEventName, </del> <ins> Name:      FederationEventName, Namespace: namespace, </ins> }, InvolvedObject: v1.ObjectReference{ Kind:       "Pod", Name:       "pod-name", <del> Namespace:  "default", </del> <ins> Namespace:  namespace, </ins> UID:        "C934D34AFB20242", APIVersion: "version", },

https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
assigning to you because it looks like you changed this recently, feel free to reassign as appropriate.
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e

suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created

and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events

[Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e suite}
https://k8s- Failed: [] Federation events [Feature:Federation] Event objects should be created and deleted successfully {Kubernetes e2e


Question: Param("fieldValidation", "Strict"). Body(yamlBody). DoRaw(ctx) <del> if !(strings.Contains(string(result), `line 11: key "foo" already set in map`)) { </del> <ins> if !(strings.Contains(string(result), `line 9: key "foo" already set in map`)) { </ins> framework.Failf("error missing duplicate field: %v:n%v", err, string(result)) } })

After running k8s conformance testing on kubernetes v1.27, we have leftover custom resources of the format that should not exist. For example) Basic test instructions: We use the latest official kubernetes kubekins image: We extract kubetest binary from kubekins latest-master docker image. We run serial conformance tests as follows. We also run parallel conformance tests as follows.
/sig api-machinery
The following script is used to cleanup the leftover CRDs:
/triage accepted /assign /cc
/area conformance
/sig architecture


Question: type readWriteOncePodTest struct { config *storageframework.PerTestConfig <del> cs     clientset.Interface volume *storageframework.VolumeResource pods   []*v1.Pod </del> <ins> cs            clientset.Interface volume        *storageframework.VolumeResource pods          []*v1.Pod priorityClass *schedulingv1.PriorityClass </ins> migrationCheck *migrationOpCheck }

For 1.27 we should graduate the ReadWriteOncePod feature to beta. This is needed to rollout the feature to GA. In beta this feature will support preemption of pods using PVCs. See:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig storage /sig scheduling


Question: var prober volume.DynamicPluginProber = nil // TODO (#51147) inject mock kubelet.volumePluginMgr, err = <del> NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, token.NewManager(kubelet.kubeClient.CoreV1()), allPlugins, prober) </del> <ins> NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, token.NewManager(kubelet.kubeClient), allPlugins, prober) </ins> require.NoError(t, err, "Failed to initialize VolumePluginMgr") kubelet.mounter = &mount.FakeMounter{}

/kind bug /priority failing-test /priority important-critical /milestone v1.11 /assign cc All GKE jobs are failing in various release dashboards. Code freeze for 1.11 is a couple of hours away.
Common error appears to be:
Failures seem to align with when was merged. But nothing in that PR seems suspicious. Having verify. Bringing up my own cluster on head to try and get a live cluster.
I have a quick try to reproduce one of the failure test cases by gcloud container clusters create --addons=HttpLoadBalancing,HorizontalPodAutoscaling,KubernetesDashboard --machine-type=n1-standard-2 --image-type=gci --num-nodes=3 --cluster-version=1.11.0-beta.1.222+ test- The cluster isn't up. What I observed is the following: 1) kubelet on master is in crash-loop due to a panic No other master components could be up. 2) kubelet is in crashloop in node as well.
/sig node Looks like has a patch up at
/priority critical-urgent
/status approved-for-milestone
Waiting for new runs to verify the fix. Will reopen this issue if needed
/reopen Reopening this one until its actually verified to ensure it shows up in our tracking queries.
I see the GKE tests cleaning up in the . still doesn't seem to have the change (will have to wait for cutover?). The fix looks promising. Will wait for a couple more runs before I close
[MILESTONENOTIFIER] Milestone Issue Needs Attention Action required: During code freeze, issues in the milestone should be in progress. If this issue is not being actively worked on, please remove it from the milestone. If it is being worked on, please add the label so it can be tracked with other in-flight issues. Action Required: This issue has not been updated since Jun 6. Please provide an update. Note: This issue is marked as , and must be updated every 1 day during code freeze. Example update: <details<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : Fixes a bug discovered during the current release.

</details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
all the GKE ones still look good /close


Question: } } <ins> func TestStoreGracefulDeleteWithResourceVersion(t *testing.T) { podA := &example.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec:       example.PodSpec{NodeName: "machine"}, } testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), "test") destroyFunc, registry := NewTestGenericStoreRegistry(t) defer destroyFunc() defaultDeleteStrategy := testRESTStrategy{scheme, names.SimpleNameGenerator, true, false, true} registry.DeleteStrategy = testGracefulStrategy{defaultDeleteStrategy} // test failure condition _, _, err := registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, nil) if !errors.IsNotFound(err) { t.Errorf("Unexpected error: %v", err) } // create pod _, err = registry.Create(testContext, podA, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if err != nil { t.Errorf("Unexpected error: %v", err) } // try to get a item which should be deleted obj, err := registry.Get(testContext, podA.Name, &metav1.GetOptions{}) if errors.IsNotFound(err) { t.Errorf("Unexpected error: %v", err) } accessor, err := meta.Accessor(obj) if err != nil { t.Errorf("Unexpected error: %v", err) } resourceVersion := accessor.GetResourceVersion() options := metav1.NewDeleteOptions(0) options.Preconditions = &metav1.Preconditions{ResourceVersion: &resourceVersion} // delete object _, wasDeleted, err := registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, options) if err != nil { t.Errorf("Unexpected error: %v", err) } if !wasDeleted { t.Errorf("unexpected, pod %s should have been deleted immediately", podA.Name) } // try to get a item which should be deleted _, err = registry.Get(testContext, podA.Name, &metav1.GetOptions{}) if !errors.IsNotFound(err) { t.Errorf("Unexpected error: %v", err) } } </ins> // TestGracefulStoreCanDeleteIfExistingGracePeriodZero tests recovery from // race condition where the graceful delete is unable to complete // in prior operation, but the pod remains with deletion timestamp

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: When developing unit tests related to PDBs/Deletion, mutates pod resource via adding deletion timestamp, and this directly conflicts with checking resource version. What you expected to happen: No resource version mutation. How to reproduce it (as minimally and precisely as possible): I'm attempting to add preconditions to delete call in eviction API: Anything else we need to know?: Happens locally and in CI. Environment: Kubernetes version (use ): Head of master: Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
can you provide more details about what you're observing?
You can see an example here: Here's the relevant section of the tests: There really shouldn't be anything modifying the pod after creation until the delete request, thus I don't believe we should be hitting this precondition failure. Resource Version in precondition I believe to be correct, but when we contact etcd to verify, it's increased by one. It's not clear why this is happening, it appears to be a bug.
Okay, I've sorted this out. We have a conflict with 'graceful' deletion with this approach: This updates the object to add a deletion timestamp, thus the first call to eviction will always fail if we need to set ResourceVersion.


Question: nodes = append(nodes, nodeList.Items...) } <del> allocatable := make(map[string]v1.ResourceList) </del> <ins> availableResources := make(map[string]v1.ResourceList) </ins> for _, n := range nodes { <del> allocatable[n.Name] = n.Status.Allocatable </del> <ins> if !o.ShowCapacity { availableResources[n.Name] = n.Status.Allocatable } else { availableResources[n.Name] = n.Status.Capacity } </ins> } <del> return o.Printer.PrintNodeMetrics(metrics.Items, allocatable, o.NoHeaders, o.SortBy) </del> <ins> return o.Printer.PrintNodeMetrics(metrics.Items, availableResources, o.NoHeaders, o.SortBy) </ins> } func getNodeMetricsFromMetricsAPI(metricsClient metricsclientset.Interface, resourceName string, selector labels.Selector) (*metricsapi.NodeMetricsList, error) {

What happened: We ran the following, and got the following: How/why do we have a node reporting 100% memory usage? (There seems to be plenty of memory on the host as reported by the kernel's statistic. (multiple gigabytes) What you expected to happen: Memory usage can't exceed 100%, no? How to reproduce it (as minimally and precisely as possible): we unfortunately don't know Anything else we need to know?: no swap on these VMs. We're curious what kernel memory statistic goes into computing the total from Kubernetes; it's my understanding that there is various ways to go over 100%, e.g., by summing RSS over several processes. (E.g., shared and resident pages would get double-counted.) Environment: Kubernetes version (use ):Cloud provider or hardware configuration: Azure OS (e.g: ): AKS instance? Kernel (e.g. ): (unsure, since this is Azure AKS; we don't have good access to this piece of data…) Install tools: Azure? Network plugin and version (if this is a network-related bug): N/A / Azure Others: None
(Taking my best shot here, but if this is wrong please freely adjust it.) /sig instrumentation
Other than 101% reporting issue, do you observe any other abnormality ?
Well, yes, the node didn't seem to be actually at 100% memory use; as mentioned, it seemed to have significant headroom.
how about
Node Memory utilization is ratio of node Working Set Bytes and node allocatable memory. Allocatable memory is available on node object: Node Working Set Bytes are available on api: Please provide results from those commands so we can distinquish if problem is in kubectl top or in metrics pipeline.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with .

Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I have similar problem: kubectl describe node secondary-pool-dqiqzzikb5 kubectl get --raw I have no idea what's causing it or how to fix :/
You can't reopen an issue/PR unless you authored it or you are a collaborator. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen We have the same issue.
Reopened this issue. <detailsIn response to : /reopen We have the same issue. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-lifecycle rotten /triage accepted /assign
Seeing same issue here too
Another data point (on AWS EKS): Seems like happen only on smaller nodetypes (I see it with both micro/nano, others are medium) Usage is displayed correctly with NodeMetrics:
Why kubelet node memory utilization can exceed 100% (same can happen for CPU)? Simple, because it's not utilization of physical device, but utilization of resources allocated for pods and system daemons. How utilization of resources allocated is calculated? Simple, it's sum of resources used by on node divided by all resources allocated. How Kubelet allocates resources for pods? It takes all resources available on VM and substracts resources that it reserves for itself, kernel etc. How Kubelet knows how much resources it needs to reserve? User provides it in flags.

There is no error, there is no Kubernetes magic that gets us over 100%, it's just question of how you define utilization. Kubelet reserves some resources for system, they are not included when calculating node utilization, which means it can go above 100% when pods start using reserved resources (this is what over-committing means in "kubectl describe node" output). Some math based on your kubectl describe node output Node capacity: 968424Ki Node allocatable: 559848Ki Node Usage 717664Ki Node utilization: Node usage / node allocatable = 717664Ki / 559848Ki = 128% VM utilization: Node usage / node capacity = 717664Ki / 968424Ki = 74%
Based on my reading of the , I don't think "Node Usage" include only usage from pods. The includes usage by system daemons, so I'm not sure it makes sense to compare to allocatable, which is meant to exclude system daemons.
Hmm, I was not aware of that. This is a good point, as utilization was implemented in kubectl by different SIG (CLI), it's possible that it was not properly reviewed by other stakeholders (SIG Instrumentation/SIG Node). I would make sense to revisit what should be displayed as node utilization.


Question: genericfeatures "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server/egressselector" utilfeature "k8s.io/apiserver/pkg/util/feature" <ins> utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" </ins> "k8s.io/apiserver/pkg/util/x509metrics" restclient "k8s.io/client-go/rest" "k8s.io/client-go/transport"

On a cluster with many APIs provided by aggregated apiservers, with controllers managing those APIs, the number of occupied seats effectively has a floor of 1 per watch. If the combined max-inflight limit is reduced low enough, priority levels tend to become saturated by these long-running watch requests and other requests are starved. Watch requests consume a seat only during initialization. a PriorityLevelConfiguration and a corresponding FlowSchema to match watches against a resource served by an aggregated apiserver. some watch requests. the metric climb on the priority level per watch until all seats are exhausted. Kubernetes version (use ): (OpenShift 4.9) /sig api-machinery
cc
D'oh! I can see two obvious possible ways to cope: one apiserver not manage watch requests the are delegated to another apiserver; or a marker into the notification stream so that the non-origin apiservers can detect when initialization is done.
I would even say that for regular (non-long-running) requests, it's not obvious that they should be managed by P&F even. [I agree that for watch initialization it's particularly bad though.] - do we have a simple way to detect if the request will be served from our kube-apiserver or from some other aggregated apiserver?
There is not currently a way to know that ahead of time in the filters today. It require a list/watch, but it ought not be excessively expensive to make a 99% correct prediction, but the actual decision is made by the aggregated apiserver later in the request.
/triage accepted
: is there a way that we can plumb that knowledge forward to the filters?


Question: client := fake.NewSimpleClientset( &v1.Service{Spec: v1.ServiceSpec{Selector: map[string]string{"foo": ""}}}, ) <del> ctx := context.Background() </del> <ins> ctx, cancel := context.WithCancel(context.Background()) defer cancel() </ins> informerFactory := informers.NewSharedInformerFactory(client, 0) _ = informerFactory.Core().V1().Services().Lister() informerFactory.Start(ctx.Done())

The ctx object () is context.Background(). Thus, its Done() method returns a nil channel. Later, the nil channel is used to call and as their argument. The goroutines created in the two functions are leaked. No goroutine leaks in TestSelectorSpreadScore(). Run TestSelectorSpreadScore() No response all versions after <details</details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling (It sounds like leas in unit tests, which may be negligible. Does it cause some test failures?)
the test can pass.
/assign
/unassign /assign


Question: // referenced by the pod spec. If visitor returns false, visiting is short-circuited. // Transitive references (e.g. pod -> pvc -> pv -> secret) are not visited. // Returns true if visiting completed, false if visiting was short-circuited. <del> func VisitPodSecretNames(pod *api.Pod, visitor Visitor) bool { </del> <ins> func VisitPodSecretNames(pod *api.Pod, visitor Visitor, containerType ContainerType) bool { </ins> for _, reference := range pod.Spec.ImagePullSecrets { if !visitor(reference.Name) { return false } } <del> VisitContainers(&pod.Spec, AllContainers, func(c *api.Container, containerType ContainerType) bool { </del> <ins> VisitContainers(&pod.Spec, containerType, func(c *api.Container, containerType ContainerType) bool { </ins> return visitContainerSecretNames(c, visitor) }) var source *api.VolumeSource

Fully described in , and will visit these fields for all containers, regardless of whether they're hidden behind a feature gate (e.g. ). This is usually correct, these methods are often used to drop fields and in this case should visit all containers that are persisted regardless of whether they're disabled, but the callers of these methods should signal that explicitly rather than relying on the implicit behavior. /cc /priority important-longterm /kind bug /sig node
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : Fully described in , and will visit these fields for all containers, regardless of whether they're hidden behind a feature gate (e.g. ). /priority important/long-term /kind bug /sig node Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
are you interested in taking this one, too?
VisitPodSecretNames and VisitPodConfigmapNames already specify AllContainers for VisitContainers Can you let me know what else needs to be done?
Callers should specify whether they want to visit all containers or not. There are valid reasons for each, so we shouldn't assume the desired behavior, even if 100% of the current users want to visit all containers. Jordan's original comment in has some examples of current uses. I don't think this needs to be as configurable as , so my naive first attempt to implement this would probably just be a new to and


Question: .PHONY:	build push PREFIX=gcr.io/google_containers <del> TAG = 1.34 </del> <ins> TAG = 1.35 </ins> build: docker build --pull -t $(PREFIX)/fluentd-gcp:$(TAG) .

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): no What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): fluentd timezone timeformat Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): What happened: Log lines from docker have the wrong timestamp. Anything else do we need to know: We are building a custom Docker Image for fluentd/td-agent which is based on your Dockerfile but with some slight changes. One of which is that we set the timezone to something else than UTC. In this case fluentd will produce wrong timestamps for log lines since the string only includes a simple Z in the end. Changing the time_format to fixed the problem for us. Of course this is not really a bug in kubernetes but changing the default to the above will probably help others in the future.
Thanks for reporting! It seems to be a bug


Question: e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" ) <del> //go:embed flexvolume guestbook kubectl sample-device-plugin.yaml scheduling/nvidia-driver-installer.yaml statefulset storage-csi </del> <ins> //go:embed cluster-dns flexvolume guestbook kubectl sample-device-plugin.yaml scheduling/nvidia-driver-installer.yaml statefulset storage-csi </ins> var e2eTestingManifestsFS embed.FS func GetE2ETestingManifestsFS() e2etestfiles.EmbeddedFileSource {

What happened: We've been doing some release readiness and auditing all tagged ginkgo tests... Looks like The test seems to expect a GOPATH when running. Im seeing some failures in sonobuoy and suspect a possible relation there but havent dove into it. Either way, its clear this test might need some updating/fixing/cleanuping and parts can be rewritten/improved. What you expected to happen: No reliance on GOPATH for an e2e test ... I dont remember the whole history on this, but no tests that rely on static files currently look in gopath, from what i can tell ? instead we plumbed in a repo-root/ option a few years ago. There may be a state of the art in terms of how we do this.... but i thought it was at one point which was an option passed in. How to reproduce it (as minimally and precisely as possible): Try to run this in sonobuoy or any other environment where there's no GOPATH or non-standard source trees, pretty sure it might not work. Anything else we need to know?: I think the fix here is to just embed the YAML in golang like we do in other tests. along the way there's other cleanup we can do to improve this test. Environment: Kubernetes version (use ): master
/sig network
/assign
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
the YAML files are pretty small, so +1 to embed them in i don't think testing the YAML from the separate repository () directly is required here. ref /help /kind cleanup /priority backlog
/remove-help
/assign Also I should say, I talked to Jay on slack and he is ok with me taking a look at it, if thats ok.
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at .

/lifecycle stale
/remove-lifecycle stale
xref: go 1.16 has built in support for embedding go 1.17 looks to be removing GOPATH
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/help wanted
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
/remove-lifecycle rotten still a problem. FYI re: GOPATH removal
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at .

/lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
/assign