Question: framework.ConformanceIt("should perform rolling updates and roll backs of template modifications", func() { By("Creating a new StatefulSet") ss := framework.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels) <del> sst := framework.NewStatefulSetTester(c) sst.SetHttpProbe(ss) ss, err := c.AppsV1().StatefulSets(ns).Create(ss) Expect(err).NotTo(HaveOccurred()) sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) ss = sst.WaitForStatus(ss) currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision Expect(currentRevision).To(Equal(updateRevision), fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s", ss.Namespace, ss.Name, updateRevision, currentRevision)) pods := sst.GetPodList(ss) for i := range pods.Items { Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(currentRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to current revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[apps.StatefulSetRevisionLabel], currentRevision)) } sst.SortStatefulPods(pods) sst.BreakPodHttpProbe(ss, &pods.Items[1]) Expect(err).NotTo(HaveOccurred()) ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) newImage := NewNginxImage oldImage := ss.Spec.Template.Spec.Containers[0].Image By(fmt.Sprintf("Updating StatefulSet template: update image from %s to %s", oldImage, newImage)) Expect(oldImage).NotTo(Equal(newImage), "Incorrect test setup: should update to a different image") ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { update.Spec.Template.Spec.Containers[0].Image = newImage }) Expect(err).NotTo(HaveOccurred()) By("Creating a new revision") ss = sst.WaitForStatus(ss) currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision Expect(currentRevision).NotTo(Equal(updateRevision), "Current revision should not equal update revision during rolling update") By("Updating Pods in reverse ordinal order") pods = sst.GetPodList(ss) sst.SortStatefulPods(pods) sst.RestorePodHttpProbe(ss, &pods.Items[1]) ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) ss, pods = sst.WaitForRollingUpdate(ss) Expect(ss.Status.CurrentRevision).To(Equal(updateRevision), fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal update revision %s on update completion", ss.Namespace, ss.Name, ss.Status.CurrentRevision, updateRevision)) for i := range pods.Items { Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(newImage), fmt.Sprintf(" Pod %s/%s has image %s not have new image %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, newImage)) Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(updateRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to update revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[apps.StatefulSetRevisionLabel], updateRevision)) } By("Rolling back to a previous revision") sst.BreakPodHttpProbe(ss, &pods.Items[1]) Expect(err).NotTo(HaveOccurred()) ss, pods = sst.WaitForPodNotReady(ss, pods.Items[1].Name) priorRevision := currentRevision currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision ss, err = framework.UpdateStatefulSetWithRetries(c, ns, ss.Name, func(update *apps.StatefulSet) { update.Spec.Template.Spec.Containers[0].Image = oldImage }) Expect(err).NotTo(HaveOccurred()) ss = sst.WaitForStatus(ss) currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision Expect(currentRevision).NotTo(Equal(updateRevision), "Current revision should not equal update revision during roll back") Expect(priorRevision).To(Equal(updateRevision), "Prior revision should equal update revision during roll back") By("Rolling back update in reverse ordinal order") pods = sst.GetPodList(ss) sst.SortStatefulPods(pods) sst.RestorePodHttpProbe(ss, &pods.Items[1]) ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name) ss, pods = sst.WaitForRollingUpdate(ss) Expect(ss.Status.CurrentRevision).To(Equal(priorRevision), fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal prior revision %s on rollback completion", ss.Namespace, ss.Name, ss.Status.CurrentRevision, updateRevision)) for i := range pods.Items { Expect(pods.Items[i].Spec.Containers[0].Image).To(Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to previous image %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Spec.Containers[0].Image, oldImage)) Expect(pods.Items[i].Labels[apps.StatefulSetRevisionLabel]).To(Equal(priorRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to prior revision %s", pods.Items[i].Namespace, pods.Items[i].Name, pods.Items[i].Labels[apps.StatefulSetRevisionLabel], priorRevision)) } </del> <ins> rollbackTest(c, ns, ss) </ins> }) /*

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: It would be good to test rolling updates with RWO PVCs that cannot be attached to multiple nodes at the same time.
There have been reports of this scenario


Question: if lbaas.opts.ManageSecurityGroups { err := lbaas.ensureSecurityGroup(clusterName, apiService, nodes, loadbalancer) if err != nil { <del> // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(ctx, clusterName, apiService) return status, err </del> <ins> return status, fmt.Errorf("Error reconciling security groups for LB service %v/%v: %v", apiService.Namespace, apiService.Name, err) </ins> } }

Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): Master Environment: Cloud provider or hardware configuration: OpenStack OS (e.g. from /etc/os-release): n/a Kernel (e.g. ): n/a Install tools: n/a Others: What happened: now can update a LB instead of deleting it and recreating it. However the security group code is still set to try and create security group rules, and if it gets an error, delete the LB and start over. What you expected to happen: Updating a service does not delete the LBaaS LB How to reproduce it (as minimally and precisely as possible): Enable in the section of Restart a Anything else do we need to know:
you're absolutely right, this is a critical issue (if you enable security group management). The dangerousness of aggressive cleanups has already been discussed in detail in this thread: I hope you guys can fix this.
Yup - I have a larger refactor, that ensures this, and a few other edge cases we have found in our development. It is in massive need of a rebase though, so I will need a day or 2 of uninterrupted time to get it together
Are you still working on this? If not, I can take a look.
ping :)
/close
We just ran into this where Kubernetes recreated an LB with a new Floating IP because the OpenStack API was not available for a few seconds while the LB was updated. Could this issue be re-opened? Since this is a major bug for us, we will start working on a fix for this today.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available .

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: "LimitRange":  true, "PodPreset":   true, <ins> // ignored for being disruptive in an e2e, and getting automatically deleted by a controller "Node": true, </ins> // ignored temporarily while waiting for bug fix. "ComponentStatus":    true, "ClusterRoleBinding": true,

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: gce-master-scale-correctness Which test(s) are failing: [sig-cli] Kubectl client kubectl get output should contain custom columns for each resource Since when has it been failing: 11/6 Testgrid link: https://k8s- Reason for failure: Anything else we need to know: n/a /sig cli /sig scalability /cc
The label(s) cannot be applied. These labels are supported: <detailsIn response to : <!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --gce-master-scale-correctness [sig-cli] Kubectl client kubectl get output should contain custom columns for each resource 11/6 https://k8s- Anything else we need to know: n/a /sig scalability /cc Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Doesn't seem to be scale-related - failed for me the same way in the presubmit:
- do we run correctness test in smaller scale (so we can detect whether failures are scale related or not)? I can't find them on neither https://k8s- nor https://k8s- dashboards.
This is a new test that seem to be flaky at head: - seems that you this in Can you please take a look?
/assign
/reopen Let's wait for the test to pass /assign
Reopened this issue. <detailsIn response to : /reopen /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
That seems to be creating one of every object from the etcd data test in an e2e test. The failing object looks like a node object, which is likely getting deleted by the Node controller as soon as it is created.
opened to skip nodes for this test
/reopen The job is not green
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository.

</details
failed test run started from a commit prior to the fix merging ( started from ) /close
Closing this issue. <detailsIn response to : failed test started from a commit prior to the fix merging ( started from ) Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return nil, diagnosis, err } <del> feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) </del> <ins> feasibleNodesAfterExtender, err := findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) </ins> if err != nil { return nil, diagnosis, err } <del> return feasibleNodes, diagnosis, nil </del> <ins> if len(feasibleNodesAfterExtender) != len(feasibleNodes) { // Extenders filtered out some nodes. // // Extender doesn't support any kind of requeueing feature like EnqueueExtensions in the scheduling framework. // When Extenders reject some Nodes and the pod ends up being unschedulable, // we put framework.ExtenderName to pInfo.UnschedulablePlugins. // This Pod will be requeued from unschedulable pod pool to activeQ/backoffQ // by any kind of cluster events. // https://github.com/kubernetes/kubernetes/issues/122019 if diagnosis.UnschedulablePlugins == nil { diagnosis.UnschedulablePlugins = sets.New[string]() } diagnosis.UnschedulablePlugins.Insert(framework.ExtenderName) } return feasibleNodesAfterExtender, diagnosis, nil </ins> } func (sched *Scheduler) evaluateNominatedNode(ctx context.Context, pod *v1.Pod, fwk framework.Framework, state *framework.CycleState, diagnosis framework.Diagnosis) ([]*v1.Node, error) {

When Extender filters out some Nodes, we don't set any unschedulable plugins at all. It means Extender is completely ignored during the requeueing process. So, what's happening is: If Extender filters out all Nodes during scheduling, this Pod is soon retried because this Pod doesn't have any plugin name in unschedulable plugins. If Extender filters out some Nodes during scheduling and plugins filter out all other Nodes, this Pod is retried based on plugins' QueueingHint. Even if any cluster events happen and it could change Extender's decision (but not any plugins' decision), this Pod won't be requeued to activeQ/backoffQ. The latter case is serious because it could make Pods being stuck in unschedulable pod pool in 5min. We should have a short-term solution for the latter case. We can requeue Pods, which were rejected by Extender, by any kind of cluster events because we cannot know which events make Pods schedulable. Eventually, this issue makes us wonder how Pods rejected by Extender should be requeued. We cannot implement QHint equivalent in Extenders because it'd be too slow to call Extender every time any cluster events happen. Probably, somehow implementing EventsToRegister equivalent in Extender? Use Extender which does something in Filter. No response master <detailsn/a </details<details</details<details</details<details</details<details</details
/assign
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
/reopen We'll close it when all cherry-picks are done.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/close Cherry-picks are done.


Question: for k, v := range labels { labelMap[k] = v } <ins> for k, v := range lvMap { labelMap[k] = v } </ins> return &DataItem{ Labels: labelMap, Data: map[string]float64{

<!-- Please only use this template for submitting enhancement requests --What would you like to be : run got the below error message, We should provide the metric data for different extension points from the metric of "schedulerframeworkextensionpointdurationseconds" Why is this needed: Those metrics have been removed by this commit - Those metrics are deprecated cause it is suggested to use - , but simply remove those metrics definitely is not a nice way to address the issue. /sig scheduling
/triage accept
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /triage accept Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale


Question: f := framework.NewDefaultFramework("critical-pod-test") ginkgo.Context("when we need to admit a critical pod", func() { ginkgo.It("[Flaky] should be able to create and delete a critical pod", func() { <del> configEnabled, err := isKubeletConfigEnabled(f) framework.ExpectNoError(err) if !configEnabled { e2eskipper.Skipf("unable to run test without dynamic kubelet config enabled.") } </del> // because adminssion Priority enable, If the priority class is not found, the Pod is rejected. node := getNodeName(f) // Define test pods

It is unclear why the test needs the dynamic kubelet config. Need to clean it up. /sig node /priority important-soon /help Test is not a part of any testgrid. Need to understand why and return it back to the grid. Also it is marked as flaky. Need to be deflaked. unknown No response No response No response_ /sig node
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : ### Which jobs are failing? /priority important-soon /help ### Which tests are failing? Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hi, I am new to this project and would really love to start contributing and thought this might be a good start. Could you tell me a little more about this problem?
glad to have you, welcome! Some step by step you may start with: Find the test mentioned, understand what it is doing If dependency on DynamicKubeletConfig is found, try to work around them and remove that line. Try running the test locally. If you can repro flakiness - try to fix it If test passes: Try removing the and see if passes in serial tests on presubmits
Thank you so much!

/assign
/cc
it had a non-op config change in the past - - I potentially missed this when removing that
so, if I'd like to fix it, what should I do?
see Since we understood item 2 - it is not needed, just remove the skipper lines. Ideally test needs to be de-flaked as well, but it it not necessarily in the scope of this change
this is a simple issue, so I tried to fix it since I've found it some days.


Question: expectNoError(err) }) <del> It("should create and stop a replication controller", func() { defer cleanup(nautilusPath) By("creating a replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) </del> <ins> Describe("update-demo", func() { var ( updateDemoRoot = filepath.Join(testContext.repoRoot, "examples/update-demo/v1beta1") nautilusPath   = filepath.Join(updateDemoRoot, "nautilus-rc.yaml") kittenPath     = filepath.Join(updateDemoRoot, "kitten-rc.yaml") ) It("should create and stop a replication controller", func() { defer cleanup(nautilusPath, updateDemoSelector) By("creating a replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) }) It("should scale a replication controller", func() { defer cleanup(nautilusPath, updateDemoSelector) By("creating a replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) By("scaling down the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1") validateController(c, nautilusImage, 1) By("scaling up the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2") validateController(c, nautilusImage, 2) }) It("should do a rolling update of a replication controller", func() { // Cleanup all resources in case we fail somewhere in the middle defer cleanup(updateDemoRoot, updateDemoSelector) By("creating the initial replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) By("rollingupdate to new replication controller") runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath) validateController(c, kittenImage, 2) }) </ins> }) <del> It("should scale a replication controller", func() { defer cleanup(nautilusPath) By("creating a replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) By("scaling down the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1") validateController(c, nautilusImage, 1) By("scaling up the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2") validateController(c, nautilusImage, 2) }) </del> <ins> Describe("guestbook", func() { var guestbookPath = filepath.Join(testContext.repoRoot, "examples/guestbook") It("should create and stop a working application", func() { defer cleanup(guestbookPath, frontendSelector, redisMasterSelector, redisSlaveSelector) </ins> <del> It("should do a rolling update of a replication controller", func() { // Cleanup all resources in case we fail somewhere in the middle defer cleanup(updateDemoRoot) </del> <ins> By("creating all guestbook components") runKubectl("create", "-f", guestbookPath) </ins> <del> By("creating the initial replication controller") runKubectl("create", "-f", nautilusPath) validateController(c, nautilusImage, 2) By("rollingupdate to new replication controller") runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath) validateController(c, kittenImage, 2) </del> <ins> By("validating guestbook app") validateGuestbookApp(c) }) </ins> }) }) <del> func cleanup(filePath string) { </del> <ins> func validateGuestbookApp(c *client.Client) { // Wait for frontend to serve content. serving := false for start := time.Now(); time.Since(start) < guestbookResponseTimeout; time.Sleep(5 * time.Second) { entry, err := makeRequestToGuestbook(c, "get", "") if err == nil && entry == `{"data": ""}` { serving = true break } } if !serving { Failf("Frontend service did not start serving content in %v seconds", guestbookResponseTimeout.Seconds()) } // Try to add a new entry to the guestbook. added := false for start := time.Now(); time.Since(start) < guestbookResponseTimeout; time.Sleep(5 * time.Second) { result, err := makeRequestToGuestbook(c, "set", "TestEntry") if err == nil && result == `{"message": "Updated"}` { added = true break } } if !added { Failf("Cannot added new entry in %v seconds", guestbookResponseTimeout.Seconds()) } // Verify that entry is correctly added to the guestbook. entry, err := makeRequestToGuestbook(c, "get", "") if err != nil { Failf("Request to the guestbook failed with: %v", err) } if entry != `{"data": "TestEntry"}` { Failf("Wrong entry received: %v", entry) } } func makeRequestToGuestbook(c *client.Client, cmd, value string) (string, error) { result, err := c.Get(). Prefix("proxy"). Resource("services"). Name("frontend"). Suffix("/index.php"). Param("cmd", cmd). Param("key", "messages"). Param("value", value). Do(). Raw() return string(result), err } func cleanup(filePath string, selectors ...string) { </ins> By("using stop to clean up resources") runKubectl("stop", "-f", filePath) <del> resources := runKubectl("get", "pods,rc", "-l", updateDemoSelector, "--no-headers") if resources != "" { Failf("Resources left running after stop:n%s", resources) </del> <ins> for _, selector := range selectors { resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers") if resources != "" { Failf("Resources left running after stop:n%s", resources) } </ins> } }

Implement guestbook test as a kubectl-driven test in . this should be a fairly straightforward follow up from , if you are interested/have cycles.
As I wrote in the email the plan is: problem w guestbook example (). test becomes actual test (). it do ginkgo. Should be done this week, but it depends on how fast 1. can be done.


Question: {"range nodes capacity", `{range .items[*]}[{.metadata.name}, {.status.capacity}] {end}`, nodesData, "[127.0.0.1, map[cpu:4]] [127.0.0.2, map[cpu:8]] "}, {"user password", `{.users[?(@.name=="e2e")].user.password}`, &nodesData, "secret"}, <ins> {"hostname", `{.items[0].metadata.labels.kubernetes.io/hostname}`, &nodesData, "127.0.0.1"}, {"hostname filter", `{.items[?(@.metadata.labels.kubernetes.io/hostname=="127.0.0.1")].kind}`, &nodesData, "None"}, </ins> } testJSONPath(nodesTests, t)

I love using but I cannot figure out how to print an annotation that contains a special character like or or . This does not work: nor this: nor this: nor this: kubectl get pods tmp--dhpx0 -o jsonpath="{$['metadata']['annotations']['']}" (the last one should work according to what I can find about jsonPath, but doesn't, and anyhow is hard to get shell escaping right for. Need a way to quote a
A use case for this is extracting "created-by" or "init-container" annotations. I've wanted to do both in the last month and fought jsonpath.
FYI, this is possible with go-template. We should still fix the jsonpath though. Similar issue:


Question: } // InstallTracingFilter installs OpenTelemetry tracing filter with the restful Container. <del> func (s *Server) InstallTracingFilter(tp oteltrace.TracerProvider) { s.restfulCont.Filter(otelrestful.OTelFilter("kubelet", otelrestful.WithTracerProvider(tp))) </del> <ins> func (s *Server) InstallTracingFilter(tp oteltrace.TracerProvider, opts ...otelrestful.Option) { s.restfulCont.Filter(otelrestful.OTelFilter("kubelet", append(opts, otelrestful.WithTracerProvider(tp))...)) </ins> } // addMetricsBucketMatcher adds a regexp matcher and the relevant bucket to use when

Follow-up to Discussed in As part of kubelet tracing beta, we should add tracing support to the kubelet's read-only endpoint, which currently does not have tracing. It should use WithPublicEndpoint to prevent callers from influencing sampling decisions. Part of
/sig node /sig instrumentation
/triage accepted /assign
can i help you with this issue?
I don't think we'll be able to start with this until merges
Ok, I'm waiting to see if I can help you. Thank you
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale is merged now, so this can be worked on.
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
Could I help with this issue?
Sure. See We pass a NoOp Tracer provider to disable tracing for the endpoint. Instead, we need to pass the tracerprovider, but make sure that the eventual uses withPublicEndpoint. Looking closer, i'm not sure its currently possible to use WithPublicEndpoint with the otelrestful package... doesn't contain that option (unlike otelhttp: ), so we may need to add that upstream first.

Another thing to keep in mind is that updating otel libraries is currently blocked by (see my attempt in )
I gave the whole thing another try. At least locally, the dependencies seem now to be able to be updated. :raccoon: Could you maybe have a look at it :)
He's on leave for the time being.
oh, can you provide some feedback or point me to the right people?
I'm back now. Feel free to assign me to PRs related to this, and I'll review
Welcome back :) Could you re-run the CI ? Its just the dependencies. The PR itself shouldn't be that much afterwards.
otel is now up-to-date enough. here:
Hi, I would like to ask about one trouble that caused by WithPublicEndpoint. Aggregated APIServer that I customized uses kubernetes's RecommendedConfig. Due to the fact that WithPublicEndpoint is enabled, it can't be tracked by the full trace for its span-link model. And I think child-parent span model is more appropriate. So, could you please give me some advice? Looking forward to your reply.
thanks. We know about this, and are tracking it in


Question: t.Helper() assert.ElementsMatch(t, expected, r.evaluations) } <ins> func TestPrioritizePods(t *testing.T) { isController := true sampleOwnerReferences := []struct { ownerRefs []metav1.OwnerReference }{ { ownerRefs: []metav1.OwnerReference{ { UID:        uuid.NewUUID(), Controller: &isController, }, }, }, { ownerRefs: []metav1.OwnerReference{ { UID:        uuid.NewUUID(), Controller: &isController, }, }, }, { ownerRefs: []metav1.OwnerReference{ { UID:        uuid.NewUUID(), Controller: &isController, }, }, }, } var pods []*corev1.Pod randomSource := rand.NewSource(time.Now().Unix()) for _, sampleOwnerRef := range sampleOwnerReferences { // Generate multiple pods for a controller for i := 0; i < rand.New(randomSource).Intn(5)+len(sampleOwnerReferences); i++ { pods = append(pods, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ OwnerReferences: sampleOwnerRef.ownerRefs, }, Spec: corev1.PodSpec{}, }) } } a := &Admission{} prioritizedPods := a.prioritizePods(pods) controllerRef := make(map[types.UID]bool) for i := 0; i < len(sampleOwnerReferences); i++ { if controllerRef[metav1.GetControllerOfNoCopy(prioritizedPods[i]).UID] { assert.Fail(t, "Pods are not prioritized based on uniqueness of the controller") } controllerRef[metav1.GetControllerOfNoCopy(prioritizedPods[i]).UID] = true } if len(prioritizedPods) != len(pods) { assert.Fail(t, "Pod count is not the same after prioritization") } } </ins>

When updating policy labels on a namespace, existing pods are evaluated to see if they are compatible with the new policy. If there are too many pods, or if evaluation takes too long, we short circuit and warn that not all pods were checked. For that reason, we want to prioritize checking a variety of pods before checking essentially identical pods, so that unique warnings can be surfaced before reaching the pod count cutoff or evaluation timeout. Pods created by the same controller are most likely to be identical (e.g. replicas from a replicaset or job, copies of pods from a statefulset or daemonset, etc) cc
This is an interesting fix I'd like to take this up Can I take this up? I am fairly new to Kubernetes source code so might need some help
I am guessing this is where the logic you mentioned should be implemented right?
thanks for the interest; this is targeted at the beta version... right now, effort is going toward the alpha-blocking work for 1.22
Should I wait for the code freeze before working on this? Sorry for lame questions
If you want to start looking at it, that's fine, but it's possible the code in the ValidateNamespace function will change before we're ready to start merging 1.23 work, and there isn't really reviewer bandwidth to take a look at work-in-progress stuff for next release yet.
Sure, I'll just for code freeze before I start with implementation By the time I'll just assign this to myself /assign
Does it make sense to take it up now? I think we already have a couple of RC builds after the code freeze
sure, it's fine to start work in a branch in your fork and have a WIP PR open, even before master reopens for 1.23 work
Awesome, I'll start working on it Thank you
Hello! Are you still working on this? If you don't have time, I can take this issue. If not, maybe I can give some help
Since this was not an urgent issue I was working on some other issues last week I have started on this, Will be raising a PR maybe today or tomorrow I Will ask you if I get stuck Thank you


Question: DefaultEndpointReconcilerInterval = 10 * time.Second // DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer DefaultEndpointReconcilerTTL = 15 * time.Second <del> // DefaultStorageEndpoint is the default storage endpoint for the lease controller DefaultStorageEndpoint = "kube-apiserver-endpoint" </del> ) type ExtraConfig struct {

/kind bug The lease reconciler creates the kubernetes endpoint with a prefix of kube-apiserver-endpoint, however that is not the prefix that the rest of kube uses. The end result is that the kubernetes service doesn't have an endpoint, and kubectl won't list the kubernetes endpoint. The lease reconciler also creates lease files with a really long lease time instead of the default 15 seconds.
/cc
cc xref
kube-apiserver-endpoint is used for the lease itself, from what I remember. I don't think that's an issue by itself.
Oh but it does look like the endpointRegistry for the lease reconciler is misconfigured to use that
i'll look into it
what is the name that should be used?
services/endpoints afaik -
Yeah, making that change locally fixed it for me
But I'm not seeing the masterleases/apiServerIPInfo data in etcd, or I'm issuing the wrong command to find it
I had that same problem. The lease timeout is set way high, but for some reason that resulted in the lease files being created for only a very short time. I'm not sure why. Playing with the ttl amount got the lease files to stay around....forever. I've got a PR I'm about to push to fix both issues.
FYI has a PR for this now
cc


Question: "os" "path/filepath" "reflect" <ins> "runtime" </ins> "strings" "sync" "testing"

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: unit tests fail on osx. What you expected to happen: How to reproduce it (as minimally and precisely as possible): To see this unit test fail, you need to run go test on an osx machine. Environment: OS: macOS High Sierra v10.13.3
/sig storage /assign


Question: shift $((OPTIND - 1)) # Use eval to preserve embedded quoted strings. <del> eval "goflags=(${GOFLAGS:-})" </del> eval "testargs=(${KUBE_TEST_ARGS:-})" # Used to filter verbose test output.

What happened: When building kubernetes from source for 1.10 it was possible to limit the parallel linker calls using GOFLAGS="-p 1". (I understand that the new format is GOFLAGS="-p=1" so I am using that) With kubernetes 1.11 the build aborts: What you expected to happen: The build should complete, using just one process for linking. How to reproduce it (as minimally and precisely as possible): export GOFLAGS="-p=1" make -j1 Anything else we need to know?: The environment is quite memory constrained, therefore preventing the build to fork a linker for each CPU is required. Any other option instead of the GOFLAGS would work as well. Environment: Kubernetes version (use ): 12.0 OS (e.g. from /etc/os-release): Arch Linux ARM Others: go version go1.11.1 linux/arm64
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig testing
This is because a change in go 1.11, see , and related go issues: , etc. I've opened a PR to address this: .
/assign


Question: } if host == "" { <del> return nil, fmt.Errorf("couldn't find external IP address for node %s", node.Name) </del> <ins> // No external IPs were found, let's try to use internal as plan B for _, a := range node.Status.Addresses { if a.Type == v1.NodeInternalIP { host = net.JoinHostPort(a.Address, sshPort) break } } } if host == "" { return nil, fmt.Errorf("couldn't find any IP address for node %s", node.Name) </ins> } Logf("SSH %q on %s(%s)", cmd, node.Name, host)

Currently IssueSSHComand in only checks for External IP address to ssh, this PR adds check for internal IP too. /sig storage /kind bug


Question: podCopy := pod.DeepCopy() // NominatedNodeName is updated only if we are trying to set it, and the value is // different from the existing one. <del> if !podutil.UpdatePodCondition(&podCopy.Status, condition) && (len(nominatedNode) == 0 || pod.Status.NominatedNodeName == nominatedNode) { </del> <ins> if !podutil.UpdatePodCondition(&podCopy.Status, condition) && pod.Status.NominatedNodeName == nominatedNode { </ins> return nil } <del> if nominatedNode != "" { podCopy.Status.NominatedNodeName = nominatedNode } </del> <ins> podCopy.Status.NominatedNodeName = nominatedNode </ins> return patchPod(client, pod, podCopy) }

What happened: Scheduler decided that it wants to schedule Pod P on Node N after preempting some pods from N. It set the P.nominatedNodeName to N. After a while (before scheduling actually happened) node N was removed from the cluster. Scheduler did not reset P.nominatedNodeName to empty value. What you expected to happen: I would expect scheduler to unset P.nominatedNodeName when the node is no longer valid. The trivial reason for node not being valid any more is it is removed from the cluster. But there are more cases to cover. Out of the top of my head I may think of: extra Pod was scheduled to node and there is not enough resources any more either scheduled by scheduler or directly as static pod/daemonset node became unschedulable How to reproduce it (as minimally and precisely as possible): We did not try to reproduce issue manually. Yet following setup should work. create cluster with nodes A and B. ensure all system pods are running on A. Fill up A so it does not have spare resources. add a low priority Pod L with long termination, with requests matching allocatable of B. It should be scheduled to B. add a high priority Pod H. It should get B as nominatedNodeName kill node B the H.nominatedNodeName will still point to B which is no longer in the cluster Anything else we need to know?: Current behavior does not play well with Cluster Autoscaler which assumes that Pod with nominatedNodeName will be scheduled in a while. Therefore such pods do not trigger scale-up. If we can have P.nominatedNodeName set incorrectly for unbounded period of time CA will never provision node for P to run. Environment: Kubernetes version (use ): We observed the behavior during presubmit tests which run k8s built from HEAD of master on 27.11.2019.

Versions logged in test log: I1125 18:55:11.691] Client Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.0.1183+", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-11-25T17:04:18Z", GoVersion:"go1.13.4", Compiler:"gc", Platform:"linux/amd64"} I1125 18:55:11.692] Server Version: version.Info{Major:"1", Minor:"18+", GitVersion:"v1.18.0-alpha.0.1183+", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-11-25T17:04:18Z", GoVersion:"go1.13.4", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: GCE /sig scheduling
/assign
/cc
Another interesting case would be creating a new pod with pod antiaffinity blocking the pod with nominated node name from scheduling: up a cluster with pods, including a long-terminating low-priority pod A. high-priority pod B that has the same requests as A. pod C with negligible resource requests and zonal topology pod antiaffinity matching pod B (and no other pods in the cluster). What I'd expect: Pod C keeps pending, nominatedNodeName is cleared. Pod A is restarted on the same node it was running before. I haven't actually tested this scenario, so I'm not sure what will actually happen. If report is correct, this may be a more convenient way of reproducing the issue (no need to add / remove nodes, it's not important which pod will go to which node).
I'm not sure if we need to clear nominatedNodeName on Pod when node is no longer valid immediately. And we have other logic to clear nominatedNodeName. Reuse your example. create cluster with nodes A and B. ensure all system pods are running on A. Fill up A so it does not have spare resources.

add a low priority Pod L with long termination, with requests matching allocatable of B. It should be scheduled to B. add a high priority Pod H. It should get B as nominatedNodeName kill node B the H.nominatedNodeName will still point to B which is no longer in the cluster After this, The H will be scheduled again. Obviously it's going to fail. H will enter the preemption process and the is empty. It will return itself as preemption process will clean nominatedNodeName on the Pod H.
when I have some time I will check if reproduction steps I describe actually work. But we surely observed pods with lingering nominatedNodeName which pointed to inexistent node, during our E2E tests. Quick thought. The method has many return points. If it returned at the place you linked it seems should be cleared. But there are many return points below, which set the list of pods to be cleared to nil. Is it possible that for a pod to consistently exit from with one of those return points?
I think you are right. Actually there are many return points to return nil. I think maybe we need to clear the of the pod after the filter failed again and before the new preemption process to avoid potential risks. WDYT?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or .

/lifecycle stale
/remove-lifecycle stale
I believe it's a bug as in old logic, e.g., v1.18.3, if the Pod (with nominatedNodeName set) cannot be scheduled and also preemption doesn't help ( nil potentialNodes is a corner case, it's more common the pod can't fit on all potentialNodes), it won't hit L413: then in old recordingFailure logic, nominatedNodeName remains as is. In the latest codebase, introduced a fix to combine setting nominatedNodeName as well update the pod's unschedulable condition. However, this doesn't help in this issue either as it ignores the case when the pod's nominatedNodeName is nil:
/reopen as regresses - we need to come up with a complete fix.
Reopened this issue. <detailsIn response to : /reopen as regresses - we need to come up with a complete fix. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle rotten
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.

If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle rotten
what are the implications of not clearing the nominated node name?
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: //      The data directory itself is a link to a timestamped directory with //      the real data: //        <target-dir>/..data          -> ..2016_02_01_15_04_05.12345678/ <del> //  8.  A symlink to the new timestamped directory ..data_tmp is created that will //      become the new data directory //  9.  The new data directory symlink is renamed to the data directory; rename is atomic </del> <ins> //      NOTE(claudiub): We need to create these symlinks AFTER we've finished creating and //      linking everything else. On Windows, if a target does not exist, the created symlink //      will not work properly if the target ends up being a directory. </ins> // 10.  Old paths are removed from the user-visible portion of the target directory // 11.  The previous timestamped directory is removed, if it exists func (w *AtomicWriter) Write(payload map[string]FileProjection) error {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Creating a pod with a nested subpath volume will cause the pod to fail to start on Windows containerd nodes. What you expected to happen: The pods should start normally. How to reproduce it (as minimally and precisely as possible): Create the following pod on a Windows node with containerd: apiVersion: v1 kind: Pod metadata: annotations: e2e-test-privileged-psp name: pod-subpath-test-downwardapi spec: containers: args: mounttest --filecontentinloop=/test-volume --retrytime=20 image: claudiubelu/agnhost:2.21 name: test-container-subpath-downwardapi volumeMounts: mountPath: /test-volume name: test-volume subPath: downward/podname volumes: downwardAPI: defaultMode: 420 items: fieldRef: apiVersion: v1 fieldPath: path: downward/podname name: test-volume Anything else we need to know?: This issue is causing the following tests to fail on Windows Containerd nodes: [sig-storage] Subpath Atomic writer volumes should support subpaths with downward pod [LinuxOnly] [Conformance] [sig-storage] Subpath Atomic writer volumes should support subpaths with projected pod [LinuxOnly] [Conformance] The tests have the ` tag because we did not support Windows Containerd nodes at the time, and the tests still do not work for Windows Docker nodes. After a fix for this bug is merged, these tests will have to be renamed to enable them for Windows Containerd environments but not for Windows Docker environments. /sig windows Environment: Kubernetes version (use ): v1.21.0-alpha.0.493+ Cloud provider or hardware configuration: N/A OS (e.g: ): Windows Server 1809, Containerd Kernel (e.g. ): 10.0. N/A Build Install tools: N/A Network plugin and version (if this is a network-related bug): N/A Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance.

The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale


Question: framework.ExpectNodeHasLabel(cs, nodeName, k, v) defer framework.RemoveLabelOffNode(cs, nodeName, k) <del> By("Trying to create a pod(pod1) with hostport 80 and hostIP 127.0.0.1 and expect scheduled") creatHostPortPodOnNode(f, "pod1", ns, "127.0.0.1", v1.ProtocolTCP, nodeSelector, true) </del> <ins> port := int32(54321) By(fmt.Sprintf("Trying to create a pod(pod1) with hostport %v and hostIP 127.0.0.1 and expect scheduled", port)) creatHostPortPodOnNode(f, "pod1", ns, "127.0.0.1", port, v1.ProtocolTCP, nodeSelector, true) </ins> <del> By("Trying to create another pod(pod2) with hostport 80 but hostIP 127.0.0.2 on the node which pod1 resides and expect scheduled") creatHostPortPodOnNode(f, "pod2", ns, "127.0.0.2", v1.ProtocolTCP, nodeSelector, true) </del> <ins> By(fmt.Sprintf("Trying to create another pod(pod2) with hostport %v but hostIP 127.0.0.2 on the node which pod1 resides and expect scheduled", port)) creatHostPortPodOnNode(f, "pod2", ns, "127.0.0.2", port, v1.ProtocolTCP, nodeSelector, true) </ins> <del> By("Trying to create a third pod(pod3) with hostport 80, hostIP 127.0.0.2 but use UDP protocol on the node which pod2 resides") creatHostPortPodOnNode(f, "pod3", ns, "127.0.0.2", v1.ProtocolUDP, nodeSelector, true) </del> <ins> By(fmt.Sprintf("Trying to create a third pod(pod3) with hostport %v, hostIP 127.0.0.2 but use UDP protocol on the node which pod2 resides", port)) creatHostPortPodOnNode(f, "pod3", ns, "127.0.0.2", port, v1.ProtocolUDP, nodeSelector, true) </ins> }) It("validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP", func() {

The test uses port 80, which is often in use on clusters (in our case, masters have an ingress controller listening on 80). The test should use something in the high range that is unlikely to conflict or use a smarter algorithm for finding a port to validate. fails in openshift
I am looking into it.
I looked into the test and it seems that another option would be for the test to not use 127.0.0.1 to avoid such conflict with infra (or any other important) pods on 127.0.0.1. As the test is using 127.0.0.1 and 127.0.0.2, I think we could just use 127.0.0.2 and 127.0.0.3 instead.
so it seems there are following options we can use: 127.0.0.2 and 127.0.0.3 instead of 127.0.0.1 and 127.0.0.2. pick some high random port number. something like following to pick unused port: and then extract random port from it.
with option 1 as it seems safer than 2 to me. Let me know if there is any concern, or if we would want to go with 2 or 3 option or something else.


Question: // https://github.com/kubernetes/kubernetes/issues/57744 func TestConstructVolumeSpec(t *testing.T) { <ins> if runtime.GOOS == "darwin" { t.Skipf("TestConstructVolumeSpec is not supported on GOOS=%s", runtime.GOOS) } </ins> tmpDir, err := utiltesting.MkTmpdir("rbd_test") if err != nil { t.Fatalf("error creating temp dir: %v", err)

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: unit tests fail on osx. What you expected to happen: How to reproduce it (as minimally and precisely as possible): To see this unit test fail, you need to run go test on an osx machine. Environment: OS: macOS High Sierra v10.13.3
/sig storage /assign


Question: t.Errorf("Exepcted RequiresRemount to be false, got %t", has) } } <ins> func TestGetRbdImageSize(t *testing.T) { for i, c := range []struct { Output     string TargetSize int }{ { Output:     `{"name":"kubernetes-dynamic-pvc-18e7a4d9-050d-11e9-b905-548998f3478f","size":10737418240,"objects":2560,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.9f4ff7238e1f29","format":2}`, TargetSize: 10240, }, { Output:     `{"name":"kubernetes-dynamic-pvc-070635bf-e33f-11e8-aab7-548998f3478f","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.670ac4238e1f29","format":2}`, TargetSize: 1024, }, } { size, err := getRbdImageSize([]byte(c.Output)) if err != nil { t.Errorf("Case %d: getRbdImageSize failed: %v", i, err) continue } if size != c.TargetSize { t.Errorf("Case %d: unexpected size, wanted %d, got %d", i, c.TargetSize, size) } } } </ins>

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: kubectl create -f ceph- and result is ok now i want to resize pv from 1G to 2G, edit kubectl describe pvc pvc-expand-test What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: ceph storageclass yaml: ceph secret: Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.5", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-11-26T14:41:50Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.5", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-11-26T14:31:35Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration:OS (e.g. from /etc/os-release): CentOS Linux release 7.4.1708 (Core) Kernel (e.g. ): Linux m01 3.10.0-693.el7.x8664 SMP Tue Aug 22 21:09:27 UTC 2017 x8664 x8664 x8664 GNU/Linux Install tools: kubeadm Others: ceph version 13.2.2 () mimic (stable) <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
What's the version of your ceph cluster? k8s uses the command to get rbd image's size, and it expects the size unit in the output is . However, in your environment the unit is :
ceph version 13.2.2 () mimic (stable)
It seems that the output format is different between different ceph versions.

I will try to fix it.
/assign
your PR has been merged to master branch, and when will Kubernetes v1.11 fix this bug?
Sorry for the late, I've send cherry-pick PRs:
It seems this change is not working in some cases: I think this is due to two concurrent conditions: 1- ceph client still complain on stderr about not being able to load config file even if there is no need as a key is supplied and the result is (hence the output on stderr of ). This happens not only in hyperkube. 2- it seems we capture both stdout and stderr from the execution of the command (please note the output in the error from ) and then there is then a problem trying to unmarshall the JSON (to be clear, the unuseful warning is on stderr, the json output on stdout). I am able to have a workaround creating empty files for and so that the ceph client does not output anymore but I feel it would be much better to parse only stdout. A reference on the source code for Thanks edit:typo
could you tell where did you put the empty and ceph.keyring files? On the node running kubelet?
Inside the pod of the as the rbd resize is executed by this component.
I had the same problem, I ceph. conf and ceph. Keyring and solved it, but I think k8s got the default user of CEPH before going back to ceph. conf and ceph. Keyring. The following code validates my conjecture: //num 672 Ultimately, I want k8s to get those two files through configurable CEPH parameters. What do you think?
as far as I know you do not need to configure those two files if you use a key. That is still an hypothesis as in the version of Ceph we are using this switch seems to be undocumented: Anyway the warning can be safely ignored and I do not see why we should parse when there is no error
I two files with empty content before, no error; later I a soft connection called and ceph. Keyring (pointing to the ceph configuration file and key ring generated by my cluster), but ceph did not report an error. . I think ceph only needs two files to exist and doesn't care about its contents.

Fortunately, the current measures can solve the problem.
this matches my experience (see above, I created the empty files too and it gives no error) but another possibility is just not parse which I find "cleaner" as there is no error in this case even from the error code, and even what we get is just a warning.
Same issue as here. The warnings about the missing etc files break the JSON output. Adding empty files in the container fixes resizing.
Shouldn't this be reopened until the bug is fixed? Creating the two files also "solves" the issue for us.
/reopen cc
Reopened this issue. <detailsIn response to : /reopen cc Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
which version of ceph cluster is in use in your setup ?
We are running a ceph cluster with . kube-controller-manager is running which internally uses . Also just to prevent any confusion: I've used to resize the PVC, not rancher.
This also seems related to .
We are running
Are there plans to resolve this issue?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available .

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Answering myself - apparently it has been fixed in k8s 1.20:


Question: desc := fmt.Sprintf("latency/resource should be within limit when create %d pods with %v interval", itArg.podsNr, itArg.interval) It(desc, func() { itArg.createMethod = "batch" runDensityBatchTest(f, itArg) }) } })

Kubelet and System-reserve are implemented on Windows along with pod/container reserves which can help prevent overprovisioning, but we still need to test OOM behavior. We need a test case that will: kubelet & system reserve pods until node is full something that will take up system ram so that OOM is triggered /sig windows /milestone v1.14
I have started to look into this test case and have a few questions: What is the expected behavior we are testing for? What does "run something that will take up system ram" mean? Does that mean run a pod that uses up memory or run a system service that uses memory? After setting the reserves and as mentioned in we should be able to schedule pods until full and then once we trigger OOM (either by pod or system service) kubelet should continue running becuase of the reserves? Do the two settings need to be tested independently or they should be tested together? I found that seems to be doing something similar but it is in node-e2e. Is sig-windows running node-e2e tests currently?
/assign
GitHub didn't allow me to assign the following users: jsturtevant. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
After researching this further and talking with I found that the does test the allocatable memory but uses CGroups to set reserves to test the reported vs actual. As windows does not have CGroups this test will not port as is. There is and the is not implemented so will not work either. The proposed testing solution would be two parts: kubelet & system reserve and verify allocatable is reporting properly. Schedule pods with known memory consumption values and verify the allocatable is correclty updated.Schedule until full and verify scheduling is not allowed. kubelet & system reserve and verify allocatable is reporting properly. Schedule Schedule until full and verify scheduling is not into machine and run process that consumes memory and verify kubelet and pods remain running. There is an open question on the second test as to whether kubelet and will be have properly.

If it does not we will submit test 1 and open an issue to resolve 2.
I would suggest adding a third case that would test the behavior when excessive memory pressure is exerted on the node from the pods. This is perhaps a more common use case than 2 since the pod workloads are more likely to exhibit greater resource variability than node system processes. It would also carry over as a test for when/if node allocatable enforcement get implemented for windows nodes. Additionally, to complement this work would it be suitable to test windows pod memory limit enforcement? Or is this tested elsewhere?
Agreed this is a more common case. I am not sure that it is. Is this tested on linux somewhere?
I am seeing strange numbers reported when performing a test case. I scheduled a few pods to test memory limits and then schedule a pod with no memory limit and a process that will consume all of the available memory 500mb at a time: After the pod consumes all available memory, When I compare Node Allocatable/Node usage/Pod Usage I am not seeing the numbers add up properly: My node is staying online and kubelet is reporting no memory pressure. The total node memory consumption is over the Allocatable memory. What is the expected behavior and what should the test look like?
I believe node allocatable will always just be capacity minus reserves (system and kubelet) - So in this scenario it looks like you have and/or set? The memory % on the node is relative to the allocatable amount, and still seems to be under the total memory capacity - are you sure that all available memory has been consumed by your oom pod? The node should be tainted with memory-pressure when a hard or soft eviction threshold is met (). What do you have set for and ?
Thanks for the clarification. I currently have these settings: And on the machine I am seeing most memory being used: If I understand correctly, I start to see memory-pressure once I get into the setting above. In that case I should start to see pods get evicted, so for a valid test I could adjust the values to be easier to hit then test to see that pods get evicted properly which would be similar the .

I am going to adjust a few of the settings and manually test that this is working then see what it will take to port the PriorityMemoryEviction, which currently uses cgroups which won't work on windows. This combined with testing that Memory Allocatable is being properly set should give us a good test coverage for windows on this scenario.
I performed manual testing and found that even when memory was used up on the machine the "Memory Pressure" notification was not fired. After looking deeper I found is not implemented for windows. The , which uses cgroups, provides the . This means even though is being it isn't used. On Windows, Node allocatable when used with limits and requests does prevent the node from over scheduling. I believe the the test and  (this only shows cpu and I could not find a test that demonstrated memory) demonstrate this works. But this doesn't stop a pod from consuming all the memory on the node and since eviction (hard/soft/node allocatable) is not implemented on windows this means a node can end up resource starved. I think the only valid test to be written for this scenario, as it stands now, is validating that Node Allocatable is being reported properly. Beyond that there is no functionality that can be validated around OOM behavior for the windows node.
Yeah, that sounds to be in agreement with what I've observed on the windows nodes that I run. To prevent resource starvation (as a workaround), would setting mem/cpu requests equal to mem/cpu limits for all windows pods work? I think the request / limit values get passed down into the docker daemon when it starts a container, so is it possible that docker could enforce the memory limit?
Yes, setting limits==requests would help the scheduler not over provision the nodes. I think the best way to approach this particular issue for 1.14 is to make sure docs call out what works/doesn't work and give guidance to on how to prevent resource starvation on nodes. Is there a place in the docs that would make sense for this information?

The call outs would be: , , and flags do update Node Allocatable is not implemented and are not implemented Condition is not implemented in Windows There are no OOM actions taken by kubelet As a work around to previous, and should always be equal to prevent memory resource starvation as this allows the scheduler to to use limits and Kubelet running on the windows node does not have memory restrictions. and system-reserve does not set limits on kubelet or processes running the host. This means kubelet or a process on the host could cause memory resource starvation outside the and scheduler
thank you for the thorough investigation


Question: nodes := makeNodeList([]string{"3", "2", "1"}) scheduler := makeScheduler(predicates, nodes) <del> _, predicateMap, err := scheduler.findNodesThatFit(nil, &v1.Pod{}, nodes) </del> <ins> _, predicateMap, err := scheduler.findNodesThatFit(nil, &v1.Pod{}, schedulertesting.FakeNodeLister(nodes)) </ins> if err != nil { t.Errorf("unexpected error: %v", err)

In : It seems that nodes are only used in the case there are no predicates. This call can be moved inside . Moving it there saves us from creating the node slice unless it is necessary. Checking if there are no nodes (line 202 below) can be done using . /kind cleanup /sig scheduling
I can work on this


Question: var _ Interface = new(Mock) <ins> func (c *Mock) Start() error { args := c.Called() return args.Error(1) } </ins> // ContainerInfo is a mock implementation of Interface.ContainerInfo. func (c *Mock) ContainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) { args := c.Called(name, req)

This is blocking (reboot test). In running a test that reboots all of the nodes and then runs the e2e tests, we saw a failure with the influxdb / heapster monitoring test: The heapster logs, from running (), shows a ton of the following kubelet errors for all containers (this one happens to be for skydns): The shows a bunch of 500 errors, such as: However, the apiserver thinks that all the pods are running ( gives this ) Docker thinks that the pods are running ( on that node gives this ) In summary: apiserver and docker think containers are running but kubelet can't find them this manifested in the influxdb / heapster monitoring e2e test after node reboot I've still got the cluster around if anyone would like to debug. +cc (helped me debug this (thanks!)) +cc (node) +cc (interested in getting in, which this is blocking)
Can I access your test nodes?
Is this an active problem? I'd check if the cAdvisor thinks the containers exist (it is the one returning the error). I'd then scan the logs to see if cAdvisor had any errors related to that container.
Unfortunately during debugging, I restart kubelet service on that node, the issue is gone. The problem could be cadvisor cannot get stats, but I couldn't find any cadvisor related error messages. How easy to reproduce the issue here? Next time we might access cadvisor's page to see if there is no such container information. Thanks!
(and Thanks for looking into this! I believe I can reproduce easily but I'll confirm. Where do I look for cAdvisor info? I don't see a log in or , and I don't see a cAdvisor container running?
cAdvisor is running as part of kubelet now. If you can reproduce the issue, could you please change kubelet's log level to 4 so that we have more cadvisor log here? Thanks!
Yes! How do I do that?

I can reproduce this by doing the following: After this, I got two failures: ... and I have another node with more of the same kubelet errors: (If I need to change the Kubelet log level at compile-time I can do that and repro on a new cluster; otherwise, I've got another cluster with the issue that can be introspected...)
There is a race between cadvisor initialization and docker. cadvisor failed to register docker factory, and bailed out. is going to make cadvisor more resilient to the issue of docker is not available, and have some retry logic. I am going to change kubelet initialization logic a little bit to make sure docker is running before we start cadvisor.
Awesome, thank you Dawn! Are there issues or PRs I can follow to know when this is unblocked?
is one of them. is working on the second.


Question: fi # shellcheck disable=SC2024 <del> sudo "${GO_OUT}/hyperkube" proxy  </del> <ins> sudo "${GO_OUT}/hyperkube" kube-proxy  </ins> --v="${LOG_LEVEL}"  --config=/tmp/kube-proxy.yaml  --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" 2>&1 &

Hyperkube supports a number of short aliases for commands, such as , , , , etc, rather than the longer , , , etc. There are comments indicating this is "unfortunate" and we still currently create these aliases in the hyperkube docker image: It'd be nice to simplify this, assuming we can find all uses of the short aliases and remove them. /kind cleanup Not sure which sig should own this, so tagging a few. /sig architecture cluster-lifecycle cli cc
interestingly, the command doesn't create the short aliases, though we also don't really use anywhere either.
I think we can just go ahead and start the deprecation cycle for this. WDYT? cc
cc as well
+1 to nuke it
/assign :wink: ok?
SGTM
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
/help
This request has been marked as needing help from a contributor. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /help Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I would like to work on it!
It seems like this issue was already solved in ?
I don't think the hyperkube code was changed in that PR, only the image?
Yes, thanks for your comment, can i try to fix it ?
I made a PR here: . Review/feedback/suggestions appreciated! :)
/assign /unassign
GitHub didn't allow me to assign the following users: Rand01ph.

Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign /unassign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-sig architecture cli /sig release Does anyone even still use hyperkube? Wouldn't they use kubeadm or k3s instead?
AFAIK, SAP (the company) still use hyperkube.
aks-engine deployer ( ) still uses hyperkube
Since this ticket is completed now. Can this todo be removed?
we are removing the whole thing :)


Question: ) set -x <del> /usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" -v="${E2E_VERBOSITY}" > >(tee "${RESULTS_DIR}"/e2e.log) ret=$? </del> <ins> /usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" -v="${E2E_VERBOSITY}" > >(tee "${RESULTS_DIR}"/e2e.log) && ret=0 || ret=$? </ins> set +x saveResults exit ${ret}

Same as an issue that occurred for v1.14.0: the failing test causes ginkgo to exit non-zero which causes the script to abort without running the routine. A recent PR to this script fixed a race but we didn't realize that now that the command was running in the foreground (and not the background) that the non-zero status would impact that script. Currently, Sonobuoy can get around this by just using the Golang based runner via . The Go-runner does not seem to have a similar vulnerability. This is going to be the default for Sonobuoy runs starting in Sonobuoy v0.16.0 if the PR gets merged. Honestly, I think that we may want to consider just removing the bash script all-together for v1.17.0 if the go runner works well for this entire cycle. See the original reporting of the issue and diagnosis here: /sig testing /area conformance /priority important-soon
/assign The PR in question that re-emerged this problem: FWIW Sonobuoy is adding an integration suite that will be able to test this going forward (soon). I think we need to fix the script (and get it cherrypicked into v1.16.x branches) I think that if the go runner proves stable over v1.16 then we default to it going forward We consider removing entirely if we think we can do that without impacting other workflows. Maybe that wont be possible due to other workflows though. cc
is probably never called due to , it can probably be fixed by adding a or before line 70.
Yes that is exactly why it is but in the past when I recommended removing/modifying the errexit code I've gotten pushback from a googler just due to convention: see Just will add something like to the lines that need this


Question: } type testClientConfig struct { <ins> rawconfig          *clientcmdapi.Config </ins> config             *restclient.Config namespace          string namespaceSpecified bool

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: In-cluster namespace isn't overridden by kubeconfig if the context is specified via kubectl flag. When using , the override works as expected. What you expected to happen: If I specify for a context in kubeconfig, then I expect to use as the default namespace, regardless of whatever's in . How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: supposedly fixed this issue, but not for the case where the context is specified via a command line flag (). This is because this line checks for . Environment: Kubernetes version (use ): v1.10.5 (details above) Cloud provider or hardware configuration: - OS (e.g. from /etc/os-release): - Kernel (e.g. ): - Install tools: - Others: - <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
fixed in


Question: // isCoreDNSConfigMapMigrationRequired checks if a migration of the CoreDNS ConfigMap is required. func isCoreDNSConfigMapMigrationRequired(corefile, currentInstalledCoreDNSVersion string) (bool, error) { var isMigrationRequired bool <del> if corefile == "" || migration.Default("", corefile) { </del> <ins> // Current installed version is expected to be empty for init if currentInstalledCoreDNSVersion == "" { </ins> return isMigrationRequired, nil } deprecated, err := migration.Deprecated(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: does not retaining coredns configmap using v1.19.4 makes no difference What you expected to happen: should follow documentation and retain coredns configmap How to reproduce it (as minimally and precisely as possible): upgrading from v1.18.6 to v1.19.3 Anything else we need to know?: works like a charm Environment: Kubernetes version (use ): v1.18.6 Cloud provider or hardware configuration: OS (e.g: ): CentOS 7 Kernel (e.g. ): 4.20.13- Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cluster-lifecycle
seems to be a regression ?
/assign
Would you check you dns configmap for ? As , federation is removed from coredns. If you have federation enabled in your coredns configmap, the check may fail in
We don't have any parameter in our coredns configmap, or any kube-dns parameters in kubeadm config
Looking closely at the dry run messages above, this may not be a regression. The apparent reason it works in 1.18.12 is that support for CoreDNS 1.7.0 was not until 1.19. i.e. 1.18 sees an unsupported version of CoreDNS and therefore skips the corefile migration. Whereas in 1.19, CoreDNS 1.7.0 is supported, and the migration takes place. What does your complete Corefile look like before the migration?
If it's possible, can you also attach the entire dry run logs?
Sure.
Thanks.

I think the issue may be in the detection of the "default" configuration. That is, in the upgrade process, if it detects that you are using the "default" coredns corefile, it simply installs the new default over top of it, rather than migrating the existing one. I think since the default config stanza is in there in its original form, the whole Corefile is incorrectly being detected as a default. If i'm right, the fix could either go in the package (should at least eventually be fixed there), and/or kubeadm could just always migrate the existing config instead of deploying the default over top of the existing one.
I just tested against the package manually - and it correctly identifies the config as "non-default", so that's not it... And I also tested a migration on the Corefile, and the result included the extra stanzas as expected. So, the issue doesn't appear to lie in the migration package.
After testing with the code, I find the issue in , I fixed it in . A workaround for you is to create an empty kube-dns configmap. ———————————————————— edited new comments here I find still fix a problem , when kube-dns configmap is using federations.
Thanks, I'll try it
empty kube-dns configmap doesn't helped.
sorry for my mistake. According to my test, the create or update will trigger is no migrating. Even your corefile will meet some dep that need to be migrated. max_concurrent is new default. However, there is a typo in . Below is a workaround(not good one), add upstream to your corefile in kubernetes. ! It works for me.
if your current coredns is 1.6.7, upstream is deprecated but not removed. In new 1.7.0 coredns, upstream is removed.
thanks for the discussion and PRs this one is mergable for 1.21 as non critical: what do you think about this one and should we backport it as critical (also, to which k8s versions)?
/area kubeadm
It's not obvious to me how this relates to . The kubeadm dry run log appears to be replacing the existing config with the default. It should only be doing this if the existing config is semantically the same as a default Corefile, i.e.

if is true.
just explains that the migration is not triggered at all. if is not a fix for this issue, please remove this line from the PR description.
I think issue here is in/around , and how the result is interpreted... The code correctly assumes that if a Corefile is a default Corefile, then no migration is required. However later it incorrectly assumes the reciprocal: If migration is not required, then the corefile must be a default. Hence, it ends up replacing any Corefile that does not require migration with a default… even if it wasn't a default to begin with.
yes. The need be fixed as well, I will learn to fix it later if no one is assigned. I changed releaseNote to None.


Question: } if svcInfo.NodePort != 0 { <del> lp := utilproxy.LocalPort{ Description: "nodePort for " + svcNameString, IP:          "", Port:        svcInfo.NodePort, Protocol:    protocol, </del> <ins> addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) if err != nil { glog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) continue </ins> } <del> if proxier.portsMap[lp] != nil { glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else { socket, err := proxier.portMapper.OpenLocalPort(&lp) if err != nil { glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue </del> <ins> lps := make([]utilproxy.LocalPort, 0) for address := range addresses { lp := utilproxy.LocalPort{ Description: "nodePort for " + svcNameString, IP:          address, Port:        svcInfo.NodePort, Protocol:    protocol, </ins> } <del> if lp.Protocol == "udp" { isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP) conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, clientv1.ProtocolUDP) </del> <ins> if utilproxy.IsZeroCIDR(address) { // Empty IP address means all lp.IP = "" lps = append(lps, lp) // If we encounter a zero CIDR, then there is no point in processing the rest of the addresses. break </ins> } <del> replacementPortsMap[lp] = socket } // We're holding the port, so it's OK to install ipvs rules. </del> <ins> lps = append(lps, lp) } // For ports on node IPs, open the actual port and hold it. for _, lp := range lps { if proxier.portsMap[lp] != nil { glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else { socket, err := proxier.portMapper.OpenLocalPort(&lp) if err != nil { glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } if lp.Protocol == "udp" { isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP) conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, clientv1.ProtocolUDP) } replacementPortsMap[lp] = socket } // We're holding the port, so it's OK to install ipvs rules. } </ins> // Nodeports need SNAT, unless they're local. // ipset call

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: kube-proxy was still binding to ::{port} for a node port service after it was started with the --nodeport-addresses argument What you expected to happen: kube-proxy to only bind to interfaces in the CIDR ranges specified by --nodeport-addresses. How to reproduce it (as minimally and precisely as possible): Start kube-proxy with --nodeport-addresses "127.0.0.1,::1" Create a service of type NodePort Observe kube-proxy bind to all listening interfaces Anything else we need to know?: In the proposal for the --nodeport-addresses feature, it specifies being able to have other services answering or listening on a given port. Since kube-proxy binds to all interfaces, this is still not possible so it appears one of the initial intents of the proposal was net met with the existing solution. [] Doesn't set an interface. Given that nodeport-addresses is specified as CIDR ranges extra work must be done to determine which IP's on a host exist that could be bound to. Alternatively, a flag such as --nodeport-addresses-bind could allow an operator to set an explicit value. To test functionality I was able to hardcode an address here and observe the desired behavior could be achieved. Environment: Kubernetes version:Cloud provider or hardware configuration: AWS OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Custom fabricated Others: As expected, this behaves the same way regardless of iptables or ipvs being used as the forwarding mechanism.
/sig network
/assign
Nice catch! I will fix it ASAP~
Please check


Question: "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" <del> "k8s.io/kubernetes/pkg/api/legacyscheme" </del> <ins> fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/restmapper" coretesting "k8s.io/client-go/testing" </ins> api "k8s.io/kubernetes/pkg/apis/core" kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" <del> // Manually initialize legacy scheme to make test rest mapper work with built-in resources. // See issue #70192 for more details _ "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" </del> ) type fakeAuthorizer struct{}

/kind flake /sig testing /sig auth we are injecting a restMapper into this admisson controller while in its test, the restMapper somehow become empty when we remove internal clientset/informer dependency from it (). now we come up with a walkaround in which we manually init the legacy schema via a dry import so that the built-in resources can be installed. this failing test suggests that the test is not quite correct logically.
/assign Are you up for trying to figure out how to avoid requiring the internal types?
What test does this break? Links?
link here: w/o the dry import, then will fail with output:
sure. will be looking into it
OK, setting priority so that I know this isn't a 1.13 blocker. /priority important-longterm


Question: // parsing successful, so we now know the proper value for .Parts requestInfo.Parts = currentParts <ins> // Raw should have everything not in Parts requestInfo.Raw = requestInfo.Raw[:len(requestInfo.Raw)-len(currentParts)] </ins> // if there's another part remaining after the kind, then that's the resource name if len(requestInfo.Parts) >= 2 {

The apiserver proxy rewrites links to the form "/proxy/ns/default/...": But this form stopped working recently, seemingly around The "/proxy/ns/default/..." form always returns "404: Page Not Found", similar to the problem seen in If you remove "/ns/default" from the URL, it works.
/cc
I can investigate this as well.
I see the issue, sending a PR.
The proxy path needs to be built with knowledge of the apiversion of the incoming proxy request, and it needs to be updated to use the knowledge from the restmapper. This is a little trickier than I have time for right now, but it should definitely not be hardcode appending the /ns form. Not clear why this is not also caught by our test case. I can try and look at this more in the AM unless someone else wants to take a look in interim. /cc
Let me steal this one from you, I think I can fix it.
Have at it. /cc me on your PR
Sorry, I see you've already got a PR out!
No problem. I would be happy if you finished my PR relative to the commit I started. It will be a few days before I can go back to looking at this issue. I think v1beta1/v1beta2 proxy urls in go-restful need to always but the namespace scope in the path as the ultimate fix. I do want the proxy to have access to the restmapper per my PR. ----- Original Message ----- From: "Daniel Smith" To: "GoogleCloudPlatform/kubernetes" Cc: "Derek Carr" Sent: Friday, February 13, 2015 4:56:19 PM Subject: Re: [kubernetes] apiserver proxy stopped accepting namespaces () Sorry, I see you've already got a PR out! Reply to this email directly or view it on GitHub:
Any updates ?
This is breaking monitoring since it requires the proxy. Bumping up the priority to P0.
Sorry for dropping this. Working on a fix.


Question: github.com/golangplus/fmt => github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 github.com/golangplus/testing => github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e github.com/google/btree => github.com/google/btree v1.0.0 <del> github.com/google/cadvisor => github.com/google/cadvisor v0.38.6 </del> <ins> github.com/google/cadvisor => github.com/google/cadvisor v0.38.7 </ins> github.com/google/go-cmp => github.com/google/go-cmp v0.5.2 github.com/google/gofuzz => github.com/google/gofuzz v1.1.0 github.com/google/martian => github.com/google/martian v2.1.0+incompatible

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Kubelet v1.19.0 doesn't start on nodes with CRI-O 1.17.2 (storagedriver="btrfs") with BTRFS as root filesystem. Related/previous issues: This error shows up in the journal: The is referring to the btrfs root filesystem, mounted at the time. : : What you expected to happen: Kubelet starts and detects the filesystem backing , even if it is BTRFS. How to reproduce it (as minimally and precisely as possible): Install kubeadm/kubectl 1.19.0 Install CRI-O, I used 1.17.2 from: Configure CRI-O to use Perform "kubeadm init" creating a new cluster Observe journal (check for kubelet issues) Anything else we need to know?: A possible workaround is to make sure a bind mount exists which allows kubelet's logic to find the backing fileystem. Eg. add the following fstab entry and then perform : The resulting mount should look somewhat like this: The workaround has been tested and seems to make kubelet work as expected. Environment: Kubernetes version: 1.19.0 OS: Debian Bullseye (Testing) Kernel: Linux 5.8.5 Others: The package has been installed
/sig node
cc xref
I am encounter the same issue. thank you for your workaround. It seems to work.
I got the same issue with latest stable Docker on Debian (btrfs) and k8s v1.19.2.
I confirm the same issue shows up with on btrfs (on LVM on LUKS, Debian buster), running k3s. No btrfs subvols. The workaround works for me; for k3s I also need to bind mount . I did not experience this issue with 1.18.
This also reproduces with Docker runtime. Workaround with bind mount helps, however automatically unmounts it so don't get confused. is on root that is mounted like this: , and docker is configured to use .

Reproduced with k8s 1.19.3 on Gentoo. The rootfs is btrfs, is on rootfs, docker is configured with storage drigver . automatically mount and unmount it on . Also confirmed bind mount workaround helps.
/assign
I think almost anyone using BTRFS as the root filesystem has encountered this issue. If is merged and applied to the kubernetes, this can be fixed. Please take a look at this ?
The has been merged to google/cadvisor. I think if Kubernetes v1.21 includes the fix by using google/cadvisor v0.39, the issue will be resolved.
Just a note that I too have stumbled on this issue after reconfiguring CRI-O storage to use btrfs driver pointing with graphroot directly to a BTRFS subvolume (the root filesystem is not BTRFS). After reading this issue I tried mounting the BTRFS subvolume explicitly on some mountpoint and pointed with graphroot to this explicit subvolume mountpoint which solved the problem. The problem seems to occur only when pointing to a subvolume within some parent subvolume as this is not a real mountpoint.
Ok, so I just want to make sure I understand. Currently when i try to run kubeadm to create a new cluster while using btrfs, it gives me With Kubernetes v1.21, that will no longer be the case?
In my experience (with CRI-O 1.19 and k8s 1.19), the btrfs driver (in ) is supported, but the combination does not work correctly when graphroot is pointed to a btrfs subvolume within some parent subvolume (I don't have experience with graphroot poited to some subdirectory of a default subvolume and/or when default subvolume is also a root filesystem, just when pointed directly to a subvolume within some parent subvolume which is not a root filesystem). For example, when btrfs filesystem is mounted like the following (): when I then create a subvolume: then use it as a graphroot (): But when I explicitly mount the subvolume to some mountpoint (): then use the explicitly mounted path as the graphroot (): it works correctly without any problems.

I guess some logic (either in CRI-O/containers or in kubelet) tries to find the "mountpoint" of the filesystem used for graphroot or something related and can't do that correctly when graphroot is located in a btrfs subvolume that is not explicitly mounted somewhere.
Ah, I see what you meant by saying that is not supported in . I haven't checked that since I created k8s clusted with default (overlay) driver and later migrated to . It is possible that disallows using because of problems like the one described above. But in my experience, when a subvolume is mounted explicitly, the btrfs works as expected - at least I didn't have problems with it.
Can the fix be backported to 1.19? only supports updating one version at a time: () So this leaves people on btrfs with with no upgrade path to which is not really nice.
We've discussed this issue at Regarding Maybe, there are quite a lot of people using btrfs than I expected. What do you think about this? Thanks!
I am running up against this issue as well. I am stuck with using k8s on 1.18
It sounds like for full fix we'll need to cherrypick to be included in k8s 1.19 and 1.20. To do that the following is needed: to cAdvisor v0.37 and v0.38 branches k8s 1.19 uses cAdvisor v0.37.3 - k8s 1.20 uses cAdvisor v0.38.6 - new cAdvisor patch release for v0.37 and v0.38 against k8s on 1.19 and 1.20 bumping to newly released cAdvisor (v0.37 / v0.38) with the fix I'm happy to cut the new cAdvisor release from my side if you want to proceed and think this is critical enough bug that is worth cherrypicking.
New cAdvisor patch versions have been cut with the BTRFS fix: v0.37.4 - v0.38.7 - /cc


Question: // If the network plugin is not ready, only start the pod if it uses the host network if rs := kl.runtimeState.networkErrors(); len(rs) != 0 && !kubecontainer.IsHostNetworkPod(pod) { <del> kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "network is not ready: %v", rs) return fmt.Errorf("network is not ready: %v", rs) </del> <ins> kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, rs) return fmt.Errorf("%s: %v", NetworkNotReadyErrorMsg, rs) </ins> } // Create Cgroups for the pod and apply resource parameters

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Pods are synchronized by kubelet before network is ready. Synchronization in such case fails and pod is synchronized after 10 seconds. What you expected to happen: Kubelet should wait until network is ready before starting synchronizing pods. How to reproduce it (as minimally and precisely as possible): Configure cluster with Cluster Autoscaler enabled. Start pod so cluster is scaled up. In logs you might find following error: E0907 13:18:49. 972 ] Error syncing pod xxxxxx ("xxxxxxxxx(xxxxxxxxxxx)"), skipping: network is not ready: [runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: Kubenet does not have netConfig. This is most likely due to lack of PodCIDR] Environment: Build from HEAD Cloud provider or hardware configuration: GKE
/kind bug /sig node