Question: <ins> // +build linux /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "testing" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" ) func TestNodeAllocatableReservationForScheduling(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") cpuMemCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList expected       v1.ResourceList capacity       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("150m", "150Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("150m", "250Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getResourceList("150m", "694157320"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("", ""), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("", "150Mi"), }, } for idx, tc := range cpuMemCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved  v1.ResourceList expected      v1.ResourceList capacity      v1.ResourceList hardThreshold evictionapi.ThresholdValue }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList("100Mi"), }, { kubeReserved: getEphemeralStorageResourceList("100Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity: getEphemeralStorageResourceList("10Gi"), expected: getEphemeralStorageResourceList("200Mi"), }, { kubeReserved: getEphemeralStorageResourceList("150Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getEphemeralStorageResourceList("694157320"), }, { kubeReserved: v1.ResourceList{}, capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList(""), }, } for idx, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableForEnforcement(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") testCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList capacity       v1.ResourceList expected       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("10", ""), }, } for idx, tc := range testCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.getNodeAllocatableAbsolute() { expected, exists := tc.expected[k] assert.True(t, exists) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableInputValidation(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") highMemoryEvictionThreshold := resource.MustParse("2Gi") cpuMemTestCases := []struct { kubeReserved         v1.ResourceList systemReserved       v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), }, { kubeReserved:   getResourceList("5", "10Gi"), systemReserved: getResourceList("5", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &highMemoryEvictionThreshold, }, capacity:             getResourceList("10", "11Gi"), invalidConfiguration: true, }, } for _, tc := range cpuMemTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved         v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("500Mi"), }, { kubeReserved: getEphemeralStorageResourceList("20Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity:             getEphemeralStorageResourceList("20Gi"), invalidConfiguration: true, }, } for _, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } } // getEphemeralStorageResourceList returns a ResourceList with the // specified ephemeral storage resource values func getEphemeralStorageResourceList(storage string) v1.ResourceList { res := v1.ResourceList{} if storage != "" { res[v1.ResourceEphemeralStorage] = resource.MustParse(storage) } return res } </ins>

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: After running some tests, we're seeing that the combined usage of kubelet + docker can be around 2Gi. We need to be able to account for this in scheduling to avoid crashing these services. The first step towards this is using kube-reserved and system-reserved to avoid oversubscribing the virtual memory space available on the Windows nodes with too many containers. What you expected to happen: It should reduce node allocatable memory by kube-reserved + system-reserved. How to reproduce it (as minimally and precisely as possible): This logs an error in the kubelet log such as: code: Anything else we need to know?: /milestone v1.13 /assign feiskyer /sig windows This blocks a PR where I want to set it by default to 2Gi Environment: Kubernetes version (use ): v1.11.3 (also v1.12) Cloud provider or hardware configuration: Azure OS (e.g. from /etc/os-release): Windows Server version 1803 Install tools: acs-engine
You must be a member of the github team to set the milestone. <detailsIn response to : Is this a BUG REPORT or FEATURE REQUEST?: What happened: What you expected to happen: This logs an error in the kubelet log such as: code: /milestone v1.13 /assign feiskyer Kubernetes version (use ): v1.11.3 (also v1.12) Cloud provider or hardware configuration: Azure OS (e.g. from /etc/os-release): Windows Server version 1803 Install tools: acs-engine Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/milestone v1.13
/priority important-soon


Question: if err := o.Complete(tf, &cobra.Command{}); err != nil { t.Errorf("Unexpected error: %v", err) } <del> if err := o.Validate(); err != nil { </del> <ins> if err := o.Validate(nil); err != nil { </ins> t.Errorf("Unexpected error: %v", err) } <ins> if err := o.Validate([]string{"extraParameter0"}); !strings.Contains(err.Error(), "extra arguments") { t.Errorf("Unexpected error: should fail to validate the args length greater than 0") } </ins> if err := o.Run(); err != nil { t.Errorf("Cannot execute version command: %v", err) }

ran instead of and got no error and then though that was the right version kubectl should fail for unexpected extra arguments
/sig cli
/assign
/triage accepted


Question: ### Create replication controller frontend from JSON # Pre-condition: no replication controller is running <del> kube::test::get_object_assert rc "{{range.items}}{{.id}}:{{end}}" '' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" '' </ins> # Command kubectl create -f examples/guestbook/frontend-controller.json "${kube_flags[@]}" # Post-condition: frontend replication controller is running <del> kube::test::get_object_assert rc "{{range.items}}{{.id}}:{{end}}" 'frontend-controller:' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" 'frontend-controller:' </ins> ### Resize replication controller frontend with current-replicas and replicas # Pre-condition: 3 replicas <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '3' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '3' </ins> # Command kubectl resize --current-replicas=3 --replicas=2 replicationcontrollers frontend-controller "${kube_flags[@]}" # Post-condition: 2 replicas <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '2' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '2' </ins> ### Resize replication controller frontend with (wrong) current-replicas and replicas # Pre-condition: 2 replicas <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '2' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '2' </ins> # Command ! kubectl resize --current-replicas=3 --replicas=2 replicationcontrollers frontend-controller "${kube_flags[@]}" # Post-condition: nothing changed <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '2' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '2' </ins> ### Resize replication controller frontend with replicas only # Pre-condition: 2 replicas <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '2' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '2' </ins> # Command kubectl resize  --replicas=3 replicationcontrollers frontend-controller "${kube_flags[@]}" # Post-condition: 3 replicas <del> kube::test::get_object_assert 'rc frontend-controller' '{{.desiredState.replicas}}' '3' </del> <ins> kube::test::get_object_assert 'rc frontend-controller' "{{.$rc_replicas_field}}" '3' </ins> ### Delete replication controller with id # Pre-condition: frontend replication controller is running <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' 'frontend-controller:' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" 'frontend-controller:' </ins> # Command kubectl delete rc frontend-controller "${kube_flags[@]}" # Post-condition: no replication controller is running <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' '' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" '' </ins> ### Create two replication controllers # Pre-condition: no replication controller is running <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' '' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" '' </ins> # Command kubectl create -f examples/guestbook/frontend-controller.json "${kube_flags[@]}" kubectl create -f examples/guestbook/redis-slave-controller.json "${kube_flags[@]}" # Post-condition: frontend and redis-slave <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' 'frontend-controller:redis-slave-controller:' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" 'frontend-controller:redis-slave-controller:' </ins> ### Delete multiple controllers at once # Pre-condition: frontend and redis-slave <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' 'frontend-controller:redis-slave-controller:' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" 'frontend-controller:redis-slave-controller:' </ins> # Command kubectl delete rc frontend-controller redis-slave-controller "${kube_flags[@]}" # delete multiple controllers at once # Post-condition: no replication controller is running <del> kube::test::get_object_assert rc '{{range.items}}{{.id}}:{{end}}' '' </del> <ins> kube::test::get_object_assert rc "{{range.items}}{{.$id_field}}:{{end}}" '' </ins> #########

This works: Are we using /var/run for testing?
Curiously travis shows a similar error message regarding /var/run/kubernetes, but it isn't failing: I suspect the /var/run/kubernetes issue is not what's causing the failure here.
However, I am able to reproduce the failure on my machine. I note that in your case, the fatal error says For me, it says The ip addresses showing through here (which appear to be in the GCE space?) are worrying.
It seems it still failing. I've run it locally and it failed with the following error: The aforementioned line: I think it is expected that there are some pods started by k8s (e.g. elasticsearch). Does it mean the test has a bug?
Strangely the same code () passes tests on travis (https://travis-)
I think you're reading it backwards. The test is expecting no pods to be started, but finding the k8s add-on pods, which is normal if it were running against a normal master. In Travis, it's a synthetic environment with no add-ons. This should be assigned to whoever the checks (on mobile.. I know I merged the PR but don't remember who :)). We need to exclude the add-ons (perhaps by finally getting around to putting them in a separate namespace).
the assertions, so CC'ing for comment.
I looked into this with and it seems that test- may fail locally depending on the kubeconfig. Some commands are using defaults for kubectl and thus sometimes they contact another cluster. I'll send a PR to fix this.


Question: storageutils.VerifyExecInPodSucceed(f, pod2, "[ ! -f /mnt/test-0/hello-world ]") } <del> defer StopPod(f.ClientSet, pod2) </del> <ins> defer StopPodAndDependents(f.ClientSet, pod2) </ins> return nil }

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: Which test(s) are failing: Since when has it been failing: Testgrid link: Reason for failure: I0722 06:26:18.056] I0722 06:26:18.057] [Driver: csi-hostpath] I0722 06:26:18.057] I0722 06:26:18.057] [Testpattern: Generic Ephemeral-volume (default fs) [Feature:GenericEphemeralVolume] (late-binding)] ephemeral skipped 448 lines unfoldmore I0722 06:30:33.845] Jul 22 06:30:33.845: INFO: Wait up to 5m0s for pod "inline-volume-tester2-hhcbd" to be fully deleted I0722 06:31:06.117] Jul 22 06:31:06.117: INFO: Pod inline-volume-tester-74jqh has the following logs: /dev/sda1 on /mnt/test-0 type ext4 (rw,nosuid,nodev,noexec,relatime,commit=30) I0722 06:31:06.117] I0722 06:31:06.117] Jul 22 06:31:06.117: INFO: Deleting pod "inline-volume-tester-74jqh" in namespace "ephemeral-9141" I0722 06:31:06.155] Jul 22 06:31:06.155: INFO: Wait up to 5m0s for pod "inline-volume-tester-74jqh" to be fully deleted I0722 06:31:38.264] Jul 22 06:31:38.264: FAIL: no dangling PVCs I0722 06:31:38.264] Expected I0722 06:31:38.265] <[]v1.PersistentVolumeClaim | len:1, cap:1: [ I0722 06:31:38.265] { I0722 06:31:38.265] TypeMeta: {Kind: "", APIVersion: ""}, I0722 06:31:38.

265] ObjectMeta: { skipped 315 lines unfold_more I0722 06:32:16.896] STEP: Destroying namespace "ephemeral-9141" for this suite. I0722 06:32:16.931] Jul 22 06:32:16.931: INFO: Namespace ephemeral-9141 was already deleted I0722 06:32:16.932] STEP: Destroying namespace "ephemeral-522" for this suite. I0722 06:32:16.966] Jul 22 06:32:16.966: INFO: Namespace ephemeral-522 was already del Anything else we need to know: /cc /sig storage /priority critical-urgent /milestone v1.19
I have tracked this down and it looks like that changes were made in that affected how Pods were being deleted in the e2e framework (specifically commit ). This caused failures in other tests, so the commit was reverted in I will have a fix open shortly that uses foreground deletion only for these storage tests. /assign /cc Side note: I see already saw this


Question: package v1alpha1 import ( <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" )

no explanation for metadata filed of NodeMetrics/PodMetrics, code: should be like $ kubectl explain NodeMetrics KIND: NodeMetrics VERSION: DESCRIPTION: NodeMetrics sets resource usage metrics of a node. FIELDS: apiVersion <stringAPIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: kind <stringKind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: metadata <Objecttimestamp <string-required- The following fields define time interval from which metrics were collected from the interval [Timestamp-Window, Timestamp]. usage <map[string]string-required- The memory usage is the memory working set. window <string-required- and $ kubectl explain PodMetrics KIND: PodMetrics VERSION: DESCRIPTION: PodMetrics sets resource usage metrics of a pod. FIELDS: apiVersion <stringAPIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: containers <[]Object-required- Metrics for all containers are collected within the same time window. kind <stringKind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: metadata <Objecttimestamp <string-required- The following fields define time interval from which metrics were collected from the interval [Timestamp-Window, Timestamp]. window <string-required- able to see the explanation for metadata filed of NodeMetrics/PodMetrics Kubernetes Version: v1.22.0-rc.0+
/sig instrumentation
/kind documentation /assign /triage accepted


Question: * [kubectl config](kubectl_config.md)	 - config modifies kubeconfig files <del> ###### Auto generated by spf13/cobra on 8-Dec-2015 </del> <ins> ###### Auto generated by spf13/cobra on 8-Jan-2016 </ins> <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/kubectl/kubectl_config_view.md?pixel)]()

It acts like a boolean flag but requires an argument, unlike boolean flag is pretty much every environment. We need to find a better way to express the things that this is expressing. Maybe a add a tristate to the pflag lib that behaves the same way but doesn't demand an argument?
cc
The handles "true", "false", "unset" and has a default value. I seem to recall you were looking at changing it, but we needed to preserve the value. The complain here is that you have to do instead of just . /cc
fixing is really easy....


Question: <ins> /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kubeadm // Role labels are applied to Nodes to mark their purpose.  In particular, we // usually want to distinguish the master, so that we can isolate privileged // pods and operations. // // Originally we relied on not registering the master, on the fact that the // master was Unschedulable, and on static manifests for master components. // But we now do register masters in many environments, are generally moving // away from static manifests (for better manageability), and working towards // deprecating the unschedulable field (replacing it with taints & tolerations // instead). // // Even with tainting, a label remains the easiest way of making a positive // selection, so that pods can schedule only to master nodes for example, and // thus installations will likely define a label for their master nodes. // // So that we can recognize master nodes in consequent places though (such as // kubectl get nodes), we encourage installations to use the well-known labels. // We define NodeLabelRole, which is the preferred form, but we will also recognize // other forms that are known to be in widespread use (NodeLabelKubeadmAlphaRole). const ( // NodeLabelKubeadmAlphaRole is a label that kubeadm applies to a Node as a hint that it has a particular purpose. // Use of NodeLabelRole is preferred. NodeLabelKubeadmAlphaRole = "kubeadm.alpha.kubernetes.io/role" ) </ins>

used to show masters, but as of it no longer does. This is not in accordance with our deprecation policy I believe. cc
That commit/PR is not relevant as there are no supported users/deployment methods that are using label. kubeadm v1.4 or v1.5 was the last. What you're really looking for is who/what removed printing things based on the label...
No blame on you The regression is that we have stopped printing masters in . There may be more to it that a simple revert of , but we should probably start there.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
This has been fixed in recent versions


Question: // (see `applyPlatformSpecificDockerConfig` and `performPlatformSpecificContainerCleanup` // methods for more info). containerCleanupInfos map[string]*containerCleanupInfo <ins> cleanupInfosLock      sync.RWMutex </ins> } // TODO: handle context.

What happened: Windows kubelet panic occasionally on nodes which have high volume of pods run and delete. Full logs: Preliminary investigate result: How to reproduce it (as minimally and precisely as possible): Run high volume of pods and delete them when they completed in Windows nodes. Environment: Kubernetes version (use ): 1.17.7 Cloud provider or hardware configuration: Azure AKS OS (e.g: ): Windows Server 2019 Kernel (e.g. ): 10.0..1282
/assign I would like to give this a shot.
/remove-sig windows I think this is not a windows specific problem. /sig node /area kubelet
Hi! Thanks for your report, I have filed to address this issue.
This issue is a bit urgent for our situation, it occurs every few days in production clusters. Seems the PR is pending a few days, is it possible to increase the priority to merge it trunk and 1.17/1.18 branch? Thanks
I understand how urgent this issue is, but I am not privileged to do anything further :(
Do you know anyone who can help?
looks like a simple fix, add a lock for , I could help.
There is already a fix by , the problem is the PR is pending, not sure if you or someone can increase the priority to merge it trunk and 1.17/1.18 branch asap. Our AKS clusters are affecting by this frequently.
Thanks guys.


Question: // ServiceAnnotationLoadBalancerMixedProtocols is the annotation used on the service // to create both TCP and UDP protocols when creating load balancer rules. ServiceAnnotationLoadBalancerMixedProtocols = "service.beta.kubernetes.io/azure-load-balancer-mixed-protocols" <ins> // serviceTagKey is the service key applied for public IP tags. serviceTagKey = "service" // clusterNameKey is the cluster name key applied for public IP tags. clusterNameKey = "kubernetes-cluster-name" </ins> ) var (

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We deployed two Kubernetes clusters to the same Azure Resource Group. Created services of load balancer type on both clusters. One of the services never received a public IP. Upon investigation, we noticed that the first Kubernetes cluster deleted the Azure Public IP that was created for the service by the other cluster. Apparently, all public IPs in the resource group are retrieved: then public IPs which don't match this criteria: are deleted. This logic besides the name of the kubernetes service should also consider the K8s cluster where the public IP originated from to avoid stepping on each other's toes when multiple clusters share the same resource group. What you expected to happen: In case multiple clusters run in the same resource group a cluster should delete only resources created by itself. How to reproduce it (as minimally and precisely as possible): Create two Kubernetes clusters (1.14) with different cluster names to the same resource group than create kubernetes services of load balancer type in each cluster. Anything else we need to know?: Environment: Kubernetes version (use ): 1.4 Cloud provider or hardware configuration: Azure OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
thanks for reporting the issue, this is the bug we should fix for sharing resource group. /sig azure /kind bug /assign


Question: TTY:       tty, }, api.Scheme) <del> return p.Attach.Attach("POST", req.URL(), p.Config, stdin, p.Out, p.Err, tty) </del> <ins> err = p.Attach.Attach("POST", req.URL(), p.Config, stdin, p.Out, p.Err, tty) if err != nil { return err } if p.Stdin && tty && pod.Spec.RestartPolicy == api.RestartPolicyAlways { fmt.Fprintf(p.Out, "Session ended, resume using 'kubectl attach %s -c %s -i -t' command when the pod is runningn", pod.Name, containerToAttach.Name) } return nil </ins> } // GetContainer returns the container to attach to, with a fallback.

After a times out, you can usually reattach with something like . But not all users realize that. It would be nice, when a or session ends, to print


Question: } if len(srcSpec.PodName) != 0 && len(destSpec.PodName) != 0 { <del> if _, err := os.Stat(args[0]); err == nil { return o.copyToPod(fileSpec{File: args[0]}, destSpec, &exec.ExecOptions{}) } return fmt.Errorf("src doesn't exist in local filesystem") </del> <ins> return fmt.Errorf("one of src or dest must be a local file specification") </ins> } if len(srcSpec.PodName) != 0 {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Running silently fails if that source file doesn't exist. What you expected to happen: I expected kubectl to exit with a non-zero status code and to print a standard error message like so: How to reproduce it (as minimally and precisely as possible): Find any running pod, try to copy a non-existent file: Anything else we need to know?: works right. For example: I'm sure this has something to do with the fact that uses under the hood. However, itself will report an error if you try to extract a file that doesn't exist, so maybe this is just a matter of not including the to-be-extracted-path(s) on the end of the tar command line. This might be fixed by appending (or rather the basename of it) to the end of the array here: (haven't tried it). Environment: Kubernetes version (use ): and Cloud provider or hardware configuration: docker-for-desktop 2.0.4.1 () and GKE running 1.13.6-gke-0 OS (e.g: ): macOS 10.14.5 Kernel (e.g. ): Install tools: Installed docker from dmg, using the included Network plugin and version (if this is a network-related bug): Others:
/assign /sig cli
any update on this?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Same issue when the file doesn't exist on the pod
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Beep
Rotten issues close after 30d of inactivity.

Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
You can't reopen an issue/PR unless you authored it or you are a collaborator. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen </details
This is the first example where I found the kubectl cli to be sub-par (otherwise the experience has been excellent). Is anyone looking at this?
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen /remove-lifecycle rotten
Reopened this issue. <detailsIn response to : /reopen /remove-lifecycle rotten Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
For me never copies anything and never writes any info to console. Even if I point it to non-existing pod, it just exists with no trace.
I've opened with a fix.
: what version of kubectl are you using? I tried to upload a file to a non-existent pod and it failed with an error message (using the master branch) :
Issues go stale after 90d of inactivity. Mark the issue as fresh with .

Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
Any maintainer here? I had opened with a fix.
Would be great if would return a simple error in the CLI. Our pipeline was failing for a month and none realised because all the kubectl results were green :heavycheckmark:


Question: ${IP_ALIAS_SUBNETWORK} fi fi <del> if [[ ${SERVICE_CLUSTER_IP_SUBNETWORK} == ${INSTANCE_PREFIX}-subnet-services ]]; then echo "Removing auto-created subnet ${NETWORK}:${SERVICE_CLUSTER_IP_SUBNETWORK}" if [[ -n $(gcloud beta compute networks subnets describe  --project "${PROJECT}"  --region ${REGION}  ${SERVICE_CLUSTER_IP_SUBNETWORK} 2>/dev/null) ]]; then gcloud --quiet beta compute networks subnets delete  --project "${PROJECT}"  --region ${REGION}  ${SERVICE_CLUSTER_IP_SUBNETWORK} fi fi </del> } # Generates SSL certificates for etcd cluster. Uses cfssl program.

When using IP aliases on GCE, use a secondary range rather than subnetwork to reserve cluster IPs.
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals


Question: ctx, cancel := context.WithTimeout(req.Context(), timeout) defer cancel() <del> ctx = request.WithNamespace(ctx, namespace) </del> outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope) if err != nil { scope.err(err, w, req)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: It appears that validating admission requests for server-side apply are populating the for requests to create Namespace objects. Non server-side apply requests don't do this, and a missing namespace field is the only way to distinguish cluster-scoped resources from namespace-scoped resources. This appears to be a breaking behavioral change. See What you expected to happen: Requests to create namespaces would have a consistent signature between server-side apply and non-server-side apply contexts. How to reproduce it (as minimally and precisely as possible): See Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: kind OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig sig-api-machinery
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig sig-api-machinery Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
/wg api-expression
/assign
/assign
Requests made to namespaced URLs (URLs which contain a segment) populate the attribute in AdmissionReview requests. The documentation of the AdmissionReview field is as follows: From Update/Patch/Delete requests to specific namespaces have consistently populate this attribute for modifications to Namespace objects. Server-side apply is a type of patch request, so even when it is creating an object, that attribute will be populated. The difference is caused by a difference in the URL used by POST and apply-based creates. Creation via POST requests are made against the endpoint, so no attribute was populated. Namespace computed from request path: Namespace admission attributes for POST-based creates: Creation via server-side apply is done via a PATCH request to , so the attribute is populated. Namespace computed from request path: Namespace admission for apply-based creates: This is currently working as intended. If we want to make the behavior consistent between the two, I'd consider populating the namespace attribute for POST-based creation of Namespace objects.

Thanks for the thorough analysis Jordan, the problem OP is trying to solve is that they want to know if the object is namespaced or not, and used the presence of the field to detect that. According to your analysis, that seems wrong. The typical way to know if an object is namespaced is to use the discovery information, would that be the right way to solve that?
They could use discovery, or can continue to use the presence of the attribute, special-casing requests (namespaces are the only cluster-scoped resource which can have the attribute populated in AdmissionReview requests).
the documentation should be clearer, and I'd consider updating the POST-based create behavior to include this information to be consistent with update/patch/delete requests
do you want to update the doc and/or the post-based create behavior?
Sure! I am happy to take care of both.
where do you think it makes sense to add the namespace to the post-based behavior? We could add it to , but that might interfere with authorization, right? Would it be better to add the namespace directly to the ?
The info from the request body is not available when parsing requestinfo, only data in the URL. That's sort of what I expected, around the point we do this to pull the name from the incoming object:
Makes sense, I will give it a go, thank you!
that will also require sweeping admission plugins that check the namespace attribute. examples:
I think that we don't need to change anymore now. Do you know if there are other places in the docs where I should check?
yeah, I think we're good now. thanks for helping identify and clean this up


Question: Pod:         st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(), NumAllNodes: 2, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "1": framework.NewStatus(framework.UnschedulableAndUnresolvable, "injected unschedulable status").WithFailedPlugin("FakePreFilter"), "2": framework.NewStatus(framework.UnschedulableAndUnresolvable, "injected unschedulable status").WithFailedPlugin("FakePreFilter"), }, </ins> PreFilterMsg:         "injected unschedulable status", UnschedulablePlugins: sets.New("FakePreFilter"), },

When PreFilter returns UnschedulableAndUnresolvable, we don't need to run the preemption, but currently we run. This unexpected preemption could cause errors from the preemption by trying to read non-exist PreFilter data. As pointed out (), actually we didn't run the preemption in such cases in the past versions, but introduced this bug. /kind bug /kind regression /sig scheduling /triage accepted /priority important-soon
/assign
will fix this.


Question: return result } <ins> // getContainerUsageNanoCores gets usageNanoCores based on cached usageCoreNanoSeconds. func (p *criStatsProvider) getContainerUsageNanoCores(stats *runtimeapi.ContainerStats) *uint64 { if stats == nil || stats.Cpu == nil || stats.Cpu.UsageCoreNanoSeconds == nil { return nil } p.mutex.Lock() defer func() { // Update cache with new value. p.cpuUsageCache[stats.Attributes.Id] = stats.Cpu p.mutex.Unlock() }() cached, ok := p.cpuUsageCache[stats.Attributes.Id] if !ok || cached.UsageCoreNanoSeconds == nil { return nil } nanoSeconds := stats.Cpu.Timestamp - cached.Timestamp usageNanoCores := (stats.Cpu.UsageCoreNanoSeconds.Value - cached.UsageCoreNanoSeconds.Value) * uint64(time.Second/time.Nanosecond) / uint64(nanoSeconds) return &usageNanoCores } func (p *criStatsProvider) cleanupOutdatedCaches() { p.mutex.Lock() defer p.mutex.Unlock() for k, v := range p.cpuUsageCache { if v == nil { delete(p.cpuUsageCache, k) } if time.Since(time.Unix(0, v.Timestamp)) > defaultCachePeriod { delete(p.cpuUsageCache, k) } } } </ins> // removeTerminatedContainer returns the specified container but with // the stats of the terminated containers removed. func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container {

Related: On Windows, kubelet reports only the cumulative for cpu usage. On Linux, kubelet reports both and the instantaneous . Although In the future, most components/monitoring pipelines will switch to using the cumulative metrics, a few features still depend on the instantaneous metrics today. Examples are: , HPA/VPA. Kubelet should report this metrics on Windows to support the existing features. /cc who explained to me what features depend on the cpu metrics. /cc /cc
- is this something that's clear enough in the CRI layer that you can implement it, or do we need to fix that first?
/assign
do you have any notes or a doc you can point to that better describes what features depend on which metrics? It would be good for us to share and see if anything other than is broken
The horizontal and vertical pod autoscalers also depend on the
Documentation on metrics API and metrics server (i.e., the aggregator): The metrics-server pod keeps spewing errors like this: Running : I haven't verified whether pod autoscaling works in my cluster, but I assume they won't since metrics server seems to discard the metrics from the windows pods. I believe the implementation is in kubelet (for the docker integration), so this should not need CRI changes.
I think this is generally an issue for non-Docker CRIs (not necessarily Windows-exclusive). The kubelet source code suggests that these metrics will be gathered from the CRI (if the CRI is not Docker). The struct that kubelet expects returned from the ContainerStats RPC does not include usageNanoCores. Does this mean that, until this is resolved, custom CRIs are unable to satisfactorily report metrics to the 'metrics-server'?
- are you able to work on this?
that is a good point. For CRI runtimes, the kubelet should calculate the average over 10s to produce this metric.
Good point. We designed CRI around core metrics API, but some CRI runtimes have integrated with cadvisor directly to fill the gap while we wait for core metrics API. I think it's reasonable for kubelet to do this on behalf of all CRI runtimes. I spoke to offline, and this would probably require no more than 2 days of work. If sig-windows (e.g., can pick this up, that'd be great.

has agreed to review the changes (thanks!) If not, we should clearly document that this is not supported in Windows GA for 1.14.
thanks for the update. we will wait to hear back from
Currently, is got from docker's stats API on Windows. From docker stats API , only is available, hence is not set on Windows: Do you know any way to get instantaneous for Windows containers?
I have no idea. any idea?
I think this is broader than just a Windows issue. Officially supported tools such as the kubernetes metrics-server expect the field, but custom CRIs that conform to the interfaces defined in the package are not given the liberty to return this value.
suggested previously to cache the stats temporarily and compute the stats using the diff. David, could you chime in on how this could be done? Yes, this was mentioned in the previous comments too. There are plans to transition those components off this field, but it's WIP. We are looking for a workaround in kubelet for now.
my biggest worry is that if we say this impacts the horizonal POD autoscaler, how is that passing in our end2end and conformance tests?
Do you test HPA? It's not part of conformance, and I doubt this is included in windows testing.
As to how this could be done, in and we could just add a in the struct, which we assign the previous CpuUsage metric for each container. Then, when we need to calculate the rate, we pass the previous CpuUsage along with the current one to , and calculate a rate based on the timestamp and values.
ListPodCPUAndMemoryStats() we could just add a map[string]*runtimeapi.CpuUsage in the criStatsProvider struct I've done some tinkering on this myself and I originally thought that it would be as simple as that. It gets much more complicated when you have to work in between the existing cAdvisor-based logic. I assume the intent is to cut it out entirely eventually, but the information passed through the CRI doesn't even come close to being so comprehensive. So far I've found that this approach fixes the problem for the "/stats/summary" api, but not necessarily for the other apis. In particular, "/stats/container" seems to be completely based on cAdvisor, being unimpacted by the changes we made to those two functions.

Some monitoring tools and metrics scrapers rely on the "/stats/container" api for their metrics, which means it is insufficient to update one api without doing the other.
We don't need to fill in everything in the summary API, just "usageNanoCores". The "/stats/summary" api is the one used by the metrics server, which is why we care about it in this case. The "/stats/container" isn't something we need to implement to support this.
we a couple of tickets to validate if HPA for Windows containers works. My guess is that it works for memory and not for CPU metrics because of this issue. If this is complicated to fix, i wonder if waiting for the CRI containerD changes in windows would be the best option for supporting this.
containerd and cri-o bypass this issue by leveraging cadvisor as a temporary solution. I don't think that'd work for windows.
Sorry the for late response. HPA is actually working for Windows containers. e.g. in my cluster (v1.12.5), cpu-based HPA is working as expected: Which version of kubernetes are you referring for HPA issues?
are you running heapster, or the metrics server to serve HPA? Heapster uses the cumulative metric, but the metrics-server uses the 10 average rate metric.
metrics-server is used in my cluster.
See .
/milestone v1.14


Question: if externalCloudProvider { // If --cloud-provider=external and node address is already set, // then we return early because provider set addresses should take precedence. <del> // Otherwise, we try to look up the node IP and let the cloud provider override it later </del> <ins> // Otherwise, we try to use the node IP defined via flags and let the cloud provider override it later </ins> // This should alleviate a lot of the bootstrapping issues with out-of-tree providers if len(node.Status.Addresses) > 0 { return nil } <ins> // If nodeIPs are not specified wait for the external cloud-provider to set the node addresses. // Otherwise uses them on the assumption that the installer/administrator has the previous knowledge // required to ensure the external cloud provider will use the same addresses to avoid the issues explained // in https://github.com/kubernetes/kubernetes/issues/120720. // We are already hinting the external cloud provider via the annotation AnnotationAlphaProvidedIPAddr. if !nodeIPSpecified { return nil } </ins> } if cloud != nil { cloudNodeAddresses, err := nodeAddressesFunc()

The kubelet logic used to get the Node addresses when using an external provider, temporary assigns an address discovered from the node , this address can be overriden later by the external cloud provider This was implemented in to improve the bootstrapping problems with some external cloud providers The side effect of this behavior is that if the external cloud provider sets a different address/es, then the pods with will be showing the "first" addresses as their , since we don't mutate those IPs once set to avoid problems I think that we should revert the existing logic for external cloud providers, since is causing a temporary and inconsistent state and it seem to address mainly a problem with kubeadm but kubeadm folks seems to not need this functionality Announced mailing list /kind bug /sig network /sig node
/assign /cc
So we don't want the external provider to update the IP of the node, right? But I think even if it updates the address, for pods with hostnetwork, it still gets the first address from nodeStatus as podIPs, why is it inconsistent?
the contrary, we don't want to shortcut the address assignment, if cloud provider is external the node should wait for the cloud provider to assign the address. Let me put an example, a node with IPs 10.0.0.1 and 2001:db2::3 and cloud-provider external that sets the IPv6 address first. Right, now, the Node will use prefer IPv4 when it starts and assign the IP 10.0.0.1 to the node, hostNetwork pods will get that IP. and later the cloud provider will override the IPs and sets 2001:db2::3 as the address, so new hostNetwork pods will use the IPv6 address but old hostNetwork pods will have 10.0.0.1
I see, Now I think it makes sense, but we need to confirm that kubeadm really doesn't need it anymore.
^^^
I don't know the history of why it works the way it currently does, but I agree that it's weird and seems likely to lead to problems, so if it turns out it's not necessary, then let's change it.
This is why we taint nodes initially and only remove the taint once cloud provider has initialized the node.

So this is reallly an issue for pods tolerating this taint
i don't understand what is being asked exactly, but if the Q is again whether kubeadm supports self-hosted control plane components in a DS, then no. kubeadm does support the --cloud-privider=external on paper, but we don't have tests. i don't think we want to maintain them either. do you have context?
For current behavior, the workaround would be if I understand correctly. I don't know if there are any cloud provider installers that depend on this behavior. I prefer to fix the weird behavior to avoid potential problems. Or a problem detector has to cover this to check if a hostNet pod is not using the correct node IP.
yeah, but I think that is fair to give those pods the responsibility to deal with the known behavior (node will not have addresses until cloud provider external initializes) , than make the kubelet have two different addresses during its lifetime (that we discussed extensively and ) We discarded this behavior because most components, apps and features like downward API depend on the existing behavior of not changing the original IP For the temporal status without addresses we can just set the node as notReady
NetworkUnavailable might be more appropriate, but overall makes sense I think
is there a reported issue or bug that is reproducible based on what you describe in the isuse description? Logically what you said makes sense but I'm wondering how common the problem actually is
Ah, is it just or are there others?
Oh -- is not even related, because they are not setting --cloud-provider=external
yeah, I opened this issue because when I tried to bring up an IPv6 only cluster with kops in GCE it failed and one of the problems was that the hostNetwork pods were using IPv4 as pod ips ... is not likely anybody hits this with IPv4 or dual stack, since cloud-provider and kubelet will agree most of the times, but as soon that you use IPv6 only or IPv6 primary you ends with this problem
Isn't this a configuration issue on the kubelet and not with external cloud providers then? You should configure kubelet to bootstrap initially with an IPv6 address and not IPv4.

Still does not work for dual-stack, or if there are multiple addresses on the node, or multiple interfaces .... I don't see the reason why kubelet should configure the node addresses in the first place just to wait for the cloud-provider to override that ... the pods bootstrap problem is another thing, kubelet should not be ready if there are no addresses in the node , but current behavior is compensating the problem of having an external source for the address allocation, that is clearly not generalizable
yeah, I opened this issue because when I tried to bring up an IPv6 only cluster with kops in GCE it failed and one of the problems was that the hostNetwork pods were using IPv4 as pod ips ... is not likely anybody hits this with IPv4 or dual stack, since cloud-provider and kubelet will agree most of the times, but as soon that you use IPv6 only or IPv6 primary you ends with this problem For cloud-provider-vsphere, what we did to workaround is to identify the preferred ip address to get, which interface, which rank, then in bootstrap time we get it and pass it to kubelet flag . Now support dual stack address with external cloud-provider. Without the kubelet starting node with some IP, if there is issue in CPI code or in the config, then it would be hard for us to debug CPI since getting the log will getting error that node doesn't have prefered ip. Might need to get the container log from containerd directly by ssh into the node. You mentioned this is for GCE, I remember brought it up in here that now the is also if CCM has route controller enable, which doesn't have clear ownership between CMM and CNI which should reconcile it. And according to the CCM logic , in GCE case it will managed the in CCM, and probably will mark it as false once the route is ready, then node could possibly becomes ready. So maybe that's why in your case the node becomes ready even though the node doesn't have the address you preferred.

I found it trying with GCE, but that is not important, I can hit this with Openshift or Rancher or Kind ... my goal is to define a general behavior of the kubelet that is consistent and easier to think about, without breaking anyone of course, but current implementation is a footgun and seems tailor made... do we enforce all cloud providers to pass the --node-ip flag to kubelet to match their IPs? I will put this for the next sig-network agenda so we can go deeper and talk more about it
Another example of this problem in Azure and CAPG
Yeah, the NetworkUnavailable condition/taint is not extensible or reusable. Any change to its current semantics will not work correctly in some environments.
Ohk we ran into similar issue today, We didn't pass --node-ips to kubelet, and we are setting ipv6 as primary ipfamily, but kubelet chose ipv4 address at the beginning, so all hostNetwork pod has ipv4 address. Now even after CPI has fixed the node ip to put ipv6 address as primary address, all following pods still has ipv4 addresses regardless. Example: Node addresses:
so , does it mean you agree we need to fix this as in
I'm in favor of removing the behavior to bootstrap initial addresses as long as we are confident it is not breaking some exitsing behavior that some provider is depending on. I still find the scenario in a little odd, those pods should not be running on the node if the CCM has no initialized them yet. I wonder if the actual problem here is that node addresses are only updated after the first initialization: Which means pods can be admitted between the time time the cloud provider taint is removed and when addresses are updated.
so , does it mean you agree we need to fix this as in At least from cloud-provider-vsphere it shouldn't depend on this behavior to bootstrap. It's just that it would be harder for people to debug the CPI issue or issues in other component that comes before CPI like KCM/kube-apiserver/etcd in the future if something goes wrong before CPI could initialize the node. Example issue: I still find the scenario in a little odd, those pods should not be running on the node if the CCM has no initialized them yet. Yeah this is weird. For pod like apiserver pod has ipv4 address, which make sense since it comes before CPI.

Kubelet could choose the wrong IP at the beginning now CPI already removes the taints and updated the node with correct IP address, also the node has correct podCIDRs, with ipv6 address at the front. Still if I create a new pod on that node, it will be assigned with a ipv4 address as primary ip. For example that test pod was created long after the CPI initialized the node. And we only experience issue with pod that get created on this node, all other pod on other node will have ipv6 address as primary address.
/triage accepted /priority important-longterm
that should not be a problem if having node addresses is a requirement of node readiness
The AWS EKS AMIs that we provide specify which avoids this issue. If that flag is not passed, then the same problem exists along with a slightly worse one that we intend to fix,
good catch is event more complex, since the ccm will not add addresses meanwhile the node is tainted


Question: } // Mounter.SetUp() <del> if err := csiMounter.SetUp(nil); err != nil { </del> <ins> fsGroup := int64(2000) if err := csiMounter.SetUp(&fsGroup); err != nil { </ins> t.Fatalf("mounter.Setup failed: %v", err) } path := csiMounter.GetPath()

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When a volume for a CSI-based PV being mounted, the code does not apply the fsGroup value when the PV is marked as readOnly = false. What you expected to happen: if pv.readOnly == false, then the fsGroup ownership value should be applied to the mounted volume.
/assign /sig storage


Question: }, []string{"resource"}, ) <ins> WatchCacheReadWait = compbasemetrics.NewHistogramVec( &compbasemetrics.HistogramOpts{ Namespace:      namespace, Subsystem:      subsystem, Name:           "read_wait_seconds", Help:           "Histogram of time spent waiting for a watch cache to become fresh.", StabilityLevel: compbasemetrics.ALPHA, Buckets:        []float64{0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3}, }, []string{"resource"}) </ins> ) var registerMetrics sync.Once

Part of From For now posted as a "help wanted" issue, however if we cannot hit the timeline for 1.30 code freeze () I would take it over so we don't miss. Still issue is simple enough that it can be quickly implemented. Implementation tips: Add a metric around waitUnitlFreshAndBlock method Metric should be a histogram with latency buckets similar to api latency Metric should have label for resource version. A way to monitor watch cache impact on request latency.
/sig api-machinery /triage accepted /help
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /sig api-machinery /triage accepted /help Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: if err := s.Features.ApplyTo(genericConfig); err != nil { return nil, nil, err } <ins> if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil { return nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err) } </ins> // Use protobufs for self-communication. // Since not every generic apiserver has to support protobufs, we

See: This code should make several attempts before ing. Alternatively, we could consider not listening for connections until this code has succeeded.
Feel free to correct me if I'm wrong on any of this since I've never looked at kubernetes code base before. The code for the and the both appear to make multiple attempts before returning an error which leads to a Fatal.
, I think we had waited for to start at . Do we need to wait more or make it configurable?
I would like to work on this if nobody else is.
No need to ask for permission! Just say here that you're working on it, that should be sufficient to prevent duplicate work. :)
Can I have a go at this?
I have a nearly completed solution for this already. Just working on testing/test cases. WIll hopefully sumbit a PR shortly.
Ok.
I wondered why we care about this. After all, kubelet restarts failing pods. For posterity, after talking with the reason is that the apiserver may fail so fast that the kubelet begins to back off on restarts of a failing pod. That backoff may be too aggressive. (Maybe it causes spurious alerts, or maybe it allows too much downtime.) So, let's fix the apiserver. That's fine, but clearly we're not the only user that might encounter this. You may be running a server you don't want to (or can't) modify, and you might wish you could tweak the kubelet's backoff heuristics for your pod. Alternately, maybe the kubelet could choose not to enter the backoff if apiserver exited with a different code. anything like this already in place?
That's a wonderful explanation that hadn't occurred to me. If there isn't a feature request for this already I'll work on something and reference this issue.
AFAIK, the backoff mechanism is there to protect kubelet from being overwhelmed by failed user containers. Trusting the containers with returning a specific exit code sounds like a big "no" to me. Making this configurable per pod might be ok, but it's still not good in general to allow aggressive restarts.
The referred PR () only fixes the issue for kube-apiserver. We still need a generic fix in genericapiserver (Ref ).


Question: --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 & KUBELET_PID=$! <del> kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25 </del> <ins> kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet" </ins> # Start kube-apiserver kube::log::status "Starting kube-apiserver"

Example:
Is kubelet healthz coupled with the docker daemon being accessible? ?
That particular error is coming from cAdvisor and should not be causing the test to block. The Kubelet does wait up to 5min for Docker to come up though. Do we use a real Docker in these tests?
Shippable is running on different os dist? ContainerOsVersion:Ubuntu precise?
Yes today kubelet healthz coupled with docker daemon. We even periodically healthcheck docker and kubelet, and restart them even there are issues.
Another example:
bumping priority because we have to get these flaky tests fixed.
This has been marked P0 for 2 months but hasn't had any updates. Is this still happening on Shippable? If so, can we just increase a timeout in the test?
This is ridiculous, I'll fix it myself.


Question: <ins> - job-template: name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}' disabled: false description: 'Upgrade multijob from {version-old} to {version-new}. Test owner: ihmccreery.' project-type: multijob triggers: - timed: '@hourly' builders: # TODO(ihmccreery) In theory, we could get ourselves into trouble by # editing these things in the middle of a run.  Jenkins Job Builder # would delete jobs, and they'd leave resources lying around.  We # should either (1) make this not a multi-job, or (2) add a script here # to update these jobs only at the beginning of a multijob run. # # This pain would also pretty much disappear with #18119, too. - multijob: name: Deploy condition: SUCCESSFUL projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step1-deploy' - multijob: name: Kubectl Test New condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step2-kubectl-e2e-new' - multijob: name: Upgrade Master condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step3-upgrade-master' - multijob: name: Test Old condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step4-e2e-old' - multijob: name: Upgrade Cluster condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step5-upgrade-cluster' - multijob: name: Test Old condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step6-e2e-old' - multijob: name: Test New condition: COMPLETED projects: - name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step7-e2e-new' - job-template: name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step1-deploy' description: 'Deploy a cluster at {version-old} to be tested and upgraded to {version-new}. Test owner: ihmccreery.' logrotate: daysToKeep: 7 builders: - shell: | curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/{branch}/hack/jenkins/e2e.sh" | bash - properties: - mail-watcher publishers: - claim-build # No junit-publisher, since we're not running any tests - gcs-uploader - log-parser - email-ext: recipients: 'ihmccreery@google.com' wrappers: - ansicolor: colormap: xterm - timeout: timeout: 60 fail: true - timestamps - workspace-cleanup - job-template: name: 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}' description: '{description} Test owner: ihmccreery.' # Use the same workspace as step1 workspace: /var/lib/jenkins/jobs/kubernetes-upgrade-{provider}-{version-old}-{version-new}-step1-deploy/workspace/ logrotate: daysToKeep: 7 builders: - shell: | curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/{branch}/hack/jenkins/e2e.sh" | bash - properties: - mail-watcher publishers: - claim-build - junit-publisher - gcs-uploader - log-parser - email-ext: recipients: 'ihmccreery@google.com' wrappers: - ansicolor: colormap: xterm - timeout: timeout: 300 abort: true fail: true - timestamps # Don't clean the workspace; we want to keep configs intact across steps in the multijob - job-group: name: '{provider}-{version-old}-{version-new}-upgrades' jobs: - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-step1-deploy': branch: '{branch-old}' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step2-kubectl-e2e-new' branch: '{branch-new}' description: 'Run {version-new} kubectl tests against the cluster running {version-old}.' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step3-upgrade-master' branch: '{branch-new}' description: 'Upgrade the master from {version-old} to {version-new}.' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step4-e2e-old' branch: '{branch-old}' description: 'Run {version-old} e2e tests against the cluster with master at {version-new} and nodes still at {version-old}.' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step5-upgrade-cluster' branch: '{branch-new}' description: 'Upgrade the nodes from {version-old} to {version-new}.' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step6-e2e-old' branch: '{branch-old}' description: 'Run {version-old} e2e tests against the cluster with master and nodes at {version-new}.' - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step7-e2e-new' branch: '{branch-new}' description: 'Run {version-new} e2e tests against the cluster with master and nodes at {version-new}.' - project: name: 'upgrade-gke' jobs: - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gke' version-old: '1.1' version-new: 'master' branch-old: 'release-1.1' branch-new: 'master' - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gke' version-old: '1.0' version-new: 'master' branch-old: 'release-1.0' branch-new: 'master' - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gke' version-old: '1.0' version-new: 'current-release' branch-old: 'release-1.0' branch-new: 'release-1.1' - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gke' version-old: 'stable' version-new: 'current-release' branch-old: 'release-1.1' branch-new: 'release-1.1' - project: name: 'upgrade-gce' jobs: - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gce' version-old: '1.1' version-new: 'master' branch-old: 'release-1.1' branch-new: 'master' - '{provider}-{version-old}-{version-new}-upgrades': provider: 'gce' version-old: 'stable' version-new: 'current-release' branch-old: 'release-1.1' branch-new: 'release-1.1' </ins>

got us through the v1.1 release, but it didn't go very smoothly and should be better for v1.2.


Question: NodeSelector: map[string]string{ "kubernetes.io/os": "windows", }, <ins> NodeName: node.Name, </ins> }, } <del> pods = append(pods, &pod) </del> <ins> _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) framework.ExpectNoError(err) } podName := "mem-failure-pod" failurePod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  podName, Image: imageutils.GetPauseImageName(), Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ v1.ResourceMemory: *resource.NewQuantity(1024*1024*1024, resource.BinarySI), }, }, }, }, NodeSelector: map[string]string{ "kubernetes.io/os": "windows", }, }, </ins> } <ins> failurePod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), failurePod, metav1.CreateOptions{}) framework.ExpectNoError(err) gomega.Eventually(func() bool { eventList, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err) for _, e := range eventList.Items { // Look for an event that shows FailedScheduling if e.Type == "Warning" && e.Reason == "FailedScheduling" && e.InvolvedObject.Name == failurePod.ObjectMeta.Name { framework.Logf("Found %+v event with message %+v", e.Reason, e.Message) return true } } return false }, 3*time.Minute, 10*time.Second).Should(gomega.Equal(true)) </ins> <del> return pods </del> } // getNodeMemory populates a nodeMemory struct with information from the first

The test : is failing because it assumes the memory is the same across all nodes. A change in the configuration changed that assumption and the test started to fail. the test to assign the correct amount of memory to each of the nodes to fill them up. create a Windows cluster with 3 nodes. Two have 6gb of allocatable memory, 1 has 14gb of allocatable memory. Run the test above and it will fail because it tries to create 3 pods with 9 gb of memory. The first pod will land on the node with 14gb of memory but the other two pods won't be able to find a node with enough memory (9gb pod 6 gb nodes). fixes this in the configuration /sig windows <details</details<details</details<details</details<details</details<details</details<details</details
/triage accepted /priority important-long-term
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /triage accepted /priority important-long-term Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Please inform that Am I reasonable here or not. We need to update the in all the cases inside the (, and ) similar to what was done in PR, but now we need to update it in all cases. If you could please confirm that I am reasonable here then I can begin to make these changes in a PR.
sorry I could have been clearer in the issue. The test it's self needs to be updated to handle nodes that do not have consistent memory. The logic in the test is wrong if the nodes don't all have the same memory values: This results in pods that won't fit on some of the nodes and leaves the pods pending. The test should handle this situation by making sure pods filled up memory but can be scheduled across the nodes. Let me know of need more clarification.
So, here we have to fix the logic because the logic is wrong for the case when the nodes don't have the same memory values.

Is the node memory value is equal to which gets assigned to variable in ?My understanding of tackling the issue : So, we need to change because it is the total memory and when it is fixed then the pods present will get appropriate memory limit by . So, if updating the logic for the value of is our focus, such that the it (memory limit) is changed according to the nodes, how can we achieve that ? Am I thinking reasonable here? If my understanding is correct till now then how we will be changing the code for assigning the memory value for a node such that each of the nodes get relevant amount of memory based on there need to work properly.
Yes, I think think the simplest thing to so would be get a list of windows nodes iterate though each one, looking up the allocatable memory for the node, creating a pod to fill that memory and assign it to that node for scheduling via , ensuring the scheduling was successful. Then once the nodes are all full deploy the pod that should go into pending (code should stay the same). pseudo code:
Do we have some examples in the repo where we do something similar for other nodes ? First we need to make function or is it already present ? Confirming : Also Do we need to run the whole code(After Modifying) inside in loop for every windows node and if even one return fails, the test fails.
I need some guidance. Could you please take a look on previous comment.
example of getting the nodes and memory total: no, basically need to replace some of the code in that function with the sudo code in
Why are we not deploying both the pods of in one loop only (if we want memory of pending pods memory same as nodes memory). Also after we deploys both the pods, do we will be running another loop similar to for checking the failure. I am not much sure about my changes, can I make a PR so that I can discuss about my code and make appropriate changes further ?
Sounds good
Does functions , and already exists or we have to make them because I am not able to find, have you used these names as short forms for other functions which do same as their names ?


Question: if err != nil { return err } <del> otelDialer := func(ctx context.Context, addr string) (net.Conn, error) { return egressDialer(ctx, "tcp", addr) </del> <ins> if egressDialer != nil { otelDialer := func(ctx context.Context, addr string) (net.Conn, error) { return egressDialer(ctx, "tcp", addr) } opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer))) </ins> } <del> opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer))) </del> } resourceOpts := []resource.Option{

While trying to enable API Server Tracing while using k3s, the API Server component crashes. The only way to avoid this crash is to disable egress selectors in k3s entirely. I originally opened an issue in the k3s repo (see ), and we determined that the issue seems to be an assumption that a dialer will always be returned if there's no error, but in this case there's no grpc egress configured so is returning and causing a panic at API Server Tracing to function properly. Please see the linked k3s issue for a repro case; However, I believe the minimal setup for this would be to configure cluster egress without defining a grpc egress selector and then attempting to enable API server tracing. panic stack trace -- contents of egress selector config file in direct mode- contents of egress selector config in 'agent' mode - contents of api server tracing config file -- <details25.2+k3s1 </details<detailsDigital Ocean Droplet </details<details</details<details</details<details</details<details</details
/sig instrumentation
/assign /triage accepted Thanks


Question: } t.Logf("Created service: %s", svcNodePort.Name) } <ins> // Regresion test for https://issues.k8s.io/115316 func TestServiceCIDR28bits(t *testing.T) { serviceCIDR := "10.0.0.0/28" client, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.ServiceClusterIPRanges = serviceCIDR }, }) defer tearDownFn() // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) if err != nil { return false, err } return true, nil }); err != nil { t.Fatalf("creating kubernetes service timed out") } ns := framework.CreateNamespaceOrDie(client, "test-regression", t) defer framework.DeleteNamespaceOrDie(client, ns, t) service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test-1234", }, Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, Ports: []v1.ServicePort{{ Port: int32(80), }}, Selector: map[string]string{ "foo": "bar", }, }, } _, err := client.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) if err != nil { t.Fatalf("Error creating test service: %v", err) } } </ins>

kube-apiserver observed a panic: kube-apiserver should not panic Not sure yet -- will add more details as I investigate. No response v1.25.4 N/A N/A N/A N/A N/A
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network /cc
max is greater than offset?
/assign


Question: } return true, obj, nil }) <ins> nodes = &v1.NodeList{} nodeNamePrefix := "mynode" for i := 0; i < 5; i++ { var nodeName string if i != 0 { nodeName = fmt.Sprintf("%s-%d", nodeNamePrefix, i) } else { // We want also the "mynode" node since all the testing pods live there nodeName = nodeNamePrefix } attachVolumeToNode("lostVolumeName", nodeName) } </ins> fakeClient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &v1.NodeList{} <del> nodeNamePrefix := "mynode" for i := 0; i < 5; i++ { var nodeName string if i != 0 { nodeName = fmt.Sprintf("%s-%d", nodeNamePrefix, i) } else { // We want also the "mynode" node since all the testing pods live there nodeName = nodeNamePrefix } node := v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, Labels: map[string]string{ "name": nodeName, }, Annotations: map[string]string{ util.ControllerManagedAttachAnnotation: "true", }, }, Status: v1.NodeStatus{ VolumesAttached: []v1.AttachedVolume{ { Name:       TestPluginName + "/lostVolumeName", DevicePath: "fake/path", }, }, }, } obj.Items = append(obj.Items, node) } </del> <ins> obj.Items = append(obj.Items, nodes.Items...) </ins> return true, obj, nil }) <ins> volumeAttachments = &storagev1.VolumeAttachmentList{} fakeClient.AddReactor("list", "volumeattachments", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &storagev1.VolumeAttachmentList{} obj.Items = append(obj.Items, volumeAttachments.Items...) return true, obj, nil }) fakeClient.AddReactor("create", "volumeattachments", func(action core.Action) (handled bool, ret runtime.Object, err error) { createAction := action.(core.CreateAction) va := createAction.GetObject().(*storagev1.VolumeAttachment) volumeAttachments.Items = append(volumeAttachments.Items, *va) return true, createAction.GetObject(), nil }) pvs = &v1.PersistentVolumeList{} fakeClient.AddReactor("list", "persistentvolumes", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &v1.PersistentVolumeList{} obj.Items = append(obj.Items, pvs.Items...) return true, obj, nil }) fakeClient.AddReactor("create", "persistentvolumes", func(action core.Action) (handled bool, ret runtime.Object, err error) { createAction := action.(core.CreateAction) pv := createAction.GetObject().(*v1.PersistentVolume) pvs.Items = append(pvs.Items, *pv) return true, createAction.GetObject(), nil }) </ins> fakeWatch := watch.NewFake() fakeClient.AddWatchReactor("*", core.DefaultWatchReactor(fakeWatch, nil))

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Background: The attach/detach controller create objects for CSI Volumes to indicate that a volume needs to be attached to a node. It deletes the object to indicate that the volume should be detached. Problem: If the Attach/Detach controller crashes, and in the meantime the pod using that volume is deleted, when the AD controller restarts, it won't know that the volume was previously attached and should be cleaned up. What you expected to happen: Proposal: We should have the AD controller periodically list all objects and for any that are not accounted for, update its internal actual/desired state caches so that the volume is gracefully detached. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: /sig storage Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
could I take it?
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available .

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/lifecycle frozen /reopen
Reopened this issue. <detailsIn response to : /lifecycle frozen /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
should address this


Question: func makePodSpec(readinessProbe, livenessProbe *api.Probe) *api.Pod { pod := &api.Pod{ <del> ObjectMeta: api.ObjectMeta{Name: "nginx-" + string(util.NewUUID())}, </del> <ins> ObjectMeta: api.ObjectMeta{Name: "test-webserver-" + string(util.NewUUID())}, </ins> Spec: api.PodSpec{ Containers: []api.Container{ { <del> Name:           "nginx", Image:          "nginx", </del> <ins> Name:           "test-webserver", Image:          "gcr.io/google_containers/test-webserver", </ins> LivenessProbe:  livenessProbe, ReadinessProbe: readinessProbe, },

If the container_probe test runs on a node without the nginx image, it sometimes takes 90s to get the pod in running. Most of this time goes in pulling the image, so I'm assuming it's registry related hiccup. Using might make it more reliable. Example failure: Note the following event in the logs:
a) we should not be pulling from docker in our tests - too hit-or-miss b) we should replace nginx (132 MB) with (4.5 MB) whenever possible. , Prashanth B wrote:


Question: ginkgo.By("waiting for RC to be added") eventFound := false <del> ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) </del> <ins> ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) </ins> defer cancel() _, err = watchUntilWithoutRetry(ctx, retryWatcher, func(watchEvent watch.Event) (bool, error) { if watchEvent.Type != watch.Added {

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: - Triage link for more jobs that are affected: Which test(s) are failing: Since when has it been failing: Since October 23 Testgrid link: Reason for failure: Anything else we need to know: Example spyglass link: /sig apps /cc /priority important-soon
/cc missed this in the main body
/assign /triage accepted
/remove-priority important-soon /priority critical-urgent top flake for pull-kubernetes-conformance-kind-ipv6-parallel, 2nd top flake for per /area conformance this is a conformance test /remove-kind failing-test /kind flake this is not failing continuously, but is instead flaking
/assign
I increased the timeout while waiting for available Replicas.
Test grid green since Merge 5 Nov 2020 /cc


Question: if getUnschedulablePod(q, affinityPod) != nil { t.Error("affinityPod is still in the unschedulableQ.") } <ins> q.lock.RLock() </ins> if _, exists, _ := q.activeQ.Get(newPodInfoNoTimestamp(affinityPod)); !exists { t.Error("affinityPod is not moved to activeQ.") } <ins> q.lock.RUnlock() </ins> // Check that the other pod is still in the unschedulableQ. if getUnschedulablePod(q, &unschedulablePod) == nil { t.Error("unschedulablePod is not in the unschedulableQ.")

<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --Which jobs are flaking: pull-kubernetes-bazel-test Which test(s) are flaking: pkg/scheduler/internal/queue Testgrid link: Reason for failure: Anything else we need to know: /sig scheduling /assign
I just sent a pr for it according to the error message, but I'm not very sure if it will make things all right.
/assign


Question: "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" <del> "k8s.io/apiserver/pkg/features" </del> "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" <del> utilfeature "k8s.io/apiserver/pkg/util/feature" </del> ) // strategy implements behavior for CustomResources.

This is the tracking issue of promoting CRD validation rules to GA: Listed the blocking issues/features below: [x] [x] [x] [x] Issues be aware of but should not be a GA blocker [ ] [ ] [ ] [ ] [ ] /sig api-machinery /triage accepted
/assign
/cc


Question: // syncResourceQuota runs a complete sync of current status func (rm *ResourceQuotaManager) syncResourceQuota(quota api.ResourceQuota) (err error) { <ins> // quota is dirty if any part of spec hard limits differs from the status hard limits dirty := !api.Semantic.DeepEqual(quota.Spec.Hard, quota.Status.Hard) </ins> // dirty tracks if the usage status differs from the previous sync, // if so, we send a new usage with latest status // if this is our first sync, it will be dirty by default, since we need track usage <del> dirty := quota.Status.Hard == nil || quota.Status.Used == nil </del> <ins> dirty = dirty || (quota.Status.Hard == nil || quota.Status.Used == nil) </ins> // Create a usage object that is based on the quota resource version usage := api.ResourceQuota{

Resource quota update does not take effect until create/delete another ResourceQuota. I am not sure it is an issue or design.
I am 99% sure that the issue is the update is changing the Spec and the Kubectl client is showing the Status. The Status will update at the next quota sync interval to align with the desired Spec. Can you describe what KUBERNETES environment you are using so I can verify? In particular, how is kube-controller-manager started? You can confirm this by doing kubectl get quota -o json And you should see the spec update occurred and if you wait 1 minute you will see the status take effect. A few options to improve user experience: Improve latency of updating status with spec. This has some potential side effects that I need to think through with how the current controller is structured. Change kubectl output to show spec if it differs from status so it's clear that the system will ultimately drive to its desired state. This seems like a good thing to do in general, will experiment with some options to present this information, but I am open to your input. I think even a single line noting that the enforced usage limits are different than the desired state but will reconcile soon. Sent from my iPhone
Sure to append config of kube-controller-manager: Tried the cmd, the spec updated as you said, however, the status refuse to take effect after more than 5 mins. I am not sure if it is caused by my version v0.13.2.
I see the issue, will send a PR. It appears the quota status was only getting marked as dirty for update if the usage changed, and not the underlying limit itself. So I suspect if you removed a pod (assuming you were tracking total number of pods), you would see the status kept updated to the desired spec.
The status changed correctly after resizing a rc and wait a while (8s).
I want to update the Kubectl output to show the desired value if it differs from the status value. I will look to do that in the next few days. I think that would eliminate all confusion. Sent from my iPhone


Question: err := decorated2.Do(func() { defer close(executed) if !delegated { <del> t.Fatalf("expected delegated execution") </del> <ins> t.Errorf("expected delegated execution") </ins> } }) if err == nil {

Can we get this fixed ASAP? Thanks!
yep, thanks for reporting , Brendan Burns wrote:
how reproduceable is this? I have some changes that I think will resolve it but I couldn't reproduce locally even when all my cpu's were pretty tied up and using a high GOMAXPROCS.
flakey label process?
the labelling is due to me failing @ github's label interface. Bulk editing labels removes them if any of the selected things have the label sigh Thanks for the quick fix.


Question: }) // Rc <del> output := framework.RunKubectlOrDie("describe", "rc", "redis-master", nsFlag) </del> requiredStrings := [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns},

https://k8s- replica set controller maybe?
The rc seems to be updated almost a second after the failure. Maybe the test needs to retry
And the real log for the failure is
/assign flakes PRs at least once a day The can filter down to focus on a single test:
For when you have some time.
Reassign to you
Output of has been change in 1.6. That's why upgrade tests are failing. But I'm not sure why it is not failing 100% from the triage board. Do you know why? Output of 1.5 or earlier: Output in 1.6 or later:
The timeout is now. Is it reasonable to give it a little more time?
That timeout waits for the pods to run based on the API server. The rc controller is using its own cache so it may lag a bit before it actually updates the rc status. The test needs to retry for some time if the output we get back from describe does not match.
In other word, should also have its own retries.
SGTM. Going to send a fix today.
FYI, this test continues to flake PRs at least once per day:
should it show up in
Hm, this seems to fail in the upgrade tests. this failure seems to be related to an image name change as you already pointed out before. Mind finding what changed the test to use that image and fix the image name?
It is actually the key name has been changed in the key-value pair, not the value. As I pointed out in before expected key: now expected key:
Hm, this is unfortunate. These kubectl tests should really be unit tests.
the 24 hr test history is largely focused on failures right now, and this is highlighting flakes specifically (aka which tests are both passing and failing at the same commit the most often). Please follow the reports I'm sending to kubernetes-dev and related sigs. Also you can generate this info yourself using PS: also try out for finding failures (I'm sending out a separate email highlighting the biggest sources of failure). PPS: we are in the process of creating infrastructure to automate creating these reports, but automation never works as well as a fellow human asking for attention.
We still need to patch the release-1.4 and release-1.5 branch to make the upgrade tests passing.

I will send PRs later this week.
We didn't fix
The change mentioned in was introduced in 1.6. The change covers describer and kubectl e2e test. After discussing with we think we are not going to backport the changes to 1.4 and 1.5, instead we are going to skip the tests in version skew cases. WDYT?
should fix this issue
looks like the PR is resolved, should we close?
Please wait until fix to be cherrypick into 1.4.
Sure! Which PR is this?
Is it possible to cherrypick to 1.4 by just changing the milestone to 1.4?
no you need to create a new PR with the target branch of 1.4
Ready to close?
Yup. /close