Question: <ins> /* Copyright 2014 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package common import ( "fmt" "os" "path" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("[sig-storage] Projected secret", func() { f := framework.NewDefaultFramework("projected") /* Release : v1.9 Testname: Projected Volume, Secrets, volume mode default Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key with default permission mode. Pod MUST be able to read the content of the key successfully and the mode MUST be -rw-r--r-- by default. */ framework.ConformanceIt("should be consumable from pods in volume [NodeConformance]", func() { doProjectedSecretE2EWithoutMapping(f, nil /* default mode */, "projected-secret-test-"+string(uuid.NewUUID()), nil, nil) }) /* Release : v1.9 Testname: Projected Volume, Secrets, volume mode 0400 Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key with permission mode set to 0x400 on the Pod. Pod MUST be able to read the content of the key successfully and the mode MUST be -r—-—————. */ framework.ConformanceIt("should be consumable from pods in volume with defaultMode set [NodeConformance]", func() { defaultMode := int32(0400) doProjectedSecretE2EWithoutMapping(f, &defaultMode, "projected-secret-test-"+string(uuid.NewUUID()), nil, nil) }) /* Release : v1.9 Testname: Project Volume, Secrets, non-root, custom fsGroup Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key. The volume has permission mode set to 0440, fsgroup set to 1001 and user set to non-root uid of 1000. Pod MUST be able to read the content of the key successfully and the mode MUST be -r—-r————-. */ framework.ConformanceIt("should be consumable from pods in volume as non-root with defaultMode and fsGroup set [NodeConformance]", func() { defaultMode := int32(0440) /* setting fsGroup sets mode to at least 440 */ fsGroup := int64(1001) uid := int64(1000) doProjectedSecretE2EWithoutMapping(f, &defaultMode, "projected-secret-test-"+string(uuid.NewUUID()), &fsGroup, &uid) }) /* Release : v1.9 Testname: Projected Volume, Secrets, mapped Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key with default permission mode. The secret is also mapped to a custom path. Pod MUST be able to read the content of the key successfully and the mode MUST be -r—-—————— on the mapped volume. */ framework.ConformanceIt("should be consumable from pods in volume with mappings [NodeConformance]", func() { doProjectedSecretE2EWithMapping(f, nil) }) /* Release : v1.9 Testname: Projected Volume, Secrets, mapped, volume mode 0400 Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key with permission mode set to 0400. The secret is also mapped to a specific name. Pod MUST be able to read the content of the key successfully and the mode MUST be -r—-—————— on the mapped volume. */ framework.ConformanceIt("should be consumable from pods in volume with mappings and Item Mode set [NodeConformance]", func() { mode := int32(0400) doProjectedSecretE2EWithMapping(f, &mode) }) It("should be able to mount in a volume regardless of a different secret existing with same name in different namespace [NodeConformance]", func() { var ( namespace2  *v1.Namespace err         error secret2Name = "projected-secret-test-" + string(uuid.NewUUID()) ) if namespace2, err = f.CreateNamespace("secret-namespace", nil); err != nil { framework.Failf("unable to create new namespace %s: %v", namespace2.Name, err) } secret2 := secretForTest(namespace2.Name, secret2Name) secret2.Data = map[string][]byte{ "this_should_not_match_content_of_other_secret": []byte("similarly_this_should_not_match_content_of_other_secretn"), } if secret2, err = f.ClientSet.CoreV1().Secrets(namespace2.Name).Create(secret2); err != nil { framework.Failf("unable to create test secret %s: %v", secret2.Name, err) } doProjectedSecretE2EWithoutMapping(f, nil /* default mode */, secret2.Name, nil, nil) }) /* Release : v1.9 Testname: Projected Volume, Secrets, mapped, multiple paths Description: A Pod is created with a projected volume source ‘secret’ to store a secret with a specified key. The secret is mapped to two different volume mounts. Pod MUST be able to read the content of the key successfully from the two volume mounts and the mode MUST be -r—-—————— on the mapped volumes. */ framework.ConformanceIt("should be consumable in multiple volumes in a pod [NodeConformance]", func() { // This test ensures that the same secret can be mounted in multiple // volumes in the same pod.  This test case exists to prevent // regressions that break this use-case. var ( name             = "projected-secret-test-" + string(uuid.NewUUID()) volumeName       = "projected-secret-volume" volumeMountPath  = "/etc/projected-secret-volume" volumeName2      = "projected-secret-volume-2" volumeMountPath2 = "/etc/projected-secret-volume-2" secret           = secretForTest(f.Namespace.Name, name) ) By(fmt.Sprintf("Creating secret with name %s", secret.Name)) var err error if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil { framework.Failf("unable to create test secret %s: %v", secret.Name, err) } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-projected-secrets-" + string(uuid.NewUUID()), }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: volumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: name, }, }, }, }, }, }, }, { Name: volumeName2, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: name, }, }, }, }, }, }, }, }, Containers: []v1.Container{ { Name:  "secret-volume-test", Image: imageutils.GetE2EImage(imageutils.Mounttest), Args: []string{ "--file_content=/etc/projected-secret-volume/data-1", "--file_mode=/etc/projected-secret-volume/data-1"}, VolumeMounts: []v1.VolumeMount{ { Name:      volumeName, MountPath: volumeMountPath, ReadOnly:  true, }, { Name:      volumeName2, MountPath: volumeMountPath2, ReadOnly:  true, }, }, }, }, RestartPolicy: v1.RestartPolicyNever, }, } f.TestContainerOutput("consume secrets", pod, 0, []string{ "content of file "/etc/projected-secret-volume/data-1": value-1", "mode of file "/etc/projected-secret-volume/data-1": -rw-r--r--", }) }) /* Release : v1.9 Testname: Projected Volume, Secrets, create, update delete Description: Create a Pod with three containers with secrets namely a create, update and delete container. Create Container when started MUST no have a secret, update and delete containers MUST be created with a secret value. Create a secret in the create container, the Pod MUST be able to read the secret from the create container. Update the secret in the update container, Pod MUST be able to read the updated secret value. Delete the secret in the delete container. Pod MUST fail to read the secret from the delete container. */ framework.ConformanceIt("optional updates should be reflected in volume [NodeConformance]", func() { podLogTimeout := framework.GetPodSecretUpdateTimeout(f.ClientSet) containerTimeoutArg := fmt.Sprintf("--retry_time=%v", int(podLogTimeout.Seconds())) trueVal := true volumeMountPath := "/etc/projected-secret-volumes" deleteName := "s-test-opt-del-" + string(uuid.NewUUID()) deleteContainerName := "dels-volume-test" deleteVolumeName := "deletes-volume" deleteSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, Name:      deleteName, }, Data: map[string][]byte{ "data-1": []byte("value-1"), }, } updateName := "s-test-opt-upd-" + string(uuid.NewUUID()) updateContainerName := "upds-volume-test" updateVolumeName := "updates-volume" updateSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, Name:      updateName, }, Data: map[string][]byte{ "data-1": []byte("value-1"), }, } createName := "s-test-opt-create-" + string(uuid.NewUUID()) createContainerName := "creates-volume-test" createVolumeName := "creates-volume" createSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, Name:      createName, }, Data: map[string][]byte{ "data-1": []byte("value-1"), }, } By(fmt.Sprintf("Creating secret with name %s", deleteSecret.Name)) var err error if deleteSecret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(deleteSecret); err != nil { framework.Failf("unable to create test secret %s: %v", deleteSecret.Name, err) } By(fmt.Sprintf("Creating secret with name %s", updateSecret.Name)) if updateSecret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(updateSecret); err != nil { framework.Failf("unable to create test secret %s: %v", updateSecret.Name, err) } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-projected-secrets-" + string(uuid.NewUUID()), }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: deleteVolumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: deleteName, }, Optional: &trueVal, }, }, }, }, }, }, { Name: updateVolumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: updateName, }, Optional: &trueVal, }, }, }, }, }, }, { Name: createVolumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: createName, }, Optional: &trueVal, }, }, }, }, }, }, }, Containers: []v1.Container{ { Name:    deleteContainerName, Image:   imageutils.GetE2EImage(imageutils.Mounttest), Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/projected-secret-volumes/delete/data-1"}, VolumeMounts: []v1.VolumeMount{ { Name:      deleteVolumeName, MountPath: path.Join(volumeMountPath, "delete"), ReadOnly:  true, }, }, }, { Name:    updateContainerName, Image:   imageutils.GetE2EImage(imageutils.Mounttest), Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/projected-secret-volumes/update/data-3"}, VolumeMounts: []v1.VolumeMount{ { Name:      updateVolumeName, MountPath: path.Join(volumeMountPath, "update"), ReadOnly:  true, }, }, }, { Name:    createContainerName, Image:   imageutils.GetE2EImage(imageutils.Mounttest), Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/projected-secret-volumes/create/data-1"}, VolumeMounts: []v1.VolumeMount{ { Name:      createVolumeName, MountPath: path.Join(volumeMountPath, "create"), ReadOnly:  true, }, }, }, }, RestartPolicy: v1.RestartPolicyNever, }, } By("Creating the pod") f.PodClient().CreateSync(pod) pollCreateLogs := func() (string, error) { return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, createContainerName) } Eventually(pollCreateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("Error reading file /etc/projected-secret-volumes/create/data-1")) pollUpdateLogs := func() (string, error) { return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, updateContainerName) } Eventually(pollUpdateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("Error reading file /etc/projected-secret-volumes/update/data-3")) pollDeleteLogs := func() (string, error) { return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, deleteContainerName) } Eventually(pollDeleteLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-1")) By(fmt.Sprintf("Deleting secret %v", deleteSecret.Name)) err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(deleteSecret.Name, &metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred(), "Failed to delete secret %q in namespace %q", deleteSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Updating secret %v", updateSecret.Name)) updateSecret.ResourceVersion = "" // to force update delete(updateSecret.Data, "data-1") updateSecret.Data["data-3"] = []byte("value-3") _, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Update(updateSecret) Expect(err).NotTo(HaveOccurred(), "Failed to update secret %q in namespace %q", updateSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Creating secret with name %s", createSecret.Name)) if createSecret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(createSecret); err != nil { framework.Failf("unable to create test secret %s: %v", createSecret.Name, err) } By("waiting to observe update in volume") Eventually(pollCreateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-1")) Eventually(pollUpdateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-3")) Eventually(pollDeleteLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("Error reading file /etc/projected-secret-volumes/delete/data-1")) }) }) func doProjectedSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int32, secretName string, fsGroup *int64, uid *int64) { var ( volumeName      = "projected-secret-volume" volumeMountPath = "/etc/projected-secret-volume" secret          = secretForTest(f.Namespace.Name, secretName) ) By(fmt.Sprintf("Creating projection with secret that has name %s", secret.Name)) var err error if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil { framework.Failf("unable to create test secret %s: %v", secret.Name, err) } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "pod-projected-secrets-" + string(uuid.NewUUID()), Namespace: f.Namespace.Name, }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: volumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: secretName, }, }, }, }, }, }, }, }, Containers: []v1.Container{ { Name:  "projected-secret-volume-test", Image: imageutils.GetE2EImage(imageutils.Mounttest), Args: []string{ "--file_content=/etc/projected-secret-volume/data-1", "--file_mode=/etc/projected-secret-volume/data-1"}, VolumeMounts: []v1.VolumeMount{ { Name:      volumeName, MountPath: volumeMountPath, }, }, }, }, RestartPolicy: v1.RestartPolicyNever, }, } if defaultMode != nil { //pod.Spec.Volumes[0].VolumeSource.Projected.Sources[0].Secret.DefaultMode = defaultMode pod.Spec.Volumes[0].VolumeSource.Projected.DefaultMode = defaultMode } else { mode := int32(0644) defaultMode = &mode } if fsGroup != nil || uid != nil { pod.Spec.SecurityContext = &v1.PodSecurityContext{ FSGroup:   fsGroup, RunAsUser: uid, } } modeString := fmt.Sprintf("%v", os.FileMode(*defaultMode)) expectedOutput := []string{ "content of file "/etc/projected-secret-volume/data-1": value-1", "mode of file "/etc/projected-secret-volume/data-1": " + modeString, } f.TestContainerOutput("consume secrets", pod, 0, expectedOutput) } func doProjectedSecretE2EWithMapping(f *framework.Framework, mode *int32) { var ( name            = "projected-secret-test-map-" + string(uuid.NewUUID()) volumeName      = "projected-secret-volume" volumeMountPath = "/etc/projected-secret-volume" secret          = secretForTest(f.Namespace.Name, name) ) By(fmt.Sprintf("Creating projection with secret that has name %s", secret.Name)) var err error if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil { framework.Failf("unable to create test secret %s: %v", secret.Name, err) } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-projected-secrets-" + string(uuid.NewUUID()), }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: volumeName, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { Secret: &v1.SecretProjection{ LocalObjectReference: v1.LocalObjectReference{ Name: name, }, Items: []v1.KeyToPath{ { Key:  "data-1", Path: "new-path-data-1", }, }, }, }, }, }, }, }, }, Containers: []v1.Container{ { Name:  "projected-secret-volume-test", Image: imageutils.GetE2EImage(imageutils.Mounttest), Args: []string{ "--file_content=/etc/projected-secret-volume/new-path-data-1", "--file_mode=/etc/projected-secret-volume/new-path-data-1"}, VolumeMounts: []v1.VolumeMount{ { Name:      volumeName, MountPath: volumeMountPath, }, }, }, }, RestartPolicy: v1.RestartPolicyNever, }, } if mode != nil { //pod.Spec.Volumes[0].VolumeSource.Projected.Sources[0].Secret.Items[0].Mode = mode pod.Spec.Volumes[0].VolumeSource.Projected.DefaultMode = mode } else { defaultItemMode := int32(0644) mode = &defaultItemMode } modeString := fmt.Sprintf("%v", os.FileMode(*mode)) expectedOutput := []string{ "content of file "/etc/projected-secret-volume/new-path-data-1": value-1", "mode of file "/etc/projected-secret-volume/new-path-data-1": " + modeString, } f.TestContainerOutput("consume secrets", pod, 0, expectedOutput) } </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Running the non- conformance tests with produces unpredictable results, sometimes tests show up again with a on the end. This is due to the following tests having multiple tests with identical names: - - What you expected to happen: Tests should have unique names. How to reproduce it (as minimally and precisely as possible): run the conformance tests with a skip on and Anything else we need to know?: example here: https://k8s- /sig node /sig testing cc
er this is probably better /sig storage cc
/assign /assign
/area conformance
/milestone v1.13


Question: } } <ins> func TestSingularObjectNoExtension(t *testing.T) { obj, err := NewBuilder(latest.RESTMapper, api.Scheme, fakeClient()). NamespaceParam("test").DefaultNamespace(). FilenameParam(false, "../../../examples/pod"). Flatten(). Do().Object() if err != nil { t.Fatalf("unexpected error: %v", err) } pod, ok := obj.(*api.Pod) if !ok { t.Fatalf("unexpected object: %#v", obj) } if pod.Name != "nginx" || pod.Namespace != "test" { t.Errorf("unexpected pod: %#v", pod) } } </ins> func TestSingularRootScopedObject(t *testing.T) { node := &api.Node{ObjectMeta: api.ObjectMeta{Name: "test"}, Spec: api.NodeSpec{ExternalID: "test"}} r := streamTestObject(node)

It works when I pipe but not with filename.
broke in this commit cc
Did you try to add an extension .json to the file? We filter for file extensions
ahhh ok. so working as intended. it used to autodetect :( ... thanks
That was unintended. The file extensions should only be used for deciding which files in a directory to process. For a single file, any extension should be allowed. We don't actually infer the content type from the extensions. For stdin and http sources, for instance, there are no extensions. cc
Fix is pretty straightforward (ignore file extensions when dealing with single file); I have it working with manual test, just adding unit test. Will send shortly.
Thanks, but this is v1.0-post, so no rush


Question: // test when the pod anti affinity rule is not satisfied, the pod would stay pending. It("validates that InterPodAntiAffinity is respected if matching 2", func() { <del> // launch a pod to find a node which can launch a pod. We intentionally do // not just take the node list and choose the first of them. Depending on the // cluster and the scheduler it might be that a "normal" pod cannot be // scheduled onto it. By("Trying to launch a pod with a label to get a node which can launch it.") </del> <ins> // launch pods to find nodes which can launch a pod. We intentionally do // not just take the node list and choose the first and the second of them. // Depending on the cluster and the scheduler it might be that a "normal" pod // cannot be scheduled onto it. By("Launching two pods on two distinct nodes to get two node names") CreateHostPortPods(f, "host-port", 2, true) defer framework.DeleteRC(f.Client, f.Namespace.Name, "host-port") podList, err := c.Pods(ns).List(api.ListOptions{}) ExpectNoError(err) Expect(len(podList.Items)).To(Equal(2)) nodeNames := []string{podList.Items[0].Spec.NodeName, podList.Items[1].Spec.NodeName} Expect(nodeNames[0]).ToNot(Equal(nodeNames[1])) By("Applying a random label to both nodes.") k := "e2e.inter-pod-affinity.kubernetes.io/zone" v := "china-e2etest" for _, nodeName := range nodeNames { framework.AddOrUpdateLabelOnNode(c, nodeName, k, v) framework.ExpectNodeHasLabel(c, nodeName, k, v) defer framework.RemoveLabelOffNode(c, nodeName, k) } By("Trying to launch another pod on the first node with the service label.") </ins> podName := "with-label-" + string(uuid.NewUUID()) pod, err := c.Pods(ns).Create(&api.Pod{ TypeMeta: unversioned.TypeMeta{

https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
Looking into this...
trying to track this flake down. It looks like the pod is launched despite AntiAffinity in . In my local setup I saw the same, but noticed that the new admission controller was not activate.

I have activate it and now get this:
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
https://k8s- Failed: [] SchedulerPredicates [Serial] validates that InterPodAntiAffinity is respected if matching 2 {Kubernetes e2e suite}
Should be fixed by


Question: ) require ( <ins> github.com/emicklei/go-restful/v3 v3.11.0 // indirect </ins> github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect

We're trying to upgrade to () and found the issue on executing (). The problem is that on () missed dependency that not allow to install () No error To reproduce the same what happening on , just execute this command on code-generator folder: I think it's happens after this commit <details</details<details</details<details</details<details</details<details</details<details</details
/cc
/assign
/sig api-machinery
I was able to reproduce, but just because I had an older version of kube-openapi in my GOPATH. Using the prefix in the command fixed it. I think line needs to be
Yeah, with works for me too.
Either that or we pin to a more recent commit, for example . For some reason if you ommit the version it is using a from Feb 27.
thanks for investigating. Question, if we use would this not cause problems in the future if we try to build a release branch from the past? Maybe this is not a big deal. OTOH if we hardcode a tag, it may create maintenance costs. I'm wondering if there is a better way, say to version and use the corresponding tag, say ?
Using tags would be a way to have more reproducible builds, they still need to be tagged from time to time tho. That is still better than using or micro managing commit revisions.
Thanks and for the research. Is either of you thinking on submitting a fix for this? Thanks!
I can do it.
Thanks! /assign mbobrovskyi
Yes, please don't use Generally instead we'd control the version of a tool in the hack/tools module and install it from there (see e.g.
This allows us to also gain for the package while not polluting the main module with dev-only dependencies.
Not sure if we need to make another Because it's not just a dev-only dependency. I propose to add it to of code-generator.
This is about semversioning
/triage accepted


Question: func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.PerTestConfig, func()) { // Create secondary namespace which will be used for creating driver driverNamespace := utils.CreateDriverNamespace(f) <del> ns2 := driverNamespace.Name ns1 := f.Namespace.Name </del> <ins> driverns := driverNamespace.Name testns := f.Namespace.Name </ins> ginkgo.By("deploying csi mock driver") cancelLogging := utils.StartPodLogs(f, driverNamespace)

Tracking bug for /sig storage /important-longterm /assign
/sig storage /triage accepted
/priority important-longterm


Question: "time" "k8s.io/api/core/v1" <del> "k8s.io/apimachinery/pkg/api/errors" </del> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels"

SchedulerPredicates [Serial] validates that a pod with an invalid NodeAffinity is rejected Has been failing consistently since 4/17. Here are the /kind bug /priority failing-test /priority important-soon /sig scheduling /milestone 1.11 cc /assign for triage.
The provided milestone is not valid for this repository. Milestones in this repository: [, , , , , , , , , , ] Use to clear the milestone. <detailsIn response to : ## Failing Job SchedulerPredicates [Serial] validates that a pod with an invalid NodeAffinity is rejected /priority failing-test /priority important-soon /sig scheduling /milestone 1.11 Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/milestone v1.11
/status approved for milestone
Also fails as part of upgrade job as seen below
Also fails as part of following jobs https://k8s- https://k8s- https://k8s- https://k8s-
xref
/assign
[MILESTONENOTIFIER] Milestone Issue Needs Approval Action required: This issue must have the label applied by a SIG maintainer. If the label is not applied within 6 days, the issue will be moved out of the v1.11 milestone. <details<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
Seems like this is failing in (just with slightly different versions)


Question: func (g *dockerConfigKeyProvider) Provide() credentialprovider.DockerConfig { // Read the contents of the google-dockercfg metadata key and // parse them as an alternate .dockercfg <del> if cfg, err := credentialprovider.ReadDockerConfigFileFromUrl(dockerConfigKey, g.Client, metadataHeader); err == nil { </del> <ins> if cfg, err := credentialprovider.ReadDockerConfigFileFromUrl(dockerConfigKey, g.Client, metadataHeader); err != nil { glog.Errorf("while reading 'google-dockercfg' metadata: %v", err) } else { </ins> return cfg }

It looks like we always try and hit the metadata server even if we aren't on GCE. We then print an error out. This should either be silent (assume after X attempts that we aren't on GCE?) or should only be activated with an option. Email thread: !topic/google-containers/w-ei5Xs6K0Y
cc:
FYI, the way the credentialprovider logic is set up, providers are asked once per keyring creation whether they are enabled. If they are enabled, then they are asked to provide their contribution to .dockercfg each time the lazy keyring is accessed. For efficiency, there is a caching provider, which composes with heavier providers and stores their .dockercfg contribution for a predetermined TTL. What was happening here was that "Enabled()" for our GCE-metadata implementations was logging about errors fetching "http://metadata", where the entire "Enabled()" check is "err == nil".
I believe that this was fixed by


Question: zero := int64(0) <ins> // Some distributions (Ubuntu 16.04 etc.) don't support the proc file. _, err := framework.IssueSSHCommandWithResult( "ls /proc/net/nf_conntrack", framework.TestContext.Provider, clientNodeInfo.node) if err != nil && strings.Contains(err.Error(), "No such file or directory") { framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name) } framework.ExpectNoError(err) </ins> clientPodSpec := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "e2e-net-client",

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: e2e test "[sig-network] Network [It] should set TCP CLOSEWAIT timeout" requires the test target OS supports /proc/net/nfconntrack, but Ubuntu 16.04 doesn't support it. Then the test fails today like the following: What you expected to happen: The test succeeded or was skipped. How to reproduce it (as minimally and precisely as possible): Just kick Anything else we need to know?: The existence of /proc/net/nfconntrack depends on the Linux kernel config CONFIGNFCONNTRACKPROCFS, but that is disabled on Ubuntu 16.04. In addition, the test is skipped if the node number is less than 2. So I guess that is one of reasons this issue was not present until today. Environment: Kubernetes version (use ): v1.11.3 Cloud provider or hardware configuration: OpenStack OS (e.g. from /etc/os-release): Ubuntu 16.04.4 LTS Kernel (e.g. ): 4.4.0-134-generic Install tools: kubeadm Others:
/sig network
Some kube-proxy functionalities depends on conntrack, is it possible to run a conformance cluster without having conntrack enable?


Question: <ins> /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1beta2 import ( "fmt" apps "k8s.io/api/apps/v1beta2" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) // StatefulSetListerExpansion allows custom methods to be added to // StatefulSetLister. type StatefulSetListerExpansion interface { GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error) } // StatefulSetNamespaceListerExpansion allows custom methods to be added to // StatefulSetNamespaceLister. type StatefulSetNamespaceListerExpansion interface{} // GetPodStatefulSets returns a list of StatefulSets that potentially match a pod. // Only the one specified in the Pod's ControllerRef will actually manage it. // Returns an error only if no matching StatefulSets are found. func (s *statefulSetLister) GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error) { var selector labels.Selector var ps *apps.StatefulSet if len(pod.Labels) == 0 { return nil, fmt.Errorf("no StatefulSets found for pod %v because it has no labels", pod.Name) } list, err := s.StatefulSets(pod.Namespace).List(labels.Everything()) if err != nil { return nil, err } var psList []*apps.StatefulSet for i := range list { ps = list[i] if ps.Namespace != pod.Namespace { continue } selector, err = metav1.LabelSelectorAsSelector(ps.Spec.Selector) if err != nil { return nil, fmt.Errorf("invalid selector: %v", err) } // If a StatefulSet with a nil or empty selector creeps in, it should match nothing, not everything. if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) { continue } psList = append(psList, ps) } if len(psList) == 0 { return nil, fmt.Errorf("could not find StatefulSet for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels) } return psList, nil } </ins>

As a prerequisite of moving the , , , and controllers to use the apps/v1beta2 APIs, we need to add list expansions for , , , and to .
/assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig apps


Question: genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/storage/names" "k8s.io/kubernetes/pkg/api" <ins> "k8s.io/kubernetes/pkg/api/pod" </ins> "k8s.io/kubernetes/pkg/apis/settings" "k8s.io/kubernetes/pkg/apis/settings/validation" )

says "Before persisting the object to storage, clear disabled alpha fields". There is that actually implements it but there are some discussions about it. This issues is a reminder that introduced by needs to be cleared too when the discussion in settles down. /assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig api-machinery
kind/cleanup priority/important-soon
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/api-machinery: Issue will be escalated to these SIGs if needed. priority/important-soon: Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</a</details


Question: import ( "flag" "fmt" <ins> "io/ioutil" </ins> "os" "time" <ins> "github.com/golang/glog" </ins> "github.com/onsi/ginkgo/config" "github.com/spf13/viper" <ins> restclient "k8s.io/client-go/rest" </ins> "k8s.io/client-go/tools/clientcmd" <ins> clientcmdapi "k8s.io/client-go/tools/clientcmd/api" </ins> "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" "k8s.io/kubernetes/pkg/kubemark"

/kind feature I am running self-hosted E2E conformance tests in a pod on every freshly deployed cluster (similar as in ). is started without any host, kubeconfig configuration and picks up the InClusterConfig correctly. All conformance tests pass nicely, except from kubectl executions. kubectl gets executed by as and therefore fails on connection refused. In the container running kubectl without --server, will work as expected, since I have service accounts set up correctly. Looking into code, for some reason defaultHost is set to http://127.0.0.1:8080 ( ). That value is not used within as it seems, but is passed on to kubectl, only. Omitting would let the kubectl run with no problem, why is it set to that value (default proxy port is 8001)? Currently I am mitigating by pre-exectuting before running
/sig Testing
This is known, and there is a means to circumvent by passing in a kubeconfig. I've had a todo to fix this for a while, but I'll prioritize to get this fixed sooner. fyi -
can you please leave some notes on how you wanted to fix this? i can take a shot at it
if you can write the in-cluster config out and set KUBECONFIG prior to the kubectl tests that would help.
: ack. : Can you please send me the list of tests that fails? So i can be sure my changes will work for you?
- reviewing the code paths PR should do the trick, but would like to get that list of failed tests so i can confirm. Thanks, Dims
every conformance test that's shelling out kubectl is failing (20-some of the 150 conformance tests). as elegant as your PR looks, I think it won't work, due to authentication: kubectl will work out of the box on a pod with InClusterConfig, however, as soon as you start specifying one single parameter (such as --server), even if you do not change it compared to InClusterConfig, you will need to specify the rest, too. There might be two ways to do it: write out a kubeconfig file and reference it omit when shelling out to kubectl IMHO, the second option is much more elegant, but would need to test to confirm it works.

e2e AND kubectl work with InClusterConfig by themselves, why not leave it that way and only change anything, iff e2e gets any parameters (-host) specified. Otherwise, InClusterConfig should work as designed.
can you give me the list of the 20 tests please?
Sure, rerunning the tests, might take an hour.
Here's the first one: The rest will follow as a list.
It is a 1.7.2 test running against a 1.7.7 cluster.
See, there are three variants on how to invoke kubectl: Only the third one gets to the API and and gets authorized. If I had a kubeconfig file, that would also work.
Excellent! thanks will take a look at the scenarios. thanks for digging in.
/assign
[MILESTONENOTIFIER] Milestone Issue Current <details open<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details


Question: import ( goflag "flag" <ins> "fmt" </ins> "strings" "github.com/spf13/pflag"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --, is not assigned is assigned Use to parse the command line parameters I am preparing to merge pr to solve this problem Kubernetes version (use ): N/A Cloud provider or hardware configuration: N/A OS (e.g: ): N/A Kernel (e.g. ): N/A Install tools: N/A Network plugin and version (if this is a network-related bug): N/A Others: N/A
/sig cloud-provider /area provider/gcp
/area test
how about 'flags.VarP('
The pointer of 's values is assigned to the new address, and the relationship with the input parameter is broken. The 'flags.VarP(' also to no avail
/assign /triage accepted


Question: expectedGVK = *testData.ExpectedGVK } <ins> // if previous releases had a non-alpha version of this group/kind, make sure the storage version is understood by a previous release fixtureFilenameGroup := expectedGVK.Group if fixtureFilenameGroup == "" { fixtureFilenameGroup = "core" } // find all versions of this group/kind in all versions of the serialization fixture testdata previousReleaseGroupKindFiles, err := filepath.Glob("../../../staging/src/k8s.io/api/testdata/*/" + fixtureFilenameGroup + ".*." + expectedGVK.Kind + ".yaml") if err != nil { t.Error(err) } if len(previousReleaseGroupKindFiles) == 0 && !allowMissingTestdataFixtures[expectedGVK] { // We should at least find the HEAD fixtures t.Errorf("No testdata serialization files found for %#v, cannot determine if previous releases could read this group/kind. Add this group-version to k8s.io/api/roundtrip_test.go", expectedGVK) } // find non-alpha versions of this group/kind understood by previous releases previousNonAlphaVersions := sets.NewString() for _, previousReleaseGroupKindFile := range previousReleaseGroupKindFiles { if serverVersion := filepath.Base(filepath.Dir(previousReleaseGroupKindFile)); serverVersion == "HEAD" { continue } parts := strings.Split(filepath.Base(previousReleaseGroupKindFile), ".") version := parts[len(parts)-3] if !strings.Contains(version, "alpha") { previousNonAlphaVersions.Insert(version) } } if len(previousNonAlphaVersions) > 0 && !previousNonAlphaVersions.Has(expectedGVK.Version) { t.Errorf("Previous releases understand non-alpha versions %q, but do not understand the expected current storage version %q. "+ "This means a current server will store data in etcd that is not understood by a previous version.", previousNonAlphaVersions.List(), expectedGVK.Version, ) } </ins> actualGVK := output.getGVK() if actualGVK != expectedGVK { t.Errorf("GVK for %s does not match, expected %s got %s", kind, expectedGVK, actualGVK)

Improve the to ensure that the expected storage version for a resource is one understood by previous releases. We have API testdata from prior versions available in When checking each group/version/kind in the test: accumulate all the versions of the group/kind available in previous (non-HEAD) releases if there is a beta or stable version of the group/kind available in previous releases, then the storage version must be in the set of previously available versions It is too easy to accidentally change the storage version prematurely and edit the test/integration/etcd test to match the bug cc
There are no sig labels on this issue. Please add an appropriate label by using one of the following commands: - Please see the for a listing of the SIGs, working groups, and committees available. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: period5Name, periodC5Name, <del> period5Name, </del> <ins> periodB5Name, </ins> }, { period5Name, periodC5Name, <del> period5Name, </del> <ins> periodB5Name, </ins> periodA5Name, }, { period5Name, periodC5Name, <del> period5Name, </del> <ins> periodB5Name, </ins> periodA5Name, periodCritical5Name, },

/kind failing-test <!-- If this is a flake, please add: /kind flake --/sig node
/triage accepted
/assign
This appears to be due to timeout being reached and the kubelet not updating the status of the terminated pod in time before the timeout is hit.
Testgrid: and


Question: metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt" <ins> metadata="${metadata},kube-master-internal-route=${KUBE_ROOT}/cluster/gce/gci/kube-master-internal-route.sh" </ins> metadata="${metadata},${MASTER_EXTRA_METADATA}" local disk="name=${master_name}-pd"

If you create a private cluster via kube-up using provider GCE, you won't be able to restart your master nodes. If you restart you master, the nodes won't be able to connect to your master. This is due to this line - It gets executed on master on cluster-up, but it won't be re-executed on master restart. We should fix that, especially if we're to implement /sig scalability /priority important-soon
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale /good-first-issue I'll ask whether someone in SIG Scalability public meeting would like to take on this one
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /remove-lifecycle stale /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hello, Can I start work on this one as my first issue? :)
I don't see anyone else claiming it, so your contributions are welcome :)
let me know if you need any explanations here
Thank you :) I've made some changes and assign you can you tell me is this will be good approach?
Thanks, I replied in the PR


Question: package kuberuntime import ( <del> "runtime" </del> v1 "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" <ins> "k8s.io/kubernetes/pkg/kubelet/winstats" </ins> "k8s.io/kubernetes/pkg/securitycontext" )

When submitting a windows pod using resource limits on a node with more than one processor group, the specified count is incorrect due to the usage of which does not take processor groups into account. For example: node with 96 cores cpu limit 94 returns This results in a CPU maximum of since the range is from [1, ]: Our initial goal for this was to limit a process to not use all available CPU and therefore render the node uncontrollable. A CPU maximum of instead of for that specific case. For the calculation, it should use the CPU count for all available processor groups instead of just the current one. Launch a node with a lot of CPUs, for example and schedule the following pod on it: Then SSH into the node and inspect the running container using . The value will be set to . (which just uses ) is able to correctly find out the number of CPUs available to a system. Using for calculation or incorporating that code might solve this issue. <details</details<detailsAzure in-tree </details<details</details<details</details<detailscontainerd://1.6.10 </details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig windows
There is already some code which uses GetActiveProcessorCount:
/assign
/triage accepted


Question: By("Deleting second pod") framework.DeletePodOrFail(config.client, config.ns, pod2.Name) }) <del> It("should not set different fsGroups for two pods simultaneously", func() { fsGroup1, fsGroup2 := int64(1234), int64(4321) By("Create first pod and check fsGroup is set") pod1 := createPodWithFsGroupTest(config, testVol, fsGroup1, fsGroup1) By("Create second pod and check fsGroup is still the old one") pod2 := createPodWithFsGroupTest(config, testVol, fsGroup2, fsGroup1) ep := &eventPatterns{ reason:  "AlreadyMountedVolume", pattern: make([]string, 2), } ep.pattern = append(ep.pattern, fmt.Sprintf("The requested fsGroup is %d", fsGroup2)) ep.pattern = append(ep.pattern, "The volume may not be shareable.") checkPodEvents(config, pod2.Name, ep) By("Deleting first pod") framework.DeletePodOrFail(config.client, config.ns, pod1.Name) By("Deleting second pod") framework.DeletePodOrFail(config.client, config.ns, pod2.Name) }) </del> }) })

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Based on discussion , events can be throttled and should not be used to validate failure conditions in e2e tests. We should remove the event checks from subpath and local pv tests and just wait for pod failure. <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/assign
could you help look at this for local PV tests?
sure /assign
/area deflake
"Invalid AWS KMS key" also waits for an event

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: fsgroup tests are flaking 1-2 times a day: https://k8s- Some test logs: https://k8s- https://k8s-
could you take a look?
I'll look into this.
I found why the tests failed, see my analysis of logs: I created a PR () trying to fix the issue. I am still investigating kubelet and trying to reproduce the issue locally.
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
/lifecycle frozen


Question: type UnstructuredCopier struct{} func (UnstructuredCopier) Copy(obj runtime.Object) (runtime.Object, error) { <ins> if _, ok := obj.(runtime.Unstructured); !ok { // Callers should not use this UnstructuredCopier for things other than Unstructured. // If they do, the copy they get back will become Unstructured, which can lead to // difficult-to-debug errors downstream. To make such errors more obvious, // we explicitly reject anything that isn't Unstructured. return nil, fmt.Errorf("UnstructuredCopier can't copy type %T", obj) } </ins> // serialize and deserialize to ensure a clean copy buf := &bytes.Buffer{} err := unstructured.UnstructuredJSONScheme.Encode(obj, buf)

BUG REPORT Kubernetes version (use ): v1.6.2 Environment: Cloud provider or hardware configuration: GKE What happened: Create TPR "MyResource" Create namespace "soak-test" Create a "MyResource" "a" under "soak-test":Delete namespace "soak-test"wait namespace to be deleted MyResource "a" still exists after ns is deleted.
/area third-party-resource
/sig cli
I'd like to take a look at the problem if nobody else is working on this
Root Cause of this is the fact that Namespace on ThirdPartyResources is getting ignored (both passed as command --namespace, or the one present in yaml) ThirdPartyResources always end up in default namespace (which is causing confusion reported in bug)
How long has that bug existed? might related to your other findings. We know that bug doesn't exist in CRDs (we have tests specifically checking namespace handling), so I'd suggest converting to those as soon as possible.
Isn't CRD available in 1.7+ ?
Yes. If the bug has existed for multiple releases (not a regression), I don't think this is a blocker for 1.7. It could still be patched, just not a blocker for the current release.
Migration to CRD still is not automated and will still take some time. Have you considered that?
Yes, but if the behavior is not a regression, then this does not block the release. The fact that we have a clear path forward to a working resource is gravy.
This bug existed in 1.4 (), got fixed in 1.5, but appears again in 1.6 .
Ok. Not a blocker for 1.7, but if the patch is reasonable as a fix we can merge it into the service stream. For 1.7, I'd still recommend moving to CRD as quickly as possible.
I was able to reproduce this with CRD on master HEAD. What level of tests are there? Anything e2e?
most of them and specifically a storage one. You're saying we have the from above?
No, I don't see any evidence that they always end up in the default namespace (neither for TPR nor CRD). I'm not sure what that's referring to.

What I was able to reproduce is the bug described at the top of this issue: Create item in namespace, delete namespace, item remains. I observed this for TPR on and , and for CRD on .
The problem seems to be that is misspelled as in this line: If I fix that, namespace cleanup works for TPR. However, for CRD, now namespace cleanup gets stuck and controller manager logs this: Edit: To clarify, I "fixed" the check in the linked code by removing the entire highlighted block. It seems like the singular is not the only problem. It was also using the internal version of when was expected.
Regarding namespace cleanup getting stuck for CRD after the "fix", the error on the apiserver side is: According to the stack trace, the error comes from here:
yes, not only the is misspelled, but also the mismatch of the version. this is why in my original patch, I tried to ignore the version of .
I'd like to keep this open until we have a regression test since this is at least the third time we've fixed this. I'll add a test as part of making sure this fix works for CRD.


Question: deleteFlags.AddFlags(cmd) <del> cmd.Flags().Bool("wait", true, `If true, wait for resources to be gone before returning.  This waits for finalizers.`) </del> cmdutil.AddIncludeUninitializedFlag(cmd) return cmd }

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /sig CLI What happened: Running leads to nil pointer dereference: What you expected to happen: is expected to run delete with wait in that case How to reproduce it (as minimally and precisely as possible): (with any valid resource yaml) Anything else we need to know?: Discovered in when accidentally enabled wait for deletion from other commands than and saw failing integration tests in https://k8s- Environment: Kubernetes version (use ): latest master Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
This bug was introduced in : is set in that requires to be non-nil flag is being set in is normally being set in method which is not invoked in . a result, the invocation of resource finder leads to nil pointer dereference (see stack trace in the description).
/cc


Question: ${GCE_PARALLEL_SKIP_TESTS[@]:+${GCE_PARALLEL_SKIP_TESTS[@]}}  ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}}  ${GCE_PARALLEL_FLAKY_TESTS[@]:+${GCE_PARALLEL_FLAKY_TESTS[@]}}  <ins> ${AWS_REQUIRED_SKIP_TESTS[@]:+${AWS_REQUIRED_SKIP_TESTS[@]}}  </ins> )"} : ${ENABLE_DEPLOYMENTS:=true} # Override AWS defaults.

For example: I'm disabling the test in But we need to re-enable it, or roll-back the test.
fyi
Shouldn't this is part slow test, and the fix should be either: 1) Increase Kubernetes-e2-gke-ci timeout value if decide to include slow to that project or 2) disable all slow tests on that project? cc/ Which one is required here?
The test runs 50 pods per node, which is over the default --max-pod limit of kubelet. The test relies on the flag --max-pods=100 being set in cluster/gce/config- I will disable it for all GKE tests in On the other hand, it's not clear to me where the timeout is set for a specific build. /cc
Timeouts for each Jenkins build are in the Jenkins project configurations themselves, which are not currently saved in version control anywhere. :
Discussed with about the test offline. There are several issues: Do we want to run any kind of kubelet resource monitoring test on GKE? Is it useful? We only override max-pods in GCE. Should the test itself figure out what platform it can run based on the #pods in the test (see ) Per-test configuration allows greater flexibility (e.g., we may run 100 pods on GCE to test kubelet's scalability), but it'd make the tests more complicated as they need to be aware of the test environment (which many e2e tests already do); there is also the burden of updating the tests once the max-pod value changes. Does this test belong to SLOW tests or should it belong to a new category? Not all slow tests have the same attributes. Arguably this test could be enabled only on selective builds. The downside would be more flags -- less readable/manageable e2e configs. Not all builds which run SLOW tests have the proper timeout value to run this test. We need to manually change them every time a slow test is . Re-assigning this back to since she has more concerns about the PR.
Had offline discussion with We are going to split current Kubelet resource usage tracking test into two, one is targeted for GCE, another one is for GKE. In this case, we won't mess up the test code itself by introducing knobs. Re-assign this back to


Question: Pod and the service must now have empty set of endpoints. release: v1.9 file: test/e2e/network/service.go <ins> - testname: Endpoint resource lifecycle codename: '[sig-network] Services should test the lifecycle of an Endpoint [Conformance]' description: Create an endpoint, the endpoint MUST exist. The endpoint is updated with a new label, a check after the update MUST find the changes. The endpoint is then patched with a new IPv4 address and port, a check after the patch MUST the changes. The endpoint is deleted by it's label, a watch listens for the deleted watch event. release: v1.19 file: test/e2e/network/service.go </ins> - testname: ConfigMap, from environment field codename: '[sig-node] ConfigMap should be consumable via environment variable [NodeConformance] [Conformance]'

[x] Original approval issue: [x] Two weeks soak end date: This issue is created to allow edit by /wip /hold This issue proposes the writing of a new test for the API endponts , , , with the intention of promoting it to Conformance. This issue contains a very basic mock test in order to start the conversation about what a good e2e test would look like for this endpoint. The code below is verified to be hitting the intended endpoint, show as per the queries in APIsnoop's live view of the cluster's audit logs. According to this APIsnoop query, there are still some remaining Endpoints endpoints which are untested. a Service in a namespace all Endpoints in all Namespaces find the created Service(1) /update the Namespaced Endpoint for the Service(1) the Namespaced Endpoint for the Service(1) the Namespaced Endpoint for the Service(1) Namespaced Endpoint(1) via a Collection with a LabelSelector Discover useragents: If a test with these calls gets merged, Conformance coverage will go up by 4 points* /sig testing /sig architecture /area conformance
/reopen Wait for Promotion PR to Merge.
Reopened this issue. <detailsIn response to : /reopen Wait for Promotion PR to Merge. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: // DefaultClusterDNSIP defines default DNS IP DefaultClusterDNSIP = "10.96.0.10" // DefaultKubernetesVersion defines default kubernetes version <del> DefaultKubernetesVersion = "stable-1.9" </del> <ins> DefaultKubernetesVersion = "stable-1.10" </ins> // DefaultAPIBindPort defines default API port DefaultAPIBindPort = 6443 // DefaultAuthorizationModes defines default authorization modes

In following with history we will be curating action items here, but it's still a WIP, atm: All non-bug-fixes for this release cycle should be merged, but there is still an ongoing list of bugs that folks may execute against for the remainder of the freeze and we will evaluating on a case-by-case basis. /cc
Thank you for creating this issue!
/cc
Adding required tags. Please use this issue to supply a frequent summary update of the issues you are tracking in the kubeadm repo and their status. Especially, please call out and link to any bugs that are blockers for 1.10. /kind feature /priority critical-urgent
can we have a general status update on the kubeadm issues? The issues themselves don't generally have status updates. This is the last week before release, where are you with the kubeadm changes?
We're all green except for the 1.9-1.10 upgrade tests which appear to have other issues that are not kubeadm specific. Final default update PR is which will auto-close this issue.
^ I don't know why we have this policy but it's ridiculous...
are there any 1.9-1.10 upgrade issues which aren't already represented by issues against kubernetes/kubernetes? that is, are there any in the kubadm repo only?
Not that I see atm, I still need to triage the inbound issues.
/status in-progress
/status in-progress
[MILESTONENOTIFIER] Milestone Issue: Up-to-date for process Note: This issue is marked as , and must be updated every 1 day during code freeze. Example update: <details open<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : New functionality. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details


Question: work := e.work[ns] delete(e.work, ns) delete(e.dirtyWork, ns) <del> if len(work) != 0 { e.inProgress.Insert(ns) return ns, work, false } e.queue.Done(ns) e.inProgress.Delete(ns) return ns, []*admissionWaiter{}, false </del> <ins> e.inProgress.Insert(ns) return ns, work, false </ins> } // prettyPrint formats a resource list for usage in errors

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: I wrote a simple driver program () that creates ConfigMap objects as fast as it can, and ran it like this: . In the CSV file produced, two of the creates recorded an error that simply says "timeout": Here is an excerpt from the api-server's log: A few minutes later I checked that my etcd cluster is healthy; it was: I also looked at the on each host for etcd, and each reported that etcd had been running for over a day. What you expected to happen: I expected no such problem, or at least a more informative error message. How to reproduce it (as minimally and precisely as possible): That test driver is pretty minimal. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: SoftLayer bare metal machines, SSD storage on the etcd servers OS (e.g. from /etc/os-release): Ubuntu 16.04.4 LTS Kernel (e.g. ): Linux mjs-api-dal10-b 4.4.0-122-generic -Ubuntu SMP Mon Apr 23 15:34:04 UTC 2018 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm driven by an Ansible playbook I wrote, based on Others:
That's weird, I'm not aware of a 10s timeout anywhere (there is a 60s one). does the etcd client have such a timeout?
FYI:
- can you try etcd without SSL/TLS? it could be ( https://k8s-*&repos=kubernetes )
Yes, I will try that, the next time it is convenient to tear everything down and start over.
OK, I tried again with etcd NOT using TLS (neither client/server nor server/server). I still got a "timeout" error. The driver at commit .
I am running kube release 1.10.3 now. I turned v to 5 in the api-server driven by the client, and ran another trial.

Here are the errors logged by the driver (with timestamps in UTC): I found this in the stderr of the api-server (the timestamp seems to be in UTC):
The etcd cluster should have no trouble with that write rate.
Around that time, the etcd server that was loaded by the kube api-server logged the following (in US CDT):
try tweaking and ?
I to the loaded api-server's command line, and still got 10s timeouts.
The doc string for seems to say it is irrelevant for create operations.
However, making the client specify a timeout of 17 sec (see ) warded off the problem --- according to my first and only trial so far.
However, my second trial was much more interesting. The api-server logged an error on create --- but the client did not! And the subsequent update and delete operations on the object in question got 404s. Here is what the driver logged about that object: Here is what the api-server logged around that time: The etcd leader logged nothing unusual around that time.
has suggested the mysterious timeout might be due to . I do indeed have the admission controller enabled. However, I have no ResourceQuota objects in any namespace.
I tried removing from the admission controller list, and that made the mysterious 10 second timeouts go away. Why is the ResourceQuota admission controller causing these 10 second timeouts? Remember, they are rare: I can do hundreds of thousands of create operations in rapid fire, and get a single digit number of timeouts, while the other create operations complete in well under 10 seconds. See for an example.
Here is another example (now with and the etcd cluster using TLS): <img width="578" alt="screen shot 2018-05-25 at 1 29 51 pm" src="https://user-"
So I slowed down from 3000 to 2000 kube writes per second, but I still got a timeout. <img width="576" alt="screen shot 2018-05-25 at 2 13 59 pm" src="https://user-"And no, that is not a really tight cluster of data points at (17:35:25.870, 10.036) --- it really is just a single data point. No other operation took anywhere near that long.

Interestingly, there is a roughly 50 ms bubble of latency around the time of the start of the operation that timed out. As you can see from the graph, there are other bubbles of latency. All much smaller than 10 sec. Here are 36 consecutive lines of driver data from around the timeout, sorted by preop time.
Looking in , I see something odd. In two places --- and in --- there is a special case for an empty list of objects. Now, it is not easy to imagine how the list could ever be empty when the namespace is in the queue --- but if this ever comes to pass then something bad happens: is called twice (first in and second in ).
So I made to test the hypothesis that the double-call of is a critical part of the pathology. I ran a few trials, and got no timeouts. Here is the longest:
wanna file a WIP PR so we can see the exact changes?
yeah, when I get a chance. Weekend now.
very nice investigation you've done here!


Question: CompiledVariables map[string]CompilationResult } <del> func (c *CompositionEnv) AddField(name string) { c.MapType.Fields[name] = apiservercel.NewDeclField(name, apiservercel.DynType, true, nil, nil) </del> <ins> func (c *CompositionEnv) AddField(name string, celType *cel.Type) { c.MapType.Fields[name] = apiservercel.NewDeclField(name, convertCelTypeToDeclType(celType), true, nil, nil) </ins> } func NewCompositionEnv(typeName string, baseEnvSet *environment.EnvSet) (*CompositionEnv, error) {

Variables field is to ValidatingAdmissionPolicy in 1.28. Currently the usage of variables would expect a operator associated. e.g. (thanks to Mariam Fahmy for raising the issue) Above will fail with error: Meanwhile the will pass. The reason is when variables be evaluated to non dyn type(like bool or string), it will require special handling(like casting) since we use dyn type everywhere. It might confuse user.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted /sig api-machinery /cc
Tests I'd expect to see verify this behavior: PASS: A variable is of type bool, an expression returns the variable PASS: A variable is of type string, a messageExpression returns the variable PASS: A variable is of type dyn, at runtime the dyn is a bool, an expression returns the variable PASS: A variable is of typedyn, at runtime the dyn is a string, an messageExpression returns the variable PASS: A expression is of type dyn, at runtime the dyn is of type bool PASS: A messageExpression is of type dyn, at runtime the dyn is of type string COMPILE ERROR: A variable is of type bool, an expression tries to perform a non-bool (e.g. string replace) operation on the variable, this should resultsin a compile error COMPILE ERROR: A variable is of type string, an expression tries to perform a non-string (e.g. less than) operation on the variable, this should result in a compile error RUNTIME ERROR: A variable is of type dyn, at runtime the dyn is a bool, an expression tries to perform a non-bool (e.g. string replace) operation on the variable, this should result in a runtime error I'd also expect to see some cases covering variables of other types (maps, lists, duration, quantity, ...) to make sure we're over-fitting for bool and string.
/retitle ValidatingAdmissionPolicy: variable expression type not recognized as return type in expression that evaluates the variable
/assign


Question: klog.V(4).InfoS("Pod was deleted and sandbox failed to be created", "pod", klog.KObj(pod), "podUID", pod.UID) return } <del> metrics.StartedPodsErrorsTotal.WithLabelValues(err.Error()).Inc() </del> <ins> metrics.StartedPodsErrorsTotal.Inc() </ins> createSandboxResult.Fail(kubecontainer.ErrCreatePodSandbox, msg) klog.ErrorS(err, "CreatePodSandbox for pod failed", "pod", klog.KObj(pod)) ref, referr := ref.GetReference(legacyscheme.Scheme, pod)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --While reviewing labels in prometheus noticed this metric whose label value is large (700+ characters) since it was a error log message. Example: The metric is from It doesn't look like we should have error/ log message as a metric Query the metric in prometheus UI or through api where kubelet is one of the target in prometheus Kubernetes version (use ): v1.22.0-rc.0 Cloud provider or hardware configuration: NA OS (e.g: ): NA Kernel (e.g. ): NA Install tools: Network plugin and version (if this is a network-related bug): Others: cc:
I agree that error logs should never be part of metrics, it is not the correct vector to convey this kind of information. Logging would be more appropriated here. /sig instrumentation /sig node /cc
I want to work on this. /assign
/triage accepted
/triage accept
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /triage accept Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: s.Authorization.RemoteKubeConfigFileOptional = true s.Authorization.AlwaysAllowPaths = []string{"/healthz"} <del> s.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" </del> <ins> // Set the PairName but leave certificate directory blank to generate in-memory by default s.SecureServing.ServerCert.CertDirectory = "" </ins> s.SecureServing.ServerCert.PairName = "kube-controller-manager" s.SecureServing.BindPort = ports.KubeControllerManagerPort

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --BUG REPORT: /kind bug What happened: Kubernetes distro testing v1.12.0-rc.2 shows in setups with self-hosted control planes (i.e. apiserver, controller-manager, scheduler run as pods) that previously worked on v1.11.3 and prior, the kube-controller-manager now crash loops. Notably, the kube-controller-manager is run with: which has been fine through to v1.11.3. Full . What you expected to happen: Controller manager may not assume it has root. Or should clarify if an additional mount is now required. May have been introduced in cc How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.12.0-rc.2 Cloud provider or hardware configuration: GCE, DigitalOcean, AWS OS (e.g. from /etc/os-release): Container Linux 1855.4.0 Kernel (e.g. ): 4.14.67-coreos Install tools: Typhoon
Seems like distros like Typhoon, Tectonic, and Tectonic/Openshift may work around this explicitly passing some other path to kube-controller-manager's or creating an emptyDir mount for . Need to understand the expectations around these certs (multiple pods, restarts, etc.) A release note seems needed to cover the new requirements. Appears this was also noticed by juju in
/sig release
cc
WDYT? Can we get folks to write a note to cover this problem in known issues?
I've an mount to address that crashloop, which seems to work preliminary (). was able to proceed with self-signed cert creation and hit a new bug I've filed separately EDIT: Perhaps a suitable release note for this change is:
the cert should not need to be persisted to disk.

I don't think adding a writeable directory requirement just for a self-signed cert is a very good idea /assign
Possible fixes enable by default if no cert is provided, rather than self-signing the self-sign cert in memory only, no disk persistence needed 't enable by default at all 2 would be ideal but is likely too big a change at this point. 3 would be unfortunate. 1 could be a good compromise if the code change is contained.
It's up to the SIG if this is a 1.12.0 blocker, or a Known Issue and 1.12.1 material. I'm inclined to think the latter.
thanks for the clarifications. I would prefer to avoid my current workaround of giving it a writeable location so those all sound fine to me. is there a full story around these controller-manager changes somewhere? I could mint actual certs, mount from a secret, update clients. If there's merit. Like, how big of a deal is it that this is now always enabled vs before? I'll probably read through the various diffs after vacation.
If this is going to get some text in release notes Known Issues (seems like it should), somebody needs to draft the text and make a suggestion of it in ASAP.
/assign
Do we want to fix this for 1.12.1 (on friday)?
Self signing in memory means you get a different cert each time it starts. It's probably better than nothing but it's not how things are supposed to work (it's too inconvenient to manually verify the cert out of band in places users should trust it). OTOH if you want to do that, maybe you'll provide a writable directory in the first place, or even a real cert. So I guess in-memory is probably OK.
Second thought: is a terrible default for the cert path. It should have the word "certs" in the path somewhere. If folks (inadvisably) run apiserver and controller manager on the same system, they should at least not both try to use certs from the same place, or worse, write them there. I suggest as the default.
What components are now verifying these certs or supposed to be? I'd expect something to be failing since random certs are generated each time the pod restarts, but don't see any impact (implying this isn't yet improving security).

Seems like controller-manager just an HTTPS endpoint, but the HTTP one is still what's used right?
I guess that aligns with the comment
The autogenerated certs only have SANs for localhost and 127.0.0.1.
The auto generated certs are self signed, right? If so, and callers won't be able to validate the signer, which SANs they have don't really matter.
Yeah, they're self-signed. So looks like the 1.12 controller-manager serves on HTTP/ (always on) and HTTPS/ (unless --secure-port=0). I guess the point of the original change was so some client could now choose to use the HTTPS endpoint with insecureSkipVerify. Main reason I was interested in the SAN is to get some clue about how a client would be expected to access these controller-manager pods (for contrast, apiserver SAN has DNS:kubernetes and an internal record that always resolves to a kube-apiserve pod).
is this updated for the openstack cloud provider too?
This issue wasn't related to any particular cloud provider. The cloud provider / hardware config field is just a typical part of filing an issue in case it had been related.
I have a writeable (I've tried other writeable directories too.) I'm encountering this: Should I post elsewhere in a new thread, or continue here?


Question: if err != nil { t.Errorf("%s test case: unexpected error: %v", tc.name, err) } <del> expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", tc.origin, tc.dest) + familyParamStr(isIPv6(tc.origin)) </del> <ins> expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", tc.origin, tc.dest) + familyParamStr(IsIPv6String(tc.origin)) </ins> execCommand := strings.Join(fcmd.CombinedOutputLog[i], " ") if expectCommand != execCommand { t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand)

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: There is a bug in ipv6, see: What you expected to happen: The judgement logic should be: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: /sig network /area ipv6
/assign cc


Question: for _, tc := range testCases { result := checkerNames(tc.have...) t.Run(tc.desc, func(t *testing.T) { <del> if reflect.DeepEqual(tc.want, result) { </del> <ins> if !reflect.DeepEqual(tc.want, result) { </ins> t.Errorf("want %#v, got %#v", tc.want, result) } }) } <ins> } func TestFormatQuoted(t *testing.T) { n1 := "n1" n2 := "n2" testCases := []struct { desc     string names    []string expected string }{ {"empty", []string{}, ""}, {"single name", []string{n1}, ""n1""}, {"two names", []string{n1, n2}, ""n1","n2""}, {"two names, reverse order", []string{n2, n1}, ""n2","n1""}, } for _, tc := range testCases { result := formatQuoted(tc.names...) t.Run(tc.desc, func(t *testing.T) { if result != tc.expected { t.Errorf("expected %#v, got %#v", tc.expected, result) } }) } </ins> }

What happened: The logic in the checkerNames test for is faulty. Failing the test when the 'wanted' value equals the actual result is the opposite behavior of what we want. What you expected to happen: I expect the test to fail when the desired output is not the same as the actual output. <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/sig api-machinery /assign


Question: <del> #!/usr/bin/env bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Script to test cluster/update-storage-objects.sh works as expected. set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" # The api version in which objects are currently stored in etcd. KUBE_OLD_API_VERSION=${KUBE_OLD_API_VERSION:-"v1"} # The api version in which our etcd objects should be converted to. # The new api version KUBE_NEW_API_VERSION=${KUBE_NEW_API_VERSION:-"v1"} KUBE_OLD_STORAGE_VERSIONS=${KUBE_OLD_STORAGE_VERSIONS:-""} KUBE_NEW_STORAGE_VERSIONS=${KUBE_NEW_STORAGE_VERSIONS:-""} KUBE_STORAGE_MEDIA_TYPE_JSON="application/json" KUBE_STORAGE_MEDIA_TYPE_PROTOBUF="application/vnd.kubernetes.protobuf" ETCD_HOST=${ETCD_HOST:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-2379} ETCD_PREFIX=${ETCD_PREFIX:-randomPrefix} API_PORT=${API_PORT:-8080} API_HOST=${API_HOST:-127.0.0.1} RUNTIME_CONFIG="" ETCDCTL=$(which etcdctl) KUBECTL="${KUBE_OUTPUT_HOSTBIN}/kubectl" UPDATE_ETCD_OBJECTS_SCRIPT="${KUBE_ROOT}/cluster/update-storage-objects.sh" DISABLE_ADMISSION_PLUGINS="ServiceAccount,NamespaceLifecycle,LimitRanger,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PersistentVolumeLabel,DefaultStorageClass,StorageObjectInUseProtection" function startApiServer() { local storage_versions=${1:-""} local storage_media_type=${2:-""} kube::log::status "Starting kube-apiserver with..." kube::log::status "                        storage-media-type: ${storage_media_type}" kube::log::status "                            runtime-config: ${RUNTIME_CONFIG}" kube::log::status "                 storage-version overrides: ${storage_versions}" "${KUBE_OUTPUT_HOSTBIN}/kube-apiserver"  --insecure-bind-address="${API_HOST}"  --bind-address="${API_HOST}"  --insecure-port="${API_PORT}"  --storage-backend="etcd3"  --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}"  --etcd-prefix="/${ETCD_PREFIX}"  --runtime-config="${RUNTIME_CONFIG}"  --disable-admission-plugins="${DISABLE_ADMISSION_PLUGINS}"  --cert-dir="${TMPDIR:-/tmp/}"  --service-cluster-ip-range="10.0.0.0/24"  --storage-versions="${storage_versions}"  --storage-media-type="${storage_media_type}" 1>&2 & APISERVER_PID=$! # url, prefix, wait, times kube::util::wait_for_url "http://${API_HOST}:${API_PORT}/healthz" "apiserver: " 1 120 } function killApiServer() { kube::log::status "Killing api server" if [[ -n ${APISERVER_PID-} ]]; then kill ${APISERVER_PID} 1>&2 2>/dev/null wait ${APISERVER_PID} || true kube::log::status "api server exited" fi unset APISERVER_PID } function cleanup() { killApiServer kube::etcd::cleanup kube::log::status "Clean up complete" } trap cleanup EXIT SIGINT make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver kube::etcd::start echo "${ETCD_VERSION}" > "${ETCD_DIR}/version.txt" ### BEGIN TEST DEFINITION CUSTOMIZATION ### # source_file,resource,namespace,name,old_version,new_version tests=( "test/e2e/testing-manifests/rbd-storage-class.yaml,storageclasses,,slow,v1beta1,v1" ) KUBE_OLD_API_VERSION="networking.k8s.io/v1,storage.k8s.io/v1beta1,extensions/v1beta1" KUBE_NEW_API_VERSION="networking.k8s.io/v1,storage.k8s.io/v1beta1,storage.k8s.io/v1,extensions/v1beta1,policy/v1beta1" KUBE_OLD_STORAGE_VERSIONS="storage.k8s.io/v1beta1" KUBE_NEW_STORAGE_VERSIONS="storage.k8s.io/v1beta1,storage.k8s.io/v1" ### END TEST DEFINITION CUSTOMIZATION ### ####################################################### # Step 1: Start a server which supports both the old and new api versions, # but KUBE_OLD_API_VERSION is the latest (storage) version. # Additionally use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding. ####################################################### RUNTIME_CONFIG="api/all=false,api/v1=true,apiregistration.k8s.io/v1=true,${KUBE_OLD_API_VERSION}=true,${KUBE_NEW_API_VERSION}=true" startApiServer ${KUBE_OLD_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} # Create object(s) for test in "${tests[@]}"; do IFS=',' read -ra test_data <<<"$test" source_file=${test_data[0]} kube::log::status "Creating ${source_file}" ${KUBECTL} create -f "${KUBE_ROOT}/${source_file}" # Verify that the storage version is the old version resource=${test_data[1]} namespace=${test_data[2]} name=${test_data[3]} old_storage_version=${test_data[4]} if [ -n "${namespace}" ]; then namespace="${namespace}/" fi kube::log::status "Verifying ${resource}/${namespace}${name} has storage version ${old_storage_version} in etcd" ETCDCTL_API=3 ${ETCDCTL} --endpoints="http://${ETCD_HOST}:${ETCD_PORT}" get "/${ETCD_PREFIX}/${resource}/${namespace}${name}" | grep "${old_storage_version}" done killApiServer ####################################################### # Step 2: Start a server which supports both the old and new api versions, # but KUBE_NEW_API_VERSION is the latest (storage) version. # Still use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding. ####################################################### RUNTIME_CONFIG="api/all=false,api/v1=true,apiregistration.k8s.io/v1=true,${KUBE_OLD_API_VERSION}=true,${KUBE_NEW_API_VERSION}=true" startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} # Update etcd objects, so that will now be stored in the new api version. kube::log::status "Updating storage versions in etcd" ${UPDATE_ETCD_OBJECTS_SCRIPT} # Verify that the storage version was changed in etcd for test in "${tests[@]}"; do IFS=',' read -ra test_data <<<"$test" resource=${test_data[1]} namespace=${test_data[2]} name=${test_data[3]} new_storage_version=${test_data[5]} if [ -n "${namespace}" ]; then namespace="${namespace}/" fi kube::log::status "Verifying ${resource}/${namespace}${name} has updated storage version ${new_storage_version} in etcd" ETCDCTL_API=3 ${ETCDCTL} --endpoints="http://${ETCD_HOST}:${ETCD_PORT}" get "/${ETCD_PREFIX}/${resource}/${namespace}${name}" | grep "${new_storage_version}" done killApiServer ####################################################### # Step 3 : Start a server which supports only the new api version. # However, change storage encoding to KUBE_STORAGE_MEDIA_TYPE_PROTOBUF. ####################################################### RUNTIME_CONFIG="api/all=false,api/v1=true,apiregistration.k8s.io/v1=true,${KUBE_NEW_API_VERSION}=true" # This seems to reduce flakiness. sleep 1 startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_PROTOBUF} for test in "${tests[@]}"; do IFS=',' read -ra test_data <<<"$test" resource=${test_data[1]} namespace=${test_data[2]} name=${test_data[3]} namespace_flag="" # Verify that the server is able to read the object. if [ -n "${namespace}" ]; then namespace_flag="--namespace=${namespace}" namespace="${namespace}/" fi kube::log::status "Verifying we can retrieve ${resource}/${namespace}${name} via kubectl" # We have to remove the cached discovery information about the old version; otherwise, # the 'kubectl get' will use that and fail to find the resource. rm -rf "${HOME}/.kube/cache/discovery/localhost_8080/${KUBE_OLD_STORAGE_VERSIONS}" ${KUBECTL} get "${namespace_flag}" "${resource}/${name}" done killApiServer </del>

The cluster/update-storage- are not maintained. It will be replaced by We still run the hack/test-update-storage- Its invocation will be removed in We can remove the script. /sig api-machinery /assign
Do you mind if I work on this issue ?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen /remove-lifecycle rotten
Reopened this issue. <detailsIn response to : /reopen /remove-lifecycle rotten Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
are we ready to pull the plug on these shell scripts?
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity.

If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten


Question: existed = true } <ins> // If address exists, get it by IP, because name might be different. // This can specifically happen if the IP was changed from ephemeral to static, // which results in a new name for the IP. if existingIP != "" { addr, err := s.GetRegionAddressByIP(region, existingIP) if err != nil { return "", false, fmt.Errorf("error getting static IP address: %v", err) } return addr.Address, existed, nil } // Otherwise, get address by name </ins> addr, err := s.GetRegionAddress(name, region) if err != nil { return "", false, fmt.Errorf("error getting static IP address: %v", err)

/kind bug What happened: If the cloud provider attempts to reserve an IP and it's already reserved, it should continue syncing. Instead, it's returning an error and causing the service controller to retry. The above code only checks for 409 status code, but we need to also check for code 400. If I recall correctly, 409 happens for naming conflicts while 400 occurs for external address conflict. How to reproduce it (as minimally and precisely as possible): a load balancer service address to static with a random name. an LB sync. /sig network /area platform/gce
I don't think we gain anything by backporting the func to 1.7, right? Checking for both status codes seems to be sufficient.
Correct. It could be a minor performance optimization, but wouldn't help with the correctness. Agreed.
Actually, it's not sufficient. Right after that block of code, it tries to retrieve the address by name - a name which may be the user's. I'm thinking this issue is not worth fixing. I don't want to make more changes for a minor edge case.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/lifecycle frozen
/remove-lifecycle stale

We allow users to specify in the Service object. If the user later removes this from the Service Spec, the controller will fail to sync. For example: creates an LB for Service.Spec.LoadBalancerIP = "1.1.1.1" with a user-owned static IP address. updates the Service to remove the requested LoadBalancerIP tries to used by the existing forwarding rule (i.e., "1.1.1.1"). It encounters a conflict. Controller assumes it owns the IP and by name but fails. We can fix the controller to continue syncing and using the user-owned IP for the LB, or create a new address for the LB. The former seems less disruptive to me, but the downside is that if user will not be able to let other components use this IP (i.e., 1.1.1.1) unless they explicitly delete the Service. I think it's ok but would like more opinions. thoughts?
If a service is updated to remove a static address, then it seems like that would be a deliberate move on the part of the user. It seems like it should probably release the address and provision a dynamic one, even if this is disruptive. How do we handle the reverse? If a user has an already existing LoadBalancer service, and they add the after the fact? Would this release the dynamic address and recreate the forwarding rule with the reserved static address?
It seems like the creating a new address makes sense here... General comment: given the combinations (and the bugs we have been running into), I think this is a problem that all of the potential state transitions for the load balancer have not been fully spec'd out.
Creating a new address seems nice but the implementation is a nightmare. The service controller doesn't give you a previous Service spec, so you don't know what's changed (controller might not know either if spec was modified while offline). Given a desired service spec without a how do you decide to ditch your current IP and grab a new one? If we were to call to evaluate the situation, it's highly likely users will accidentally cause LBs to change IPs during typical usage. Imagine a user has a dynamically allocated IP for a service, but they want to promote it for safe keeping. The controller would think it needs to surrender the IP to the user's reservation.

The only correct process would be to set before promoting the address; however, that will surely be a missed point in docs and result in unhappy users. I lean towards the LB continuing use of the IP. Besides the easy option of deleting & re-creating the service, they could change the to another static address then demote the address. I'm not too familiar with the ideological boundaries of controllers and spec, so let me know if this crosses the line. What if the service controller set to the created IP on first sync if the IP wasn't pre-specified? The controller could rely on loadBalancerIP to always indicate which IP to use. If the field is empty (blanked by user), then it's a signal to get a new IP.
By spec, I mean we should write down the careful reasoning that you just wrote in a permanent place :-)
would it then make sense for to be immutable on an already created service?
+1 +1 Was thinking the same. Unlike the many annotations we support, updates to this actual field could be validated and blocked at the apiserver... Changing IP is disruptive anyway. Could we simply ask the user to re-create the Service?
Of course, this could be an API-breaking change, and other cloud providers may actually be supporting this properly.
I just ran across this today, but in a more likely (IMO) case. 1) create the Service, type=LoadBalancer 2) let the IP be auto-allocated 3) manually promote that IP to static and give it a meaningful name 4) observe errors from service controller trying to allocate that IP Since my Service does not spec "loadBalancerIP", the question is, do we: a) use the IP, but leave it as static b) use the IP, but demote it back to ephemeral c) allocate and use a new IP C is completely hostile. B is subtly hostile. A seems correct. Can it be implemented? If we find that the IP exists but the name of it is not what we expect, we should treat it as user-provided and leave it alone. This is still nasty because it logs a failure in the GCP activities log for every loop, but I don't know how to avoid that without getting a list of IPs and scanning for the IP we want.
Filed for the looping.

+1 as long as we can rely on the "naming" of the address to decide whether the controller owns the address and can promote/demote it.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale Still relevant
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale Still relevant
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
any update on this?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/lifecycle frozen
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.