Question: <ins> // +build !providerless /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package vsphere import ( "context" "fmt" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "k8s.io/klog/v2" "k8s.io/legacy-cloud-providers/vsphere/vclib" ) type sharedDatastore struct { nodeManager         *NodeManager candidateDatastores []*vclib.DatastoreInfo } type hostInfo struct { hostUUID   string hostMOID   string datacenter string } const ( summary       = "summary" runtimeHost   = "summary.runtime.host" hostsProperty = "host" nameProperty  = "name" ) func (shared *sharedDatastore) getSharedDatastore(ctcx context.Context) (*vclib.DatastoreInfo, error) { nodes := shared.nodeManager.getNodes() // Segregate nodes according to VC-DC dcNodes := make(map[string][]NodeInfo) nodeHosts := make(map[string]hostInfo) for nodeName, node := range nodes { nodeInfo, err := shared.nodeManager.GetNodeInfoWithNodeObject(node) if err != nil { return nil, fmt.Errorf("unable to find node %s: %v", nodeName, err) } vcDC := nodeInfo.vcServer + nodeInfo.dataCenter.String() dcNodes[vcDC] = append(dcNodes[vcDC], nodeInfo) } for vcDC, nodes := range dcNodes { var hostInfos []hostInfo var err error hostInfos, err = shared.getNodeHosts(ctcx, nodes, vcDC) if err != nil { if vclib.IsManagedObjectNotFoundError(err) { klog.Warningf("SharedHost.getSharedDatastore: batch fetching of hosts failed - switching to fetching them individually.") hostInfos, err = shared.getEachNodeHost(ctcx, nodes, vcDC) if err != nil { klog.Errorf("SharedHost.getSharedDatastore: error fetching node hosts individually: %v", err) return nil, err } } else { return nil, err } } for _, host := range hostInfos { hostDCName := fmt.Sprintf("%s/%s", host.datacenter, host.hostMOID) nodeHosts[hostDCName] = host } } if len(nodeHosts) < 1 { msg := fmt.Sprintf("SharedHost.getSharedDatastore unable to find hosts associated with nodes") klog.Error(msg) return nil, fmt.Errorf("") } for _, datastoreInfo := range shared.candidateDatastores { dataStoreHosts, err := shared.getAttachedHosts(ctcx, datastoreInfo.Datastore) if err != nil { msg := fmt.Sprintf("error finding attached hosts to datastore %s: %v", datastoreInfo.Name(), err) klog.Error(msg) return nil, fmt.Errorf(msg) } if shared.isIncluded(dataStoreHosts, nodeHosts) { return datastoreInfo, nil } } return nil, fmt.Errorf("SharedHost.getSharedDatastore: unable to find any shared datastores") } // check if all of the nodeHosts are included in the dataStoreHosts func (shared *sharedDatastore) isIncluded(dataStoreHosts []hostInfo, nodeHosts map[string]hostInfo) bool { result := true for _, host := range nodeHosts { hostFound := false for _, targetHost := range dataStoreHosts { if host.hostUUID == targetHost.hostUUID && host.hostMOID == targetHost.hostMOID { hostFound = true } } if !hostFound { result = false } } return result } func (shared *sharedDatastore) getEachNodeHost(ctx context.Context, nodes []NodeInfo, dcVC string) ([]hostInfo, error) { var hosts []hostInfo for _, node := range nodes { host, err := node.vm.GetHost(ctx) if err != nil { klog.Errorf("SharedHost.getEachNodeHost: unable to find host for vm %s: %v", node.vm.InventoryPath, err) return nil, err } hosts = append(hosts, hostInfo{ hostUUID:   host.Summary.Hardware.Uuid, hostMOID:   host.Summary.Host.String(), datacenter: node.dataCenter.String(), }) } return hosts, nil } func (shared *sharedDatastore) getNodeHosts(ctx context.Context, nodes []NodeInfo, dcVC string) ([]hostInfo, error) { var vmRefs []types.ManagedObjectReference if len(nodes) < 1 { return nil, fmt.Errorf("no nodes found for dc-vc: %s", dcVC) } var nodeInfo NodeInfo for _, n := range nodes { nodeInfo = n vmRefs = append(vmRefs, n.vm.Reference()) } pc := property.DefaultCollector(nodeInfo.dataCenter.Client()) var vmoList []mo.VirtualMachine err := pc.Retrieve(ctx, vmRefs, []string{nameProperty, runtimeHost}, &vmoList) if err != nil { klog.Errorf("SharedHost.getNodeHosts: unable to fetch vms from datacenter %s: %w", nodeInfo.dataCenter.String(), err) return nil, err } var hostMoList []mo.HostSystem var hostRefs []types.ManagedObjectReference for _, vmo := range vmoList { if vmo.Summary.Runtime.Host == nil { msg := fmt.Sprintf("SharedHost.getNodeHosts: no host associated with vm %s", vmo.Name) klog.Error(msg) return nil, fmt.Errorf(msg) } hostRefs = append(hostRefs, vmo.Summary.Runtime.Host.Reference()) } pc = property.DefaultCollector(nodeInfo.dataCenter.Client()) err = pc.Retrieve(ctx, hostRefs, []string{summary}, &hostMoList) if err != nil { klog.Errorf("SharedHost.getNodeHosts: unable to fetch hosts from datacenter %s: %w", nodeInfo.dataCenter.String(), err) return nil, err } var hosts []hostInfo for _, host := range hostMoList { hosts = append(hosts, hostInfo{hostMOID: host.Summary.Host.String(), hostUUID: host.Summary.Hardware.Uuid, datacenter: nodeInfo.dataCenter.String()}) } return hosts, nil } func (shared *sharedDatastore) getAttachedHosts(ctx context.Context, datastore *vclib.Datastore) ([]hostInfo, error) { var ds mo.Datastore pc := property.DefaultCollector(datastore.Client()) err := pc.RetrieveOne(ctx, datastore.Reference(), []string{hostsProperty}, &ds) if err != nil { return nil, err } mounts := make(map[types.ManagedObjectReference]types.DatastoreHostMount) var refs []types.ManagedObjectReference for _, host := range ds.Host { refs = append(refs, host.Key) mounts[host.Key] = host } var hs []mo.HostSystem err = pc.Retrieve(ctx, refs, []string{summary}, &hs) if err != nil { return nil, err } var hosts []hostInfo for _, h := range hs { hosts = append(hosts, hostInfo{hostUUID: h.Summary.Hardware.Uuid, hostMOID: h.Summary.Host.String()}) } return hosts, nil } </ins>

Currently whenever a vSphere volume is provisioned, we are looking at all nodes in the cluster and then fetching their hosts and then their datastores and then ensuring that selected datastore in present on all the nodes. This is done every time a new PVC is provisioned. It can be pretty expensive in large clusters and number of involved API calls can be big. /sig storage /area vsphere cc
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : Currently whenever a vSphere volume is provisioned, we are looking at all nodes in the cluster and then fetching their hosts and then their datastores and then ensuring that selected datastore in present on all the nodes. This is done every time a new PVC is provisioned. /area vsphere Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted
/assign


Question: WatchEvents, WatchEventsSizes, currentInflightRequests, <ins> requestTerminationsTotal, </ins> } )

What happened: When the kube-apiserver timeouts on a request, we end up doubling the counts measured in : apiserverrequestcounts apiserverrequesttotal apiserverrequestlatencies apiserverrequestlatenciessummary apiserverrequestdurationseconds apiserverresponsesizes What you expected to happen: We only count our requests a single time in our metrics. How to reproduce it (as minimally and precisely as possible): Add a 60 second sleep in the func . You may want to filter by resource (like pods or something) and verb for testing (otherwise this will be quite annoying). Manually hit that endpoint and look at the metric output. /assign
/sig api-machinery instrumentation /cc
Thanks for filing. Can we, while fixing this, add a metric counting timed-out requests? You can't tell from the latency distribution, even if we made the buckets more useful.
Yeah, that's the current plan. I've relayed the idea to offline and he has concurred.


Question: for i := 0; i < diff; i++ { go func(ix int) { defer wait.Done() <del> if err := jm.podControl.DeletePod(job.Namespace, activePods[ix].Name); err != nil { </del> <ins> if err := jm.podControl.DeletePod(job.Namespace, activePods[ix].Name, job); err != nil { </ins> defer util.HandleError(err) // Decrement the expected number of deletes because the informer won't observe this deletion jm.expectations.DeletionObserved(jobKey)

Currently RC emits an event when it's creating a pod: But nothing is emitted when a pod is deleted. We should create event for scaling both ways. cc


Question: # Optional: Create autoscaler for cluster's nodes. ENABLE_CLUSTER_AUTOSCALER="${KUBE_ENABLE_CLUSTER_AUTOSCALER:-false}" <del> if [[ "${KUBE_ENABLE_NODE_AUTOSCALER:-false}" == "true" || "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then ENABLE_CLUSTER_AUTOSCALER="true" </del> <ins> if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then </ins> AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-}" AUTOSCALER_MAX_NODES="${KUBE_AUTOSCALER_MAX_NODES:-}" AUTOSCALER_ENABLE_SCALE_DOWN="${KUBE_AUTOSCALER_ENABLE_SCALE_DOWN:-true}"

In kube-up we use var while in GKE, documentation, issues etc I think we call it cluster autoscaler. We should make it consistent.
I assume this is what you were already thinking, but: I think we should change it to cluster autoscaler (everywhere). You scale the cluster by adding/removing nodes. You scale a node by adding/removing resources to/from the node, so node autoscaler isn't the right term IMO.


Question: expectedDevice: "/dev/disk/by-id/wwn-0x000000000", }, <del> // Attach call fails </del> <ins> // Disk is already attached </ins> { <del> name:   "Attach_Negative", attach: attachCall{diskName, nodeName, attachError}, </del> <ins> name:           "Attach_Positive_AlreadyAttached", diskIsAttached: diskIsAttachedCall{diskName, nodeName, false, diskCheckError}, attach:         attachCall{diskName, nodeName, nil}, </ins> test: func(testcase *testcase) (string, error) { attacher := newAttacher(testcase) return attacher.Attach(spec, nodeName) }, <del> expectedError: attachError, </del> <ins> expectedDevice: "/dev/disk/by-id/wwn-0x000000000", </ins> }, // Detach succeeds

Currently we don't check if disk is already attached : from what i remember it can cause some problems, but not sure, sorry for the mass tag
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig storage /assign


Question: // nginxImage defined in kubectl.go nginxImageName = "nginx" <del> redisImage     = "gcr.io/google_containers/redis:e2e" </del> <ins> redisImage     = "gcr.io/k8s-testimages/redis:e2e" </ins> redisImageName = "redis" )

Kubernetes version (use ): master What happened: Pulling the redis image takes a while, like 30 seconds in GCE. This might be leading to periodic timeouts in tests like . What you expected to happen: Tests shouldn't depend on large images that take a potentially long time to pull. proposed fix to reduce flakiness we can increase the replica count on this test to 2 or make the redis image we use smaller (or not use redis at all and just use a simpler smaller image)
cc (considering this as a bug since we shouldnt need a 500MG image just to test kubectl :))
This should be as simple as changing the current to use alpine and install redis, instead of basing it off debian, installing gcc, and compiling redis from source :-) The image should be ~10MB.


Question: _, err = client.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", namespace) <del> // kubectl create -f auth-reader.yaml _, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler-auth-reader", Annotations: map[string]string{ rbacv1.AutoUpdateAnnotationKey: "true", </del> <ins> if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { // kubectl create -f auth-reader.yaml _, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler-auth-reader", Annotations: map[string]string{ rbacv1.AutoUpdateAnnotationKey: "true", }, </ins> }, <del> }, RoleRef: rbacv1.RoleRef{ APIGroup: "", Kind:     "Role", Name:     "extension-apiserver-authentication-reader", }, Subjects: []rbacv1.Subject{ { Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "", Kind:     "Role", Name:     "extension-apiserver-authentication-reader", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating role binding %s in namespace %s", "wardler-auth-reader", "kube-system") </del> <ins> Subjects: []rbacv1.Subject{ { Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating role binding %s in namespace %s", "wardler-auth-reader", "kube-system") } </ins> // Wait for the extension apiserver to be up and healthy // kubectl get deployments -n <aggregated-api-namespace> && status == Running

[X] Test approval issue: [X] Test PR: [x] Two weeks soak end date: [X] Test promotion PR: According to this APIsnoop query, there are still some remaining APIService endpoints which are untested. Using a number of existing e2e test practices the has to cover the outstanding APIService endpoints. There has been some around how the current Conformance test is using RBAC, which is optional and not part of Conformance. Those calls will now only happen if is true. The e2e logs for this test are listed below.

This query shows the following APIService endpoints are hit within a short period of running this e2e test example endpoint | useragent ---------------------------------------------+
createApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] deleteApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] deleteApiregistrationV1CollectionAPIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] listApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] patchApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] patchApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] readApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] readApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] replaceApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] replaceApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] (10 rows) If a test with these calls gets merged, test coverage will go up by 4 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted


Question: ) func TestMain(m *testing.M) { <ins> // see: https://github.com/kubernetes/kubernetes/issues/104265 setNoUmask() </ins> pkiutiltesting.RunWithPrivateKeyFixtureDirectory(m) }

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Running locally. Today? There is none, but I'm not filing this as a flaking test because it's a dependable failure, not a flake. I suspect this depends on the user's umask given the error: originally /area kubeadm /sig cluster-lifecycle
/assign
there were related changes to TestDownloadCerts last month: we could pin the mode for these newly created files in kubeadm. /triage accepted
but the problem seems odd. kubeadm uses client-go utils for writing a cert file with mode 644: i don't understand why something would override this mode to 0640.
i don't understand why something would override this mode to 0640. From the initial issue filing: -- users may be running in an environment with a less permissive umask, but the test depends on a full permissive umask.
ok, right. a quick fix would be to get the current process umask and expect it to be applied to the file permissions: but note: the test should be made Unix only too.
I think it would be reasonable to just set umask without getting it in the TestMain for the package. IIRC go runs tests as a binary / process per package, so it would only affect that package. We could make setting the umask only happen on unix without making the tests unix only as well, by using a conditionally compiled wrapper. I can send a PR for this, I suspect the system umask is google's workstation customization (~~debian testing), and probably relatively unusual elsewhere. I recall now we've run into this in the past during builds where e.g. does not result in the permissions on other systems so we've chmod to e.g.
the test would fail on Windows because -would always return there. so we should os-branch the checks.
I'm not sure I understand the windows part tbh, I think this particular failure can be addressed without affecting windows though. After meetings I got around to confirming on my personal debian stable instance reports while on my "gLinux" (~debian testing) workstation it reports . AFAICT should be more or less the standard across distros but debian and ubuntu at least provide a mechanism out of the box to configure it for the system.

yeah, the test is already broken on non-Unix, so we could just set the umask (syscall.Umask) to make the test happy. the non-Unix fix seems like something that can be done separately.
filed to set it to 0 in test main so it would be technically not umask sensitive anymore on linux and macOS at least.


Question: // - part 3: other plugins (excluded by part 1 & 2) in regular extension point. newPlugins := reflect.New(reflect.TypeOf(e.slicePtr).Elem()).Elem() // part 1 <del> for _, name := range enabledSet.list { </del> <ins> for _, name := range slice.CopyStrings(enabledSet.list) { </ins> if overridePlugins.has(name) { newPlugins = reflect.Append(newPlugins, reflect.ValueOf(pluginsMap[name])) enabledSet.delete(name)

Scheduler's default plugin returns an invalid score that is not in the range of [0, 100], resulting in no Pods being scheduled: After investigation, I found that there was a bug in the code that built the framework when the scheduler started. When using the plugins declared in the regular extension point to override the plugins declared in the MultiPoint, the traversed variable will be modified in the following loop: This may cause a same scoring plugin (e.g. ) to be loaded multiple times: Further, this results in a node being processed multiple times by a plugin's (e.g. 's) method, resulting in an invalid score that is not in the range of [0, 100]: If the scheduler's MultiPoint feature is used (either through the default plugins or through manual configuration), and the configuration (e.g. the scoring weight) of a certain plugin is overridden in a regular extension point, the plugin will only be loaded once in this extension point, and no illegal scores will be returned. An example KubeSchedulerConfiguration: No response K8s v1.24.15 or higher None <details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
Can I work on this?
this is being worked on. see
I see, Is there any other task that you are aware of and I can work on? Thanks.


Question: actualSum := actualHist.GetSampleSum() num := math.Abs(actualSum - ee.sum) den := math.Max(math.Abs(actualSum), math.Abs(ee.sum)) <del> if num > den/1e14 { t.Errorf("At idx=%d, expected sum %v but got %v, err=%v", idx, ee.sum, actualSum, actualSum-ee.sum) </del> <ins> relErr := num / den // Issue 120112 reports relative errors as high as 9.55994394104272e-14 if relErr > 1e-13 { t.Errorf("At idx=%d, expected sum %v but got %v, err=%v, relativeErr=%v", idx, ee.sum, actualSum, actualSum-ee.sum, relErr) </ins> } } }

pull-kubernetes-unit see title 8/17? ref No response /sig instrumentation
/triage accepted /assign
an example flake in of sig-release/master-blocking test grid. (It flakes a few times, and it is not a blocker.)
/cc still flake
I tested it locally, and it did reproduce the error. I'll take a look at this problem.
I looked into this part of the code. The problem should be the loss of precision. as the denominator of is too large, that is to say, occasionally satisfies the condition( ≈0.16%), and the allowed precision error is too small. So, occasionally this test flakes. I modified the code and printed out the results of in the log(It can be seen that the value is greater than ): see stress test result details from the . I don't know much about this, how did you decide to use as the denominator? Maybe we can adjust the precision error( for example 1e-14 -1e-13 or other value)? Can you take a look? FYI: In the prometheus histogram test , this error value is set to
I do not recall exactly how I arrived at the limit of 1e-14. That is about 46.5 bits, which should be well within the accuracy of a . To get this test passing, I have increased the limit by a factor of 10 (in ), based on the relative errors reported above.


Question: }, }, }, <ins> { name: "update existing instance with .spec.ordinals.start", old: apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{Name: "abc.123.example", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, Selector:            &metav1.LabelSelector{MatchLabels: validLabels}, Template:            validPodTemplate.Template, UpdateStrategy:      apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, update: apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{Name: "abc.123.example", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, Selector:            &metav1.LabelSelector{MatchLabels: validLabels}, Template:            validPodTemplate.Template, UpdateStrategy:      apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, Ordinals: &apps.StatefulSetOrdinals{ Start: 3, }, }, }, }, </ins> } errorCases := []testCase{

This applies to StatefulSetStartOrdinal feature gate. Field validation should be done regardless of the feature gate state. From
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/sig apps


Question: kubectl run testmetadata --image=nginx --replicas=2 --port=80 --expose --service-overrides='{ "metadata": { "annotations": { "zone-context": "home" } } } ' # Check result kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'testmetadata:' <del> kube::test::get_object_assert 'service testmetadata' "{{.metadata.annotations}}" "map[endpointslice.kubernetes.io/managed-by-setup:true zone-context:home]" </del> <ins> kube::test::get_object_assert 'service testmetadata' "{{.metadata.annotations}}" "map[zone-context:home]" </ins> ### Expose deployment as a new service # Command kubectl expose deployment testmetadata  --port=1000 --target-port=80 --type=NodePort --name=exposemetadata --overrides='{ "metadata": { "annotations": { "zone-context": "work" } } } ' # Check result <del> kube::test::get_object_assert 'service exposemetadata' "{{.metadata.annotations}}" "map[endpointslice.kubernetes.io/managed-by-setup:true zone-context:work]" </del> <ins> kube::test::get_object_assert 'service exposemetadata' "{{.metadata.annotations}}" "map[zone-context:work]" </ins> # Clean-Up # Command

What happened: kube-controller-manager panic for What you expected to happen: no panic. How to reproduce it (as minimally and precisely as possible): No idea. I guess it's hard to reproduce. Anything else we need to know?: Found from here: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: /assign cc Should we do something for v1.17? (v1.17 code freeze now, but still have the opportunity to fix) can be removed safely in v1.18, right?
/sig scheduling
Working on this in /sig network /priority critical-urgent /assign
Sorry, I didn't realize it. I should have a search before firing an issue. I recommend adding this issue to your PR. e.g. `
Updated, thanks!
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
This seems to be writing to a cached (should be const) service. I think we need to DeepCopy() it before we mutate the Go type. does that sound right? Is this still our best state of the art?
yes, copy before change
we have a tool to catch this (the cache mutation detector), but it got disabled in our test runs two years ago (xref )
yes great catch - I was always convinced this is enabled...


Question: client internalclientset.Interface // these flags control which items will be checked to deny exec/attach <del> hostIPC    bool hostPID    bool privileged bool </del> <ins> hostNetwork bool hostIPC     bool hostPID     bool privileged  bool </ins> } var _ admission.ValidationInterface = &DenyExec{}

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: currently only are checked in admission controller What you expected to happen: hostNetwork should also be checked to deny exec /attach How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/kind bug /sig auth
cc


Question: }) } <ins> // NOTE(resouer): nodeInformer to substitute deprecated taint key (notReady -> not-ready). // Remove this logic when we don't need this backwards compatibility nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: util.CreateAddNodeHandler(func(node *v1.Node) error { return nc.doFixDeprecatedTaintKeyPass(node) }), UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error { return nc.doFixDeprecatedTaintKeyPass(newNode) }), }) </ins> nc.nodeLister = nodeInformer.Lister() nc.nodeInformerSynced = nodeInformer.Informer().HasSynced

Almost all style strings use dashes, rather than javascript names. vs .
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
Thanks for notification, a quick fix is submitted.
/cc
The PR here isn't merged yet. Can we make that happen please? We're still in alpha, so should be ok
Should I add the v1.8 milestone to this issue?
Hope so. We are pending in , need approve from
I opened with safe changes. "not-ready" change needs to be discussed more, as it's a breaking change. I believe it's best with fixing that to the time we'll be moving to beta, as we'll need to do another breaking change, i.e. remove 'alpha' from the key.
Aren't we doing this at the 11th hour? Is this going to hit 1.8?
Strings like this that are part of the core API should be part of a design doc that is formally reviewed by an API group. We need to find ways to catch this stuff earlier. We should also have an evolving API style guide that lists stuff like this. /cc
Thanks , let me make some summary: quick fix for 1.8 is , will be merged soon. final fix is , we are collaborating to reach a final solution and it's not target 1.8
Agree with Joe, can someone open a PR to that documents that taint names should follow the naming conventions for annotations?


Question: return nil } <del> func (v *validator) getLoadedProfiles() (map[string]bool, error) { profilesPath := path.Join(v.appArmorFS, "profiles") profilesFile, err := os.Open(profilesPath) if err != nil { return nil, fmt.Errorf("failed to open %s: %v", profilesPath, err) } defer profilesFile.Close() profiles := map[string]bool{} scanner := bufio.NewScanner(profilesFile) for scanner.Scan() { profileName := parseProfileName(scanner.Text()) if profileName == "" { // Unknown line format; skip it. continue } profiles[profileName] = true } return profiles, nil } // The profiles file is formatted with one profile per line, matching a form: //   namespace://profile-name (mode) //   profile-name (mode) // Where mode is {enforce, complain, kill}. The "namespace://" is only included for namespaced // profiles. For the purposes of Kubernetes, we consider the namespace part of the profile name. func parseProfileName(profileLine string) string { modeIndex := strings.IndexRune(profileLine, '(') if modeIndex < 0 { return "" } return strings.TrimSpace(profileLine[:modeIndex]) } </del> func getAppArmorFS() (string, error) { mountsFile, err := os.Open("/proc/mounts") if err != nil {

What happened: Pods with invalid container-level seccomp profiles allow its init containers to be executed. Pods with unloaded apparmor profiles blocks the entire pod. What you expected to happen: Consistency between both. In one hand AppArmor blocks all pod execution regardless of the containers having or not a profile attached to it. On the other, Seccomp allows for an init container to deploy its own profile - as currently done on . How to reproduce it (as minimally and precisely as possible): Apply the yaml below: Resulting in only the init container from the being executed: ! Anything else we need to know?: AppArmor is currently being target to GA on 1.21 (). Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2020-12-08T17:59:43Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"21+", GitVersion:"v1.21.0-alpha.0.292+", GitCommit:"", GitTreeState:"clean", BuildDate:"2020-12-12T23:13:21Z", GoVersion:"go1.15.6", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ):Install tools: Network plugin and version (if this is a network-related bug): Others: /sig node /area security
/cc
/sig security /triage accepted
I think we can consider this as a bug, right? Then we could work on a fix for v1.21 interdependently of the AppArmor GA graduation.
Sounds like a good plan :+1:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with .

Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
/assign


Question: trap "teardown" EXIT <del> POD_ID_LIST=$($CLOUDCFG -json -l name=myNginx list pods | jq ".items[].id") </del> <ins> POD_ID_LIST=$($CLOUDCFG '-template={{range.Items}}{{.ID}} {{end}}' -l name=myNginx list pods) </ins> # Container turn up on a clean cluster can take a while for the docker image pull. ALL_RUNNING=0 while [ $ALL_RUNNING -ne 1 ]; do

We have an e2e test that runs the guestbook example. We should also have one that runs through the other example.


Question: "BackendConfig":         true, "NodeMetrics":           true, "PodMetrics":            true, <del> "ScalingPolicy":         true, </del> "VolumeSnapshotClass":   true, "VolumeSnapshotContent": true, "VolumeSnapshot":        true, <del> // A CRD created by other e2e tests without any test data. "Noxu": true, </del> } apiGroups, err := c.Discovery().ServerPreferredResources()

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: gce-master-scale-correctness Which test(s) are failing: [sig-cli] Kubectl client kubectl get output should contain custom columns for each resource Since when has it been failing: 11/6 Testgrid link: https://k8s- Reason for failure: Anything else we need to know: n/a /sig cli /sig scalability /cc
The label(s) cannot be applied. These labels are supported: <detailsIn response to : <!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --gce-master-scale-correctness [sig-cli] Kubectl client kubectl get output should contain custom columns for each resource 11/6 https://k8s- Anything else we need to know: n/a /sig scalability /cc Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Doesn't seem to be scale-related - failed for me the same way in the presubmit:
- do we run correctness test in smaller scale (so we can detect whether failures are scale related or not)? I can't find them on neither https://k8s- nor https://k8s- dashboards.
This is a new test that seem to be flaky at head: - seems that you this in Can you please take a look?
/assign
/reopen Let's wait for the test to pass /assign
Reopened this issue. <detailsIn response to : /reopen /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
That seems to be creating one of every object from the etcd data test in an e2e test. The failing object looks like a node object, which is likely getting deleted by the Node controller as soon as it is created.
opened to skip nodes for this test
/reopen The job is not green
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository.

</details
failed test run started from a commit prior to the fix merging ( started from ) /close
Closing this issue. <detailsIn response to : failed test started from a commit prior to the fix merging ( started from ) Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return nil } <ins> // patchAllocatedValues allows clients to avoid a read-modify-write cycle while // preserving values that we allocated on their behalf.  For example, they // might create a Service without specifying the ClusterIP, in which case we // allocate one.  If they resubmit that same YAML, we want it to succeed. func patchAllocatedValues(newSvc, oldSvc *api.Service) { if needsClusterIP(oldSvc) && needsClusterIP(newSvc) { if newSvc.Spec.ClusterIP == "" { newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP } if len(newSvc.Spec.ClusterIPs) == 0 { newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs } } if needsNodePort(oldSvc) && needsNodePort(newSvc) { // Map NodePorts by name.  The user may have changed other properties // of the port, but we won't see that here. np := map[string]int32{} for i := range oldSvc.Spec.Ports { p := &oldSvc.Spec.Ports[i] np[p.Name] = p.NodePort } for i := range newSvc.Spec.Ports { p := &newSvc.Spec.Ports[i] if p.NodePort == 0 { p.NodePort = np[p.Name] } } } if needsHCNodePort(oldSvc) && needsHCNodePort(newSvc) { if newSvc.Spec.HealthCheckNodePort == 0 { newSvc.Spec.HealthCheckNodePort = oldSvc.Spec.HealthCheckNodePort } } } </ins> // NormalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not // consider any other fields. func NormalizeClusterIPs(oldSvc, newSvc *api.Service) {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: was closed as not a problem with kubectl, but with the server What you expected to happen: 'kubectl replace' with no change on a service should work. How to reproduce it (as minimally and precisely as possible): reproduction included in this issue Anything else we need to know?: Seems to me like this comment sums up the options I got here from this issue Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig cli
After a long day of reading and experimenting with the Kubernetes API i found a work-around I want to share with you. Problem in my case: I'm using the Java Client K8 API, especially I got the exact same error like written in and here in this issue summary: I send an update of a service WITHOUT specifying the spec.clusterip within the V1Service object. The request gets then rejected with the reason . Workaround: First load the existing service that contains the current clusterIP. Set the old clusterIp to the updated V1Service. Also load the old metadata.resourceVersion value and set it to the new V1Service object, because leaving the value out using the Java API Client results in an error: Please do not close this issue due to inactivity again, like it was done with . Even if there are some workarounds, this issue will pop up again and again, because there really is a problem with the API restrictions or at least the messages you will get when interacting with the API. Constructive Suggestion: e.g. instead of throwing the error you could throw something like and for the resourceVersion:
Seriously, this has been going on for years. is over 2 years old and there have been dozens of bug reports/feature requests (in Kubernetes, Kubectl, Helm, and elsewhere) showing the problems the current behavior is causing. (List available on request.) is from over 3 years ago. is from 2015.

The old justifications for keeping the current behavior ("works as designed") have proven to be insufficient. It is time for a redesign. The API should accept as meaning "assign a clusterIP" on creation and "keep the current clusterIP" on update. Since, when you set on creation, the resource does not end up with , this means a resource cannot validly have , and since the field is immutable, could never be a correct value for modification under the current API. While this could be considered a breaking change, I doubt there is anyone who would complain about it. Rejecting as a modification is not helping anyone. It is not rejecting an accidental change or avoiding confusion because the meaning is unclear, and it is hard for me to imagine a use case where someone is relying on the current behavior and would be upset about the proposed new behavior (with the possible exception of people writing compliance tests). It is just causing problems for the sake of some kind of theoretical purity that is in the eye of the beholder. Treating as meaning "system should replace this with a valid setting" is equally pure.
cc long-term-issue (note to self)
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
keepalive
Is there something I can do to move this forward?
/remove-sig cli /sig network /assign this is sort of the inverse of the issue fixed in Using with a static manifest on an object that has spec fields populated by the API server (like clusterIP in service) or controller (like replicas in deployment) is an anti-pattern. is a better match for this usage. Alternatively, a controller that did a 'read -modify -replace' could be ok, since it would be starting with the current API server state.
This is not impossible to fix in the PUT path (), but it basically turns PUT into PATCH. We have distinct PATCH operations for a reason, and that is almost certainly what you want in this case. should "just work" as a replacement for . Is there a reason why you can't use PATCH or apply?

- while I continue to argue that is wrong here, maybe we should just cave and make it quietly patch the value of clusterIP in this specific case? I really dislike it, not just for purity reasons, but because I have a hard time seeing if that will bite us later. Thoughts?
I really dislike it as well. "replace all spec content" is a bad thing to teach clients to do on resources that have fields set by different actors.
Agree, and while we might be able to quietly patch this SPECIFIC case, we can't do that in general. - My inclination is to close this. I know it's not what you want, but I do think you and most users will be better served by patch or apply in almost all cases (even knowing that Service has an ambiguity around changing ports that overlap each other which is not easily fixed because it is breaking). Is there a reason why you can't use patch or apply?
This really seems related to bug in Helm, as far as what I have read, and some recent (Jan2021) posts suggest that was fixed. Have not tested this out myself. Agree that adhering to expected PUT/PATCH api behavior would be best. Thanks for the followup!
I reject the idea that this is a bug in Helm, even though Helm is working hard (and failing) to handle the situation. I agree with you when you "Agree that adhering to expected PUT/PATCH api behavior would be best". My point is that the expected "PUT" behavior is that if a "PUT" request succeeds, the exact same "PUT" request given a second time should succeed as well. That is what is broken and I am asking to have fixed. Is there a better reason why I should be prohibited from using replace on a Service, when I can use it for every other resource? Why should Service require special-case handling by the user? I ask you to think of this in the larger context of using Infrastructure as Code (IaC) to manage resources, and the automation around that. In the IaC definition of a Service, the has to be set to in order to get a system-assigned address. Semantically, says "make the resource exactly like this". It can be, and is, used to ensure there is not "drift" in the configuration, i.e.

that the configuration stored in the IaC code base is live. It is the correct thing to use when you want to say "here is the configuration that should be active", and it is what automation uses to reset the configuration after it has drifted. Users should not have to treat Services specially, they should be able to any resources with the same spec that created them. Stating this another way, (A user should be able to the same thing repeatedly, successfully, and without causing changes after the first accepted .) By not accepting for in , you are breaking the contract of making idempotent. (As a matter of practicality, the Helm maintainers are trying to make up for the fact that is not idempotent, but that is not an excuse for Kubernetes to leave this API in its current state.) As I stated above, I am not even advocating for this case to be handled as a . As I see it, the value of is handled specially but not idempotently. The value of currently means one thing if the resource does not exist and something else if it does exist. I am asking instead that it consistently, idempotently be taken as meaning "any valid address: the existing one if one is set, otherwise one randomly assigned by the controller".
Agree, and while we might be able to quietly patch this SPECIFIC case, we can't do that in general. Unfortunately, to make PUT idempotent in this case, it would need to turn into a variant of PATCH. I'm not even saying I totally disagree, but that is what it means. As for PUT being idempotent, I don't think that's ever really been a hard objective. In this case (and look, I am not saying I am happy about it) that means reading existing state and patching it into the PUT. We can do that (probably) but I can't say for sure that there are no other APIs where this will happen (there probably are not). Note that to do this we would need to fixup , , and probably the newer dual-stack-related fields. Can someone help me understand the "bug"? Is it about using PUT instead of apply? I could see two paths to fixing this, if we want to (and I am still not 100%). 1) Change clusterIP and nodePort fields to pointers.

If not-specified on PUT AND the object exists AND the old warrants a value AND the new warrants a value, copy the existing values. This is a bit tricky for nodePort we'd have to at least specify what happens if the list is reordered or otherwise modified. For example, if the existing object has: and the update specifies: Do I track the move by name? What if the names changed, does that lose the tracking? We've had innumerable issues here because the API-specified merge-key was (is) ambiguous. The only way to get exactly what you want in ALL cases is to RMW the resource. Explicitly setting "" and 0 would work on initialization (compat) but would not be auto-patched. 2) Leave the fields as non-pointers and accept "" and 0 as "DWIM" mode. I dislike this because it makes it less obvious when a user mistakenly tries to clear a field - it used to fail and will now "succeed" but silently change the value. I am not sure I see a clever 3rd option. The only reason I entertain this at all is because of the issues around apply/patch and ports. NOTE: My willingness to think it thru is not a commitment to implement it. I am swamped right now :)
wrote I am shocked that you would think that, although I don't really know what you mean by "hard objective". The doc certainly implies should be idempotent, via such statements as and Moving on to how to do make it so, the idea of is that at the end of the operation, the object is in the state described by the action. If you PUT ports P and Q and the existing object has ports R and S, you delete R and S and add P and Q. If adding or deleting a port is not allowed, then you reject the PUT. I think to some extent you are overthinking/overdesigning a solution here. In Kubernetes, every PUT is to some extent a patch because there are some fields that are not specifiable via PUT, like all the managed fields, or , and fields like and are special in a whole different way. But that stuff should already be handled by the more generalized resource handling architecture.

The general rule about PUT is that PUT on an existing resource should adjust that resource to be in the same working configuration as if you used that PUT to create a new resource. Calling PUT a second time with the same input should be acceptable and should not change the resource. I believe that Kubernetes generally has a canonical ordering of all fields in a resource (and if there is no clear way to order a list, the canonical order is simply the order in which the list was originally specified on resource creation), so what you do is enumerate the fields of the existing resource in canonical order, filter out the fields like that PUT cannot specify, and then compare it to the PUT resource, after normalizing it into canonical order, and add/delete/replace field by field. Clearly something like this is already happening in handling PUT and PATCH. This should not be a lot of new work. The ongoing community complaint is that when PUT says and it is matched against the exiting the API rejects it, saying you can't change the . All we are asking for is that this case be interpreted differently and allowed. is documented as an allowable value in the input, but it is never an allowable value in the realized resource, so it cannot possibly mean to be interpreted as "force this value on the resource". Its meaning is that the system should assign a valid value and the user does not care what it is. So on PUT, when is matched against the exiting , he API should just confirm that is a valid value and, if so, accept the input.
As for PUT being idempotent, I don't think that's ever really been a hard objective. What I meant is it's just never been a primary question in reviewing APIs. Service is a weird case, far outside the norm and I'm not excusing it - just explaining what I meant. Moving on to how to do make it so, the idea of is that at the end of the operation, the object is in the state described by the action. If you PUT ports P and Q and the existing object has ports R and S, you delete R and S and add P and Q. If adding or deleting a port is not allowed, then you reject the PUT. For historical reasons, the "key" for ports was not the name but the port value, then the value + protocol. So we can key off that, I think.

I think to some extent you are overthinking/overdesigning a solution here. In Kubernetes, every PUT is to some extent a patch because there are some fields that are not specifiable via PUT, like all the managed fields, or , and fields like and are special in a whole different way. Those are all outside of . I don't know of any resource that PUT-PATCH inside the spec. Hilariously, I just noticed that nodePort is even worse - it accepts the non-value and then re-allocates the nodePort. I am going to call this a bug, but now we have a compatibility discussion to have. If a user does not specify a value, we can reasoinably argue that we should retain existing values. But if they specify , do we stay compatible and re-allocate the node port, or do we say that was always a bug? Is it a useful "feature" to be able to re-allocate? We disallow it explicitly gfor ClusterIPs. Clearly something like this is already happening in handling PUT and PATCH. This should not be a lot of new work. You haven't spent much time in the Service REST code, I guess. I have a large rework of that on-going, but this was not part of it. :) The ongoing community complaint is that when PUT says and it is matched against the exiting the API rejects it, saying you can't change the . Given the previous point about nodePort, would it be fair to modify your statement as: "when PUT says (or does not specify at all) ..." With the corollart that if you actually specify you will get the current (immutable error) behavior. This seems like it would satisfy helm and other users who are not specifying literal "" - they are not specifying any value at all. Thoughts?
At this point I am willing to call it a bug, albeit an unpleasant one.
wrote I have had regular run-ins with this issue for years, which is why I'm being insistent on getting it fixed properly. I do not recall having similar problems with and so I am less informed on those issues. I don't know what means in the spec or in practice, so I don't have a specific opinion as to what it should do to an existing resource.

I will simply restate the general principle which should apply to just like everything else: what ever you allow as a setting on creation should be allowed in any and should not cause a change if repeatedly applied. The very practical reason for this is so that I can record some place (like a checked into a VCS) that "this is how the resource should be configured" and use that (1) to create the resource and (2) to return the resource to expected configuration after some external operation changed it, using a standard API while otherwise treating the resource as opaque.
/cc
/cc (i know i am going to regret this)
It is a bug. Not because it does not match rest api semantics. But because it is not consistent with the rest of the apis. In other words, If user can do on other resources they should be able to do it on as well. It is hard (and weird) to ask users to do one thing for all resources, but then do a totally different thing for Services. As discussed over sig-network the problem will be user intent. Some ideas for a solution: If irrespective of changes in other we can assume that the user is replacing and does not care about ClusterIP value, hence copying to new should be fine. It is basically like the user saying so copying old=new should be Ok. If that is a failure case, because it breaks ClusterIP immutability rules ( we should really, really remove ClusterIP immutability but i digress). We will still require the validation to work as is. e.g. replacing with with a type that does not require ClusterIP and not setting it to should fail. I can look into building a POC for this.
I am probably missing some subtlety here, but as in I suspect that Helm is behaving properly and the problem is in way Kubernetes handles s: the of a resource should never be rewritten with variable environmental information; that is what is for. In other words could be set in the unusual case that you wished to pick an IP yourself but would normally be null; would reflect the service’s current value, expected to be retained across unrelated patches.
nailed it on the head with his last comment. I do agree that we should not be re-defining the behaviour of a PUT request.

But the inconsistency of how a Service updates the field compared to other resources results in inconsistencies in the PUT behaviour. Once a resource defines fields in the object, normally Kubernetes leaves it unchanged. My understanding is that Kubernetes uses this field to use as a reference point to be the "ideal state of the world", using the field to record the "current state of the resource". The only offending case I've seen in the wild is a , as it updates both the field AND the field. The end result is that a with a defining results in differing behaviour than the rest of the resource APIs. From Helm's point of view, a chart author's only course of action is to a) omit from the manifest and all subsequent updates, or b) if you happened to define in your first install, find the value Kubernetes provided and inject it into your template so that the value Kubernetes provided is preseved. Failing to do so results in a failed PUT () request. I wrote this behaviour down in-depth in this Helm comment: I'd propose an alternative solution than solutions: instead of updating after an IP has been assigned to the Service, use the API, i.e. . It would be a breaking change to the API, but it should bring Services back in line with the rest of the resource APIs, leaving as a user-defined field.
I agree
I'm afraid that breaking an API is not a feasible option
That document does allow for breaking changes under the apiVersion. Instead of making a breaking change to v1 services, you can introduce Service v2. Existing v1 services migrate their spec.clusterIP’s existing value (auto-generated or not) to the new version. spec.clusterIP and status.clusterIP would now match their corresponding relationships, and Kubernetes retains backwards/forwards compatibility.
(If it's not clear whether the API changes document does or doesn't allow the change from last comment, let's raise a separate improvement request against that document).
Does the case is important ? Because here I can read or ...
I have a patch. I'll try to get it out for review tonight.
Has anyone yet checked that the purported fix actually addresses the issue in Helm?
Please do!
I can still see the error but I'm on GKE 1.18, will this only apply to Kubernetes 1.21/22?

It will be part of 1.22 or the next 1.21.X (I don't remember now what is the latest stable)
Can anyone confirm that this issue was actually fixed in k8s version 1.21.4 or 1.22.1 ? I do not see it listed in the release notes/change log
The change was released in the 1.22.0-beta.2, so yes it should be available in 1.22. See I do not believe it was backported to 1.21.
Thanks for confirming and providing the details


Question: } } <ins> func TestValidateObjectMetaUpdateIgnoresCreationTimestamp(t *testing.T) { if errs := ValidateObjectMetaUpdate( &api.ObjectMeta{Name: "test", CreationTimestamp: util.NewTime(time.Unix(10, 0))}, &api.ObjectMeta{Name: "test"}, ); len(errs) != 0 { t.Fatalf("unexpected errors: %v", errs) } if errs := ValidateObjectMetaUpdate( &api.ObjectMeta{Name: "test"}, &api.ObjectMeta{Name: "test", CreationTimestamp: util.NewTime(time.Unix(10, 0))}, ); len(errs) != 0 { t.Fatalf("unexpected errors: %v", errs) } if errs := ValidateObjectMetaUpdate( &api.ObjectMeta{Name: "test", CreationTimestamp: util.NewTime(time.Unix(11, 0))}, &api.ObjectMeta{Name: "test", CreationTimestamp: util.NewTime(time.Unix(10, 0))}, ); len(errs) != 0 { t.Fatalf("unexpected errors: %v", errs) } } </ins> // Ensure trailing slash is allowed in generate name func TestValidateObjectMetaTrimsTrailingSlash(t *testing.T) { errs := ValidateObjectMeta(&api.ObjectMeta{Name: "test", GenerateName: "foo-"}, false, nameIsDNSSubdomain)

After the DST change: We should be ignoring creationTimestamp on update.
Could you please explain the cause of the failure in more detail? Do we not normalize the timezone in the representation used in the API?
When we serialize to JSON we are using a value that has a timezone embedded in it (time.Time). As a result the representation changed. We should normalize it as well during serialization, but in general creationTimestamp should not prevent an update since it can safely be ignored.
I think we should allow creationTimestamp to be empty and also convert time to UTC() before marshaling.
More specifically: We'd also want to change: to and to
Makes sense to me - clients should always control their timezone and with distributed masters you might see different values. ----- Original Message -----
I'll own the follow up on this. ----- Original Message -----
+1 for UTC serialization
Opened to marshal to utc when serializing


Question: pod := podInfo.Pod scheduleResult, err := sched.SchedulePod(ctx, fwk, state, pod) if err != nil { <ins> defer func() { metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInSeconds(start)) }() </ins> if err == ErrNoNodesAvailable { status := framework.NewStatus(framework.UnschedulableAndUnresolvable).WithError(err) return ScheduleResult{nominatingInfo: clearNominatedNode}, podInfo, status

schedulingCycle only reports algorithmduration when it's a successful scheduling attempt: Also report scheduling algorithm latency when the pods is unschedulable Create a Pod that is unschedulable (perhaps node selector for a node that doesn't exist) and observe metrics No response <detailsANY </details<detailsANY </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
/assign
let me try


Question: } func addEndpointSubset(subsets []v1.EndpointSubset, pod *v1.Pod, epa v1.EndpointAddress, <del> epp v1.EndpointPort, tolerateUnreadyEndpoints bool) ([]v1.EndpointSubset, int, int) { </del> <ins> epp *v1.EndpointPort, tolerateUnreadyEndpoints bool) ([]v1.EndpointSubset, int, int) { </ins> var readyEps int = 0 var notReadyEps int = 0 <ins> ports := []v1.EndpointPort{} if epp != nil { ports = append(ports, *epp) } </ins> if tolerateUnreadyEndpoints || podutil.IsPodReady(pod) { subsets = append(subsets, v1.EndpointSubset{ Addresses: []v1.EndpointAddress{epa}, <del> Ports:     []v1.EndpointPort{epp}, </del> <ins> Ports:     ports, </ins> }) readyEps++ } else if shouldPodBeInEndpoints(pod) { glog.V(5).Infof("Pod is out of service: %s/%s", pod.Namespace, pod.Name) subsets = append(subsets, v1.EndpointSubset{ NotReadyAddresses: []v1.EndpointAddress{epa}, <del> Ports:             []v1.EndpointPort{epp}, </del> <ins> Ports:             ports, </ins> }) notReadyEps++ }

What happened: When creating a headless service locally with set as , field has to be specified. Otherwise, endpoint controller will not be able to create an endpoint for the headless service successfully. This is because port number of to-be-created endpoint is defaulted to 0 when the field is not specified, which results in an error (0 is a reserved number not available for TCP and UDP usage). This explicitly suggests that the field is optional, which can be confusing to those unfamiliar with kubernetes networking. What you expected to happen: Port number of to-be-created endpoint should be defaulted to any value from 1 to , not 0. Apart from this, the tutorial should not suggest the field as optional until the defaulting is fixed. How to reproduce it (as minimally and precisely as possible): : an to refer, but remember to specify field into controller manager log and search for errors related to endpoint creation for the headless service Environment: Kubernetes version (use ):Cloud provider or hardware configuration: local cluster OS (e.g. from /etc/os-release): /cc
In endpoints controller we have explicit logic allowing headless service not to have ports: Though confirmed creating a headless service with no ports defined hit the exact same error: Will investigate more to see if this is a recent regression on endpoint API validation.
Ref , which reported the same bug.
Thanks for looking at the issue! Tell me if I can be of any help. :)
Sure thing :) I have a fix opened . /assign

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When deploying a headless service without port definition, there is no DNS endpoint created. What you expected to happen: That there would be a valid DNS entry for the service that has just been created. How to reproduce it (as minimally and precisely as possible): Use the example configuration from , but leave out the following section: Anything else we need to know?: Searching trough the issues, it seems I expect the "unexpected" behavior from ticket , which might been fixed in and PR Environment: Kubernetes version (use ):Cloud provider or hardware configuration:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
allows a headless service without ports and updates service endpoints for the service. I note that you mentioned DNS endpoint. How did you check the DNS endpoint? Thanks!
If I copy the complete example found in the you can resolve the endpoints as follows: If we recreate the example leaving out the ports, we get the following result: Describing the also does not show any endpoints:
In I'm encountering this problem too. I create a headless service with an empty Ports list Create a statefulset referring to that service Pods created by the stateful set cannot resolve each others stable names. I see the following errors in the cluster logs
Also experiencing something similar though it may not be the exact same thing. I have GlusterFS configured in a 1.8 cluster. The gluster replicas were created via a stateful set and thus has a headless service to go with it. If I try to use the glusterfs volume in a pod, in the same namespace (default) then the pod comes online fine and is able to mount the volume. I tried using the same gluster volume in a pod in a different namespace but it fails to resolve the endpoints. I've tried , and but all result in the same issue saying endpoint cannot be found. I moved the deployment into the default namespace and it started working as expected (using , haven't tested the other names but I expect they now work too).

Hi I believe there's a bug in that is not allowing the endpoints for a headless service with no ports to be created. At and , the endpoints controller is adding a port with to the EndpointSubset that is eventually included in the newEndpoints which gets created/updated. In the API, this EndpointSubset is being validated at , and the port with is failing validation at This results in the server returning a 422, and the endpoints controller dropping the service out of its queue: Logs from API: Logs from endpoint controller: Steps to reproduce (logs above from this example): Hope this helps.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Open for fixing.
/assign
I had been missing this issue for a long time. Just found it again a couple days ago but haven't found a solution yet. Thanks for the fixing!


Question: ginkgo.It("each node by switching off the network interface and ensure they function upon switch on", func() { // switch the network interface off for a while to simulate a network outage // We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down. <del> testReboot(f.ClientSet, "nohup sh -c 'sleep 10 && sudo ip link set eth0 down && sleep 120 && sudo ip link set eth0 up && (sudo dhclient || true)' >/dev/null 2>&1 &", nil) </del> <ins> cmd := "nohup sh -c '" + "sleep 10; " + "echo Shutting down eth0 | sudo tee /dev/kmsg; " + "sudo ip link set eth0 down | sudo tee /dev/kmsg; " + "sleep 120; " + "echo Starting up eth0 | sudo tee /dev/kmsg; " + "sudo ip link set eth0 up | sudo tee /dev/kmsg; " + "sleep 10; " + "echo Retrying starting up eth0 | sudo tee /dev/kmsg; " + "sudo ip link set eth0 up | sudo tee /dev/kmsg; " + "echo Running dhclient | sudo tee /dev/kmsg; " + "sudo dhclient | sudo tee /dev/kmsg; " + "echo Starting systemd-networkd | sudo tee /dev/kmsg; " + "sudo systemctl restart systemd-networkd | sudo tee /dev/kmsg" + "' >/dev/null 2>&1 &" testReboot(f.ClientSet, cmd, nil) </ins> }) ginkgo.It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {

Which jobs are failing: Which test(s) are failing: Testgrid link: Reason for failure: check Anything else we need to know: /cc /milestone v1.18 /sig cloud-provider-gcp
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : Which jobs are failing: Which test(s) are failing: Testgrid link: check /cc /milestone v1.18 /sig cloud-provider-gcp </details
/sig cloud-provider
/area provider/gcp
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /area provider/gke </details
Probably related to
shows the merge shows the errant merge that inadvertantly disabled the test This test was probably failing since forever.
Any news?
this one is on cloud provider GCP.
Can you please take a look?
Follow up from today's release meeting: If the test has been broken for as long as we know it, theres value in just removing it. If the test owners don't have capacity to look into it, CI Signal can investigate.
Any advice on which team is responsible? It seems there isn't any maintainer for this job.
try talking to please
/assign
Turning to the question of whether these tests are actually useful... One of the tests that has been continuously failing is this one Which essentially does because From the logs (example ) it waits for 2 minutes (120 seconds) for each node to not be ready and then for the next couple minutes it just checks this condition until it fails So it looks like the e2e tests are not failing because of inherent infra issues (i.e., kube-up decided to do some weird thing) but the issue seems to be with the time to reach a condition. Again if we look at the logs we can see that switching the network interface does have an effect and leads to the nodes reaching a non-ready status but ultimately fail to come back up within the desired 5m time mark - this bit of the tests is where it all goes bad . The question then turns to turning off the network interface for a node in such a way () and then expecting the node to come back to live within 5 minutes of turning the interfac back on something reasonable?

If it is, then either we have to look at why it takes longer than 5 minutes or we can increase the timeouts If it isn't, then how can we test the ability of nodes to be rebooted?
/assign
Wanted to triple check on the test looking for nodes to be back up after being rebooted: I dont think 5 minutes is a hard deadline - only SLOs I could find were for scalability


Question: type ILBOptions struct { // AllowGlobalAccess Indicates whether global access is allowed for the LoadBalancer AllowGlobalAccess bool <ins> // SubnetName indicates which subnet the LoadBalancer VIPs should be assigned from SubnetName string </ins> } // GetLoadBalancerAnnotationAllowGlobalAccess returns if global access is enabled

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: ILB ip addresses come from the subnet that the K8s cluster uses. There is no ability to specify a different subnet. What you expected to happen: ILB ip addresses should be allocated from a different subnet if specified. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: GCE OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/area platform/gce /assign
/sig network


Question: case syncErr == nil: // No error; requeue at the regular resync interval. p.workQueue.Enqueue(uid, wait.Jitter(p.resyncInterval, workerResyncIntervalJitterFactor)) <ins> case strings.Contains(syncErr.Error(), NetworkNotReadyErrorMsg): // Network is not ready; back off for short period of time and retry as network might be ready soon. p.workQueue.Enqueue(uid, wait.Jitter(backOffOnTransientErrorPeriod, workerBackOffPeriodJitterFactor)) </ins> default: // Error occurred during the sync; back off and then retry. p.workQueue.Enqueue(uid, wait.Jitter(p.backOffPeriod, workerBackOffPeriodJitterFactor))

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Pods are synchronized by kubelet before network is ready. Synchronization in such case fails and pod is synchronized after 10 seconds. What you expected to happen: Kubelet should wait until network is ready before starting synchronizing pods. How to reproduce it (as minimally and precisely as possible): Configure cluster with Cluster Autoscaler enabled. Start pod so cluster is scaled up. In logs you might find following error: E0907 13:18:49. 972 ] Error syncing pod xxxxxx ("xxxxxxxxx(xxxxxxxxxxx)"), skipping: network is not ready: [runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: Kubenet does not have netConfig. This is most likely due to lack of PodCIDR] Environment: Build from HEAD Cloud provider or hardware configuration: GKE
/kind bug /sig node


Question: return false, nil } <ins> // Exists checks if VM exists and is not terminated func (vm *VirtualMachine) Exists(ctx context.Context) (bool, error) { vmMoList, err := vm.Datacenter.GetVMMoList(ctx, []*VirtualMachine{vm}, []string{"summary.runtime.powerState"}) if err != nil { klog.Errorf("Failed to get VM Managed object with property summary. err: +%v", err) return false, err } // We check for VMs which are still available in vcenter and has not been terminated/removed from // disk and hence we consider PoweredOn,PoweredOff and Suspended as alive states. aliveStates := []types.VirtualMachinePowerState{ types.VirtualMachinePowerStatePoweredOff, types.VirtualMachinePowerStatePoweredOn, types.VirtualMachinePowerStateSuspended, } currentState := vmMoList[0].Summary.Runtime.PowerState for _, state := range aliveStates { if state == currentState { return true, nil } } return false, nil } </ins> // GetAllAccessibleDatastores gets the list of accessible Datastores for the given Virtual Machine func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*DatastoreInfo, error) { host, err := vm.HostSystem(ctx)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We have k8s cluster with version . And I noticed that if I deleted worker node VM from IAAS, then node stuck in state, I created statefulset before and the pod scheduled on that node was stuck in state, and no new pod was created to replace that. Node should be deleted if no instance is found Related log When I check the log of I could see following log keeps popping up. Looks like node_controller noticed that the VM is gone but it doesn't delete the node. Related code here will return if no instance is found. But it should check if it's VM is deleted not shutdown, then return And it should cherry pick this fix avoiding deleting shutdown node What you expected to happen: After VM is deleted, the node should be deleted, pods assigned to the node should be rescheduled to other available nodes How to reproduce it (as minimally and precisely as possible): k8s cluster. In our case we use PKS. workloads on node worker node VM from IAAS Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: vSphere Cloud Provider
/sig cloud-provider /sig vmware
Same issue as this one The issue and is cluster API repo. We should fix it inside node_controller in k/k
Hi this issue was found when testing windows cluster deployed using PKS. We do have a workaround by manually deleting node using kubectl. Still we wish the fix could go in ASAP. Could you ask someone from upstream to to review the PR that fix this issue? Thank you
, we will try to push this through for v1.20. 1.19 is approaching code thaw, so we can't push this through
Thanks Michael. I'll let my team know.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale


Question: results <- err return } <ins> discoveryURL.Path = "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version </ins> errCh := make(chan error) go func() { <ins> // be sure to check a URL that the aggregated API server is required to serve </ins> newReq, err := http.NewRequest("GET", discoveryURL.String(), nil) if err != nil { errCh <- err

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: runs as a APIService but it fails with and the endpoint returns on kubernetes clusters with v1.15. It works with all previous cluster versions perfectly but gives the following status on v1.15 but I also don't see any changes in Releasing notes that effects APIService What you expected to happen: APISerice should run correctly and the endpoint should be responding. How to reproduce it (as minimally and precisely as possible): Linkerd dosent have to be installed to check this, and discovery happens directly at the url without any communication with the underlying mesh. In the repo, running this generates the required certs and applies the config to the cluster To check the status of the APIService, can be used Anything else we need to know?: I did check with clean 1.15 clusters and its the same. I also don't see any changes in 1.15 release notes that effects APIServices. Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: Testing is done on kind but its the same with minikube
/sig api-machinery
that sounds bad.. i didnt remember any breaking change except for making the port confiugrable.. /cc
what does interrogating the service directly return?
On Clusters 1.15, where the status of APIService is it gives On Clusters 1.14.3, where the status of APIService is it gives
looks related If the aggregated server returns non-200 status code, it is treated as unavailable. what endpoint is checked by the available controller?
i skimmed implementation from smi-metrics.. it only serves on prefixes so when the available controller checks the root endpoint it returns a 404.
but then it dosent fail on clusters running v1.14, In that version the apiService is marked as
fixed by , picked to 1.15.x in , will be in v1.15.1
Awesome Thanks for the quick fix :)


Question: // indicating whether processing should be retried; zero means no-retry; otherwise // we should retry in that Duration. func (s *ServiceController) processServiceUpdate(cachedService *cachedService, service *v1.Service, key string) (error, time.Duration) { <ins> if cachedService.state != nil { if cachedService.state.UID != service.UID { err, retry := s.processLoadBalancerDelete(cachedService, key) if err != nil { return err, retry } } } </ins> // cache the service, we need the info for service deletion cachedService.state = service err, retry := s.createLoadBalancerIfNeeded(key, service)

BUG REPORT: Kubernetes version (use ): Environment: Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: CC: What happened: There are resource leaks in the cloud with respect to PublicIp and LoadBalancer Frontend Ip Pool and HealthProbe What you expected to happen: All created cloud resources, associated with a deleted service, should be destroyed. This is causing the Azure Load Balancer to fill up its frontend IP Pool which is currently limited to 10 entries. After a few resource leak, the new kuberntes services cannot get valid External IP since the Azure Load Balancer is full. How to reproduce it (as minimally and precisely as possible): Create 5 namespaces and deploy the guestbook-all-in- example. (making sure you turn on the LoadBalancer type for the frontend service) Wait for all of the 5 PublicIP to be provisioned in the Azure Load Balancer. Delete all of the 5 namespaces using the Poll the namespaces until they are totally gone from kubernetes and recreate them. Redeploy the guestbook-all-in- in all 5 namespaces. You will probably end up with 5 new IPs BUT not all of the previous ones will be deleted. Anything else we need to know: Kube-Controller-Manager Logs Service IP after the first pod creation Service IP after the re-creation Azure frontend Pool Screen Shot <img width="799" alt="screen shot 2017-03-27 at 22 17 38" src=""
The following PublicIP were never removed
Does this repro on a single master cluster?
Yes, to ensure that all the logs were in the same file, I stopped all services on the other two kube-master. (e.g. kube-controller-manager, kube-apiserver, kube-scheduler) The log might still show entries when the configuration was setup as multi master
Update: I have retested with the deletion of the service/deployment instead of the namespace. I still have the same issue. As you can see 3 IP should have been delete from Azure.

52.167.217.196 52.167.223.67 52.167.222.61 <img width="721" alt="screen shot 2017-03-28 at 18 34 55" src=""Find the Debug Log for the kube-controller-manager here (start from the last Service Restart Entry)
I'm seeing that the controller is calling delete for only , which seems to indicate that the controller only processed a single delete callback for the load balancers. I don't have an azure cluster handy. -- if it's possible, can you repro with: Start with empty cluster Ceate guestbook example Delete guestbook example, but don't create again. see if the IP resources leak, and post the kube-controller-manager logs. I would like to understand if this occurs without having to recreate the objects immediately.
"delete the guestbook example" == "delete the guestbook namespace" or "manually delete the guestbook's svc"? (I only ask to clarify because I suspect that if simple "single service delete" were failing, we'd see a lot more noise/issues about it)
I can confirm that if I wait after the deletion that all of the cleanup job occurs there is no leak. The problem is when the deleted service is recreated with the same name under the same namespace.
The creation needs to occur while the deletion of the resource in Azure is still ongoing. The apiserver returns that the resources are gone and cleaned which allow us to recreate.
Can you post the kube-controller-manager log for the simplified repro? (create -delete -create) with one copy of the guestbook app)?
let me extract that information from the logs. I created test4 last, deleted it, then recreated in a continuous sequence. without deploying anything else at the same time.
(Not to interrupt any diagnosing, but FYI: I tried to intentionally use the value from the cloudprovider-levlel in order to avoid any sort of conflict like this: That value is used to name the LB rules/probes and NSG rules, with the hope that it would avoid these sorts of conflicts. But I'm not sure how is formed... if it's deterministic based on the names of things, then I can imagine there could be some conflicts occurring.)
do you have a preferred logging level ? it seems that level 10 logs too much... The journal output is missing a bunch of lines.

My original Log looks better and doesn't have missing lines. I think the first one was set at level 3
Here are the Azure Logs for the first test where test5/frontend's IP leak for example azureloadbalancer The azureloadbalancer with test5/frontend only shows thats the cloud provider was involved only for the creation and not the deletion. If we look at the we can see an example of the logs when the azure_loadbalancer gets called for deletion Adding a bit of logs from the controller related to The First time the Service was created and configured in Azure was at this time and line number () The second time it was created was at this time and line number () All of the specific logs below
the UID you use looks different between each delete/create First create/delete for test1/frontend Second Create
I think that the answer lies between line and in the first log file I attached from the OP
More investigation on my side For the 's service, we see this in the log when the controller manager tries to see if it needs to delete the service For the 's service we see the following instead The main difference between the two are those two lines  When it cannot delete the code is here. Which simply seems to return false, and ignores the deletion of the service all together. The weird part that I'm not too sure why, is that the deletion failures occur before the namespace was recreated and repopulated with resources. this is the last trace I can find for the deletion of the test5 namespace at the Namespace and its content get recreated here, at which is almost 1m30m later
I guess the difference can be explained by the that looks global for the client. The first pass, it get sets to which then trigger the exit right away.... So from the code, it just seems that the call to ` returns nothing....
and I have reproduced the issue with the smallest possible interaction. The following manifest (2 services with type:LoadBalancer) was used The Logs of the kube-controller-manager at lvl4 logging level from the time range of the test only. The IPs after the second create are The following IP and FrontEnd configuration leaked The commands executed were PS: Note that I was not able to reproduce with a manifest containing a single service object. As soon as I the second to the manifest, I was able to reproduce the issue right away.

Azure's LB snapshot <img width="788" alt="screen shot 2017-04-06 at 17 38 43" src=""
So this happens even outside of the namespace deletion? Is there anything obvious in the k-c-m logs? I'm very curious to know if this is a bug in the ServiceController or in the Azure cloudprovider code.
Also, do you see any errors in the Audit Log in the Portal for this resource group?
there are no issues in Azure' audit log nor the Activity logs for this RG. Regarding your question about the source of the issue, I would tend to say that the issue is in the ServiceController. Here's my reasoning. For the service we have the following lines Creation 1 Deletion Creation 2 Now for the service we have the following lines Creation 1 Deletion Creation 2 As you can see there is not even a call from the class to issue a delete based on this log entry ,hence the class is never called. Now, as to why the class forgets about the deletion and moves on to the recreation. So far, my best bet is that the deletion process and handled one at the time, and since it takes a while for the IP to be removed in Azure, the logic in the detect that frontend2 was created and is in need of an IP. Right before the frontend2 second creation I see this. Also, the order of creation/deletion is creates test/frontend creates test/frontend2 deletes test/frontend creates test/frontend2 creates test/frontend
So I looked at the code and I see that my hypothesis was true. (if I'm not mistaken) Here's the break down of my analysis. The Cloud-controller-manager calls the command with ConcurrentServiceSyncs hardcoded to (it cannot be change either from the command line) The service controller creates a single worker thread that uses a workingQueue to get the list of services to process. The worker's sole purpose is to synch the Service. Upon the deletion of both services, the queue gets filled up with two entries: and The worker pops out and starts processing the deletion. It takes around 3mins. Before the end of the deletion of , both services are recreated and the queue gets repopulated. After is deleted, is removed from the queue to be processed.

In the method, the service is queried to see if it is present Since it was readded (with the same name) there are no Error returned, hence no deletion requested. From here, I'm not 100% sure what would be best. what is your take here ? should the SynchService stop querying for the Service existance in the apiserver and be given the Instruction to add/remove the service directly from the Informer's DeleteFunc method? Knowing that there is only a single worker. The queue could look something like this So all of the creation and deletion would be process in the right order. The only problem I see with this right now is that there seem to be a retry mechanism that could re-queue a delete action after a recreate... I guess the safest bet is really have a way to prevent the apiserver from accepting the same service to be created if the CloudProviderPublicIP has not been removed... This is going back to this Issue I guess(). Even though the issue is requesting that the user be able to query the deletion Status, here we would want the apiserver to not delete the service from his list until it is fully deleted...
This looks to be me like an A-B-A () problem with the cloud resource. Let me do some digging...
This looks to be another instance of If the reconciliation was performed periodically, the IPs will be garbage collected.
could you explain me what the reconciliation would provide to the serviceController ? I've read the issues you linked, but I'm still not 100% sure I understand the proposed logic. (probably because I still don't know how the service controller fully works -- mostly with respect to caching)
The idea is there would be a continual loop over the cloud load balancers to see if there are any configured that should be there. If there is a mismatch, the load balancer will be deleted. Admittedly, this is not a hugely satisfying answer. It's along the lines of: there is no sync up of the Kubernetes state with the cloud provider state in the service controller and this can lead to stranded/inconsistent resources.
I see. thanks for the explanation. What do you think about having the master prevent the creation of a service that was previously deleted until the cloud controller manager is done with his clean up job ? (e.g.

all configuration are deleted in the cloud's LB) That should cover the use case where services are deleted and recreated almost back to back.
I still don't even understand why there's a conflict when they're deleted/recreated back to back. The Service UUID should be different. Or is it because the SerivceController stops trying to delete the previous service because ServiceController only tracks by name and not by the UUID?
I guess either way there's still agreement that this is a bug in ServiceController layer... please tell me if this isn't the case and we need to be more actively engaged from the Azure side.
-- as far as I can tell, it is tracked by name and not UUID
correct, the key used to track consist of the string:
This seems to be a result of using the namespace/service name in . I can reproduce this easily by causing the ServiceController to get busy syncing a new service and then deleting and recreating a different service like this: In the controller-manager I see logs like this (I'm using the Openstack provider in this case): You can see that the ServiceController essentially "lost" the delete because the name was the same. Because the calling syncService does not work concurrently (here ), by the time it finished handling and started on , the service had already been recreated. So when sync service checked it (here ) it already existed again and dropped into the case instead of the case like we would want. The end result is that the not only the public IP but the cloud provider's load balancer is orphaned.
Any updates here?
No time to take this up in this release -- punting.
/sig azure
/area platform/azure
remove from 1.7 milestone
Move to next milestone per sig/network's request.
any chance this could be picked up for 1.8 ?
I'll take a stab at this... Note I'm going to remove the sig/azure since this seems to be a global issue in the
I've run into this today (I think - at least the symptoms seem to be the same, with services perpetually stuck in for public IP, and I have been creating, deleting and recreating services perhaps too quickly). Is there a workaround?

(I saw this problem on a test cluster, so I've resorted to deleting the cluster and re-creating it for now, but I'd like to avoid that in the future if I can, since it takes a while... Also, it's not done yet, so I don't know if it actually solves my problem - but if it doesn't, my problem is likely not related to this issue.) When I do there's only three IP:s listed for my subscription, so I know I'm not hitting the limit of simultaneously provisioned IP:s (the defaults allow 20) - do you know of other limits I could be hitting? Are there other resources except the public IP addresses that leak, that could be subject to similar caps? I'm running k8s 1.9.2 on the client, and 1.8.7 on the cluster.


Question: configDir string } <del> // NewFSLoader returns a Loader that loads a KubeletConfiguration from the files in `configDir` func NewFSLoader(fs utilfs.Filesystem, configDir string) (Loader, error) { </del> <ins> // NewFsLoader returns a Loader that loads a KubeletConfiguration from the files in `configDir` func NewFsLoader(fs utilfs.Filesystem, configDir string) (Loader, error) { </ins> _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() if err != nil { return nil, err

From in The most important things I want to see through node e2e tests: 1) the kubelet is started from last-known-good-config file 2) kubelet can download the new config properly to the node; and restart itself from the new config file 3) kubelet can mark the config bad properly after several round of crashing caused the bad config 4) Kubelet can roll back the last-known-good-config and restart itself from that good one 5) kubelet can properly reports its configuration status Note: (2) is covered by existing e2enode tests (1, 4, 5) were covered by the , all of which I plan to turn into automated tests after this PR merges - but I'm trying not to increase the size of this PR any further ~(3) we need to figure out how to test this. I don't want to rely on a "valid" config that causes a crash loop - because the very existence of such a config should be viewed as a bug. I also don't want to include a "please crash loop for our tests" field in the config, because that's an ugly API. The best solution is not clear here.~ We decided to remove crash-loop detection, see . Filing this issue to track this work: into e2e_node tests [x] remote config [x] local (from file) config [x] ~test the crash-loop detection~ removed, see above /cc
[MILESTONENOTIFIER] Milestone Removed Important: This issue was missing labels required for the v1.8 milestone for more than 7 days: kind: Must specify exactly one of [, , ]. priority: Must specify exactly one of [, , ]. Removing it from the milestone. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
The test cases in the OP have been covered by our e2e_node tests for a while.


Question: "fmt" "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/util/rand" </ins> utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" "k8s.io/kubernetes/pkg/features"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Volume binding mode doesn't consider the volume attach limit per node. What you expected to happen: volume binding mode should have respected the attach limit, similar to binding mode. How to reproduce it (as minimally and precisely as possible): In CSI driver, configured the volume-attach limit to 1 per node on a 3 node cluster. In volume binding mode, only 3 pods with 3 PV/PVCs runs. In binding mode, all 5 pods runs with 5PV/PVCs on 3 node cluster. (Actually 4 pods scheduled on 1 node and 1 on other node, 3rd didn't have any pod scheduled) Anything else we need to know?: I was able to reproduced it with minimal node and configuration, and also hit this for stress case with 100 Volume limit and 500 pods running on the 5 mode cluster. Environment: Kubernetes version (use ): ~ kubectl version Client Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-10-30T21:39:38Z", GoVersion:"go1.11.1", Compiler:"gc", Platform:"darwin/amd64"} Server Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-12-03T20:56:12Z", GoVersion:"go1.11.2", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: On-prem linux server OS (e.g. from /etc/os-release): Centos 7Kernel (e.g. ): 3.10.0 Install tools: Others:
cc
/sig storage
/assign Currently the predicate PVCs that are not bound. It should lookup the storageclass to get the provisioner/driver name. This solution will require a StorageClass object is even if you don't use dynamic provisioning.

I had a thought that it wouldn't be so simple to count unbound PVCs of other pods since binding is still in progress, but I think it's fine actually. We only count other pods that have been assigned to the same node. Those pods are assigned to the node AND still unbound because of one of these reasons: It's actually been scheduled to the node and someone manually came in and unbound the PVC: unlikely and an unsupported sequence since we implemented StorageProtection The Pod was initially created with nodeName already set, bypassing the scheduler The Pod is going to be scheduled (it's been assumed), and it's currently going through binding. This is normal with delayed binding. Either way, the fact that the Pod's nodename is set to this node means that the pod should be counting towards the node's limit.
Hmm, so after opening the PR that fixes this, I realize that - from storageclass we don't really know if the volume we are evaluating is CSI volume or not. I think a similar problem exists for existing predicate though. Like AzureDisk predicate does not know if PVC it is over counting is of type AzureVolume or not. So, I suspect that those 4 hardcoded predicates are counting all delayed binding volumes.
if the storageclass.provisioner name is not found, can we just ignore it?
I think we can ignore it. The PR currently does that.