Question: _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating secret %s in namespace %s", secretName, namespace) <del> // kubectl create -f clusterrole.yaml _, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ </del> <ins> if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { // kubectl create -f clusterrole.yaml _, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ </ins> <del> ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver-reader"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(), }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role %s", "sample-apiserver-reader") </del> <ins> ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver-reader"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(), }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role %s", "sample-apiserver-reader") </ins> <del> _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":sample-apiserver-reader", }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "sample-apiserver-reader", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":sample-apiserver-reader", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":sample-apiserver-reader") </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "sample-apiserver-reader", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":sample-apiserver-reader") </ins> <del> // kubectl create -f authDelegator.yaml _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":auth-delegator", }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "system:auth-delegator", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> // kubectl create -f authDelegator.yaml _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":auth-delegator", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":auth-delegator") </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "system:auth-delegator", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":auth-delegator") } </ins> // kubectl create -f deploy.yaml deploymentName := "sample-apiserver-deployment"

[X] Test approval issue: [X] Test PR: [x] Two weeks soak end date: [X] Test promotion PR: According to this APIsnoop query, there are still some remaining APIService endpoints which are untested. Using a number of existing e2e test practices the has to cover the outstanding APIService endpoints. There has been some around how the current Conformance test is using RBAC, which is optional and not part of Conformance. Those calls will now only happen if is true. The e2e logs for this test are listed below.

This query shows the following APIService endpoints are hit within a short period of running this e2e test example endpoint | useragent ---------------------------------------------+
createApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] deleteApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] deleteApiregistrationV1CollectionAPIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] listApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] patchApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] patchApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] readApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] readApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] replaceApiregistrationV1APIService | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] replaceApiregistrationV1APIServiceStatus | Should be able to support the 1.17 Sample API Server using the current Aggregator [Conformance] (10 rows) If a test with these calls gets merged, test coverage will go up by 4 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted


Question: go_library( name = "go_default_library", srcs = [ <ins> "autoscaling_timer.go", </ins> "cluster_size_autoscaling.go", "dns_autoscaling.go", "horizontal_pod_autoscaling.go",

Currently the time it takes for HPA and CA to scale up after load increase is not measured at all, and this can lead to regressions that will be missed. This issue is about adding an e2e test that will prevent such regressions.
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list .
Filip, can you reopen? I cannot do this myself, and this issue is clearly not closed yet.
Thanks Wojtek!


Question: for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { pod, err := c.Pods(ns).Get(podName) if err != nil { <del> Logf("Get pod failed, ignoring for 5s: %v", err) </del> <ins> Logf("Get pod %v in ns %v failed, ignoring for 5s: %v", podName, ns, err) </ins> continue } done, err := condition(pod)

Testing how kubelet client rotation works, I discovered that on startup kubelet uses the cert in () and not the new cert in () This prevent kubelet to start kubelet uses the latest client certificate a 1 node cluster with kubespray (kubeadm should do) ntpd/chronyd the date 11 months in the future master certs kubelet that you have a new client cert the date 11 months more in the future and restart kubelet kubelet fails to start because it uses the cert embedded in instead of Both and are set, if I remove kubelet boots but fails to renew the cert Here an easy fix: Also I noticed that is RSA where is EC/P-256 Kubernetes version (use ): v1.18.15 (I can retest in more recent version but haven't seen anything in the release notes) Cloud provider or hardware configuration: VM OS (e.g: ): Kernel (e.g. ): Install tools:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cluster-lifecycle (not sure if this is more kubelet or kubeadm)
is the embedding certificates? see the second warning under here:
you can call on the same node to make the point to the rotatable file. newer versions of kubeadm do this change automatically.
i don't see a core bug here. i missed that you are using kubespray, which uses kubeadm. please report any issues with the kubespray issue tracker first: /kind support /close
Closing this issue. <detailsIn response to : i don't see a core bug here. please triage any issues with the kubespray issue tracker first: /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks kubespray indeed doesn't call anywhere, I'll go look at how to fix that
like i said, newer versions of do this automatically.

this might be the result of an old cluster being upgrade using but the is remaining stale. the manual step in or calling the on a node is required.
It was a fresh deployment of k8s 1.18 / kubespray 2.15, I'm doing a fresh deployment of k8s 1.20 / kubespray master right now, but maybe kubespray is using kubeadm in weird ways / include old workarounds that leave us with having embedded certs instead of pointing to .
Found the culprit: Kubespray regenerate the conf files using kubeadm to make sure the server is up to date Thanks for the help and sorry for the spam
it would make sense to just include the kubelet-finalize phase in there too.
I prefer to just change the server using one of lineinfile/sed/kubectl so next time there is a new step like kubelet-finalize we don't break again.


Question: ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns) jig := e2eservice.NewTestJig(cs, ns, serviceName) <ins> jig.ExternalIPs = true </ins> servicePort := 8080 svc, err := jig.CreateTCPServiceWithPort(nil, int32(servicePort)) framework.ExpectNoError(err)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We ran sonobuoy on one of our clusters running on AWS and it reported two failed tests: This happened because the pod employed to conduct the tests wasn't able to reach the service using the node's public IP address: What you expected to happen: The security groups on that cluster don't allow for connections to arbitrary ports through the public IP address. The tests should still succeed because that configuration is seen as a security feature. How to reproduce it (as minimally and precisely as possible): Spin up a Kubernetes cluster on AWS EC2 instances, configure SGs to only allow traffic to the instances from other instances (or from an ELB). Run sonobuoy. Anything else we need to know?: Environment: AWS Kubernetes version (use ): 1.16.8 Cloud provider or hardware configuration: AWS OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/area conformance /sig network
related:
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
I am also seeing this issue when running conformance testing. The nodes in our test cluster (where we run conformance tests) have public IP addresses, but nearly all ports all firewalled. We are also running in AWS. These tests should only rely on the internal IP address of the nodes.
/assign
Yes, I have seen this before, that if the nodes have an external address it will use that one, which means the firewall must be open for that. If the nodes do not have an external IP then the test will pass just fine. It's not as much a bug as an inconvenient implementation. But we can treat it as a bug and fix it. The intent of the test is to make sure the service is plumbed properly.
/remove-triage unresolved
Good point. However, we only assign an external IP address to be able to SSH into the host and thus block all other ports.
Issues go stale after 90d of inactivity.

Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
I'm seeing the same issue, though I noticed that it happens only on one node in the whole cluster. Even after re-creating a cluster from scratch, the test always fails only on one specific node. However this node is identical to others, but the test succeed on rest of the nodes.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
In some user scenarios, the external ip nodeport is for access from APPs outside this cluster/zone. However, in most scenarios, external IP is just for SSH or special use(Ingress Gateway, etc.) Different users have different proposals. Why not disable e2e tests checking on external IP with a flag? Will send a PR for this.
That's not all that useful for those running conformance tests. Would be better to change it to always use the internal IP, we don't need the external access for conformance purposes.
/assign
agree, also most of the network tests now run from the cluster, no need to reach externalIPs please tag me in the PR
So kubernetes applications should not expect to be able to access them with an external node address? FWIW we also require upstream / public internet access in the DNS tests, so needing somewhat unrestricted networking to test the cluster behavior is not new.
I'm not extremely familiar with the conformance tests. Are the DNS tests you are referring to about being able to access public services from inside of the cluster (egress)? Because our problem is that our default Kubernetes deployment doesn't allow ingress other than SSH and we haven't seen any problems with the DNS tests so far when checking our product for conformance. That's why I like approach of disabling that specific test with a flag.

Maybe that could also be generalized to let all conformance tests know this is a locked-up cluster with basically no ingress. But that may be a matter of a subsequent PR/issue.
Disabling the tests with a flag is not really an acceptable approach for conformance. The whole point of the conformance program is to validate the Kubernetes implementation against a standard set of tests across all vendors etc. If we don't think access like this is a standard Kubernetes behavior applications should expect anymore, the tests should be removed from conformance and we can consider instead running them just to test the Kubernetes project but not in conformance by moving them to a new test. IMHO we usually only increase the scope of conformance behavior though, not decrease, and it's expected that some users will have more secure clusters that fail conformance (for example we test some privileged behaviors IIRC) ...
After reading comment, I agree that the test should just be removed. However, I'd like to first understand the reasoning behind it being there in the first place. Maybe there's a good reason that such "secured cluster" aren't deemed compliant?!
so, the next step is to review the conformance definition for nodeport and modify the test accordingly Node.Status.Addresses are not a great thing, bad semantics, ... this KEP and this PR summaries perfectly why I personally, would move ExternalIP from Conformance entirely until it has a clear definition
So we should remove it now and then add a new test if needed after a conclusion from the discussion in the KEP. Edited. I may misunderstand. A proposal is to move those external addr NodePort test from conformance to normal e2e test cases In Conformance Case, don't check external ip's NodePort. In e2e test case, check them.
My point is that Conformance folks (cc ) should decide if they want to consider ExternalIPs as part of the conformance definition, I pointed to some issues related to the topic here sig-network will keep testing externalIPs, are part of the API and we need to verify them, Conformance has a narrower scope
The fact that these tests are using ExternalIPs does not need to be in conformance and in fact really shouldn't be. But we still need the tests working against InternalIPs for conformance. It's more of an historical artifact that they use ExternalIPs than an intentional choice.

So, agree with that sig-network should test them, but we should remove the use of the external IPs from conformance, and leave in the tests but against the internal ips only. I resolved this, for example, in GKE conformance runs by using a private GKE cluster that does not give the nodes external IPs.


Question: fi } <ins> # Creates a new DHCP option set configured correctly for Kubernetes # Sets DHCP_OPTION_SET_ID function create-dhcp-option-set () { case "${AWS_REGION}" in us-east-1) OPTION_SET_DOMAIN=ec2.internal ;; *) OPTION_SET_DOMAIN="${AWS_REGION}.compute.internal" esac DHCP_OPTION_SET_ID=$($AWS_CMD create-dhcp-options --dhcp-configuration Key=domain-name,Values=${OPTION_SET_DOMAIN} Key=domain-name-servers,Values=AmazonProvidedDNS | json_val '["DhcpOptions"]["DhcpOptionsId"]') add-tag ${DHCP_OPTION_SET_ID} Name kubernetes-dhcp-option-set add-tag ${DHCP_OPTION_SET_ID} KubernetesCluster ${CLUSTER_ID} $AWS_CMD associate-dhcp-options --dhcp-options-id ${DHCP_OPTION_SET_ID} --vpc-id ${VPC_ID} echo "Using DHCP option set ${DHCP_OPTION_SET_ID}" } </ins> # Verify prereqs function verify-prereqs { if [[ "$(which aws)" == "" ]]; then

I was successfully able to get a cluster to start up in us-west-2, the default region for kube-up, but for some reason I'm not successfully able to start up a cluster in us-east-1. I'm not seeing anything in the logs that seems to be a huge problem, so I'm not really sure how to resolve this. Kubelet log here: I'm not really sure what else to look at, I'm happy to do more investigation, but not really sure where to look. Most others log files are empty with the exception of kube- which is failing to start addons due to the unstarted api server.
cc/ Could you please help us with this?
Sure ... this sounds like a name problem - as I recall the name is supposed to be ip-172-20-0- (or similar), not just ip-172-20-0-9. I'll try in us-east-1. What revision of the software is this ?
I built from the latest master, but I have seen the same issue on recent releases.
Testing now... I wonder if this is related to and
if you entries in etcd were the fqdn of the machines then you would have been bit by the same issue as us.
I was skeptical about Running that curl returns "ip-172-20-0-9" and not "ip-172-20-0-9.ec2.internal" as we might expect?
Nvm, I tried just appending ".ec2.internal" to no avail.
I just tried it, and I was able to bring up a cluster in us-east with the latest code. I checked on my kubelet on the master, and did see your error: So I think that error is a red herring. Of course we should fix it, though I don't know if the problem is that it is missing the .ec2.internal suffix or whether the problem is that we're not registering the master as a node at all. Do you see the API server in (or ) on the master. I suspect it isn't running. Or maybe it isn't configured on your kubelet - what does show?
Nothing seems to be running:
That is odd... here is my output: Note no Note that I have arguments to kubelet (a lot of them!)

The lack of arguments matches with the " No api server defined - no node status update will be sent." message in your kubelet log (which I don't have) On the "unable to resolve host ip-172-20-0-9" error, I can . It isn't in hostname, but rather I wonder if you can't ; a possible reason would be that your VPC doesn't have name resolution enabled. Did the script create it for you? Are there any hints in ? Here is mine:
VPC was created by the script and dns support and dns hostnames are both turned on. Interestingly, I'm unable to ping ip-172-20-0-9 and my is missing the search line: looks pretty close, but noticed some extra lines re: Jinja:
I think it might be the DHCP Options Set. I have another VPC in east that already had a DHCP Option Set and it appears that the kubernetes cluster is using that same set. When I looked in west I found that the set was configured as such: Whereas in east, my DHCP option set just looked like this: Perhaps we should be explicitly creating a new DHCP option set to avoid conflicts with existing VPCs? I'd be happy to create a PR if this turns out to be the issue and if you think the above is a good solution.
That is almost certainly it - nice find! I don't fully understand how this actually happened (I guess a new VPC defaults to whatever DHCP option set already exists, although that isn't what the docs say), but I would welcome a patch that created a working DHCP option set when creating a new VPC, and/or warned if a VPC had a DHCP option set which didn't include a domain-name.
Yep, cluster in us-east-1 is up and running! Thanks for all of the troubleshooting help! Another option might be to set the DHCP option set to default, which should be configured correctly. It doesn't seem like you can modify the default, so it might be a safe choice. Plus, we don't have to set up the option set, we can just assign it through the . I'll do some experimentation and make a PR with what seems to be the most sane.

The documentation for option sets seems contradictory, in one place it says it will create a new option set and in another it says it will use the default. It seems it does a little bit of both. :)
should resolve once it merges.
Unfortunately this issue is still exist for me. Even tough a DHCP option is created for the Kubernetes VPC I've get an endless loop.
I figured it out, The issue was that as part of the dhcp set creation it didn't set a domain-name to be "ec2.interna" as described here - and only set the domain-server to AWS defaults. Once I changed that dhcp set it worked perfectly. I can see you a function for that issue in , it might not merged to the master? Thanks


Question: // print the current object if !isWatchOnly { var objsToPrint []runtime.Object <ins> writer := printers.GetNewTabWriter(out) </ins> if isList { objsToPrint, _ = meta.ExtractList(obj) } else { objsToPrint = append(objsToPrint, obj) } for _, objToPrint := range objsToPrint { <del> if err := printer.PrintObj(objToPrint, out); err != nil { </del> <ins> if err := printer.PrintObj(objToPrint, writer); err != nil { </ins> return fmt.Errorf("unable to output the provided object: %v", err) } } <ins> writer.Flush() </ins> } // print watched changes

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): Is this a BUG REPORT or FEATURE REQUEST? (choose one): <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --BUG REPORT Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"6+", GitVersion:"v1.6.0-beta.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-03-11T04:20:22Z", GoVersion:"go1.7.5", Compiler:"gc", Platform:"darwin/amd64"} Server Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.4", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-03-07T23:34:32Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"} Environment: Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: What happened: With kubectl version 1.6.0-beta.3, the output of is not aligned: What you expected to happen: With kubectl version 1.5.4, the output is as follows: How to reproduce it (as minimally and precisely as possible): Run using kubectl version 1.6.0-beta.3. Anything else we need to know:
will take a look
any progress on this?

hi, tony, sorry, I haven't look at this problem yet, maybe tomorrow, just really busy recent day, you can take it if you already find the problem, thanks
Happened to see the same issue on my laptop, I can help fix this issue if you do not have time, thx
thanks, I've been really busy this week, go ahead
for this ticket, want to discuss one more thing with you: at present, for each of the update event, command with option will print out a line showing the status change of this event. But this behavior is different with what we saw from the output of linux's command, which I think is not a very good user experience. For this kind of command, we shall reset all the old output and re-print all the pod's information in the standard output, this will make this command more like a real command. do we have any plans on this right now? welcome your ideas on this.
I think both output format has it's advantage, for current output format, user can copy paste the whole information like checking logs, for linux's command style output which user can stare on only one screen(if it's not too many pods or sth else).
yep, we do have plan to do this, and I really like a more linux style command, you can refer for more detail. Anyway let's just get this fixed at first.


Question: } } if usage != nil && usage.Cmp(requests) > 0 { <del> message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String()) </del> <ins> message += fmt.Sprintf(containerMessageFmt, container.Name, usage.String(), requests.String(), resourceToReclaim) </ins> containers = append(containers, container.Name) containerUsage = append(containerUsage, usage.String()) }

when the pod is evicted due to low imagefs.available, the prompt is difficult to understand. it showed: But the container was evicted not by exceeding its request of 0 ephemeral-storage, but due to reaching the evictionHard of the node. The correct prompt message points out the reason for the pod eviction. don't print "which exceeds its request of 0.", because the container was evicted not by exceeding its request of 0. % oc label mcp worker custom-kubelet-eviction=small-pods evictionHard by creating a kubeletconfig % oc create -f custom-kubelet- custom-kubelet- several minutes, the mcp worker finish rolling out to the latest mc a pod % oc create -f hello-pod- hello-pod- a large file in container, make imagefs.available < 90Gi (the current available disk size is 111G ) command terminated with exit code 137 the node status the pod status % oc get pod -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES hello-pod-2 0/1 ContainerStatusUnknown 1 7m58s 10.131.0.7 ip-10-0-213- <none<none% oc get pod hello-pod-2 -o yaml No response <details</details<detailsaws </details<details</details<details</details<details</details<details</details
/sig node
/assign
/assign
/triage accepted


Question: } } <del> func TestEndpointAddressNodeNameInvalidDNS1123(t *testing.T) { </del> <ins> func TestEndpointAddressNodeNameInvalidDNSSubdomain(t *testing.T) { </ins> // Check NodeName DNS validation <del> endpoint := newNodeNameEndpoint("illegal.nodename") </del> <ins> endpoint := newNodeNameEndpoint("illegal*.nodename") </ins> errList := ValidateEndpoints(endpoint) if len(errList) == 0 { t.Error("Endpoint should reject invalid NodeName") } } <ins> func TestEndpointAddressNodeNameCanBeAnIPAddress(t *testing.T) { endpoint := newNodeNameEndpoint("10.10.1.1") errList := ValidateEndpoints(endpoint) if len(errList) != 0 { t.Error("Endpoint should accept a NodeName that is an IP address") } } </ins>

introduced a new requirement for node names to be DNS1123Labels. In case of hack/local-up-, an IP address is used as node name and thus endpoints aren't created for services, e.g. kube-dns doesn't work anymore. There are other projects that use IP addresses as node names for running simplified k8s dev environment: Probably there are other projects where it's convenient to have just an IP address as a node name. Verified against . On affected installs, service latency e2e test hangs, see e.g. Example ( some newlines):


Question: }, }, }) var pausePids []int ginkgo.By("checking infra container's oom-score-adj") gomega.Eventually(func() error {

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: ci-kubernetes-node-kubelet-serial Which test(s) are failing: [] Container Manager Misc [Serial] Validate OOM score adjustments [NodeFeature:OOMScoreAdj] once the node is setup pod infra containers oom-score-adj should be -998 and best effort container's should be 1000 Since when has it been failing: Testgrid link: Reason for failure: Anything else we need to know:
/assign
Let's monitor this one and see if worked /reopen /milestone v1.17 /priority critical-urgent
Reopened this issue. <detailsIn response to : Let's monitor this one and see if worked /reopen /milestone v1.17 /priority critical-urgent </details
It seems work well.
These now have 8 successful runs without a single flake. Looks like worked! :smile: 's%20should%20be% cc
thanks /close
Closing this issue. <detailsIn response to : thanks Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: } // Handle filesystem notify event. <ins> // Files names: // - MUST NOT start with a '.' </ins> func (w *Watcher) handleCreateEvent(event fsnotify.Event) error { klog.V(6).Infof("Handling create event: %v", event)

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug This is more of an enhancement. What happened: The Plugin Watcher is setup to trigger on file activities when they occur in the subdirectory that it scans. During startup, the Watcher is passed (from within the Kubelet) the Kubelet's root plugin directory as a location to scan for newly plugins - Because the kubelet's plugins subdirectory is also used by existing in-tree drivers (including csi) as a location to store plugins artifacts, the Plugin Watcher is getting triggered by directory/file activities occurring in that subdirectory that has nothing to do with plugin registration as shown in the output below: What you expected to happen: The Plugin Watcher should not be getting triggered by file/directory activities not related to plugin registration/unregistration. One possibility is to have the Watcher scan one directory below plugins like or something similar How to reproduce it (as minimally and precisely as possible): Run plugin watcher with a CSI driver. cc
/sig node
I think you should only place the plugin socket file in the plugin registration directory but not the plugin data.
agree with we can only place the registration file in the specified path instead of placing all data into it. On the other hand, you can place data into the /var/lib/kubelet/plugins/registration even if registration created, that's a no-end loop.
As I mentioned in issue, existing in-tree volume plugins were already using that directory as a location to store storage operation artifacts (not just csi). Here is a shortened list of driver locations that use /var/lib/kubelet/plugins in their code already: And several more locations. The fix would be to have plugin watcher scan a subdirectory of plugins (say ) instead of plugins directly as it is doing today. The watcher functions as intended. Except now it picks up all directory activities whether related or not to plugin registration.
Well given the fact that the pluginwatcher is beta this is a bit of an issue. My understanding of the "beta" guarantees is that we want to be backwards compatible for all plugins published with the beta API. Meaning that even if we move the registration directory to a new location, we'll need to keep scanning the old directory (to pick up v1beta plugins).

From a 10 000 feet point of view it seems easier to me if we move the in-tree volume plugin's data. Though it might be a lot of work. Another less painful option, but a bit grey is the following: Enforce plugins to end with this suffix is recommended for device plugins (though not enforced) and I believe is a convention on linux. This is a bit grey because we would still be breaking v1beta users but since we're moving a convention from "recommended" to "enforced" it seems "less" breaking to me (and I haven't seen device plugins not using that extension). What do you think ?
I like the suffix idea. It can be a recommendation (which csi is already doing).
Hmm, I've just deployed a new k8s cluster on 1.12.0 and noticed this repeating in the logs on a few of our pods. Just for my own sanity, is this something I should be worried about or can I ignore it until a fix is released?
you can ignore these errors. if it's a recommendation then we'll keep spamming the logs with all the data files. Another thing to keep in mind, it seems like CSI plugins puts a lot of data (and creates a lot of directories). It might be worth considering limiting the recursion to one level. I'm concerned we might exhaust the amount of fds by tracking the plugins data.
yeah CSI keeps track of volume operation metadata in the plugins dir. This probably will not change. I understand this is beta. However, besides CSI and Device plugin, are there any other implementers out there for this feature ? Would it be possible to propose a new location (i.e. <pluginsDir/reg/?
let me know if you open a PR for this as discussed.
thanks for open the issue with detail logs, i can see two problems by looking at logs and code. 1) plugin watcher trying to dial non-socket files. 2) too many logs generated (CREATE and DELETE). I a suggestion to fix of this issue we can fix those problems by: 1) check file mode when handle CREATE event, only attempt to dial file when file mode is socket.

2) increase log verbose level for both 'handleDeleteEvent()' and 'handleCreateEvent()' plugin watcher still has activity when any file create/delete under the directory, but they become light weighted (CREATE file will only trigger file mode check, DELETE file event will trigger plugin searching) i am not sure we want to go further to limit directory depth where plugin can live. it has two drawbacks IMO: 1) it is hard to enforce, not sure everyone will read code/readme. 2) it provides less flexibility to plugins: if want to organize many plugin socket files with directory.
These are definitely good suggestions! It seems like there might be some confusion as to why we are thinking of doing this: Most plugins use the plugin directory to store data (actually a lot of files / dirs) Watching all the directories and files created inside the plugin directory puts more burden on the Kubelet in terms of cpu / memory consumption Watching all the directories and files created inside the plugin directory quickly raises the number of fd consumed By limiting the search of the plugins to one directory, we would be severely limiting the ability of the module to unintentionally harm the overall Kubelet. It seems actually pretty easy to enforce, as it's a simple condition. As for not everyone reading code / readme, we can add the documentation to the gRPC interface if this is a concern. In terms of flexibility, is there a real need to create a plugin in a lower directory? Is there a plugin that has so many sockets that a single directory underneath the plugin directory isn't enough? Naively it seems to me that is enough for the different plugins we enable.
ping on the above :)
Naively it seems to me that $ is enough for the different plugins we enable. i can not speak from plugin developer, but both below looks reasonable to me , or as long as we can communicate to plugin developers clearly and effectively (gRPC interface documentation is needed), the directory depth restriction is reasonable to me.
I like . However, why not put plugintype one level up as . For instance
Is the suggestion to hard code the number of levels that the recursion algorithm will search to 3? Or just expect a socket only at a hard coded level (e.g. socket must be nested 3 deep)? Copying my comment from for consistency: Another option I can think of: recursively scanning but before scanning a directory check if it is mounted ().

This will be backwards compatible and will prevent scanning of volume data Thoughts?
i like this idea better if it is always the case that: volume data is mounted, we can modify the watcher code to not watch the mounted directory in the first place.
Skipping mounted dir along with restructuring search path to known (shallow) dir sounds like a good plan. Though with the new suggested paths, sockets will be placed at a location where mounts should not exist.
I think that would be a safe assumption. The plugin directory is only used as a target path for global mounts.
Ephemeral volume types may not need to mount, it could just be a symlink or plain directory. Can we assume any ephemeral volume CSI driver won't implement global mount?
Yes, in-tree ephemeral volumes today don't do a global mount so the mounts directory is which is not the plugin dir. So no issue there. For CSI the will be the global mount path (this plugin dir). Emphermal volumes shouldn't implement call -- but nothing prevents them from doing so. Maybe let's start with just checking for mount and ignoring mount directories. If it is still an issue we can consider adding a configurable black list of paths. Thoughts?
That sounds good. Also, putting the socket files in a specific location that does not clash with mounts will also help (i.e. )
/milestone v1.13 /priority important-soon
I just want to point out that one of the issue we have with recursively watching all directories is that it will significantly increase the amount of fd consumed. Given that plugins actually store data there and in a dynamic way my fear is that it will exhaust the available fds for no real good reasons. Limiting the depth sounds like a reasonable solution to that issue.
btw, because of this verbosity in my cluster writes 5G logs every day and filled up my rootfs (which is small). Is there any way to decrease this verbosity?
We can do both. Don't search mounted dirs, and max depth of search. Who is working on this?
That's pretty bad, can you open another issue to track that.
we have a PR in flight discussing this issue (recursive scanning) here: Thinking more about this, the fact that too much logs is generated is clearly linked to this specific issue and how we process plugins.

In any case I'll be fixing that issues too
how confident are we in getting this fixed by Code freeze tomorrow? 5pm PST
I expect things to wind up today, changes in the above PR aren't very contentious and don't involve significant changes


Question: export PATH=${GOPATH}/bin:${PWD}/third_party/etcd:/usr/local/go/bin:${PATH} <del> retry go get github.com/tools/godep && godep version </del> retry go get github.com/jstemmer/go-junit-report retry go get github.com/cespare/prettybench

Didn't see an issue for this. Godep recently updated its version to v80. This broke several builds that directly godep, grabbing latest. /kind bug cc Example https://k8s- Here are some examples where we grab master:
We have helpers to grab godep. We should use them everywhere.
/cc
/assign


Question: // Some of this port stuff is under-documented voodoo. // See http://stackoverflow.com/questions/20428302/binding-a-port-to-a-host-interface-using-the-rest-api var protocol string <del> switch strings.ToUpper(string(port.Protocol)) { case "UDP": </del> <ins> switch port.Protocol { case runtimeapi.Protocol_UDP: </ins> protocol = "/udp" <del> case "TCP": </del> <ins> case runtimeapi.Protocol_TCP: </ins> protocol = "/tcp" default: glog.Warningf("Unknown protocol %q: defaulting to TCP", port.Protocol)

This is in 1.6.0-beta.4: Some context: The line number suggests it is here: I think this is because dockershim now uses a protobuf object and Protocol is now an int. And is allowed in go (apparently) - is "A". And the comedic error is because Protocol implements If this is the error, there's something fishy going on because all our UDP tests should fail?
Aha ... I think it is only UDP hostports that would be "broken" by the issue. I wouldn't be surprised if we don't rely on any, and that we might not have any test coverage. I don't think this is a regression - I think this code has been there since the start of dockershim, but maybe CRI is now the default? I don't know how widely UDP hostports are used. Given how close we are to the planned release date, cc and
There is something wrong in dockershim's protocol parsing. Will file an PR to fix this. /assgin
/assign
Opened to fix this.
thanks for filing the issue. Thanks for quickly addressing the bug. We need node e2e tests to capture the kind of regression at the first place.


Question: type RegistryList struct { DockerLibraryRegistry string `yaml:"dockerLibraryRegistry"` E2eRegistry           string `yaml:"e2eRegistry"` <del> EtcdRegistry          string `yaml:"etcdRegistry"` </del> GcRegistry            string `yaml:"gcRegistry"` PrivateRegistry       string `yaml:"privateRegistry"` SampleRegistry        string `yaml:"sampleRegistry"`

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: I see only amd64 architecture etcd is used for the which will create issues for the other architectures like ppc64le, arm etc. We need to replace this etcd image with the k8s maintained fat manifest image present in
/sig testing


Question: package factory import ( <del> "math" </del> "testing" "github.com/stretchr/testify/assert"

MaxInt const is exported in scheduler api although it is not needed to be exported. It is only used by . discussed that the common practice is to have API constants in /sig scheduling
cc
The problem is that it is part of a v1 API :(
Ok, it turns out they are not actually considered part of the API, and so we can remove them:
Should I open a PR to address removing those const values from ?
Yes, we can remove those in a followup PR to the one handling sized values.
Yes, this was the plan. /assign
remember to remove as well :)


Question: clientCA, err := s.getClientCA() if err != nil { <del> return authenticatorfactory.DelegatingAuthenticatorConfig{}, err </del> <ins> if _, ignorable := err.(ignorableError); !ignorable { return authenticatorfactory.DelegatingAuthenticatorConfig{}, err } else { glog.Warning(err) } </ins> } requestHeader, err := s.getRequestHeader() if err != nil {

Is this a BUG REPORT or FEATURE REQUEST?: /kind feature What happened: When I start the core API server with , it populates the ConfigMap in namespace with the requestheader CA. This allows the extension API server to dynamically discover the CA and use it for validating the client certificate of aggregator layer. See for documentation However, if the core API server doesn't use certificate based authentication for it's clients i.e. flag isn't set in core API server then the extension API server fails to start with error message I believe this is coming from bootstrap code in extension API server . What you expected to happen: Extension API server should be able to start and not require that both and be present in the ConfigMap. This allows Kubernetes clusters which don't use certificate based authentication for core API server to be able to dynamically discover the requestheader-ca instead of having to mount it as a secret. How to reproduce it (as minimally and precisely as possible): Create a Kubernetes cluster which uses non certificate based authentication for core API server like Webhook based authentication. Example: You can launch an Amazon EKS cluster which uses Webhook based authentication following steps outlined . Setup aggregator layer by adding flags mentioned . CA certificate and client certificate for aggregator layer can be generated using and . Deploy metrics server to cluster following steps in Deployment section of README Get logs of metrics server pod and it should be failing with error message above Anything else we need to know?: It's possible there is a reason which is not evident to me as to why extension API server would require both client-ca-file and requestheader-client-ca-file to be present in the ConfigMap. Environment: Kubernetes version (use ): 1.10.3 Cloud provider or hardware configuration: Amazon EKS OS (e.g. from /etc/os-release): Kubernetes control plane and nodes = Amazon Linux 2 (2017.12) LTS Release Candidate (4.14.42).
/sig api-machinery /cc
since it looks like the at-mention didn't take
/sig auth /assign
Thanks Jordan. This is something we are interested in contributing to and ask for being cherrypicked to patch releases of 1.10.x and 1.11.x. That conversation can wait once we have the PR out.

Quick update: I'm working on sending a PR for this issue by this week
- Now that this is merged, can we open up the possibility of cherry picking to 1.10.x and 1.11.X please?
sure, opened , ,


Question: if err != nil { return false } <del> return b </del> <ins> return b && api.IsPodReady(pod) </ins> } // isDying returns true if the pod has a non-nil deletion timestamp. Since the

This is mostly motivated by petset. Currently we create a governing service that is responsible for the network identity of each pet. The requirements of this governing service differ from a normal service in the following ways: have internal protocols to notice peer health, and a list of different checks (eg: ). In kube, peers use DNS to communicate. So we don't want a readiness check failure to remove the pet from DNS and abruptly cut it off from the set. "governing service" is headless so kube-proxy doesn't need to NAT the service ip. will not be using this Service, so it doesn't matter that unready pets show up in DNS under the svcName sub domain. In addition to these, there's one important reason some pets need to have DNS before readiness. To provide the guarantees required to safely bootstrap a pet, we need: 0 starts before pet1.. 't start pet1 till pet0 is running/ready pet1 starts, it sees pet0 in DNS The easiest way to guarantee 3 today is to do one of: poll in the entrypoint of pet0, till it sees itself poll in the init container of pet0, till it sees itself poll in the init containers of pet {1..N-1} till it sees all previous pets Third option is not a great idea because we don't know who else is supposed to exist, and for the first 2 to work with a readiness probe, we need a DNS record before the Service is ready.
I have a wip that does this through an annotation:
+1. this is useful even in traditional n rc's and n+1 services setup for clustered systems like n nodes zookeeper/etcd. Currently, there isn't a easy way to specify a readiness check in such setups, because the check in these cases would look for whether the pod member has joined the cluster successfully as follower, which wouldn't work as they discover each other through them appearing in service endpoints. this prevents one from being able to do a rolling upgrade of cluster members safely, because there is no readiness feedback after a member pod is upgraded.
Suggestion to hold this for alpha in 1.5, and in 1.6 spend a bit of time talking about how services are best intermixed with stateful sets (there's a lot of correlation between them).

That avoids us breaking people in 1.5 and gets a bit more think time.
SGTM
See also
Can we claim that if a pod is requesting pet DNS (i.e DNS name via the hostname/subdomain annotation) it never wants kube-style readiness, because it has its own internal protocol for peer health? If that's true we can get rid of the annotation and teach kubedns to always serve records for pet DNS ignoring readiness. My intuition tells me the answer to that question is no (i.e there exist pets that use DNS for discovery, and don't have an internal protocol for readiness, so they must use kube readiness to remove themselves from the peer group), and so the ability to toggle it on/off is useful, but I also want others to consider it.
I'd love to nix an annotation but I have not the expertise to answer this one.
I don't have a strong feel either way - I'd say readiness controls on a service feel orthogonal to independent names though. Since we don't have per port readiness, we need to consider that at some point we might need more sophistication on services or pods to control membership (unless we make label flipping easier). I would like to see us tease apart the "leader / follower" service member concept a bit more because that might inform how we handle readiness (seems like a separate bit, but we may want atomic changes).
They are somewhat orthogonal, but you presumably only want to discover healthy peers, unless you have a way to probe every peer you discover for health. Applying this policy based on endpoint info also feels a little weird, because it's harder to create overlapping Servics with different behavior. I think doing the common thing always and by default is fine, as long as we can annotate our way back to what we have today, but what form would the opposite of today's Service level annotation take? i.e something that says "even though my endpoints need pet names, apply the regular rediness behavior to their DNS records"
How bad would it be to start without an annotation, assume that pet-names == tolerate-unready, and see if anyone screams? Who should we consult?
SGTM, that's also what I was prposing in the previous comment
I guess the question is why do we even want to do that? Why make a change that couples logic vs leaving as is with decoupled logic?

We have people who want unready endpoints for deployments to form gossip pools - so I guess that counts as a use case where pet names is not coupled to unready. Is leaving the annotation on for another release really a problem? , Prashanth B <:
That's exactly the kind of use case I was looking for, something that doesn't require pet names. I get the feeling there will be other apps but I couldn't produce an example.
JBoss EAP uses it to form initial quorum for jgroups, which isn't a strong consensus system. So they use deployments and a minimum set to form an initial quorum and deal with split brain like jgroups normally does (infinispan has multiple consistent modes, some of which might prefer loose pools). Arguably smaller sets and some infinispan quorums might prefer statefulsets, but those use cases are also more tolerant of partitions in general. , Prashanth B <:
we are using tolerate-unready-endpoints annotation in our zookeeper deployments (not using pet/stateful set). Without unready endpoints, it is impossible to define a meaningful pod readiness check for zk, and without a readiness check, it is hard to do rolling updates (one ensemble member at a time).
The question is whether there's a need to have StatefulSets without tolerate-unready-endpoints. It sounds like maybe yes, which makes me grumpy, but I'll acquiesce if Clayton and Prashanth say so. , ravilr <:
isn't it enough to establish a need to tolerate-unready without statefulsets?
I have two services pointing to my etcd stateful set - one that is endpoints (peers) and tolerates unready. I have one that is for naive clients that uses readiness. On Dec 1, 2016, at 2:59 AM, Prashanth B <: isn't it enough to establish a need to tolerate-unready without statefulsets? — You are receiving this because you commented. Reply to this email directly, view it on GitHub , or mute the thread
fine.
/sig network
Right. This is exactly why etcd operator uses the alpha feature. Can we make it beta in 1.8 release?
/cc
We had discussed promoting since we need it for StatefulSet and no one has come up with anything better.

We should make a decision and include it in our GA roadmap for workload controllers.
Be sure to include sig-network On Jul 6, 2017, at 2:40 PM, Anthony Yeh <: had discussed promoting tolerate-unready-endpoints since we need it for StatefulSet and no one has come up with anything better. We should make a decision and include it in our GA roadmap for workload controllers. — You are receiving this because you commented. Reply to this email directly, view it on GitHub , or mute the thread
This is also useful for ingresses that need to retain the source IP using OnlyLocal node ports, which is important for firewalling with things like Calico. The problem that we see now is when a deployment occurs, the pod is removed from the node port before the PreStop completes, which doesn't give the upstream AWS load balancer time for the pod to fail out using normal health checks.
/close
Headless Services are a thing now


Question: // DefaultKubeProxyConfiguration assigns default values for the kube-proxy ComponentConfig func DefaultKubeProxyConfiguration(internalcfg *kubeadmapi.ClusterConfiguration) { <del> externalproxycfg := &kubeproxyconfigv1alpha1.KubeProxyConfiguration{} </del> <ins> externalproxycfg := &kubeproxyconfigv1alpha1.KubeProxyConfiguration{FeatureGates: make(map[string]bool)} </ins> kind := "KubeProxyConfiguration" // Do a roundtrip to the external version for defaulting

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: kubeadm init --feature-gates get panic ! What you expected to happen: no panic How to reproduce it (as minimally and precisely as possible): kubeadm init --feature-gates Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
what version of kubeadm are you using?
/area kubeadm /sig cluster-lifecycle
cc


Question: # # We specify -X to avoid a race condition that can cause minion failure to # install.  See https://github.com/saltstack/salt-bootstrap/issues/270 <del> curl -L http://bootstrap.saltstack.org | sh -s -- -X </del> <ins> curl -L http://bootstrap.saltstack.com | sh -s -- -X </ins>

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: After deleting a pod and scaling up the statefulset with different valid envs in the same container simultaneously, then the output events of executing the command is below: What you expected to happen: update successfully. How to reproduce it (as minimally and precisely as possible): a statefulset. apply this statefulset's yaml: After a while, execute , and the pod is running a new stateful's yaml, which a new env key and more replicas. the pod and apply a new stateful's yaml the statefulset Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.4", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-02-28T13:37:52Z", GoVersion:"go1.11.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.4", GitCommit:"", GitTreeState:"clean", BuildDate:"2019-02-28T13:30:26Z", GoVersion:"go1.11.5", Compiler:"gc", Platform:"linux/amd64"}
/sig apps
Meet same problem while using kube-addon-manager. vs and patchStrategy
+1
In some cases, add in the env which using can solve the problem.
/assign
I think the root cause of this is the strategic patch. As mentioned, we have to use the to fix the problem which tells the strategic patch to remove the field so the spec could pass the validation process of statefulsets. It would lead to API broken if we change the behaviour of strategic merge patch, I'll dig around in the apps validation process and try to find out the solution to this.
Thank you for your reply. According to your advice, I tried to add in my second statefulset's yaml, but I still could not update statefulset correctly. How to fix my yamls to get around the problem?

/unassign Sorry, I do not have enough time for this, anyone interested could take it from here.
I don't think the problem comes from your YAML, the problem comes from the controller that doesn't update the fields properly.
/assign


Question: } }, "topologyKey": { <del> "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. For PreferredDuringScheduling pod anti-affinity, empty topologyKey is interpreted as "all topologies" ("all topologies" here means all the topologyKeys indicated by scheduler command-line argument --failure-domains); for affinity and for RequiredDuringScheduling pod anti-affinity, empty topologyKey is not allowed.", </del> <ins> "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", </ins> "type": "string" } }

The field is marked as optional: The comment also describes a use case for empty topologykey: But the validation rule says it's required:
cc
cc could you take a look? Thank you. The validation rule is by
the release note of should go into the "action-required" section of the , but it's currently in "Other notable changes".
/assign
It seems that we should also update the external api type comments. Right?
I will look into it today.
If you didn't work on this, I am pleasure to send PR to fix this. :)
yeah sure. thanks.


Question: virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec) fileAlreadyExist := false task, err := dummyVM.Reconfigure(ctx, virtualMachineConfigSpec) <ins> if err != nil { glog.Errorf("Failed to reconfig. err: %v", err) return "", err } </ins> err = task.Wait(ctx) if err != nil { fileAlreadyExist = isAlreadyExists(vmdisk.diskPath, err)

Hi, I am currently working on k8s cloud provider and I ran in pkg/cloudprovider package. Turned out that some providers code have unused variables and logic problems. Here is the report: session.New is deprecated some bugs and unused values oauth2.NoContext is deprecated, overwritten ctx and miss error checks missing error check unused values missing error check, loop logic issues Hope this report can help to cloud provider working groups. :)
/kind bug /sig cloud-provider /sig vmware
PR open for vsphere cloud provider
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return false, fmt.Errorf("requested ip %q is neither static nor assigned to the LB", requestedIP) } <del> func (gce *GCECloud) createTargetPool(svc *v1.Service, name, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, affinityType v1.ServiceAffinity, hc *compute.HttpHealthCheck) error { </del> <ins> func (gce *GCECloud) ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation bool, svc *v1.Service, loadBalancerName, clusterID, ipAddressToUse string, hosts []*gceInstance, hcToCreate, hcToDelete *compute.HttpHealthCheck) error { serviceName := types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name} lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName) if tpExists && tpNeedsRecreation { // Pass healthchecks to DeleteExternalTargetPoolAndChecks to cleanup health checks after cleaning up the target pool itself. var hcNames []string if hcToDelete != nil { hcNames = append(hcNames, hcToDelete.Name) } if err := gce.DeleteExternalTargetPoolAndChecks(svc, loadBalancerName, gce.region, clusterID, hcNames...); err != nil { return fmt.Errorf("failed to delete existing target pool for load balancer (%s) update: %v", lbRefStr, err) } glog.Infof("ensureTargetPoolAndHealthCheck(%s): Deleted target pool.", lbRefStr) } // Once we've deleted the resources (if necessary), build them back up (or for // the first time if they're new). if tpNeedsRecreation { createInstances := hosts if len(hosts) > maxTargetPoolCreateInstances { createInstances = createInstances[:maxTargetPoolCreateInstances] } if err := gce.createTargetPoolAndHealthCheck(svc, loadBalancerName, serviceName.String(), ipAddressToUse, gce.region, clusterID, createInstances, hcToCreate); err != nil { return fmt.Errorf("failed to create target pool for load balancer (%s): %v", lbRefStr, err) } if hcToCreate != nil { glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created health checks %v.", lbRefStr, hcToCreate.Name) } if len(hosts) <= maxTargetPoolCreateInstances { glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created target pool.", lbRefStr) } else { glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created initial target pool (now updating the remaining %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) if err := gce.updateTargetPool(loadBalancerName, hosts); err != nil { return fmt.Errorf("failed to update target pool for load balancer (%s): %v", lbRefStr, err) } glog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) } } else if tpExists { // Ensure hosts are updated even if there is no other changes required on target pool. if err := gce.updateTargetPool(loadBalancerName, hosts); err != nil { return fmt.Errorf("failed to update target pool for load balancer (%s): %v", lbRefStr, err) } glog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)) } return nil } func (gce *GCECloud) createTargetPoolAndHealthCheck(svc *v1.Service, name, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, hc *compute.HttpHealthCheck) error { </ins> // health check management is coupled with targetPools to prevent leaks. A // target pool is the only thing that requires a health check, so we delete // associated checks on teardown, and ensure checks on setup.

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /sig network /area cloudprovider What happened: The cloud provider interface that should . The GCE provider does this for but not for . Every other in-tree provider implements by fully updating the load balancer including updated backend nodes: , , , And the following out-of-tree providers do as well and Oracle (not oss yet). What you expected to happen: I expect that all cloud providers implement by creating or fully updating the load balancer if it already exists. Related discussion: /cc
/assign Make sense to make implementation in GCE consistent with others for the ease of up-coming service controller enhancement. This sounds like a 1.10 task.
is this task blocking you on another attempt at refactoring the service controller sync logic?
yup


Question: ) func (c *FakeEvents) CreateWithEventNamespace(event *v1.Event) (*v1.Event, error) { <del> action := core.NewRootCreateAction(eventsResource, event) </del> <ins> var action core.CreateActionImpl </ins> if c.ns != "" { action = core.NewCreateAction(eventsResource, c.ns, event) <ins> } else { action = core.NewCreateAction(eventsResource, event.GetNamespace(), event) </ins> } obj, err := c.Fake.Invokes(action, event) if obj == nil {

Opening this issue based on what I ran into on with namespace as follows While unit testing the controller, we create a fake client and setup the controller to run using the client. This ends up creating a having the namespace set to When the call eventually reaches The action gets associated with a namespace only if the object has a namespace associated with it. Which in this will be Which will result in the error in to manage the event like does. In case if the namespace is set to Above is a test case that can easily simulate the issue in question. Referring back to the test under PR NA <details</details<details</details<details</details<details</details<details</details<details</details
/sig api-machinery
/help /triage accepted
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /help /triage accepted Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign


Question: fi fi <del> if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}" if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSIDriverRegistry" ]]; then # If not explicitly specified, default to true. FEATURE_GATES="${FEATURE_GATES},CSIDriverRegistry=true" fi if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSINodeInfo" ]]; then # If not explicitly specified, default to true. FEATURE_GATES="${FEATURE_GATES},CSINodeInfo=true" fi fi </del> # Optional: Install cluster DNS. # Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS. CLUSTER_DNS_CORE_DNS="${CLUSTER_DNS_CORE_DNS:-true}"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Probably caused by <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
https://k8s- /assign
I understand that moved crd creation logic to e2e test's k8s cluster deployment code. On the other hand, I tested it with manually deployed k8s cluster. So, in my case, I guess that in addition to enable and feature gates enabled, I need to create crd manually before tests, like below: I could make two of three SkipAttach tests pass in my environment by above way. For "non-attachable volume does not need VolumeAttachment" case, I'm still looking into it. (It doesn't seem to skip creating volumeAttachment, properly)
I found a typo in 's argument. While hostpath driver's connects driver name and unique name without "-", it connects them with "-". This makes "non-attachable" test fail, due to the mismatch between volume name in crd and provisioner name, as a result, creating volumeAttachment isn't skipped. I will create a PR to fix it. For above bug, is unrelated. It just changed the way in how to test in manually deployed k8s cluster as I mentioned in above comment. So, there should be no problem for the PR, at least for the CI testing if it works well in CI environments.
I can verify that the CRDs are not getting installed on the alpha cluster. Investigating why.
/reopen since I see the Alpha tests failing
/milestone v1.13 /priority important-soon /kind failing-tests
/kind failing-test
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I don't think this is fully resolved. There are still failures on https://k8s- /reopen
Reopened this issue. <detailsIn response to : I don't think this is fully resolved. There are still failures on https://k8s- Instructions for interacting with me using PR comments are available .

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
can you help take a look at these failures? It's testing CSI filesystem volume mode. I think it is unrelated to the crd installation issue that Saad fixed.
I tested with AllAlpha flag and manually created csi crds and confirmed that filesystem volume mode test passed in my environment and also other tests passed there. If I delete csi crds, other tests also fail, so I agree that the issue that Saad fixed is not related. However, above also means that this issue can't be reproduced in my environment.
I1109 14:56:01.141] Nov 9 14:56:01.131: INFO: At 2018-11-09 14:50:54 +0000 UTC - event for security-context--e42e-11e8-9b95-: {attachdetach-controller } FailedAttachVolume: AttachVolume.Attach failed for volume "pvc--e42e-11e8-880b-" : node "bootstrap-e2e-minion-group-3wtg" has no NodeID annotation
Hostpath also seems to be failing because of driver registration issues? Not sure why other alpha tests are passing if that's the case
This test is the only test that uses  to create pod with CSI driver volume and expects the result to be success. One of the pod parameters specified there might affect this failure.
Those 2 failing tests are newly introduced in and started to fail from the beginning. Is that PR blocking 1.13? If not, can we revert and fix later? I have recently set up optional presubmit tests for alpha features at , which will pick up those 2 tests. Currently this failure is blocking
apart from the 2 failing tests, I see which addressed with is also flaky. These failures are now blocking PRs as mentioned above. As we enter code slush today, its very critical to have the PRs get in sooner than later. Can we plz revert the feature and the tests until they are fully resolved and ready to go. Thanks
I prefer disabling the specific tests instead of reverting the entire pr. We will lose important signal for csi
That works for me.

Can you cc me on the PR that disables those tests? When you re-enable the tests, remember to try them out on . :)
Let me consider if we can just skip these test for CSI, instead of reverting the entire patch.
you mean just skipping them in Alpha cluster right?
I was thinking about skipping the below test with the combination of below drivers: [Test] [Testpattern: Dynamic PV (filesystem volmode)] volumeMode[Feature:BlockVolume] should create sc, pod, pv, and pvc, read/write to the pv, and delete all created resources [Driver] csi-hostpath Won't it enough? Or, do we need to skip it only with alpha cluster?
If we just disable the tests alone and not revert the feature, we are just masking the issue. Ideally if we are not able to fix the tests and reenable them the feature should be reverted from 1.13 as well. Are you also actively working on fixing the tests and this skip is just to unblock PRs in the meantime? if so can we timebox by when we can have the tests fixed and re-enabled / the feature reverted? My fear is we might forget the fact that its disabled since there's no active reporting on it.
Actually, there is no feature in either of culprit commits. It tests or intended to improve the way it is tested. Therefore, it must be a test issue or another issue which is found by the test , not an issue introduced by the commits themselves. In the code, there is a clear comment that the test is skipped due to this issue with the reference to this issue#, so this won't be forgot. If you still worry about this issue is not tracked, we might be able to keep this issue open.
IIUC the test that were failing (and are being reverted) were for the new CSI festures going to Alpha in 1.13, right? so even these 2 culprit commits do not contain feature work, it is testing some new code additions to 1.13 and now those will be untested until we re-enable these tests. Is there an ETA to re-enable them, preferably sometime early next week? Yes +1 to keeping this issue open to track the skipped tests.
The two failing tests themselves are not for new alpha features that are in v1.13.

Instead, they are just to test csi driver works well as it works in existing in-tree driver. (And failed because they might be affected by some other alpha features or some of specific configurations in gce.) In addition, ~50 tests for csi driver, so it would be better not to remove all of the tests by reverting the PR, just for the 2 failing tests. I think that just reverting the patch won't help improving quality of k8s. As for ETA to solve the issue, I found difficulty in debuging in my local environment. It seems that the two tests fail in gce environment w/ AllAlpha=true, however, as I've already mentioned, at least csi-hostpath tests in my local environment w/ AllAlpha=true doesn't fail.(And I don't have test environment for gce.) Of course, I will help solving the issue next week. However, I will need someone's help for testing combinations of alpha features in gce environment to narrow down which flag really affects this issue. At least I would like to know the results: w/ MountPropagation=true,CSIPersistentVolume=true,BlockVolume=true,CSIBlockVolume=true all alpha flags except CSIDriverRegistry and CSINodeInfo
/reopen Thanks for the detailed explanation. I think the revert is fine in this case then. I am keeping this issue open however to track and fix the failures in these 2 tests can any of you help answer w.r.t alpha features on gce environment. thanks
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I can help debug when I'm back on Tuesday. In the meantime, I would suggest looking through the failing test logs to try to triage what's going on
Here's one failing run:
I looked into the csi-hostpath failure log. As seen in the for , where the pod which has a csi-hostpath volume is being attached is scheduled, mount volume fails with these errors (1). Related code to this log is (2), so the driver doesn't seem to be registered to . As for the registration code for csiDriver, it is called as (3).

Therefore, will be logged if the driver is triggered for registration. As seen (4), registration is only triggered in , which is different from the one the pod is scheduled. In addition, driver-registrar seems to be deployed only on the node where registration is triggered (5). Is it right behaviour, while registrar is deployed as daemonset? I guess that just this fact seems to cause an issue to make csi-hostpath driver work properly. (Actually, I only tested with 1 node environment, so this might only happen in multiple nodes environment.) <details- (1) (2) Result of grep to check whether registrar is deployed to each node [ for bootstrap-e2e-minion-group-50r7][ for bootstrap-e2e-minion-group-k781][ for bootstrap-e2e-minion-group-z5cg] </details
The hostpath plugin should be deployed as a DaemonSet, so it's strange that it's only getting registered on only one node.
/assign looking into this as well
could you link the PR for skipping [external-gcepd] tests into this issue when you do that as well
skipping gcePD-external is here:
Since gce-cos-alphafeatures is now passing, do we need to keep this issue open? Should it morph into a non-blocking issue for longterm work on a better test?
We have root caused 2 of the 3 failures, which were problems in the test and test conifg. Fixes will be out soon. Still triaging the 3rd failure, which could be a real bug in alpha clusters
I've root caused csi-hostpath failure issue. It just failed to set the node properly in test code side. I will make a PR to fix it.
Or should it be done in your PR too, to avoid modifying twice in the separate PR? Actually, the fix are as below: (Failing to set node before pod creation)
Created for csi-hostpath fix, to prepare for the case that we need a separate PR. We can discuss more on this there, if needed.
/unassign found possible root cause for GCE issue: CSI NodeID annotation is not present on the Node object, could be caused by failed retries on updating Node object.

(apparently when alpha features are open we shouldn't be looking at the Node object anyway except we seem to be, so that might also be a bug). has the most context for the fix /assign
The symptom of the remaining test failure is the test pod fails to create because it fails to attach with the error: . The test succeeds the first time it's run in a cluster, and always fails after that. Two issues caused this error message. the CSI external-attacher tries to fetch the object to get the NodeID, it does so through an informer. The first time the test is run, objects are created for the very first time, and the informer returns the correct object. In subsequent test runs, s from the first run is kept, and for some reason the informer fails to detect the object and returns empty. Making the API call directly works, I'll patch the external-provisioner with this to mitigate the issue, and this is sufficient to get the test to pass. The informer bug will need to be eventually fixed. fails to post the NodeID annotation sometimes, likely due to node status patch not being retried correctly. This is crucial for clusters without feature flag set, and this is the default for 1.13. Will add the appropriate retry mechanism.
Opened to track the informer issue
Are and potential PR for the only 2 fixes being actively worked on for this issue?
Only is needed. We have a workaround for
will you have a followup PR to enable the CSI alpha tests back? or is that already in flight?
also reenables the tests
OK with merged we should see the CSI alpha tests re-enable. to keep an eye out for CI signal
The tests got reenabled 5 days ago, and alpha tests are passing. Closing this issue. Thanks everyone for your rapid turnaround! /close
Closing this issue. <detailsIn response to : The tests got reenabled 5 days ago, and alpha tests are passing. Closing this issue. Thanks everyone for your rapid turnaround! Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The tests are actually running (and passing) in the normal suite now, now that the raw block feature moved to beta.


Question: // this error since all subsequent creations will fail. return } <del> if errors.IsTimeout(err) { // Pod is created but its initialization has timed out. // If the initialization is successful eventually, the // controller will observe the creation via the informer. // If the initialization fails, or if the pod keeps // uninitialized for a long time, the informer will not // receive any update, and the controller will create a new // pod when the expectation expires. return } </del> } if err != nil { defer utilruntime.HandleError(err)

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: In some case, controller manager failed to create pod as replicas number defined in ReplicaSet spec. Normally if replicaset controller fails to create a pod, it will try later. () But func in ignore timeout error for some reason(), then if a create pod request really timeout during accessing apiserver, this failed will not add to the queue again. Then the pod should be created will never be created again. () I saw this happened in a production environment. What you expected to happen: Controller manager will retry and create pod again. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.10.1 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): centos 7 Kernel (e.g. ): Install tools: Others: /cc
/sig api-machinery
/sig apps /remove-sig api-machinery
In : The controllers ignore the timeout error when creating the pods, depending on how the initialization progress: the initialization is successful in less than 5 mins, the controller will observe the creation via the informer. All is good. the initialization fails, server will delete the pod, but the controller won't receive any event. The controller will not create new pod until the Creation expectation expires in 5 min. the initialization takes too long (5 mins), the Creation expectation expires and the controller will create extra pods. Note 2,3 that if the creation request failed with timeout, the ReplicaSetController will resync: And the expectation creation has already been in if diff < 0:
I got the similar situation. After getting type: 'Warning' reason: 'FailedCreate' Error creating: Timeout: request did not complete within allowed duration, Pod is not created. Its ReplicaSet has .spec.replicas = 0, but .status.replicas stays 0 without any reason. k8s version is 1.11.0.
hello , , i am thinking the code in causes the inconsistency above.

it will only RE-enqueue the RS when ReadyReplicas == Replicas && AvailableReplicas != Replicas, while there are situations that ReadyReplicas != Replicas.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Also seeing this in 1.12 on GKE (1.12.8-gke.6).
Just got this now on AWS (v1.12.6-eks-): Using Istio 1.1.7, if that helps at all. I updated some nodes and switched to a new node type, and whilst it was supposed to be bringing these pods back up, it just never did. It's only in our staging environment right now at least!
Ooooh, we're also running Istio. (And I just saw the issue again in a different GKE cluster.) With the automatic sidecar-injection. Maybe an interplay problem with that admission controller.
Hitting the same issue on EKS 1.13.7, and we are using Istio as well (1.2.0). We ran into the issue while doing a rolling-upgrade of the cluster, with a lot of pod churn, so I assume that the above hypothesis around admission-controller / sidecar injection might be correct. :) Edit: we're trying to mitigate this by making the sidecar injector HA; by default it is running a single instance.
did scaling the sidecar injector work? I'm still hitting this issue with Istio 1.2.2, and it's really making me nervous about upgrading the production server. It's consistently not bringing pods back.

it seems to have mitigated the problem for us, but I'm not sure we've done another full rolling-update since my previous message. In any case, scaling up the injector can't hurt
Happens on my cluster two, but we're not using Istio. Message is following, with no additional clue : One thing to note is that it happened after pods were Evicted, and OutofEphemeral condition. Disk was full during the eviction.
I actually found the issue I've been having "with Istio". We're using CloudFormation to create the nodes in AWS (almost the default CloudFormation template given by AWS). The problem with this is that it doesn't drain the nodes before killing them, so even if you replace nodes in batches, you're still killing pods. If you kill the Istio pods, it can then prevent other pods from starting up (giving you the aforementioned error). From that point I think some of the fault lies with Kubernetes too, because there are times when a node has been terminated, and I can go look at a ReplicaSet and it'll think it's got the right amount of pods running, then I can go look at pods, and there'll be nothing there for that service. I think this prevents Istio from recovering as quickly as it could. Istio then seems to be very picky about the order that services start in, so it takes quite a bit of time for Istio to recovery even once the Istio pods start coming back. So what I've done now is labels to the nodes (e.g. or ), and then when I want to upgrade the nodes, I create an entire new node group with the opposite deployment label. Then using that label I taint and drain the old nodes. At this point you may need to scale up some services to allow nodes to be drained because of PodDisruptionBudgets. Once the nodes are drained, I destroy the CloudFormation stack for the old node (alternatively I suppose you could set the desired number of nodes to 0). Luckily this isn't something that really needs to happen very often.

Also, I'm pretty sure using would ease some of this, but we're using Terraform for our infrastructure, and I'd rather keep it all in once place... Next step for me is trying to figure out how to replace nodes in batches, because ideally I don't want twice as many nodes as I need running at one point. I might hit some limits in AWS for certain node types.
I think I ran into this situation as well. It was caused by a similar situation as described by . A change to a Terraform module resulted in ASG replacement. The change was supposed to be deployed by hand on each ASG individually, which though not as graceful as using taints to drain the nodes in the old ASG, would have probably allowed Kubernetes to reschedule the unexpectedly terminated pods without too much difficulty. Instead, the change was inadvertently applied to a large number of ASGs all at roughly the same time. This caused many ReplicaSets to report 0 current and 0 ready pods, and to have the "Error creating: the server was unable to return a response in the time allotted, but may still be processing the request (post pods)" event. Luckily, manually scaling each ReplicaSet to 0 and then back to the original scale gave Kubernetes the kick it needed to create the pods.
I think I am hitting this bug, too. My situation is that I have an validating webhook admission controller attached to the namespace. If the admission controller goes offline, the replica set for regular Deployments on that namespace will not be able to create pods as they can't get past the admission controller. They list 'Timeout: request did not complete within allowed duration' errors, and then never try again to create the pod. Once the admission controller is back online. These replica sets just sit there. The only way to get the Deployment to deploy pods again is to delete its replica set. Then, it will make a new replica set and that one will happily deploy the pods. In particular, this is problematic if you have some sort of moderately widespread worker node outage. If a bunch/all of workers go offline at once, you could lose your admission controllers temporarily and then get into a situation where nothing comes back due to stuck replica sets.
Got the same situation in EKS (we have a nice monitoring alert that matches events containing ), server version .

Fixed using the commands: Delete stuck replicasets, Kubernetes creates another instead and if you don't have any other problems (like resource quotas, for example), they will start new pods.
Observed on GKE 1.13.6-gke.13 or 1.13.7-gke.24, as it happened during an upgrade. Lots of FailedCreate events. We got scheduling back on track by scaling deployments to zero replicas and then back to the intended number.
Found the following in ReplicaSetController#manageReplicas :
I am hitting this bug, too. If ReplicaSetController create pod request really timeout during accessing apiserver, this failed key will not add to the queue again. Then the pod should be created will never be created again. in manageReplicas, if create pod timeout < 5 minute, the queue has no key, the controller never sync when no key , so it never create a new pod even through the expectation expires. rs status kubectl get rs --all-namespaces NAMESPACE NAME DESIRED CURRENT READY AGE kube-system kubenab- 3 2 2 5d9h kube-controller-manager log: I1204 04:13:23. 1 ] Event(v1.ObjectReference{Kind:"ReplicaSet", Namespace:"kube-system", Name:"kubenab-", UID:"-b142-4db2-a914-", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Warning' reason: 'FailedCreate' Error creating: the server was unable to return a response in the time allotted, but may still be processing the request (post pods) kube-apiserver log: W1204 04:12:53. 1 ] Failed calling webhook, failing open kubenab- failed calling webhook "kubenab-": Post context deadline exceeded (Client.Timeout exceeded while awaiting headers) E1204 04:12:53. 1 ] failed calling webhook "kubenab-": Post context deadline exceeded (Client.Timeout exceeded while awaiting headers) E1204 04:13:23. 1 ] apiserver was unable to write a JSON response: http: Handler timeout E1204 04:13:23. 1 ] apiserver received an error that is not an metav1.Status: &errors.errorString{s:"http: Handler timeout"} I1204 04:13:23.

1 ] Trace[]: "Create /api/v1/namespaces/kube-system/pods" (started: 2019-12-04 04:12:23. +0000 UTC m=+.) (total time: 1m0.011414107s): Trace[]: [30.010012786s] [30.00986498s] About to store object in database Trace[]: [1m0.011414107s] [30.001401321s] END W1204 04:13:23. 1 ] Failed calling webhook, failing open kubenab- failed calling webhook "kubenab-": Post context deadline exceeded (Client.Timeout exceeded while awaiting headers) E1204 04:13:23. 1 ] failed calling webhook "kubenab-": Post context deadline exceeded (Client.Timeout exceeded while awaiting headers)
/assign /assign /assign It appears that the timeout code here: may be out-of-date now that initializers have been removed. The solution may be to just remove the code swallowing the timeout.


Question: <ins> # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM gliderlabs/alpine MAINTAINER Mehdy Bohlool <mehdy@google.com> RUN apk-install bash ADD etcd-empty-dir-cleanup.sh etcd-empty-dir-cleanup.sh ADD etcdctl etcdctl ENV ETCDCTL /etcdctl ENV SLEEP_SECOND 3600 RUN chmod +x etcd-empty-dir-cleanup.sh CMD bash /etcd-empty-dir-cleanup.sh </ins>

I came upon a cluster that had had the namespace code thoroughly exercised. No namespaces currently existed, but many had been created and deleted in the past. Its etcd had lots of empty directories under each resource that had held items when those namespaces were alive (secrets, serviceaccounts, pods...). Do we need to be doing something to clean these empty directories up? It seems like this etcd was consuming a lot of memory.
or might know the answer to this question.
Right. We should delete them. It might get tricky for concurrent cases like A checks if dir is empty, B creates, then A deletes it. What does it look like currently in creating and deleting namespace?
How much memory does a directory entry take? Does it really sound possible to run out of memory while snapshotting on account of too many empty directories?
-- have you guys encountered this? I think you run with more namespaces than we do. But maybe yours are mostly populated?
Yes, we've seen it. c.f.
Approximate memory use we saw from empty dirs:
cc
The patch we debated carrying is here: Our hope was to just wait to etcd v3 where this would not be a problem.
can you link to your clean-up script here?
I needed another PR to run the pod automatically on master.


Question: key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) defer ss.vmssVMCache.Delete(key) <del> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) _, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM) </del> <ins> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI) resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err) retryErr := ss.UpdateVmssVMWithRetry(nodeResourceGroup, ssName, instanceID, newVM) if retryErr != nil { err = retryErr klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err) } } </ins> if err != nil { <del> klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) </del> <ins> klog.Errorf("azureDisk - detach disk(%s, %s) from %s failed, err: %v", diskName, diskURI, nodeName, err) </ins> } else { <del> klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) </del> <ins> klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI) </ins> } return err

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Current azure cloud provider would fail to detach azure disk when there is server side error, need to add retry mechanism for detach disk operation, while for attach disk operation, it's not necessary since k8s pv-controller will try if first try failed. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.10 ~ master Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Others: /sig azure /assign
/milestone v1.14 We're entering and this looks relevant to the 1.14 release. Please if I am incorrect


Question: {ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}}, {ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}}, }, <ins> expectedAntiAffinity: topologyToMatchedTermCount{}, expectedAffinity: topologyToMatchedTermCount{ {key: "region", value: "r1"}: 2, {key: "zone", value: "z11"}:  2, }, </ins> }, }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Maybe it should be What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig Scheduling
I feel it is a valid error. But need confirmation from sig-scheduling maintainers. ... is it okay if I work on this?
ok
Yeah, this is a bug in preemption
/assign


Question: }, time.Minute, time.Second*4).Should(BeNil()) }) <del> It("should be possible to delete [NodeConformance]", func() { </del> <ins> /* Release : v1.13 Testname: Kubelet, failed pod, delete Description: Create a Pod with terminated state. This terminated pod MUST be able to be deleted. */ framework.ConformanceIt("should be possible to delete [NodeConformance]", func() { </ins> err := podClient.Delete(podName, &metav1.DeleteOptions{}) Expect(err).To(BeNil(), fmt.Sprintf("Error deleting Pod %v", err)) })

What happened: These tests are errantly listed as (ref: ) but aren't actually run as part of Conformance. They need to be moved to test/e2e/common to be included. To my eyes they may meet the criteria for conformance. There's definitely nothing about them that is node specific. They should be running as part of our regular CI, and they should probably be running as presubmits. Porting them to test/e2e/common would cause this to happen. /kind cleanup /sig node /area test /assign
GitHub didn't allow me to assign the following users: mkurylec. Note that only and repo collaborators can be assigned. For more information please see <detailsIn response to : What happened: These tests are errantly listed as (ref: ) but aren't actually run as part of Conformance. They need to be moved to test/e2e/common to be included. /sig node /area test /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details