Question: <ins> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package auth import ( "context" "fmt" authorizationv1 "k8s.io/api/authorization/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/kubernetes/test/e2e/framework" "github.com/onsi/ginkgo/v2" ) var _ = SIGDescribe("SubjectReview", func() { f := framework.NewDefaultFramework("subjectreview") ginkgo.It("should support SubjectReview API operations", func() { AuthClient := f.ClientSet.AuthorizationV1() ns := f.Namespace.Name saName := "e2e" ginkgo.By(fmt.Sprintf("Creating a Serviceaccount %q in namespace %q", saName, ns)) sa, err := f.ClientSet.CoreV1().ServiceAccounts(ns).Create(context.TODO(), &v1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: saName, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "Unable to create serviceaccount %q", saName) saUsername := serviceaccount.MakeUsername(sa.Namespace, sa.Name) framework.Logf("saUsername: %q", saUsername) saGroups := []string{"system:authenticated"} saGroups = append(saGroups, serviceaccount.MakeGroupNames(sa.Namespace)...) framework.Logf("saGroups: %#v", saGroups) saUID := string(sa.UID) framework.Logf("saUID: %q", saUID) ginkgo.By(fmt.Sprintf("Creating clientset to impersonate %q", saUsername)) config := rest.CopyConfig(f.ClientConfig()) config.Impersonate = rest.ImpersonationConfig{ UserName: saUsername, UID:      saUID, Groups:   saGroups, } impersonatedClientSet, err := kubernetes.NewForConfig(config) framework.ExpectNoError(err, "Could not load config, %v", err) ginkgo.By(fmt.Sprintf("Creating SubjectAccessReview for %q", saUsername)) sar := &authorizationv1.SubjectAccessReview{ Spec: authorizationv1.SubjectAccessReviewSpec{ ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb:      "list", Resource:  "configmaps", Namespace: ns, Version:   "v1", }, User:   saUsername, Groups: saGroups, UID:    saUID, }, } sarResponse, err := AuthClient.SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{}) framework.ExpectNoError(err, "Unable to create a SubjectAccessReview, %#v", err) framework.Logf("sarResponse Status: %#v", sarResponse.Status) expectedAllowed := sarResponse.Status.Allowed ginkgo.By(fmt.Sprintf("Verifying as %q api 'list' configmaps in %q namespace", saUsername, ns)) _, requestErr := impersonatedClientSet.CoreV1().ConfigMaps(ns).List(context.TODO(), metav1.ListOptions{}) actuallyAllowed := false switch { case apierrors.IsForbidden(requestErr): actuallyAllowed = false case requestErr != nil: framework.Fail("Unexpected error") default: actuallyAllowed = true } if actuallyAllowed != expectedAllowed { framework.Fail(fmt.Sprintf("Could not verify SubjectAccessReview for %q in namespace %q: SubjectAccessReview allowed: %v, request allowed: %v, request error: %v", saUsername, ns, expectedAllowed, actuallyAllowed, requestErr)) } framework.Logf("SubjectAccessReview has been verified") ginkgo.By(fmt.Sprintf("Creating a LocalSubjectAccessReview for %q", saUsername)) lsar := &authorizationv1.LocalSubjectAccessReview{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, }, Spec: authorizationv1.SubjectAccessReviewSpec{ ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb:      "list", Resource:  "configmaps", Namespace: ns, Version:   "v1", }, User:   saName, Groups: saGroups, UID:    saUID, }, } lsarResponse, err := AuthClient.LocalSubjectAccessReviews(ns).Create(context.TODO(), lsar, metav1.CreateOptions{}) framework.ExpectNoError(err, "Unable to create a LocalSubjectAccessReview, %#v", err) framework.Logf("lsarResponse Status: %#v", lsarResponse.Status) expectedAllowed = lsarResponse.Status.Allowed if actuallyAllowed != expectedAllowed { framework.Fail(fmt.Sprintf("Could not verify LocalSubjectAccessReview for %q in namespace %q: LocalSubjectAccessReview allowed: %v, request allowed: %v, request error: %v", saUsername, ns, expectedAllowed, actuallyAllowed, requestErr)) } framework.Logf("LocalSubjectAccessReview has been verified") }) }) </ins>

[x] test approval issue : [x] test pr : [x] two weeks soak end date : 31 Dec 2022 [X] test promotion pr : According to following two APIsnoop queries, there are still two authorization endpoints which are untested. example endpoint kind ---------------------------------------------------------+
-+-------------------------- createAuthorizationV1NamespacedLocalSubjectAccessReview LocalSubjectAccessReview (1 row) sql-mode select distinct endpoint, testhit AS "e2e Test", conftesthit AS "Conformance Test" from public.auditevent where endpoint ilike '%SubjectAccessReview' and useragent like '%e2e%' and not conftesthit order by endpoint limit 10; example endpoint Conformance Test ------------------------------------------+----------+------------------ createAuthorizationV1SubjectAccessReview f (1 row) [It] should support SubjectReview API operations STEP: Create pod "pod-x6fhh" in namespace "subjectreview-1704" 12/07/22 21:33:12.332 Dec 7 21:33:12.441: INFO: Waiting up to 5m0s for pod "pod-x6fhh" in namespace "subjectreview-1704" to be "running" Dec 7 21:33:12.502: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 61.265222ms Dec 7 21:33:14.512: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 2.070783878s Dec 7 21:33:16.520: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 4.078677494s Dec 7 21:33:18.513: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 6.07168643s Dec 7 21:33:20.510: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 8.068598657s Dec 7 21:33:22.509: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false.

Elapsed: 10.068442532s Dec 7 21:33:24.514: INFO: Pod "pod-x6fhh": Phase="Pending", Reason="", readiness=false. Elapsed: 12.073076728s Dec 7 21:33:26.511: INFO: Pod "pod-x6fhh": Phase="Running", Reason="", readiness=true. Elapsed: 14.069523123s Dec 7 21:33:26.511: INFO: Pod "pod-x6fhh" satisfied condition "running" Dec 7 21:33:26.515: INFO: "pod-x6fhh" in namespace "subjectreview-1704" is "Running" Dec 7 21:33:26.515: INFO: serviceaccount name: "system:serviceaccount:subjectreview-1704:default" STEP: Creating SubjectAccessReview in "subjectreview-1704" namespace 12/07/22 21:33:26.515 Dec 7 21:33:26.521: INFO: sarResponse Status: v1.SubjectAccessReviewStatus{Allowed:false, Denied:false, Reason:"", EvaluationError:""} STEP: Creating clientset to impersonate "system:serviceaccount:subjectreview-1704:default" 12/07/22 21:33:26.521 STEP: Verifying api 'get' call to "pod-x6fhh" as "system:serviceaccount:subjectreview-1704:default" 12/07/22 21:33:26.522 Dec 7 21:33:26.525: INFO: api call by "system:serviceaccount:subjectreview-1704:default" was denied Dec 7 21:33:26.525: INFO: SubjectAccessReview has been verified STEP: Creating a LocalSubjectAccessReview in "subjectreview-1704" namespace 12/07/22 21:33:26.525 Dec 7 21:33:26.530: INFO: lsarResponse Status: v1.SubjectAccessReviewStatus{Allowed:false, Denied:false, Reason:"", EvaluationError:""} STEP: Verifying api 'get' call to "pod-x6fhh" as "system:serviceaccount:subjectreview-1704:default" 12/07/22 21:33:26.

53 Dec 7 21:33:26.532: INFO: api call by "system:serviceaccount:subjectreview-1704:default" was denied Dec 7 21:33:26.532: INFO: LocalSubjectAccessReview has been verified sql-mode select distinct substring(endpoint from 'w+') AS endpoint, right(useragent,43) AS useragent from testing.auditevent where useragent like 'e2e%should%' and releasedate::BIGINT round(((EXTRACT(EPOCH FROM NOW()))::numeric)*1000,0) - and endpoint ilike '%subject%' order by endpoint limit 10; example endpoint | useragent ---------------------------------------------------------+--------------------------------------------- createAuthorizationV1NamespacedLocalSubjectAccessReview | should support SubjectReview API operations createAuthorizationV1SubjectAccessReview | should support SubjectReview API operations (2 rows) If a test with these calls gets merged, test coverage will go up by 2 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: close(stopChan) // wait for r.ForwardPorts to actually return <del> select { case err := <-doneChan: if err != nil { t.Fatalf("%d: unexpected error: %s", err) } case <-time.After(200 * time.Millisecond): t.Fatalf("%d: timeout waiting for ForwardPorts to finish") </del> <ins> err = <-doneChan if err != nil { t.Fatalf("%d: unexpected error: %s", err) </ins> } if e, a := len(testCase.Send), len(conn.streams); e != a {

this is you
Hmm, I had upped waiting from 100ms to 500ms and that seemed to fix it. Do you have any better ideas?
Don't wait? :)
If the only reason you would timeout would be the code is broken, that's why the test runs have -timeout set.
Then the test fails immediately. It's waiting for the net.Listen calls to finish so it can connect to the specified local ports.
*finish enough so that socket connections work
So I can remove the timeout gating in my tests?
Yes ----- Original Message -----
Just failed again: https://travis-
Seems to be failing frequently now.
If this is test is failing again, my gut says it's not the same root cause. The original failure was a timeout issue. This new one is connection reset by peer. I've got the test running in a loop waiting for a failure but haven't seen one yet. is this still failing frequently?
No. It had failed twice in ~10 builds when I reopened this issue, but hasn't failed since. Reclosing. Will file a new issue if it recurs.


Question: // Represents time when the job was completed. It is not guaranteed to // be set in happens-before order across separate operations. // It is represented in RFC3339 form and is in UTC. <ins> // The completion time is only set when the job finishes successfully. </ins> // +optional CompletionTime *metav1.Time `json:"completionTime,omitempty" protobuf:"bytes,3,opt,name=completionTime"`

The API docs for jobs are unclear as to what fields a caller would use to determine job success or failure in 1.11 (and we haven't updated them in a while) A casual reader would infer that completionTime is set when the job fails (but it isn't). Linking out to the docs is ok, but I shouldn't have to click through the links to understand how to use the Job object. The Job description and the job status fields should clearly indicate what fields are set in completion states on the job, because that's 100% of the point of docs. Taking a summary of the jobs-run-to-completion link and putting it in , , and job would improve the actual utility of our API doc. We should also do a pass on other objects and make the actual in server docs more useful.
as a consumer of the API, I had to read way too much to know how to use jobs correctly and how to write an API client that properly handled the conditions. Also, we should consider either setting completionTime on failures, or adding failedTime (the former may not be forwards compatible), because reading the conditions is too complex for a fundamental use case like this. This matches our decision w.r.t. conditions vs. fields - conditions are useful for extension and extra data, but status should have fields that allow an API consumer to easily understand state.
Also... our docs don't actually explain how to determine the job is complete via the API, so there's arguably no place this is captured.
I have the same problem right now, of correctly identifying a finished job, when it does not complete successfully and after reading the documentation and the reference and all the examples I could find, it's still not clear how I can check the conditions and make sure the job is finished when I do so: -what does each mean? can I rely on , is that set only when the job finishes, whether via or ? what other reasons are possible? It would definitely help to have specified on failure as well, and the differentiation to be made via the presence of .
/assign soltysh
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with .

Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
I have to say this issue is still very relevant. As of now, it is very hard to design something around Job Statuses. Anyway to move this forward?
Completely agree with previous comments! Even I found the way how to detect completed/failed jobs, still not sure am I covered all use cases. This part definitely needs clarifications/improvements/better documentation.
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
With regards to: If you actually follow this link and then search on that page for "conditions" you will get exactly zero hits. This is extremely unhelpful.
This documentation really, really needs work. (So does the API, for that matter.)
So what is the correct way to determine if a job is finished and whether it completed successfully?
When a Job completes (finished successfully), its will be set, and Job conditions will have one with . When a Job fails, Job conditions will have one with .
Hi, if nobody has been working on it, can I take this one? I have gone through the issue which is mentioned here and I would like to work on this. Can you please help me to get started as I am beginner to the project.
I totally agree with this- currently the only way to get the full picture of a Job is to read its conditions along with status fields which is error prone and a lot of extra work. I think a good start would be to add a new field as suggested by so as to not break compatibility for users already depending on
This sound reasonable. I'm still unclear when can contain more than one element. Also what does a or mean?
Still, open -- anyone actively working on this? I would like to attempt if not. :)
Are you still working on this?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.

If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
bump
This does not match what I just witnessed in my cluster… digging deeper to see if I need to file a bug but I have a Job with a single Pod that was executed. The Pod failed but the Job was marked as complete. This is the one (and only) pod that executed as my job: And this is the job that owns it (from kubectl describe): This is the job status object: on v1.16.7 but not finding any similar reports. just found this page as my only relevant search result.
I think it not only needs documentation, but also a change: It would be good to have 2 fields (probably needs better names), this way it's easy and clear when a job is finished and if it failed or not and when (someone can look at failed, succeeded and conditions for details): finishedtime: only set when the job finished (either completely failed or completely finished) finishtype: 'failed or completed'


Question: if len(event.ReportingController) == 0 { allErrs = append(allErrs, field.Required(field.NewPath("reportingController"), "")) } <del> for _, msg := range validation.IsQualifiedName(event.ReportingController) { allErrs = append(allErrs, field.Invalid(field.NewPath("reportingController"), event.ReportingController, msg)) } </del> <ins> allErrs = append(allErrs, ValidateQualifiedName(event.ReportingController, field.NewPath("reportingController"))...) </ins> if len(event.ReportingInstance) == 0 { allErrs = append(allErrs, field.Required(field.NewPath("reportingInstance"), "")) }

After got merged, we have new helper function here: It will be good for us to use this simpler helper function to replace previous function. /good-first-issue
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : After got merged, we have new helper function here: </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig usability
I would like to take this one /assign
I worked in on this as well, but kept your changed files untouched (as written in my PR description). same for you. I hope this is OK for you.
This should be reopened until and got merged (my PR excluded the changes from the other PRs).
/reopen per last comment
Reopened this issue. <detailsIn response to : This should be reopened until and got merged (my PR excluded the changes from the other PRs). per last comment Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
can you assign me
/assign
/assign
I'd love to work on this issue.

/assign
Hi, , I am a first commitor for kubernetes and I have committed a PR in , could you help me to review it, or give me some suggestions. Thanks a lot.
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: "github.com/google/gofuzz" ) <ins> func TestIsList(t *testing.T) { tests := []struct { obj    runtime.Object isList bool }{ {&api.PodList{}, true}, {&api.Pod{}, false}, } for _, item := range tests { if e, a := item.isList, runtime.IsListType(item.obj); e != a { t.Errorf("%v: Expected %v, got %v", reflect.TypeOf(item.obj), e, a) } } } </ins> func TestExtractList(t *testing.T) { pl := &api.PodList{ Items: []api.Pod{

Had a report of this over email. Filing this issue so we don't forget to investigate.
Can't reproduce.


Question: } if *build { <del> if !runBash("build", `test-build-release`) { </del> <ins> if !run("build-local", "hack/build-go.sh") { log.Fatal("Error building. Aborting.") } if !runBash("build-release", `test-build-release`) { </ins> log.Fatal("Error building. Aborting.") } }

building release, hopefully it's just binary missing; but i expect e2e-test to build binary automatically.
Steps to reproduce? /cc
Just run hack/e2e-, will fail. The _output/go/bin directory only has integration binary. (tried run first, still fails).
Did you try hack/build- first? e2e is in the default target list:
It's also in the client target list in the new build infrastructure:
Ah, I think that will do the trick. I didn't run hack/build-, as I saw e2e- output something like this (A little misleading if it won't build the binary). Re-running test now, will close the issue if it works. But it would be great if the binary will be built automatically.
I agree it should be built automatically. Otherwise, we'd at least have to update our e2e documentation.
Ah, yeah, I ran into this too and haven't fixed it yet. I this cmd/e2e to the build system, where does it end up? Apparently not in _output...


Question: # arguments: # $@: all stuff that goes after 'gcloud compute ' function run-gcloud-compute-with-retries { <ins> echo "" > /tmp/gcloud_retries </ins> for attempt in $(seq 1 ${RETRIES}); do <del> if ! gcloud compute $@; then </del> <ins> if ! gcloud compute $@ &> /tmp/gcloud_retries; then if [[ $(grep -c "already exists" /tmp/gcloud_retries) -gt 0 ]]; then if [[ "${attempt}" == 1 ]]; then echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 exit 1 fi echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" return 0 fi </ins> echo -e "${color_yellow}Attempt $(($attempt+1)) failed to $1 $2 $3. Retrying.${color_norm}" >& 2 sleep $(($attempt * 5)) else

https://k8s- Failed: Kubemark {} Previous issues for this test:
Builds: ci-kubernetes-kubemark-5-gce Failed: Kubemark {}
There are few things here: 3443 run - can you please invstigate it? 3480 run - gcloud copy-files should be retried 3550 run - we should better handle "already exists" errors (and not retry them). I think was looking into it at some point, so he may have already did something
https://k8s- Failed: Kubemark {}
I was looking at 'already exists' problem in RouteController, not in Kubemark.
Builds: ci-kubernetes-kubemark-5-gce Failed: Kubemark {}
Here are the causes of the above tests to fail and fixes: Test Fix ------ ------ ci-kubernetes-kubemark-5-gce/3443 Ignore (as its super flaky) or change create-certs/generate-certs to use a different directory ci-kubernetes-kubemark-5-gce/3480 Run the copy-files with retries as it is idempotent ci-kubernetes-kubemark-5-gce/3550 Don't retry when resource already exists ci-kubernetes-kubemark-5-gce/3666 Not worry as it is flaky or add retrying logic to all gclouds in that script ci-kubernetes-kubemark-5-gce/3787 Run the ssh with retries as it is idempotent
This PR fixes issues with the 2nd, 3rd and 5th tests in the above table. Could you PTAL at the PR? Also, I think we can safely ignore the 1st test (which is super flaky) and what do you suggest about the 4th test (do we add retrying logic to just call inside detect-master function of or let it stay as it is because there are a lot of other such gcloud calls inside the file, all of which should be changed rightfully)?
Let's not fix everything - fix just the things that you need, and direct vicinity.
Ok. So then let's ignore the 1st and the 4th tests in the above table as they are not really important. 2nd, 3rd and 5th are solved by PR . And regarding failure of , it looks like it is a flaky issue as well.

The density test succeeded, but the load test failed. From the logs it appears to be an issue with the client caches returning old resource versions for pod watches, with many such log statements: Does this worry you or you'd deem it to be just some weird flake?
It might mean a number of things, but most likely something gets overloaded/congested. It might be hard to debug, as we're seeing this from time to time, but it doesn't look very serious so I think we never investigated it thoroughly.


Question: t.Errorf("Expected expectedVolumesNeedDevicePath:n%vn got:n%v", expectedVolumes, rcInstance.volumesNeedUpdateFromNodeStatus) } <del> expectedVolumes = make([]v1.UniqueVolumeName, len(tc.expectedVolumesNeedReportedInUse)) for i := range tc.expectedVolumesNeedReportedInUse { expectedVolumes[i] = v1.UniqueVolumeName(tc.expectedVolumesNeedReportedInUse[i]) } if !reflect.DeepEqual(expectedVolumes, rcInstance.volumesNeedReportedInUse) { t.Errorf("Expected volumesNeedReportedInUse:n%vn got:n%v", expectedVolumes, rcInstance.volumesNeedReportedInUse) } </del> volumesFailedReconstruction := sets.NewString() for _, vol := range rcInstance.volumesFailedReconstruction { volumesFailedReconstruction.Insert(vol.volumeSpecName)

remove and was introduced in to fix This is to overcome the situation that volume is to DSW at first and then deleted before it is to ASW, which causes mounted volume neither in DSW nor ASW and won't execute unmount/detach: The reason the volume cannot be to ASW in time is , which is . However, there is another PR also solved the issue. What it did is to check whether ASW already has the volume before deleting it from DSW, this makes sure that volume can be detached/unmounted after it is deleted from DSW. I suggest that we can keep the second solution and remove related code to make the reconstruction process and reconcile loop cleaner?
/sig storage
cc
Should I proceed with this issue? Your guidance is appreciated.
I think we'd better wait for the feedback from
cc
Sorry for a late reply. This is how behind we are with reviews :-(. Reading Now that is on by default, it reconstructs volumes first and always puts them to ASW, it should never happen that a reconstructed volume is in DSW and not in ASW, right? So we can remove easily. I would not be so sure without , because the code is 5 years old and who knows all the implications. You could be right though, it may not be necessary there too.
/triage accepted
/assign


Question: } ginkgo.BeforeEach(func() { <ins> // Precautionary check that kubelet is healthy before running the test. waitForKubeletToStart(context.TODO(), f) </ins> ginkgo.By("setting up the pod to be used in the test") e2epod.NewPodClient(f).Create(context.TODO(), testCase.podSpec) })

ci-cgroupv1-containerd-node-arm64-e2e-serial-ec2-eks ci-cgroupv1-containerd-node-e2e-serial-ec2-eks It has been failing consistently for a long time. No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
There's another bug that was filed a long time ago. But there's no update on it and the same test in gce jobs seem to be green now.
cc


Question: The following environment variables should refer to keys and certificates on the host that are specific to that host. - `FLOCKER_CONTROL_SERVICE_CA_FILE` should refer to the full path to the cluster certificate file <del> - `FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE` should refer to the full path to the key file for the API user - `FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE` should refer to the full path to the certificate file for the API user </del> <ins> - `FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE` should refer to the full path to the [api key](https://docs.clusterhq.com/en/latest/config/generate-api-plugin.html) file for the API user - `FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE` should refer to the full path to the [api certificate](https://docs.clusterhq.com/en/latest/config/generate-api-plugin.html) file for the API user </ins> More details regarding cluster authentication can be found at the documentation: [Flocker Cluster Security & Authentication](https://docs.clusterhq.com/en/latest/concepts/security.html) and [Configuring Cluster Authentication](https://docs.clusterhq.com/en/latest/config/configuring-authentication.html). ### Create a pod with a Flocker volume <del> **Note**: A new dataset must first be provisioned using the Flocker tools. For example, using the [Volumes CLI](https://docs.clusterhq.com/en/latest/labs/volumes-cli.html)), create a new dataset called 'my-flocker-vol' of size 10GB: </del> <ins> **Note**: A new dataset must first be provisioned using the Flocker tools or Docker CLI *(To use the Docker CLI, you need the [Flocker plugin for Docker](https://clusterhq.com/docker-plugin/) installed along with Docker 1.9+)*. For example, using the [Volumes CLI](https://docs.clusterhq.com/en/latest/labs/volumes-cli.html), create a new dataset called 'my-flocker-vol' of size 10GB: </ins> ```sh <del> flocker-volumes create -m name=my-flocker-vol -s 10G </del> <ins> flocker-volumes create -m name=my-flocker-vol -s 10G -n <node-uuid> # -n or --node= Is the initial primary node for dataset (any unique # prefix of node uuid, see flocker-volumes list-nodes) </ins> ``` The following *volume* spec from the [example pod](flocker-pod.yml) illustrates how to use this Flocker dataset as a volume. <ins> > Note, the [example pod](flocker-pod.yml) used here does not include a replication controller, therefore the POD will not be rescheduled upon failure. If your looking for an example that does include a replication controller and service spec you can use [this example pod including a replication controller](flocker-pod-with-rc.yml) </ins> ```yaml volumes: - name: www-root

We found that some users coming to the document were getting a little confused. It should include more detail about where to install Flocker services, how to set environment variables, that you can create volumes now with Docker CLI and give more detail in example including how to use it with a spec that includes a replication controller.


Question: // NewProxyServer returns a new ProxyServer. func NewProxyServer(o *Options) (*ProxyServer, error) { <del> return newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master) </del> <ins> return newProxyServer(o.config, o.CleanupAndExit, o.scheme, o.master) </ins> } func newProxyServer( config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, <del> cleanupIPVS bool, </del> scheme *runtime.Scheme, master string) (*ProxyServer, error) {

What would you like to be : Remove any logic in kube-proxy that tries to automatically clean up proxy rules based on the proxy mode. Only attempt to clean up if is set. Why is this needed: This is a continuation of a discussion from There's a lot of complexity that comes with switching the proxy mode on kube-proxy because it tries to delete proxy rules created from other proxy modes. This has a lot of side-effects and we should be encouraging users to reboot thier nodes along with switching proxy modes. One example of this can be found
/sig network
I can take this on - does this issue have/need a KEP?
I think not but will defer to
I think a micro-KEP is appropriate. Specifically to talk about compatibility with regards to flags and to talk about docs (not sure which, but we have to write this down :)
also to talk about --cleanup (I think it should probably clean up its own mode, not the others -- or maybe I am wrong, please think about the right thing here :)
/assign (Yay, I can do this now!)
/priority important-soon

What happened: When restarting kube-proxy in iptables mode, there will be several seconds of timeouts because it flushes the KUBE-SERVICES nat chain and several others. What you expected to happen: Restarting kube-proxy should not impact traffic in any way. How to reproduce it (as minimally and precisely as possible): an HTTP server behind a ClusterIP service an http client in a loop making requests to the HTTP server -15 the kube-proxy process on the client node timeout for a few seconds until kube-proxy re-syncs the iptables rules Anything else we need to know?: Running with makes it pretty clear what's happening: it says it's cleaning up inactive rules, but these are crucial to the iptables proxier. You can trivially reproduce by just running: Environment: Kubernetes version (use ): 1.10.11 Cloud provider or hardware configuration: amazon OS (e.g: ): custom Kernel (e.g. ): 4.14.77-hs623.el6.x86_64 Install tools: custom Others: iptables 1.6.2 We're running kube-proxy 1.10.11, but I've confirmed this issue with kube-proxy 1.13.4 too
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Possibly a bug in the proxy. It flushes iptabes if the proxier is able to use IPVS which seems like the wrong behavior to me if there are shared tables/chains between the IPVS and iptables proxier. One workaround I can think of right now is to unload the ipvs kernel module to skip this check.
/assign
oh, that's an interesting workaround, I'll give that a shot, thanks!
np, I'll try to work on the bug fix in the meantime!
/triage unresolved
note that unloading the ipvs module is not actually enough of a workaround because kube-proxy will load the kernel module if it sees that it is available. If I unload the ipvs module and then rename to something else in then it does the right thing. Dug slightly further: the way it probes for modules is by actually running modprobe.

So the simplest hack that allows us to keep ipvs loaded for other things on the system is to put a modprobe script on its PATH that just always exits 1 for the ip_vs modules. (We can stop mounting the modules dir into the kube-proxy container in our kubernetes clusters, but we also run kube-proxy on non-kubernetes nodes via the init system, so the PATH hack works well enough there). This should hold us over fine until we get to a version with your fix in it.
Good to know, thanks for sharing!
/assign
Quick update on this issue from today's SIG Network call: we're going to try to get rid of the automatic proxy clean up altogether for v1.14.1 since this is considered a bug. is working on the KEP & implementation.


Question: func Test_growSlice(t *testing.T) { type args struct { initialCapacity int <ins> initialLen      int </ins> v               reflect.Value maxCapacity     int sizes           []int

has by default, and since 1.22 has the same. They seem to return wrong results in certain situations combined with / . And / also use , so they also don't work correctly. outputs partial list of pods on the node, doesn't evict some of the pods. To make it work correctly, I had to specify works correctly with any , since it's just a performance optimizing parameter. I can't provide an actual concrete example (no = default ), but instead I show how to reproduce it with a smaller I believe that the same problem occurs with create two deployments. there are 5 pods in each. with --selector=app=another-server and , it shows only 4 pods. I could confirm that the problem happens in both case of and regardless namespace, even with , or <details</details<detailsGCP / GKE Standard </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
/sig node
I think this is indeed a bug, after some simple testing, it seems to be an bug unique to 1.24, let me fix it. /assign
/assign
I can reproduce this error in a 1.24 cluster, but not in a 1.23.6 cluster. All clusters are created via kind, and I'm guessing gke may have cherry-picked some commits or had some of their own changes to their branch. So I think this fix should at least cherry-pick to 1.23 to help other distributions fix this issue.
/remove-sig api-machinery


Question: COPY --from=prep /dig-dir /dig COPY --from=prep /vcruntime140.dll /Windows/System32/ <ins> # include nltest COPY --from=servercore-helper /Windows/System32/nltest.exe /Windows/System32/nltest.exe COPY --from=servercore-helper /Windows/System32/en-US/nltest.exe.mui /Windows/System32/en-US/nltest.exe.mui COPY --from=servercore-helper /Windows/System32/ntdsapi.dll /Windows/System32/ntdsapi.dll </ins> # include powershell and its Module analysis cache. COPY --from=ps-helper ["/PowerShell", "/Program Files/PowerShell"]

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI -- [sig-windows] [Feature:Windows] GMSA Kubelet [Slow] kubelet GMSA support when creating a pod with correct GMSA credential specs passes the credential specs down to the Pod's containers [sig-windows] [Feature:Windows] GMSA Full [Serial] [Slow] GMSA support works end to end ** pause image using in image was transitioned to the official pause image which no longer has in the image related to /sig windows /triage accepted


Question: glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. <del> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec, &b.pod, *b.opts) </del> <ins> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, *b.opts) </ins> if err != nil { return err }

Data as of June 3rd, 9:00 PM Eastern. Latest data can be found here: Prow link to the 2 jobs: * Additional Context:
/sig storage cc
I'm working on , it's hard to get a job yaml right at the first time. I'll check iscsi. /assign
/triage accepted /kind failing-test
ci-kubernetes-e2e-storage-kind-disruptive fixed: has 1 green run pull-kubernetes-e2e-storage-kind-disruptive fixed: , it fails when it should fail, it's green otherwise, I'll continue with iscsi.
/reopen it's still red, but at least it installs now:
Reopened this issue. <detailsIn response to : /reopen it's still red, but at least it installs now: Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
e2e-gce-iscsi-serial job passes now: e2e-gce-iscsi is getting greener: I'll let it run for few more days to see if today's failures are flakes. /reopen
Reopened this issue. <detailsIn response to : It's getting greener: /reopen </details
iscsi-serial is green:


Question: }, enableSchedulingGates: true, }, <ins> { name: "addition to nodeAffinity is allowed for gated pods with nil affinity", create: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "fake-name", Image: "fakeimage", }, }, SchedulingGates: []v1.PodSchedulingGate{{Name: "baz"}}, }, }, update: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "fake-name", Image: "fakeimage", }, }, Affinity: &v1.Affinity{ NodeAffinity: &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ // Add 1 MatchExpression and 1 MatchField. NodeSelectorTerms: []v1.NodeSelectorTerm{ { MatchExpressions: []v1.NodeSelectorRequirement{ { Key:      "expr", Operator: v1.NodeSelectorOpIn, Values:   []string{"foo"}, }, }, MatchFields: []v1.NodeSelectorRequirement{ { Key:      "metadata.name", Operator: v1.NodeSelectorOpIn, Values:   []string{"foo"}, }, }, }, }, }, }, }, SchedulingGates: []v1.PodSchedulingGate{{Name: "baz"}}, }, }, enableSchedulingGates: true, }, </ins> } for _, tc := range cases { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodSchedulingReadiness, tc.enableSchedulingGates)()

When a gated pod is created with nil affinity and a user/controller tries to update its nodeAffinity, the validation fails. The pod gets updated with the new affinity as per rules in a pod as in the following using : to update the pod as with: No response <details</details<detailsTested on kind and OpenShift </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling


Question: <del> /* Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package label import ( "bytes" "context" "errors" "fmt" "io" "sync" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" cloudprovider "k8s.io/cloud-provider" cloudvolume "k8s.io/cloud-provider/volume" volumehelpers "k8s.io/cloud-provider/volume/helpers" persistentvolume "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" api "k8s.io/kubernetes/pkg/apis/core" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" ) const ( // PluginName is the name of persistent volume label admission plugin PluginName = "PersistentVolumeLabel" ) // Register registers a plugin func Register(plugins *admission.Plugins) { plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { persistentVolumeLabelAdmission := newPersistentVolumeLabel() return persistentVolumeLabelAdmission, nil }) } var _ = admission.Interface(&persistentVolumeLabel{}) type persistentVolumeLabel struct { *admission.Handler mutex            sync.Mutex cloudConfig      []byte azurePVLabeler   cloudprovider.PVLabeler vspherePVLabeler cloudprovider.PVLabeler } var _ admission.MutationInterface = &persistentVolumeLabel{} var _ kubeapiserveradmission.WantsCloudConfig = &persistentVolumeLabel{} // newPersistentVolumeLabel returns an admission.Interface implementation which adds labels to PersistentVolume CREATE requests, // based on the labels provided by the underlying cloud provider. // // As a side effect, the cloud provider may block invalid or non-existent volumes. func newPersistentVolumeLabel() *persistentVolumeLabel { // DEPRECATED: in a future release, we will use mutating admission webhooks to apply PV labels. // Once the mutating admission webhook is used for Azure, and GCE, // this admission controller will be removed. klog.Warning("PersistentVolumeLabel admission controller is deprecated. " + "Please remove this controller from your configuration files and scripts.") return &persistentVolumeLabel{ Handler: admission.NewHandler(admission.Create), } } func (l *persistentVolumeLabel) SetCloudConfig(cloudConfig []byte) { l.cloudConfig = cloudConfig } func nodeSelectorRequirementKeysExistInNodeSelectorTerms(reqs []api.NodeSelectorRequirement, terms []api.NodeSelectorTerm) bool { for _, req := range reqs { for _, term := range terms { for _, r := range term.MatchExpressions { if r.Key == req.Key { return true } } } } return false } func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) { if a.GetResource().GroupResource() != api.Resource("persistentvolumes") { return nil } obj := a.GetObject() if obj == nil { return nil } volume, ok := obj.(*api.PersistentVolume) if !ok { return nil } volumeLabels, err := l.findVolumeLabels(volume) if err != nil { return admission.NewForbidden(a, err) } requirements := make([]api.NodeSelectorRequirement, 0) if len(volumeLabels) != 0 { if volume.Labels == nil { volume.Labels = make(map[string]string) } for k, v := range volumeLabels { // We (silently) replace labels if they are provided. // This should be OK because they are in the kubernetes.io namespace // i.e. we own them volume.Labels[k] = v // Set NodeSelectorRequirements based on the labels var values []string if k == v1.LabelTopologyZone || k == v1.LabelFailureDomainBetaZone { zones, err := volumehelpers.LabelZonesToSet(v) if err != nil { return admission.NewForbidden(a, fmt.Errorf("failed to convert label string for Zone: %s to a Set", v)) } // zone values here are sorted for better testability. values = zones.List() } else { values = []string{v} } requirements = append(requirements, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values}) } if volume.Spec.NodeAffinity == nil { volume.Spec.NodeAffinity = new(api.VolumeNodeAffinity) } if volume.Spec.NodeAffinity.Required == nil { volume.Spec.NodeAffinity.Required = new(api.NodeSelector) } if len(volume.Spec.NodeAffinity.Required.NodeSelectorTerms) == 0 { // Need at least one term pre-allocated whose MatchExpressions can be appended to volume.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]api.NodeSelectorTerm, 1) } if nodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) { klog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.", requirements, volume.Spec.NodeAffinity) } else { for _, req := range requirements { for i := range volume.Spec.NodeAffinity.Required.NodeSelectorTerms { volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions = append(volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions, req) } } } } return nil } func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) { existingLabels := volume.Labels // All cloud providers set only these two labels. topologyLabelGA := true domain, domainOK := existingLabels[v1.LabelTopologyZone] region, regionOK := existingLabels[v1.LabelTopologyRegion] // If they don't have GA labels we should check for failuredomain beta labels // TODO: remove this once all the cloud provider change to GA topology labels if !domainOK || !regionOK { topologyLabelGA = false domain, domainOK = existingLabels[v1.LabelFailureDomainBetaZone] region, regionOK = existingLabels[v1.LabelFailureDomainBetaRegion] } isDynamicallyProvisioned := metav1.HasAnnotation(volume.ObjectMeta, persistentvolume.AnnDynamicallyProvisioned) if isDynamicallyProvisioned && domainOK && regionOK { // PV already has all the labels and we can trust the dynamic provisioning that it provided correct values. if topologyLabelGA { return map[string]string{ v1.LabelTopologyZone:   domain, v1.LabelTopologyRegion: region, }, nil } return map[string]string{ v1.LabelFailureDomainBetaZone:   domain, v1.LabelFailureDomainBetaRegion: region, }, nil } // Either missing labels or we don't trust the user provided correct values. switch { case volume.Spec.AzureDisk != nil: labels, err := l.findAzureDiskLabels(volume) if err != nil { return nil, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err) } return labels, nil case volume.Spec.VsphereVolume != nil: labels, err := l.findVsphereVolumeLabels(volume) if err != nil { return nil, fmt.Errorf("error querying vSphere Volume %s: %v", volume.Spec.VsphereVolume.VolumePath, err) } return labels, nil } // Unrecognized volume, do not add any labels return nil, nil } // getAzurePVLabeler returns the Azure implementation of PVLabeler func (l *persistentVolumeLabel) getAzurePVLabeler() (cloudprovider.PVLabeler, error) { l.mutex.Lock() defer l.mutex.Unlock() if l.azurePVLabeler == nil { var cloudConfigReader io.Reader if len(l.cloudConfig) > 0 { cloudConfigReader = bytes.NewReader(l.cloudConfig) } cloudProvider, err := cloudprovider.GetCloudProvider("azure", cloudConfigReader) if err != nil || cloudProvider == nil { return nil, err } azurePVLabeler, ok := cloudProvider.(cloudprovider.PVLabeler) if !ok { return nil, errors.New("Azure cloud provider does not implement PV labeling") } l.azurePVLabeler = azurePVLabeler } return l.azurePVLabeler, nil } func (l *persistentVolumeLabel) findAzureDiskLabels(volume *api.PersistentVolume) (map[string]string, error) { // Ignore any volumes that are being provisioned if volume.Spec.AzureDisk.DiskName == cloudvolume.ProvisionedVolumeName { return nil, nil } pvlabler, err := l.getAzurePVLabeler() if err != nil { return nil, err } if pvlabler == nil { return nil, fmt.Errorf("unable to build Azure cloud provider for AzureDisk") } pv := &v1.PersistentVolume{} err = k8s_api_v1.Convert_core_PersistentVolume_To_v1_PersistentVolume(volume, pv, nil) if err != nil { return nil, fmt.Errorf("failed to convert PersistentVolume to core/v1: %q", err) } return pvlabler.GetLabelsForVolume(context.TODO(), pv) } func (l *persistentVolumeLabel) findVsphereVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) { pvlabler, err := l.getVspherePVLabeler() if err != nil { return nil, err } if pvlabler == nil { return nil, fmt.Errorf("unable to build vSphere cloud provider") } pv := &v1.PersistentVolume{} err = k8s_api_v1.Convert_core_PersistentVolume_To_v1_PersistentVolume(volume, pv, nil) if err != nil { return nil, fmt.Errorf("failed to convert PersistentVolume to core/v1: %q", err) } labels, err := pvlabler.GetLabelsForVolume(context.TODO(), pv) if err != nil { return nil, err } return labels, nil } func (l *persistentVolumeLabel) getVspherePVLabeler() (cloudprovider.PVLabeler, error) { l.mutex.Lock() defer l.mutex.Unlock() if l.vspherePVLabeler == nil { var cloudConfigReader io.Reader if len(l.cloudConfig) > 0 { cloudConfigReader = bytes.NewReader(l.cloudConfig) } cloudProvider, err := cloudprovider.GetCloudProvider("vsphere", cloudConfigReader) if err != nil || cloudProvider == nil { return nil, err } vspherePVLabeler, ok := cloudProvider.(cloudprovider.PVLabeler) if !ok { // GetCloudProvider failed return nil, errors.New("vSphere Cloud Provider does not implement PV labeling") } l.vspherePVLabeler = vspherePVLabeler } return l.vspherePVLabeler, nil } </del>

In 1.30, the PersistentVolumeLabel admission plugin blocks creation of in-tree AzureDisk and vSphere PVs: The reason is that the cloud provider was removed (), but the admission plugin still calls GetCloudProvider there, which fails: The PV should be admitted + created. Enable PersistentVolumeLabel admission plugin. For example, when running local-up- Create AzureDisk PV on any 1.30 cluster, you don't need Azure cloud: /sig storage /priority important-soon <detailsToday's master </details<detailsAzure </detailsNo response <detailslocal-up- </details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I am fixing the worst bug in and I plan to backport it to 1.30. However, we should consider removing the whole PersistentVolumeLabel admission plugin. Most of the cloud providers were removed, only GCE remains and IMO even that one will be removed sooner or later. The admission plugin has no way to get volume zone + region and is therefore useless. The admission said (in 1.26, where we had all in-tree cloud providers): Such webhook never materialized for AWS, Azure nor vSphere and I am not aware of any plans for GCE.
It did materialize in So removing it from k/k is the best way forward.


Question: } func podWithSecrets(ns, podName string, toAttach secretsToAttach) *v1.Pod { <del> return podWithSecretsAndUID(ns, podName, "", toAttach) </del> <ins> return podWithSecretsAndUID(ns, podName, fmt.Sprintf("%s/%s", ns, podName), toAttach) </ins> } func podWithSecretsAndUID(ns, podName, podUID string, toAttach secretsToAttach) *v1.Pod {

qps of configmap get request is too high in kube-apiserver. when there are many configmaps in cluster mounted by running pod, the qps of configmap get request will be high and the request will also need to get configmap resource from etcd instead of returned from kube-apiserver cache. DO NOT get configmap from etcd. Get it from cache of kube-apiserver. kubelet configMapAndSecretChangeDetectionStrategy to Cache; kube-apiserver --v=3 lots of configmaps mount to pods kube-apiserver metrics kube-apiserver log I have tested it in v1.22.13 and v1.28.0. The logic to set resourceversion=0 seems never be triggered <details</details<details</details<details</details<details</details<details</details<details</details
/sig node
Thanks can you also fill-in at least the kubernetes version, but the more info the better in your initial comment?
k8s version.
/remove-sig node /sig apimachinery
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /remove-sig node /sig apimachinery Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig apiserver
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /sig apiserver Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
I think this issue is related to kubelet not kube-apiserver and i have found the root cause i think.
/sig node
Could you add concrete data for some metrics? the request rate you describe timing for the requests. Is there a slowdown?
request rate depends on the total number of configmaps mounted to pods. what i care is not reduce the qps but to use kube-apiserver cache instead of direct request to etcd. for now, the request will redirect to etcd. there is no resourceVersion in the request query.
/cc /triage accepted
/cc
can you add some steps to reproduce the issue?

and a link to the code you think is causing the problem?
steps: code: The invocation chain is quite lengthy and the reason like blow: kubelet has a ticker with 1s interval and will try to get pods from queue to sync if there is data in syncCh; is default set to 1m. pod will be enqueued with a random time between 1m ~ 1.5m by default; is background reconcicler in file. it will run every 100ms to try to mountAttachVolumes which could trigger configmap get operation when the volume is not mounted or require remount; .object will be set to nil each time pod to sync via . pay attention to the comment. to mount or remount volume, a configmap get request will be triggered and because the is nil, the resourceversion query will not be which result to the request goes to etcd finally. volume mounted, the flag will be set to false in ; when next mountAttachVolumes called but volume has been mounted and not require remount, no need to get configmap from kubelet's cache. therefore there will no http request to apiserver to get configmap; may be some configmap get operation can be satisfied by cachebasedmanager by directly returned from local cache or get it from apiserver's cache when cache expired because same configmap can be used by many pods and each pod has a goroutine to execute syncPod but only one goroutine to execute mountAttachVolumes. I think we can always set resourceversion=0 to the query and ignore whether it's the first time to fetch the configmap or just the local cache expired. The implementation of informer is also just to list all specified resources with resourceversion=0 when to start.
That was intentional, so that a newly scheduled pod would be sure to get latest content from the API. However, RegisterPod is called on every resync, and the cacheBasedManager#RegisterPod implementation is calling AddReference on all references every time, not just changed references. I agree we don't want to do that. No, we want to actually fetch a live object the first time we get data for a new pod. I think the fix needs to happen in cacheBasedManager#RegisterPod to only call AddReference for items which are not in c.getReferencedObjects(prev)
make sense to me.


Question: t.Fatal(err) } t.Run(mapping.Resource.String(), func(t *testing.T) { <del> if _, ok := ignoreList[mapping.Resource]; ok { t.Skip() } </del> status, ok := statusData[mapping.Resource] if !ok { status = statusDefault

It looks like the openapi is not routed properly to set-up the apiservice so apply requests will fail because the fieldmanager is not set-up properly. /cc /sig api-machinery /priority important-soon /kind bug
/wg apply /cc
OpenAPIModels is set to nil for the kube-aggregator server config (which serves the APIServices API): This is done because the kube-aggregator has a special openapi handler that aggregates all the other server's openapi docs. The field controls (at least) two things: /openapi endpoint serving (which we want kube-aggregator to continue to customize) creation of the field manager (which we want kube-aggregator to behave consistently on) We need to decouple presence of OpenAPIConfig and construction of StaticOpenAPISpec (which is needed by the field manager) from construction/installation of OpenAPIVersionedService. That might be a little tricky to untangle, since construction of StaticOpenAPISpec includes paths/operations. The field manager only needs the gvk/schemas.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/lifecycle frozen This is not something we want closed by accident.
/assign
For reference:
Additional lines for reference: This is roughly how SSA consumes the openapi spec. openapi spec -openapi proto model -typeconverter -SSA field manager For built in types, openapi spec -openapi model openapi model -typeconverter A similar thing happens for CRDs As liggitt mentioned, SSA only requires a subset of the information in the openapi generation, and the openapi generation in the aggregator slightly different than the process for builtins and CRDs.
/unassign
/assign


Question: } func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { <ins> klog.V(4).Infof("try to provide secret for image %s", image) </ins> cfg := credentialprovider.DockerConfig{} <del> ctx, cancel := getContextWithCancel() defer cancel() </del> if a.config.UseManagedIdentityExtension { <del> klog.V(4).Infof("listing registries") result, err := a.registryClient.List(ctx) if err != nil { klog.Errorf("Failed to list registries: %v", err) return cfg } for ix := range result { loginServer := getLoginServer(result[ix]) klog.V(2).Infof("loginServer: %s", loginServer) cred, err := getACRDockerEntryFromARMToken(a, loginServer) if err != nil { continue </del> <ins> if loginServer := parseACRLoginServerFromImage(image); loginServer == "" { klog.V(4).Infof("image(%s) is not from ACR, skip MSI authentication", image) } else { if cred, err := getACRDockerEntryFromARMToken(a, loginServer); err == nil { cfg[loginServer] = *cred </ins> } <del> cfg[loginServer] = *cred </del> } } else { // Add our entry for each of the supported container registry URLs

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: PR() fixed pull image error from a cross-subscription azure container registry by service principal, while for MSI, this issue is not fixed. Need to find a way how to fix this issue for MSI, related code is here: Here the difficulty is could not list all ACR since it's based on one subscriptionID: So if we could not get in a different subscription, how could we set the . What you expected to happen: How to reproduce it (as minimally and precisely as possible): error msg would be like following when pull a repository from an ACR in another subscription: Anything else we need to know?: Environment: Kubernetes version (use ): v.9 - v1.12 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: /sig azure /assign
By further investigation, I think we should implement LazyProvider for azure acrProvider: Related code:
Update: It's hard to fix this since in the beginning we cannot list ACR in another subscription, will always list acr under current subscription, and without target loginServer address of ACR, we cannot get : And in func, we cannot get target repo, it's not passed on
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
-- is this still an issue? If not, let's it out.
it's still an issue, and currently it should be low priority, let's keep it
Rotten issues close after 30d of inactivity.

Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-lifecycle rotten /reopen
Reopened this issue. <detailsIn response to : /remove-lifecycle rotten /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: } } totalSizeErrorCases := []map[string]string{ <del> {"a": strings.Repeat("b", totalAnnotationSizeLimitB)}, </del> <ins> {"a": strings.Repeat("b", TotalAnnotationSizeLimitB)}, </ins> { <del> "a": strings.Repeat("b", totalAnnotationSizeLimitB/2), "c": strings.Repeat("d", totalAnnotationSizeLimitB/2), </del> <ins> "a": strings.Repeat("b", TotalAnnotationSizeLimitB/2), "c": strings.Repeat("d", TotalAnnotationSizeLimitB/2), </ins> }, } for i := range totalSizeErrorCases {

Expose totalAnnotationSizeLimitB in the validation package: This value is useful for validating annotations outside of the validation package, such as in SSA: Otherwise, size validation on annotations outside of the package would diverge and cause issues later. /kind cleanup /wg api-expression cc
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign


Question: if err != nil { return err } <del> if !dv.CanAddr() { </del> <ins> if !dv.CanAddr() && !dv.CanSet() { </ins> return fmt.Errorf("can't write to dest") } sv, err := EnforcePtr(src)

In Go you cannot take the address of a because the map implementation hides that from you. Our conversion library assumes that any reflected map value can have Addr() taken, which causes a panic when you try to do the following for PodInfo conversion: The conversion code tries to call Addr() on the reflect.Value (the map's value, ContainerStatus value object) which fails because it cannot have an address. I worked around it by registering a conversion function on *PodInfo and then manually converting the map.
Oh you mean the reflection code can't addr that? ok sounds like a bug. Non-reflection code can certainly do it:
That playground is taking the address of a value before it's in the struct. You can't take the address of m ----- Original Message -----
Right, because go does magical c++ style references there. But I thought the map copying code handled that case specially already?
I think I see what you're talking about. I'll make a test.
I repro'd. Will fix.
It was actually the source value that caused the trouble. Thanks for the report!
Sorry, should have been clearer. ----- Original Message -----
Note that this map violates our API conventions. It should be a list of named subobjects.
Yeah this field has always been problematic. It's still a bug regardless-- could have triggered copying other maps.


Question: resourceWithReq := &lazyResource{req: req.WithContext(ctx)} assert.Equal(t, "resource", fmt.Sprintf("%v", resourceWithReq)) } <ins> func TestLazyScope(t *testing.T) { assert.Equal(t, "unknown", fmt.Sprintf("%v", &lazyScope{})) scopeWithEmptyReq := &lazyScope{&http.Request{}} assert.Equal(t, "unknown", fmt.Sprintf("%v", scopeWithEmptyReq)) req := &http.Request{} ctx := request.WithRequestInfo(context.TODO(), &request.RequestInfo{Namespace: "ns"}) scopeWithReq := &lazyScope{req: req.WithContext(ctx)} assert.Equal(t, "namespace", fmt.Sprintf("%v", scopeWithReq)) } </ins>

One of the use-cases for APIServer tracing would be to try and debug SLO-violating requests served by the API Server. Right now, it adds just some basic attributes: For API Server , SLO-related metrics have the following labels: "verb", "dryrun", "group", "version", "resource", "subresource", "scope", "component", "code" We should add those labels to at least one of the spans created by API Server requests. It isn't a graduation requirement for , but would make the feature more helpful.
/sig instrumentation
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
We might be able to do something using RequestInfo, similar to how the dropped request metric is recorded:


Question: // Now do the more expensive test initialization. l.config, l.testCleanup = driver.PrepareTest(f) <ins> l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName) </ins> l.resource = createGenericVolumeTestResource(driver, l.config, pattern) if l.resource.volSource == nil { framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name) } <del> l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName) </del> } cleanup := func() {

Example:
/assign
cc
For all those PD instances, tests are being skipped due to this line: Fixing it by removing that skip as suggested by
/open until we verify the issue is fixed
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Looks like diffResources became green on 5/27 /close
Closing this issue. <detailsIn response to : Looks like diffResources became green on 5/27 Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: 123.45.67.89	some.domain # Entries added by HostAliases. <del> 123.45.67.89	foo 123.45.67.89	bar 123.45.67.89	baz </del> <ins> 123.45.67.89	foo	bar	baz </ins> `, }, {

What would you like to be : Currently, specifying multiple aliases for a single host using results in a line for each alias in . Example: leads to an file containing something like Why is this needed: Having multiple entries for the same IP address in can break DNS resolution for some software. See for more details <!-- DO NOT EDIT BELOW THIS LINE --/kind feature
/sig node


Question: spec: containers: - name: fluentd-cloud-logging <del> image: gcr.io/google_containers/fluentd-gcp:1.12 </del> <ins> image: gcr.io/google_containers/fluentd-gcp:1.13 </ins> resources: limits: cpu: 100m

When I run Kubernetes cluster on GKE, it generates a lot of system logs coming from docker and kubelet. Even if my cluster runs just a few pods, I can see dozens or hundreds of log entries per second available in Google Cloud Logging in docker and kubelet logs. This makes the actual logs from my application get lost among them, if I do not set appropriate filter. Also, I am afraid that collecting massive amounts of logs imposes unnecessary workload on fluentd pod in my cluster. It would be good if I could just disable collection of kubelet/docker logs (and potentially other Kubernetes system logs) assuming that I do not want to have these logs exported.
/cc
since you tend to have a vested interest in node debug-ability, what's the minimum you'd want to have available here? From GKE's perspective, our users typically have no interest in these logs, so turning them off by default, even without an easy option to re-enable them, makes sense. Any user sophisticated enough to want them shouldn't have problems SSH'ing to the node for them. Is the story all that different in open-source kubernetes? Does anyone even care to have the kubelet and docker logs collected from the nodes? I've never used cloud logging to debug issues with them, it's always been easier to just SSH to the node. I'd lean towards not collecting them on the nodes in general. Also cc in case he has wisdom he'd like to impart
All I can say is that personally I have never accessed kubelet etc. logs "for real" when debugging -- as Alex says -- it is usually more direct just to SSH in. I would support not collecting these logs to improve cluster stability (but perhaps have a switch / technique to make it easy to collect them if the admin does care about this?).
One last call for to complain, because I'm starting on this now
Also
We want to avoid storing logs onto local disk in the future. In-order to achieve that, we need the existing cluster level logging infrastructure to work. Can the GKE problem be solved by moving system logs to a separate namespace? , Alex Robinson wrote:
How far in the future? Unless that's intended to happen soon, collecting the kubelet and docker logs in the meantime offers little value with a nontrivial resource (cpu+memory+egress) cost.

But yes, we can handle the system logs separately if we'd like. I think the most natural way would be to report them as compute engine logs, which could be done by adding an extra config option into the gcp output plugin.
, Alex Robinson wrote:
-- just to confirm, do you think this fix will be sufficient for you? In GKE, logs from containers will be listed under Container Engine, while logs from Docker and the Kubelet will be listed under Compute Engine, with the logs for each effectively being entirely seperated in the log viewer.
Yes, this will be sufficient in terms of not polluting Container Engine logs with system logs. I am still concerned about unnecessary load on fluentd caused by exporting Docker and Kubelet logs so ideally it would be great not to export these system logs if a user is not interested in them. But this is of slightly lower priority.
Hi, What's the status from this request? Is this available in 1.7.x or higher? Thanks.


Question: func (pm *basicManager) DeleteOrphanedMirrorPods() { podFullNames := pm.getOrphanedMirrorPodNames() for _, podFullName := range podFullNames { <del> pm.MirrorClient.DeleteMirrorPod(podFullName) </del> <ins> pm.MirrorClient.DeleteMirrorPod(podFullName, nil) </ins> } }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: of a static pod (really the mirror pod) results in multiple attempts to re-create the mirror pod before it finally succeeds (note that in some runs it goes into Running or Terminating state -- it just depends upon the exact timing of the . Note that in the below trace the pod name is elided from the output. Note that the actual static pod continues to run as normal. What you expected to happen: Mirror pod should start without additional state transitions How to reproduce it (as minimally and precisely as possible): Create a static pod, it, and Anything else we need to know?: Environment: Kubernetes version (use ): 1.14/OpenShift 4.2 (master has the same code) Cloud provider or hardware configuration: AWS (not relevant) OS (e.g: ): Not relevant Kernel (e.g. ): Not relevant Install tools: Not relevant Network plugin and version (if this is a network-related bug): N/A Others:
/sig node
More details: The bug appears to be a race condition between deleting the old mirror pod, the API server deleting its record of it, and creating a replacement that's not related to skew. I suspect the reason I didn't see it bouncing back and forth in my earlier tests was simply that I wasn't running the kubectl get node frequently enough; if I run it continuously, I see it bounce. The sequence is: 1) Somebody kubectl deletes the mirror pod. 2) The static pod code in (around line 1630) notices that the pod has been marked deleted (its deletion timestamp is not nil and it is a mirror pod), deletes it (in DeleteMirrorPod in , via the apiserverClient), and creates a new one (again, via the apiserverClient). 3) The next time around, the static pod code may see either the old pod (that has been deleted) or the new one as the mirror pod.

If the old pod is the one that's seen, it sees that there's a deletion timestamp, so it deletes it again -- which, since pod deletion is by name (UID isn't involved) results in the new pod being deleted and yet another new pod being created. If the new pod is the one that's seen, then we break out with the spin cycle complete. Still TBD is why the mirror pod isn't getting replaced right away. But here are traces of the relevant sequences: Bad sequence: Good sequence: The messages at lines 7 and 18 were printed immediately after the call to apiServerClient...Create(). But the key is at lines 9 and 19. At line 9, note that the UID (...) is the UID of the old mirror pod, which has a deletion timestamp and therefore the mirror code deletes the pod -- but the pod that's actually in existence at the time is the new pod. At line 19, however, note that the mirror pod is now the new one, which is not marked for deletion, and is therefore left alone. The solution (I'll open a PR shortly) is to check both the pod name and the UID to make sure that we're in fact trying to delete the pod we think we want to delete.
Thanks for the detailed issue :) Really awesome!
I'm seeing this issue on an upgrade from v1.14.4 to v1.15.1 following the on a CentOS 7 system. After installing the updated kubelet package and restarting kubelet per the instructions, the cluster went into a loop of continually restarting the etcd pod from one control plane node and the scheduler pod from another control plane node. This continued for approximately 10 minutes before the etcd pod finally stayed up. It was another 5 minutes before the scheduler pod finally ran. I noticed v1.15.2 was released although the change log doesn't show this issue included. Was there a plan to back-port this fix to 1.15 and 1.14 (not to the same extent, but I have seen this behavior on a 1.14.4 cluster)? Since this behavior can happen to the apiserver pod, it has the potential to be very disruptive to a cluster.
The actual etcd pod, or the mirror pod (which is more likely)?

I hadn't specifically planned on backporting it, but it shouldn't be a problem; this code has been stable.
Ah, good point--the actual pod does appear to remain running. That makes it less important than I was initially thinking. However, it is a bit disconcerting for someone who happens to be looking at the state of the cluster (or for monitoring) to see the pod appear to be stuck in a restart loop. If it were just a few iterations, it might not matter too much. However, I'm seeing this happen for minutes at a time. For example, to verify that the real pod remained available I did a on the apiserver pod for one master, and output showed it in the restart loop for 15 minutes before it settled down. IMO that's worth backporting if it's low risk.
I opened a cherrypick against 1.15 for this.
Appreciated. Thanks!


Question: } } <ins> handlerSyncReadyCh := make(chan struct{}) handlerSyncCheck := healthz.NamedCheck("sched-handler-sync", func(_ *http.Request) error { select { case <-handlerSyncReadyCh: return nil default: } return fmt.Errorf("waiting for handlers to sync") }) readyzChecks = append(readyzChecks, handlerSyncCheck) </ins> // Start up the healthz server. if cc.SecureServing != nil { <del> handler := buildHandlerChain(newHealthzAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) </del> <ins> handler := buildHandlerChain(newHealthEndpointsAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks, readyzChecks), cc.Authentication.Authenticator, cc.Authorization.Authorizer) </ins> // TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve if _, _, err := cc.SecureServing.Serve(handler, 0, ctx.Done()); err != nil { // fail early for secure handlers, removing the old error loop from above

Quoted from 's Splitting healthz/ into livez/ and readyz/
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
/sig scheduling
I read all the content of the PR comment and am willing to work on this issue Do we need to continue some discussion for this issue
Hi , thanks for offering your help. But I was working in progress before knowing you can work on this. However, please feel free to contribute the similar issue
/assign
The Kubernetes project currently lacks enough contributors to adequately respond to all issues. This bot triages un-triaged issues according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue as fresh with Close this issue with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale


Question: <ins> /* Copyright 2014 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "fmt" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Services", func() { var c *client.Client BeforeEach(func() { c = loadClientOrDie() }) It("should provide DNS for the cluster", func() { if testContext.provider == "vagrant" { By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)") return } podClient := c.Pods(api.NamespaceDefault) //TODO: Wait for skyDNS // All the names we need to be able to resolve. namesToResolve := []string{ "kubernetes-ro", "kubernetes-ro.default", "kubernetes-ro.default.kubernetes.local", "google.com", } probeCmd := "for i in `seq 1 600`; do " for _, name := range namesToResolve { probeCmd += fmt.Sprintf("wget -O /dev/null %s && echo OK > /results/%s;", name, name) } probeCmd += "sleep 1; done" // Run a pod which probes DNS and exposes the results by HTTP. By("creating a pod to probe DNS") pod := &api.Pod{ TypeMeta: api.TypeMeta{ Kind:       "Pod", APIVersion: "v1beta1", }, ObjectMeta: api.ObjectMeta{ Name: "dns-test-" + string(util.NewUUID()), }, Spec: api.PodSpec{ Volumes: []api.Volume{ { Name: "results", Source: api.VolumeSource{ EmptyDir: &api.EmptyDir{}, }, }, }, Containers: []api.Container{ { Name:  "webserver", Image: "kubernetes/test-webserver", VolumeMounts: []api.VolumeMount{ { Name:      "results", MountPath: "/results", }, }, }, { Name:    "pinger", Image:   "busybox", Command: []string{"sh", "-c", probeCmd}, VolumeMounts: []api.VolumeMount{ { Name:      "results", MountPath: "/results", }, }, }, }, }, } By("submitting the pod to kuberenetes") _, err := podClient.Create(pod) if err != nil { Fail(fmt.Sprintf("Failed to create %s pod: %v", pod.Name, err)) } defer func() { By("deleting the pod") defer GinkgoRecover() podClient.Delete(pod.Name) }() By("waiting for the pod to start running") waitForPodRunning(c, pod.Name) By("retrieving the pod") pod, err = podClient.Get(pod.Name) if err != nil { Fail(fmt.Sprintf("Failed to get pod %s: %v", pod.Name, err)) } // Try to find results for each expected name. By("looking for the results for each expected name") var failed []string for try := 1; try < 100; try++ { failed = []string{} for _, name := range namesToResolve { _, err := c.Get(). Prefix("proxy"). Resource("pods"). Namespace("default"). Name(pod.Name). Suffix("results", name). Do().Raw() if err != nil { failed = append(failed, name) fmt.Printf("Lookup using %s for %s failed: %vn", pod.Name, name, err) } } if len(failed) == 0 { break } fmt.Printf("lookups using %s failed for: %vn", pod.Name, failed) time.Sleep(10 * time.Second) } Expect(len(failed)).To(Equal(0)) // TODO: probe from the host, too. fmt.Printf("DNS probes using %s succeededn", pod.Name) }) It("should provide RW and RO services", func() { svc := api.ServiceList{} err := c.Get(). Namespace("default"). AbsPath("/api/v1beta1/proxy/services/kubernetes-ro/api/v1beta1/services"). Do(). Into(&svc) if err != nil { Fail(fmt.Sprintf("unexpected error listing services using ro service: %v", err)) } var foundRW, foundRO bool for i := range svc.Items { if svc.Items[i].Name == "kubernetes" { foundRW = true } if svc.Items[i].Name == "kubernetes-ro" { foundRO = true } } Expect(foundRW).To(Equal(true)) Expect(foundRO).To(Equal(true)) }) }) </ins>

Create a ginkgo suite to test services. Initially this could contain the following tests: TestClusterDNS TestKubernetesROService
Part of the ginko-ization of the e2e tests
Fixed by .


Question: ginkgo.Context("attempt to deploy past allocatable memory limits", func() { ginkgo.It("should fail deployments of pods once there isn't enough memory", func() { <del> overrideAllocatableMemoryTest(f, 4) </del> <ins> overrideAllocatableMemoryTest(f, framework.TestContext.CloudConfig.NumNodes) </ins> }) })

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: None, test is excluded Which test(s) are failing: Since when has it been failing: Unsure Testgrid link: Currently excluded Reason for failure: All 3 nodes have , of which is allocatable. The test was trying to schedule pods with 11009703Ki if I'm understanding the logs correctly. I think 3 pods should have started and 4th should have failed. Maybe the test needs to be updated to wait for 3/4 to start, not 4/4 but that's just a guess. Anything else we need to know: /sig windows
/assign


Question: type JobStatus struct { // The latest available observations of an object's current state. <ins> // When a job fails, one of the conditions will have type == "Failed". </ins> // +optional Conditions []JobCondition

The API docs for jobs are unclear as to what fields a caller would use to determine job success or failure in 1.11 (and we haven't updated them in a while) A casual reader would infer that completionTime is set when the job fails (but it isn't). Linking out to the docs is ok, but I shouldn't have to click through the links to understand how to use the Job object. The Job description and the job status fields should clearly indicate what fields are set in completion states on the job, because that's 100% of the point of docs. Taking a summary of the jobs-run-to-completion link and putting it in , , and job would improve the actual utility of our API doc. We should also do a pass on other objects and make the actual in server docs more useful.
as a consumer of the API, I had to read way too much to know how to use jobs correctly and how to write an API client that properly handled the conditions. Also, we should consider either setting completionTime on failures, or adding failedTime (the former may not be forwards compatible), because reading the conditions is too complex for a fundamental use case like this. This matches our decision w.r.t. conditions vs. fields - conditions are useful for extension and extra data, but status should have fields that allow an API consumer to easily understand state.
Also... our docs don't actually explain how to determine the job is complete via the API, so there's arguably no place this is captured.
I have the same problem right now, of correctly identifying a finished job, when it does not complete successfully and after reading the documentation and the reference and all the examples I could find, it's still not clear how I can check the conditions and make sure the job is finished when I do so: -what does each mean? can I rely on , is that set only when the job finishes, whether via or ? what other reasons are possible? It would definitely help to have specified on failure as well, and the differentiation to be made via the presence of .
/assign soltysh
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with .

Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
I have to say this issue is still very relevant. As of now, it is very hard to design something around Job Statuses. Anyway to move this forward?
Completely agree with previous comments! Even I found the way how to detect completed/failed jobs, still not sure am I covered all use cases. This part definitely needs clarifications/improvements/better documentation.
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
With regards to: If you actually follow this link and then search on that page for "conditions" you will get exactly zero hits. This is extremely unhelpful.
This documentation really, really needs work. (So does the API, for that matter.)
So what is the correct way to determine if a job is finished and whether it completed successfully?
When a Job completes (finished successfully), its will be set, and Job conditions will have one with . When a Job fails, Job conditions will have one with .
Hi, if nobody has been working on it, can I take this one? I have gone through the issue which is mentioned here and I would like to work on this. Can you please help me to get started as I am beginner to the project.
I totally agree with this- currently the only way to get the full picture of a Job is to read its conditions along with status fields which is error prone and a lot of extra work. I think a good start would be to add a new field as suggested by so as to not break compatibility for users already depending on
This sound reasonable. I'm still unclear when can contain more than one element. Also what does a or mean?
Still, open -- anyone actively working on this? I would like to attempt if not. :)
Are you still working on this?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.

If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
bump
This does not match what I just witnessed in my cluster… digging deeper to see if I need to file a bug but I have a Job with a single Pod that was executed. The Pod failed but the Job was marked as complete. This is the one (and only) pod that executed as my job: And this is the job that owns it (from kubectl describe): This is the job status object: on v1.16.7 but not finding any similar reports. just found this page as my only relevant search result.
I think it not only needs documentation, but also a change: It would be good to have 2 fields (probably needs better names), this way it's easy and clear when a job is finished and if it failed or not and when (someone can look at failed, succeeded and conditions for details): finishedtime: only set when the job finished (either completely failed or completely finished) finishtype: 'failed or completed'


Question: "hybrid_network.go", "memory_limits.go", "networking.go", <ins> "service.go", </ins> "volumes.go", ], importpath = "k8s.io/kubernetes/test/e2e/windows",

We need a test case that will verify that NodePort works on all nodes in the cluster for a Windows application. Here's an example manual test: Run with this yaml Once the pod is running, get the nodeport and make sure it's accessible on each node IP for that port. /good-first-issue /kind test /sig windows
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : We need a test case that will verify that NodePort works on all nodes in the cluster for a Windows application. /good-first-issue /kind test /sig windows </details
/assign adelina-t
Something is wierd here: What I did: Fresh environment, 2 linux nodes agents, 2 windows agents, deployed with aks-engine v.29.1. Using k8s 13.2. I used the exact spec you indicated: As you can see, it is assigned to the node: 2923k8s011. This is the first ever pod ( other than kube-system, that don't land on windows ) created on the windows nodes. CURL on the first windows node: Curl on the other cluster nodes works: kubeproxy logs on the first node only contain this: After I create a bunch of other pods, to make sure one gets scheduled on the windows node, the initial pod will be accessible via nod port from the first node as well. What I figure is that kubeproxy can't start because there is no HNS-Network called azure on the node until the first pod lands on the node ( c:kkubeletstart.ps1 deletes it ).
Cc
/milestone v1.14
per - Azure-CNI only creates the network when a pod is scheduled. Kube-proxy waits for the network to be created. Therefore if no pod has been scheduled, kube-proxy won't be active on that node. This won't affect Flannel or OVN deployments because they have a daemon already running that creates the network.
is going to take a look cc
NodePort service doesn't work on my mixed-OS clusters (1 k8s linux master + 1 k8s linux worker + 1 windows worker) as well.

I depends on as network solution for windows node, other linux node use calico for networking. all works well except accept access NodePort service on windows node, it seems the Port is not get stared on windows node, anyone who want to get more info for this issue, please let me know, thanks
this message show in the kube-proxy logs periodically.
please send us the debug logs.
thanks, below is my log, please take a look,
i looked at the logs i dont see any LB Policy programmed in HNS for the IP aspnetappsvc NodePort 10.0.15.35 <none80:/TCP 8m14s. Could you please confirm that this is the nodeport we are looking to debug? In the log files you collected please look @ and see if the policy you were trying to program is there or not. if its there and it doesnt work then its an HNS issue. if the policy is not there in that file then its a KubeProxy issue where the policy itself is not programmed in HNS. Then look at Kubeproxy to see why its failing to plumb the policy.
thanks for your suggestion, i will retry it and append the result here.
Below is my retest full steps and output: it seems the ip of aspnetapp and aspnetappsvc not in , but I don't see any error in kube-proxy logs except Any comments ? I need more debug on kube-proxy according to your suggestion
I just tried it with and it works for me.

I get 200. azureuser curl -s -o /dev/null -w "%{httpcode}" http://10.255.255.5: 200azureuser -s -o /dev/null -w "%{httpcode}" http://10.255.255.5: azureuser curl -s -o /dev/null -w "%{httpcode}" http://10.240.0.4: 200azureuser kubectl get nodes -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME 1116k8s000 Ready agent 5d15h v1.13.3 10.240.0.35 <noneWindows Server Datacenter 10.0..316 docker://18.9.0 1116k8s001 Ready agent 5d15h v1.13.3 10.240.0.4 <noneWindows Server Datacenter 10.0..316 docker://18.9.0 k8s-master--0 Ready master 5d16h v1.13.3 10.255.255.5 <noneUbuntu 16.04.5 LTS 4.15.0-1037-azure docker://3.0.4 azureuser curl -s -o /dev/null -w "%{httpcode}" http://10.240.0.35: 200azureuser get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE iis NodePort 10.0.179.188 <none80:/TCP 5d kubernetes ClusterIP 10.0.0.1 <none443/TCP 5d16h azureuser exit
/assign immuzz
GitHub didn't allow me to assign the following users: immuzz. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign immuzz Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
CC I'm able to create a test for it which is passing successfully. I will be doing a PR shortly
it looks like this needs a rebase.

There were a lot of merges yesterday and unfortunately this one was the victim.
/milestone clear
Yeah .. I just merged. Let's hope this time works.