Question: }) f.It("should fail if image volume is not existing", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: invalidImageRef}}}}, []v1.VolumeMount{{Name: volumeName, MountPath: volumePathPrefix}}, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to fail", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to fail", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodContainerToFail(ctx, f.ClientSet, f.Namespace.Name, podName, 0, images.ErrImagePullBackOff.Error(), time.Minute) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> }) f.It("should succeed if image volume is not existing but unused", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: invalidImageRef}}}}, nil, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> ginkgo.By(fmt.Sprintf("Verifying the volume mount is not used for path: %s", volumePathPrefix)) output := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/ls", filepath.Dir(volumePathPrefix)) gomega.Expect(output).NotTo(gomega.ContainSubstring(strings.TrimPrefix(volumePathPrefix, "/"))) }) <ins> f.It("should succeed with multiple pods and same image on the same node", func(ctx context.Context) { node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) framework.ExpectNoError(err, "Failed to get a ready schedulable node") baseName := "test-pod" anotherSELinuxLevel := "s0:c100,c200" for i := range 2 { podName := fmt.Sprintf("%s-%d", baseName, i) var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { if i == 0 { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } } else { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: anotherSELinuxLevel, } } ginkgo.By(fmt.Sprintf("Using SELinux on pod %q: %v", podName, selinuxOptions)) } createPod(ctx, podName, node.Name, []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: validImageRef}}}}, []v1.VolumeMount{{Name: volumeName, MountPath: volumePathPrefix}}, selinuxOptions, ) ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) ginkgo.By(fmt.Sprintf("Verifying the volume mount contents for path: %s", volumePathPrefix)) firstFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "dir", "file")) gomega.Expect(firstFileContents).To(gomega.Equal("1")) secondFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "file")) gomega.Expect(secondFileContents).To(gomega.Equal("2")) } podName := baseName + "-0" ginkgo.By(fmt.Sprintf("Rechecking the pod (%s/%s) after another pod is running as expected", f.Namespace.Name, podName)) err = e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) ginkgo.By(fmt.Sprintf("Verifying the volume mount contents for path: %s", volumePathPrefix)) firstFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "dir", "file")) gomega.Expect(firstFileContents).To(gomega.Equal("1")) secondFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "file")) gomega.Expect(secondFileContents).To(gomega.Equal("2")) }) </ins> })

Related to and SIG storage meeting notes: From me: It is related to image volume plugin and selinux. Is it a known limitation of all the volume plugins when using selinux? I have a different opinion. If two pods don't share the same mountpoint (hostpath), they can run the same node when the selinux is enabled. the current implementation of cri-o is sharing the same hostpath when two pods use the same image volume. More details about my idea: From Sharing each volume between namespaces is kind of natural. Sharing PVs not possible but sharing image quite common. From Although for most volume plugins it is true that, they can't be mounted with 2 different selinux context on same node, we want to understand why is this not possible for Imagevolume source. For a traditional storage persistent storage plugins - this limitation is underlying limitation of storage backend/Linux kernel etc. For ImageVolume source, it seems like a simple solution which unpacks image to two different hostpath would work and not run into this limitation. can you specify, why this limitation exists for ImageVolume plugins? From we choose to keep it simple and follow the lead of other volume types. supporting multiple selinux types would mean making more complicated the way we mount the image into the container AFAIR. But agreed we can chat about this elsewhere, in the meantime for the purposes of this PR: this is consistent with behavior of other volume plugins and is expected behavior. Let's discuss the SELinux behavior for image volumes in this issue. cc /sig node /sig storage
In my opinion, from an end-user perspective, the image volume may act like a read-only with a pre-populated content via image. Two pods from different namespaces can share the same image and run on the same node.
Pasted from According to , the containerd doesn't have the same limitation.
Given that OCI/Image Volume Source support is in alpha, I'd prefer that we consider discussing this as part of the original feature. We can discuss this here but I don't know if we need a separate KEP for this yet.

Yes, so far, in our conversations at the containerd community meetings we decided to start this feature out using a single pod scope for the pod scoped image volume host path because in containerd each runtime handler may have a different snapshotter (possibly on a different fs), or at least that's the plan. Additionally, the pods may have different os versions on windows indicating a different platform can be selected on the same node and this sort of versioning may extend beyond to other os providers. We could ask Kirtana to confirm the versioning issue and status of platform version selection over the platform index manifest, if the snapshotter reason isn't enough to at least start with per pod scope. If this is insufficient support we can certainly discuss. Cheers, Mike
No need a separate KEP. We can update the original KEP once this issue is resolved.
I'm working on a technical solution (just a PoC) in , not sure how feasible that is as time of writing, though.
yeah I agree I don't think we need a different feature for this, especially because it's mostly an implementation detail in the runtime. would you open this as an issue in kubernetes/kubernetes instead of the enhancements repo, or even better in the cri-o repo?
/transfer kubernetes
I moved this issue into kubernetes repo.
/triage accepted /priority important-soon
thanks!
Copid from kep: As merged, do we need to change the KEPs to reflect the new behavior?
no, nothing needs to be changed from a Kubernetes any more :+1:
/remove-sig storage


Question: "testing" "k8s.io/kubernetes/pkg/api" <ins> "k8s.io/kubernetes/pkg/api/errors" </ins> "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/runtime"

We sorta fixed this-- now it's a real status error, but we left the status code 500, so it still produces stack traces and potentially confuses clients. we should figure out the correct status code.
I think there is a lot more going on related to this in . Clients are re-listing after the error but not rewatching.
/sub
It turned out this was not causing job e2e flakes. Just sending 500s that were confusing. So lowering priority.
How about a NewBadRequest error?
that certainly sounds like an improvement.


Question: LIST and WATCH operations may specify label selectors to filter the sets of objects returned using a query parameter. Both requirements are permitted: <del> * _equality-based_ requirements: `?labelSelector=key1%3Dvalue1,key2%3Dvalue2` * _set-based_ requirements: `?labelSelector=key+in+%28value1%2Cvalue2%29%2Ckey2+notin+%28value3%29` </del> <ins> * _equality-based_ requirements: `?labelSelector=environment%3Dproduction,tier%3Dfrontend` * _set-based_ requirements: `?labelSelector=environment+in+%28production%2Cqa%29%2Ctier+in+%28frontend%29` </ins> <del> Kubernetes also currently supports two objects that use label selectors to keep track of their members, `service`s and `replicationcontroller`s: </del> <ins> Both label selector styles can be used to list or watch resources via a REST client. For example targetting `apiserver` with `kubectl` and using _equality-based_ one may write: </ins> <del> * `service`: A [service](services.md) is a configuration unit for the proxies that run on every worker node.  It is named and points to one or more pods. </del> <ins> ```shell $ kubectl get pods -l environment=production,tier=frontend ``` or using _set-based_ requirements: ``` $kubectl get pods -l 'environment in (production),tier in (frontend)' ``` As already mentioned _set-based_ requirements are more expressive.  For instance, they can implement the _OR_ operator: ```shell $kubectl get pods -l 'environment in (production, qa)' ``` or restricting negative matching via _exists_ operator: ```shell $kubectl get pods -l 'environment,environment notin (frontend)' ``` Kubernetes also supports two objects that use label selectors to keep track of their members, `service`s and `replicationcontroller`s: * `service`: A [service](services.md) is a configuration unit for the proxies that run on every worker node.  It is named and points to one or more pods. </ins> * `replicationcontroller`: A [replication controller](replication-controller.md) ensures that a specified number of pod "replicas" are running at any one time. <del> The set of pods that a `service` targets is defined with a label selector. Similarly, the population of pods that a `replicationcontroller` is monitoring is also defined with a label selector. For management convenience and consistency, `services` and `replicationcontrollers` may themselves have labels and would generally carry the labels their corresponding pods have in common. </del> <ins> The set of pods that a `service` targets is defined with a label selector. Similarly, the population of pods that a `replicationcontroller` is monitoring is also defined with a label selector. For management convenience and consistency, `services` and `replicationcontrollers` may themselves have labels and would generally carry the labels their corresponding pods have in common. Labels selectors for both objects are defined in `json` or `yaml` files using `map` and only _equality-based_ requirement selectors are supported: ```json "selector": { "name" : "redis", } ``` or ```yaml selector: name: redis ``` this selector (respectively in `json` or `yaml` format) is equivalent to `name=redis` or `name in (redis)`. At the moment in `json` or `yaml` format there is no way to represent inequalities or existence operators see [#341](https://github.com/kubernetes/kubernetes/issues/341). For example [redis-controller.yaml](../../examples/redis/redis-controller.yaml)`redis` `replicationcontroller` will monitor pods selected by the `name=redis` _equality-based_ requirement. Similarly for [redis-sentinel-service.yaml](../../examples/redis/redis-sentinel-service.yaml) `redis-sentinel` `service` will target pods selected by the `redis-sentinel=true` _equality-based_ requirement. </ins> Sets identified by labels could be overlapping (think Venn diagrams). For instance, a service might target all pods with `"tier": "frontend"` and  `"environment" : "prod"`.  Now say you have 10 replicated pods that make up this tier.  But you want to be able to 'canary' a new version of this component.  You could set up a `replicationcontroller` (with `replicas` set to 9) for the bulk of the replicas with labels `"tier" : "frontend"` and `"environment" : "prod"` and `"track" : "stable"` and another `replicationcontroller` (with `replicas` set to 1) for the canary with labels `"tier" : "frontend"` and  `"environment" : "prod"` and `"track" : "canary"`.  Now the service is covering both the canary and non-canary pods.  But you can mess with the `replicationcontrollers` separately to test things out, monitor the results, etc.

The suggests we support both "equality-based" and "set-based" label selectors, but actually we only support "==" selector right now. This is confusing. People are asking questions on IRC. We should update the doc in both github and
cc
We support the set-based selectors for filtering GET/watch requests. We support equality-based selectors, represented as maps, for API resources, such as ReplicationController, Service, and Pod nodeSelector.
Even if it's technically correct people are getting confused, so we need to fix the documentation.
Ref
sorry I see this only now. I'm ok to better explain the difference between set-based and equality based but I would need of some example of questions on IRC. May you have some of them? Otherwise we may clearly write that kubectl <command-l <selectorsupports set-based and equality based selector while API resources supports only equality-based.
I was thinking of making the difference between selectors in kubectl and in API clear.
cool, I can work on this if you've not already started.
By all means - please do:)
:)


Question: "description": "Selects the pods to which this NetworkPolicy object applies. The array of ingress rules is applied to any pods selected by this field. Multiple network policies can select the same set of pods. In this case, the ingress rules for each are combined additively. This field is NOT optional and follows standard label selector semantics. An empty podSelector matches all pods in this namespace." }, "policyTypes": { <del> "description": "List of rule types that the NetworkPolicy relates to. Valid options are Ingress, Egress, or Ingress,Egress. If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include "Egress" (since such a policy would not include an Egress section and would otherwise default to just [ "Ingress" ]). This field is beta-level in 1.8", </del> <ins> "description": "List of rule types that the NetworkPolicy relates to. Valid options are "Ingress", "Egress", or "Ingress,Egress". If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include "Egress" (since such a policy would not include an Egress section and would otherwise default to just [ "Ingress" ]). This field is beta-level in 1.8", </ins> "items": { "type": "string" },

Is this a BUG REPORT or FEATURE REQUEST?: bug report /kind bug What happened: I was reading the documentation for as defined What you expected to happen: To have clear documentation How to reproduce it (as minimally and precisely as possible): Read the following, as seen in the 1.12 docs: Anything else we need to know?: Nope Environment: Kubernetes version (use ): N/A Cloud provider or hardware configuration: N/A OS (e.g. from /etc/os-release): N/A Kernel (e.g. ): N/A Install tools: N/A Others: N/A
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Is it a case whereby only can be specified as , or only , or both (note the ) ? If that's the case, do you think the following would make it clearer?
It is confusing to have commas used for English grammar and array element delineation in the same sentence. What about simply this?
sounds like a great fix! Putting out a quick pr with this change now :)


Question: cloudController, } <del> makeSymlinksFlag := false </del> cmd := &cobra.Command{ Use:   "hyperkube", Short: "Request a new project", Run: func(cmd *cobra.Command, args []string) { <del> if len(args) != 0 || !makeSymlinksFlag { </del> <ins> if len(args) != 0 { </ins> cmd.Help() os.Exit(1) } <del> if err := makeSymlinks(os.Args[0], commandFns); err != nil { fmt.Fprintf(os.Stderr, "%vn", err.Error()) } </del> }, } <del> cmd.Flags().BoolVar(&makeSymlinksFlag, "make-symlinks", makeSymlinksFlag, "create a symlink for each server in current directory") cmd.Flags().MarkHidden("make-symlinks") // hide this flag from appearing in servers' usage output cmd.Flags().MarkDeprecated("make-symlinks", "This feature will be removed in a later release.") </del> for i := range commandFns { cmd.AddCommand(commandFns[i]())

Follow up for the deprecation we did in 1.14:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale /good-first-issue
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /remove-lifecycle stale /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig release
I am willing to take this, if it is available :)
go for it. All you need to do is remove and associated code -
I ended up getting hit with the removal of the --make-symlinks flag. I looked at the threads linked to the original PR, but I did not find the actual reason behind the removal. Do you recall was the reason behind removing that capability?
the original request was from


Question: import ( "context" <ins> "errors" </ins> "time" authentication "k8s.io/api/authentication/v1beta1"

What happened: I configured my API server with a Webhook token authentication server by way of the "authentication-token-webhook-config-file" command-line flag, and confirmed that the API server does pass bearer tokens through, issuing HTTP POST requests with a JSON-encoded body. When my Webhook server indicates authentication failure, it responds with HTTP status code 200 and a JSON-encoded in the body, this time with the object's "Error" field populated. Here's an example: When the API server receives this response, it refuses the original motivating request with an HTTP status code 401 response, but the API server writes the following into its log file: This use of this default error message is due to an omission in file , where , and . What you expected to happen: The API server should instead log a message that includes the value from . Per a brief discussion with , he agreed that the API server log message should include such an error message from the Webhook server, provided it's present and nonempty. How to reproduce it (as minimally and precisely as possible): Configure an API server to delegate to a Webhook token authentication server. Force that Webhook server to respond "successfully" with an HTTP 200 response, but with the "Authenticated" field set to false (or omitted) and with a particular error message in the "Error" field. Make an HTTP request of the API server with an HTTP "Authentication" header with a value starting with the string "Bearer " followed by at least one more character. Inspect the logs from the API server, looking for a message that begins with "Unable to authenticate the request due to an error." Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: AWS EC2 OS (e.g. from /etc/os-release):Kernel (e.g. ):Install tools: kubeadm_
/sig auth
/assign


Question: fieldSelector: fields.ParseSelectorOrDie("status.phase=Pending"), expectMatch:   false, }, <ins> { in: &api.Pod{ Status: api.PodStatus{PodIP: "1.2.3.4"}, }, fieldSelector: fields.ParseSelectorOrDie("status.podIP=1.2.3.4"), expectMatch:   true, }, { in: &api.Pod{ Status: api.PodStatus{PodIP: "1.2.3.4"}, }, fieldSelector: fields.ParseSelectorOrDie("status.podIP=4.3.2.1"), expectMatch:   false, }, </ins> } for _, testCase := range testCases { m := MatchPod(labels.Everything(), testCase.fieldSelector)

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: I was attempting to filter pods with a using client-go selecting by its . I tried multiple variations until I ended up with . When filtering by , list of pod(s) came back empty even if pod with the IP we are filtering by was running. Then I tried reproducing it by calling the Kubernetes API directly which had the same result, list of pods came back empty: What you expected to happen: When passing the IP of the previously retrieved running Pod, I expected to get that Pod returned in the result. How to reproduce it (as minimally and precisely as possible): create a simple , get its IP (For example using - query Kubernetes: Anything else we need to know?: If I leave out the pod IP, I get all the pods back. I also tried using different formats when doing the curl, not one of them return anything. Also it seems like field is . As a side note I retrieved the URL which I used from debugging which calls client-go made. Environment: Kubernetes version (use ):Cloud provider or hardware configuration**: minikube version: v0.20.0 OS (e.g. from /etc/os-release): minikube OS Kernel (e.g. ): minikube kernel: Install tools: minikube Others:
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig api-machinery
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Currently the for only support , , , and . I've already submitted a PR in . PTAL.
cc


Question: glog.Warningf("No API client: %v", err) } <ins> credentialprovider.SetPreferredDockercfgPath(*rootDirectory) </ins> kcfg := standalone.KubeletConfig{ Address:                 address, AllowPrivileged:         *allowPrivileged,

As per We should look for the .dockercfg in three places: ./.dockercfg, ~/.dockercfg, and $rootDir/.dockercfg
I'll tackle this.
Bunch of questions: we also add a flag that overrides the search mechanism suggested in this issue? Perhaps, ? to the user's homedir requires we use os/user [0] . The os/user package and cross-compiling don't play nice [1]. So, if we add to the search path, cross-compiled kubelet binaries will not actually be able to take advantage of it. Is this an issue? is the proper lookup order? I suggest: if provided, , , . walking through the lookups, do we stop at the first file that exists, or the first file that exists and is parsed as a valid ? [0] [1] /cc
I would absolutely love a --dockercfg option, and it would also be helpful if it logged which .dockercfg it ultimately used.
1) Seems like YAGNI to me 2) Is all of os/user broken? We will eventually want actual user support. I'm fine with dropping it for now. 3) I would say $rootDir/.dockercfg, ./.dockercfg (assuming we drop ~) 4) First successful parse sounds good to me. And can we NOT log errors if files don't exist :)
would it make sense to default to the kubelet config directory if provided rather than the working directory of the kubelet process? then no extra flag required.
What ever happened with this idea?
/cc
It never got high enough on my weekend TODO list :( I can keep telling myself that I need to do it, or we can find someone who has a bit more time than me.
I can probably get to this today.
Cool, thank you! , Mike Danese wrote:


Question: } } <del> func (plugin *kubenetNetworkPlugin) clearBridgeAddressesExcept(keep string) { </del> <ins> func (plugin *kubenetNetworkPlugin) clearBridgeAddressesExcept(keep *net.IPNet) { </ins> bridge, err := netlink.LinkByName(BridgeName) if err != nil { return

This is at least partially responsible for - some tests can't run when dns is down. Logs from just before the crash: It got killed because Kubelet decided to kill it (readiness probe failed): It started happening quite recently - I'm trying to locate a build which caused this. Repro: run "Kubelet should not restart containers across restart" test - it caused this behavior for me multiple times today. cc - I applied the P0/v1.3 label/milestone, as I think it should be a blocker. Feel free to decrease the priority if you disagree
gce-serial 1511 seems to be the first one with this problem.
Or 1510 - hard to tell, as it was a broken run. There's no PR around that looks too suspicious though:/
I locally reverted and it seems that DNS is no longer failing. Sadly I have to run to catch a train, but maybe someone can pick it up here ?
Taking a look now. The PR should be unrelated...I am checking to see if the Federation DNS changes went in yet. Add nethealth container to prepull manifest
I search the restart counts for jobs from 1510-latest and gce-serial Job 1540 is the first one to show a non-zero kubedns restart count. If there is a commit responsible, it would be between 1539 and 1540 (June 21 7 AM -11 AM) 1539 end of run 1540 end of run Latest
The kubedns:1.3 container hasn't been updated since (Sat, 04 Jun 2016 03:29:37.847 GMT). GET
cc The kubedns container code is from June 04th. Dawn suggested that its networking issues that are causing kubedns container to fail its readiness check. I will need to add logic to print kubedns container logs during e2e tests, I am trying to search for any logs that might help, otherwise I will have to debug the next run live.
As reported, a simple kubelet restart triggers this issue - it looks like 10.0.0.1 is unreachable from the kubedns container.

The kubedns container blocks indefinitely waiting for a connection to the kubernetes service at 10.0.0.1 before setting up the healthz handlers - the readiness probe will fail until kubedns successfully connects to the API server.
- you can try reverting and running it again - it worked for me for some reason (it might be by accident though)
I think the root cause is that cbr0 doesn't have an IP address assigned after kubelet restart - all the pods running behind cbr0 are isolated from the network. Not only kube-dns, but after my kubelet restart, the other addon pods are also stuck in a crash loop.
This sounds like a blocker even more now.
Looks like a regression. I am investigating.
Found the root cause. There is a subnet ip mismatch. vs . Hence kubenet thinks they are different and clears the address. Will submit a fix soon.
i found coredns status is CrashLoopBackOff and i check the logs : i got tips from so i restart my docker , i check the logs , there is no error in logs again


Question: } } <ins> // doEviction does the fake eviction and returns the status of eviction operation. func (nc *nodeController) doEviction(fakeNodeHandler *testutil.FakeNodeHandler) bool { var podEvicted bool zones := testutil.GetZones(fakeNodeHandler) for _, zone := range zones { nc.zonePodEvictor[zone].Try(func(value TimedValue) (bool, time.Duration) { uid, _ := value.UID.(string) deletePods(fakeNodeHandler, nc.recorder, value.Value, uid, nc.daemonSetStore) return true, 0 }) } for _, action := range fakeNodeHandler.Actions() { if action.GetVerb() == "delete" && action.GetResource().Resource == "pods" { podEvicted = true return podEvicted } } return podEvicted } </ins> // TestCloudProviderNoRateLimit tests that monitorNodes() immediately deletes // pods and the node when kubelet has not reported, and the cloudprovider says // the node is gone.

Running latest from master (currently commit ), I'm seeing TestMonitorNodeStatusEvictPodsWithDisruption is flakey: test -c ./pkg/controller/node true; do if ! ; then break; fi; done for failure I've narrowed it down , which is supposed to process all the items (nodes) in the queue. This queue has a capacity of 1, and at least in the test code, a refill rate of QPS. If the call to fails , it will stop processing wherever it happened to be in the queue. I've some extra logging locally, and when it fails, I see it processing nodes 1 and 2 before TryAccept fails. The issue is that the pod it's trying to evict is on node0, and it doesn't process node0 before TryAccept returns false. cc
I'm surprised we haven't hit this in jenkins yet
and removed the "leak" option referred to in the godoc, but it seems like it kept the leak behavior, not the retry behavior when the rate limiter is hit
Still seeing this issue
I didn't understand how this is related to sig/scheduling. Am I missing anything here?
guessing the flake is related to this line: anything on the order of milliseconds is going to be unreliable in testenvs that sometimes peg the CPU with parallel tests or it's related to the single item burst and the "abort when TryAccept fails" behavior
I can extend this sleep to something bigger, but I don't have cycles to debug it properly:/
any thoughts on that?
Generally we retry when TryAccept fails, but not in tests. So either we need to rewrite the test, or lengthen the sleep time. It's not perfect, but there's no way for "wait until rate limiter is reloaded".
I was working on this issue and I tried the change that you suggested and I notice that it fails with this change too() and I think rewriting the test with retry seems to be the option as I tried increasing 5000 ms and still it failed. I am new to the codebase, if there is piece of code where retry is available, I can check on that.
- what you wrote is concerning. I wish I had time to debug it:/ Frankly - those tests are old. Like really, really old. So they start to deserve a rewrite.

On the flip side - we'll need to rewrite them when making taint-based evictions default, so we'll need to allocate time for that anyway. If lengthening sleep time reduce the flakyness I vote for doing that, and just wait for upcoming rewrite.
having a delay in the loop before caused it to run for close 4 hours without terminating on 8GB,4 core machine. Is this something normal? I know, is not happy with this change but since we are rewriting this test anyways, this could be a work around that removes the flakyness. Let me know your thoughts. If this does work, can we remove the flake label and keep this issues as this is not a release blocker for 1.6?
- I don't understand which delay in which loop you mean. You a delay inside the "Try" call? What do you mean by 'run for close 4 hours without terminating'? That the test didn't fail? If so, then I'm all for such solution for stopping the bleeding.
, Yes I introduced a sleep in loop for zone, like Yes the test didn't fail but it didn't complete either even after 4 hours, so I am wondering if this causes timeout in testing infra.
It's strange indeed... Do you have any logs for the test by any chance? Adding this delay shouldn't be noticeable, as it just inserts some delay here: Sleeping under the lock is usually not the best idea, but 50 microseconds in test shouldn't make a lot of difference...
Unfortunately no, I can try reproducing this today. Probably by afternoon EST, I can have something, The way I am doing as of now is the way suggested by in the comments above. I can redirect that to a file and see. Let me know, if you are ok with approach.
I'm OK with anything that produces logs:) I generally do:
- out of curiosity. How often does this flake?
If I run my loop, it takes about 8-10 seconds to flake. , Marek Grabowski <:
I think I made a mistake. I did not take that while true loop out because of which it ran in infinite loop. I am sorry about that.

It means that with that delay introduced that I mentioned earlier it never failed even after running for 4 hours, which means we got rid of atleast test flakyness. if you can confirm if it is working, I will raise a PR which introduces this delay.
giving the ratelimiter time to refill inside the loop is consistent with the theory that is causing problems. If the rate limiter runs out of tokens in Try(), it returns with a comment, but the unit test does not try again later. I'm having a hard time following the code to know if non-test invocations have the same problem or if this is just a unit test issue.
The sleep addition seems to have helped. rgudimet <:
It's test only problem: Runs in , so it'll retry in the next round. I guess the problem here was actually introduces later than what suggests - I don't think that the removal of leaky behavior was problematic, but introducing a test that evicts more than one Node was. This means that if processing queue is fast enough, then rate limiter fails to reload, so adding the sleep there is actually a correct solution (as long as we don't have good rate limiter mock/testable rate limiter).
Thanks for confirmation. I will raise a PR for that now. As far as I understood, the issue is with unit test. If rate limiter fails to reload for a particular node, it cannot retry(unless the we separate out the logic which tells if the rate limiter has reloaded or not. - if we have that logic we can ask Try to run again by returning a false just like ).
Still an issue: https://k8s-
but not a blocker (the test is at fault)
Thanks, I will check this out after 1.6 release hopefully, with rewrites..
Has there been any recent incidents of this issue? It's marked for 1.7.
Closing - shows near nothing.
- this flakes out sporadically, I was able to reproduce this when debugging on my system, if I run it for long time.
https://k8s-


Question: Corefile: | .:53 { errors <del> log stdout </del> <ins> log </ins> health <del> kubernetes __PILLAR__DNS__DOMAIN__ __PILLAR__CLUSTER_CIDR__ </del> <ins> kubernetes __PILLAR__DNS__DOMAIN__ __PILLAR__CLUSTER_CIDR__ { pods insecure } </ins> prometheus proxy . /etc/resolv.conf cache 30

Is this a BUG REPORT or FEATURE REQUEST?: BUG What happened: When CoreDNS is deployed by default using feature-gates of kubeadm, it does not log anything when processing a DNS query. That is due to change of log option of CoreDNS in latest version. that version was bumped up in kubeadm, but the manifest was not adapted to the new log option. initial change of 'log stdout' in CoreDNS : version of CoreDNS 1.0.0 bumped into Kubeadm : NOTE: I guess we should also changed manifests for kube-up : see The change in manifest expected is to remove the option "stdout" in the line "log ...". As it is now interpreted as a scope of domain to log. Invalid manifest: Environment: Kubernetes version (use ): 1.9 and MAIN Cloud provider or hardware configuration: any Install tools: kubeadm, kube-up the configmap for coredns kubectl get configmaps -n kube-system coredns -o yaml kubectl replace -f - the coredns deployment kubectl scale deployment --replicas=0 -n kube-system coredns kubectl scale deployment --replicas=1 -n kube-system coredns
/cc /sig network
NOTE: only log is impacted. CoreDNS working as expected.
/cc
I have a question : the initial manifest is done for the image 0.9.10 of CoreDNS. Bumping the version of CoreDNS was done by changing the version in file (see ) which means that the image's version is STILL 0.9.10 in the manifests ... and the corefile config is correct against the version of the manifest. QUESTION: when modifying the , is there a way to upgrade the manifest by program ? I guess at the time the new version number is applied on the manifest ...
thanks for reporting this! I send PR to update the Corefile. I don't think of a way so far. I guess we can't unless there is a way to get the Corefile from some place of upstream (CoreDNS).


Question: } const yamlSeparator = "n---" <del> const separator = "---n" </del> <ins> const separator = "---" </ins> // splitYAMLDocument is a bufio.SplitFunc for splitting YAML streams into individual documents. func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {

Kubernetes version (use ): Environment: Cloud provider or hardware configuration: Any OS (e.g. from /etc/os-release): Any Kernel (e.g. ): Any Install tools: Any What happened: If I use a template with the seperator, and if the line with the separator has trailing whitespace characters, then kubectl considers that to be the EOF instead of reading further. e.g. template What you expected to happen: It should not stop reading if there are trailing whitespace characters, it should just trim the separator line and ignore the whitespaces after the . This is how it worked until v1.3.0 kubectl version. How to reproduce it (as minimally and precisely as possible): Just use the template from above and you'll notice that it won't create the Service, but it'll create the Replication controller. Anything else do we need to know: This is on kubectl version 1.4.0
kubectl regression. cc
This bug is general (can be found on get, apply, delete, ...). May be somewhere in resource builder.
Might be , Janet Kuo wrote:
Confirmed - is the cause. would you take a look?
We may need to cherrypick the fix
Yaml rules are separator must be at beginning of line, three dashes, and new line. Did we change how strict we were in that PR? On Sep 27, 2016, at 7:46 PM, Janet Kuo wrote: We may need to cherrypick the fix — You are receiving this because you are on a team that was mentioned. Reply to this email directly, view it on GitHub , or mute the thread .
Yes, and the test was changed too (originally tests allow with trailing spaces)
According to spec (if my spec-alese is correct) newline + three dashes is the only requirement to terminate a document. Whitespace or prefixes are allowed afterwards, but for it to be valid YAML those must be valid at the beginning of a doc as well (as far as I can tell). So we should split and remove on any line starting with a newline and three dashes - but not consume the remaining whitespace (I believe) , Janet Kuo wrote:
We should probably exit non-0 and have an error message if we see something in the file that we don't expect. E.g.

is neither a separator nor parseable yaml.
We should add a test that uses a YAML document prefix rule and verify that it correctly passes / fails when YAML goes to parse. From reading the spec it appears that anything that is invalid when n--- is present is also invalid in a single document context, and vice versa. , Phillip Wittrock wrote:
I have rechecked the code, and I have located the root cause, since I am outing for a training these days, I will fix this right now after the training on Friday.
I am quite sorry about the regression. :(
should we allow trailing whitespace characters after , if so we can trim the whitespaces when we read line by line?
is it possible to include this fix to one of the 1.4 releases? We'd really like our users to have of k8s alpha features (Scheduled jobs is one of them) that can be configured only using 1.4 client; w/o them potentially hitting this regression.
I will double check this regression, and open a cherry-pick PR.
I am thinking create a cherry-pick PR for this to include it in 1.4, wdyt?
I would like to see this in 1.4 since as you note it may break scripts. On Oct 19, 2016, at 8:19 PM, TonyAdo wrote: I am thinking create a cherry-pick PR for this to include it in 1.4, wdyt? — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub , or mute the thread .
Do we have a test case now? :) On Oct 19, 2016, at 8:19 PM, TonyAdo wrote: I am thinking create a cherry-pick PR for this to include it in 1.4, wdyt? — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub , or mute the thread .
we already test this, you can see


Question: v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" <ins> "k8s.io/apimachinery/pkg/types" </ins> "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait"

According to this APIsnoop query, there are still some remaining Endpoints endpoints which are untested. Create a Service in a namespace List all Endpoints in all Namespaces find the created Service(1) Replace/update the Namespaced Endpoint for the Service(1) Patch the Namespaced Endpoint for the Service(1) Get the Namespaced Endpoint for the Service(1) Delete Namespaced Endpoint(1) via a Collection with a LabelSelector Discover useragents: If a test with these calls gets merged, Conformance coverage will go up by 4 points /sig testing /sig architecture /area conformance
When you create a selector, race conditions can occur between controllers and our test code. We have services that test endpoint code, we could do these as explicit tests that are NOT handled by the endpoints controller. Let's try this with are managed by the endpoints controller w/ a selector. Endpoints normally go along with services: we want to create a service sans selector. This keeps it out of the endpoints controller, so we can test directly. Agreed for backlog. Double check that we don't already have test for that, whether it's being done by the endpoints controller that doesn't hit the API. Look at ALL the user agents that hit the endpoint.
/reopen /open Needs promotion!
Reopened this issue. <detailsIn response to : /reopen /open Needs promotion! Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/retitle Promote CoreV1Endpoints test - +4 coverage
This doesn't have an APISnoop PR because (the exact process wasn't in place properly when the original mock test ticket was made)
/retitle Write CoreV1Endpoints test+promote - +4 coverage
please close this Issue in favour of and place in In Progress Issues on org dashboard
/close Close this Issue in favour of and place in In Progress Issues on org dashboard
Closing this issue. <detailsIn response to : /close Close this Issue in favour of and place in In Progress Issues on org dashboard Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/close Close this Issue in favour of and place in In Progress Issues on org dashboard


Question: // // As a side effect, the cloud provider may block invalid or non-existent volumes. func NewPersistentVolumeLabel() *persistentVolumeLabel { <ins> // DEPRECATED: cloud-controller-manager will now start NewPersistentVolumeLabelController // which does exactly what this admission controller used to do. So once GCE and AWS can // run externally, we can remove this admission controller. glog.Warning("PersistentVolumeLabel admission controller is deprecated. " + "Please remove this controller from your configuration files and scripts.") </ins> return &persistentVolumeLabel{ Handler: admission.NewHandler(admission.Create), }

In , we a Persistent Volume Controller with existing code that runs in cloud-controller-manager In , we cleaned up dependencies for AWS/GCE for the new controller Next steps, [x] Deprecate the PersistentVolumeLabel admission controller [ ] Remove the PersistentVolumeLabel admission controller once GCE and AWS can run as external cloud controller
/sig cluster-lifecycle
cc
/reopen
Changed the timeline for this to v1.10, sounds good as an initial plan?
/area platform/gce /sig gcp
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/unassign
adding kind/cleanup to this issue so that it can stay in 1.10. Please update the kind if that is incorrect. kind cleanup
/remove-lifecycle stale /lifecycle frozen is working on running the cloud controller in gce, but it's not there yet, so we shouldn't remove the PV label admission controller yet.
and should we drop the v1.10 milestone then for this issue?
reminder: this feature will be automatically dropped from 1.10 at Code Freeze tonight.
[MILESTONENOTIFIER] Milestone Removed From Issue Important: This issue was missing labels required for the v1.10 milestone for more than 3 days: kind: Must specify exactly one of , or . <details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
PVL admission controller was set for deprecation in 1.8, in 1.11 we'll be turning it off by default, users can still enable it if they need it which seems to be the use for GCE.
^ I'm not 100% aware of the status of PVLs in AWS/GCE, this is just the deprecation plan we had in place a while back. If there are any known issues with doing this please let me know
As long as the admission controller is not removed from cluster/gce/config-, then GCE should continue to work.

/remove-sig cluster-lifecycle
/assign /triage accepted
is it time to try this again?
I don't think we're ready. There's no replacement for it in the cloud controllers. The initial attempt at replacement using initializers was abandoned. I think we should consider a replacement using an admission webhook. cc
Could I suggest we make this the primary topic in an upcoming meeting? Maybe April 22nd Cloud Provider Extraction WG meeting? Assuming and can make it?
Yes that sounds good.
What do we think the way forward looks like?
This issue is labeled with but has not been updated in over 90 days, and should be re-triaged. Important-soon issues must be staffed and worked on either currently, or very soon, ideally in time for the next release. You can: Confirm that this issue is still relevant with (org members only) Deprioritize it with or Close this issue with For more details on the triage process, see /remove-triage accepted
/remove-sig api-machinery Since sig cloud provider has taken care of this one. :)
do you know what the latest status is on this?
i saw this go by the other day, please see (search for )
xref:
!
For the record, replacement webhook:
Is this done now?
/close yes! deleted in
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: By("Saturating pet set " + ps.Name) pst.saturate(ps) <ins> By("Verifying petset mounted data directory is usable") ExpectNoError(pst.checkMount(ps, "/data")) </ins> cmd := "echo $(hostname) > /data/hostname; sync;" By("Running " + cmd + " in all pets") <del> pst.execInPets(ps, cmd) </del> <ins> ExpectNoError(pst.execInPets(ps, cmd)) </ins> By("Restarting pet set " + ps.Name) pst.restart(ps) pst.saturate(ps) <ins> By("Verifying petset mounted data directory is usable") ExpectNoError(pst.checkMount(ps, "/data")) </ins> cmd = "if [ "$(cat /data/hostname)" = "$(hostname)" ]; then exit 0; else exit 1; fi" By("Running " + cmd + " in all pets") <del> pst.execInPets(ps, cmd) </del> <ins> ExpectNoError(pst.execInPets(ps, cmd)) </ins> }) It("should handle healthy pet restarts during scale [Feature:PetSet]", func() {

Eg: Suspect it's because of:
If this hasn't been seen in 3 months, okay to close?
Yes we fixed it with


Question: } func toInternalPodOrError(obj runtime.Object) (*api.Pod, error) { <del> pod := &api.Pod{} </del> switch t := obj.(type) { case *v1.Pod: <del> if err := v1.Convert_v1_Pod_To_api_Pod(t, pod, nil); err != nil { </del> <ins> converted, err := api.Scheme.ConvertToVersion(obj, api.SchemeGroupVersion) if err != nil { </ins> return nil, err } <ins> if pod, ok := converted.(*api.Pod); ok { return pod, nil } else { return nil, fmt.Errorf("expect *api.Pod, got %v", converted) } </ins> case *api.Pod: <del> pod = t </del> <ins> return t, nil </ins> default: return nil, fmt.Errorf("expect *api.Pod or *v1.Pod, got %v", t) } <del> return pod, nil </del> } // podMatchesScopeFunc is a function that knows how to evaluate if a pod matches a scope

The pod evaluator used by the resource quota controller made direct calls to an unsafe pod conversion function which mutates the pod argument. With multiple resource quota controller workers, concurrent processing of the same pod from a shared informer can result in a panic when the conversion code attempts to write to a map field in the pod. Observed in this and discovered through analyzing core dumps from the controller manager. The bug is only present in <=1.7: the conversion code in 1.8 no longer mutates the input. In my branch, I a small unit test to which reliably reproduces the panic: /kind bug
Fixed by


Question: return nil } <ins> type patcher struct { encoder runtime.Encoder decoder runtime.Decoder mapping *meta.RESTMapping helper  *resource.Helper backOff clockwork.Clock } func NewPatcher(encoder runtime.Encoder, decoder runtime.Decoder, mapping *meta.RESTMapping, helper *resource.Helper) *patcher { return &patcher{ encoder: encoder, decoder: decoder, mapping: mapping, helper:  helper, backOff: clockwork.NewRealClock(), } } func (p *patcher) patchSimple(obj runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) { // Serialize the current configuration of the object from the server. current, err := runtime.Encode(p.encoder, obj) if err != nil { return nil, cmdutil.AddSourceToErr(fmt.Sprintf("serializing current configuration from:n%vnfor:", obj), source, err) } // Retrieve the original configuration of the object from the annotation. original, err := kubectl.GetOriginalConfiguration(p.mapping, obj) if err != nil { return nil, cmdutil.AddSourceToErr(fmt.Sprintf("retrieving original configuration from:n%vnfor:", obj), source, err) } // Create the versioned struct from the original from the server for // strategic patch. // TODO: Move all structs in apply to use raw data. Can be done once // builder has a RawResult method which delivers raw data instead of // internal objects. versionedObject, _, err := p.decoder.Decode(current, nil, nil) if err != nil { return nil, cmdutil.AddSourceToErr(fmt.Sprintf("converting encoded server-side object back to versioned struct:n%vnfor:", obj), source, err) } // Compute a three way strategic merge patch to send to server. patch, err := strategicpatch.CreateThreeWayMergePatch(original, modified, current, versionedObject, true) if err != nil { format := "creating patch with:noriginal:n%snmodified:n%sncurrent:n%snfor:" return nil, cmdutil.AddSourceToErr(fmt.Sprintf(format, original, modified, current), source, err) } _, err = p.helper.Patch(namespace, name, api.StrategicMergePatchType, patch) return patch, err } func (p *patcher) patch(current runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) { var getErr error patchBytes, err := p.patchSimple(current, modified, source, namespace, name) for i := 1; i <= maxPatchRetry && errors.IsConflict(err); i++ { if i > triesBeforeBackOff { p.backOff.Sleep(backOffPeriod) } current, getErr = p.helper.Get(namespace, name, false) if getErr != nil { return nil, getErr } patchBytes, err = p.patchSimple(current, modified, source, namespace, name) } return patchBytes, err } </ins>

From an e2e test failure: The command should retry resource version mismatches like scale and rollingupdate. cc
/cc fyi.
From perspective , Isn't the failure intentional (i.e. optimistic mutate) ? or... maybe I'm (probably) missing something. I would think its the user's job to retry... ?
The desired behavior that we settled on previously is to detect version mismatches at the client and error. User can force replacement using --force. See , and .
What is the mismatch here? If it is just resourceVersion, kubectl should retry. It should only bail on a meaningful conflict. We're working on eliminating flaky tests (and flaky parts of the system). Also, that log message isn't very useful.
per discussion on , we can't determine the correct behavior when the resource version changes without more information, so best to catch version mismatch at the client and bail, and let the user replace the object with --force if they want. We can also make the error message less verbose.
I don't see any mention of resourceVersion in . It shouldn't be serialized in the annotation, because the user shouldn't specify it in their config. The resourceVersion-based precondition is intended to guard against concurrent read-modify-write operations. kubectl apply should be able retry the entire diff&patch in the case of this type of conflict. That said, it's not obvious to me from the log message what the actual problem was in this case.
I'm sorry, my mistake. You're right that resourceVersion is not in the annotation, and that we should retry on resourceVersion failure. I was confusing this with the api version change issue discussed in . This is actually a much simpler issue, in the sense that it's an optimistic lock conflict. If we encounter a lock conflict, then we should fetch a fresh copy of the object, and try again to apply the new configuration supplied by the user. If we don't get a merge conflict when we resubmit, then all's well. If we get a merge conflict because the object has changed out from under the user, then we report it in those terms, and the user can decide what to do (e.g., replace using --force). Does this fit your expectation of how it should be handled?
This is causing test flake:
yes, your understanding is correct.

Do you have time to fix this? We're trying to get all our tests fixed. Additionally this sounds like a legit system bug, not just a flaky test problem.
Maybe. Swamped right now, but can fit it in given enough notice. When is the fix needed?
asap. We'll find someone else.
This should just be a generic GET/Update retry loop, eg:
Go for it. Jack Greenfield | Google , Prashanth B wrote:
Chao, do you have cycles to take a look?
I thought Patch is retried in the server. I'll investigate.
Patch is retried at server-side five times before an error is returned: , so I expect this issue to occur very rarely. In fact, it has passed last ~400 kubernetes-gce-e2e tests. If we want to make this test less flaky, I'd recommend increase the number of the retries at the server-side. What do you guys think?
It'll fail if an actual conflicting change was made, though. Do we know what the nature of the conflict was? Maybe first we need to add some log messages to figure out what the conflict was next time it occurs. , Chao Xu wrote:
This happened last week:
you are right. I thought an actual conflicting change would result in a different error message but it's not. I'll add the logs.
In I a log entry if there is a meaningful conflict. That said, I don't think we have a meaningful conflict in this particular e2e test. After taking a closer look at the implementation of , I think the submitted is precise, so we shouldn't retry at the client side if server reports a meaningful conflict. [edit] And if the conflict is not meaningful, we should just rely on the server-side patch handler to retry.
does the milestone on this need to be changed to v1.2?
How often is the test flaking?
Last time I checked, I'll check it again.
Last time I checked the results was two days ago, in the past 48 hours, the test has consecutively passed another 124 builds ( to ).
I only see 3 reports of failures since October, so I'm going to say, no, it doesn't need to be fixed in 1.2.
any progress on this now?

I returned back to the kubectl apply issues, and hope I can fix all apply related issues before 1.3.0 release.
I'll try to check the flake rate again. IMO if the flake rate is non-negligible, we should increase the retry limits of Patch, or more aggressively, just remove the limit.
Infinite retry still scares me, mostly because we may have failure modes that could race for a very long time. , Chao Xu wrote:
I don't think anyone has worked on this. Though the original failing test/logs weren't recorded, it appears to be a failure of "should apply a new configuration to an existing RC": There is a retry loop in the Patch handler in apiserver, but if resourceVersion is specified, the retry loop will always fail in the case of a conflict. (We should detect that and bail out, but I don't think we do.) In the case of such a conflict, apply should diff again, create another patch, and try again. I agree with that it shouldn't retry forever. We should use the same retry limits we use for other commands.
we have an infinite loop in GuaranteedUpdate (), maybe we should apply a retry limit there as well?
Agree. Just note that the test flake doesn't specify resourceVersion.
Since this is causing flakey test failures, it is something we should focus on. Is this something I should pick up?
FYI: has a PR out for this under review
Making this a P3 (no release block) since I looks like we haven't seen many instances since Feb 2016


Question: echo "Using VPC $VPC_ID" <ins> create-dhcp-option-set </ins> if [[ -z "${SUBNET_ID:-}" ]]; then SUBNET_ID=$($AWS_CMD describe-subnets --filters Name=tag:KubernetesCluster,Values=${CLUSTER_ID} | get_subnet_id $VPC_ID $ZONE) fi if [[ -z "$SUBNET_ID" ]]; then echo "Creating subnet." SUBNET_ID=$($AWS_CMD create-subnet --cidr-block $INTERNAL_IP_BASE.0/24 --vpc-id $VPC_ID --availability-zone ${ZONE} | json_val '["Subnet"]["SubnetId"]')

I was successfully able to get a cluster to start up in us-west-2, the default region for kube-up, but for some reason I'm not successfully able to start up a cluster in us-east-1. I'm not seeing anything in the logs that seems to be a huge problem, so I'm not really sure how to resolve this. Kubelet log here: I'm not really sure what else to look at, I'm happy to do more investigation, but not really sure where to look. Most others log files are empty with the exception of kube- which is failing to start addons due to the unstarted api server.
cc/ Could you please help us with this?
Sure ... this sounds like a name problem - as I recall the name is supposed to be ip-172-20-0- (or similar), not just ip-172-20-0-9. I'll try in us-east-1. What revision of the software is this ?
I built from the latest master, but I have seen the same issue on recent releases.
Testing now... I wonder if this is related to and
if you entries in etcd were the fqdn of the machines then you would have been bit by the same issue as us.
I was skeptical about Running that curl returns "ip-172-20-0-9" and not "ip-172-20-0-9.ec2.internal" as we might expect?
Nvm, I tried just appending ".ec2.internal" to no avail.
I just tried it, and I was able to bring up a cluster in us-east with the latest code. I checked on my kubelet on the master, and did see your error: So I think that error is a red herring. Of course we should fix it, though I don't know if the problem is that it is missing the .ec2.internal suffix or whether the problem is that we're not registering the master as a node at all. Do you see the API server in (or ) on the master. I suspect it isn't running. Or maybe it isn't configured on your kubelet - what does show?
Nothing seems to be running:
That is odd... here is my output: Note no Note that I have arguments to kubelet (a lot of them!)

The lack of arguments matches with the " No api server defined - no node status update will be sent." message in your kubelet log (which I don't have) On the "unable to resolve host ip-172-20-0-9" error, I can . It isn't in hostname, but rather I wonder if you can't ; a possible reason would be that your VPC doesn't have name resolution enabled. Did the script create it for you? Are there any hints in ? Here is mine:
VPC was created by the script and dns support and dns hostnames are both turned on. Interestingly, I'm unable to ping ip-172-20-0-9 and my is missing the search line: looks pretty close, but noticed some extra lines re: Jinja:
I think it might be the DHCP Options Set. I have another VPC in east that already had a DHCP Option Set and it appears that the kubernetes cluster is using that same set. When I looked in west I found that the set was configured as such: Whereas in east, my DHCP option set just looked like this: Perhaps we should be explicitly creating a new DHCP option set to avoid conflicts with existing VPCs? I'd be happy to create a PR if this turns out to be the issue and if you think the above is a good solution.
That is almost certainly it - nice find! I don't fully understand how this actually happened (I guess a new VPC defaults to whatever DHCP option set already exists, although that isn't what the docs say), but I would welcome a patch that created a working DHCP option set when creating a new VPC, and/or warned if a VPC had a DHCP option set which didn't include a domain-name.
Yep, cluster in us-east-1 is up and running! Thanks for all of the troubleshooting help! Another option might be to set the DHCP option set to default, which should be configured correctly. It doesn't seem like you can modify the default, so it might be a safe choice. Plus, we don't have to set up the option set, we can just assign it through the . I'll do some experimentation and make a PR with what seems to be the most sane.

The documentation for option sets seems contradictory, in one place it says it will create a new option set and in another it says it will use the default. It seems it does a little bit of both. :)
should resolve once it merges.
Unfortunately this issue is still exist for me. Even tough a DHCP option is created for the Kubernetes VPC I've get an endless loop.
I figured it out, The issue was that as part of the dhcp set creation it didn't set a domain-name to be "ec2.interna" as described here - and only set the domain-server to AWS defaults. Once I changed that dhcp set it worked perfectly. I can see you a function for that issue in , it might not merged to the master? Thanks


Question: return err } <del> // Invalidate the cache since we would update it. if err = ss.deleteCacheForNode(vmName); err != nil { return err } </del> <ins> // Invalidate the cache since right after update defer ss.deleteCacheForNode(vmName) </ins> // Update vmssVM with backoff. ctx, cancel := getContextWithCancel()

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: vmss dirty cache issue, caused by What you expected to happen: clean vmss cache should happen after disk attach/detach operation, now it's before those operations, which would lead to dirty cache. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: /kind bug /assign /priority important-soon /sig cloud-provider /area provider/azure


Question: namespace, name, err := namer.Name(req) if err != nil { <del> notFound(w, req.Request) </del> <ins> errorJSON(err, codec, w) </ins> return } ctx := ctxFn(req)

e.g. REST.Update should never return an arbitrary error value. Maybe we should change the return code to something stricter than generic error.
/cc
FYI: is for remaining cases where the API doesn't return JSON/Status errors.
Re-opening this until we
I think etcd errors require more abstraction, also:
There are some admission control plugins that did not always follow that practice which I will resolve as part of
Are there any known instances of this remaining?
None that I know of.


Question: cd "${KUBE_ROOT}/third_party" if [[ $(uname) == "Darwin" ]]; then download_file="etcd-v${ETCD_VERSION}-darwin-amd64.zip" <del> curl -fsSLO --retry 3 https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/"${download_file}" </del> <ins> curl -fsSLO --retry 3 --keepalive-time 2 https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/"${download_file}" </ins> unzip -o "${download_file}" ln -fns "etcd-v${ETCD_VERSION}-darwin-amd64" etcd rm "${download_file}" else <del> curl -fsSL --retry 3 https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz | tar xzf - </del> <ins> curl -fsSL --retry 3 --keepalive-time 2 https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz | tar xzf - </ins> ln -fns "etcd-v${ETCD_VERSION}-linux-amd64" etcd fi kube::log::info "etcd v${ETCD_VERSION} installed. To use:"

http://pr- Seems like a flake? Could probably retry this, but not sure whether it's worth it.
we most definitely want to retry! Will you send a PR?
I just hit this in PR
We should close this, should be fixed by:
that didn't actually solve the problem. Per the docs, only works on "transient" errors, which the manpage defines as We might want to try passing , which some googling suggests might help. The default is 60s, which may be too long.
Please reactive this if you see it after June 15


Question: mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(test.existingVM, nil) cloud.VirtualMachinesClient = mockVMClient mockNICClient := mockinterfaceclient.NewMockInterface(ctrl) <del> mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil) </del> <ins> test.existingNIC.VirtualMachine = &network.SubResource{ ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/k8s-agentpool1-00000000-1"), } mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil).Times(2) </ins> mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) cloud.InterfacesClient = mockNICClient

What happened: introduced new dependency on Availability Sets which is too restrictive for some existing installations: What you expected to happen: Azure cloud provider to successfully manage LBs, like it had been doing before. Instead, it died [with][1]: How to reproduce it (as minimally and precisely as possible): Fails every time in OpenShift CI. Can you point us to documentation around the infrastructure options that the Azure cloud provider expects for LB management? https://kubernetes- doesn't say anything about NIC names, and it doesn't have specifics around what things should look like without Availability Sets, or whether "without Availability Sets" is not supported (when previous iterations of the cloud provider had no problem with our lack of Availability Sets). Documenting cloud-provider assumptions about host hardware would have also helped avoid ; CC /sig cloud-provider /area provider/azure [1]:
/triage accepted
/assign
Notes on OpenShift's Azure infrastructure in ARM templates [here][1], in case that helps clarify the infra we use today. Those are OpenShift 4.6 docs, and 4.6 is based on Kubernetes 1.19, and things worked fine. The trouble we've been having when we took the same Azure-infra approach and tried to use it with Kubernetes 1.20, after openshift/kubernetes. [1]:
could you share an example of resourceID (please replace your subscription/resourceName with fake ones)?
Example CI run [here][1] from the NIC-renaming openshift/installer. Installer logs [here][2] include: I'm hoping that the subscription name is not sensitive, because we've been dumping them in public logs for a long time now . Should be IDs for any other resources you're interested in in those logs as well. [1]: [2]: https://gcsweb-
Looks like this is about the master node. Are you using standard LB or a basic one? For standard LB, the master node would be excluded from the LB by default.
[Standard][1] (and [here][2]). Is this documented somewhere? We've been using standard LBs [for years][3], with no problems until the bump to the Kube 1.20 Azure cloud-provider. [1]: [2]: [3]:
Are you referring to the built-in logic of Kube to never target master nodes with a service type LoadBalancer?

That used to be a problem (), but my understanding is that it has since been fixed.
Can we re-open this? kubernetes-sigs/cloud-provider-azure was [not sufficient][1] to get the cloud-provider happy on our infra, which again, contains no Availability Sets. There are some more notes about our infrastructure [here][2] (with discussion in [rhbz][3]), in case that helps. [1]: [2]: [3]:
who has a better grasp on this than me, also just opened . Sounds like fixing that broader issue might obsolete this ticket, because "the cloud-provider expects Availability Sets which OpenShift infra doesn't have in order to figure out which nodes to remove" doesn't matter if the cloud-provider isn't removing any OpenShift nodes in the first place.
can you please confirm the standalone VM scenario is supported? It's a critical use case for CAPZ (our control plane VMs in zone-enabled regions are not placed in availability sets)

What happened: introduced new regex which is too restrictive for some existing installations: This new restriction causes LBs to fail to sync if just one ipConf fails regex. What you expected to happen: This change was to enable optional behavior, any ipConf that fails regex should just be ignored. This will allow existing installs to work as they did prior to the change. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
Suggest putting a continue here: This will restore previous functionality and appears to have no undesirable impact.
/sig cloud-provider /area provider/azure
Unclear if we should also put a continue here as well:
/assign
could you share an example of resourceID (please replace your subscription/resourceName with fake ones)? /triage accepted
Here's the error we're seeing: Just the string


Question: // ExtraArgs are extra arguments provided to the etcd binary // when run inside a static pod. <ins> // A key in this map is the flag name as it appears on the // command line except without leading dash(es). </ins> ExtraArgs map[string]string `json:"extraArgs,omitempty"` // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert.

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --I was reading the doc about NodeRegistration, with an interest in specifying some . I noticed that the doc (e.g., ) does not tell me exactly how to identify each arg: exactly as on the command line (including leading dash(es)), as on the command line but without leading dash(es), or camelCase. I expected the doc to enable me to write some with confidence. Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cluster-lifecycle
FWIW the answer is flag names as on the command name but without any leading dashes. This also applies to controllerManager.extraArgs, scheduler.extraArgs, etc. which hopefully should all cover this similarly (if not now, after fixing ...)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --I was reading about how to use kubeadm and noticed a typo in the API documentation. speaks of the "node subnet" where I think "pod subnet" is intended. Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cluster-lifecycle
/area kubeadm


Question: return lbStatus.Ingress[0].IP, nil } <del> func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel string) (*network.PublicIPAddress, error) { </del> <ins> func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel, clusterName string) (*network.PublicIPAddress, error) { </ins> pipResourceGroup := az.getPublicIPAddressResourceGroup(service) pip, existsPip, err := az.getPublicIPAddress(pipResourceGroup, pipName) if err != nil {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We deployed two Kubernetes clusters to the same Azure Resource Group. Created services of load balancer type on both clusters. One of the services never received a public IP. Upon investigation, we noticed that the first Kubernetes cluster deleted the Azure Public IP that was created for the service by the other cluster. Apparently, all public IPs in the resource group are retrieved: then public IPs which don't match this criteria: are deleted. This logic besides the name of the kubernetes service should also consider the K8s cluster where the public IP originated from to avoid stepping on each other's toes when multiple clusters share the same resource group. What you expected to happen: In case multiple clusters run in the same resource group a cluster should delete only resources created by itself. How to reproduce it (as minimally and precisely as possible): Create two Kubernetes clusters (1.14) with different cluster names to the same resource group than create kubernetes services of load balancer type in each cluster. Anything else we need to know?: Environment: Kubernetes version (use ): 1.4 Cloud provider or hardware configuration: Azure OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
thanks for reporting the issue, this is the bug we should fix for sharing resource group. /sig azure /kind bug /assign


Question: cancelContextAfter int attemptsExpected   int errExpected        error <ins> timer              Timer </ins> }{ { name: "condition successful is only one attempt",

While trying to update to use latest version of package and to migrate away from deprecated , function, our tests fail as it seems new function immediately executes given condition twice, which is not mentioned anywhere. Result of executed test specified below: Since has been deprecated and it recommends using , I expected it to have the equivalent semantics to allow easy migration from deprecated functions, as no changelog or documentation mentions new semantics. Playground: Local: file with the following content: as . No response_ <details</details<details</details<details</details<details</details<details</details<details</details
/sig api-machinery
CC since you were involved in
/assign /triage accepted
I'd like to work on this one if it's allowed I think this is a simple fix with few lines removed: i.e.: instead of only wait a loop if immediate is false, we wait a time loop after special execution also if the immediate is true.
/assign
/assign


Question: Status: v1.PodStatus{ NominatedNodeName: "node1", }, <del> }), framework.NewPodInfo(&v1.Pod{ </del> <ins> }) unschedulablePodInfo = framework.NewPodInfo(&v1.Pod{ </ins> ObjectMeta: metav1.ObjectMeta{ Name:      "up", Namespace: "ns1",

The scheduler cache entered a corrupted state where a Pod, according to the cache dump, was nominated to node A, while the status said it was nominated to node B. This lead to unschedulable low priority pods due to the phantom pod in the nominator cache. The cache to match the Pod status Setup: two nodes with all the resources used. A high priority pod comes in that is able to preempt at least one pod. This might cause these sequence of events X doesn't fit, it gets nominated to and back to scheduling queue. start a second scheduling attempt for Pod X. get a Pod update for the . This puts the pod back into the scheduling queue. second scheduling attempt finishes and schedules the pod successfully in start a third scheduling attempt for Pod X (due to 3) get a Pod update for the . This should have cleared the nominated node in the cache, but there is nothing currently there. the third scheduling attempt, the pod doesn't fit, so the scheduler nominates it to . Nothing clears this nomination afterwards Usually step 2 happens after 3, so everything works as expected. But in a busy cluster, 3 might be delayed. No response <detailsGKE </details<details</details<details</details<details</details<details</details
/sig scheduling cc
This Pod actually got scheduled: The cache dump was obtained much later at : I think the problem is that we got an update for the Pod (probably the nominated node itself), scheduled it again and assigned a second nominated node for it. The second scheduling attempt failed, but we didn't clear the cache.
/triage accepted
/priority important-soon
Sounds like is related?
It could be the same, or it could be another bug. Note that this is a second scheduling attempt for a Pod that was already scheduled.
My theory of events: X doesn't fit, it gets nominated to and back to scheduling queue. start a second scheduling attempt for Pod X. get a Pod update for the . This puts the pod back into the scheduling queue. second scheduling attempt finishes and schedules the pod successfully in start a third scheduling attempt for Pod X (due to 3) get a Pod update for the . This should have cleared the nominated node in the cache, but there is nothing currently there.

the third scheduling attempt, the pod doesn't fit, so the scheduler nominates it to . Not sure why this pod update fails, but nevertheless, we have a corrupted cache at this point.
thanks for raising up this and the fix. Could you update the "How can we reproduce it (as minimally and precisely as possible)?" section a bit? Per our offline conversation, the issue will leave a "phantom" pod holding a number of resources in the internal nominator cache. And due to that unnecessary resource reservation, low-priority pods may not get scheduled while they should have been. This might be the key trigger of this issue. In practice, it can be caused by a (way-too-quick) movePodBackToActiveQ call (due to relevant/irrelevant events, or backoff's completion) after step 1 and before step 3.
Updated the description. I will update the PRs on Monday.


Question: fileMode        = "file_mode" dirMode         = "dir_mode" vers            = "vers" <del> defaultFileMode = "0700" defaultDirMode  = "0700" </del> <ins> defaultFileMode = "0755" defaultDirMode  = "0755" </ins> defaultVers     = "3.0" )

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Using azureFile volume after 1.8.x mount the share onto host with both dirmode and filemode options set to 0700, which was changed in this commit. This disallow any container run with non root user (which is the suggested way to run app) to access the contents. What you expected to happen: The mode should at least set to 755 or even 777, which seem to be fine since we can still configure of volume or of volumeMount. How to reproduce it (as minimally and precisely as possible): Use azureFile at 1.8.x up versions. Anything else we need to know?: might solve this too, but closed :( Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"8", GitVersion:"v1.8.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-10-24T21:07:53Z", GoVersion:"go1.9.1", Compiler:"gc", Platform:"darwin/amd64"} Server Version: version.Info{Major:"1", Minor:"8", GitVersion:"v1.8.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-10-24T19:38:10Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: Bare metal on Azure VM OS (e.g. from /etc/os-release): Ubuntu 16.04.2 Kernel (e.g. ): Linux -Ubuntu SMP Tue Aug 8 11:56:56 UTC 2017 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm Others:
/sig azure
/assign
I would like to set the mode as 777, let me know your concern about this, thx.
Should any user from the container have full access by default to the volume? I think is a bit too permissive. The default value could be concisely overwritten by the when the permissions are too strict. Also the option is set by default to false if not specified. In general, I would go by default with , and provide the option to increase the privileges when required.

does storage class support mountOptions in v1.8? From the doc: it's already supported, but I never make it work for azure disk and azure file, below is my config:
apparently no :) Need to get mount options by calling MountOptionFromSpec and join to PV options
while for azure file, it's already joining: point is it says invalid for my azure file config:
sorry, correct format should be like this, azure file is provisioned successfully:
so currently azure file/disk does not support mountMounts, I would create a task to support it, seems like it's only a little code change. :)
I have a for this, use mount options in storage class, below is an example:
IT's a real show-stopper using Azurefile! Like any rails app using non-root uid will not be able to Access root-only mounts (like phusion / passenger). I hope this get's fixed soon to add fsgroup properly! It's also very weird that also chown within container is not able to Change ownership of an azure file cifs mount Point!
Hi fsgroup could be set by k8s security context, is that what you want?
Thanks for the hint, but this ist not working for azurefiles Volume, because it's not a usual PVC and a dedicated Volume mount Parameter, without the possibility of fsgroup and fsuid. Would be trivial to fix, because it's a regular cifs 3.0 mount in the Background... I was absolutely surprised that this ist still not fixed in the Go File for the azurefile class in Kubernetes...
you can set uid, gid in mountOptions, the code is already in v1.9.0, you can set filemode and dirmode in mountOptions too, see below example:
Thanks a lot. I the storage class with my uid/gid, but this storage class is not used.. volumes: name: azure azureFile:
I have cherry-pick this mountOptions feature to v1.8, see: Seems you are using azure file static provisioning, here is an example: BTW, set the dirmode & filemode as 0700 is really too strict, customers complain this a lot. I am planing to change it, which value would you suggest? ? Thanks.
AWESOME, working perfectly now!! I really appreciate your immediate Feedback.

I owe you a beer, next time you visit Germany (Hamburg) ! :-) I'd the challenge to set set uid & guid to 0777 to my share that the rails app was working properly, which I've to figure out. I'd recommend to go with 0744 as Default Settings! GREAT COMMUNITY SUPPORT, thanks a lot!! Rgds, Gerald aka Kontainerschiff
I am happy that it works on your side. Your encourage would be my power!! Germany is in my traveling list :-)
Same problem here, still get dirmode=0700 and filemode0700 when using K8s storage class with azure file. When inspecting the PV I see that the storage class has passed the correct mountOptions that are dirmode=0777 and filemode=0777, but when the volume is mounted, it's being mounted with dirmode=0700 and filemode0700. - Cloud provider Service
Hi , I just did a check, PR has already into v1.8, while it's not in v1.8.3 or v1.8.4, it is in v1.8.5 definitely
and in v1.8.3, azure file code would ignore mountOptions in strorage class, mountOptions would be really used from v1.8.5
Thanks for the info, that's good to know so that I can skip using Azure file plugin for now.
It#s really a mess. I just upgraded to 1.8.4 and file_mode only mounts 0700 instead of necessary 0777. I'm on Azure Container Services,so now 1.8.5 upgrade possibility :-(
I'm in production in 1.8.4 and have to switch is a nightmare...
BTW: 1.8.2 was working !
1.8.2 is working? I have checked the code, it's still using 0700, If you are using v1.8.x, I would suggest you to use v1.8.5, it would support mountOptions. 0700 is really too restrict, I have created a to set as 0755.
well..1.8.5 still not available for ACS-engine ...therefore no upgrade on my cluster.

I'm using azureFile in my deployment yaml, but it does not have symbolic link support. I've read that I need to include "mfsymlinks" in my CIFS mount command.. sudo mount -t cifs //<your-storage-acc-<storage-file-share-name/media/shared -o vers=3.0,username=<storage-account-name,password='<storage-account-key',dirmode=0777,filemode=0777,mfsymlinks so how would I define this under mountOptions? Or ???
for dyanmic provisioning, using: for static provisioning, using: mount options support of azure file is available from v1.8.5
thanks andyzhangx! that did it.
hi guys, I am thinking add as default mountoptions into azurefile mount, thus to support symbolic link in azurefile mount, let me know your thoughts, thx
PTAL ^^
Andy, I am using static provisioning where I am providing the volume information to the pod (actually deployment to be exact) like the following: I am not using persistent volume or persistent volume claim as it seems to be required for dynamic provisioning only. I do have an issue where my program can read successfully from the Azure file mount but cannot write to it. What do you suggest I need to do to fix this? I tried using Azure AKS 1.9.2 and Azure AKS 1.9.6 clusters.
looks like I need to document this somewhere, you still need to use PV/PVC to set mountOptions:
Thank you Andy, it worked. If you look at the documentation here, unfortunately, it doesn't talk about it here:
would you add the above doc? Thanks.
Hi, I was wondering, what does this size mean in kubernetes? I created a 5TB file share on Azure, should this need to match and it should be 5TB or 5Ti?
if you are using PVC/PV matching, storage size is only for matching in PVC bounding process, the actual size could be different than ; while if you use dynamic provision of azure file, k8s will create a 5TB file share for you.

,Hi andy,I come from AKS documents.I have a question like .please see the below configuration If I have set up an azure file share which quota is 1G on azure storage service,how much capacity should I specify in the PV defination or storage in PVC.I am confusing about the actual size of mount data. there are three size. real size(quota) in azure file share in PV in PVC please help me understand.
the size in three configs should be identical. If you bring your own file share, k8s won't check if size in 2 or 3 is not identical to in 1. To make your config more simple, it's better make them identical.


Question: // check the status of containers. for idx, container := range pod.Spec.Containers { containerStatus := podStatus.FindContainerStatusByName(container.Name) <ins> // Call internal container post-stop lifecycle hook for any non-running container so that any // allocated cpus are released immediately. If the container is restarted, cpus will be re-allocated // to it. if containerStatus != nil && containerStatus.State != kubecontainer.ContainerStateRunning { if err := m.internalLifecycle.PostStopContainer(containerStatus.ID.ID); err != nil { glog.Errorf("internal container post-stop lifecycle hook failed for container %v in pod %v with error %v", container.Name, pod.Name, err) } } </ins> // If container does not exist, or is not running, check whether we // need to restart it. if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning {

/kind bug /sig node When the is set to and a container that was granted exclusive CPUs terminates with a normal exit status, the CPU Manager never receives a call. Kuberuntime should be updated to invoke the container manager's hook on detecting normal termination for these containers. This method is already idempotent since it is currently invoked both from kuberuntime's and (the latter of which is invoked by GC.) How to reproduce it (as minimally and precisely as possible): Start singleton cluster with static CPU manager policy enabled. Start a guaranteed pod with a single container that consumes 1 CPU (to get an exclusive allocation) with restartPolicy never that simply exits . Try to start a similar container that won't fit on the node unless the first container's CPU is reaped. Observe the second container fails with insufficient CPUs, and this problem does not resolve even after an extended period of time. Environment: Kubernetes version (use ): v1.8 beta Cloud provider or hardware configuration**: all OS (e.g. from /etc/os-release): all cc
/assign
Yikes, that's a fun issue. I guess we need to listen for PLEG events?
There's a spot in the kuberuntime sync where we think we can make the call.
well, i knew i would miss something ;-) good catch.
All thanks to workload perf testing
By the way, where is of CPU manager called if is set to or ? invoked by GC?
Hi, Besides, I wonder do we have a mechanism to recover things in state of CPU manager when kubelet restarts? Here I have a trial, seems it doesn't work as expected, that is: a few logs before and after the action of addcontainer and removecontainer in policy, to see the content of state: a pod that declares an exclusive CPU, and saw logs: kubelet the pod, to compare the state with that , I think if recovered, the content of them should be the same. but saw logs: Seems that the content in state did not recovered, this way, the CPU manager would behavior far from what we expected if kubelet restart. Did I miss something important?
is called as a part of internal container lifecycle pre-stop and post-stop hooks (see and ).

Regarding Kubelet restarts, it is a known issue and captured in
priority/important-soon
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/node: Issue will be escalated to these SIGs if needed. priority/important-soon: Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details
Well, I understand business in , what I wonder is upper behavor, that is, the behavior of in containerRuntime, When a container with restartStrategy exits, It seems that the runtime try to restart new container directly, without calling things like . So I failed to see the place where remove related hooks called in this case, so here I have a trial: a pod whose container is in restartStrategy of and would exit after a perid exited after a peiod: try to restart it but failed: restart after quite a preiod: So I think the resource is not recycled synchronously, and the remove related hook is called in GC. Am I right? Does it works in the way we expected?
Yes, that's exactly correct. The linked PR with the fix for this issue also helps the situation you described to converge faster.


Question: v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), }, }, <ins> StorageClassName: cfg.StorageClassName, </ins> }, } } <del> func CreatePDWithRetry() (string, error) { </del> <ins> func createPDWithRetry(zone string) (string, error) { </ins> var err error for start := time.Now(); time.Since(start) < PDRetryTimeout; time.Sleep(PDRetryPollTime) { <del> newDiskName, err := createPD() </del> <ins> newDiskName, err := createPD(zone) </ins> if err != nil { Logf("Couldn't create a new PD, sleeping 5 seconds: %v", err) continue

For statically created PVs, the persistent volume label admission controller adds the zone label to the PV object. If the admission controller is not enabled or has some bugs, then an e2e test would be helpful for catching these errors. /assign
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list


Question: cmc.Check(t) } <ins> func TestOnEndpointSliceUpdate(t *testing.T) { _, esController := newController([]string{"node-1"}, time.Duration(0)) ns := metav1.NamespaceDefault serviceName := "testing-1" epSlice1 := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name:      "matching-1", Namespace: ns, Labels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   controllerName, }, }, AddressType: discovery.AddressTypeIPv4, } epSlice2 := epSlice1.DeepCopy() epSlice2.Labels[discovery.LabelManagedBy] = "something else" assert.Equal(t, 0, esController.queue.Len()) esController.onEndpointSliceUpdate(epSlice1, epSlice2) err := wait.PollImmediate(100*time.Millisecond, 3*time.Second, func() (bool, error) { if esController.queue.Len() > 0 { return true, nil } return false, nil }) if err != nil { t.Fatalf("unexpected error waiting for add to queue") } assert.Equal(t, 1, esController.queue.Len()) } </ins> // Ensure SyncService handles a variety of protocols and IPs appropriately. func TestSyncServiceFull(t *testing.T) { client, esController := newController([]string{"node-1"}, time.Duration(0))

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Mistake use obj as prevObj prevEndpointSlice := obj.(discovery.EndpointSlice) What you expected to happen: prevEndpointSlice := prevObj.(discovery.EndpointSlice) How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: cc
/sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved PR for fixing it Thanks