Question: <ins> /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package apimachinery import ( "context" "encoding/json" "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" "github.com/onsi/ginkgo" "k8s.io/apimachinery/pkg/types" ) const ( eventRetryPeriod  = 1 * time.Second eventRetryTimeout = 1 * time.Minute ) var _ = ginkgo.Describe("[sig-api-machinery] Events", func() { f := framework.NewDefaultFramework("events") /* Release : v1.19 Testname: Event resource lifecycle Description: Create an event, the event MUST exist. The event is patched with a new message, the check MUST have the update message. The event is deleted and MUST NOT show up when listing all events. */ framework.ConformanceIt("should ensure that an event can be fetched, patched, deleted, and listed", func() { eventTestName := "event-test" ginkgo.By("creating a test event") // create a test event in test namespace _, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).Create(context.TODO(), &v1.Event{ ObjectMeta: metav1.ObjectMeta{ Name: eventTestName, Labels: map[string]string{ "testevent-constant": "true", }, }, Message: "This is a test event", Reason:  "Test", Type:    "Normal", Count:   1, InvolvedObject: v1.ObjectReference{ Namespace: f.Namespace.Name, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create test event") ginkgo.By("listing all events in all namespaces") // get a list of Events in all namespaces to ensure endpoint coverage eventsList, err := f.ClientSet.CoreV1().Events("").List(context.TODO(), metav1.ListOptions{ LabelSelector: "testevent-constant=true", }) framework.ExpectNoError(err, "failed list all events") foundCreatedEvent := false var eventCreatedName string for _, val := range eventsList.Items { if val.ObjectMeta.Name == eventTestName && val.ObjectMeta.Namespace == f.Namespace.Name { foundCreatedEvent = true eventCreatedName = val.ObjectMeta.Name break } } framework.ExpectEqual(foundCreatedEvent, true, "unable to find the test event") ginkgo.By("patching the test event") // patch the event's message eventPatchMessage := "This is a test event - patched" eventPatch, err := json.Marshal(map[string]interface{}{ "message": eventPatchMessage, }) framework.ExpectNoError(err, "failed to marshal the patch JSON payload") _, err = f.ClientSet.CoreV1().Events(f.Namespace.Name).Patch(context.TODO(), eventTestName, types.StrategicMergePatchType, []byte(eventPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch the test event") ginkgo.By("fetching the test event") // get event by name event, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).Get(context.TODO(), eventCreatedName, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to fetch the test event") framework.ExpectEqual(event.Message, eventPatchMessage, "test event message does not match patch message") ginkgo.By("deleting the test event") // delete original event err = f.ClientSet.CoreV1().Events(f.Namespace.Name).Delete(context.TODO(), eventCreatedName, metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete the test event") ginkgo.By("listing all events in all namespaces") // get a list of Events list namespace eventsList, err = f.ClientSet.CoreV1().Events("").List(context.TODO(), metav1.ListOptions{ LabelSelector: "testevent-constant=true", }) framework.ExpectNoError(err, "fail to list all events") foundCreatedEvent = false for _, val := range eventsList.Items { if val.ObjectMeta.Name == eventTestName && val.ObjectMeta.Namespace == f.Namespace.Name { foundCreatedEvent = true break } } framework.ExpectEqual(foundCreatedEvent, false, "should not have found test event after deletion") }) ginkgo.It("should delete a collection of events", func() { eventTestNames := []string{"test-event-1", "test-event-2", "test-event-3"} ginkgo.By("Create set of events") // create a test event in test namespace for _, eventTestName := range eventTestNames { eventMessage := "This is " + eventTestName _, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).Create(context.TODO(), &v1.Event{ ObjectMeta: metav1.ObjectMeta{ Name:   eventTestName, Labels: map[string]string{"testevent-set": "true"}, }, Message: eventMessage, Reason:  "Test", Type:    "Normal", Count:   1, InvolvedObject: v1.ObjectReference{ Namespace: f.Namespace.Name, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create event") framework.Logf("created %v", eventTestName) } ginkgo.By("get a list of Events with a label in the current namespace") // get a list of events eventList, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{ LabelSelector: "testevent-set=true", }) framework.ExpectNoError(err, "failed to get a list of events") framework.ExpectEqual(len(eventList.Items), len(eventTestNames), "looking for expected number of pod templates events") ginkgo.By("delete collection of events") // delete collection framework.Logf("requesting DeleteCollection of events") err = f.ClientSet.CoreV1().Events(f.Namespace.Name).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{ LabelSelector: "testevent-set=true"}) framework.ExpectNoError(err, "failed to delete the test event") ginkgo.By("check that the list of events matches the requested quantity") err = wait.PollImmediate(eventRetryPeriod, eventRetryTimeout, checkEventListQuantity(f, "testevent-set=true", 0)) framework.ExpectNoError(err, "failed to count required events") }) }) func checkEventListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) { return func() (bool, error) { var err error framework.Logf("requesting list of events to confirm quantity") eventList, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{ LabelSelector: label}) if err != nil { return false, err } if len(eventList.Items) != quantity { return false, err } return true, nil } } </ins>

[x] Original approval issue: [x] Two weeks soak end date: 6 July 2020 This issue is created to allow edit by /wip /hold According to this APIsnoop query, there are still an endpoint which is untested. a set of 3 Events with a static label within a Namespace. that all 3 Events with the label in the Namespaces are created. the set of Namespaced Events with a label via DeleteCollection. that all Events with the label have been deleted. Discover useragents: List endpoints hit by the test: Display endpoint coverage change: Two other endpoints tested here have tests in progress to conformance already. If a test with these calls gets merged, test coverage will go up by 1 points This test is also created with the goal of conformance promotion.
/sig testing /sig architecture /area conformance
During Conformance Meeting: Is there a way to write this test so it is resource agnostic? If we are just changing the resource... can be parameterize the resource-type
/assign
GitHub didn't allow me to assign the following users: heyste. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen PR Merge and soaking
Reopened this issue. <detailsIn response to : /reopen PR Merge and soaking Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return nil, err } domainNameLabel := getPublicIPDomainNameLabel(service) <del> pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel) </del> <ins> pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel, clusterName) </ins> if err != nil { return nil, err }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We deployed two Kubernetes clusters to the same Azure Resource Group. Created services of load balancer type on both clusters. One of the services never received a public IP. Upon investigation, we noticed that the first Kubernetes cluster deleted the Azure Public IP that was created for the service by the other cluster. Apparently, all public IPs in the resource group are retrieved: then public IPs which don't match this criteria: are deleted. This logic besides the name of the kubernetes service should also consider the K8s cluster where the public IP originated from to avoid stepping on each other's toes when multiple clusters share the same resource group. What you expected to happen: In case multiple clusters run in the same resource group a cluster should delete only resources created by itself. How to reproduce it (as minimally and precisely as possible): Create two Kubernetes clusters (1.14) with different cluster names to the same resource group than create kubernetes services of load balancer type in each cluster. Anything else we need to know?: Environment: Kubernetes version (use ): 1.4 Cloud provider or hardware configuration: Azure OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
thanks for reporting the issue, this is the bug we should fix for sharing resource group. /sig azure /kind bug /assign


Question: "type": "array" } }, <del> "required": [ "acceptedNames", "storedVersions" ], </del> "type": "object" }, "io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionVersion": {

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Currently, the field on must be explicitly set to their zero-values due to the fact that they are not tagged . Ref: Why is this needed: If using a system that automatically compares the provided configuration to the current, it will produce a diff on the updated status after creation. In addition, making sure to explicitly set to zero-values is not the best user experience. I am happy to provide the fix if this change is approved! /cc /sig api-machinery /area custom-resources
marking them as seems ok, so that client-side validation doesn't complain if they are absent when submitting manifests, but they shouldn't need to be set to omitempty (and at this point, we likely wouldn't change the serialization) Those fields are changed by controllers after creation, so expecting the live API object to exactly match the status is not going to go well anyway.
Good point! I’ll open a fix for this. Yep! The specific system in question ignores fields that are not set at creation time by default when doing comparison, so if status fields are not included then it will not report a diff.
/assign
I opened a fix for this but CC'ed both you and Let me know if any additional effort is required. Thanks!


Question: if len(az) < 1 { return "", fmt.Errorf("invalid (empty) AZ") } <del> region := az[:len(az)-1] </del> <ins> r := regexp.MustCompile(`^([a-zA-Z]+-)+d+`) region := r.FindString(az) if region == "" { return "", fmt.Errorf("invalid AZ: %s", az) } </ins> return region, nil }

<!-- Please only use this template for submitting enhancement requests --What would you like to be : AWS release [Local Zones] (). One user enable local zones, using us-west-2 as an example, there's AZ and AWS right now uses method here to infer to region. This won't work for local zone region. Why is this needed: Update to cover this case
/sig cloud-provider These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry.
/area provider/aws


Question: exit 0; } <ins> kube::golang::build_binaries_for_platform() { local platform=$1 local use_go_build=${2-} local -a statics=() local -a nonstatics=() for binary in "${binaries[@]}"; do if kube::golang::is_statically_linked_library "${binary}"; then kube::golang::exit_if_stdlib_not_installed; statics+=($binary) else nonstatics+=($binary) fi done if [[ -n ${use_go_build:-} ]]; then # Try and replicate the native binary placement of go install without # calling go install.  This means we have to iterate each binary. local output_path="${KUBE_GOPATH}/bin" if [[ $platform != $host_platform ]]; then output_path="${output_path}/${platform////_}" fi for binary in "${binaries[@]}"; do local bin=$(basename "${binary}") if [[ ${GOOS} == "windows" ]]; then bin="${bin}.exe" fi if kube::golang::is_statically_linked_library "${binary}"; then kube::golang::exit_if_stdlib_not_installed; CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}"  "${goflags[@]:+${goflags[@]}}"  -ldflags "${version_ldflags}"  "${binary}" else go build -o "${output_path}/${bin}"  "${goflags[@]:+${goflags[@]}}"  -ldflags "${version_ldflags}"  "${binary}" fi done else # Use go install. if [[ "${#nonstatics[@]}" != 0 ]]; then go install "${goflags[@]:+${goflags[@]}}"  -ldflags "${version_ldflags}"  "${nonstatics[@]:+${nonstatics[@]}}" fi if [[ "${#statics[@]}" != 0 ]]; then CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}"  -ldflags "${version_ldflags}"  "${statics[@]:+${statics[@]}}" fi fi } # Return approximate physical memory in gigabytes. kube::golang::get_physmem() { local mem # Linux, in kb if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then echo $(( ${mem} / 1048576 )) return fi # OS X, in bytes. Note that get_physmem, as used, should only ever # run in a Linux container (because it's only used in the multiple # platform case, which is a Dockerized build), but this is provided # for completeness. if mem=$(sysctl -n hw.memsize 2>/dev/null); then echo $(( ${mem} / 1073741824 )) return fi # If we can't infer it, just give up and assume a low memory system echo 1 } </ins> # Build binaries targets specified # # Input:

Filing for This code is wrong The local needs to be on a separate line from the command execution, and we need to be resilient to not existing on OS X, like:
For once I was actually watching the output of a build and noticed that I was seeing an error about md5sum not being found. I'm building on a mac using boot2docker. The short_hash function () works fine on my mac as md5 does exist.
Actually, I'm just going to fix this as a follow-on to .
Thank you!


Question: // verifyRunAsNonRoot verifies RunAsNonRoot. func verifyRunAsNonRoot(pod *v1.Pod, container *v1.Container, uid int64) error { effectiveSc := securitycontext.DetermineEffectiveSecurityContext(pod, container) <del> if effectiveSc == nil || effectiveSc.RunAsNonRoot == nil { </del> <ins> // If the option is not set, or if running as root is allowed, return nil. if effectiveSc == nil || effectiveSc.RunAsNonRoot == nil || !*effectiveSc.RunAsNonRoot { </ins> return nil }

What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): kubelet securitycontext Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): 1.7.0-beta/1.8.0 (HEAD) Environment: Cloud provider or hardware configuration: gce OS (e.g. from /etc/os-release): gci Kernel (e.g. ): Install tools: Others: docker runtime (rkt seems to have a different code path, but I haven't checked it) What happened: containers could not run as root when "RunAsNonRoot=false" What you expected to happen: containers could run as root How to reproduce it (as minimally and precisely as possible): add a securitycontext to a container, don't set privileged, do set RunAsNonRoot, use an image that runs as root. The PodSecurityPolicy admission controller will always set a securitycontext if it's not specified, and cause this to be hit more often. Anything else we need to know: The bug is in the checks here: I have a fix in but that going in depends on an extension being granted. We may want to fix the bug regardless.
/sig node
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
I'll send a fix shortly.


Question: "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" volumevalidation "k8s.io/kubernetes/pkg/volume/validation" "k8s.io/kubernetes/third_party/forked/golang/expansion" <ins> utilnet "k8s.io/utils/net" </ins> ) const (

What happened: The is taken from without regard for the main family of the cluster. No warning or error is given if gets the wrong family. What you expected to happen: That is taken from the "main" ip family, or at the very least the POD is not started and an error is given if it's not. How to reproduce it (as minimally and precisely as possible): Use the CNI-plugin and ipam. Use a single node cluster to avoid that assign the same address to PODs in different nodes. This may be CRI-plugin dependent, I use . The PODs are assigned dual addresses which has been supported since v1.9. Note the order; IPv6 first. In a dual-stack or a single-stack cluster with main family IPv4 will get an IPv6 address even if the main family is IPv4; Anything else we need to know?: First, note that this is not a dual-stack problem. may get the wrong family in a single-stack cluster if the PODs has dual addresses. The bug may however have been introduced with the dual-stack support, before K8s may have selected the correct family, I have not checked that. This is reported and discussed in There it is considered as a CNI-plugin problem, the CNI-plugin must present addresses in the order K8s wants them. Problem is that CNI-plugins doesn't know this. The addresses takes the path; The problem can be addressed in either of these places. The current situation is that the full responsibility lies on the , it must send the addresses in the "correct order". This is not the best place. The CNI-plugins are not a K8s-only thing and to require that a "prefered-family" or something similar must be supported by CNI-plugins to be "Kubernetes compliant" should be avoided. The CRI-plugin is a K8s-only thing and a "prefered-family" can be introduced so the CRI-plugin can sort the address array before sending it to . This is however undesirable since it adds a configuration complexity, it put a responsibility on the user or installation tool to configure the "prefered-family" to all CNI-plugins now and in the future. The best is to handle this in K8s itself where the "main" family is known. Is this really necessary?

All communication works even if is not of the main family. EndpointSlices seem to get this right regardless of order. Endpoint on the other hand seem to take the (old) podIP. So perhaps; is sufficient? If really must be , then K8s should sort the array so belongs to the main family. The problem exist on all supported K8s versions and on "master". If installed in the default way the situation is; Calico always send IPv4 first Cilum always send IPv6 first I don't think you can control the order, but I have not asked. Environment: Kubernetes versions: v1.17.12, v1.18.9, v1.19.2, v1.20.0-alpha.1, master v1.20.0-alpha.1.257+ Cloud provider or hardware configuration: None OS (e.g: ): xcluster Kernel: linux-5.8.1 Install tools: None Network plugin and version: bridge, host-local v0.8.7 Others: CRI-plugin: cri-o 1.18.3
/sig network
For the record; all communication via services works perfectly fine even if the order in is "faulty" with not belonging to the main family in k8s v1.18.8.
I think this is a duplicate of CNI plugins must report multiple IPs in the order expected by the cluster. The podIPs[0] == podIP requirement is for backwards compatibility with clients that do not read podIPs. Server-side reordering causes problems with clients that continually write/reconcile until the server list exactly matches their local list.
The problem exist in single-stack K8s clusters when PODs has dual addresses (supported since K8s v1.9, i.e way before K8s dual-stack which came in v1.16) And this issue is not a duplicate of . That issue is about a problem when the CNI-plugin uses the "correct" order. This issue is about the fact that K8s puts undocumented requirements on the CNI-plugins. There is no requirement on order in the . And since this requirement is undocumented it is not supported for instance by cilium and calico. The itself should be the requirement is for backwards compatibility with clients that do not read podIPs.

Since they doesn't read podIPs the value of should not matter to them, or? I think this should be handled in K8s, but if the responsibility is pushed downhill to the CNI-plugins then two things must be done in K8s; extra requirement on "Kubernetes compliant" CNI-plugins (outside the CNI spec) must be formally documented and communicated to the CNI-plugin providers. a "faulty" CNI-plugin accidently put's an IP-address of the "wrong" family in the first item then K8s must report an error and not start the POD. Please note again that this is not a K8s dual-stack problem. In a single-stack IPv6 K8s cluster the CNI-plugins must provide an IPv6 address first in the list.
The reason podIP must match podIPs[0] is related to updates to an existing object by an old client. See for a related problem when this was not done. An old client updated an existing object, setting only the podIP field (which is the only one it was aware of). The API server needed to detect that and populate the podIPs field correctly for new clients to read. The detection is based on the mismatch between podIP and podIPs[0], and the server responds by populating . That means that new clients (aware of both fields) must set podIP and podIPs[0] to match.
So this is weird, but... dual-stack clusters do not actually have a "main family", because decisions about IP families are made by multiple components, and kubernetes clusters do not have a single source of configuration truth that applies to all components. So it is explicitly allowed to have kube-apiserver default to IPv4-primary IPv6-secondary for services, and have your network plugin default to IPv6-primary IPv4-secondary for pod IPs, and to have some nodes have a single IPv4 address and some have a single IPv6 address and some have dual-stack addresses. It's not super clear why you'd want to do that, but we decided we had to allow it, simply because there's no easy way to prevent it. Of course in scenarios where the entire cluster is being installed/deployed by a single tool, one would expect that tool to configure all of the components in a consistent way. There is no "order K8s wants them".

Kubernetes doesn't want the addresses in any order. You want them in a particular order. Therefore you must configure the CNI plugin to return them in that order. Eg, in the example above, you should reverse the two in the config. Kubernetes's interpretation of this is that if Cilium sends the IPv6 address first, then it is telling Kubernetes that Kubernetes ought to prefer the IPv6 one. It appears that the CNI spec doesn't actually say that though... It doesn't seem to say anything about the ordering of multiple IPs... So maybe that should be clarified. If the CNI authors feel that the ordering of returned addresses is not meaningful then perhaps kubelet's behavior could be changed.
Thoughts on this?
CNI doesn't make any statement about ordering of IPs. It definitely doesn't have any notion of a "primary IP". It seems that containerd itself does have some
I mean that's literally what I just said :stuckouttongue:. The question is more if that is intentional and/or changeable. It seems strange that the decision about whether to allocate IPv4 and/or IPv6 IPs is left up to the network plugin, with no input from the runtime, but the decision about what to do with IPv4 and/or IPv6 IPs is then left up to the runtime, with no input from the network plugin.
That is very good an as it should be but... The quote is from the initial commant. Please note that this is a single-stack cluster! And that has a main family even though the PODs have dual addresses.
Right; don't comment on issues just before going to bed. Ooops. I don't think we want to introduce the notion of IP ordering / or preference in to CNI for a few reasons -- It's a bit of a breaking change, depending on how you read the spec It would be awkward to push the "preferred family" from k8s all the way down through to CNI It doesn't match how Linux works - interfaces don't have primary IPs (legacy aside). So fixing this in the apisever is probably the right place? BTW, what, exactly, defines a cluster's address family? ClusterCIDR, but that's optional (as is PodCIDR). ServiceCIDR, certainly. Is there an explicit config flag?

no flag, you configure the cluster with IPv4 or IPv6 addresses, of course if you have pod with IPv4 addresses and service with IPv6 that will not work kubelet set the status.PodIPs In a single-stack cluster you don't have a main family, you only have ONE family If we don't have PodIPs and only PodIP in a single-stack cluster this will be clearly a configuration problem in the CNI,no? I personally, think that this is a configuration problem and that we created it because we pluralised PodIP without a feature gate
Yes, and this is how it looks in an IPv4 single-stack cluster ( NOT set) if the CNI-plugin has been configured for dual addresses on PODs but returns the assigned addresses in the "wrong" order (ipv6 first);
Why is requesting an IPv6 address for pods in an IPv4 cluster not a configuration error? To me it seems a bit like running an ARM image on an AMD64 machine; Kubernetes will go through some motions but will not make it work or tell you what you did wrong.
Dual addresses in PODs has been supported from K8s v1.9, i.e. long before K8s dual-stack so that is not a configuration error. In the CNI specification there is nothing about required order for addresses or "prefered family" so it is not a configuration error for the CNI-plugin either. For K8s functions the order in does not matter. But design of upgrade requires that see Problem is that K8s (kubelet) sets "podIP = podIPs[0]" without checking the family. Here some error should be in place, like "CNI-plugin not compatible with K8s" and the POD should not be started, or better, sort the addresses or remove the requirement. This is a K8s problem. CNI-plugins should not have to care about order of reported addresses at least not without a CNI specification update.
This doesn't appear to answer my question. What is the meaning of an IPv6 pod address in an IPv4 cluster?
A cluster does not have an address family. Usually it seems like it does but that's just because the administrator has configured all of the individual components in a consistent way. But most components' configuration only affects what that component does (though in some cases that will have knock-on effects).

kube-controller-manager sets based on its (assuming is set) The network plugin (indirectly) sets (which may be affected by kcm's choice of ) based on its own configuration Either kubelet or an external cloud provider sets : based on , if passed, in the non-cloud-provider case based on cloud-provider-specific configuration (or hardcoded defaults) in the cloud case kubelet sets from kube-apiserver sets based on its kube-controller-manager creates endpoints, mostly based on the IP families chosen by kube-apiserver, except in the case of headless selectorless services, where it lets the user set endpoints of whatever family it wants, because it doesn't usually know whether the cluster is single or dual stack. kube-proxy determines whether rules are actually created for IPv4 and/or IPv6 services, based on the passed in if the dual stack feature gate is disabled, and unconditionally dual stack if the feature gate is enabled the cloud provider determines whether services get IPv4 and/or IPv6 IPs, based on cloud-provider-specific config or hardcoded defaults. (EDIT: well, and probably ) This... is kind of a mess, but there's no real way to fix it centrally given the way that kubernetes works. But we could add a config option to kubelet to unambiguously specify whether it should act like it's in a SSIPv4/SSIPv6/DSIPv4Primary/DSIPv6Primary cluster, and then make assignment make use of that.
Oof. Well. Maybe it makes sense for kubelet to stop setting PodIP, and leave that decision to the apiserver? It could make that decision based on ServiceCIDR, and order the families accordingly?
No, the dual stack KEP even explicitly calls out the fact that that it doesn't work that way: There is no single source of truth for kubernetes cluster config, and thus there is no single truth about whether the cluster is IPv4, IPv6, dual-stack-IPv4-primary, or dual-stack-IPv6-primary. If we want kubelet to impose some policy on PodIPs, then we need to tell it what policy to impose, because it has no entirely-reliable way of determining what policies any other component is imposing.
That is still much better than to require this function from the CNI-plugin which is not even K8s and follows a specification.

Please note that the networking e.g services, external access, works fine whatever order are in and whatever value has. I have not found any communication problem in K8s = v1.19 caused by this. The value of seem more "cosmetic". But it looks wrong to see ipv6 addresses of PODs in an ipv4-only cluster (). The problem seem to be an unfortunate upgrade design that imposed the constraint If that can some way be relaxed this becomes a no-problem as I see it.
/triage accepted /assign
What is the material impact of this behavior?
we can not relax because that will break existing clients. a side note. Using dualstack configured CNI on a none dual stack cluster was pre dualstack hack, if it must remain in use then CNI should only report one IP not two.
what's your reasoning to change CNI rather than kubelet?
We can't just put undocumented requirements on the CNI-plugins, they are not part of K8s, they have a formal specification and they have other users beside K8s. At least K8s must formally document the extra requirements on CNI-plugins that K8s has for the CNI-plugin to be K8s compliant.
As highlighted on the discussion there is no single truth of what clusterIP family is. Also kubelet will have to depend on which is not always used by CNIs. You can have a perfectly functional cluster that does not use PodCIDRs at all (I believe Cilium for example does not use it). Also I am not saying change CNI. A well configured CNI (single family for single stack cluster) will not have this problem.
Is the latter statement a retraction of the former? If not, how can they be reconciled?
He's saying that an administrator deploying a CNI plugin in a single-stack kubernetes cluster ought to configure that CNI plugin to return a single IP address, of the family being used by the kubernetes cluster. A CNI plugin that does not allow you configure what address families it uses may not be compatible with all kubernetes clusters. Eg, a CNI plugin that only supports IPv4 cannot be used as the network plugin in a single-stack IPv6 kubernetes cluster.
So, to sum up: Kubelet will always use the first IP returned by the CNI plugin as , regardless of any other cluster configuration.

Some CNI plugins, when configured to do dual stack, do not give you a way to configure which IP comes first, meaning kubelet may pick a that is not the IP family the admin considers the cluster's primary IP family In dual-stack clusters, this does not appear to cause any actual problems, other than that the "default" pod IPs shown in "" may not be the ones the administrator wants Nothing in the current kubelet configuration is supposed to indicate whether kubelet should prefer IPv4 or IPv6 pod IPs. It would be possible to add a new config option indicating this, or we could just declare that should also affect the sorting of pod IPs, because really why wouldn't you want that? In single-stack clusters, it has been suggested that the right answer is "don't do that then"; ie, the administrator should configure the CNI plugin to only return a single IP, and if they can't do that, then that's the CNI plugin's problem, not Kubernetes's
oh, and: if the CNI plugin returns multiple IPs which are not a dual-stack pair (eg, if it returns 2 IPv4 IPs, or 1 IPv4 IP and 2 IPv6 IPs), then kubelet will fail to update the pod at all, because it assumes it can just copy the IPs returned from CNI into , but the apiserver will only accept if it is either a single IP or a dual-stack pair.
That would be excellent! Then it would not be necessary to stop supporting dual-stack pods in a single stack K8s. And Cilium users wanting to upgrade their ipv4-only cluster to dual stack would not suddenly see ipv6 addresses on a .
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Just for the record, this is an actual problem when trying to deploy rook-ceph. Rook makes strong assumption about family being the same as family and just fails to create Ceph cluster when they do not match. are configured to listen only on but the advertised addresses are the corresponding service .

This result in listening only on pod IPv6 address but being advertised using the service IPv4 address.


Question: utilexec "k8s.io/utils/exec" ) <del> // exclude those used by azure as resource and OS root in /dev/disk/azure </del> <ins> // exclude those used by azure as resource and OS root in /dev/disk/azure, /dev/disk/azure/scsi0 // "/dev/disk/azure/scsi0" dir is populated in Standard_DC4s/DC2s on Ubuntu 18.04 </ins> func listAzureDiskPath(io ioHandler) []string { <del> azureDiskPath := "/dev/disk/azure/" </del> var azureDiskList []string <del> if dirs, err := io.ReadDir(azureDiskPath); err == nil { for _, f := range dirs { name := f.Name() diskPath := azureDiskPath + name if link, linkErr := io.Readlink(diskPath); linkErr == nil { sd := link[(libstrings.LastIndex(link, "/") + 1):] azureDiskList = append(azureDiskList, sd) </del> <ins> azureResourcePaths := []string{"/dev/disk/azure/", "/dev/disk/azure/scsi0/"} for _, azureDiskPath := range azureResourcePaths { if dirs, err := io.ReadDir(azureDiskPath); err == nil { for _, f := range dirs { name := f.Name() diskPath := filepath.Join(azureDiskPath, name) if link, linkErr := io.Readlink(diskPath); linkErr == nil { sd := link[(libstrings.LastIndex(link, "/") + 1):] azureDiskList = append(azureDiskList, sd) } </ins> } } }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Cannot mount azure disk on Standard_DC4s/DC2s instances details: What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: /kind bug /assign /priority important-soon /sig cloud-provider /area provider/azure


Question: "path" rt "runtime" "strconv" <ins> "strings" </ins> "time" "k8s.io/kubernetes/pkg/admission"

Is this expected? Does it mean that we cannot drop and replace alpha api without waiting for the new gcloud version?
cc:
cc
cc:
My money is on the TPR discovery code that is trying to identify TPR versions and hacks them into the list of registered APIs.
I guess it's not on HEAD? Do you have the PR#?
had a PR moving StatefulSet from v1alpha1 to v1beta1 and she didn't have the same problem. It's interesting why graduating PDB will have the problem
My PR has similar failure but different symptom. I created a GCE cluster locally. doesn't give me that error, but addon-manager isn't working properly. From addon-manager container log, I saw similar error:
Ah, I was using new kubectl
was using the newly built kubectl so we didn't see the problem at first. Addon-manager is using 1.4.4 kubectl, so we observed problem later.
Hmm, if the problem is the old versions of kubectl unable to dealing with new REST endpoints, then the problem is hard. I'll take a look.
is correct. See this .
I thought the TPR code was inert by default and required a flag to kick in
It seems the flag is enabled for most commands:
We need to support 1 minor version skew for kubectl, so we have to workaround in the server. We can hide those new APIVersions in the discovery API from old kubectl. I'm going to send a PR. In the mean time, if someone can come up with a better idea.
kubectl bug is fixed in , need to cherrypick the change to 1.4. is working on server side temp fix
should fix the problem.
The issue was "fixed" because the "unable to register" error was ignored (not returned, just logged), see:
that line in doesn't seem like a fix... it seems like it's leaving the client restmapper in an unknown state, which seems bad to me
Yeah it's not really a fix
Looks like kubectl should try to register the version to the existing group. IMO this isn't a blocker for
can you make sure there's a 1.5 issue tracking the failure to register the new versions into existing groups correctly, or we'll have a related issue in the future
Filed to track this


Question: func TestApiserverMetrics(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> // Make a request to the apiserver to ensure there's at least one data point // for the metrics we're expecting -- otherwise, they won't be exported.

http://pr- is another failure.
Also seen in http://pr-
Again. Not assigned to anyone, so I am sending to for routing
Again: Is there anyone working on this? I will spend some time looking into this.
I think that the name of this issue is misleading - the issue seems to be unrelated to etcd_watcher Basically, if you look into this part of log: So it seems that, the WaitGroup that is problematic is in httptest.Server. So my feeling is that we are simply talking to a server that is already being close and this is ~roughly duplicate of We should just find where exactly the problem is and try to work-around it.
I think this is exactly what we had before. I will send a PR with the fix today.


Question: - user: root - name: /opt/cni - makedirs: True <del> - source: https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz </del> <ins> - source: https://storage.googleapis.com/kubernetes-release/network-plugins/cni-0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff.tar.gz </ins> - tar_options: v <del> - source_hash: md5=dd11b04dcb0b9aca00b5287ba18dcfaa </del> <ins> - source_hash: md5=afbb526e7d976f98353ac96f73043031 </ins> - archive_format: tar - if_missing: /opt/cni/bin

Started as a tangent of Many CNI plugins don't properly handle a action on missing resources. The spec says, , however many improperly return an error. I see the steps moving forward to be: [x] Modify CNI plugins bundled with Kubernetes to properly support del (flannel already done; bridge, ipvlan, macvlan, and ptp in containernetworking/cni) [x] Create release of containernetworking/cni Does this make sense? cc/
SGTM - I'm working on the right now and can do the CNI release too. On the 3rd check box - why is that still a thing? Doesn't the CNI project provide the required artifacts as part of its release? Maybe not something to resolve now but I'd like to make sure the CNI project is providing the right things to projects that use it.
I just want to say thanks to everyone for jumping on this.
status tracking doc
I can test out patches as needed. I currently only have test cases for Weave and Calico, but it should not be too difficult to add others.
It would be great to get flannel tested too. You probably need a master build of the flannel CNI plugin and possibly some flannel patches to be merged. Feel free to DM on slack if you need more info.
s390x builds are missing still. If that's , we can switch to v0.5.1 in time for v1.7. However, I really don't want to bump from v0.3.0-ish to v0.5.0 one day before release. This is a v1.7 issue to ensure all CNI plugins (including the official) work as intended with Kubernetes and an issue to make sure we update to CNI v0.5.0 w/ spec v0.3.0 in time for v1.7. I think will solve the short-term solutions. Also it seems like Weave, Calico and maybe Canal has fixed this as-is. I think Flannel works with
I'm a little confused! You mean s390x builds for flanneld? And what do you mean by This is a v1.7 issue I thought this was a 1.6 issue?

CNI still needs s390x binaries to be consumable by k8s: Two other conclusions: 1) We should go with as the immediate solution 2) We should upgrade to CNI v0.5.1 (when released( officially for Kubernetes in time for v1.7 and enforce the latest spec (v0.3.0) for all network plugins
We should get the PR in (after testing), but this is not a really solution as it only avoids hitting this problem at cluster setup.
OK, that makes sense on the s390x binaries. I've just released v0.5.1 of CNI, which updates the plugins with the new del behaviour. They should all still be back compatible with the older spec versions so I recommend that K8s takes this CNI release.
I'm working on a patch for that bump now
Be aware that has just been merged. If you sync to HEAD, you won't be able to reproduce this problem at cluster setup easily.
New cni packaged tarball (based on v0.5.1) has been pushed:
-- can this be closed?
we can't close until all the consumers have been updated to point to the new tarballs
What is the ETA for resolving this?
Ok, I see that is at the top of the submit queue.


Question: import ( "context" <ins> "fmt" </ins> apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/runtime"

An ingress resource with both annotation and set cannot be created. Ingress resources can be updated to include both the annotation and the without issue. Kubernetes response: The ingress should be deployed with both annotation and set, as it is supported by updating an existing ingress. Apply this resource: The code causing this issue is here: <details</details<details</details<detailsLinux, Mac, mulitple OSs with the same issue. It's caused by code as linked above </details<details</details<details</details<details</details
/sig network
that code referenced is only used on Create update has a different path is this based on an observation of the code or are you hitting this problem?
, correct. This only happens on create. Our work around now is to create an ingress resource with the annotation, and then update the resource, adding the . This involves 2 MRs and approval processes We are hitting this problem regularly as we scale out the number of kubernetes clusters we operate, and create new ingress resources. It's quite painful as we're starting to use , but cannot drop the ingressClass annotation yet as some third party services have not ported over to use the .
but why do you create it with the annotaiton?
Sorry, I don't understand your question. I need to be able to create an ingress resource with both the ingressClass annotation () and both set
yeah, that is my question, if you are creating the ingress with v1 you don't need the annotation, it was deprecated by the field, the annotation has to be there for the people that upgrades from v1beta1 to v1
As far as I know, External DNS still requires the annotation. We use external DNS, and thus cannot drop the annotation yet
I think what you're doing as a workaround is a valid approach . A better fix would be for the ingress controller you're using to migrate to properly use the v1 API for Ingress and IngressClass. Please consider filing a bug against that ingress controller or other component. We could also return a header when someone sets the annotation on an Ingress that already has set, to make it clear that this is not recommended.
It seems that PR got stuck , but is not a kubernets bug
IMO we should probably not hard-fail if these are both set, but a warning would be appropriate.

, i'd completely agree. Kubernetes is happy to have both set side by side, but the logic linked in the first comment blocks it. Seemingly needlessly.
Discussed at sig-net. We don't object to loosening this rule - an API warning seems sufficient. PRs welcome /triage accepted
Hi , I'm willing to help with this issue. I'm not familiar with , I read the relevant code and I think it seems that to add an API warning, I just need to implement for in . Am I right? And in the same time, don't check these two fields in validation?
Pretty much! See how Service works (though you probably don't need to put it into a different package, not sure why we did that..) kidddddddddddddddddddddd <
/assign


Question: } r, _, _ := hc.stabilizeRecommendationWithBehaviors(arg) assert.Equal(t, tc.expectedStabilizedReplicas, r, "expected replicas do not match") <del> if !assert.Len(t, hc.recommendations[tc.key], len(tc.expectedRecommendations), "stored recommendations differ in length") { return } for i, r := range hc.recommendations[tc.key] { expectedRecommendation := tc.expectedRecommendations[i] assert.Equal(t, expectedRecommendation.recommendation, r.recommendation, "stored recommendation differs at position %d", i) </del> <ins> if tc.expectedRecommendations != nil { if !assert.Len(t, hc.recommendations[tc.key], len(tc.expectedRecommendations), "stored recommendations differ in length") { return } for i, r := range hc.recommendations[tc.key] { expectedRecommendation := tc.expectedRecommendations[i] assert.Equal(t, expectedRecommendation.recommendation, r.recommendation, "stored recommendation differs at position %d", i) } </ins> } }) }

When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement.

Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)


Question: requireMapListKeysMapSetValidation bool // preexistingExpressions tracks which CEL expressions existed in an object before an update. May be nil for create. preexistingExpressions preexistingExpressions <ins> // versionsWithUnchangedSchemas tracks schemas of which versions are unchanged when updating a CRD. // Does not apply to creation or deletion. // Some checks use this to avoid rejecting previously accepted versions due to a control plane upgrade/downgrade. versionsWithUnchangedSchemas sets.Set[string] // suppressPerExpressionCost indicates whether CEL per-expression cost limit should be suppressed. // It will be automatically set during Versions validation if the version is in versionsWithUnchangedSchemas. suppressPerExpressionCost bool </ins> celEnvironmentSet *environment.EnvSet }

We were using cel-go v0.12.7 in Kubernetes 1.27 release and cel-go v0.16.0 in Kubernetes 1.28 release. There are many changes from cel-go side between those two releases and some may break backward compatible of CRD validation rules. An example would be the estimated cost for CRD validation rules(Thanks to who brought this to our attention with the following example). In the example CRD validation rules below: It will pass perfect in Kubernetes 1.27 version but fail in Kubernetes 1.28 with error: This is due to the estimator fix happened in cel-go(, ) The changes should not cause backward incompatible of CRD validation rules. OR a stability test suite should be in place to prevent the changes causing backward compatible issue before GA. Apply the above CRD in Kubernetes 1.27 and 1.28, 1.27 cluster should pass and 1.28 cluster should fail. Since the fix in cel-go side makes sense to me and the feature is in Beta now, I am not quite sure how strict we should be on backward compatible. cc As a followup, a stability test suite for estimate cost is worth adding before GA to make sure any future changes be caught before the feature promoted to GA. <details</details<details</details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hi I agree that the changes in cel-go should not break the backward compatibility of CRD validation rules, especially for users who have already defined and deployed their custom resources using the previous version of Kubernetes. One suggestion I have is to use a version flag for the CEL library, so that users can specify which version of cel-go they want to use for their CRD validation rules. This way, they can avoid unexpected errors or behavior changes when upgrading their Kubernetes cluster.

For example, they can add something like in their CRD schema to indicate that they want to use the old version of cel-go.
Another suggestion I have is to use a cost limit parameter for the CEL library, so that users can adjust the budget for their CRD validation rules. This way, they can fine-tune the trade-off between performance and complexity of their rules. For example, they can add something like in their CRD schema to indicate that they want to allow more expensive rules.
I agree this is the type of bug fix that is legitimate to make during beta. when the size changes came in, did our existing size estimation test coverage notice this cost increased? If not, is there a systemic change we could make to exercise each type of estimation combination so we have visibility to any future changes. Existing persisted CRD objects should continue to work as-is on upgrade, but the CEL rules and/or schema would be required to fit within fixed cost estimates before making future updates/changes to the CEL rules there should be unit tests ensuring updates to a CRD that don't change CEL rules are allowed even if the existing CEL rules exceed cost estimates, but can you verify that is the case? That's not workable, both from an implementation perspective and from a fragmentation perspective. We explicitly don't want to allow opting into loopholes / bugs in cost estimation that allow CRD authors to make API servers do excessive amounts of work. Similar to above, the responsibility for API server resource consumption lies with the kube-apiserver admin, not the CRD author, so exposing this in the API is not something I would expect.
/assign
Thank you! I guess the next step would be: the existing test suite regarding with the cel cost evaluation. Add estimated cost tests for every single operation, function, type, etc if it's not existing now. updates to a CRD that don't change CEL rules are allowed even if the existing CEL rules exceed cost estimates. Currently it is not working like this but would definitely help here. cc , please correct me if I'm wrong :) Those should be done before the feature goes to GA. I am not sure about the policy of a GA feature relying on alpha/beta feature is acceptable or not. is going to Beta in this release. The worst case is we have to wait further until the feature is ready.

I don't think that helps... CRD ratcheting is about validation of the custom resources. This is about validation of the CRD itself.
thanks for clarifying! Are you suggesting while updating CRD, if the rules matching what's in the existing object, we should skip cost estimation check? That makes sense. I could start a PR if my understanding aligns :)
something along those lines, though now that I think about it, other changes in the schema could affect the number of times a CEL expression could be run (dropping or increasing maxItems, maxProperties, etc) we already collect preexisting expressions when updating a CRD in order to allow already-persisted expressions (possibly persisted via a newer API server) making use of new CEL features: I think that is what I was thinking of. I would probably focus on: coverage of the edges of the cost estimation so we don't pick up changes there without knowing coarse-grained skipping of schema/cel validation for a particular version in a CRD for updates which don't change anything under that version's schema (would let someone make unrelated updates to the CRD and leave an existing version's schema / CEL rules alone)
Thank you! We have those work planned for the current cycle. /assign /assign


Question: "fmt" "os" <ins> "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" </ins> v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework"

I am trying to run Azure e2e tests: The test creates a volume in the cloud + runs a pod with the volume as in-line. The pod never starts: The test creates a "blob" disk, while the Azure machine expects "managed" disk. Does anyone run storage e2e tests on Azure regularly? Was it ever working? Managed disks are provisioned . Should the test suite create "managed" disks by default too? Environment: Kubernetes version (use ): v1.17.0-alpha.0.1089+-dirty
/sig azure
/sig cloud-provider
managed disk should be the default, the azure disk storage class in the test case may have
The volume is created outside of kubernetes dynamic provisioning to test pre-provisioned and inline volumes: I wonder if anyone actually runs the tests.


Question: }) } } <ins> func Test_getDeprecatedTopologyNodeNames(t *testing.T) { testcases := []struct { name              string endpointSlice     *discovery.EndpointSlice expectedNodeNames sets.String }{ { name: "2 nodes", endpointSlice: &discovery.EndpointSlice{ Endpoints: []discovery.Endpoint{ {DeprecatedTopology: map[string]string{corev1.LabelHostname: "node-1"}}, {DeprecatedTopology: map[string]string{corev1.LabelHostname: "node-2"}}, }, }, expectedNodeNames: sets.NewString("node-1", "node-2"), }, { name: "duplicate values", endpointSlice: &discovery.EndpointSlice{ Endpoints: []discovery.Endpoint{ {DeprecatedTopology: map[string]string{corev1.LabelHostname: "node-1"}}, {DeprecatedTopology: map[string]string{corev1.LabelHostname: "node-3"}}, {DeprecatedTopology: map[string]string{corev1.LabelHostname: "node-3"}}, }, }, expectedNodeNames: sets.NewString("node-1", "node-3"), }, { name: "unset", endpointSlice: &discovery.EndpointSlice{ Endpoints: []discovery.Endpoint{ {DeprecatedTopology: map[string]string{"other": "value"}}, {DeprecatedTopology: map[string]string{"foo": "bar"}}, {DeprecatedTopology: nil}, }, }, expectedNodeNames: sets.NewString(), }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { actualNames := getDeprecatedTopologyNodeNames(tc.endpointSlice) if !tc.expectedNodeNames.Equal(actualNames) { t.Errorf("Expected %+v node names, got %+v", tc.expectedNodeNames, actualNames) } }) } } </ins>

On the first update of an EndpointSlice using the v1 API, we drop deprecatedTopology with EndpointSlice strategy. Although we only do this if at least 1 endpoint has been modified, there is no guarantee that node name has been populated for all endpoints within the EndpointSlice. This is most noticeable if the first update using the v1 API is only adding endpoint(s). In that case the EndpointSlice controller does not update EndpointSlices and instead just adds the new endpoint(s). That will cause node names to be dropped from the deprecated topology field without having the newer node name field populated. Node name to be retained in deprecated topology until the new field was populated. an EndpointSlice with the v1beta1 API. an endpoint in that EndpointSlice: No response This is most notable after upgrading a 1.20 cluster to 1.21, but can technically occur in any 1.21 or newer cluster. Any Any Any N/A N/A
I'm working on a fix for this, hoping to have it out later today. /triage accepted /sig network /cc
I think this is better characterized as a controller bug, but is the safest mitigation I can think of in the short term, since we don't want to rewrite 100% of endpointslices on startup. Can you open a separate issue to work through the controller reconciliation so that v1 clients can depend on the nodeName field more reliably?


Question: endpointsHandler.ValidateRequestCount(t, 1) endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data) } <ins> func TestSyncEndpointsItemsExcludeNotReadyPodsWithRestartPolicyNeverAndPhaseFailed(t *testing.T) { ns := "other" testServer, endpointsHandler := makeTestServer(t, ns) defer testServer.Close() endpoints := newController(testServer.URL) endpoints.endpointsStore.Add(&v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", Labels: map[string]string{ "foo": "bar", }, }, Subsets: []v1.EndpointSubset{}, }) addNotReadyPodsWithSpecifiedRestartPolicyAndPhase(endpoints.podStore, ns, 1, 1, v1.RestartPolicyNever, v1.PodFailed) endpoints.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ Selector: map[string]string{"foo": "bar"}, Ports:    []v1.ServicePort{{Port: 80, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}}, }, }) endpoints.syncService(ns + "/foo") data := runtime.EncodeOrDie(testapi.Default.Codec(), &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", }, Subsets: []v1.EndpointSubset{}, }) endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data) } func TestSyncEndpointsItemsExcludeNotReadyPodsWithRestartPolicyNeverAndPhaseSucceeded(t *testing.T) { ns := "other" testServer, endpointsHandler := makeTestServer(t, ns) defer testServer.Close() endpoints := newController(testServer.URL) endpoints.endpointsStore.Add(&v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", Labels: map[string]string{ "foo": "bar", }, }, Subsets: []v1.EndpointSubset{}, }) addNotReadyPodsWithSpecifiedRestartPolicyAndPhase(endpoints.podStore, ns, 1, 1, v1.RestartPolicyNever, v1.PodSucceeded) endpoints.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ Selector: map[string]string{"foo": "bar"}, Ports:    []v1.ServicePort{{Port: 80, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}}, }, }) endpoints.syncService(ns + "/foo") data := runtime.EncodeOrDie(testapi.Default.Codec(), &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", }, Subsets: []v1.EndpointSubset{}, }) endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data) } func TestSyncEndpointsItemsExcludeNotReadyPodsWithRestartPolicyOnFailureAndPhaseSucceeded(t *testing.T) { ns := "other" testServer, endpointsHandler := makeTestServer(t, ns) defer testServer.Close() endpoints := newController(testServer.URL) endpoints.endpointsStore.Add(&v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", Labels: map[string]string{ "foo": "bar", }, }, Subsets: []v1.EndpointSubset{}, }) addNotReadyPodsWithSpecifiedRestartPolicyAndPhase(endpoints.podStore, ns, 1, 1, v1.RestartPolicyOnFailure, v1.PodSucceeded) endpoints.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ Selector: map[string]string{"foo": "bar"}, Ports:    []v1.ServicePort{{Port: 80, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}}, }, }) endpoints.syncService(ns + "/foo") data := runtime.EncodeOrDie(testapi.Default.Codec(), &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name:            "foo", Namespace:       ns, ResourceVersion: "1", }, Subsets: []v1.EndpointSubset{}, }) endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data) } // There are 3*5 possibilities(3 types of RestartPolicy by 5 types of PodPhase). Not list them all here. // Just list all of the 3 false cases and 3 of the 12 true cases. func TestShouldPodBeInEndpoints(t *testing.T) { testCases := []struct { name     string pod      *v1.Pod expected bool }{ // Pod should not be in endpoints cases: { name: "Failed pod with Never RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, }, Status: v1.PodStatus{ Phase: v1.PodFailed, }, }, expected: false, }, { name: "Succeeded pod with Never RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, }, Status: v1.PodStatus{ Phase: v1.PodSucceeded, }, }, expected: false, }, { name: "Succeeded pod with OnFailure RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyOnFailure, }, Status: v1.PodStatus{ Phase: v1.PodSucceeded, }, }, expected: false, }, // Pod should be in endpoints cases: { name: "Failed pod with Always RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyAlways, }, Status: v1.PodStatus{ Phase: v1.PodFailed, }, }, expected: true, }, { name: "Pending pod with Never RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, }, Status: v1.PodStatus{ Phase: v1.PodPending, }, }, expected: true, }, { name: "Unknown pod with OnFailure RestartPolicy", pod: &v1.Pod{ Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyOnFailure, }, Status: v1.PodStatus{ Phase: v1.PodUnknown, }, }, expected: true, }, } for _, test := range testCases { result := shouldPodBeInEndpoints(test.pod) if result != test.expected { t.Errorf("%s: expected : %t, got: %t", test.name, test.expected, result) } } } </ins>

If you create a job in which pods are crashing and a service spawning those pods, this may result in generating extremely huge Endpoints object (which will cause different performance issues). [Based on situation from one of our customers cluster]. After 30m from creating the objects below in my 3 node-cluster I have an endpoint which has almost 1MB of size: Example objects which are enough to cause this : :
- I think that we want to have "failed pods" that have "restart policy" "Always" to be part of "NotReadyAddresses". But do we want to keep pod that are in Failed state and have restart policy "Never"? I think it doesn't make sense. And this would be a quick and easy fix to this particular problem...
Can you update the issue title to "Failed pods with restart policy=Never should not be in the Endpoints.NotReadyAddresses"
done Do you agree that this is a reasonable request?
Geniunely interested: in what case do you want a service and a job? cc:
I can imagine a map-reduce-like thing that serves status from each worker, and uses a headless Service for discovery.
Yes - this is the example I've heard during discussions on last Kubecon.
Well - I don't know if it was map-reduce-like-thing, but it was about headless Service for discovery (and also reporting status from workers).
/assign
GitHub didn't allow me to assign the following users: xiangpengzhao. Note that only can be assigned. <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
We should also not put such pods in the Endpoints.NotReadyAddresses: Succeeded pods with restart policy=Never Succeeded pods with restart policy=OnFailure
- thanks for working on it. I commented on the PR.


Question: serviceName := "nodeport-update-service" ns := f.Namespace.Name jig := e2eservice.NewTestJig(cs, ns, serviceName) <ins> jig.ExternalIPs = true </ins> ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns) tcpService, err := jig.CreateTCPService(nil)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We ran sonobuoy on one of our clusters running on AWS and it reported two failed tests: This happened because the pod employed to conduct the tests wasn't able to reach the service using the node's public IP address: What you expected to happen: The security groups on that cluster don't allow for connections to arbitrary ports through the public IP address. The tests should still succeed because that configuration is seen as a security feature. How to reproduce it (as minimally and precisely as possible): Spin up a Kubernetes cluster on AWS EC2 instances, configure SGs to only allow traffic to the instances from other instances (or from an ELB). Run sonobuoy. Anything else we need to know?: Environment: AWS Kubernetes version (use ): 1.16.8 Cloud provider or hardware configuration: AWS OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/area conformance /sig network
related:
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
I am also seeing this issue when running conformance testing. The nodes in our test cluster (where we run conformance tests) have public IP addresses, but nearly all ports all firewalled. We are also running in AWS. These tests should only rely on the internal IP address of the nodes.
/assign
Yes, I have seen this before, that if the nodes have an external address it will use that one, which means the firewall must be open for that. If the nodes do not have an external IP then the test will pass just fine. It's not as much a bug as an inconvenient implementation. But we can treat it as a bug and fix it. The intent of the test is to make sure the service is plumbed properly.
/remove-triage unresolved
Good point. However, we only assign an external IP address to be able to SSH into the host and thus block all other ports.
Issues go stale after 90d of inactivity.

Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
I'm seeing the same issue, though I noticed that it happens only on one node in the whole cluster. Even after re-creating a cluster from scratch, the test always fails only on one specific node. However this node is identical to others, but the test succeed on rest of the nodes.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
In some user scenarios, the external ip nodeport is for access from APPs outside this cluster/zone. However, in most scenarios, external IP is just for SSH or special use(Ingress Gateway, etc.) Different users have different proposals. Why not disable e2e tests checking on external IP with a flag? Will send a PR for this.
That's not all that useful for those running conformance tests. Would be better to change it to always use the internal IP, we don't need the external access for conformance purposes.
/assign
agree, also most of the network tests now run from the cluster, no need to reach externalIPs please tag me in the PR
So kubernetes applications should not expect to be able to access them with an external node address? FWIW we also require upstream / public internet access in the DNS tests, so needing somewhat unrestricted networking to test the cluster behavior is not new.
I'm not extremely familiar with the conformance tests. Are the DNS tests you are referring to about being able to access public services from inside of the cluster (egress)? Because our problem is that our default Kubernetes deployment doesn't allow ingress other than SSH and we haven't seen any problems with the DNS tests so far when checking our product for conformance. That's why I like approach of disabling that specific test with a flag.

Maybe that could also be generalized to let all conformance tests know this is a locked-up cluster with basically no ingress. But that may be a matter of a subsequent PR/issue.
Disabling the tests with a flag is not really an acceptable approach for conformance. The whole point of the conformance program is to validate the Kubernetes implementation against a standard set of tests across all vendors etc. If we don't think access like this is a standard Kubernetes behavior applications should expect anymore, the tests should be removed from conformance and we can consider instead running them just to test the Kubernetes project but not in conformance by moving them to a new test. IMHO we usually only increase the scope of conformance behavior though, not decrease, and it's expected that some users will have more secure clusters that fail conformance (for example we test some privileged behaviors IIRC) ...
After reading comment, I agree that the test should just be removed. However, I'd like to first understand the reasoning behind it being there in the first place. Maybe there's a good reason that such "secured cluster" aren't deemed compliant?!
so, the next step is to review the conformance definition for nodeport and modify the test accordingly Node.Status.Addresses are not a great thing, bad semantics, ... this KEP and this PR summaries perfectly why I personally, would move ExternalIP from Conformance entirely until it has a clear definition
So we should remove it now and then add a new test if needed after a conclusion from the discussion in the KEP. Edited. I may misunderstand. A proposal is to move those external addr NodePort test from conformance to normal e2e test cases In Conformance Case, don't check external ip's NodePort. In e2e test case, check them.
My point is that Conformance folks (cc ) should decide if they want to consider ExternalIPs as part of the conformance definition, I pointed to some issues related to the topic here sig-network will keep testing externalIPs, are part of the API and we need to verify them, Conformance has a narrower scope
The fact that these tests are using ExternalIPs does not need to be in conformance and in fact really shouldn't be. But we still need the tests working against InternalIPs for conformance. It's more of an historical artifact that they use ExternalIPs than an intentional choice.

So, agree with that sig-network should test them, but we should remove the use of the external IPs from conformance, and leave in the tests but against the internal ips only. I resolved this, for example, in GKE conformance runs by using a private GKE cluster that does not give the nodes external IPs.


Question: `)) Expect(err.Error()).To(Equal("[kind not set, apiVersion not set]")) }) <ins> It("is fine with crd resource with List as a suffix kind name, which may not be a list of resources", func() { err := validator.ValidateBytes([]byte(` apiVersion: fake.com/v1 kind: FakeList metadata: name: fake spec: foo: bar `)) Expect(err).To(BeNil()) }) </ins> })

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: I've created a CRD with the name . There is no validation specified on the CRD object. Creating the CRD itself succeeds, but attempting to create CRD resources with of that type fails with a validation error. What you expected to happen: I expect to be able to create CRD resources of type without a validation error. How to reproduce it (as minimally and precisely as possible): A normal working CRD example, : Applying this succeeds as expected: Simply changing the name of the CRD kind to breaks however, : CRD creates, but the actual resource fails validation: If you supply the object does create correctly. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: Digitalocean VM cluster OS (e.g. from /etc/os-release): Server (Ubuntu 14.04.5) Client (Ubuntu 16.04.4) Kernel (e.g. ): Server (4.4.0-111-generic) Client (4.13.0-38-generic) Install tools: Others:
/sig api-machinery /area custom-resources
is a special suffix for name. When you create a object using , which will be render as instead of plain . That's why error occurs due to missing item list.
This sounds like a bug in , not a thing that should be "fixed" by restricting names.
I think the magic suffix is somewhat unexpected, but in the case that it doesn't change -- I would expect a warning or error when creating a CRD with that name. As for this specific behavior, I would consider it a bug -- there's no indication that the kubectl validation is against a different than the one explicitly specified in the resource, and confusingly disabling validation causes the operation to succeed normally (creating the explicitly defined resource , regardless of magic suffix).
cc
cc
PTAL . It is a kubectl bug.
Thanks! Fix works for me


Question: # must make sure the output from parallel runs is not mixed. To achieve this, # we spawn a subshell for each parallel process, redirecting the output to # separate files. <del> printf "%sn" "${@}" | xargs -I{} -n1 -P${KUBE_COVERPROCS}  </del> <ins> # cmd/libs/go2idl/generator is fragile when run under coverage, so ignore it for now. # see: https://github.com/kubernetes/kubernetes/issues/24967 printf "%sn" "${@}" | grep -v "cmd/libs/go2idl/generator"| xargs -I{} -n1 -P${KUBE_COVERPROCS}  </ins> bash -c "set -o pipefail; _pkg="{}"; _pkg_out=${_pkg////_};  go test ${goflags[@]:+${goflags[@]}}  ${KUBE_RACE}

I can reproduce it every time. and /cc
It show passes when doing this patch:
I do not know the detail, but it looks like the test actually report error at line 77 and not 78.
What go version are you using?
If I remember correctly, it could be reproduced when running with go1.5.3 and go1.6.1. To be more specific, the go test passes, and this issue only happened when enabled with .
Interesting. I guess code coverage changes the behavior of line reporting? I'm willing to take a PR to fix this!
the coverage tool rewrites source code. Seems like a bug that it changes line numbers in the files. Skipping it wasn't the right fix IMO, we should file a bug with them.
Closing due to age, reopen if this is re-creatable.


Question: <del> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Launches a container and verifies it can be reached. Assumes that # we're being called by hack/e2e-test.sh (we use some env vars it sets up). set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. : ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} : ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} : ${KUBE_CONFIG_FILE:="config-test.sh"} export KUBECTL KUBE_CONFIG_FILE source "${KUBE_ROOT}/cluster/kube-env.sh" source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" prepare-e2e if [[ "${KUBERNETES_PROVIDER}" != "gce" ]] && [[ "${KUBERNETES_PROVIDER}" != "gke" ]]; then echo "WARNING: Skipping certs.sh for cloud provider: ${KUBERNETES_PROVIDER}." exit 0 fi # Set KUBE_MASTER detect-master # IMPORTANT: there are upstream things that rely on these files. # Do *not* fix this test by changing this path, unless you _really_ know # what you are doing. for file in kubecfg.key kubecfg.crt ca.crt; do echo "Checking for ${file}" "${GCLOUD}" compute ssh --zone="${ZONE}" "${KUBE_MASTER}" --command "ls /srv/kubernetes/${file}" done </del>

Part of umbrella issue . cc But this is my favorite shell test; it's so fast!


Question: framework.ExpectNoError(podErr) framework.Logf("completed pod") }) <ins> ginkgo.It("should run through the lifecycle of a ServiceAccount", func() { testNamespaceName := f.Namespace.Name testServiceAccountName := "testserviceaccount" testServiceAccountStaticLabels := map[string]string{"test-serviceaccount-static": "true"} testServiceAccountStaticLabelsFlat := "test-serviceaccount-static=true" ginkgo.By("creating a ServiceAccount") testServiceAccount := v1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name:   testServiceAccountName, Labels: testServiceAccountStaticLabels, }, } _, err := f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Create(context.TODO(), &testServiceAccount, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create a ServiceAccount") ginkgo.By("watching for the ServiceAccount to be added") resourceWatchTimeoutSeconds := int64(180) resourceWatch, err := f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Watch(context.TODO(), metav1.ListOptions{LabelSelector: testServiceAccountStaticLabelsFlat, TimeoutSeconds: &resourceWatchTimeoutSeconds}) if err != nil { fmt.Println(err, "failed to setup watch on newly created ServiceAccount") return } resourceWatchChan := resourceWatch.ResultChan() eventFound := false for watchEvent := range resourceWatchChan { if watchEvent.Type == watch.Added { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find %v event", watch.Added) ginkgo.By("patching the ServiceAccount") boolFalse := false testServiceAccountPatchData, err := json.Marshal(v1.ServiceAccount{ AutomountServiceAccountToken: &boolFalse, }) framework.ExpectNoError(err, "failed to marshal JSON patch for the ServiceAccount") _, err = f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Patch(context.TODO(), testServiceAccountName, types.StrategicMergePatchType, []byte(testServiceAccountPatchData), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch the ServiceAccount") eventFound = false for watchEvent := range resourceWatchChan { if watchEvent.Type == watch.Modified { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find %v event", watch.Modified) ginkgo.By("finding ServiceAccount in list of all ServiceAccounts (by LabelSelector)") serviceAccountList, err := f.ClientSet.CoreV1().ServiceAccounts("").List(context.TODO(), metav1.ListOptions{LabelSelector: testServiceAccountStaticLabelsFlat}) framework.ExpectNoError(err, "failed to list ServiceAccounts by LabelSelector") foundServiceAccount := false for _, serviceAccountItem := range serviceAccountList.Items { if serviceAccountItem.ObjectMeta.Name == testServiceAccountName && serviceAccountItem.ObjectMeta.Namespace == testNamespaceName && *serviceAccountItem.AutomountServiceAccountToken == boolFalse { foundServiceAccount = true break } } framework.ExpectEqual(foundServiceAccount, true, "failed to find the created ServiceAccount") ginkgo.By("deleting the ServiceAccount") err = f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) framework.ExpectNoError(err, "failed to delete the ServiceAccount by Collection") eventFound = false for watchEvent := range resourceWatchChan { if watchEvent.Type == watch.Deleted { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find %v event", watch.Deleted) }) </ins> }) var reportLogsParser = regexp.MustCompile("([a-zA-Z0-9-_]*)=([a-zA-Z0-9-_]*)$")

[ ] Promotion created According to this APIsnoop query, there are still some remaining ServiceAccount endpoints which are untested. Create a ServiceAccount with a static label Create a Secret Patch the ServiceAccount with a new Label and a new Secret Get the ServiceAccount to ensure it's patched List all ServiceAccounts in all Namespaces find the ServiceAccount(1) ensure that the ServiceAccount is found and is patched Delete Namespaced ServiceAccount(1) via a Collection with a LabelSelector Delete the Secret Discover useragents: List endpoints hit by the test: Display endpoint coverage change: If a test with these calls gets merged, test coverage will go up by 3 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
Fine with moving forward You don't need to check the status of the secret as part of the test. In other places we check that the resource in question happens, we don't have to follow.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/retitle Write CoreV1ServiceAccount resource lifecycle test+promote - +3
/close close this Issue in favour of and place in In Progress Issues on org dashboard
Closing this issue. <detailsIn response to : /close close this Issue in favour of and place in In Progress Issues on org dashboard </details


Question: }) } } <ins> func TestEnvelopeLogging(t *testing.T) { klog.InitFlags(nil) flag.Set("v", "6") flag.Parse() testCases := []struct { desc     string ctx      context.Context wantLogs []string }{ { desc: "no request info in context", ctx:  testContext(t), wantLogs: []string{ `"encrypting content using envelope service" uid="UID" key="0123456789" group="" version="" resource="" subresource="" verb="" namespace="" name=""`, `"decrypting content using envelope service" uid="UID" key="0123456789" group="" version="" resource="" subresource="" verb="" namespace="" name=""`, }, }, { desc: "request info in context", ctx: genericapirequest.WithRequestInfo(testContext(t), &genericapirequest.RequestInfo{ APIGroup:    "awesome.bears.com", APIVersion:  "v1", Resource:    "pandas", Subresource: "status", Namespace:   "kube-system", Name:        "panda", Verb:        "update", }), wantLogs: []string{ `"encrypting content using envelope service" uid="UID" key="0123456789" group="awesome.bears.com" version="v1" resource="pandas" subresource="status" verb="update" namespace="kube-system" name="panda"`, `"decrypting content using envelope service" uid="UID" key="0123456789" group="awesome.bears.com" version="v1" resource="pandas" subresource="status" verb="update" namespace="kube-system" name="panda"`, }, }, } for _, tc := range testCases { tc := tc t.Run(tc.desc, func(t *testing.T) { var buf bytes.Buffer klog.SetOutput(&buf) klog.LogToStderr(false) defer klog.LogToStderr(true) envelopeService := newTestEnvelopeService() fakeClock := testingclock.NewFakeClock(time.Now()) envelopeTransformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, func(ctx context.Context) (string, error) { return "1", nil }, func(ctx context.Context) error { return nil }, aestransformer.NewGCMTransformer, 1*time.Second, fakeClock) dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) transformedData, err := envelopeTransformer.TransformToStorage(tc.ctx, originalText, dataCtx) if err != nil { t.Fatalf("envelopeTransformer: error while transforming data to storage: %v", err) } // advance the clock to trigger cache to expire, so we make a decrypt call that will log fakeClock.Step(2 * time.Second) _, _, err = envelopeTransformer.TransformFromStorage(tc.ctx, transformedData, dataCtx) if err != nil { t.Fatalf("could not decrypt Envelope transformer's encrypted data even once: %v", err) } klog.Flush() klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf capturedOutput := buf.String() // replace the uid with a constant to make the test output stable and assertable capturedOutput = regexp.MustCompile(`uid="[^"]+"`).ReplaceAllString(capturedOutput, `uid="UID"`) for _, wantLog := range tc.wantLogs { if !strings.Contains(capturedOutput, wantLog) { t.Errorf("expected log %q, got %q", wantLog, capturedOutput) } } }) } } </ins>

Add additional logging at level that associates UID with the metadata from the request. xref: /sig auth
/triage accepted
/milestone v1.26
Mind if I take a look at this? /assign
Hello I am wondering how's this coming along. Is there anything I can help with?
Sorry. I got a bit occupied with a different project.. Will have a PR opened for this for review tomorrow.. I am trying to find a way to inject the Verbs into log. But haven't found the right way to include that into the context.. Any suggestion on what can be done?
any suggestion on what is the best way to get the additional metadata for verbs into this log? Move that info across using ?
Hello , bug triage team here, just wanted to touch base with you for updates on this ticket. Thanks!
Thanks! This is being worked on and is still on track for .
There is a Pull request opened on this already as well. But Waiting on some clarification to see if that needs additional changes.
/reopen /assign /unassign
Reopened this issue. <detailsIn response to : /reopen /assign /unassign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/milestone v1.27
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
Hi Bug Triage Team for v1.27 here o/ Just wanted to touch base with you for updates on this ticket. Thanks!
I am not working on this anymore Carlos. This was re-assigned
This is tracked for . I've linked the PR.


Question: } } <del> apiPodStatus := kl.generatePodStatus(pod, podStatus) </del> <ins> apiPodStatus := kl.generateAPIPodStatus(pod, podStatus) </ins> // Record the time it takes for the pod to become running. existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID) if !ok || existingStatus.Phase == api.PodPending && apiPodStatus.Phase == api.PodRunning &&

We moved reason cache and status convert function from to by , , and . After this, we should also migrate related old unit test to kubelet level. For we should add test for: [x] Different image pull policy. [x] Pod creation failure. [x] Pod image failure. For we should add test for: [x] ContainerStatuses order. [x] Last termination state. [x] How reason cache will affect the PodPhase: [x] Pull image failure reason [x] Pod creation failure reason
Time to close this issue? :)
Not yet, The last 3 tests are in . :)


Question: all: push # Set this to the *next* version to prevent accidentally overwriting the existing image. <del> TAG = 1.5 </del> <ins> TAG = 1.4 </ins> webserver: webserver.go CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' ./webserver.go

It's failing on both GCE and GKE-CI. It appears to have broken somewhere around build number 2509/2510 which includes a likely candidate PR (). /cc
I was about to debug this one. I'll take it.
Root cause is almost definitely one of these: Summary secrets example pod (details) the permissions on python files. (details) the implementation of DeepCopy to use reflection instead of Gob encode/decode. (details) networking test (details) back to executable. (details) protected docker processes (-900) (details) proxySocket out of to (details) Commit by Fix secrets example pod The file was modified Commit by Fixing the permissions on python files. The file was modified cluster/juju/charms/trusty/kubernetes- The file was modified cluster/juju/return-node- The file was modified cluster/juju/charms/trusty/kubernetes- Commit by Changing the implementation of DeepCopy to use reflection instead of Gob encode/decode. The file was The file was modified The file was modified Commit by Fix networking test The file was modified contrib/for-tests/network- The file was modified contrib/for-tests/network- The file was modified The file was modified contrib/for-tests/network-tester/Makefile Commit by Changing back to executable. The file was modified Commit by OOM protected docker processes (-900) The file was modified The file was modified The file was modified Commit by Move proxySocket out of to Moves proxySocket out of to new in proxy package in order to start separating proxy logic and implementation and make proxier more manageable to review. The file was modified The file was
Pretty sure it's commit , so rolling back now. cc .
The above rollback solved that immediate problem, but now the test, while proceeding healthily, is timing out. More debugging or optimization required: Networking should function for intra-pod communication ..... [irrelevant stuff deleted] ..... INFO: Proxy status call returned in 4.861944ms INFO: Attempt 35/60: test still running INFO: About to make a proxy status call Build timed out (after 90 minutes). Marking the build as aborted. Build timed out (after 90 minutes). Marking the build as failed. Ran 40 of 46 Specs in 5069.444 seconds FAIL!

-- 18 Passed 5 Skipped Build was aborted Recording test results
I'm not entirely sure why, but the test is running normally again, completing consistently in 13 seconds. Closing this issue for now. !


Question: framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second)) }) <ins> It("should provision storage with mount options", func() { framework.SkipUnlessProviderIs("gce", "gke") test := storageClassTest{ "HDD PD on GCE/GKE", []string{"gce", "gke"}, "kubernetes.io/gce-pd", map[string]string{ "type": "pd-standard", }, "1Gi", "1Gi", func(volume *v1.PersistentVolume) error { return checkGCEPD(volume, "pd-standard") }, } class := newStorageClass(test, ns, "mountoptions") class.MountOptions = []string{"debug", "nouid32"} claim := newClaim(test, ns, "mountoptions") claim.Spec.StorageClassName = &class.Name testDynamicProvisioning(test, c, claim, class) }) </ins> // NOTE: Slow!  The test will wait up to 5 minutes (framework.ClaimProvisionTimeout) // when there is no regression. It("should not provision a volume in an unmanaged GCE zone. [Slow]", func() {

for 1.8. /assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig storage
Looking at the PR description, it seems that adds tests for ReclaimPolicy only. What about MountOptions?
those are coming also, I'll re-rename issue/point to another one
Since the feature is in 1.8, we need tests in 1.8. Adding this to 1.8 milestone.
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/storage: Issue will be escalated to these SIGs if needed. priority/important-soon: Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. kind/cleanup: Adding tests, refactoring, fixing old bugs. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details
This issue is a 1.8 blocker. The PRs for to fix it, and , are approved/LGTM'd and awaiting merge. This issue can be closed as soon as those merge.


Question: containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" <ins> kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" </ins> "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/cephfs"

All kubemark tests started failing around 11-04. https://k8s- https://k8s- The testing framework is not able to schedule pods. hollow kubelets has errors like: Finally (when the test already failed) it changes to Suspected commit range: ci-kubernetes-kubemark-500-gce.Overall 11-04 https://k8s- No response No response /sig scalability
/triage accepted
/assign
None of the commits in the range looks indeed suspicious (mostly some endpointslice and metrics changes).
I forgot that the commit in the testgrid doesn't match the tested version. The real suspected range is Probably is a suspect.
It helped !


Question: fi ###################### <ins> # Convert            # ###################### if kube::test::if_supports_resource "${deployments}"; then record_command run_convert_tests fi ###################### </ins> # Delete             # ###################### if kube::test::if_supports_resource "${configmaps}" ; then

was removed in 1.20 in favor of a plugin. However, this is not included in release artifacts. The binary is included in release artifacts, and ideally easily installable via kubectl plugin managers like . /sig cli /assign
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This question came up in a docs review of a page referencing () Once this is resolved, please ensure the website is updated to document installation/use of this tool.
/sig release