Question: // RoundUpToGiB rounds up given quantity upto chunks of GiB func RoundUpToGiB(size resource.Quantity) (int64, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSize(requestBytes, GiB), nil </del> <ins> return roundUpSizeInt64(size, GiB) </ins> } // RoundUpToMB rounds up given quantity to chunks of MB func RoundUpToMB(size resource.Quantity) (int64, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSize(requestBytes, MB), nil </del> <ins> return roundUpSizeInt64(size, MB) </ins> } // RoundUpToMiB rounds up given quantity upto chunks of MiB func RoundUpToMiB(size resource.Quantity) (int64, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSize(requestBytes, MiB), nil </del> <ins> return roundUpSizeInt64(size, MiB) </ins> } // RoundUpToKB rounds up given quantity to chunks of KB func RoundUpToKB(size resource.Quantity) (int64, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSize(requestBytes, KB), nil </del> <ins> return roundUpSizeInt64(size, KB) </ins> } <del> // RoundUpToKiB rounds up given quantity upto chunks of KiB </del> <ins> // RoundUpToKiB rounds up given quantity to chunks of KiB </ins> func RoundUpToKiB(size resource.Quantity) (int64, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSize(requestBytes, KiB), nil </del> <ins> return roundUpSizeInt64(size, KiB) } // RoundUpToB rounds up given quantity to chunks of bytes func RoundUpToB(size resource.Quantity) (int64, error) { return roundUpSizeInt64(size, 1) </ins> } // RoundUpToGiBInt rounds up given quantity upto chunks of GiB. It returns an // int instead of an int64 and an error if there's overflow func RoundUpToGiBInt(size resource.Quantity) (int, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSizeInt(requestBytes, GiB) </del> <ins> return roundUpSizeInt(size, GiB) </ins> } // RoundUpToMBInt rounds up given quantity to chunks of MB. It returns an // int instead of an int64 and an error if there's overflow func RoundUpToMBInt(size resource.Quantity) (int, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSizeInt(requestBytes, MB) </del> <ins> return roundUpSizeInt(size, MB) </ins> } // RoundUpToMiBInt rounds up given quantity upto chunks of MiB. It returns an // int instead of an int64 and an error if there's overflow func RoundUpToMiBInt(size resource.Quantity) (int, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSizeInt(requestBytes, MiB) </del> <ins> return roundUpSizeInt(size, MiB) </ins> } // RoundUpToKBInt rounds up given quantity to chunks of KB. It returns an // int instead of an int64 and an error if there's overflow func RoundUpToKBInt(size resource.Quantity) (int, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSizeInt(requestBytes, KB) </del> <ins> return roundUpSizeInt(size, KB) </ins> } // RoundUpToKiBInt rounds up given quantity upto chunks of KiB. It returns an // int instead of an int64 and an error if there's overflow func RoundUpToKiBInt(size resource.Quantity) (int, error) { <del> requestBytes := size.Value() return roundUpSizeInt(requestBytes, KiB) </del> <ins> return roundUpSizeInt(size, KiB) </ins> } // RoundUpToGiBInt32 rounds up given quantity up to chunks of GiB. It returns an // int32 instead of an int64 and an error if there's overflow func RoundUpToGiBInt32(size resource.Quantity) (int32, error) { <del> requestBytes, ok := size.AsInt64() if !ok { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } return roundUpSizeInt32(requestBytes, GiB) </del> <ins> return roundUpSizeInt32(size, GiB) </ins> } // roundUpSizeInt calculates how many allocation units are needed to accommodate <del> // a volume of given size. It returns an int instead of an int64 and an error if // there's overflow func roundUpSizeInt(volumeSizeBytes int64, allocationUnitBytes int64) (int, error) { roundedUp := roundUpSize(volumeSizeBytes, allocationUnitBytes) roundedUpInt := int(roundedUp) if int64(roundedUpInt) != roundedUp { return 0, fmt.Errorf("capacity %v is too great, casting results in integer overflow", roundedUp) </del> <ins> // a volume of a given size. It returns an int and an error if there's overflow func roundUpSizeInt(size resource.Quantity, allocationUnitBytes int64) (int, error) { if bits.UintSize == 32 { res, err := roundUpSizeInt32(size, allocationUnitBytes) return int(res), err </ins> } <del> return roundedUpInt, nil </del> <ins> res, err := roundUpSizeInt64(size, allocationUnitBytes) return int(res), err </ins> } // roundUpSizeInt32 calculates how many allocation units are needed to accommodate <del> // a volume of given size. It returns an int32 instead of an int64 and an error if // there's overflow func roundUpSizeInt32(volumeSizeBytes int64, allocationUnitBytes int64) (int32, error) { roundedUp := roundUpSize(volumeSizeBytes, allocationUnitBytes) roundedUpInt32 := int32(roundedUp) if int64(roundedUpInt32) != roundedUp { return 0, fmt.Errorf("quantity %v is too great, overflows int32", roundedUp) </del> <ins> // a volume of a given size. It returns an int32 and an error if there's overflow func roundUpSizeInt32(size resource.Quantity, allocationUnitBytes int64) (int32, error) { roundedUpInt32, err := roundUpSizeInt64(size, allocationUnitBytes) if err != nil { return 0, err } if roundedUpInt32 > math.MaxInt32 { return 0, fmt.Errorf("quantity %s is too great, overflows int32", size.String()) </ins> } <del> return roundedUpInt32, nil </del> <ins> return int32(roundedUpInt32), nil </ins> } <del> // roundUpSize calculates how many allocation units are needed to accommodate // a volume of given size. E.g. when user wants 1500MiB volume, while AWS EBS // allocates volumes in gibibyte-sized chunks, // RoundUpSize(1500 * 1024*1024, 1024*1024*1024) returns '2' // (2 GiB is the smallest allocatable volume that can hold 1500MiB) func roundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 { </del> <ins> // roundUpSizeInt64 calculates how many allocation units are needed to accommodate // a volume of a given size. It returns an int64 and an error if there's overflow func roundUpSizeInt64(size resource.Quantity, allocationUnitBytes int64) (int64, error) { // Use CmpInt64() to find out if the value of "size" would overflow an // int64 and therefore have Value() return a wrong result. Then, retrieve // the value as int64 and perform the rounding. // It's not convenient to use AsScale() and related functions as they don't // support BinarySI format, nor can we use AsInt64() directly since it's // only implemented for int64 scaled numbers (int64Amount). // CmpInt64() actually returns 0 when comparing an amount bigger than MaxInt64. if size.CmpInt64(math.MaxInt64) >= 0 { return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) } volumeSizeBytes := size.Value() </ins> roundedUp := volumeSizeBytes / allocationUnitBytes if volumeSizeBytes%allocationUnitBytes > 0 { roundedUp++ } <del> return roundedUp </del> <ins> return roundedUp, nil </ins> }

After upgrading our clusters to Kubernetes 1.19, the provisioning of PersistentVolumes started failing with the following Events: The value matches the actual storage request of 3.2Gi. It doesn't seem special in any way, not too big, not too small, and it always worked until 1.19. We look into the issue with and we believe the PR introduced a change that broke the PersistentVolume provisioning even when it doesn’t overflow int64. Amounts of resource in Kubernetes are represented with the . In the PersistentVolume drivers, there are some functions that manipulate storage request in order to allocate a disk with the right size. Before the mentioned PR, those quantities were retrieved by calling which relies on to return the value as an int64. This worked probably fine in most of the cases, but it could happen that the represented value would overflow an int64, and in this case the result of would be wrong. The function’s description states that it’s the caller’s responsibility to check this: This is likely what the PR wanted to handle. The change replaced all calls with which has a different signature. From where it’s used now, you would think that the second return value signals an overflow: : This is probably where the error we’re seeing is generated. I believe that the signature of was misinterpreted. From my understanding of its description, returning doesn’t mean we’re overflowing . It only means that internally, the quantity is represented as a decimal amount: Internally, depending on their values, quantities are stored/represented in : for DecimalExponent (e.g. ) or DecimalSI (e.g. ) for BinarySI that don’t have a decimal part but a unit (simplified) (e.g. ) for all the rest With the PR, Kubernetes now fails to provision volumes that have a storage request represented as . Many of the storage requests are represented this way and still perfectly valid in our opinion. I would have expected the provisioning to continue to work as those requests seemed valid and were always accepted by our underlying cloud-provider in earlier version of Kubernetes.

Create a StatefulSet with a PersistentVolumeClaim that uses vSphere storage: Kubernetes version (use ): 1.19.7 Cloud provider or hardware configuration: vSphere, but I think it impacts other cloud-provider that were changed in the PR: /sig storage /kind bug /cc Possibly related to
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I think the only reason we do not use the quantity library for scaling and why we tried to roll our own rounding code is because when scaling stuff in AsScaledInt64(3) or ScaledValue(3), the library assumes decimal SI (M, 1000), not binary SI units (Mi, 1024): You are right, our self-rolled rounding code doesn't handle values with millibyte or higher precision like 3.2Gi. Our code uses AsInt64 and then performs its own scaling. If AsInt64 returns false, the int64 returned is not in bytes (it must be in millibytes or higher precision), so our rounding code won't work, so it returns that " is too great, overflows int64". In 1.18, we used Value() to round to bytes but we didn't check it against ScaleValue as the signature suggested. So the PR in question fixed cases where Value() overflowed but inadvertently broke your case. The correct way to round, which the resource library's rounding does, is to drop the useless precision and round up. , but as I said it doesn't seem to handle binary SI quantities correctly. For decimal SI quantities, we should rely on the resource library's AsScaledInt64 and be done with it. (We could also use Value() and ScaledValue as the signature suggests but calling AsScaledInt64 directly seems simpler). Like this: For binary SI quantities, we need to do something fancier involving CanonicalizeBytes I think, I'm not sure yet.
I guess for binary SI we can go back to using Value but actually check the overflow this time.

Hi Thanks for the quick feedback on the problem's description already. Indeed, the Scaled() functions don't handle BinarySI and that's why I quickly discarded the idea of using them. Since this is about storage, I suppose most people are using this format. I just finished wrapping up a PR that I hope address both the issue we are facing, but also what was meant to be fixed in the PR your reviewed back then. It doesn't make any difference between the 3 formats however.
Yeah I commented within seconds of your PR. I glanced at it , will review more soon, but it makes sense to me to just use VAlue and check ofr overflow for both decimal SI and binary SI rather than using Scaled() for one and not the other :+1:
I think it means both. i.e - when incoming value is decimal or if value is so big that type can't store it as it switches to decimal. So while my PR fixed later case, it broke the former. :/


Question: return nodes, nil } <ins> func (nc *NodeController) recordNodeEvent(node *api.Node, event string) { ref := &api.ObjectReference{ Kind:      "Node", Name:      node.Name, UID:       types.UID(node.Name), Namespace: "", } glog.V(2).Infof("Recording %s event message for node %s", event, node.Name) // TODO: This requires a transaction, either both node status is updated // and event is recorded or neither should happen, see issue #6055. nc.recorder.Eventf(ref, event, "Node %s is now %s", node.Name, event) } </ins> // For a given node checks its conditions and tries to update it. Returns grace period to which given node // is entitled, state of current and last observed Ready Condition, and an error if it ocured. func (nc *NodeController) tryUpdateNodeStatus(node *api.Node) (time.Duration, api.NodeCondition, *api.NodeCondition, error) {

/cc
Is this from head? I've never seen this before, maybe changed anything recently. Anyway, the code in question will be deleted by
I saw this in the Travis run for an unrelated PR (Kubelet test refactoring). If it's going away, then that's ho we'll fix it :)
Yup. It'll go away. I'll assign it to myself - I'll run test few times after to make sure it's gone.
another example: https://travis-
Just hit this again: https://travis-
I was hoping we could finish getting rid of this code this week, which is why I had been ignoring this. But we ran out of time and Poland has a holiday on Monday, so I guess we need to fix this race. I'll take a look.
Thanks. This is becoming a real nuisance that is masking other failures.
To repro I assume I just type the commands from Travis, e.g. the shell stuff in https://travis- ?
Never mind, I got it. Though the first time I ran it all the tests passed.
I don't think the failure is deterministic, but yes.
This is super annoying, it happens way too much of the time...
Sigh, I was hoping we can get rid of the code this week. I think the problem comes from someone changes the event recording. I'll send a PR
It seems to be basically always failing now, except when some other flake fails first.
It would be fine with me if we just temporarily disabled event generation there.
If I'm reading the stack traces correctly, what's going on is that NewNodeController [] calls StartRecordingToSink() [] which calls StartEventWatcher() which calls eventHandler() which calls StartRecordingToSink() which calls sink.Create() which is Create() in , which appends to c.Fake.Actions. At the same time, the last line of the test tries to print "item.fakeNodeHandler.Actions" and these are presumably the same variable. I'm not sure why these are running at the same time, though.
+1 to Braians idea to temporary disable event generation.
Re-titled issue.
Re-titled issue.


Question: func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (subnet network.Subnet, exists bool, err error) { var realErr error <ins> var rg string if len(az.VnetResourceGroup) > 0 { rg = az.VnetResourceGroup } else { rg = az.ResourceGroup } </ins> az.operationPollRateLimiter.Accept() glog.V(10).Infof("SubnetsClient.Get(%s): start", subnetName) <del> subnet, err = az.SubnetsClient.Get(az.ResourceGroup, virtualNetworkName, subnetName, "") </del> <ins> subnet, err = az.SubnetsClient.Get(rg, virtualNetworkName, subnetName, "") </ins> glog.V(10).Infof("SubnetsClient.Get(%s): end", subnetName) exists, realErr = checkResourceExistsFromError(err)

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: I've created a Kubernetes CLuster on Azure using acs-engine. In order to be able to communicate within a private network (Azure Vnet) between k8s resources and other Azure resources I have used a custom VNet configuration which allows to specify the Azure VNet resource you want k8s to be deployed into (which was already existing and had already some resources). When I tried to create an internal load balancer service (see following yaml) Kubernetes tells me it fails to get the subnet he wants to use for the Azure internal load balancer resource (it's the same subnet that is used by k8s masters nodes, so it does exist). What you expected to happen: Azure internal load balancer resource created How to reproduce it (as minimally and precisely as possible): Environment: Kubernetes version (use ): 1.6.6 Cloud provider or hardware configuration: Azure OS (e.g. from /etc/os-release): 16.04.2 LTS (Xenial Xerus) Kernel (e.g. ): Linux k8s-master--0 4.4.0-83-generic -Ubuntu SMP Mon Jun 26 17:54:43 UTC 2017 x8664 x8664 x86_64 GNU/Linux
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig azure /sig network
All right, I think I found the issue. K8s fails to find the subnet because in my case the VNet is not in the same resource group as the kubernetes. I'll try to make a patch to be able to configure the resource group of the VNet.
I still seem to be experiencing this problem using version 1.7.4; was it merged into the 1.7.x branch?
It has been, have you the vnetResourceGroup properties your the ?
I don't think so. I've provisioned the cluster using acs-engine.

Is the file just on the master node(s)?
definitely exists on both masters and agents, in /etc/kubernetes. Presumably both are used as masters and agents have different subnet configurations. I updated both to be safe. I've just experienced this issue myself on a fresh deployment to Azure using . Presumably acs-engine needs to be updated to populate the attribute from the attributes in the apimodel specification JSON. Specifically they can be found here: Does anyone know if this has been documented as an issue in acs-engine already?
We also encountered this issue. Can you provide an example about how to add into ?
Thanks for the reply. Can you provide more details? In our case, we use a json file like below to generate and by . After that, we can deploy the cluster by . Do you have any idea about this?
It looks like this was fixed in a PR to acs-engine last week: Thanks ! There hasn't been a binary release since then, so you'd have to clone the acs-engine repo and build from source to benefit from the fix. - as a workaround for a cluster that's already been deployed, you need to SSH to each master and worker and edit as suggested. Specifically:


Question: } } <ins> func TestScaleUpdateWithoutResourceVersion(t *testing.T) { storage, server := newStorage(t) defer server.Terminate(t) defer storage.CustomResource.Store.DestroyFunc() name := "foo" var cr unstructured.Unstructured ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) key := "/noxus/" + metav1.NamespaceDefault + "/" + name if err := storage.CustomResource.Storage.Create(ctx, key, &validCustomResource, &cr, 0, false); err != nil { t.Fatalf("error setting new custom resource (key: %s) %v: %v", key, validCustomResource, err) } replicas := int32(8) update := autoscalingv1.Scale{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: autoscalingv1.ScaleSpec{ Replicas: replicas, }, } if _, _, err := storage.Scale.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(&update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil { t.Fatalf("error updating scale %v: %v", update, err) } obj, err := storage.Scale.Get(ctx, name, &metav1.GetOptions{}) if err != nil { t.Fatalf("error fetching scale for %s: %v", name, err) } scale := obj.(*autoscalingv1.Scale) if scale.Spec.Replicas != replicas { t.Errorf("wrong replicas count: expected: %d got: %d", replicas, scale.Spec.Replicas) } } func TestScaleUpdateWithoutResourceVersionWithConflicts(t *testing.T) { storage, server := newStorage(t) defer server.Terminate(t) defer storage.CustomResource.Store.DestroyFunc() name := "foo" var cr unstructured.Unstructured ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) key := "/noxus/" + metav1.NamespaceDefault + "/" + name if err := storage.CustomResource.Storage.Create(ctx, key, &validCustomResource, &cr, 0, false); err != nil { t.Fatalf("error setting new custom resource (key: %s) %v: %v", key, validCustomResource, err) } fetchObject := func(name string) (*unstructured.Unstructured, error) { gotObj, err := storage.CustomResource.Get(ctx, name, &metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("error fetching custom resource %s: %v", name, err) } return gotObj.(*unstructured.Unstructured), nil } applyPatch := func(labelName, labelValue string) rest.TransformFunc { return func(_ context.Context, _, currentObject runtime.Object) (objToUpdate runtime.Object, patchErr error) { o := currentObject.(metav1.Object) o.SetLabels(map[string]string{ labelName: labelValue, }) return currentObject, nil } } errs := make(chan error, 1) rounds := 100 go func() { // continuously submits a patch that updates a label and verifies the label update was effective labelName := "timestamp" for i := 0; i < rounds; i++ { expectedLabelValue := fmt.Sprint(i) update, err := fetchObject(name) if err != nil { errs <- err return } setNestedField(update, expectedLabelValue, "metadata", "labels", labelName) if _, _, err := storage.CustomResource.Update(ctx, name, rest.DefaultUpdatedObjectInfo(nil, applyPatch(labelName, fmt.Sprint(i))), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil { errs <- fmt.Errorf("error updating custom resource label: %v", err) return } gotObj, err := fetchObject(name) if err != nil { errs <- err return } gotLabelValue, _, err := unstructured.NestedString(gotObj.Object, "metadata", "labels", labelName) if err != nil { errs <- fmt.Errorf("error getting label %s of custom resource %s: %v", labelName, name, err) return } if gotLabelValue != expectedLabelValue { errs <- fmt.Errorf("wrong label value: expected: %s, got: %s", expectedLabelValue, gotLabelValue) return } } }() replicas := int32(0) update := autoscalingv1.Scale{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } // continuously submits a scale update without a resourceVersion for a monotonically increasing replica value // and verifies the scale update was effective for i := 0; i < rounds; i++ { select { case err := <-errs: t.Fatal(err) default: replicas++ update.Spec.Replicas = replicas if _, _, err := storage.Scale.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(&update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil { t.Fatalf("error updating scale %v: %v", update, err) } obj, err := storage.Scale.Get(ctx, name, &metav1.GetOptions{}) if err != nil { t.Fatalf("error fetching scale for %s: %v", name, err) } scale := obj.(*autoscalingv1.Scale) if scale.Spec.Replicas != replicas { t.Errorf("wrong replicas count: expected: %d got: %d", replicas, scale.Spec.Replicas) } } } } func TestScaleUpdateWithResourceVersionWithConflicts(t *testing.T) { storage, server := newStorage(t) defer server.Terminate(t) defer storage.CustomResource.Store.DestroyFunc() name := "foo" var cr unstructured.Unstructured ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) key := "/noxus/" + metav1.NamespaceDefault + "/" + name if err := storage.CustomResource.Storage.Create(ctx, key, &validCustomResource, &cr, 0, false); err != nil { t.Fatalf("error setting new custom resource (key: %s) %v: %v", key, validCustomResource, err) } obj, err := storage.Scale.Get(ctx, name, &metav1.GetOptions{}) if err != nil { t.Fatalf("error fetching scale for %s: %v", name, err) } scale, ok := obj.(*autoscalingv1.Scale) if !ok { t.Fatalf("%v is not of the type autoscalingv1.Scale", scale) } replicas := int32(12) update := autoscalingv1.Scale{ ObjectMeta: scale.ObjectMeta, Spec: autoscalingv1.ScaleSpec{ Replicas: replicas, }, } update.ResourceVersion = "1" _, _, err = storage.Scale.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(&update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) if err == nil { t.Fatal("expecting an update conflict error") } if !errors.IsConflict(err) { t.Fatalf("unexpected error, expecting an update conflict but got %v", err) } } </ins> func setSpecReplicas(u *unstructured.Unstructured, replicas int64) { setNestedField(u, replicas, "spec", "replicas") }

What happened: Using a command such as: fails with the following error: This bug happens on each run and started occurring as of v1.15.0. With v1.14 or older, I don't have such problem, i.e. it works flawlessly. What you expected to happen: The command succeeds: How to reproduce it (as minimally and precisely as possible): Create a CRD that has the scale subresource and try to scale a CR using . Example of such CRD can be found . Anything else we need to know?: Regarding the bug, it seems that it started appearing as of . Before the PR, we always fetched the resource and it worked because it had the resource version, while now we are only fetching it in case preconditions are provided (preconditions are not provided by default/when using ). Here is the output of a command: If I manually craft the request that has the resource version (or use the flag), it works. Is it as expected that the Scale resource requires the resource version in order for update to succeed? Shouldn't it always pick the latest resource version in case it isn't provided? On that note, I also noticed that if you want to update a CR by crafting the request manually and using , you also need to specify the resource version or you will get the same error. I'm not sure if this is expected and related, but I'm writing it down as it might be related. As far as I know, this is not required for some native resources at all, but I'm not sure is this the case for CRDs. Environment: Kubernetes version (use ):
/sig api-machinery cc
/sub
could you try to see if it fix the problem?
I've created a test cluster based on and I can confirm that I'm able to scale CustomResources using as usual. I used an to test this, such as: Thanks a lot for your time spent fixing this!
Bug triage for 1.17 here with a gentle reminder that code freeze for this release is on November 14. Is this issue still intended for 1.17? I see you move it to 1.17 so I just wanted to be sure.
I guess could be merged now?

It is in the review queue and should still make 1.17
a kind reminder that code freeze for this release is on November 14.
I wanted to circle back around on this. We are 4 days out from code freeze. If this is still good based on just note here. Judging from the looks of it we are still good
Yes this should merge early in the week
Code freeze is tomorrow, this still going to make it?
yes, is still planned


Question: import ( "encoding/json" "fmt" <ins> "runtime" </ins> "time" "k8s.io/kubernetes/cmd/kubeadm/app/images"

Right now we create 3 daemonsets for in , but will only give use image names for , so this feature is broken. We should only deploy one daemon set for the native architecture, but ensure it has the right label selectors, so we don't try to run pods on nodes of different architecture. We should also make sure all addons have architecture labels, and not just . If user is building heterogeneous clusters, the will need to take about deploying addons for additional architectures, we shouldn't pre-deploy multiple things knowing that they will most likely remain unschedulable.
I have the same issue as well. I tried to create a cluster with x86 (Intel NUC) and arm (Raspberry-Pi) and I see that the kube-proxy is crashing. Here's the log entry from kube-proxy Logs from kube-proxy exec user process caused "exec format error" Attached is the kube-proxy yaml file. I see that its picking up arm image Don't know how to fix this.


Question: package gc import ( <ins> "fmt" </ins> "strings" "testing" <del> "k8s.io/apimachinery/pkg/api/meta/testrestmapper" </del> <ins> corev1 "k8s.io/api/core/v1" extensionv1beta1 "k8s.io/api/extensions/v1beta1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema"

/kind flake /sig testing /sig auth we are injecting a restMapper into this admisson controller while in its test, the restMapper somehow become empty when we remove internal clientset/informer dependency from it (). now we come up with a walkaround in which we manually init the legacy schema via a dry import so that the built-in resources can be installed. this failing test suggests that the test is not quite correct logically.
/assign Are you up for trying to figure out how to avoid requiring the internal types?
What test does this break? Links?
link here: w/o the dry import, then will fail with output:
sure. will be looking into it
OK, setting priority so that I know this isn't a 1.13 blocker. /priority important-longterm


Question: } } <ins> // isInvalidOrLocalhostAddress returns `true` if the provided `ip` is either not // parsable or the loopback address. Otherwise it will return `false`. func isInvalidOrLocalhostAddress(ip string) bool { parsedIP := net.ParseIP(ip) if parsedIP == nil || parsedIP.IsLoopback() { return true } return false } </ins> // testEndpointReachability tests reachability to endpoints (i.e. IP, ServiceName) and ports. Test request is initiated from specified execPod. // TCP and UDP protocol based service are supported at this moment // TODO: add support to test SCTP Protocol based services.

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: CRI-O e2e Which test(s) are failing: e2e suite [sig-network] Services should be able to change the type from ExternalName to NodePort e2e suite [sig-network] Services should be able to create a functioning NodePort service 19s See link: https://openshift-gce- Since when has it been failing: Anything else we need to know:
/sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/milestone v1.16 /assign FYI Looks like two other release-blocking tests too: - The changes link testgrid gives () suggests two other PR's, but I agree that seems the likely culprit
Based on the , Service is reachable on and but failing to reach on node's IP over nodePort i.e. . From the log, it is not clear why its not reachable on node's internal IP. Could you provide from the CRI-O based node when trying to reach nodePort service. Secondly, need to make sure, node is listening on allocated nodePort. As this issue seems much on system-level, I need to setup k8s cluster/node with CRI-O runtime and troubleshoot to find the root cause.
[sig-network] ESIPP [Slow] [DisabledForLargeClusters] should work from pods [sig-network] Services should only allow access from service loadbalancer source ranges [Slow] seems flaky to me. Most of time execPod starts running and might be terminating immediately after some time which causes this issue. [1.] e2e is failing as might have terminated and so no PodIP leading to invalid . [2.] empty . The same issue had been observed in for E2E and to fix the issue, we introduced execPod deployment to make sure execPod would remain in desired running state. It's better to address this change in another PR and emphasize using pausePod deployment.
Thanks, for helping me setting up CRI-O based cluster.

Kubernetes e2e suite [sig-network] Services should be able to change the type from ExternalName to NodePort e2e suite [sig-network] Services should be able to create a functioning NodePort service 19s From log, it is clear that, these services are failing to reach over allocated nodePort. But on my CRIO based testing cluster, they are reachable. or Could you please check if they failing at your end too or it's testgrid Job-specific issue?
Hey, I did some local tests and was able to reproduce the failure. I started the cluster via (check that the variable is correct and does not point to ): Executed the tests by (again, watch out for the variable to be correct): Error output: Since I'm now able to reproduce the issue I can go further in debugging and find out how to fix this. Is anyone furhter, or If not, then I will target for fixing the issue this week. :)
Thanks for reproducing this. FYI we've seem similar failure on , but I haven't reproduced the problem locally yet. We are looking to at least turn on verbose on the test command for debugging:
The issue is that we try to automatically discover the node IP by obtaining its status here: In our setup the node has IP which for sure does not work if we try to access it within the pod. I'm thinking about a proper solution and will propose it as a PR.
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
/assign


Question: {utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"}, {utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"}, {utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"}, <ins> {utiliptables.TableFilter, utiliptables.ChainOutput, kubeIPVSOutFilterChain, "kubernetes ipvs access filter"}, </ins> } var iptablesChains = []struct {

When using loadBalancerSourceRanges to allow access only from CIDRs outside the cluster segment, access from the K8s nodes (or pods with hostNetwork:true) should be rejected, but for proxy-mode=ipvs it works. /sig network /area kube-proxy /area ipvs Access from K8s nodes should be rejected in the same way as for proxy-mode=iptables Define a service with loadBalancerSourceRanges that allows access from a cidr outside the cluster, e.g. The K8s nodes have addresses on 192.168.1.0/24 and fd00::c0a8:100/120. Make sure a loadBalancerIP is assigned (in this example 10.0.0.0/32) and try to access it from a K8s node: Same for both ipv4 and ipv6. This is related to since a fix requires that the passed to the is correct. Please see v1.27.4 and master None Linux 6.4.4 None N/A N/A
wanted to work on this
Great! But please coordinate with I hope that one get accepted, and then ~the s (both families) passed to the proxiers should be usable.~ No, they are not, please see
Discussed this one in the SIG Network meeting today: we agree this is something we'd like to implement, however we weren't sure exactly how to do it with ipvs during the meeting. Whoever takes this on needs to come up with and communicate a clear plan of how it will be accomplished. /triage accepted
My comment is not feasible. For proxy-mode=ipvs the src and dst is both loadBalancerIP. So the nodeIP will not help :disappointed:
I did a small experimented with kind + metallb. I created a loadbalancer service backed by . loadBalancerIP: 172.18.255.200 podIP: 10.244.1.3 conntrack picks the src and dst both as loadBalancerIP. Tried to create an iptables rules matching source and destination with loadbalancerIP to drop packets but it didn't work. After some exploration I found and with the following rules I was able to prevent access to lbIP from k8s nodes / host network.

The solution looks hacky though, we only need this rule if source ranges are defined for the service and node ip is not covered by any of them. Will try to program this in the ipvs proxier.
Good find! :+1: I had missed the ipvs extension. I started to check nodeIP and the (flawed) option to experiment with setting a nodeIP as "src" for some routes to dodge the lbip-both-as-src-and-dst problem, and an old "vision" I have to not assign clusterIPs to at all, but have one rule to replace them all, like: It can be done, and in a cluster with 1000's of clusterIP's it would simplify


Question: {% endif %} <ins> {% if grains.cloud is defined %} {% if grains.cloud == 'gce' %} </ins> # Kubelet will run without this file but will not be able to send events to the apiserver. /var/lib/kubelet/kubernetes_auth: file.managed:

The kubelet.service EnvironmentFile /etc/sysconfig/kubelet doesn't exist on my minions.
Some extra information from master and minion-1:
I didn't think we used pki with salt. I wonder why salt is complaining about it. , David Reynolds wrote:
Is it possible this is related to salt's tar option having changed? A fix for that went in earlier today, but I don't know that we fixed vagrant yet...
Something else when I try to ./cluster/kube- This happens for each minion.
I think it's looking for eth0 in that ips list but isn't finding it?
since you're in the maintainers file for vagrant: any ideas?
- not yet, but I can take a look
- just to help debug, was this a fresh cluster you created that was working and then stopped after a push, or was this a fresh cluster that never worked?
It was a cluster that was working, I did a push and it broke. So I tore it down completely and rebuilt a fresh cluster that doesn't work either.
I am actually getting an issue where if I run more than one minion, my salt-minion is no longer able to contact the salt-master. So there is a number of things I need to look into.
So eth0 does not exist on a vagrant box for me, we look to need to use eth1 in a vagrant env.
appears to have introduced the regression on the vagrant env. is causing the issue.
Looking to see if I can specify which network interface to use for the function so we can safely use ips[0][0] when in an environment where the interface is not eth0
Just synced my branch, and found the issue is not limited to vagrant. When I ran hack/dev-build-kube-, I have the same kubeletservicecannotrestart issue due to lack of /var/lib/kubelet/kubernetesauth: But kubernetesauth is not installed: root ls /var/lib/kubelet/kubernetesauth ls: cannot access /var/lib/kubelet/kubernetes_auth: No such file or directory cc/
There are a number of issues, the master is not picking up the right network interface to use for the kube-apiserver to serve the readonly content, think we need a entry for address and publicAddressOverride per the arguments to kube-apiserver.

I don't have much more time to debug this tonight, but there are a number of issues we need to resolve here.
I ran kube- to recreate the cluster, and now /var/lib/kubelet/kubernetesauth is properly installed on nodes, and kubelet is succussfully started without any issue. Looks like we ran into saltcannotupgradeall_files issue we saw before.
- I do not see same behavior on a fresh cluster.
I am referring to gce, not vagrant. I think there are more issues on vagrant. But on gce, fresh cluster don't have issue.
I think I am close to a fix for vagrant.
Submitted a fix. Testing appreciated.
Bringing up a new cluster on vagrant. Will report back in a bit.
Works for me! Fresh cluster build
Works for me too. Awesome work. Thanks!
Derek, do you folks run vagrant e2e? If not maybe I should add that to our jenkins e2e box... --brendan , David Reynolds wrote:
Brendan, If you are willing to add it to your jenkins box that would be appreciated. -- Derek ----- Original Message ----- From: "Brendan Burns" To: "GoogleCloudPlatform/kubernetes" Cc: "Derek Carr" Sent: Wednesday, November 19, 2014 11:45:46 PM Subject: Re: [kubernetes] Kubelet not starting () Derek, do you folks run vagrant e2e? If not maybe I should add that to our jenkins e2e box... --brendan , David Reynolds wrote: Reply to this email directly or view it on GitHub:
The problem with upgrading the cluster is that it references a new file that doesn't exist if you are doing an upgrade. I'm going to check in a default version of that file to make this work and to smooth things over when the check fails.


Question: // TODO(vinaykul,InPlacePodVerticalScaling): Investigate doing this in HandlePodUpdates + periodic SyncLoop scan //     See: https://github.com/kubernetes/kubernetes/pull/102884#discussion_r663160060 if kl.podWorkers.CouldHaveRunningContainers(pod.UID) && !kubetypes.IsStaticPod(pod) { <del> kl.handlePodResourcesResize(pod) </del> <ins> pod = kl.handlePodResourcesResize(pod) </ins> } }

mutates the pod object in place (vs creating a copy first). That's illegal and can cause data races or potentially panics (the panic is unlikely here but I can't rule it out). This line should be: This should be fixed because we might serialize the object in the middle of this from somewhere else and if the object changes midway the protobuf object we create may not be valid, leading to panics in kubelets with this gate turned on. /sig node /priority critical-urgent /kind bug /assign No mutation of pod objects from within methods (in fact no mutation of should occur in most loops). N/A No response <details</details<details</details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
ACK. I'll fix it shortly.
is similar. is my try for both data races that we found.
One small , thanks.
Please review
This Issue/PR has been marked as release blocker. Please have the fix merged by eod today otherwise the feature has to be reverted. For the Issue/PR owner: please have a revert/feature disable PR open today in case the fix failed to get in. Thank you! See the -- Kubernetes 1.27 release manager


Question: Name:      "items_per_watch", Help:      "How many items an API watch returns to the reflectors", }, []string{"name"}) <ins> lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: reflectorSubsystem, Name:      "last_resource_version", Help:      "Last resource version seen for the reflectors", }, []string{"name"}) </ins> ) func init() {

I'm seeing a large number of unique prometheus series showing up (due to the logging we for anonymous reflectors). This is a small cluster but I believe this may have unbounded cardinality.
we storing the resource version in the metric name?
cc
Came into the same problem, checked the code, seems we use + + as metric names, for most reflector, they use a default name provided by which is go filename. I think the proper way is use as the metric name, and add a label, like
opened a pr to fix this, ptal thanks :)


Question: sched.finishBinding(logger, fwk, assumed, targetNode, status) }() <del> bound, err := sched.extendersBinding(assumed, targetNode) </del> <ins> bound, err := sched.extendersBinding(logger, assumed, targetNode) </ins> if bound { return framework.AsStatus(err) }

I use extenders to add webhook for default-scheduler: And set to ignore scheuler error when scheduler extender is not working. But I got an error: . I suppose work fine in predicate path, but not work fine in bind path. The Ignorable logic in kube-scheduler: 637: and, 905: the is not judge in . Schedule should not error when set . extenders to kube-scheduler config(which has filterVerb and bindVerb and ignorable flag), example: and add config flag in kube-scheduler pod start up command. a scheduler-extender() and then stop it. a pod and you will see the schedule error. No response <details</details<detailsLocal environment </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
/cc
/assign
Maybe this is not a bug but as designed, because bind is more critical than filter, what would you do in bind? I have no history context, cc
Hi, Kerthcet, in my scene, I use extender to bind IP(from my own network system) to my pod's annotaion. I'm aware of the critical of binding path, but, what If the extender is broken, how to skip it and continue? The scheduler extender design has supported ignorable, but only work for filter path make no sense for extender both has filter and bind. Maybe we can add an option to allow user to chose what can be ignored. Thanks for your reply anyway~
Reiterating the mentions to trigger a notification: <detailsIn response to : Maybe this is not a bug but as designed, because bind is more critical than filter, what would you do in bind? I have no history context, cc I'm aware of the critical of binding path, but, what If the extender is broken, how to skip it and continue?

The scheduler extender design has supported ignorable, but only work for filter path make no scence for extender both has filter and bind. Maybe we can add an option to allow user to chose what can be ignored. thanks for your reply anyway~ Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
So even the extender bind failed, will not impact your pod scheduling, right?
Yup
Right, if the field is there, we should support it. It's up to the cluster administrator to judge whether it makes sense to ignore a failed extender during bind.


Question: - name: lib-modules hostPath: path: /lib/modules <ins> tolerations: - key: CriticalAddonsOnly operator: Exists - key: {{ .MasterTaintKey }} effect: NoSchedule nodeSelector: beta.kubernetes.io/arch: {{ .Arch }} </ins> ` )

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: After joining a new ppc64le node to the k8s cluster, the kube-proxy DaemonSet fails on the new node. What you expected to happen: It was expected that the kube-proxy DS would "just work" as it does when adding an amd64 node to the cluster. How to reproduce it (as minimally and precisely as possible): Set up an amd64 based k8s cluster, then join a ppc64le node (or any non-amd64 arch) to the cluster. See the pod fail via 'kubectl -n kube-system get pods' because it's pulling the amd64 kube-proxy docker image instead of the ppc64le image. Anything else we need to know?: The yaml for the DaemonSet specifies the image: <versionIt would be preferable to have a manifest so that the image in the DS could be specified as <versionand could "just work" on any platform. See: Environment: Kubernetes version (use ): v1.10.3 Cloud provider or hardware configuration: self - cluster installed via kubeadm OS (e.g. from /etc/os-release): NAME="Ubuntu" VERSION="16.04.3 LTS (Xenial Xerus)" ID=ubuntu IDLIKE=debian PRETTYNAME="Ubuntu 16.04.3 LTS" VERSIONID="16.04" HOMEURL="" SUPPORTURL="" BUGREPORTURL="" VERSIONCODENAME=xenial UBUNTU_CODENAME=xenial Kernel (e.g. ): N/A Install tools: N/A Others: N/A
if you are running multi-architecture nodes, your daemonsets should specify a node selector to limit their arch-specific image to run on matching nodes
/close
I am well aware that a node selector can be used. That is what we are doing now. But it is a hack. The default should "just work". Adding a new non-amd64 node should not result in kube-proxy immediately failing on it because the default daemon set is amd64 arch specific. If kube-proxy is meant to "just work" clusterwide, a node selector should not be needed.

Please use the manifest-tool so that the default "just works". Please see the weave-net plugin. They are using the manifest-tool and it "just works" on amd64 and ppc64le. Their daemon set does not require/use a nodeSelector.
The current kube-proxy daemon set implementation is not properly architected, IMHO. Please re-open this
Reiterating the mentions to trigger a notification: <detailsIn response to : The current kube-proxy daemon set implementation is not properly architected, IMHO. Please re-open this </details
ah, this was a comment about the kube-proxy image, not the daemonset functionality /reopen
the kube-proxy addon daemonset is probably the multi-arch kube-proxy image to support that daemonset might be
Looks like it's coming from here: /assign /sig cluster-lifecycle Lucas, Do we want to avoid the references?
A dirty workaround for this is to pull and rename to . The kube-proxy is running as daemonset, created during . That means the kube-proxy image arch keeps consistent with your master node. Currently we don't have a multi-arch kube-proxy image yet. We have to explicitly specify the right name, like .
Hi .. Yes, we just created a separate daemonset with a ppc64le node slector that refers to the ppc64le image directly. But it's a bit of a hack. That is why I opened this ticket looking for a multi-arch image. Thanks!
I don't know that mixed-ISA clusters are supported. cc
The requirement is: only use "" in DaemonSet yaml file, and the k8s will download the correct image according to the ARCH info. It's more about image registry's feature; and it seems some registries had supported that: .
Yeah, has already supported multi-arch. But actually all the images now are architecture specific, like and . We've not built any multi-arch images right now.
for the test images we have the ball rolling here - - so if you can do the same (or similar) for other images, that would be great!
I have pending PR in releases project for creating manifest for those -
The manifest images created in kubernetes/release should work for future released versions, but will they be created during CI testing?

if you see one of the jobs that pushes images say you can grep for that's basically from the same files that got changed in so i fully expect the images pushed to CI will reflect the changes made in 516.
please confirm that i am reading this right ^ :)
I was more concerned with ci jobs that do not use cross builds, such as https://k8s-
please open an issue about pushing manifests in CI as well


Question: <ins> check process etcd with pidfile /var/run/etcd.pid group etcd start program = "/etc/init.d/etcd start" stop program = "/etc/init.d/etcd stop" if failed host 127.0.0.1 port 4001 protocol http request "/v2/keys/" then restart </ins>

We don't currently automatically restart etcd. We should.
Can we run etcd as a Pod without resource limit, so that we could benefit directly from 1) RestartPolicy: Always 2) resource monitoring and logging
When I bring up an e2e cluster (i.e. go run --up), without even running any of the e2e tests against it, it appears that monit is repeatedly killing and restarting etcd: davidopp more [UTC Feb 19 07:39:16] info : monit: generated unique Monit id and stored to '/var/lib/monit/id' [UTC Feb 19 07:39:16] info : Starting monit daemon [UTC Feb 19 07:39:16] error : State file '/var/lib/monit/state': Unable to read magic [UTC Feb 19 07:39:16] info : 'systeme2e-test-davidopp-master.c.striped-torus-760.internal' Monit started [UTC Feb 19 07:39:18] info : monit daemon with pid [5620] killed [UTC Feb 19 07:39:18] info : 'systeme2e-test-davidopp-master.c.striped-torus-760.internal' Monit stopped [UTC Feb 19 07:39:18] info : Starting monit daemon [UTC Feb 19 07:39:18] info : 'system_e2e-test-davidopp-master.c.striped-torus-760.internal' Monit started [UTC Feb 19 07:39:18] error : 'etcd' failed, cannot open a connection to INET[127.0.0.1:4001/v2/keys/] via TCP [UTC Feb 19 07:39:18] info : 'etcd' trying to restart [UTC Feb 19 07:39:18] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 07:39:18] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 07:41:18] error : 'etcd' failed, cannot open a connection to INET[127.0.0.

1:4001/v2/keys/] via TCP [UTC Feb 19 07:41:18] info : 'etcd' trying to restart [UTC Feb 19 07:41:18] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 07:41:18] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 07:43:18] error : 'etcd' failed, cannot open a connection to INET[127.0.0.1:4001/v2/keys/] via TCP [UTC Feb 19 07:43:18] info : 'etcd' trying to restart [UTC Feb 19 07:43:18] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 07:43:18] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 07:45:18] error : 'etcd' failed, cannot open a connection to INET[127.0.0.1:4001/v2/keys/] via TCP [UTC Feb 19 07:45:18] info : 'etcd' trying to restart [UTC Feb 19 07:45:18] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 07:45:18] info : 'etcd' start: /etc/init.d/etcd [.] I examined the etcd logs and there is nothing suspicious there. I suspect this may be the cause of some of the e2e test flakiness.
I think this particular problem is caused by firewall settings.
actually didn't fix the problem. Here's the [UTC Feb 19 23:22:25] info : monit: generated unique Monit id and stored to '/var/lib/monit/id' [UTC Feb 19 23:22:25] info : Starting monit daemon [UTC Feb 19 23:22:25] error : State file '/var/lib/monit/state': Unable to read magic [UTC Feb 19 23:22:25] info : 'systeme2e-test-davidopp-master.c.striped-torus-760.

internal' Monit started [UTC Feb 19 23:22:26] info : monit daemon with pid [5481] killed [UTC Feb 19 23:22:26] info : 'systeme2e-test-davidopp-master.c.striped-torus-760.internal' Monit stopped [UTC Feb 19 23:22:27] info : Starting monit daemon [UTC Feb 19 23:22:27] info : 'system_e2e-test-davidopp-master.c.striped-torus-760.internal' Monit started [UTC Feb 19 23:22:32] error : 'etcd' failed, cannot open a connection to INET[kubernetes-master:4001/v2/keys/] via TCP [UTC Feb 19 23:22:32] info : 'etcd' trying to restart [UTC Feb 19 23:22:32] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 23:22:32] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 23:24:37] error : 'etcd' failed, cannot open a connection to INET[kubernetes-master:4001/v2/keys/] via TCP [UTC Feb 19 23:24:37] info : 'etcd' trying to restart [UTC Feb 19 23:24:37] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 23:24:37] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 23:26:43] error : 'etcd' failed, cannot open a connection to INET[kubernetes-master:4001/v2/keys/] via TCP [UTC Feb 19 23:26:43] info : 'etcd' trying to restart [UTC Feb 19 23:26:43] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 23:26:43] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 23:28:48] error : 'etcd' failed,

cannot open a connection to INET[kubernetes-master:4001/v2/keys/] via TCP [UTC Feb 19 23:28:48] info : 'etcd' trying to restart [UTC Feb 19 23:28:48] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 23:28:48] info : 'etcd' start: /etc/init.d/etcd [UTC Feb 19 23:30:53] error : 'etcd' failed, cannot open a connection to INET[kubernetes-master:4001/v2/keys/] via TCP [UTC Feb 19 23:30:53] info : 'etcd' trying to restart [UTC Feb 19 23:30:53] info : 'etcd' stop: /etc/init.d/etcd [UTC Feb 19 23:30:53] info : 'etcd' start: /etc/init.d/etcd
The reason the fix didn't work is that it hard-codes the hostname as kubernetes-master but the hostname may actually be anything. For example, in e2e test the master hostname is e2e-test-${USER}-master
This is fixed by changing etcd to startup with -bind-addr 0.0.0.0:4001 I'm sending a PR now.
(And thanks to for figuring out the problem was that etcd wasn't listening on the localhost address.)


Question: # NOTE(claudiub): A non-default builder instance is needed in order to build Windows images. all-push: all-container-registry push-manifest <ins> push-manifest: SHELL:=/bin/bash </ins> push-manifest: docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)-&~g") set -x; for arch in $(ALL_ARCH.linux); do docker manifest annotate --os linux --arch $${arch} ${IMAGE}:${TAG} ${IMAGE}:${TAG}-linux-$${arch}; done # For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node can pull the proper image it needs. # we use awk to also trim the quotes around the OS version string. set -x;  <ins> # tagToKernelVersionMap maps the container images tags for different Windows Server releases (ex: ltsc2022 for Windows Server 2022) # to the kernel version for that OS release (ex: 20348 for Windows Server 2022). This is needed to fetch the servicing revision from the # pause base image manifest (which containers an entry for each Windows Server version) so we can add the approrite 'os.version' # field to the pause image manifest. declare -A tagToKernelVersionMap=( ['1809']='17763' ['20H2']='19042' ['ltsc2022']='20348' ); </ins> for arch in $(ALL_ARCH.windows);  do  for osversion in ${ALL_OSVERSIONS.windows}; do  <del> BASEIMAGE=${BASE.windows}:$${osversion};  full_version=`docker manifest inspect ${BASE.windows}:$${osversion} | grep "os.version" | head -n 1 | awk -F" '{print $$4}'` || true;  </del> <ins> full_version=`docker manifest inspect ${BASE.windows} | grep "10.0.$${tagToKernelVersionMap[$$osversion]}" | head -n 1 | awk -F" '{print $$4}'` || true;  </ins> docker manifest annotate --os windows --arch $${arch} --os-version $${full_version} ${IMAGE}:${TAG} ${IMAGE}:${TAG}-windows-$${arch}-$${osversion};  done;  done

The Windows pause images is built using buildkit on Linux hosts and this configuration does not allow for the Windows registry to me modified during the container build process. When Windows containers are built on Windows host machines part of the build process creates 'delta' registry files when get stored in the layer files. These 'delta' registry hive files are applied to the containers active registry when the containers are created. Chaning the registry files in the container image layers without having the 'delta' files present will result in the changes NOT being present in the container image. We have an increasing demand for the ability to modify registry in the pause image on Windows such as Disable some services like DiagTrack that negatively impact performance Change behavior of RPCSS so it doesn't hijack RPC port (135) for all containers in the pod Things are working as expected given the current build process. There are linux utilities like that can modify Windows registry files that I have used to test this out. After running the container produced from these files you can see the registry changes made by persisted in the container image but these changes are not present in the container's active registry. vs One possible solution is to create an intermediate image based on (the current base image for the Windows pause image) that performs all the desired registry operations that is built on a Windows machine and published someplace (MCR maybe?) and the use that as the container base image for the Windows pause image. <detailsN/A </details<detailsN/A </details<detailsN/A </details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
cc
my first concern was. will a community member be able to build this image themselves (will the sources be available and up-to-date and the image built from it). We could also do a github hosted windows runner to build / publish this new image as well. let's please revisit this once we wrap 1.24.

/milestone v1.25
We created which has the source/docker files used to build . AKS has made a pause image based on image available to some customers out of necessity to unblock some scenarios. We'd like to upstream the changes - and do so with as little destabilization to Kubernetes project as possible.
Seems like the intermediate base image step would be reasonable. I don't think community members can build the existing base windows images either for the same reason of depending on windows?
The windows pause image is currently based on 'nanoserver' which is one of several container base images provided by Microsoft (and only Microsoft can produce container base images).
yep. i'd expect community downstream teams (for example) would like to start with this nanoserver, have a set of dockerfiles to build this new image ... then build their own pause images based on this new image
or in other words if MSFT wants to roll all the changes into nanoserver itself (since we all already know that only MSFT can produce this), then there is no debate after that
We can't roll these changes into nanoserver because they may break other scenarios. Because of that - having an intermediate layer built somewhere was the best solution we came up with without forking the build process for the pause image. Microsoft is willing to host build/host the intermediate layers/images from open-sourced docker files. If we want to build the intermediate layers somewhere else too (like github hosted Windows runners) that would be fine too.
/sig windows
If this intermediate image used only by k8s then it is a good idea to build the image via GH actions in the kubernetes org so that k8s maintainers have better control on this or if this image is going to be generic for anyone to use then MSFT can build one and host it somewhere.
Picking this back up since 1.24 release is behind everyone. - Do you think a separate github repo for the dockerfile and github actions staging/prod container repos would be appropriate here? If so I can get started on creating these. Also - Is it possible to push to the staging repos from a github action job instead of PROW jobs? We'd need to use github actions to build the Windows image on a Windows machine.

I supposed we can store the .tar files as release artifacts if needed and then use a PROW to job push them to the staging repo if needed but being able to push for the action would be simpler.
/cc as FYI
I see two possible paths forward Continue to host the source for the intermediate layer at , publish the image to , and then update the Windows pause build docker files in k/k to consume this image instead of the or Create a new kubernetes-sigs repo to host the source and build steps for the image, publish the image to , and then update the Windows pause build docker files in k/k to consume this image. I'm fine with either approach but am not sure who the right folks to discuss this with are.
This looks more promising because this image you are creating is not a specific change for the pause container, IMO if we can change that image name to something generic and use the same wherever required.
I think those both sound reasonable personally. I think it's technically feasible to push from GitHub actions to staging GCR with workload identity but nobody has done this yet in Kubernetes.


Question: return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath) } device := mntPath[(ind + 1):] <del> // strip -lun- from device path ind = strings.LastIndex(device, "-lun-") </del> <ins> // strip -lun- from mount path ind = strings.LastIndex(mntPath, "-lun-") </ins> if ind < 0 { return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath) } <del> prefix := device[:ind] </del> <ins> prefix := mntPath[:ind] </ins> return device, prefix, nil } <ins> func extractIface(mntPath string) (string, bool) { re := regexp.MustCompile(`.+/iface-([^/]+)/.+`) re_output := re.FindStringSubmatch(mntPath) if re_output != nil { return re_output[1], true } return "", false } </ins> func extractPortalAndIqn(device string) (string, string, error) { ind1 := strings.Index(device, "-") if ind1 < 0 {

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): iscsi session logout Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): Environment: Cloud provider or hardware configuration: Libvirt qemu VM, 4 CPU, 8Gig RAM OS (e.g. from /etc/os-release): Ubuntu 16.04.1 LTS (Xenial Xerus) Kernel (e.g. ): Linux localhost 4.4.0-62-generic -Ubuntu SMP Wed Jan 18 14:10:15 UTC 2017 x8664 x8664 x86_64 GNU/Linux Install tools: kubeadm Others: What happened: If you have multiple pods on the same node using PVCs that are backed by iSCSI LUNs on the same external system over a single iSCSI session (same IQN but different LUN numbers), then deleting one of the pods causes the iSCSI session to terminate for all pods on that node that are using it. For example, here and are nginx pods each connected to a different PVC, which are backed by LUNs residing on the same NetApp storage system. These share a single iSCSI session, differing only by LUN number. Once pod2 is deleted, all the connected LUNs are no longer available.

[6:0:0:1] disk NETAPP LUN C-Mode 8300 /dev/sdc What you expected to happen: I expect that in-use iSCSI sessions are not terminated How to reproduce it (as minimally and precisely as possible): 2 PVs and PVCs that are backed by a single storage system that aggrates iSCSI connections into a single session. 2 pods that consume those PVCs one of the pods Anything else we need to know:
Could someone from please take a look? Thanks!
looking...
It looks like the problem is in line 86 of , in the function. The check evaluates whether a given mount path starts with the prefix (protocol + target portal + target IQN) for the LUN; this will always evaluate to false in most set-ups, since each path will look something like . Changing to appears to result in the correct behavior.
The problem is that the mount device shown in /proc/mounts are canonicalized, and thus when searching device prefix, iscsi volume failed to find a match. If we can use mount option, this the device will show up in /proc/mounts. But this is not a reliable fix since there are probable mpio paths too.
If I'm reading the code correctly, the function doesn't look at the actual device that's backing the mount point; it looks at the mount path (i.e., the second token in /proc/mounts, rather than the first). Thus, the change I proposed (HasPrefix -Contains) appears to work with some very basic testing. That said, I've not done anything with multipathing, so there may be something I'm missing; in addition, I suppose there's a chance that Contains may trigger false positives.
I see your point, yes that sounds right to me.
I think in this case, mpio device should work too (since the mount path constructed in the same way). I see you have yet another iscsi issue open, I am good with rescan. Can you create a pr to fix them?
I should be able create a PR for this issue; the LUN rescan PR will probably get handled by someone else on my team. Is the fix itself sufficient, or does it require tests?
thanks, the rescan fix doesn't have to unit test. The mount path fix needs also a unit test fix here

BUG REPORT Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-01-12T04:57:25Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-01-12T04:52:34Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"} Environment: Cloud provider or hardware configuration: Bare metal OS (e.g. from /etc/os-release): CentOS 7; reproduced in Ubuntu 14.0.4.2. Kernel (e.g. ): Linux centos7 3.10.0-514.6.1.el7.x8664 SMP Wed Jan 18 13:06:36 UTC 2017 x8664 x8664 x8664 GNU/Linux; kernel 3.16.0-77-generic What happened: On a host where an iSCSI LUN from a given target is mounted in a pod, a user who creates a new LUN on the same target and attempts to create a pod that mounts the new LUN will fail to create the pod. Specifically, depending on the operating system, the pod will hang in the ContainerCreating stage with either a "Could not attach disk: Timeout after 10s" error (CentOS 7) or an error about establishing a session when one already exists (Ubuntu 14). What you expected to happen: The pod should successfully mount the LUN. How to reproduce it (as minimally and precisely as possible): Given a working iSCSI target and a single node Kubernetes cluster: a LUN on the iSCSI target a pod that uses the LUN from step 1 (either natively or through a PV/PVC) a new LUN on the iSCSI target (note that the target portal and IQN must be identical). a pod that uses the LUN from step 3. The pod created in step 4 will fail to start.

Anything else we need to know: As far as I can tell, this is due to a bug with the Kubernetes iSCSI implementation. The AttachDisk() function in first checks if the LUN exists in /dev/disk/by-path/; if it doesn't, it attempts to log in to the target and recheck for the LUN's existence (lines 123-138 in the 1.5 branch). In this case, though, the session already exists, as it was created with the pod in step 2. In Ubuntu, attempting to log in a second time triggers an error, which gets reported in the pod description. In CentOS 7, this gets treated as a NOP; however, the new LUN will never become visible, and thus, the pod will fail to start. This can be solved manually by logging into the host and running . It should be possible to fix it in Kubernetes by adding a check in AttachDisk() to see if a session already exists for the target portal and IQN and, if so, doing a rescan of the session.
Could someone from please help with triage? Thanks!


Question: // InfoObject is an implementation of the Object interface. It gets all // the information from the Info object. type InfoObject struct { <ins> Remote  runtime.Unstructured </ins> Info    *resource.Info Encoder runtime.Encoder Parser  *parse.Factory

/kind bug /sig cli /area kubectl Hello. This is my first interaction with the Kubernetes community, so please forgive me if i have left out any crucial details, or if, perhaps, this is PEBKAC after all! I am keen to have a working feature, as described in and I am running K8s v1.9.3 and realise that this feature is in alpha, but i thought i would ask anyway. We're not on v1.9.4 yet, but the changelog doesn't mention the feature. What you expected to happen: I have some services deployed to Kubernetes with YAML files, that has been working reliably. We always use to deploy things. When a service is already deployed, and i make a minor change in the config file, for example the service port, i expect to see that in the diff. Instead, gives no output and exits with a . How to reproduce it (as minimally and precisely as possible): Consider the file : I make sure that it is correctly running on the cluster: Great, looking good. Now, i change the value to use Docker image tag and save. Now, i run: The stderr/stdout output is empty, and the exit code is . Anything else we need to know?: Environment: Kubernetes version: Client Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-02-09T21:50:44Z", GoVersion:"go1.9.4", Compiler:"gc", Platform:"darwin/amd64"} Server Version: version.Info{Major:"1", Minor:"9", GitVersion:"v1.9.3+coreos.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-02-10T01:42:55Z", GoVersion:"go1.9.2", Compiler:"gc", Platform:"linux/amd64"}
I have this same issue - if I change, say, to in my , running returns empty. If I try tweaking the version I'm diffing (which I don't understand) - e.g. - then I see a bunch of differences, but not actually the change I was expecting in the number of replicas.
I'll take this. Sorry for the inconvenience.


Question: // ValidateEvent makes sure that the event makes sense. func ValidateEvent(event *api.Event) errs.ValidationErrorList { allErrs := errs.ValidationErrorList{} <del> if event.Namespace != event.InvolvedObject.Namespace { </del> <ins> // TODO: There is no namespace required for minion if event.InvolvedObject.Kind != "Node" && event.Namespace != event.InvolvedObject.Namespace { </ins> allErrs = append(allErrs, errs.NewFieldInvalid("involvedObject.namespace", event.InvolvedObject.Namespace, "namespace does not match involvedObject")) } if !util.IsDNS1123Subdomain(event.Namespace) {

now shows NodePhase, NodeCondition, etc. information from NodeStatus. It should also show related events like what we did for pods.


Question: done echo "Validate output:" <del> kubectl_retry get cs </del> <ins> kubectl_retry get cs || true </ins> if [ "${return_value}" == "0" ]; then echo -e "${color_green}Cluster validation succeeded${color_norm}" else

cluster upgrade should maintain a functioning cluster [Feature:ClusterUpgrade] connection refused in version check /sig networking /assign See also
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
/sig network
I took a quickly look at the failures, the root cause is actually the master node is not ready. The following error msg indicates why master is not ready: It is indeed a network related issue.
I am re-assigning this back to you, please delegate. Thanks! cc/
this looks like a CNI related, can you guys take a look?
/assign
One thing I see happening at least in the GCE tests (Because it has the master log, but the GKE tests do not) I0614 04:25:10. 5 ] Node bootstrap-e2e-minion-group-2cz0 has CIDR 10.180.1.0/24, occupying it in CIDR map E0614 04:25:10. 5 ] Error starting "node" F0614 04:25:10. 5 ] error starting controllers: failed to mark cidr as occupied: cidr 10.180.1.0/24 is out the range of cluster cidr 10.100.0.0/14 I see that the test has: cluster/gce/config-"${CLUSTERIPRANGE:-10.100.0.0/14}" And this for the the pod cidr: "--pod-cidr", "10.180.0.0/24", // Assign a fixed CIDR to the node because there is no node controller. Is the above 10.100.0.0/14 supposed to be 10.180/14 ? 10.100/14 is only going to give 10.100 to 10.103 which isn't within the cluster cidr range.
I think I have an idea what happened here.

It looks like this PR changed the cluster CIDR from 10.180/14 to 10.100/14 However I see the the e2e node test assumes the pod cider is 10.180.0/24 Here: And Here: Since I believe we need to keep the 10.100/14 probably the right thing to do now is to fix the e2e node test to pick a pod cidr within 10.100/14
This was green on 6-15, if green again today i'll close this. Thanks for your attention !
should be the one fixing the issue. I will leave this open, until all UpgradeTest on GCE 1.6-1.7 are green. Thanks!
Ok, I thought we still have the issue on this, and assign this to myself. It turns out the test is still running with the old build for the upgrade test. Will continue monitoring this. cc/
fyi
After was merged, the master is up and ready, but the upgrade is still marked as failed. I looked into it this morning, and found controller_manager was running into the crashloop after master was upgraded due to the following error: I believe the issue is that we changed validation test in at which breaks the backward compatibility with 1.6 nodes. cc/ WDYT?
Had a quick discussion with network team The quick fix can be putting all nodes' cidr back 10.180.0.0/24, instead of 10.100.0.0/14 to stop the upgrade craziness. is working on this, and running the upgrade tests locally. Meanwhile, is going to discuss with on the next steps on how to rollout the feature which depends on the new ip range: 10.100.0.0/14
I think the problem now is that we have one version using 10.180.x. (1.6) and one using 10.100x.x. (1.7) And when we upgrade happens the cluster was created with 10.180 but then we upgrade we want 10.100. Can we just set the env var CLUSTERIPRANGE to 10.100 in the test so that the 1.6 cluster comes up with the right range?
In particular I think we could just change this file? To include the CLUSTERIPRANGE.

What you suggested should work theoretically. But since both branches hard-coded with two different ip range in some files, instead of respecting CLUSTERIPRANGE env variable, it won't work now. Since the failure are limited to our tests only, and upgrade test itself relies on the previous branch's latest build, not the real release, we decided to cherry pick fix to 1.6 branch. I closed my revert prs.
Cherrypicking the required changes to 1.6, 1.5:
besides cherrypicking , I believe we also need cherrypick which changes the API validation code; Otherwise, you are going to break 1.6 branch validation path on this. EDITED: Ignore this comment. All changes in are either for node e2e tests, or for validation test itself.
Thanks, looking forward to a green run :) Latest failure (in 1.6 -17 upgrade tests) is as in opening comment:
After cherrypicking those ip_range changes to 1.6 branch, now the master node is successfully upgraded to 1.7, and still accepted nodes running 1.6. The kube-controller-manager is not in crashloop anymore. The below is the output by running kubectl get nodes on the master node with https://k8s-
Also all kube-system pods are running successfully after master is upgraded:
Testing is still running, but I noticed that the cluster upgrade failed in the log at https://k8s- From above comment at , one can see that 1) now master node are up and ready, vs. notready in pre-54 runs 2) kube-controller-manager is running for 3 hour, not in crashloop in pre-54 runs kube-controller-manager-bootstrap-e2e-master 1/1 Running 3 3h 3) kubectl cs returns ok without error. There is no 'kubectl get nodes' failure, no kubect get cs failure. Above error / failure is red-herring, and looks like an autograde test infra issue. cc/
It looks like failed from the e2e test. I don't doubt that it worked from the master, but that's not where the e2e is running. I suspect an auth issue.

When the suite finishes, we can confirm by looking at the api server log for component status access from the kubectl user agent.
I checked kube-, there is no failure from the kubectl user agent, especially right before test claimed UpgradeMaster is failed, /componentstatus from kubectl is ok: I0620 00:23:45. 5 ] GET /api/v1/componentstatuses: (757.782364ms) 200 [[kubectl/v1.7.0 (linux/amd64) kubernetes/] 35.188.93.146:] In test log: https://k8s-, before previous /componentstatuses call, master is upgraded successfully, and all work-nodes are ready: But still complaint cluster is not working: But there is no retry with kubectl get cs which should be called multiple times in validate- and I checked validate- and think there is a problem in the code. The weirdness is that validation- hasn't been touched for a year. Something underneath triggers this change? is sending a quick fix.
Ok, I have more data to say that (along with ) is really buggy. Without any change, the new run of both master and nodes are successfully upgraded to 1.7 in a new run: https://k8s-
This guy is flakey: , Most recent failure about 11pm yesterday, still waiting for another run. Since the test is still failing, this issue should not have been closed - but the bot did it and don't have the authority to re-open. we're skipping some tests because of known API skew failures. Do you want to mark this test as a skip?
/reopen
Let's mark this as flakey and skip it.
we cannot skip this because it is does the real upgrade. If master upgrade failed, the test is red; if the node upgrade failed, the test is red. Sometimes, the script is flaky at the end, the test is red even the entire cluster is upgraded successfully. But we should skip this from 1.5-1.7 upgrade test suites since we only care about kubectl skews tests. There is no need to upgrade node from 1.5-1.7.
So we're going to ignore this in the 15-1.7 tests. Focus on the 1.6-1.7 tests.

Seeing new red in these (after some green) These guys remained green:
Is this related the aggregation API server ready issue?
What's the status of this issue? Should it continue blocking 1.7? If not, please remove the 'approved-for-milestone' label.
is on PTO
After we fixed the subnetwork issue, now upgrade are fine: https://k8s- There is failure for ingress And the engineer indicates it is caused by the infrastructure underneath.
I remove this as the release blocker.
This is failing in gce-1.7-master-upgrade-master: example job: https://k8s- I will apply release blocker and urgency to this. Please advice if it is not the case.
This did occur there, but it doesn't seem to occur often (hasn't happened in that suite since). The major issue with the suite that you linked is the panic you fixed yesterday.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale


Question: dnsPolicy: Default containers: - name: metadata-proxy <del> image: gcr.io/google_containers/metadata-proxy:v0.1.5 </del> <ins> image: gcr.io/google_containers/metadata-proxy:v0.1.6 </ins> securityContext: privileged: true <ins> # Request and limit resources to get guaranteed QoS. </ins> resources: requests: <del> memory: "32Mi" </del> <ins> memory: "25Mi" </ins> cpu: "30m" limits: <del> memory: "32Mi" </del> <ins> memory: "25Mi" </ins> cpu: "30m" # BEGIN_PROMETHEUS_TO_SD - name: prometheus-to-sd-exporter image: gcr.io/google_containers/prometheus-to-sd:v0.2.2 <ins> # Request and limit resources to get guaranteed QoS. resources: requests: memory: "20Mi" cpu: "2m" limits: memory: "20Mi" cpu: "2m" </ins> command: - /monitor - --stackdriver-prefix={{ prometheus_to_sd_prefix }}/addons

Offshoot of regression - We are supporting "100 pods per node" in k8s officially () and need to make sure metadata-proxy allows those many pods to access the metadata service simultaneously. cc
cc
/assign
/sig auth
Xref


Question: return config.f.ClientSet.Core().Namespaces() } <del> func CheckReachabilityFromPod(expectToBeReachable bool, namespace, pod, target string) { </del> <ins> func CheckReachabilityFromPod(expectToBeReachable bool, timeout time.Duration, namespace, pod, target string) { </ins> cmd := fmt.Sprintf("wget -T 5 -qO- %q", target) <del> err := wait.PollImmediate(Poll, 2*time.Minute, func() (bool, error) { </del> <ins> err := wait.PollImmediate(Poll, timeout, func() (bool, error) { </ins> _, err := RunHostCmd(namespace, pod, cmd) if expectToBeReachable && err != nil { Logf("Expect target to be reachable. But got err: %v. Retry until timeout", err)

Sub-issue for For AWS ELB, when an ELB is first created, is made it is very normal that it takes 2 minutes to be created, but the DNS name is reported fairly quickly.


Question: pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ <del> // A.K.A InnerVolumeSpecName required to match for Unmount Name: volume.Name, </del> <ins> // Must be unique per disk as it is used as the unique part of the // staging path Name: fmt.Sprintf("%s-%s", AzureFileDriverName, azureSource.ShareName), </ins> }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{

Minimal Repro: Create 2 different disks Create 2 Pods each using one of the disks Start both pods on the same node You will notice one of the disks gets ed to a path: , then both disks are ed from to a target path: which is similar but at least uses the pod . Both of these volumes are being treated as the same volume in the Kubelet even though they are definitely not the same volume. Note: it may not be "treated" as the same volume - it may just be that since the paths are the same the following volumes see that the path is already a mountpoint and skip the step. This is the more likely scenario based on observations. The value is the "name" of the inline-volume. These can obviously be the same across different pods - we should actually be placing it in a more unique directory (based off the translated maybe?). /cc
/sig storage
The inline volume example I'm using (ignore the fact that is a complete misnomer): I've confirmed that changing the "name" makes inline volumes work well since there is no longer a collision on the paths
/assign
Alright, problem is sort of twofold: The referred to above for inline-volumes is set here: As we can see from the comment: it is set to the because the in-tree CSI Plugin makes a lot of naming assumptions based on the which for PVC/PVs is the PV name (unique in a cluster). Primarily, during unmount the CSI Plugin tries to from a "well known ": which happens to be some path based on the which for the inline volume is just the ( above). So we run into the problem that if we use the directory is not sufficiently unique for different disks. And if we use something else like then the unmount pathway is unable to find the . I think using is a no-go. So we need to use something sufficiently unique (like ) for the ( above) and fix up some of the other codepaths to either use different "well-known paths" to write to, or to fix up path generation code so they give us the correct locations. I'm also going to check to see whether 's refactor fixes this issue (because it might since it translates things earlier)
OK. Really good news.

your refactor in its current state actually fixes this issue if we set the to something unique here: Looks like the early translation is also making the pathway from the "correct" dir for inline migration. So I will create a PR that fixes this issue that should merge after


Question: return } <ins> if hasName { // metadata.name is the canonical internal name. // generic.SelectionPredicate will notice that this is // a request for a single object and optimize the // storage query accordingly. nameSelector := fields.OneTermEqualSelector("metadata.name", name) if opts.FieldSelector != nil && !opts.FieldSelector.Empty() { // It doesn't make sense to ask for both a name // and a field selector, since just the name is // sufficient to narrow down the request to a // single object. errorJSON( errors.NewBadRequest("both a name and a field selector provided; please provide one or the other."), scope.Codec, w, ) return } opts.FieldSelector = nameSelector } </ins> if (opts.Watch || forceWatch) && rw != nil { watcher, err := rw.Watch(ctx, opts.LabelSelector, opts.FieldSelector, opts.ResourceVersion) if err != nil {

Using a watch expression on endpoints can lead to unpredictable behavior that includes returning a watch stream that ignores filters (e.g. a {name} determination) and returns items of the wrong type (e.g. changes to PODS in the ENDPOINTS stream). returns This result includes updates for services "kubernetes", "kubernetes-ro", "redis-slave", "redis-master", etc. Even though the requesting URL indicated only "guestbook" updates were to be watched. This seems to only occur when no resourceVersion is specified and the most recent resourceVersion has fallen off the etcd watch window. The expected behavior when resourceVersion is not specified I would think should be either: an initial update with the latest version of the resource, followed by filtered updates. nothing, followed by filtered updates. Though I'm not sure which of these is actually the spec. I would prefer (1) since it allows you to do a resource watch without having to first do a separate GET. But (2) coupled with a preceding GET would yield the same results.
IRC: cc
The pods mentioned in the stream are references to the pods corresponding to endpoints. See for discussion behind the feature. We probably need to distinguish the resourceVersion at which an object was last modified, which is also used as an update precondition, vs. the current (global) resourceVersion needed for watch.
Does the change to use GET with watch as a query parameter enable the GET to both return the latest version and subscribe to changes, or is a separate non-watch GET still required?
/subscribe
If you pass resourceVersion=0 you get list then watch (for single or multiple).
That behavior has been there for a while - the only challenge is you don't know when the list is complete, and we felt that list then watch more clearly delineated that.
This.
Watching with resourceVersion=0 should return an error, IMO.
I think the semantics for {watch a single resource} vs. {watch all resources of the same type} should be separated. When watching a single resource I would expect the watch to have consistent semantics. That is, I would expect it to be possible to watch an item and be guaranteed either: I saw every change to the item, or I knew that a change was missed (and so could do a raw GET to re-sync).

To this end it would be ideal if watch for a single item had the following semantics: Delivered the current state, followed by a list of all changes. If any change is missed or can't be delivered then the watch is terminated. Delivers either a guarantee of all changes that occurred on that resource from (exclusive) to present. If the server cannot prove that there were no missing changes then it should return an error and terminate the watch. This would imply that: Would always return an error unless the server can prove that it can deliver all changes to the resource (e.g. when the resource hasn't been created yet.) I'm surprised that the interface wants to provide multi-resource watch. It would seem that the semantics of this watch would either be potentially expensive to deliver or "best effort" such that it is basically useless for anything other than casual human consumption. Imagine a future where the master is sharded such that even different resources of the same type are on different shards. To provide meaningful semantics to a multi-resource watch would require that resource versions be meaningful in a global sense and in a way that spans all shards. Wouldn't this be undesirable? It would require that any write at any shard would need a round-trip message to all other shards to ensure that the logical clocks remain in sync. A multi-watch would similarly have to be fanned out to all shards in the backend. Both of these would defeat the benefits of sharding the data in the first place. I could understand providing a multi-resource watch for the creation and deletion of resources of a particular type (handled potentially by a secondary index in the sharded scheme) without providing a full watch of the contents. You could then use this watch to establish resource-specific watches for the resources you needed to track the contents of. Maintaining a secondary index wouldn't have to be transactional with changes that produced or destroyed items.
resourceVersion=0 is the same thing as saying watch resourceVersion=ObjectModifiedResourceVersion-1. Perhaps it should be more explicit, but it is generally useful. Why is blind watch bad? ----- Original Message -----
Also, I don't see it mentioned above but - everything in  is ignored. Confusing, but already fixed by the deprecation of .

cc
We don't offer multi-resource watch today, but in the future you're likely to correlate very strongly with a shard based on your namespace, rather than a shard on your type. I don't think sharding by type will be common except on a few types of resources (events, namespaces, possibly endpoints). The access pattern for most clients is going to be "in namespace Foo, get type X, get type Y, get type Z, get type A". Most UIs that want to watch anything will watch on the namespace. So the access pattern is less likely to reward sharding by type.
The query: Is a multi-resource watch. It watches for all resources of type Endpoints. But do you want the interface to guarantee that all objects in the same namespace are updated with consistency? I agree that shard boundaries are almost certainly NOT type-boundaries. For instance there is clearly a consistency relationship between a Service and its Endpoints object. I would hope that it would be impossible for me to observe a change to a Service object and then to subsequently query for its Endpoints object and get a stale read (semantically speaking). But that is a query across types. Suggesting that the appropriate shard boundaries are certainly not type-boundaries.
Sorry, I was referring to "multi-resource" as "watching over multiple resource types". It is possible to emulate a watch across a small number of stable shards by creating a resource version tuple (a vector clock, essentially) - for medium sized clusters I see that as a reasonable option. It's more likely though that infrastructure components that want to "watch the world" are either themselves shardable (such that it's acceptable for world to be defined as "my shard") or an intermediate component supports them. That intermediate component might be something like a partially ordered event stream that provides a firehose to the cluster as an aggregator of the cluster changes. Today, we require that API consumers assume resource version is internally consistent across a single type in a single namespace. Controllers are allowed to watch the world until such a time as we implement a sharding solution. There is no consistency relationship across resource types - services and endpoints are independent objects with independent lifecycles. It is by design that Endpoints can be updated out of band with Services, to allow downstream components to offload them.

We relax referential rules across resource types - there is no guarantee a referenced object will be created before or after you, or that it should exist only if you exist.
I completely agree there are possible implementations that can tolerate sharded and/or inconsistent queries. And deciding that certain kinds of JOINs, particularly across resource types and/or namespaces, are always going to be inconsistent seems like a fine place to land. I was merely surprised that the API was designed to promote the appearance of consistency for these queries today. I would have expected that the API would be more conservative to reserve these degrees of freedom around consistency for future implementations without requiring a future breaking change. Offering less functionality in V1 or functionality with weaker semantics would allow weakly consistent future implementations to emerge later without breaking existing applications. As a general rule I try to avoid APIs that give the appearance of consistency if they are not intended to actually deliver consistency. I think most developers will always assume consistency unless there is a really good reason to believe otherwise. This seems like human nature since people expect systems to exhibit the same causal behavior as the real world. Take for instance Endpoints. My impression of this after seeing the API was that Endpoints was simply a VIEW (in the SQL since) that provides a convenient way to express the JOIN between Service and Pods. From that perspective I would certainly have expected there to be a causal relationship between a change in the Service and its corresponding Endpoints object. I could easily imagine myself making that mistake when writing a application. Perhaps my program changes the Service object and then wants to send a message to all of the Endpoints associated with the new Service definition. After changing the Service, my program queries Endpoints. Did it get the new set of Endpoints? There are several possible outcomes: The Endpoints is the same as it was before I changed the Service Definition: The new Service definition just happens to result in the same set of Endpoints as before. I have not waited long enough for Endpoints to be updated. How long should I wait? What indicates that I've waited long enough? The Endpoints is different: Is this the correct one? The one that corresponds to my Service definition change? Or is this the update that occurred due to a different Service definition change that happened just before mine? If so, I should wait longer for mine.

Or is this the update that occurred due to a different Service definition change that happened just after mine? If so, I should either use this one or attempt my Service definition change again. In an inconsistent and asynchronously updated world we need to provide a way to answer these questions. (e.g. perhaps Endpoints should include the the resourceVersion of the Service definition used to compute it.) All applications need to properly deal with this extra information. (e.g. need to ignore all updated Endpoints until one arrives that has a resourceVersion = their change to the Service definition.) I'd imagine it's possible to implement this and for customers to use it correctly, but I think it offers a lot of opportunity for bugs. Users might simply wait for the next update and then take the wrong action leading to a misbehaving system. If we could decide which relationships should exhibit consistency and provide it where the developer would expect it then the API is simply and easier to use and applications are more likely to be correct. If we don't offer APIs which might lead the developer to make incorrect assumptions then applications are less likely to have those bugs.
I dug into this... it's more broken than I thought it was, so I've made a fix in . Once my fix goes in, a single object acts as if you didn't pass a name: this will be fixed directly (at least for the resource types I checked). get an appropriate resource version, you will have to LIST with a fieldselector ("?fields=name%3Dfoo" or some such-- don't recall exact syntax) instead of GETting. This way we don't have to modify the API to pass a special "watch at" resource version.