go/k8s: controller-runtime upgrade, fake client lacks functionality

Question

I am attempting to upgrade controller-runtime from version 0.13.0 to 0.16.3, which has introduced several issues with tests due to breaking changes.

I have managed to resolve most of them, but there is one specific use case that I cannot address.

The code creates a custom resource and then updates its status. While everything works as expected, the problem arises in the test phase. The test utilizes the fake client from the client/fake package.

The issue lies in the fact that when a new custom resource (CR) object is created in the fake client's memory, it appears that a corresponding object is not generated under fakeSubResourceClient for status updates. Consequently, this leads to a "not found error," which should not occur and only happens in tests, not in production.

I have attempted to seek assistance from various AI bots and have also searched for similar use cases in public GitHub repositories, but with no success.

Any help would be greatly appreciated.

Answer 1

OK so after verifying that indeed the fake client doesn't support this scenario, I came up with a workaround that works pretty well, here's the code

type ErrorInjectingFakeClient struct {
    client.WithWatch
    withSubResourceSimulation bool
    withObjects               []client.Object
    withStatusSubresource     []client.Object
    failError                 error
}

func NewErrorInjectingFakeClient(scheme *runtime.Scheme, withSubResourceSimulation bool, initObjects ...client.Object) *ErrorInjectingFakeClient {
    builder := fake.NewClientBuilder().
        WithScheme(scheme).
        WithObjects(initObjects...).
        WithStatusSubresource(initObjects...)
    return &ErrorInjectingFakeClient{
        withSubResourceSimulation: withSubResourceSimulation,
        WithWatch:                 builder.Build(),
        withObjects:               initObjects,
        withStatusSubresource:     initObjects,
    }
}

// Copied from sigs.k8s.io/[email protected]/pkg/client/fake/client.go#1217
var inTreeResourcesWithStatus = []schema.GroupVersionKind{
    {Version: "v1", Kind: "Namespace"},
    {Version: "v1", Kind: "Node"},
    {Version: "v1", Kind: "PersistentVolumeClaim"},
    {Version: "v1", Kind: "PersistentVolume"},
    {Version: "v1", Kind: "Pod"},
    {Version: "v1", Kind: "ReplicationController"},
    {Version: "v1", Kind: "Service"},

    {Group: "apps", Version: "v1", Kind: "Deployment"},
    {Group: "apps", Version: "v1", Kind: "DaemonSet"},
    {Group: "apps", Version: "v1", Kind: "ReplicaSet"},
    {Group: "apps", Version: "v1", Kind: "StatefulSet"},

    {Group: "autoscaling", Version: "v1", Kind: "HorizontalPodAutoscaler"},

    {Group: "batch", Version: "v1", Kind: "CronJob"},
    {Group: "batch", Version: "v1", Kind: "Job"},

    {Group: "certificates.k8s.io", Version: "v1", Kind: "CertificateSigningRequest"},

    {Group: "networking.k8s.io", Version: "v1", Kind: "Ingress"},
    {Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"},

    {Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"},

    {Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"},

    {Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"},

    {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema"},
    {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"},
}

func (c *ErrorInjectingFakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
    if c.failError != nil {
        return c.failError
    }

    if err := c.WithWatch.Create(ctx, obj, opts...); err != nil {
        return err
    }
    if !c.withSubResourceSimulation {
        return nil
    }

    c.withObjects = append(c.withObjects, obj)
    err := c.recreateFake(ctx)
    if err != nil {
        return err
    }

    return nil
}

func (c *ErrorInjectingFakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
    if c.failError != nil {
        return c.failError
    }

    if err := c.WithWatch.Delete(ctx, obj, opts...); err != nil {
        return err
    }

    return nil
}

func (c *ErrorInjectingFakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
    if c.failError != nil {
        return c.failError
    }
    return c.WithWatch.Update(ctx, obj, opts...)
}

// TODO add other K8S client functions you want to inject failures into

func (c *ErrorInjectingFakeClient) SetError(err error) {
    c.failError = err
}

func (c *ErrorInjectingFakeClient) recreateFake(ctx context.Context) error {
    // This is a pretty disgusting hack to get around the fact that the fake client doesn't support creating status subresources
    // Discussed here https://github.com/kubernetes-sigs/controller-runtime/issues/2386#issuecomment-1607768830
    // and here https://github.com/kubernetes-sigs/controller-runtime/issues/2362#issuecomment-1699415588
    // and here https://stackoverflow.com/questions/77489441/go-k8s-controller-runtime-upgrade-fake-client-lacks-functionality

    // We need to collect all the objects from the fake client, and then re-create the fake client with the status subresource for the created object

    gvks := sets.New(inTreeResourcesWithStatus...)
    for _, o := range c.withObjects {
        gvk, err := apiutil.GVKForObject(o, c.Scheme())
        if err != nil {
            return err
        }
        gvks.Insert(gvk)
    }

    var objs []client.Object
    for _, gvk := range gvks.UnsortedList() {
        objList := &unstructured.UnstructuredList{}
        objList.SetGroupVersionKind(schema.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind})

        err := c.List(ctx, objList)
        if err != nil {
            return err
        }

        for _, o := range objList.Items {
            objs = append(objs, o.DeepCopy())
        }
    }

    initObjs := sets.New(objs...).UnsortedList()
    c.withObjects = initObjs
    c.withStatusSubresource = initObjs

    c.WithWatch = fake.NewClientBuilder().
        WithScheme(c.Scheme()).
        WithObjects(c.withObjects...).
        WithStatusSubresource(c.withStatusSubresource...).
        Build()
    return nil
}