kartongips: paper over^W^Wfix CRD updates

Ceph CRD updates would fail with:

  ERROR Error updating customresourcedefinitions cephclusters.ceph.rook.io: expected kind, but got map

This wasn't just https://github.com/bitnami/kubecfg/issues/259 . We pull
in the 'solution' from Pulumi
(https://github.com/pulumi/pulumi-kubernetes/pull/622) which just
retries the update via a JSON update instead, and that seems to have
worked.

We also add some better error return wrapping, which I used to debug
this issue properly.

Oof.

Change-Id: I2007a7857e44128d74760174b61b59efa58e9cbc

cluster/kube/lib/rook.libsonnet

@@ -27,10 +27,7 @@ local oa = kube.OpenAPI;
         policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
 
         crds: {
-            # BUG: cannot control this because of:
-            # ERROR Error updating customresourcedefinitions cephclusters.ceph.rook.io: expected kind, but got map
-            # TODO(q3k): debug and fix kubecfg (it's _not_ just https://github.com/bitnami/kubecfg/issues/259 )
-            cephclusters:: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
+            cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
                 spec+: {
                     additionalPrinterColumns: [
                         { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },

cluster/tools/kartongips/pkg/kubecfg/update.go

@@ -102,11 +102,11 @@ func patch(existing, new *unstructured.Unstructured, schema proto.Schema) (*unst
 		tmp := unstructured.Unstructured{}
 		err := utils.CompactDecodeObject(data, &tmp)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("CompactDecodeObject original object: %w", err)
 		}
 		origData, err = tmp.MarshalJSON()
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("MarshalJSON original object: %w", err)
 		}
 	}
 
@@ -116,7 +116,7 @@ func patch(existing, new *unstructured.Unstructured, schema proto.Schema) (*unst
 	utils.DeleteMetaDataAnnotation(new, AnnotationOrigObject)
 	data, err := utils.CompactEncodeObject(new)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("CompactEncodeObject: %w", err)
 	}
 	utils.SetMetaDataAnnotation(new, AnnotationOrigObject, data)
 
@@ -124,41 +124,59 @@ func patch(existing, new *unstructured.Unstructured, schema proto.Schema) (*unst
 
 	newData, err := new.MarshalJSON()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("MarshalJSON new: %w", err)
 	}
 
 	existingData, err := existing.MarshalJSON()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("MarshalJSON new: %w", err)
 	}
 
-	var resData []byte
-	if schema == nil {
-		// No schema information - fallback to JSON merge patch
+	schemaless := func() ([]byte, error) {
 		patch, err := jsonmergepatch.CreateThreeWayJSONMergePatch(origData, newData, existingData)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("CreateThreeWayJSONMergePatch (schemaless): %w", err)
 		}
-		resData, err = jsonpatch.MergePatch(existingData, patch)
+		resData, err := jsonpatch.MergePatch(existingData, patch)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("MergePatch (schemaless): %w", err)
 		}
-	} else {
+		return resData, nil
+	}
+	schemaful := func() ([]byte, error) {
 		patchMeta := strategicpatch.NewPatchMetaFromOpenAPI(schema)
 
 		patch, err := strategicpatch.CreateThreeWayMergePatch(origData, newData, existingData, patchMeta, true)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("CreateThreeWayMergePatch (schemaful): %w", err)
 		}
-		resData, err = strategicpatch.StrategicMergePatchUsingLookupPatchMeta(existingData, patch, patchMeta)
+		resData, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(existingData, patch, patchMeta)
+		if err != nil {
+			return nil, fmt.Errorf("StrategicMergePatch (schemaful): %w", err)
+		}
+		return resData, nil
+	}
+
+	var resData []byte
+	if schema == nil {
+		resData, err = schemaless()
 		if err != nil {
 			return nil, err
 		}
+	} else {
+		resData, err = schemaful()
+		if err != nil {
+			log.Warningf("Schemaful/Three-way merge failed (%v), attempting schemaless/JSON merge...", err)
+			resData, err = schemaless()
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	result, _, err := unstructured.UnstructuredJSONScheme.Decode(resData, nil, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Decode: %w", err)
 	}
 
 	return result.(*unstructured.Unstructured), nil
@@ -171,7 +189,7 @@ func createOrUpdate(ctx context.Context, rc dynamic.ResourceInterface, obj *unst
 
 		data, err := utils.CompactEncodeObject(obj)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("CompactEncodeObject: %w", err)
 		}
 		utils.SetMetaDataAnnotation(obj, AnnotationOrigObject, data)
 
@@ -183,12 +201,12 @@ func createOrUpdate(ctx context.Context, rc dynamic.ResourceInterface, obj *unst
 		return newobj, err
 	}
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Get: %w", err)
 	}
 
 	mergedObj, err := patch(existing, obj, schema)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("patch: %w", err)
 	}
 
 	// Kubernetes is a bit odd when/how it reports
@@ -207,7 +225,7 @@ func createOrUpdate(ctx context.Context, rc dynamic.ResourceInterface, obj *unst
 	log.Debug("About to make change: ", diff.ObjectDiff(existing, mergedObj))
 	log.Info("Updating ", desc, dryRunText)
 	if dryRun {
-		return mergedObj, nil
+		return mergedObj, fmt.Errorf("ObjectDiff: %v", nil)
 	}
 	newobj, err := rc.Update(ctx, mergedObj, metav1.UpdateOptions{})
 	log.Debugf("Update(%s) returned (%v, %v)", mergedObj.GetName(), newobj, err)