[MM-51677] Basic Crossplane cluster provisioner (WIP)

cmd/cloud/cluster.go

@@ -105,6 +105,7 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error {
 
 	request := &model.CreateClusterRequest{
 		Provider:               flags.provider,
+		Provisioner:            flags.provisioner,
 		Version:                flags.version,
 		AMI:                    flags.ami,
 		Zones:                  strings.Split(flags.zones, ","),
@@ -113,11 +114,9 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error {
 		Annotations:            flags.annotations,
 		Networking:             flags.networking,
 		VPC:                    flags.vpc,
-		Provisioner:            model.ProvisionerKops,
 	}
 
-	if flags.useEKS {
-		request.Provisioner = model.ProvisionerEKS
+	if flags.provisioner == model.ProvisionerEKS {
 		request.ClusterRoleARN = flags.clusterRoleARN
 		request.NodeRoleARN = flags.nodeRoleARN
 		request.NodeGroupWithPublicSubnet = flags.nodegroupWithPublicSubnet
@@ -538,6 +537,10 @@ func defaultClustersTableData(clusters []*model.ClusterDTO) ([]string, [][]strin
 			provisionerMetadata = cluster.ProvisionerMetadataKops.GetCommonMetadata()
 			masterCount = cluster.ProvisionerMetadataKops.MasterCount
 			masterInstanceType = cluster.ProvisionerMetadataKops.MasterInstanceType
+		} else if cluster.Provisioner == model.ProvisionerCrossplane && cluster.ProvisionerMetadataCrossplane != nil {
+			provisionerMetadata = cluster.ProvisionerMetadataCrossplane.GetCommonMetadata()
+			masterCount = cluster.ProvisionerMetadataCrossplane.NodeCount
+			masterInstanceType = cluster.ProvisionerMetadataCrossplane.InstanceType
 		} else if cluster.Provisioner == model.ProvisionerEKS && cluster.ProvisionerMetadataEKS != nil {
 			provisionerMetadata = cluster.ProvisionerMetadataEKS.GetCommonMetadata()
 			masterCount = 1

cmd/cloud/cluster_flag.go

@@ -1,3 +1,7 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+//
+
 package main
 
 import (
@@ -21,6 +25,7 @@ func (flags *clusterFlags) addFlags(command *cobra.Command) {
 
 type createRequestOptions struct {
 	provider                  string
+	provisioner               string
 	version                   string
 	ami                       string
 	zones                     string
@@ -30,24 +35,24 @@ type createRequestOptions struct {
 	vpc                       string
 	clusterRoleARN            string
 	nodeRoleARN               string
-	useEKS                    bool
 	additionalNodeGroups      map[string]string
 	nodegroupWithPublicSubnet []string
 }
 
 func (flags *createRequestOptions) addFlags(command *cobra.Command) {
 	command.Flags().StringVar(&flags.provider, "provider", "aws", "Cloud provider hosting the cluster.")
-	command.Flags().StringVar(&flags.version, "version", "latest", "The Kubernetes version to target. Use 'latest' or versions such as '1.16.10'.")
+	command.Flags().StringVar(&flags.version, "version", "", "The Kubernetes version to target. Use 'latest' or versions such as '1.16.10'.")
 	command.Flags().StringVar(&flags.ami, "ami", "", "The AMI to use for the cluster hosts.")
 	command.Flags().StringVar(&flags.zones, "zones", "us-east-1a", "The zones where the cluster will be deployed. Use commas to separate multiple zones.")
 	command.Flags().BoolVar(&flags.allowInstallations, "allow-installations", true, "Whether the cluster will allow for new installations to be scheduled.")
 	command.Flags().StringArrayVar(&flags.annotations, "annotation", []string{}, "Additional annotations for the cluster. Accepts multiple values, for example: '... --annotation abc --annotation def'")
 	command.Flags().StringVar(&flags.networking, "networking", "calico", "Networking mode to use, for example: weave, calico, canal, amazon-vpc-routed-eni")
 	command.Flags().StringVar(&flags.vpc, "vpc", "", "Set to use a shared VPC")
 
+	command.Flags().StringVar(&flags.provisioner, "provisioner", "kops", "Provisioner to create cluster with.")
+
 	command.Flags().StringVar(&flags.clusterRoleARN, "cluster-role-arn", "", "AWS role ARN for cluster.")
 	command.Flags().StringVar(&flags.nodeRoleARN, "node-role-arn", "", "AWS role ARN for node.")
-	command.Flags().BoolVar(&flags.useEKS, "eks", false, "Create EKS cluster.")
 	command.Flags().StringToStringVar(&flags.additionalNodeGroups, "additional-node-groups", nil, "Additional nodegroups to create. The key is the name of the nodegroup and the value is the size constant.")
 	command.Flags().StringSliceVar(&flags.nodegroupWithPublicSubnet, "nodegroup-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroup.")
 }

cmd/cloud/server.go

@@ -34,6 +34,7 @@ import (
 	"github.com/mattermost/mattermost-cloud/internal/tools/kops"
 	"github.com/mattermost/mattermost-cloud/internal/tools/terraform"
 	"github.com/mattermost/mattermost-cloud/internal/tools/utils"
+	"github.com/mattermost/mattermost-cloud/k8s"
 	"github.com/mattermost/mattermost-cloud/model"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -55,6 +56,12 @@ func newCmdServer() *cobra.Command {
 		},
 		PreRun: func(command *cobra.Command, args []string) {
 			flags.serverFlagChanged.addFlags(command) // To populate flag change variables.
+			if err := flags.crossplaneOptions.valid(command); err != nil {
+				_ = command.Usage()
+				logger.Error(err)
+				os.Exit(1)
+			}
+
 			deprecationWarnings(logger, command)
 
 			if flags.enableLogStacktrace {
@@ -235,6 +242,10 @@ func executeServerCmd(flags serverFlags) error {
 		"disable-db-init-check":                         flags.disableDBInitCheck,
 		"enable-route53":                                flags.enableRoute53,
 		"disable-dns-updates":                           flags.disableDNSUpdates,
+		"enable-crossplane":                             flags.enableCrossplane,
+		"k8s-use-incluster-config":                      flags.k8sUseInClusterConfig,
+		"k8s-kubeconfig-path":                           flags.k8sKubeconfigPath,
+		"kube2iam-account-id":                           flags.kube2IAMAccountID,
 	}).Info("Starting Mattermost Provisioning Server")
 
 	// Warn on settings we consider to be non-production.
@@ -278,6 +289,32 @@ func executeServerCmd(flags serverFlags) error {
 		EtcdManagerEnv:          etcdManagerEnv,
 	}
 
+	// Crossplane provisioner configuration.
+	var crossplaneProvisioner *provisioner.CrossplaneProvisioner
+	if flags.enableCrossplane {
+		var k8sClient *k8s.KubeClient
+		var err error
+		if flags.k8sUseInClusterConfig {
+			k8sClient, err = k8s.NewFromInClusterConfig(logger)
+			if err != nil {
+				return errors.Wrap(err, "failed to build k8s client")
+			}
+		} else if flags.k8sKubeconfigPath != "" {
+			k8sClient, err = k8s.NewFromFile(flags.k8sKubeconfigPath, logger)
+			if err != nil {
+				return errors.Wrap(err, "failed to build k8s client")
+			}
+		}
+		crossplaneProvisioner = provisioner.NewCrossplaneProvisioner(
+			k8sClient,
+			awsClient,
+			provisioningParams,
+			sqlStore,
+			flags.kube2IAMAccountID,
+			logger,
+		)
+	}
+
 	resourceUtil := utils.NewResourceUtil(instanceID, awsClient, dbClusterUtilizationSettingsFromFlags(flags), flags.disableDBInitCheck)
 
 	kopsProvisioner := provisioner.NewKopsProvisioner(
@@ -295,7 +332,7 @@ func executeServerCmd(flags serverFlags) error {
 	)
 
 	provisionerObj := provisioner.NewProvisioner(
-		kopsProvisioner, eksProvisioner,
+		kopsProvisioner, eksProvisioner, crossplaneProvisioner,
 		provisioningParams,
 		awsClient,
 		resourceUtil,
@@ -368,6 +405,8 @@ func executeServerCmd(flags serverFlags) error {
 		multiDoer = append(multiDoer, supervisor.NewInstallationDBMigrationSupervisor(sqlStore, awsClient, resourceUtil, instanceID, provisionerObj, eventsProducer, logger))
 	}
 
+	// TODO: crossplane supervisor
+
 	// Setup the supervisor to effect any requested changes. It is wrapped in a
 	// scheduler to trigger it periodically in addition to being poked by the API
 	// layer.

cmd/cloud/server_flag.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	toolsAWS "github.com/mattermost/mattermost-cloud/internal/tools/aws"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -176,6 +177,32 @@ func (flags *serverFlagChanged) addFlags(command *cobra.Command) {
 	flags.isKeepFileStoreDataChanged = command.Flags().Changed("keep-filestore-data")
 }
 
+type crossplaneOptions struct {
+	enableCrossplane      bool
+	k8sUseInClusterConfig bool
+	k8sKubeconfigPath     string
+	kube2IAMAccountID     string
+}
+
+func (flags *crossplaneOptions) addFlags(command *cobra.Command) {
+	command.Flags().BoolVar(&flags.enableCrossplane, "enable-crossplane", false, "Whether to use Crossplane to provision resources.")
+	command.Flags().BoolVar(&flags.k8sUseInClusterConfig, "k8s-use-in-cluster-config", false, "Whether to use in-cluster config for k8s client.")
+	command.Flags().StringVar(&flags.k8sKubeconfigPath, "k8s-kubeconfig-path", "", "Path to kubeconfig file for k8s client.")
+	command.Flags().StringVar(&flags.kube2IAMAccountID, "kube2iam-account-id", "", "AWS account ID for kube2iam.")
+	command.MarkFlagsMutuallyExclusive("k8s-use-in-cluster-config", "k8s-kubeconfig-path")
+}
+
+func (flags *crossplaneOptions) valid(command *cobra.Command) error {
+	if flags.enableCrossplane && !flags.k8sUseInClusterConfig && flags.k8sKubeconfigPath == "" {
+		return errors.New("k8s-kubeconfig-path or k8s-use-in-cluster-config is required when enable-crossplane is set to true")
+	}
+
+	if flags.enableCrossplane && flags.kube2IAMAccountID == "" {
+		return errors.New("kube2iam-account-id is required when enable-crossplane is set to true")
+	}
+	return nil
+}
+
 type serverFlags struct {
 	supervisorOptions
 	schedulingOptions
@@ -185,6 +212,8 @@ type serverFlags struct {
 	dbUtilizationSettings
 	serverFlagChanged
 
+	crossplaneOptions
+
 	listen      string
 	metricsPort int
 
@@ -211,6 +240,7 @@ func (flags *serverFlags) addFlags(command *cobra.Command) {
 	flags.pgBouncerConfig.addFlags(command)
 	flags.installationOptions.addFlags(command)
 	flags.dbUtilizationSettings.addFlags(command)
+	flags.crossplaneOptions.addFlags(command)
 
 	command.Flags().StringVar(&flags.listen, "listen", ":8075", "The interface and port on which to listen.")
 	command.Flags().IntVar(&flags.metricsPort, "metrics-port", 8076, "Port on which the metrics server should be listening.")

go.mod

@@ -2,6 +2,8 @@ module github.com/mattermost/mattermost-cloud
 
 go 1.19
 
+replace github.com/mattermost/mattermost-cloud-crossplane => ../../../.go/src/github.com/mattermost/mattermost-cloud-crossplane
+
 require (
 	github.com/0xAX/notificator v0.0.0-20220220101646-ee9b8921e557
 	github.com/Masterminds/squirrel v1.5.3
@@ -33,6 +35,7 @@ require (
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.7
 	github.com/mattermost/awat v0.2.0
+	github.com/mattermost/mattermost-cloud-crossplane v0.0.0-00010101000000-000000000000
 	github.com/mattermost/mattermost-operator v1.20.0-rc.2
 	github.com/mattermost/rotator v0.2.0
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
@@ -142,11 +145,11 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
-	sigs.k8s.io/controller-runtime v0.11.0 // indirect
+	sigs.k8s.io/controller-runtime v0.11.1 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -2381,8 +2381,9 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.6.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4=
+k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-aggregator v0.18.8 h1:8VQxblQqRInpJ+DS2aGgbdWq6xP8UG/jzV6v8cFccOc=
 k8s.io/kube-aggregator v0.18.8/go.mod h1:CyLoGZB+io8eEwnn+6RbV7QWJQhj8a3TBH8ZM8sLbhI=
 k8s.io/kube-openapi v0.0.0-20180719232738-d8ea2fe547a4/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
@@ -2420,8 +2421,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PN
 sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E=
 sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
 sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
-sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
-sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
+sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
+sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
 sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
@@ -2435,8 +2436,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.0 h1:kDvPBbnPk+qYmkHmSo8vKGp438IASWofnbbUKDE/bv0=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

internal/api/cluster.go

@@ -120,6 +120,15 @@ func handleCreateCluster(c *Context, w http.ResponseWriter, r *http.Request) {
 	if createClusterRequest.Provisioner == model.ProvisionerEKS {
 		cluster.ProvisionerMetadataEKS = &model.EKSMetadata{}
 		cluster.ProvisionerMetadataEKS.ApplyClusterCreateRequest(createClusterRequest)
+	} else if createClusterRequest.Provisioner == model.ProvisionerCrossplane {
+		cluster.ProvisionerMetadataCrossplane = &model.CrossplaneMetadata{
+			// TODO: Defaults to first zone in the AWS metadata for now.
+			// FIXME: It gets the region from the first zone, but it should be inherited from the
+			// VPC or it should use zones instead of regions.
+			Region: cluster.ProviderMetadataAWS.Zones[0][:len(cluster.ProviderMetadataAWS.Zones[0])-1],
+		}
+		cluster.ProvisionerMetadataCrossplane.ApplyClusterCreateRequest(createClusterRequest)
+		cluster.ProvisionerMetadataCrossplane.SetDefaults()
 	} else {
 		cluster.ProvisionerMetadataKops = &model.KopsMetadata{}
 		cluster.ProvisionerMetadataKops.ApplyClusterCreateRequest(createClusterRequest)