From 670b68523305468db8ec5572ed7c6516d3f7c016 Mon Sep 17 00:00:00 2001 From: vinoyang Date: Tue, 7 Mar 2023 10:06:11 +0800 Subject: [PATCH] [Addon #603] Add Apache Spark as a experimental addon (#629) * [Addon #603] Add Apache Spark as a experimental addon Signed-off-by: yanghua * [Addon #579] Reformat cue file via cue fmt command Signed-off-by: yanghua * [Addon #579] Address the review suggestion Signed-off-by: yanghua * [Addon #579] Address the review suggestion Signed-off-by: yanghua * [Addon #579] Address the review suggestion Signed-off-by: yanghua * [Addon #579] Address the review suggestion Signed-off-by: yanghua * [Addon #579] Add more details about mount volume Signed-off-by: yanghua * [Addon #579] Refactor the spark-workload parameter definition Signed-off-by: yanghua * [Addon #579] Refactor the spark-workload parameter definition and add spark-py example Signed-off-by: yanghua * [Addon #603] Refactor the spark-workload parameter definition (add proxyUser option) Signed-off-by: yanghua --------- Signed-off-by: yanghua --- .../sparkapp-py.yaml | 49 +++ .../spark-kubernetes-operator/sparkapp.yaml | 44 +++ .../spark-kubernetes-operator/README.md | 295 ++++++++++++++++++ .../definitions/spark-workload.cue | 157 ++++++++++ .../spark-kubernetes-operator/metadata.yaml | 15 + .../spark-kubernetes-operator/parameter.cue | 20 ++ .../spark-kubernetes-operator/template.cue | 102 ++++++ 7 files changed, 682 insertions(+) create mode 100644 examples/spark-kubernetes-operator/sparkapp-py.yaml create mode 100644 examples/spark-kubernetes-operator/sparkapp.yaml create mode 100644 experimental/addons/spark-kubernetes-operator/README.md create mode 100644 experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue create mode 100644 experimental/addons/spark-kubernetes-operator/metadata.yaml create mode 100644 experimental/addons/spark-kubernetes-operator/parameter.cue create mode 100644 experimental/addons/spark-kubernetes-operator/template.cue diff --git a/examples/spark-kubernetes-operator/sparkapp-py.yaml b/examples/spark-kubernetes-operator/sparkapp-py.yaml new file mode 100644 index 00000000..1731b23e --- /dev/null +++ b/examples/spark-kubernetes-operator/sparkapp-py.yaml @@ -0,0 +1,49 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: spark-app-v1 + namespace: spark-cluster +spec: + components: + - name: spark-workload-component + type: spark-workload + properties: + name: my-spark-py-app + namespace: spark-cluster + type: Python + pythonVersion: "3" + mode: cluster + image: "gcr.io/spark-operator/spark-py:v3.1.1" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.streaming.JavaQueueStream + mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.py" + sparkVersion: "3.1.1" + restartPolicy: + type: OnFailure + onFailureRetries: 3 + onFailureRetryInterval: 10 + onSubmissionFailureRetries: 5 + onSubmissionFailureRetryInterval: 20 + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml new file mode 100644 index 00000000..920b68d1 --- /dev/null +++ b/examples/spark-kubernetes-operator/sparkapp.yaml @@ -0,0 +1,44 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: spark-app-v1 + namespace: spark-cluster +spec: + components: + - name: spark-workload-component + type: spark-workload + properties: + name: my-spark-app + namespace: spark-cluster + type: Scala + mode: cluster + image: "gcr.io/spark-operator/spark:v3.1.1" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.streaming.JavaQueueStream + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar" + sparkVersion: "3.1.1" + restartPolicy: + type: Never + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "1024m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" diff --git a/experimental/addons/spark-kubernetes-operator/README.md b/experimental/addons/spark-kubernetes-operator/README.md new file mode 100644 index 00000000..fc715626 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/README.md @@ -0,0 +1,295 @@ +# spark-kubernetes-operator + +A kubernetes operator for Apache Spark(https://github.com/GoogleCloudPlatform/spark-on-k8s-operator), it allows users to manage Spark applications and their lifecycle through native K8S tooling like `kubectl`. + +> Note: It's not provided by Apache Spark. But widely used by a large number of companies(https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/who-is-using.md). + +# Install + +``` +#The following steps are for enabling fluxcd and spark-kubernetes-operator in namespace called "spark-operator". + +vela addon enable fluxcd +vela addon enable spark-kubernetes-operator +``` + +# Uninstall + +``` +vela addon disable spark-kubernetes-operator +vela addon disable fluxcd +``` + +# To check the spark-kubernetes-operator running status + +* Firstly, check the spark-kubernetes-operator (and the fluxcd and we need to deploy by helm) running status + +``` +vela addon status spark-kubernetes-operator +vela ls -A | grep spark +``` + +* Secondly, show the component type `spark-workload`, so we know how to use it in one application. As a spark user, you can choose the parameter to set for your spark cluster. + +``` +vela show spark-workload ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| name | Specify the spark application name. | string | true | | +| namespace | Specify the namespace for spark application to install. | string | true | | +| type | Specify the application language type, e.g. "Scala", "Python", "Java" or "R". | string | true | | +| pythonVersion | Specify the python version. | string | false | | +| mode | Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client". | string | true | | +| image | Specify the container image for the driver, executor, and init-container. | string | true | | +| imagePullPolicy | Specify the image pull policy for the driver, executor, and init-container. | string | true | | +| mainClass | Specify the fully-qualified main class of the Spark application. | string | true | | +| mainApplicationFile | Specify the path to a bundled JAR, Python, or R file of the application. | string | true | | +| sparkVersion | Specify the version of Spark the application uses. | string | true | | +| proxyUser | Specify the user to impersonate when submitting the application. It maps to the command-line flag | string | false | | +| | “–proxy-user” in spark-submit. | | | | +| restartPolicy | Specify the policy on if and in which conditions the controller should restart an application. | [restartPolicy](#restartpolicy) | false | | +| driver | Specify the driver sepc request for the driver pod. | [driver](#driver) | true | | +| executor | Specify the executor spec request for the executor pod. | [executor](#executor) | true | | +| arguments | Specify a list of arguments to be passed to the application. | []string | false | | +| sparkConf | Specify the config information carries user-specified Spark configuration properties as they would | map[string]string | false | | +| | use the "--conf" option in spark-submit. | | | | +| hadoopConf | Specify the config information carries user-specified Hadoop configuration properties as they would | map[string]string | false | | +| | use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds | | | | +| | prefix "spark.hadoop." to Hadoop configuration properties. | | | | +| sparkConfigMap | Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The | string | false | | +| | controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| hadoopConfigMap | Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The | string | false | | +| | controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted | | | | +| | to. | | | | +| volumes | Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors. | [[]volumes](#volumes) | false | | +| deps | Specify the dependencies captures all possible types of dependencies of a Spark application. | [deps](#deps) | false | | ++---------------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ + + +## restartPolicy ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ +| type | Type value option: "Always", "Never", "OnFailure". | string | true | | +| onSubmissionFailureRetries | Specify the number of times to retry submitting an application before giving up. This is best | int | false | | +| | effort and actual retry attempts can be >= the value specified due to caching. These are required if | | | | +| | RestartPolicy is OnFailure. | | | | +| onFailureRetries | Specify the number of times to retry running an application before giving up. | int | false | | +| onSubmissionFailureRetryInterval | Specify the interval in seconds between retries on failed submissions. | int | false | | +| onFailureRetryInterval | Specify the interval in seconds between retries on failed runs. | int | false | | ++----------------------------------+------------------------------------------------------------------------------------------------------+--------+----------+---------+ + + +## driver ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| cores | Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, | int | false | | +| | respectively. | | | | +| coreLimit | Specify a hard limit on CPU cores for the pod. | string | false | | +| memory | Specify the amount of memory to request for the pod. | string | false | | +| labels | Specify the Kubernetes labels to be added to the pod. | map[string]string | false | | +| volumeMounts | Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem. | [[]volumeMounts](#volumemounts) | false | | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ + + +### volumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ + + +## executor ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ +| cores | Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, | int | false | | +| | respectively. | | | | +| coreLimit | Specify a hard limit on CPU cores for the pod. | string | false | | +| memory | Specify the amount of memory to request for the pod. | string | false | | +| instances | Specify the number of executor instances. | int | false | | +| labels | Specify the Kubernetes labels to be added to the pod. | map[string]string | false | | +| volumeMounts | Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem. | [[]volumeMounts](#volumemounts) | false | | ++--------------+------------------------------------------------------------------------------------------------------+---------------------------------+----------+---------+ + + +### volumeMounts ++-----------+-------------+--------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------+-------------+--------+----------+---------+ +| name | | string | true | | +| mountPath | | string | true | | ++-----------+-------------+--------+----------+---------+ + + +## volumes ++----------+-------------+-----------------------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++----------+-------------+-----------------------+----------+---------+ +| name | | string | true | | +| hostPath | | [hostPath](#hostpath) | true | | ++----------+-------------+-----------------------+----------+---------+ + + +### hostPath ++------+-------------+--------+----------+-----------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++------+-------------+--------+----------+-----------+ +| path | | string | true | | +| type | | string | false | Directory | ++------+-------------+--------+----------+-----------+ + + +## deps ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ +| NAME | DESCRIPTION | TYPE | REQUIRED | DEFAULT | ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ +| jars | Specify a list of JAR files the Spark application depends on. | []string | false | | +| files | Specify a list of files the Spark application depends on. | []string | false | | +| pyFiles | Specify a list of Python files the Spark application depends on. | []string | false | | +| packages | Specify a list of maven coordinates of jars to include on the driver and executor classpaths. This | []string | false | | +| | will search the local maven repo, then maven central and any additional remote repositories given by | | | | +| | the “repositories” option. Each package should be of the form “groupId:artifactId:version”. | | | | +| excludePackages | Specify a list of “groupId:artifactId”, to exclude while resolving the dependencies provided in | []string | false | | +| | Packages to avoid dependency conflicts. | | | | +| repositories | Specify a list of additional remote repositories to search for the maven coordinate given with the | []string | false | | +| | “packages” option. | | | | ++-----------------+------------------------------------------------------------------------------------------------------+----------+----------+---------+ +``` + +# Example for how to run a component typed spark-cluster in application + +**Note**: If we want to check and verify the mount volume, we need to specify `parameter.createWebhook` to be `true`. For more details, please check the [official documentation](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md#mounting-volumes). + +1. Firstly, download or copy `catalog/examples/spark-kubernetes-operator/sparkapp.yaml` + +2. Secondly, start the application: + +``` +vela up -f sparkapp.yaml +``` + +You will see the stdout like this: + +``` +Applying an application in vela K8s object format... +I0227 16:54:37.069480 361176 apply.go:121] "creating object" name="spark-app-v1" resource="core.oam.dev/v1beta1, Kind=Application" +✅ App has been deployed 🚀🚀🚀 + Port forward: vela port-forward spark-app-v1 -n spark-cluster + SSH: vela exec spark-app-v1 -n spark-cluster + Logging: vela logs spark-app-v1 -n spark-cluster + App status: vela status spark-app-v1 -n spark-cluster + Endpoint: vela status spark-app-v1 -n spark-cluster --endpoint +Application spark-cluster/spark-app-v1 applied. +``` + +3. Then, there are serval ways to check the status(or detail information) of the Spark applicaiton: + +option 1: + +``` +vela status spark-app-v1 -n spark-cluster +About: + + Name: spark-app-v1 + Namespace: spark-cluster + Created at: 2023-02-27 16:54:37 +0800 CST + Status: running + +Workflow: + + mode: DAG-DAG + finished: true + Suspend: false + Terminated: false + Steps + - id: vfgjkrxvih + name: spark-workload-component + type: apply-component + phase: succeeded + +Services: + + - Name: spark-workload-component + Cluster: local Namespace: spark-cluster + Type: spark-workload + Healthy + No trait applied +``` + +option 2: + +``` +kubectl get sparkapplications -n spark-cluster +NAME STATUS ATTEMPTS START FINISH AGE +my-spark-app RUNNING 1 2023-02-27T08:54:40Z 2m33s +``` + +option 3: + +``` +kubectl describe sparkapplication my-spark-app -n spark-cluster +Name: my-spark-app +Namespace: spark-cluster +Labels: app.oam.dev/app-revision-hash=4e5592aea53a5961 + app.oam.dev/appRevision=spark-app-v1-v1 + app.oam.dev/cluster=local + app.oam.dev/component=my-spark-application-component + app.oam.dev/name=spark-app-v1 + app.oam.dev/namespace=spark-cluster + app.oam.dev/resourceType=TRAIT + app.oam.dev/revision= + oam.dev/render-hash=640a3298d803274e + trait.oam.dev/resource=spark + trait.oam.dev/type=AuxiliaryWorkload +Annotations: app.oam.dev/last-applied-configuration: + {"apiVersion":"sparkoperator.k8s.io/v1beta2","kind":"SparkApplication","metadata":{"annotations":{"app.oam.dev/last-applied-time":"2023-02... + app.oam.dev/last-applied-time: 2023-02-27T16:54:37+08:00 + oam.dev/kubevela-version: v1.7.0 +API Version: sparkoperator.k8s.io/v1beta2 +Kind: SparkApplication +Metadata: +...... +``` + +option 4: + +``` +kubectl get app spark-app-v1 -n spark-cluster -oyaml +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: +...... +``` + +4. Show the service of spark application via this command: + +``` +kubectl get svc -n spark-cluster +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +my-spark-app-c58a1c869214bfe5-driver-svc ClusterIP None 7078/TCP,7079/TCP,4040/TCP 19m +my-spark-app-ui-svc ClusterIP xx.xx.xx.xx 4040/TCP 19m +``` + +# Architecture + +> The following content is quoted from the [original design document](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/design.md). + +The operator consists of: +* a `SparkApplication` controller that watches events of creation, updates, and deletion of +`SparkApplication` objects and acts on the watch events, +* a *submission runner* that runs `spark-submit` for submissions received from the controller, +* a *Spark pod monitor* that watches for Spark pods and sends pod status updates to the controller, +* a [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) that handles customizations for Spark driver and executor pods based on the annotations on the pods added by the controller, +* and also a command-line tool named `sparkctl` for working with the operator. + +The following diagram shows how different components interact and work together. + +![Architecture Diagram](https://raw.githubusercontent.com/GoogleCloudPlatform/spark-on-k8s-operator/master/docs/architecture-diagram.png) + +Specifically, a user uses the `sparkctl` (or `kubectl`) to create a `SparkApplication` object. The `SparkApplication` controller receives the object through a watcher from the API server, creates a submission carrying the `spark-submit` arguments, and sends the submission to the *submission runner*. The submission runner submits the application to run and creates the driver pod of the application. Upon starting, the driver pod creates the executor pods. While the application is running, the *Spark pod monitor* watches the pods of the application and sends status updates of the pods back to the controller, which then updates the status of the application accordingly. diff --git a/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue new file mode 100644 index 00000000..e655e211 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/definitions/spark-workload.cue @@ -0,0 +1,157 @@ +"spark-workload": { + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "Describes a containerized spark application that can specify resource spec." + labels: {} + type: "component" +} + +template: { + parameter: { + // +usage=Specify the spark application name + name: string + // +usage=Specify the namespace for spark application to install + namespace: string + // +usage=Specify the application language type, e.g. "Scala", "Python", "Java" or "R" + type: string + // +usage=Specify the python version + pythonVersion?: string + // +usage=Specify the deploy mode, e.go "cluster", "client" or "in-cluster-client" + mode: string + // +usage=Specify the container image for the driver, executor, and init-container + image: string + // +usage=Specify the image pull policy for the driver, executor, and init-container + imagePullPolicy: string + // +usage=Specify the fully-qualified main class of the Spark application + mainClass: string + // +usage=Specify the path to a bundled JAR, Python, or R file of the application + mainApplicationFile: string + // +usage=Specify the version of Spark the application uses + sparkVersion: string + // +usage=Specify the user to impersonate when submitting the application. It maps to the command-line flag “–proxy-user” in spark-submit + proxyUser?: string + // +usage=Specify the policy on if and in which conditions the controller should restart an application + restartPolicy?: { + // +usage=Type value option: "Always", "Never", "OnFailure" + type: string + // +usage=Specify the number of times to retry submitting an application before giving up. This is best effort and actual retry attempts can be >= the value specified due to caching. These are required if RestartPolicy is OnFailure + onSubmissionFailureRetries?: int + // +usage=Specify the number of times to retry running an application before giving up + onFailureRetries?: int + // +usage=Specify the interval in seconds between retries on failed submissions + onSubmissionFailureRetryInterval?: int + // +usage=Specify the interval in seconds between retries on failed runs + onFailureRetryInterval?: int + } + // +usage=Specify the driver sepc request for the driver pod + driver: { + // +usage=Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, respectively + cores?: int + // +usage=Specify a hard limit on CPU cores for the pod + coreLimit?: string + // +usage=Specify the amount of memory to request for the pod + memory?: string + // +usage=Specify the Kubernetes labels to be added to the pod + labels?: [string]: string + // +usage=Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem + volumeMounts?: [...{ + name: string + mountPath: string + }] + } + // +usage=Specify the executor spec request for the executor pod + executor: { + // +usage=Specify the cores maps to spark.driver.cores or spark.executor.cores for the driver and executors, respectively + cores?: int + // +usage=Specify a hard limit on CPU cores for the pod + coreLimit?: string + // +usage=Specify the amount of memory to request for the pod + memory?: string + // +usage=Specify the number of executor instances + instances?: int + // +usage=Specify the Kubernetes labels to be added to the pod + labels?: [string]: string + // +usage=Specify the volumes listed in “.spec.volumes” to mount into the main container’s filesystem + volumeMounts?: [...{ + name: string + mountPath: string + }] + } + // +usage=Specify a list of arguments to be passed to the application + arguments?: [...string] + // +usage=Specify the config information carries user-specified Spark configuration properties as they would use the "--conf" option in spark-submit + sparkConf?: [string]: string + // +usage=Specify the config information carries user-specified Hadoop configuration properties as they would use the the "--conf" option in spark-submit. The SparkApplication controller automatically adds prefix "spark.hadoop." to Hadoop configuration properties + hadoopConf?: [string]: string + // +usage=Specify the name of the ConfigMap containing Spark configuration files such as log4j.properties. The controller will add environment variable SPARK_CONF_DIR to the path where the ConfigMap is mounted to + sparkConfigMap?: string + // +usage=Specify the name of the ConfigMap containing Hadoop configuration files such as core-site.xml. The controller will add environment variable HADOOP_CONF_DIR to the path where the ConfigMap is mounted to + hadoopConfigMap?: string + // +usage=Specify the list of Kubernetes volumes that can be mounted by the driver and/or executors + volumes?: [...{ + name: string + hostPath: { + path: string + type: *"Directory" | string + } + }] + // +usage=Specify the dependencies captures all possible types of dependencies of a Spark application + deps?: { + // +usage=Specify a list of JAR files the Spark application depends on + jars?: [...string] + // +usage=Specify a list of files the Spark application depends on + files?: [...string] + // +usage=Specify a list of Python files the Spark application depends on + pyFiles?: [...string] + // +usage=Specify a list of maven coordinates of jars to include on the driver and executor classpaths. This will search the local maven repo, then maven central and any additional remote repositories given by the “repositories” option. Each package should be of the form “groupId:artifactId:version” + packages?: [...string] + // +usage=Specify a list of “groupId:artifactId”, to exclude while resolving the dependencies provided in Packages to avoid dependency conflicts + excludePackages?: [...string] + // +usage=Specify a list of additional remote repositories to search for the maven coordinate given with the “packages” option + repositories?: [...string] + } + } + + output: { + kind: "ClusterRoleBinding" + apiVersion: "rbac.authorization.k8s.io/v1" + metadata: name: parameter.name + roleRef: { + name: "edit" + apiGroup: "rbac.authorization.k8s.io" + kind: "ClusterRole" + } + subjects: [{ + name: "default" + kind: "ServiceAccount" + namespace: parameter.namespace + }] + } + + outputs: { + + "spark": { + kind: "SparkApplication" + apiVersion: "sparkoperator.k8s.io/v1beta2" + metadata: { + name: parameter.name + namespace: parameter.namespace + } + spec: { + type: parameter.type + mode: parameter.mode + image: parameter.image + imagePullPolicy: parameter.imagePullPolicy + mainClass: parameter.mainClass + mainApplicationFile: parameter.mainApplicationFile + sparkVersion: parameter.sparkVersion + driver: parameter.driver + executor: parameter.executor + if parameter.volumes != _|_ { + volumes: parameter.volumes + } + + } + } + } +} diff --git a/experimental/addons/spark-kubernetes-operator/metadata.yaml b/experimental/addons/spark-kubernetes-operator/metadata.yaml new file mode 100644 index 00000000..65d95dd6 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/metadata.yaml @@ -0,0 +1,15 @@ +description: A kubernetes operator for Apache Spark +icon: "https://spark.apache.org/images/spark-logo.png" +invisible: false +name: spark-kubernetes-operator +tags: + - GoogleCloudPlatform/spark-on-k8s-operator +version: v1beta2-1.3.8-3.1.1 +url: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator + +dependencies: + - name: fluxcd + +system: + vela: ">=1.5.0-beta.3" + kubernetes: ">=1.16" diff --git a/experimental/addons/spark-kubernetes-operator/parameter.cue b/experimental/addons/spark-kubernetes-operator/parameter.cue new file mode 100644 index 00000000..0a47568e --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/parameter.cue @@ -0,0 +1,20 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Deploy to specified clusters. Leave empty to deploy to all clusters. + clusters?: [...string] + // +usage=Namespace to deploy to + namespace: *"spark-operator" | string + // +usage=Specify if create the webhook or not + "createWebhook": *false | bool + // +usage=Specify the image repository + "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string + // +usage=Specify the image tag + "imageTag": *"v1beta2-1.3.8-3.1.1" | string + // +usage=Specify if create the sa for job or not + "createSparkServiceAccount": *false | bool +} diff --git a/experimental/addons/spark-kubernetes-operator/template.cue b/experimental/addons/spark-kubernetes-operator/template.cue new file mode 100644 index 00000000..6d9d6af4 --- /dev/null +++ b/experimental/addons/spark-kubernetes-operator/template.cue @@ -0,0 +1,102 @@ +package main + +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + type: "k8s-objects" + name: "spark-operator-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: parameter.namespace + }] + }, + { + type: "k8s-objects" + name: "spark-cluster-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: "spark-cluster" + }, + { + apiVersion: "v1" + kind: "ServiceAccount" + metadata: { + name: "spark" + namespace: "spark-cluster" + } + }] + }, + { + name: "spark-operator-helm" + type: "helm" + dependsOn: ["spark-operator-ns"] + type: "helm" + properties: { + repoType: "helm" + url: "https://googlecloudplatform.github.io/spark-on-k8s-operator/" + chart: "spark-operator" + targetNamespace: parameter["namespace"] + version: "1.1.26" + values: { + image: { + repository: parameter["imageRepository"] + tag: parameter["imageTag"] + } + + serviceAccounts: { + spark: { + create: parameter["createSparkServiceAccount"] + } + } + + serviceAccounts: { + sparkoperator: { + name: "spark-kubernetes-operator" + } + } + + webhook: { + enable: parameter["createWebhook"] + } + } + } + }, + ] + + policies: [ + { + name: "gc-dependency" + type: "garbage-collect" + properties: { + order: "dependency" + } + }, + { + type: "shared-resource" + name: "shared-resource-via-namespace" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + }, + { + type: "topology" + name: "deploy-operator" + properties: { + namespace: parameter.namespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + }, + ] + } +}