From 258a6a9c87162ba7d067bc124fad29de8ec32aea Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 31 Oct 2022 16:08:59 -0700 Subject: [PATCH 1/2] update to fit azure marketplace image --- run_nytaxi_k8s.sh | 4 ++-- run_spark_pi.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/run_nytaxi_k8s.sh b/run_nytaxi_k8s.sh index 4a05ce5..bd17313 100644 --- a/run_nytaxi_k8s.sh +++ b/run_nytaxi_k8s.sh @@ -1,5 +1,5 @@ #!/bin/bash - +IMAGE=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 ${SPARK_HOME}/bin/spark-submit \ --master k8s://https://${kubernetes_master_url}:${k8s_apiserver_port} \ --deploy-mode cluster \ @@ -9,7 +9,7 @@ ${SPARK_HOME}/bin/spark-submit \ --executor-cores 4 \ --conf spark.executor.instances=1 \ --conf spark.rpc.netty.dispatcher.numThreads=128 \ - --conf spark.kubernetes.container.image=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 \ + --conf spark.kubernetes.container.image=${IMAGE} \ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \ --conf spark.kubernetes.executor.deleteOnTermination=false \ --conf spark.kubernetes.driver.podTemplateFile=./driver.yaml \ diff --git a/run_spark_pi.sh b/run_spark_pi.sh index cff2116..c138a54 100644 --- a/run_spark_pi.sh +++ b/run_spark_pi.sh @@ -1,5 +1,5 @@ #!/bin/bash - +IMAGE=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 ${SPARK_HOME}/bin/spark-submit \ --master k8s://https://${kubernetes_master_url}:${k8s_apiserver_port} \ --deploy-mode cluster \ @@ -7,7 +7,7 @@ ${SPARK_HOME}/bin/spark-submit \ --class org.apache.spark.examples.SparkPi \ --conf spark.executor.instances=1 \ --conf spark.rpc.netty.dispatcher.numThreads=32 \ - --conf spark.kubernetes.container.image=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 \ + --conf spark.kubernetes.container.image=${IMAGE} \ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \ --conf spark.kubernetes.executor.deleteOnTermination=false \ --conf spark.kubernetes.driver.podTemplateFile=./driver.yaml \ From 17e61e35803e5508a1026974447697b7eb298e12 Mon Sep 17 00:00:00 2001 From: jenniew Date: Fri, 4 Nov 2022 13:09:09 -0700 Subject: [PATCH 2/2] add k8s scripts to image --- docker/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 81514da..9174192 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -66,5 +66,12 @@ RUN chmod a+x /opt/entrypoint.sh && \ ADD ../nytaxi/target/spark-azure-nytaxi-1.0-SNAPSHOT-jar-with-dependencies.jar /root/spark-azure-nytaxi-1.0-SNAPSHOT.jar RUN cp /root/spark-azure-nytaxi-1.0-SNAPSHOT.jar /opt/bigdl-2.1.0-SNAPSHOT/jars/ ADD ./run_azure_nytaxi.sh /root/run_azure_nytaxi.sh +ADD ./run_spark_pi.sh /root/run_spark_pi.sh +ADD ./run_nytaxi_k8s.sh /root/run_nytaxi_k8s.sh +ADD ./driver.yaml /root/driver.yaml +ADD ./executor.yaml /root/executor.yaml + +# install azure cli +RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash ENTRYPOINT [ "/opt/entrypoint.sh" ]