diff --git a/automation/src/test/resources/reference.conf b/automation/src/test/resources/reference.conf index 53e651fe8d..3879d82dc0 100644 --- a/automation/src/test/resources/reference.conf +++ b/automation/src/test/resources/reference.conf @@ -1,4 +1,5 @@ leonardo { + baseImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-base:1.0.0" rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7" pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6" hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.14" diff --git a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala index 9ae1309bf7..2232a47c87 100644 --- a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala +++ b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala @@ -12,14 +12,14 @@ object LeonardoConfig extends CommonConfig { object Leonardo { val apiUrl: String = leonardo.getString("apiUrl") val notebooksServiceAccountEmail: String = leonardo.getString("notebooksServiceAccountEmail") + val baseImageUrl: String = leonardo.getString("baseImageUrl") val rImageUrl: String = leonardo.getString("rImageUrl") val pythonImageUrl: String = leonardo.getString("pythonImageUrl") val hailImageUrl: String = leonardo.getString("hailImageUrl") val gatkImageUrl: String = leonardo.getString("gatkImageUrl") val aouImageUrl: String = leonardo.getString("aouImageUrl") - val baseImageUrl: String = leonardo.getString("baseImageUrl") val rstudioBioconductorImage = - ContainerImage(leonardo.getString("rstudioBioconductorImageUrl"), ContainerRegistry.GCR) + ContainerImage(leonardo.getString("rstudioBioconductorImageUrl"), ContainerRegistry.GAR) private val topic = ProjectTopicName.of(gcs.getString("serviceProject"), leonardo.getString("topicName")) val location: Location = Location(leonardo.getString("location")) diff --git a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala index ce3d386012..d4352ecef7 100644 --- a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala +++ b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala @@ -115,7 +115,7 @@ trait RuntimeFixtureSpec billingProject, runtimeName, getRuntimeRequest(cloudService.getOrElse(CloudService.GCE), - toolDockerImage.map(i => ContainerImage(i, ContainerRegistry.GCR)), + toolDockerImage.map(i => ContainerImage(i, ContainerRegistry.GAR)), welderRegistry ) ) diff --git a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeCreationDiskSpec.scala b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeCreationDiskSpec.scala index fe658fe86d..83e7c084bb 100644 --- a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeCreationDiskSpec.scala +++ b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeCreationDiskSpec.scala @@ -213,7 +213,7 @@ class RuntimeCreationDiskSpec extends BillingProjectFixtureSpec with ParallelTes ) val createRuntime2Request = createRuntimeRequest.copy(toolDockerImage = - Some(ContainerImage(LeonardoConfig.Leonardo.pythonImageUrl, ContainerRegistry.GCR)) + Some(ContainerImage(LeonardoConfig.Leonardo.pythonImageUrl, ContainerRegistry.GAR)) ) // this just needs to be a different image from default image Leonardo uses, which is gatk val createRuntimeCloneRequest = createRuntime2Request.copy( diff --git a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala index 5a9659aed6..0159c67794 100644 --- a/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala +++ b/automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala @@ -106,7 +106,7 @@ class RuntimeDataprocSpec extends BillingProjectFixtureSpec with ParallelTestExe false ) ), - toolDockerImage = Some(ContainerImage(LeonardoConfig.Leonardo.hailImageUrl, ContainerRegistry.GCR)) + toolDockerImage = Some(ContainerImage(LeonardoConfig.Leonardo.hailImageUrl, ContainerRegistry.GAR)) ) getRuntimeResponse <- LeonardoApiClient.createRuntimeWithWait(project, runtimeName, createRuntimeRequest) runtime = ClusterCopy.fromGetRuntimeResponseCopy(getRuntimeResponse) diff --git a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/containerModels.scala b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/containerModels.scala index 860051cc90..44e8994104 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/containerModels.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/containerModels.scala @@ -12,10 +12,12 @@ sealed trait ContainerRegistry extends EnumEntry with Product with Serializable object ContainerRegistry extends Enum[ContainerRegistry] { val values = findValues - final case object GCR extends ContainerRegistry { + final case object GAR extends ContainerRegistry { val regex: Regex = - """^((?:us\.|eu\.|asia\.)?gcr.io)/([\w.-]+/[\w.-]+)(?::(\w[\w.-]+))?(?:@([\w+.-]+:[A-Fa-f0-9]{32,}))?$""".r - override def toString: String = "GCR" + // with GCR switching to GAR but keeping the old addresses valid, this accepts both gcr.io and gar.io addresses + """^((?:us\.|eu\.|asia\.)?(?:gcr|gar)\.io)/([\w.-]+/[\w.-]+)(?::(\w[\w.-]+))?(?:@([\w+.-]+:[A-Fa-f0-9]{32,}))?$""".r + + override def toString: String = "GAR" } final case object GHCR extends ContainerRegistry { diff --git a/http/src/main/resources/init-resources/gce-init.sh b/http/src/main/resources/init-resources/gce-init.sh index 50196db598..1cee09dee7 100644 --- a/http/src/main/resources/init-resources/gce-init.sh +++ b/http/src/main/resources/init-resources/gce-init.sh @@ -537,6 +537,7 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then # Starts the locking logic (used for AOU). google_sign_in.js is likely not used anymore docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_HOME/scripts/extension/install_jupyter_contrib_nbextensions.sh \ && mkdir -p $JUPYTER_USER_HOME/.jupyter/custom/ \ + && chown -R jupyter:users $JUPYTER_USER_HOME/.jupyter \ && cp $JUPYTER_HOME/custom/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ && ls -la $JUPYTER_HOME/custom/extension_entry_jupyter.js \ && cp $JUPYTER_HOME/custom/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ diff --git a/http/src/main/resources/init-resources/init-actions.sh b/http/src/main/resources/init-resources/init-actions.sh index a48d0f2d99..b665ba6dfb 100644 --- a/http/src/main/resources/init-resources/init-actions.sh +++ b/http/src/main/resources/init-resources/init-actions.sh @@ -275,7 +275,7 @@ EOF # If any image is hosted in a GAR registry (detected by regex) then # authorize docker to interact with gcr.io. # NOTE: GCR images are now hosted on GAR, but the file paths haven't changed, they automatically redirect. - if grep -qF "gcr.io" <<< "${JUPYTER_DOCKER_IMAGE}${RSTUDIO_DOCKER_IMAGE}${PROXY_DOCKER_IMAGE}${WELDER_DOCKER_IMAGE}" ; then + if grep -qF "gcr.io|gar.io" <<< "${JUPYTER_DOCKER_IMAGE}${RSTUDIO_DOCKER_IMAGE}${PROXY_DOCKER_IMAGE}${WELDER_DOCKER_IMAGE}" ; then log 'Authorizing GCR/GAR...' gcloud auth configure-docker fi @@ -329,7 +329,6 @@ EOF # Jupyter-specific setup, only do if Jupyter is installed if [ ! -z ${JUPYTER_DOCKER_IMAGE} ] ; then - log 'Installing Jupydocker kernelspecs...' # Install notebook.json if [ ! -z ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} ] ; then @@ -476,6 +475,7 @@ EOF # Install the custom jupyter extensions needed to lock notebooks into edit or safe modes (required by AOU) docker exec -u 0 $JUPYTER_SERVER_NAME /bin/bash -c "$JUPYTER_HOME/scripts/extension/install_jupyter_contrib_nbextensions.sh \ && mkdir -p $JUPYTER_USER_HOME/.jupyter/custom/ \ + && chown -R jupyter:users $JUPYTER_USER_HOME/.jupyter \ && cp $JUPYTER_HOME/custom/google_sign_in.js $JUPYTER_USER_HOME/.jupyter/custom/ \ && ls -la $JUPYTER_HOME/custom/extension_entry_jupyter.js \ && cp $JUPYTER_HOME/custom/extension_entry_jupyter.js $JUPYTER_USER_HOME/.jupyter/custom/custom.js \ diff --git a/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml b/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml index 375e4209ee..0e47b21914 100644 --- a/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml +++ b/http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml @@ -33,8 +33,9 @@ services: R_LIBS: "${NOTEBOOKS_DIR}/packages" # The next two lines aren't great. But they're for updating PYTHONPATH, PATH in older than (inclusive) us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.2 # We should remove the two lines once we no longer support older images. In the meantime, we need to be careful updating Jupyter base images. + # NOTE: its very important to update these paths to access any tools added or moved in the base image PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages" - PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" + PATH: "/root/.local/bin:/etc/jupyter/bin:/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${NOTEBOOKS_DIR}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" env_file: - /var/custom_env_vars.env # See https://docs.docker.com/engine/reference/run/#user-memory-constraints diff --git a/http/src/main/resources/init-resources/jupyter-docker-compose.yaml b/http/src/main/resources/init-resources/jupyter-docker-compose.yaml index 34f538e706..0b0b50ce60 100644 --- a/http/src/main/resources/init-resources/jupyter-docker-compose.yaml +++ b/http/src/main/resources/init-resources/jupyter-docker-compose.yaml @@ -51,8 +51,9 @@ services: # The next two lines aren't great. But they're for updating PYTHONPATH, PATH in older than (inclusive) us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.0.2. # We should remove the two lines once we no longer support older images. # When we update base image in terra-docker next time, we should verify the paths are still valid + # NOTE: its very important to update these paths to access any tools added or moved in the base image PYTHONPATH: "/etc/jupyter/custom:/usr/lib/spark/python:${NOTEBOOKS_DIR}/packages" - PATH: "/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" + PATH: "/root/.local/bin:/etc/jupyter/bin:/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${NOTEBOOKS_DIR}/packages/bin" env_file: - /var/custom_env_vars.env # See https://docs.docker.com/engine/reference/run/#user-memory-constraints diff --git a/http/src/main/resources/init-resources/startup.sh b/http/src/main/resources/init-resources/startup.sh index 7e90819ac2..6221068a8f 100644 --- a/http/src/main/resources/init-resources/startup.sh +++ b/http/src/main/resources/init-resources/startup.sh @@ -365,8 +365,18 @@ if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then # This is to make it so that older images will still work after we change notebooks location to home dir docker exec ${JUPYTER_SERVER_NAME} sed -i '/^# to mount there as it effectively deletes existing files on the image/,+5d' ${JUPYTER_HOME}/jupyter_notebook_config.py + # Determine jupyter path based on image type + # images based on the new terra-base have jupyter installed in /etc/jupyter/bin/jupyter + # this marker is added as one of the last steps in the new base image here: + # https://github.com/DataBiosphere/terra-docker/blob/9aa343ae91ad1c94ca5a9f9e5bee02cc5d943b58/terra-base/Dockerfile#L235 + if docker exec $JUPYTER_SERVER_NAME test -f /etc/jupyter/.terra-base-marker; then + JUPYTER_PATH="/etc/jupyter/bin/jupyter" + else + JUPYTER_PATH="/opt/conda/bin/jupyter" + fi + # Start Jupyter server - docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export WELDER_ENABLED=$WELDER_ENABLED && export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || /opt/conda/bin/jupyter notebook)" + docker exec -d $JUPYTER_SERVER_NAME /bin/bash -c "export WELDER_ENABLED=$WELDER_ENABLED && export NOTEBOOKS_DIR=$NOTEBOOKS_DIR && (/etc/jupyter/scripts/run-jupyter.sh $NOTEBOOKS_DIR || $JUPYTER_PATH notebook)" fi # Configuring RStudio, if enabled diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala index cb6bd33d82..0ba9e36c91 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAO.scala @@ -107,7 +107,7 @@ class HttpDockerDAO[F[_]] private (httpClient: Client[F])(implicit logger: Logge ): F[Option[Token]] = parsedImage.registry match { // If it's a GCR repo, use the pet token - case ContainerRegistry.GCR => F.pure(petTokenOpt.map(Token)) + case ContainerRegistry.GAR => F.pure(petTokenOpt.map(Token)) // If it's a Dockerhub repo, need to request a token from Dockerhub case ContainerRegistry.DockerHub => httpClient.expectOptionOr[Token]( @@ -150,14 +150,14 @@ class HttpDockerDAO[F[_]] private (httpClient: Client[F])(implicit logger: Logge private[dao] def parseImage(image: ContainerImage)(implicit ev: Ask[F, TraceId]): F[ParsedImage] = image.imageUrl match { - case GCR.regex(registry, imageName, tagOpt, shaOpt) => + case GAR.regex(registry, imageName, tagOpt, shaOpt) => val version = Option(tagOpt) .map(Tag) .orElse(Option(shaOpt).map(Sha)) for { traceId <- ev.ask res <- version.fold(F.raiseError[ParsedImage](ImageParseException(traceId, image)))(i => - F.pure(ParsedImage(GCR, Uri.unsafeFromString(s"https://$registry/v2"), imageName, i)) + F.pure(ParsedImage(GAR, Uri.unsafeFromString(s"https://$registry/v2"), imageName, i)) ) } yield res case DockerHub.regex(imageName, tagOpt, shaOpt) => diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala index 6f4e028ecb..f697e7c52a 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BaseRuntimeInterpreter.scala @@ -159,7 +159,7 @@ abstract private[util] class BaseRuntimeInterpreter[F[_]]( .toRight(new Exception(s"Unable to update welder because current welder image is not available")) .flatMap(x => x.registry match { - case Some(ContainerRegistry.GCR) | Some(ContainerRegistry.GHCR) => + case Some(ContainerRegistry.GAR) | Some(ContainerRegistry.GHCR) => Right(config.imageConfig.welderGcrImage.imageUrl) case Some(ContainerRegistry.DockerHub) => Right(config.imageConfig.welderDockerHubImage.imageUrl) case None => Left(new Exception(s"Unable to update Welder: registry for ${x.imageUrl} not parsable")) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAOSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAOSpec.scala index d329254e58..e6ff275c50 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAOSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpDockerDAOSpec.scala @@ -6,7 +6,7 @@ import cats.effect.unsafe.implicits.global import io.circe.CursorOp.DownField import io.circe.DecodingFailure import io.circe.parser.decode -import org.broadinstitute.dsde.workbench.leonardo.ContainerRegistry.{DockerHub, GCR, GHCR} +import org.broadinstitute.dsde.workbench.leonardo.ContainerRegistry.{DockerHub, GAR, GHCR} import org.broadinstitute.dsde.workbench.leonardo.RuntimeImageType.{Jupyter, RStudio} import org.broadinstitute.dsde.workbench.leonardo.dao.HttpDockerDAO._ import org.broadinstitute.dsde.workbench.leonardo.model.InvalidImage @@ -30,10 +30,10 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll // "broadinstitute/leonardo-notebooks@sha256:bb959cf74f31d2a10f7bb8ee0f0754138d7c90f7ed8a92c3697ac994ff8b40b7" // ), // gcr with tag - ContainerImage("us.gcr.io/broad-dsp-gcr-public/leonardo-jupyter:dev", GCR), - ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6", GCR), - ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7", GCR), - ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.9", GCR), + ContainerImage("us.gcr.io/broad-dsp-gcr-public/leonardo-jupyter:dev", GAR), + ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6", GAR), + ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7", GAR), + ContainerImage("us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.9", GAR), // gcr with sha // TODO shas are currently not working // GCR( @@ -55,7 +55,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll // "rocker/rstudio@sha256:5aea617714eb38a97a21de652ab667c6d7bb486d7468a4ab6b4d515154fec383" // ), // gcr with tag - ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base:0.0.1", GCR) + ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base:0.0.1", GAR) // gcr with sha // TODO shas are currently not working // GCR( @@ -83,7 +83,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll } it should s"detect ImageParseException" in withDockerDAO { dockerDAO => - val image = ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base", GCR) // non existent tag + val image = ContainerImage("us.gcr.io/anvil-gcr-public/anvil-rstudio-base", GAR) // non existent tag val res = for { ctx <- appContext.ask[AppContext] response <- dockerDAO.detectTool(image, None, ctx.now).attempt @@ -93,7 +93,7 @@ class HttpDockerDAOSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll it should s"detect invalid GCR image if image doesn't have proper environment variables set" in withDockerDAO { dockerDAO => - val image = ContainerImage("us.gcr.io/broad-dsp-gcr-public/welder-server:latest", GCR) // not a supported tool + val image = ContainerImage("us.gcr.io/broad-dsp-gcr-public/welder-server:latest", GAR) // not a supported tool val res = for { ctx <- appContext.ask[AppContext] response <- dockerDAO.detectTool(image, None, ctx.now).attempt diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala index 92c030a7ae..841a5de6e6 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala @@ -378,7 +378,7 @@ class RuntimeServiceInterpTest it should "successfully create a cluster with an rstudio image" in isolatedDbTest { val cloudContext = CloudContext.Gcp(GoogleProject("googleProject")) val runtimeName = RuntimeName("clusterName2") - val rstudioImage = ContainerImage("some-rstudio-image", ContainerRegistry.GCR) + val rstudioImage = ContainerImage("some-rstudio-image", ContainerRegistry.GAR) val request = emptyCreateRuntimeReq.copy( toolDockerImage = Some(rstudioImage) ) @@ -564,7 +564,7 @@ class RuntimeServiceInterpTest userInfo, cloudContext, runtimeName2, - emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GCR)) + emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GAR)) ) .attempt r3 <- runtimeService @@ -634,7 +634,7 @@ class RuntimeServiceInterpTest userInfo, cloudContext, runtimeName2, - emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GCR)) + emptyCreateRuntimeReq.copy(welderRegistry = Some(ContainerRegistry.GAR)) ) .attempt diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModelSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModelSpec.scala index 9092a7cf04..6512ca3b20 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModelSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/model/LeonardoModelSpec.scala @@ -169,14 +169,16 @@ class LeonardoModelSpec extends LeonardoTestSuite with AnyFlatSpecLike { } "DockerRegistry regex" should "match expected image url format" in { - ContainerRegistry.GCR.regex.pattern.asPredicate().test("us.gcr.io/google/ubuntu1804:latest") shouldBe true - ContainerRegistry.GCR.regex.pattern.asPredicate().test("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true - ContainerRegistry.GCR.regex.pattern.asPredicate().test("us/broad-dsp-gcr-public/ubuntu1804") shouldBe false - ContainerRegistry.GCR.regex.pattern.asPredicate().test("eu.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true - ContainerRegistry.GCR.regex.pattern + ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gcr.io/google/ubuntu1804:latest") shouldBe true + ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true + ContainerRegistry.GAR.regex.pattern.asPredicate().test("us.gar.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true + ContainerRegistry.GAR.regex.pattern.asPredicate().test("us/broad-dsp-gcr-public/ubuntu1804") shouldBe false + ContainerRegistry.GAR.regex.pattern.asPredicate().test("eu.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true + ContainerRegistry.GAR.regex.pattern.asPredicate().test("eu.gar.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true + ContainerRegistry.GAR.regex.pattern .asPredicate() .test("asia.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe true - ContainerRegistry.GCR.regex.pattern + ContainerRegistry.GAR.regex.pattern .asPredicate() .test("unknown.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe false @@ -212,7 +214,7 @@ class LeonardoModelSpec extends LeonardoTestSuite with AnyFlatSpecLike { "ContainerImage.stringToJupyterDockerImage" should "match GCR first, and then dockerhub" in { ContainerImage.fromImageUrl("us.gcr.io/broad-dsp-gcr-public/ubuntu1804") shouldBe (Some( - ContainerImage("us.gcr.io/broad-dsp-gcr-public/ubuntu1804", ContainerRegistry.GCR) + ContainerImage("us.gcr.io/broad-dsp-gcr-public/ubuntu1804", ContainerRegistry.GAR) )) ContainerImage.fromImageUrl("asd/asdf") shouldBe (Some( ContainerImage("asd/asdf", ContainerRegistry.DockerHub) diff --git a/jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh b/jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh index ed11d0d158..5ca6351bfd 100755 --- a/jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh +++ b/jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh @@ -17,6 +17,7 @@ set -e -x # # Note: You can check which version of the AOU image is used in prod here: https://github.com/all-of-us/workbench/blob/main/api/config/config_prod.json#L15C1-L16C1 +terra_base="us.gcr.io/broad-dsp-gcr-public/terra-base:1.0.0" terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6" terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7" terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.7" @@ -33,7 +34,7 @@ cryptomining_detector="us.gcr.io/broad-dsp-gcr-public/cryptomining-detector:0.0. # This array determines which of the above images are baked into the custom dataproc 2.2.x image # the entry must match the var name above, which must correspond to a valid docker URI -docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou openidc_proxy anvil_rstudio_bioconductor cryptomining_detector" +docker_image_var_names="welder_server terra-base terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou openidc_proxy anvil_rstudio_bioconductor cryptomining_detector" # Comment the above and uncomment this to create the dataproc 2.1.x image # You would also need to revert the dataproc versions in the create_dataproc_image.sh like this: diff --git a/jenkins/gce-custom-images/prepare_gce_image.sh b/jenkins/gce-custom-images/prepare_gce_image.sh index 20d2ce15b0..637debd33a 100755 --- a/jenkins/gce-custom-images/prepare_gce_image.sh +++ b/jenkins/gce-custom-images/prepare_gce_image.sh @@ -15,7 +15,7 @@ set -e -x # # Constants and Global Vars # - +terra_base="us.gcr.io/broad-dsp-gcr-public/terra-base:1.0.0" terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6" terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.7" terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.7" @@ -39,7 +39,7 @@ cryptomining_detector="us.gcr.io/broad-dsp-gcr-public/cryptomining-detector:0.0. # This array determines which of the above images are baked into the custom image # the entry must match the var name above, which must correspond to a valid docker URI -docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_gatk terra_jupyter_aou terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector cos_gpu_installer google_cloud_toolbox docker_composer docker_composer_with_auth" +docker_image_var_names="welder_server terra_base terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_gatk terra_jupyter_aou terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector cos_gpu_installer google_cloud_toolbox docker_composer docker_composer_with_auth" # # Functions