From 424882d2faabdd42598fe429bb12730fa45534c8 Mon Sep 17 00:00:00 2001 From: Cheng Yin Eng Date: Wed, 14 Aug 2024 14:31:49 -0700 Subject: [PATCH 1/7] add docs for permission sharing and code for enable_trace --- .../pdf_uc_volume/03_deploy_poc_to_review_app.py | 8 ++++++-- .../pdf_uc_volume/04_create_evaluation_set.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py index ee8a8bd..1e6faab 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py @@ -136,7 +136,11 @@ # MAGIC # MAGIC Now, grant your stakeholders permissions to use the Review App. Your stakeholders do not Databricks accounts as long as you have [insert docs]. # MAGIC -# MAGIC `#TODO: add docs link` +# MAGIC +# Now, grant your stakeholders permissions to use the Review App. Your stakeholders do not need to have Databricks accounts. From the documentation: +# > If reviewers don’t have access already, account admins can use account-level SCIM provisioning to sync users and groups automatically from your identity provider to your Databricks account. You can also manually register these users and groups as you set up identities in Databricks. This allows them to be included as eligible reviewers. +# MAGIC +#[Refer to documentation here](https://docs.databricks.com/en/generative-ai/agent-evaluation/human-evaluation.html#set-up-permissions-to-the-review-app-workspace). # COMMAND ---------- @@ -151,7 +155,7 @@ # MAGIC # MAGIC If you lose this notebook's state and need to find the URL to your Review App, run this cell. # MAGIC -# MAGIC Alternatively, you can construct the Review App URL as follows: +# MAGIC Alternatively, you can cnstruct the Review App URL as follows: # MAGIC # MAGIC `https:///ml/reviews/{UC_CATALOG}.{UC_SCHEMA}.{UC_MODEL_NAME}/{UC_MODEL_VERSION_NUMBER}/instructions` diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index fd5b5b9..dfd4367 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -108,6 +108,17 @@ F.col("is_correct"), )) +# COMMAND ---------- +# MAGIC %md +# MAGIC ## Ingest chats into the Review App to be reviewed +from databricks.agents import enable_trace_reviews + +request_id_list = requests_with_feedback_df.select(F.col("request_id")).rdd.flatMap(lambda x: x).collect() +enable_trace_reviews( + model_name=UC_MODEL_NAME, + request_ids=request_id_list +) + # COMMAND ---------- # MAGIC %md From 51900b8d7a0b287770175cda4c382623336188f4 Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 14 Aug 2024 14:41:04 -0700 Subject: [PATCH 2/7] change formatting --- .../pdf_uc_volume/03_deploy_poc_to_review_app.py | 8 +++----- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py index 1e6faab..37e736d 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py @@ -134,13 +134,11 @@ # MAGIC %md # MAGIC ## Grant stakeholders access to the Review App # MAGIC -# MAGIC Now, grant your stakeholders permissions to use the Review App. Your stakeholders do not Databricks accounts as long as you have [insert docs]. +# MAGIC Now, grant your stakeholders permissions to use the Review App. Your stakeholders do not need to have Databricks accounts. From the documentation: # MAGIC -# MAGIC -# Now, grant your stakeholders permissions to use the Review App. Your stakeholders do not need to have Databricks accounts. From the documentation: -# > If reviewers don’t have access already, account admins can use account-level SCIM provisioning to sync users and groups automatically from your identity provider to your Databricks account. You can also manually register these users and groups as you set up identities in Databricks. This allows them to be included as eligible reviewers. +# MAGIC > If reviewers don’t have access already, account admins can use account-level SCIM provisioning to sync users and groups automatically from your identity provider to your Databricks account. You can also manually register these users and groups as you set up identities in Databricks. This allows them to be included as eligible reviewers. # MAGIC -#[Refer to documentation here](https://docs.databricks.com/en/generative-ai/agent-evaluation/human-evaluation.html#set-up-permissions-to-the-review-app-workspace). +# MAGIC [Refer to documentation here](https://docs.databricks.com/en/generative-ai/agent-evaluation/human-evaluation.html#set-up-permissions-to-the-review-app-workspace). # COMMAND ---------- diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index dfd4367..5243522 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -108,9 +108,11 @@ F.col("is_correct"), )) -# COMMAND ---------- # MAGIC %md # MAGIC ## Ingest chats into the Review App to be reviewed + +# COMMAND ---------- + from databricks.agents import enable_trace_reviews request_id_list = requests_with_feedback_df.select(F.col("request_id")).rdd.flatMap(lambda x: x).collect() From 48567222756c3c907fd595b97864381ec017fc9e Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 14 Aug 2024 14:44:31 -0700 Subject: [PATCH 3/7] update formatting --- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index 5243522..39abceb 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -108,9 +108,10 @@ F.col("is_correct"), )) +# COMMAND ---------- # MAGIC %md # MAGIC ## Ingest chats into the Review App to be reviewed - +# # COMMAND ---------- from databricks.agents import enable_trace_reviews From b07264d4cfefab6c6af7dc7749cbf226ca6b1f94 Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 14 Aug 2024 14:46:42 -0700 Subject: [PATCH 4/7] remove extra # --- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index 39abceb..598ed7f 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -111,7 +111,7 @@ # COMMAND ---------- # MAGIC %md # MAGIC ## Ingest chats into the Review App to be reviewed -# + # COMMAND ---------- from databricks.agents import enable_trace_reviews From 888a945c232fd35b83b67c0da8d25a20628ec136 Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 21 Aug 2024 14:25:19 -0700 Subject: [PATCH 5/7] change requests df for tracing and fix typo --- .../A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py | 2 +- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py index 37e736d..dd5a556 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/03_deploy_poc_to_review_app.py @@ -153,7 +153,7 @@ # MAGIC # MAGIC If you lose this notebook's state and need to find the URL to your Review App, run this cell. # MAGIC -# MAGIC Alternatively, you can cnstruct the Review App URL as follows: +# MAGIC Alternatively, you can construct the Review App URL as follows: # MAGIC # MAGIC `https:///ml/reviews/{UC_CATALOG}.{UC_SCHEMA}.{UC_MODEL_NAME}/{UC_MODEL_VERSION_NUMBER}/instructions` diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index 598ed7f..662f4e7 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -116,7 +116,7 @@ from databricks.agents import enable_trace_reviews -request_id_list = requests_with_feedback_df.select(F.col("request_id")).rdd.flatMap(lambda x: x).collect() +request_id_list = request_log_df.select(F.col("request_id")).rdd.flatMap(lambda x: x).collect() enable_trace_reviews( model_name=UC_MODEL_NAME, request_ids=request_id_list From 37cc28fb5bf3e3039f2c50d44e951a9a9126f0d0 Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 18 Sep 2024 16:39:09 -0700 Subject: [PATCH 6/7] change to using pandas --- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index 662f4e7..10ed9f5 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -116,7 +116,8 @@ from databricks.agents import enable_trace_reviews -request_id_list = request_log_df.select(F.col("request_id")).rdd.flatMap(lambda x: x).collect() +request_id_list = requests_with_feedback_df.toPandas()["request_id"].tolist() + enable_trace_reviews( model_name=UC_MODEL_NAME, request_ids=request_id_list From df015a9b25838ea1f8c85a85809a30bdf74a5594 Mon Sep 17 00:00:00 2001 From: chengyineng38 Date: Wed, 18 Sep 2024 16:44:28 -0700 Subject: [PATCH 7/7] change to using the right requests df and request id --- .../A_POC_app/pdf_uc_volume/04_create_evaluation_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py index 10ed9f5..9bcf0b8 100644 --- a/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py +++ b/rag_app_sample_code/A_POC_app/pdf_uc_volume/04_create_evaluation_set.py @@ -116,7 +116,7 @@ from databricks.agents import enable_trace_reviews -request_id_list = requests_with_feedback_df.toPandas()["request_id"].tolist() +request_id_list = request_log_df.toPandas()["databricks_request_id"].tolist() enable_trace_reviews( model_name=UC_MODEL_NAME,