feat: Add Weights & Biases Weave export functionality

chainforge/flask_app.py

@@ -14,6 +14,7 @@
 
 # RAG-specific imports
 from markitdown import MarkItDown
+from chainforge.providers.cf_to_weave import export_to_weave
 
 
 """ =================
@@ -1061,6 +1062,35 @@ def media_to_text(uid):
         # traceback.print_exc()
         return jsonify({"error": f"Failed to process file {uid}. Internal server error."}), 500
 
+@app.route('/api/exportToWandB', methods=['POST'])
+def export_to_wandb():
+    try:
+
+        data = request.json
+
+        if not data:
+            return jsonify({"success": False, "message": "No JSON payload received."}), 400
+
+        flow_data = data.get('flowData')
+        api_key = data.get('apiKey')
+        project_name = data.get('projectName')
+        # Ensure that flow_data is not empty before proceeding
+        if not flow_data:
+            return jsonify({"success": False, "message": "No flow data provided in the request."}), 400
+
+        print(api_key, project_name)
+
+        response = export_to_weave(flow_data, project_name, api_key)
+        print(response)
+        return jsonify(response)
+
+    except Exception as e:
+        # Catch any unexpected errors and return a consistent error message
+        print(f"Error in /api/exportToWandB: {e}", file=sys.stderr)
+        return jsonify({"success": False, "message": f"An unexpected error occurred: {str(e)}"}), 500
+
+
+
 @app.route('/api/exportFlowBundle', methods=['POST'])
 def export_flow_bundle():
     """
@@ -1311,6 +1341,12 @@ def verify_media_file_integrity(uid):
         raise ValueError(f"Hash mismatch: expected {expected_hash}, got {actual_hash}")
 
 
+@app.route('/<path:filename>')
+def serve_public_file(filename):
+    """Serve files from the public directory"""
+    public_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'react-server', 'public')
+    return send_from_directory(public_dir, filename)
+
 @app.route('/api/proxyImage', methods=['GET'])
 def proxy_image():
     """Proxy for fetching images to avoid CORS restrictions"""

chainforge/providers/cf_to_weave.py

@@ -0,0 +1,162 @@
+
+import os
+import json
+import re
+from pprint import pprint
+from datetime import datetime
+import hashlib
+import base64
+from io import BytesIO
+
+import weave
+from weave import EvaluationLogger
+from PIL import Image
+
+def sanitize_name(name):
+    """Sanitize a name to be valid for Weave (alphanumeric and underscores only)"""
+    # Replace hyphens and other invalid chars with underscores
+    sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', name)
+    # Ensure it starts with a letter or underscore
+    if sanitized and not sanitized[0].isalpha() and sanitized[0] != '_':
+        sanitized = 'model_' + sanitized
+    return sanitized
+
+def resolve_string_references(data_item, string_cache, media_cache=None):
+    """Recursively resolves integer references in data_item using the string_cache and converts image objects to PIL Images."""
+    if isinstance(data_item, int):
+        # Try to fetch the string from __s using index, otherwise return the integer itself
+        try:
+            return string_cache[data_item]
+        except (IndexError, TypeError):
+            return data_item
+    elif isinstance(data_item, dict):
+        # Check if this is an image object
+        if data_item.get("t") == "img" and "d" in data_item and media_cache:
+            image_cache_id = data_item["d"]
+            if image_cache_id in media_cache:
+                # Extract base64 data from data URI
+                base64_data = media_cache[image_cache_id]
+                if base64_data.startswith("data:image"):
+                    # Remove the data URI prefix (e.g., "data:image/jpeg;base64,")
+                    header, encoded = base64_data.split(',', 1)
+                    # Decode base64 to bytes
+                    image_bytes = base64.b64decode(encoded)
+                    # Create PIL Image from bytes
+                    pil_image = Image.open(BytesIO(image_bytes))
+                    return pil_image
+                else:
+                    # If not a data URI, return the original data
+                    return base64_data
+            else:
+                # If image not found in cache, return the original object
+                return data_item
+        else:
+            # Regular dictionary processing
+            return {k: resolve_string_references(v, string_cache, media_cache) for k, v in data_item.items()}
+    elif isinstance(data_item, list):
+        return [resolve_string_references(elem, string_cache, media_cache) for elem in data_item]
+    else:
+        return data_item
+
+def log_to_weave(data, cforge_filename="test1"):
+    """Log the .cforge file data to Weave using EvaluationLogger"""
+    cache = data.get('cache', {})
+    string_cache = cache.get('__s', []) # Get the string cache
+    media_cache = cache.get('__media', {}).get('cache', {}) # Get the media cache
+
+    # Use regex to find all evaluation keys in cache
+    eval_keys = [key for key in cache.keys() if re.search(r'Eval', key, re.IGNORECASE) and key.endswith('.json')]
+
+    if not eval_keys:
+        raise Exception("The cforge flow cannot be exported to weave because there are no evaluation nodes found.")
+
+    print(f"Found {len(eval_keys)} evaluation keys: {eval_keys}")
+
+    # Loop through each evaluation key
+    for eval_key in eval_keys:
+        eval_results = cache.get(eval_key, [])
+
+        if not eval_results:
+            raise Exception("No evaluation results found in cache. Please run the evaluators.")
+
+        # Determine model and dataset names dynamically from the first evaluation result
+        first_eval_result = eval_results[0]
+        llm_info = first_eval_result.get('llm', {})
+
+        # Handle case where llm_info can be either a string or dict
+        if isinstance(llm_info, str):
+            raw_model_name = llm_info
+        elif isinstance(llm_info, dict):
+            raw_model_name = llm_info.get('name', 'unknown_model')
+        else:
+            raw_model_name = 'unknown_model'
+
+        model_name = sanitize_name(raw_model_name)  # Sanitize the model name
+        dataset_name = f"chainforge_{cforge_filename}"
+
+        raw_eval_logger_name = eval_key.replace('.json', '')
+        eval_logger_name = sanitize_name(raw_eval_logger_name)  # Sanitize the eval logger name
+
+        print(f"Processing evaluation: {eval_logger_name}")
+        print(f"Model name: {raw_model_name} -> {model_name}")
+        print(f"Eval Logger name: {raw_eval_logger_name} -> {eval_logger_name}")
+
+        # Initialize EvaluationLogger
+        eval_logger = EvaluationLogger(
+            model=model_name,
+            dataset=dataset_name,
+            name=eval_logger_name
+        )
+
+        # Iterate through evaluation results and log each prediction
+        for eval_result in eval_results:
+            raw_inputs = eval_result.get('vars', {})
+            raw_outputs = eval_result.get('responses', [''])[0] # Take the first response
+            score = eval_result.get('eval_res', {}).get('items', [None])[0] # Take the first score
+
+            # Resolve string references for inputs and outputs, now including media cache
+            resolved_inputs = resolve_string_references(raw_inputs, string_cache, media_cache)
+            # Ensure inputs is explicitly a dictionary
+            if isinstance(resolved_inputs, dict):
+                inputs = resolved_inputs
+            else:
+                # This case should ideally not happen if 'vars' is always a dict
+                print(f"Warning: Resolved inputs for {eval_key} is not a dictionary. Converting to empty dict.")
+                inputs = {}
+
+            outputs = resolve_string_references(raw_outputs, string_cache, media_cache)
+
+            # Log the prediction input and output
+            pred_logger = eval_logger.log_prediction(
+                inputs=inputs,
+                output=outputs
+            )
+
+            # Log the score if available
+            if score is not None:
+                pred_logger.log_score(
+                    scorer=eval_logger_name, 
+                    score=score
+                )
+
+            pred_logger.finish()
+
+        # Log a summary for the evaluation run
+        eval_logger.log_summary({
+            "total_evaluations": len(eval_results),
+            "evaluation_type": eval_logger_name
+        })
+
+        print(f"Successfully logged {len(eval_results)} evaluations for '{eval_logger_name}' to Weave")
+
+def export_to_weave(data, project_name, api_key=""):
+    try:
+        # Set wandb api key
+        os.environ['WANDB_API_KEY'] = api_key
+        # Initialize Weave
+        weave.init(project_name)
+        log_to_weave(data)
+
+        return {"success": True, "message": "Successfully exported to W&B Weave"}
+    except Exception as e:
+        return {"success": False, "message": f"Error exporting to W&B Weave: {str(e)}"} 

chainforge/react-server/eslint.config.mjs

@@ -0,0 +1,96 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import js from "@eslint/js";
+import { FlatCompat } from "@eslint/eslintrc";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all,
+});
+
+export default [
+  {
+    ignores: [
+      "node_modules/**/*",
+      "build/**/*",
+      "**/craco.config.js",
+      "src/backend/pyodide/**/*",
+      "src/backend/__test__/**/*",
+    ],
+  },
+  ...compat.extends(
+    "semistandard",
+    "plugin:react/recommended",
+    "plugin:prettier/recommended",
+    "plugin:@typescript-eslint/recommended",
+  ),
+  {
+    settings: {
+      react: {
+        createClass: "createReactClass",
+        pragma: "React",
+        fragment: "Fragment",
+        version: "detect",
+        flowVersion: "0.53",
+      },
+
+      propWrapperFunctions: [
+        "forbidExtraProps",
+        {
+          property: "freeze",
+          object: "Object",
+        },
+        {
+          property: "myFavoriteWrapper",
+        },
+        {
+          property: "forbidExtraProps",
+          exact: true,
+        },
+      ],
+
+      componentWrapperFunctions: [
+        "observer",
+        {
+          property: "styled",
+        },
+        {
+          property: "observer",
+          object: "Mobx",
+        },
+        {
+          property: "observer",
+          object: "<pragma>",
+        },
+      ],
+
+      formComponents: [
+        "CustomForm",
+        {
+          name: "Form",
+          formAttribute: "endpoint",
+        },
+      ],
+
+      linkComponents: [
+        "Hyperlink",
+        {
+          name: "Link",
+          linkAttribute: "to",
+        },
+      ],
+    },
+
+    rules: {
+      semi: ["error", "always"],
+      camelcase: ["off"],
+      "react/prop-types": ["off"],
+      "@typescript-eslint/no-explicit-any": ["off"],
+      "@typescript-eslint/no-empty-function": ["off"],
+      "no-control-regex": ["off"],
+    },
+  },
+];