Add AI bot classification for event enrichment

composer.json

@@ -20,8 +20,7 @@
         "php": ">=5.0"
     },
     "require-dev": {
-        "phpunit/phpunit": "5.6.*",
-        "phpdocumentor/phpdocumentor": "2.9.*"
+        "phpunit/phpunit": "^9.6"
     },
     "autoload": {
         "files": ["lib/Mixpanel.php"]

lib/Base/MixpanelBase.php

@@ -22,7 +22,9 @@ class Base_MixpanelBase {
         "people_endpoint"   => "/engage", // host relative endpoint for people updates
         "groups_endpoint"   => "/groups", // host relative endpoint for groups updates
         "use_ssl"           => true, // use ssl when available
-        "error_callback"    => null // callback to use on consumption failures
+        "error_callback"    => null, // callback to use on consumption failures
+        "bot_detection"     => false, // enable AI bot classification
+        "bot_additional_patterns" => array() // additional bot patterns
     );
 
 

lib/BotClassifier/AiBotClassifier.php

@@ -0,0 +1,62 @@
+<?php
+require_once(dirname(__FILE__) . "/AiBotDatabase.php");
+
+/**
+ * Classifies user-agent strings against a database of known AI bot patterns.
+ */
+class BotClassifier_AiBotClassifier {
+    /** @var array The bot patterns to check against (built-in + any custom) */
+    private $_patterns;
+
+    /**
+     * @param array $additional_bots Optional additional bot patterns to prepend (checked first)
+     */
+    public function __construct($additional_bots = array()) {
+        $this->_patterns = array_merge($additional_bots, BotClassifier_AiBotDatabase::getDatabase());
+    }
+
+    /**
+     * Classify a user-agent string against the AI bot database.
+     * @param string|null $user_agent
+     * @return array Classification result with '$is_ai_bot' (always present) and optional
+     *               '$ai_bot_name', '$ai_bot_provider', '$ai_bot_category'
+     */
+    public function classify($user_agent) {
+        if ($user_agent === null || $user_agent === "" || !is_string($user_agent)) {
+            return array('$is_ai_bot' => false);
+        }
+        foreach ($this->_patterns as $bot) {
+            $match = @preg_match($bot["pattern"], $user_agent);
+            if ($match === false) {
+                continue; // Invalid regex, skip this pattern
+            }
+            if ($match) {
+                return array(
+                    '$is_ai_bot'      => true,
+                    '$ai_bot_name'     => $bot["name"],
+                    '$ai_bot_provider' => $bot["provider"],
+                    '$ai_bot_category' => $bot["category"]
+                );
+            }
+        }
+        return array('$is_ai_bot' => false);
+    }
+
+    /**
+     * Create a classifier with optional additional bot patterns (checked before built-in).
+     * @param array $options Options array with optional 'additional_bots' key
+     * @return BotClassifier_AiBotClassifier
+     */
+    public static function createClassifier($options = array()) {
+        $additional = isset($options["additional_bots"]) ? $options["additional_bots"] : array();
+        return new BotClassifier_AiBotClassifier($additional);
+    }
+
+    /**
+     * Return bot database for inspection (no regex patterns exposed).
+     * @return array
+     */
+    public function getBotDatabase() {
+        return BotClassifier_AiBotDatabase::getDatabaseForInspection();
+    }
+}

lib/BotClassifier/AiBotDatabase.php

@@ -0,0 +1,77 @@
+<?php
+/**
+ * Provides the AI bot user-agent pattern database.
+ * Each entry contains a regex pattern (for preg_match()), bot name, provider, category, and description.
+ */
+class BotClassifier_AiBotDatabase {
+    /** @var array The built-in AI bot pattern database */
+    private static $_database = array(
+        // === OpenAI ===
+        array("pattern" => "/GPTBot\//i",        "name" => "GPTBot",
+              "provider" => "OpenAI",             "category" => "indexing",
+              "description" => "OpenAI web crawler for model training data"),
+        array("pattern" => "/ChatGPT-User\//i",  "name" => "ChatGPT-User",
+              "provider" => "OpenAI",             "category" => "retrieval",
+              "description" => "ChatGPT real-time retrieval for user queries (RAG)"),
+        array("pattern" => "/OAI-SearchBot\//i",  "name" => "OAI-SearchBot",
+              "provider" => "OpenAI",             "category" => "indexing",
+              "description" => "OpenAI search indexing crawler"),
+        // === Anthropic ===
+        array("pattern" => "/ClaudeBot\//i",      "name" => "ClaudeBot",
+              "provider" => "Anthropic",           "category" => "indexing",
+              "description" => "Anthropic web crawler for model training"),
+        array("pattern" => "/Claude-User\//i",    "name" => "Claude-User",
+              "provider" => "Anthropic",           "category" => "retrieval",
+              "description" => "Claude real-time retrieval for user queries"),
+        // === Google ===
+        array("pattern" => "/Google-Extended\//i", "name" => "Google-Extended",
+              "provider" => "Google",              "category" => "indexing",
+              "description" => "Google AI training data crawler (separate from Googlebot)"),
+        // === Perplexity ===
+        array("pattern" => "/PerplexityBot\//i",  "name" => "PerplexityBot",
+              "provider" => "Perplexity",          "category" => "retrieval",
+              "description" => "Perplexity AI search crawler"),
+        // === ByteDance ===
+        array("pattern" => "/Bytespider\//i",     "name" => "Bytespider",
+              "provider" => "ByteDance",           "category" => "indexing",
+              "description" => "ByteDance/TikTok AI crawler"),
+        // === Common Crawl ===
+        array("pattern" => "/CCBot\//i",          "name" => "CCBot",
+              "provider" => "Common Crawl",        "category" => "indexing",
+              "description" => "Common Crawl bot (data used by many AI models)"),
+        // === Apple ===
+        array("pattern" => "/Applebot-Extended\//i", "name" => "Applebot-Extended",
+              "provider" => "Apple",               "category" => "indexing",
+              "description" => "Apple AI/Siri training data crawler"),
+        // === Meta ===
+        array("pattern" => "/Meta-ExternalAgent\//i", "name" => "Meta-ExternalAgent",
+              "provider" => "Meta",                "category" => "indexing",
+              "description" => "Meta/Facebook AI training data crawler"),
+        // === Cohere ===
+        array("pattern" => "/cohere-ai\//i",      "name" => "cohere-ai",
+              "provider" => "Cohere",              "category" => "indexing",
+              "description" => "Cohere AI training data crawler"),
+    );
+
+    /** @return array */
+    public static function getDatabase() {
+        return self::$_database;
+    }
+
+    /**
+     * Returns database entries without regex patterns (safe for inspection).
+     * @return array
+     */
+    public static function getDatabaseForInspection() {
+        $result = array();
+        foreach (self::$_database as $entry) {
+            $result[] = array(
+                "name"        => $entry["name"],
+                "provider"    => $entry["provider"],
+                "category"    => $entry["category"],
+                "description" => $entry["description"]
+            );
+        }
+        return $result;
+    }
+}

lib/ConsumerStrategies/BotClassifyingConsumer.php

@@ -0,0 +1,62 @@
+<?php
+require_once(dirname(__FILE__) . "/AbstractConsumer.php");
+require_once(dirname(__FILE__) . "/../BotClassifier/AiBotClassifier.php");
+
+/**
+ * Consumer wrapper that classifies AI bots in tracked event batches.
+ * Wraps any other consumer and enriches event data with bot classification
+ * properties when a user-agent string is present.
+ */
+class ConsumerStrategies_BotClassifyingConsumer extends ConsumerStrategies_AbstractConsumer {
+    /** @var ConsumerStrategies_AbstractConsumer */
+    private $_innerConsumer;
+    /** @var BotClassifier_AiBotClassifier */
+    private $_classifier;
+    /** @var string */
+    private $_userAgentProperty = '$user_agent';
+    /** @var array */
+    private $_consumers = array(
+        "file"   => "ConsumerStrategies_FileConsumer",
+        "curl"   => "ConsumerStrategies_CurlConsumer",
+        "socket" => "ConsumerStrategies_SocketConsumer"
+    );
+
+    function __construct($options = array()) {
+        parent::__construct($options);
+        $inner_key = isset($options["bot_classifying_inner_consumer"])
+            ? $options["bot_classifying_inner_consumer"] : "curl";
+        // NOTE: Do NOT merge $options["consumers"] into $_consumers here.
+        // The "consumers" key in $options is also consumed by Producers_MixpanelBaseProducer,
+        // and merging it would include "bot_classifying" => self, risking self-instantiation.
+        // Only the three hardcoded consumer types (file, curl, socket) are valid inner consumers.
+        $InnerClass = $this->_consumers[$inner_key];
+        $this->_innerConsumer = new $InnerClass($options);
+        $additional_bots = isset($options["bot_additional_patterns"])
+            ? $options["bot_additional_patterns"] : array();
+        $this->_classifier = new BotClassifier_AiBotClassifier($additional_bots);
+        if (isset($options["bot_user_agent_property"])) {
+            $this->_userAgentProperty = $options["bot_user_agent_property"];
+        }
+    }
+
+    /**
+     * Classify bot user-agents in each message and forward to inner consumer.
+     * @param array $batch
+     * @return boolean
+     */
+    public function persist($batch) {
+        foreach ($batch as &$message) {
+            if (isset($message["properties"]) && isset($message["properties"][$this->_userAgentProperty])) {
+                $classification = $this->_classifier->classify($message["properties"][$this->_userAgentProperty]);
+                $message["properties"] = array_merge($message["properties"], $classification);
+            }
+        }
+        unset($message);
+        return $this->_innerConsumer->persist($batch);
+    }
+
+    /** @return int */
+    public function getNumThreads() {
+        return $this->_innerConsumer->getNumThreads();
+    }
+}

lib/Mixpanel.php

@@ -4,6 +4,8 @@
 require_once(dirname(__FILE__) . "/Producers/MixpanelPeople.php");
 require_once(dirname(__FILE__) . "/Producers/MixpanelEvents.php");
 require_once(dirname(__FILE__) . "/Producers/MixpanelGroups.php");
+require_once(dirname(__FILE__) . "/BotClassifier/AiBotClassifier.php");
+require_once(dirname(__FILE__) . "/ConsumerStrategies/BotClassifyingConsumer.php");
 
 /**
  * This is the main class for the Mixpanel PHP Library which provides all of the methods you need to track events,
@@ -123,6 +125,9 @@ class Mixpanel extends Base_MixpanelBase {
      */
     private $_events;
 
+    /** @var BotClassifier_AiBotClassifier|null */
+    private $_botClassifier = null;
+
     /**
      * An instance of the MixpanelGroups class (used to create/update group profiles)
      * @var Producers_MixpanelPeople
@@ -148,6 +153,12 @@ public function __construct($token, $options = array()) {
         $this->people = new Producers_MixpanelPeople($token, $options);
         $this->_events = new Producers_MixpanelEvents($token, $options);
         $this->group = new Producers_MixpanelGroups($token, $options);
+        // Initialize bot classifier if bot_detection is enabled
+        if (isset($this->_options["bot_detection"]) && $this->_options["bot_detection"]) {
+            $additional_bots = isset($this->_options["bot_additional_patterns"])
+                ? $this->_options["bot_additional_patterns"] : array();
+            $this->_botClassifier = new BotClassifier_AiBotClassifier($additional_bots);
+        }
     }
 
 
@@ -200,6 +211,15 @@ public function reset() {
     }
 
 
+    /**
+     * Get the events queue (delegates to the events producer).
+     * @return array
+     */
+    public function getQueue() {
+        return $this->_events->getQueue();
+    }
+
+
     /**
      * Identify the user you want to associate to tracked events. The $anon_id must be UUID v4 format and not already merged to an $identified_id.
      * All identify calls with a new and valid $anon_id will trigger a track $identify event, and merge to the $identified_id.
@@ -216,9 +236,28 @@ public function identify($user_id, $anon_id = null) {
      * @param array $properties
      */
     public function track($event, $properties = array()) {
+        if ($this->_botClassifier !== null
+            && isset($properties['$user_agent'])
+            && !$this->_isUsingBotClassifyingConsumer()) {
+            $classification = $this->_botClassifier->classify($properties['$user_agent']);
+            $properties = array_merge($properties, $classification);
+        }
         $this->_events->track($event, $properties);
     }
 
+    /**
+     * Check if the configured consumer is BotClassifyingConsumer to avoid double-classification.
+     * @return bool
+     */
+    private function _isUsingBotClassifyingConsumer() {
+        if (!isset($this->_options['consumers']) || !is_array($this->_options['consumers'])) {
+            return false;
+        }
+        $consumerKey = $this->_options['consumer'];
+        return isset($this->_options['consumers'][$consumerKey])
+            && $this->_options['consumers'][$consumerKey] === 'ConsumerStrategies_BotClassifyingConsumer';
+    }
+
 
     /**
      * Register a property to be sent with every event.

phpunit.xml.dist

@@ -1,29 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <phpunit backupGlobals="false"
-         backupStaticAttributes="false" 
-         bootstrap="vendor/autoload.php"
-         cacheTokens="true"
          colors="false"
-         convertErrorsToExceptions="true"
-         convertNoticesToExceptions="true"
-         convertWarningsToExceptions="true"
          processIsolation="false"
          stopOnFailure="false"
-         syntaxCheck="false"
-         verbose="false">
-         
+         verbose="false"
+         bootstrap="vendor/autoload.php">
+
   <testsuites>
     <testsuite name="Mixpanel Test">
       <directory>./test/</directory>
     </testsuite>
   </testsuites>
-
-  <filter>
-    <blacklist>
-      <directory>examples</directory>
-      <directory>vendor</directory>
-      <directory>test</directory>
-    </blacklist>
-  </filter>
+
 </phpunit>

test/Base/MixpanelBaseProducerTest.php

@@ -1,19 +1,21 @@
 <?php
 
-class MixpanelBaseProducerTest extends PHPUnit_Framework_TestCase {
+use PHPUnit\Framework\TestCase;
+
+class MixpanelBaseProducerTest extends TestCase {
 
     /**
      * @var _Producers_MixpanelBaseProducer
      */
     protected $_instance = null;
     protected $_file = null;
-    protected function setUp() {
+    protected function setUp(): void {
         parent::setUp();
         $this->_file = dirname(__FILE__)."/output-".time().".txt";
         $this->_instance = new _Producers_MixpanelBaseProducer("token", array("consumer" => "file", "debug" => true, "file" => $this->_file));
     }
 
-    protected function tearDown() {
+    protected function tearDown(): void {
         parent::tearDown();
         $this->_instance->reset();
         $this->_instance = null;