diff --git a/generate-manual-config.sh b/generate-manual-config.sh old mode 100755 new mode 100644 index 37edfdd8b..171e9ffcb --- a/generate-manual-config.sh +++ b/generate-manual-config.sh @@ -28,7 +28,7 @@ LATEST_BRANCH="master" LATEST_VERSION="3.1.0-SNAPSHOT" LATEST_DIR="latest" -# Stable version (release branch) +# Stable version (release branch) STABLE_BRANCH="unomi-3.0.x" STABLE_VERSION="3.0.0" STABLE_DIR="3_0_x" @@ -82,7 +82,7 @@ PROJECT_STAGING_DIR="target/staging" # === TIMEOUTS === # Command timeouts in seconds MAVEN_TIMEOUT=1800 # 30 minutes -GIT_TIMEOUT=300 # 5 minutes +GIT_TIMEOUT=300 # 5 minutes SVN_TIMEOUT=600 # 10 minutes # Export all configuration variables diff --git a/manual/pom.xml b/manual/pom.xml index 24b0c782d..aa1b41eaf 100644 --- a/manual/pom.xml +++ b/manual/pom.xml @@ -170,7 +170,6 @@ net.nicoulaj.maven.plugins checksum-maven-plugin - 1.7 package-release-checksum diff --git a/manual/src/main/asciidoc/5-min-quickstart.adoc b/manual/src/main/asciidoc/5-min-quickstart.adoc index c78e21078..420095493 100644 --- a/manual/src/main/asciidoc/5-min-quickstart.adoc +++ b/manual/src/main/asciidoc/5-min-quickstart.adoc @@ -12,6 +12,7 @@ // limitations under the License. // +[[_five_minutes_quickstart]] === Quick start with Docker Begin by creating a `docker-compose.yml` file. You can choose between ElasticSearch or OpenSearch: @@ -30,7 +31,7 @@ services: - 9200:9200 unomi: # Unomi version can be updated based on your needs - image: apache/unomi:3.0.0 + image: apache/unomi:3.1.0 environment: - UNOMI_ELASTICSEARCH_ADDRESSES=elasticsearch:9200 - UNOMI_THIRDPARTY_PROVIDER1_IPADDRESSES=0.0.0.0/0,::1,127.0.0.1 @@ -73,7 +74,7 @@ services: - 9600:9600 unomi: - image: apache/unomi:3.0.0 + image: apache/unomi:3.1.0 environment: - UNOMI_DISTRIBUTION=unomi-distribution-opensearch - UNOMI_OPENSEARCH_ADDRESSES=opensearch-node1:9200 @@ -99,7 +100,7 @@ Try accessing https://localhost:9443/cxs/cluster with username/password: karaf/k ==== Option 1: Using ElasticSearch -1) Install JDK 17 and make sure you set the JAVA_HOME variable (see our <> guide for more information on JDK compatibility) +1) Install JDK 17 and make sure you set the JAVA_HOME variable (see our <<_jdk_compatibility,Getting Started>> guide for more information on JDK compatibility) 2) Download ElasticSearch here : https://www.elastic.co/downloads/past-releases/elasticsearch-9-2-1 (please *make sure* you use the proper version : 9.2.1) @@ -148,14 +149,51 @@ which determines which set of features and bundles are installed and started. A 9) Try accessing https://localhost:9443/cxs/cluster with username/password: `karaf/karaf` . You might get a certificate warning in your browser, just accept it despite the warning it is safe. -10) Request your first context by simply accessing : http://localhost:8181/cxs/context.js?sessionId=1234 +10) Create a tenant that will own all your data: -11) If something goes wrong, you should check the logs in `./data/log/karaf.log`. If you get errors on the search engine, +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/tenants \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "requestedId": "default", + "properties": { + "name": "Default Tenant", + "description": "Default tenant for quick start" + } + }' +---- + +Save the API keys from the response - you'll need them for API calls. + +11) Request your first context: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ + -H "Content-Type: application/json" \ + -H "X-Unomi-Api-Key: " \ + -d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "default" + } + }' +---- + +Or access it directly in a browser (requires API key in header): +`http://localhost:8181/cxs/context.js?sessionId=1234` + +NOTE: For the context request to work, you'll need to include the public API key from step 10 in the request header: `X-Unomi-Api-Key: ` + +12) If something goes wrong, you should check the logs in `./data/log/karaf.log`. If you get errors on the search engine, make sure you are using the proper version. Next steps: -- Trying our integration <> +- Trying our integration <<_samples,samples page>> Note: When using OpenSearch, make sure to: - Set up proper SSL certificates or disable SSL verification for development diff --git a/manual/src/main/asciidoc/architecture.adoc b/manual/src/main/asciidoc/architecture.adoc new file mode 100644 index 000000000..0bfaa1376 --- /dev/null +++ b/manual/src/main/asciidoc/architecture.adoc @@ -0,0 +1,246 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +=== High-Level Architecture + +[plantuml] +---- +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +LAYOUT_WITH_LEGEND() + +title Apache Unomi - High Level Architecture + +Container(webApp, "Web Application", "JavaScript", "Client application using context.js") +Container(mobileApp, "Mobile App", "Native/Hybrid", "Client using REST API") + +Container_Boundary(unomi, "Apache Unomi") { + Component(restApi, "REST API", "JAX-RS", "Public and private APIs") + Component(contextServer, "Context Server", "OSGi", "Core server components") + Component(rules, "Rules Engine", "Java", "Event processing & rule evaluation") + Component(profiles, "Profile Service", "Java", "Profile management & tracking") + Component(segments, "Segment Service", "Java", "Dynamic user segmentation") + Component(events, "Event Service", "Java", "Event collection & processing") + Component(privacy, "Privacy Service", "Java", "Consent & privacy management") +} + +ContainerDb(elastic, "Search Engine", "Elasticsearch/OpenSearch", "Data persistence") +Container(karaf, "Apache Karaf", "OSGi Container", "Runtime environment") + +Rel(webApp, restApi, "Uses", "HTTP/JSON") +Rel(mobileApp, restApi, "Uses", "HTTP/JSON") +Rel(restApi, contextServer, "Uses") +Rel(contextServer, rules, "Uses") +Rel(contextServer, profiles, "Uses") +Rel(contextServer, segments, "Uses") +Rel(contextServer, events, "Uses") +Rel(contextServer, privacy, "Uses") +Rel(contextServer, elastic, "Persists data", "REST") +Rel(karaf, unomi, "Hosts") + +@enduml +---- + +=== Service Architecture + +[plantuml] +---- +@startuml +skinparam componentStyle uml2 +skinparam component { + BackgroundColor<> LightBlue + BackgroundColor<> LightGreen + BackgroundColor<> LightYellow +} + +package "Core Services" { + [Profile Service] <> + [Event Service] <> + [Segment Service] <> + [Rules Service] <> + [Privacy Service] <> + [Definitions Service] <> +} + +package "Security" { + [Security Service] <> + [Authentication] <> + [Authorization] <> +} + +package "Persistence" { + [Persistence Service] <> + [Cache Service] <> +} + +package "HTTP APIs" { + [REST Server] + [GraphQL API] +} + +package "Extensions" { + [Router] + [Plugins] +} + +database "Search Engine" { + [Elasticsearch/OpenSearch] +} + +[Profile Service] --> [Persistence Service] +[Event Service] --> [Persistence Service] +[Segment Service] --> [Persistence Service] +[Rules Service] --> [Persistence Service] + +[Profile Service] --> [Cache Service] +[Segment Service] --> [Cache Service] +[Rules Service] --> [Cache Service] + +[REST Server] --> [Security Service] +[REST Server] --> [Profile Service] +[REST Server] --> [Event Service] +[REST Server] --> [Segment Service] +[REST Server] --> [Rules Service] + +[Persistence Service] --> [Elasticsearch/OpenSearch] + +[Router] --> [Profile Service] +[Plugins] --> [Core Services] + +note right of [Security Service] + Handles authentication, + authorization, and + tenant isolation +end note + +note right of [Cache Service] + Multi-level caching for + profiles, segments, + and rules +end note +@enduml +---- + +=== Condition Evaluation Architecture + +[plantuml] +---- +@startuml +skinparam componentStyle uml2 +skinparam component { + BackgroundColor<> LightBlue + BackgroundColor<> LightGreen + BackgroundColor<> LightYellow +} + +package "Condition Evaluation" { + [ConditionDispatcher] <> + [ConditionEvaluator] <> + [ConditionESQueryBuilder] <> + [ConditionOSQueryBuilder] <> +} + +package "Query Building" { + interface "QueryBuilder" as QB + [ElasticSearchQueryBuilder] <> + [OpenSearchQueryBuilder] <> +} + +package "Storage" { + [ElasticSearch] <> + [OpenSearch] <> +} + +[ConditionDispatcher] --> [ConditionEvaluator] +[ConditionDispatcher] --> QB + +QB <|.. [ConditionESQueryBuilder] +QB <|.. [ConditionOSQueryBuilder] + +[ConditionESQueryBuilder] --> [ElasticSearchQueryBuilder] +[ConditionOSQueryBuilder] --> [OpenSearchQueryBuilder] + +[ElasticSearchQueryBuilder] --> [ElasticSearch] +[OpenSearchQueryBuilder] --> [OpenSearch] + +note right of [ConditionDispatcher] + Routes condition evaluation: + 1. Direct evaluation + 2. Query-based evaluation +end note + +note right of QB + Abstract query building + for different search + engine implementations +end note +@enduml +---- + +==== Key Components + +1. *ConditionDispatcher* +- Routes conditions to appropriate evaluators +- Determines evaluation strategy (direct vs query-based) +- Handles caching and optimization + +2. *ConditionEvaluator* +- Performs direct condition evaluation +- Handles boolean logic and property comparisons +- Evaluates nested conditions + +3. *QueryBuilders* +- Convert conditions to search engine queries +- Handle search engine specific syntax +- Optimize query performance + +=== Data Flow + +[plantuml] +---- +@startuml +skinparam activityBackgroundColor LightBlue +skinparam activityBorderColor DarkBlue +skinparam arrowColor DarkBlue + +|Client| +start +:Send Event/Context Request; + +|REST API| +:Validate Request; +:Authenticate & Authorize; + +|Event Processing| +:Process Event; +:Apply Rules; +:Update Profile; + +|Segmentation| +:Evaluate Segments; +:Update Profile Segments; + +|Persistence| +:Save Profile; +:Save Event; + +|REST API| +:Return Response; + +|Client| +:Update UI/Take Action; +stop +@enduml +---- \ No newline at end of file diff --git a/manual/src/main/asciidoc/configuration.adoc b/manual/src/main/asciidoc/configuration.adoc index 4b191e5d2..e9da98be8 100644 --- a/manual/src/main/asciidoc/configuration.adoc +++ b/manual/src/main/asciidoc/configuration.adoc @@ -94,7 +94,7 @@ org.apache.unomi.opensearch.cluster.name=opensearch-cluster org.apache.unomi.opensearch.addresses=localhost:9200 # OpenSearch security settings (required by default since OpenSearch 3) -org.apache.unomi.opensearch.ssl.enable=true +org.apache.unomi.opensearch.sslEnable=true org.apache.unomi.opensearch.username=admin org.apache.unomi.opensearch.password=${env:OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} org.apache.unomi.opensearch.sslTrustAllCertificates=true @@ -118,6 +118,43 @@ Note: When using OpenSearch 3.x: - The default admin username is 'admin' - The initial admin password can be set via OPENSEARCH_INITIAL_ADMIN_PASSWORD environment variable +=== Refresh Policy Configuration + +Apache Unomi provides the ability to configure refresh policies for different item types in both ElasticSearch and OpenSearch. This feature allows you to optimize when changes to documents are made visible to search. + +For ElasticSearch: +[source] +---- +# refresh policy per item type in Json format. +# Valid values are WAIT_UNTIL/IMMEDIATE/NONE. The default refresh policy is NONE. +# Example: {"event":"WAIT_UNTIL","rule":"NONE"} +org.apache.unomi.elasticsearch.itemTypeToRefreshPolicy={"scheduledTask":"WAIT_UNTIL"} +---- + +For OpenSearch: +[source] +---- +# refresh policy per item type in Json format. +# Valid values are WaitFor/True/False. The default refresh policy is False. +# Example: {"event":"WaitFor","rule":"False"} +org.apache.unomi.opensearch.itemTypeToRefreshPolicy={"scheduledTask":"WaitFor"} +---- + +The refresh policy can be configured per item type and affects when changes to documents of that type become visible: + +- *WAIT_UNTIL/WaitFor*: Changes will be visible after the operation completes but will wait for a refresh. This ensures changes are immediately visible while being more efficient than an immediate refresh. +- *IMMEDIATE/True*: Changes will be made visible immediately (forces an index refresh). This is the most resource-intensive option but guarantees immediate visibility. +- *NONE/False*: Changes will be made visible when a refresh happens, either from the configured refresh interval or when manually triggered. This is the most efficient option but doesn't guarantee when changes will be visible. + +For distributed task management, it's recommended to use *WAIT_UNTIL/WaitFor* for the `scheduledTask` item type to ensure lock state changes are immediately visible to all nodes without requiring explicit refresh calls in the code. + +Authentication Rules: + +1. If JAAS authentication is provided (username/password), it grants full access to all endpoints +2. Public paths (like /context.json) require a valid public API key +3. Private paths require both tenantId and private API key +4. All other requests are denied + === Secured events configuration Apache Unomi secures some events by default. It comes out of the box with a default configuration that you can adjust @@ -135,7 +172,7 @@ The properties start with the prefix : `org.apache.unomi.thirdparty.*` and here org.apache.unomi.thirdparty.provider1.allowedEvents=${env:UNOMI_THIRDPARTY_PROVIDER1_ALLOWEDEVENTS:-login,updateProperties} The events set in allowedEvents will be secured and will only be accepted if the call comes from the specified IP -address, and if the secret-key is passed in the X-Unomi-Peer HTTP request header. The "env:" part means that it will +address, and if the secret-key is passed in the X-Unomi-Api-Key HTTP request header. The "env:" part means that it will attempt to read an environment variable by that name, and if it's not found it will default to the value after the ":-" marker. @@ -227,6 +264,289 @@ You should now have SSL setup on Karaf with your certificate, and you can test i Changing the default Karaf password can be done by modifying the `org.apache.unomi.security.root.password` in the `$MY_KARAF_HOME/etc/unomi.custom.system.properties` file +=== Tenant Management and API Access + +Apache Unomi supports multi-tenancy, allowing multiple organizations to use the same Unomi instance while keeping their data completely isolated. Each tenant has its own set of API keys for authentication. + +==== Creating and Managing Tenants + +IMPORTANT: All tenant management operations (create, list, update, delete, API key management) are restricted to administrators only and require JAAS authentication. These endpoints cannot be accessed using tenant API keys. + +To manage tenants, you need administrator access to Unomi (default credentials: karaf/karaf). You can manage tenants using either the REST API or the Karaf shell commands: + +Using REST API (requires admin credentials): +[source,bash] +---- +# Create a new tenant (JAAS auth required) +curl -X POST "http://localhost:8181/cxs/tenants" \ + -u karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "itemId": "mytenant", + "name": "My Company", + "description": "My Company tenant", + "properties": { + "address": "123 Main St", + "country": "USA" + } + }' + +# Response (HTTP 201 Created): +{ + "itemId": "mytenant", + "name": "My Company", + "description": "My Company tenant", + "properties": { + "address": "123 Main St", + "country": "USA" + }, + "itemType": "tenant", + "version": 1, + "status": "ACTIVE", + "creationDate": "2024-03-14T10:30:00Z", + "lastModificationDate": "2024-03-14T10:30:00Z" +} + +# List all tenants (JAAS auth required) +curl -X GET "http://localhost:8181/cxs/tenants" \ + -u karaf:karaf \ + -H "Accept: application/json" + +# Get tenant details (JAAS auth required) +curl -X GET "http://localhost:8181/cxs/tenants/mytenant" \ + -u karaf:karaf \ + -H "Accept: application/json" + +# Delete a tenant (JAAS auth required) +curl -X DELETE "http://localhost:8181/cxs/tenants/mytenant" \ + -u karaf:karaf +---- + +Using Karaf shell (requires admin access to Karaf console): +[source,bash] +---- +# Create a tenant +unomi:tenant-create mytenant "My Company" --description="My Company tenant" + +# List all tenants +unomi:tenant-list + +# View tenant details +unomi:tenant-show mytenant + +# Delete a tenant +unomi:tenant-delete mytenant +---- + +==== API Keys and Authentication + +Each tenant has two types of API keys: +* Public API Key: Used for client-side operations and public endpoints +* Private API Key: Used for secure operations and administrative tasks + +The API keys are automatically generated when creating a tenant. You can view them using: +[source,bash] +---- +# Using Karaf shell (requires admin access) +unomi:tenant-show mytenant + +# Output example: +Tenant Details: +ID: mytenant +Name: My Company +Description: My Company tenant +Status: ACTIVE +Creation Date: 2024-03-14T10:30:00Z +Last Modified: 2024-03-14T10:30:00Z +Public API Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c +Private API Key: 1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 +---- + +To generate new API keys (requires admin access): +[source,bash] +---- +# Using REST API (JAAS auth required) +curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PUBLIC&validityDays=30" \ + -u karaf:karaf \ + -H "Content-Type: application/json" + +# Response (HTTP 200 OK): +{ + "key": "8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c", + "type": "PUBLIC", + "expirationDate": "2024-04-13T10:30:00Z", + "creationDate": "2024-03-14T10:30:00Z" +} + +# Using Karaf shell (requires admin access) +unomi:tenant-generate-key mytenant PUBLIC 30 +---- + +==== Accessing API Endpoints + +There are three ways to authenticate with the Unomi API: + +1. Tenant Authentication (Recommended for most endpoints): +[source,bash] +---- +# List all profiles (tenant access) +curl -X GET "http://localhost:8181/cxs/profiles" \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Accept: application/json" + +# Response (HTTP 200 OK): +{ + "list": [ + { + "itemId": "profile1", + "properties": { + "firstName": "John", + "lastName": "Doe" + } + } + ], + "offset": 0, + "pageSize": 50, + "totalSize": 1 +} +---- ++ +2. Public API Access (Client-Side Operations): +[source,bash] +---- +# Get context data +curl -X POST "http://localhost:8181/cxs/context.json" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ + -H "Content-Type: application/json" \ + -d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "example" + }, + "requiredProfileProperties": ["firstName", "lastName"] + }' + +# Response (HTTP 200 OK): +{ + "profileId": "xyz123", + "sessionId": "abc456", + "profileProperties": { + "firstName": "John", + "lastName": "Doe" + } +} +---- ++ +3. Private API Access (Server-Side Operations): +[source,bash] +---- +# Get profiles using tenant credentials +curl -X GET "http://localhost:8181/cxs/profiles" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Accept: application/json" + +# Response (HTTP 200 OK): +{ + "list": [ + { + "itemId": "profile1", + "scope": "mytenant", + "properties": { + "firstName": "John", + "lastName": "Doe" + } + } + ], + "offset": 0, + "pageSize": 50, + "totalSize": 1 +} +---- + +Authentication Rules: + +1. If JAAS authentication is provided (username/password), it grants full access to all endpoints +2. Public paths (like /context.json) require a valid public API key +3. Private paths require both tenantId and private API key +4. All other requests are denied + +==== Public vs Private Endpoints + +Public endpoints (requiring only public API key): + +1. GET/POST /context.json +[source,bash] +---- +# Example request +curl -X GET "http://localhost:8181/cxs/context.json?sessionId=abc123" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" +---- ++ +2. GET/POST /eventcollector +[source,bash] +---- +# Example request +curl -X POST "http://localhost:8181/cxs/eventcollector" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ + -H "Content-Type: application/json" \ + -d '{ + "events": [{ + "eventType": "view", + "scope": "example", + "source": { + "itemId": "page1", + "itemType": "page", + "scope": "example" + }, + "target": { + "itemId": "product1", + "itemType": "product", + "scope": "example" + } + }] + }' +---- ++ +3. GET /client/* +[source,bash] +---- +# Example request +curl -X GET "http://localhost:8181/cxs/client/myapp/status" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" +---- + +All other endpoints are considered private and require either: +* JAAS authentication with admin credentials, or +* Private API key authentication with tenant credentials + +Example private endpoint access: +[source,bash] +---- +# Get segment details +curl -X GET "http://localhost:8181/cxs/segments/important-customers" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Accept: application/json" + +# Create a new segment +curl -X POST "http://localhost:8181/cxs/segments" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Content-Type: application/json" \ + -d '{ + "itemId": "high-value-customers", + "name": "High Value Customers", + "description": "Customers with high purchase value", + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "totalPurchases", + "comparisonOperator": "greaterThan", + "propertyValue": 1000 + } + } + }' +---- + === Scripting security ==== Multi-layer scripting filtering system @@ -422,7 +742,7 @@ To be valid, a Groovy action must follow a particular convention which is divide * An annotation used to define the associated action type * The function to be executed -Placed right before the function, the “@Action” annotation contains a set of parameter detailing how the action should be triggered. +Placed right before the function, the "@"Action annotation contains a set of parameter detailing how the action should be triggered. .@Action annotation |=== @@ -436,7 +756,7 @@ Placed right before the function, the “@Action” annotation contains a set of |actionExecutor |String |YES -|Action executor contains the name of the script to call for the action type and must be prefixed with “*groovy:*”. The prefix indicates to Unomi which dispatcher to use when processing the action. The name must be the file name of the groovy file containing the action without the extension (*groovy:*). +|Action executor contains the name of the script to call for the action type and must be prefixed with "*groovy:*". The prefix indicates to Unomi which dispatcher to use when processing the action. The name must be the file name of the groovy file containing the action without the extension (*groovy:*). |name |String @@ -475,24 +795,27 @@ Deploy/update an Action: [source,bash] ---- curl -X POST 'http://localhost:8181/cxs/groovyActions' \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ --form 'file=@""' ---- +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. + A Groovy Action can be updated by submitting another Action with the same id. Delete an Action: [source,bash] ---- curl -X DELETE 'http://localhost:8181/cxs/groovyActions/' \ ---user karaf:karaf +--user "TENANT_ID:PRIVATE_KEY" ---- Note that when a groovy action is deleted by the API, the action type associated with this action will also be deleted. ===== Hello World! -In this short example, we’re going to create a Groovy Action that will be adding “Hello world!” to the logs whenever a new view event is triggered. +In this short example, we're going to create a Groovy Action that will be adding "Hello world! +" to the logs whenever a new view event is triggered. The first step consists in creating the groovy script on your filesystem, start by creating the file `helloWorldGroovyAction.groovy`: @@ -513,7 +836,7 @@ Once the action has been created you need to submit it to Unomi (from the same f [source,bash] ---- curl -X POST 'http://localhost:8181/cxs/groovyActions' \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ --form 'file=@helloWorldGroovyAction.groovy' ---- @@ -523,7 +846,7 @@ Finally, register a rule to trigger execution of the groovy action: [source,bash] ---- curl -X POST 'http://localhost:8181/cxs/rules' \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "metadata": { @@ -548,22 +871,22 @@ curl -X POST 'http://localhost:8181/cxs/rules' \ }' ---- -Note that this rule contains a “location” parameter, with the value “world!”, which is then used in the log message triggered by the action. +Note that this rule contains a "location" parameter, with the value "world!", which is then used in the log message triggered by the action. -You can now use unomi to trigger a “view” event and see the corresponding message in the Unomi logs. +You can now use unomi to trigger a "view" event and see the corresponding message in the Unomi logs. -Once you’re done with the Hello World! action, it can be deleted using the following command: +Once you're done with the Hello World! action, it can be deleted using the following command: [source,bash] ---- curl -X DELETE 'http://localhost:8181/cxs/groovyActions/helloWorldGroovyAction' \ ---user karaf:karaf +--user "TENANT_ID:PRIVATE_KEY" ---- And the corresponding rule can be deleted using the following command: [source,bash] ---- curl -X DELETE 'http://localhost:8181/cxs/rules/scriptGroovyActionRule' \ ---user karaf:karaf +--user "TENANT_ID:PRIVATE_KEY" ---- ===== Inject an OSGI service in a groovy script @@ -631,13 +954,213 @@ versions will of course maintain compatibility with existing scripting solutions Apache Unomi is capable of merging profiles based on a common property value. In order to use this, you must add the MergeProfileOnPropertyAction to a rule (such as a login rule for example), and configure it with the name - of the property that will be used to identify the profiles to be merged. An example could be the "email" property, - meaning that if two (or more) profiles are found to have the same value for the "email" property they will be merged - by this action. +of the property that will be used to identify the profiles to be merged. An example could be the "email" property, +meaning that if two (or more) profiles are found to have the same value for the "email" property they will be merged +by this action. + +==== How profile merging works -Upon merge, the old profiles are marked with a "mergedWith" property that will be used on next profile access to delete -the original profile and replace it with the merged profile (aka "master" profile). Once this is done, all cookie tracking -will use the merged profile. +When a profile merge is triggered, the following process takes place: + +[plantuml] +---- +@startuml +title Apache Unomi Profile Merge Workflow + +actor "User/Website" as User +participant "Event\nProcessor" as Event +participant "MergeProfilesOnPropertyAction" as Action +participant "ProfileService" as Profile +participant "SchedulerService" as Scheduler +database "Elasticsearch" as ES + +User -> Event: Sends event (e.g., login) +activate Event + +Event -> Action: Triggers action via rule +activate Action + +Action -> ES: Query profiles with matching property +ES --> Action: Return matching profiles + +alt No matches found + Action --> Event: No merge needed +else Matches found + Action -> Profile: mergeProfiles(master, profileList) + activate Profile + + Profile -> Profile: Merge properties according to strategy + Profile -> Profile: Create aliases for merged profiles + Profile -> ES: Save master profile with updated properties + Profile -> ES: Save aliases + Profile -> ES: Delete merged profiles + Profile --> Action: Return master profile + deactivate Profile + + Action -> Event: Update event's profile reference + + Action -> Scheduler: Schedule async browsing data reassignment + activate Scheduler + Scheduler -> ES: Update sessions and events for merged profiles + Scheduler --> Action: Confirmation + deactivate Scheduler +end + +Action --> Event: Return result code +deactivate Action + +Event --> User: Event processed +deactivate Event +@enduml +---- + +==== Before and after a profile merge + +This diagram illustrates the state of profiles before and after a merge: + +[plantuml] +---- +@startuml +title Before and After Profile Merge + +package "Before Merge" { + object "Profile A" as PA { + id = "profile-a-123" + email = "user@example.com" + firstName = "John" + sessions = [session-a1, session-a2] + events = [event-a1, event-a2, event-a3] + } + + object "Profile B" as PB { + id = "profile-b-456" + email = "user@example.com" + lastName = "Doe" + sessions = [session-b1] + events = [event-b1, event-b2] + } +} + +package "After Merge" { + object "Master Profile" as MP { + id = "profile-a-123" + email = "user@example.com" + firstName = "John" + lastName = "Doe" + sessions = [session-a1, session-a2, session-b1] + events = [event-a1, event-a2, event-a3, event-b1, event-b2] + } + + object "Profile Alias" as AL { + aliasId = "profile-b-456" + profileId = "profile-a-123" + } + + MP -- AL : references +} + +PA ..> MP : becomes +PB ..> AL : becomes alias for Master +@enduml +---- + +Starting from Unomi 2: + +* The oldest profile (by firstVisit timestamp) becomes the master profile by default +* All properties from merged profiles are combined into the master profile +* An alias is created for each merged profile, pointing to the master profile +* The original merged profiles are deleted +* All sessions and events from merged profiles are reassigned to the master profile asynchronously + +==== Configuration parameters + +The `MergeProfilesOnPropertyAction` supports the following parameters: + +|=== +|Parameter |Type |Description |Required + +|mergeProfilePropertyName +|String +|The system property name used to identify profiles for merging +|Yes + +|mergeProfilePropertyValue +|String +|The value to match against the property. Often set dynamically from event properties using `eventProperty::target.properties(propertyName)` +|Yes + +|forceEventProfileAsMaster +|Boolean +|If true, forces the current event's profile to be the master profile after merging. If false, the oldest profile (by firstVisit) becomes the master. +|No (defaults to false) +|=== + +==== Security considerations + +IMPORTANT: Never trigger profile merges from unauthenticated operations such as form submissions or public-facing APIs. Always verify user identity before performing a merge. + +The following diagram highlights key security considerations: + +[plantuml] +---- +@startuml +title Profile Merge Security Considerations + +start +:Evaluate Merge Strategy; + +if (Is the event from an authenticated user?) then (yes) + :Safe to proceed; +else (no) + :SECURITY RISK!; + note right: Unauthenticated merges can lead to\nprofile information stealing + stop +endif + +if (Is the merge property sufficiently unique?) then (yes) + :Safe to proceed; +else (no) + :SECURITY RISK!; + note right: Common values could cause\nunrelated profiles to merge + stop +endif + +if (Is the merge property protected?) then (yes) + :Safe to proceed; +else (no) + :SECURITY RISK!; + note right: Unprotected properties can be\nmanipulated by malicious users + stop +endif + +:Proceed with Profile Merge; +stop +@enduml +---- + +Key security recommendations: + +* *Always authenticate users* before performing profile merges +* *Use protected properties* that require authentication to modify +* *Choose unique identifiers* like verified email or account ID +* *Implement rate limiting* for merge operations to prevent abuse +* *Consider additional verification* before merging profiles with sensitive data + +==== Troubleshooting profile merges + +If profiles aren't merging as expected, check: + +1. The merge property exists on both profiles with exactly matching values +2. The merge property is stored as a system property (`systemProperties.mergeIdentifier`) +3. The rule containing the merge action is correctly triggered +4. The profiles aren't personas or anonymous profiles (which are skipped) + +==== Performance considerations + +* The `maxProfilesInOneMerge` parameter (default: 50) limits how many profiles are merged in a single operation +* Large numbers of merges can impact system performance +* Session and event reassignment happens asynchronously to prevent blocking the event processing pipeline +* Consider the impact of merges on your Elasticsearch cluster, especially for high-traffic sites To test, simply configure the action in the "login" or "facebookLogin" rules and set it up on the "email" property. Upon sending one of the events, all matching profiles will be merged. @@ -1116,7 +1639,7 @@ For Docker deployments, you can declare a custom distribution feature repository version: '3.8' services: unomi: - image: apache/unomi:3.0.0 + image: apache/unomi:3.1.0 volumes: - ./unomi-custom-distribution-features.xml:/opt/apache-unomi/features/unomi-custom-distribution-features.xml environment: @@ -1308,3 +1831,159 @@ By default, all healthcheck providers are included but the list of those include Karaf provider is the one needed by healthcheck (always LIVE), it cannot be ignored. The timeout used for each health check can be set by setting the property `timeout` to the desired value in milliseconds. An environment variable can be used to set this property : UNOMI_HEALTHCHECK_TIMEOUT + +=== API Access Examples + +1. Basic Authentication Example: +[source,bash] +---- +# Get authentication token +curl -X POST "http://localhost:8181/cxs/login" \ + -H "Content-Type: application/json" \ + -d '{ + "username": "myuser", + "password": "mypassword" + }' + +# Response (HTTP 200 OK): +{ + "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", + "totalSize": 1 +} +---- ++ +2. Public API Access (Client-Side Operations): +[source,bash] +---- +# Get context data +curl -X POST "http://localhost:8181/cxs/context.json" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ + -H "Content-Type: application/json" \ + -d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "example" + }, + "requiredProfileProperties": ["firstName", "lastName"] + }' + +# Response (HTTP 200 OK): +{ + "profileId": "xyz123", + "sessionId": "abc456", + "profileProperties": { + "firstName": "John", + "lastName": "Doe" + } +} +---- ++ +3. Private API Access (Server-Side Operations): +[source,bash] +---- +# Get profiles using tenant credentials +curl -X GET "http://localhost:8181/cxs/profiles" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Accept: application/json" + +# Response (HTTP 200 OK): +{ + "list": [ + { + "itemId": "profile1", + "scope": "mytenant", + "properties": { + "firstName": "John", + "lastName": "Doe" + } + } + ], + "offset": 0, + "pageSize": 50, + "totalSize": 1 +} +---- + +Authentication Rules: + +1. If JAAS authentication is provided (username/password), it grants full access to all endpoints +2. Public paths (like /context.json) require a valid public API key +3. Private paths require both tenantId and private API key +4. All other requests are denied + +==== Public vs Private Endpoints + +Public endpoints (requiring only public API key): + +1. GET/POST /context.json +[source,bash] +---- +# Example request +curl -X GET "http://localhost:8181/cxs/context.json?sessionId=abc123" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" +---- ++ +2. GET/POST /eventcollector +[source,bash] +---- +# Example request +curl -X POST "http://localhost:8181/cxs/eventcollector" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ + -H "Content-Type: application/json" \ + -d '{ + "events": [{ + "eventType": "view", + "scope": "example", + "source": { + "itemId": "page1", + "itemType": "page", + "scope": "example" + }, + "target": { + "itemId": "product1", + "itemType": "product", + "scope": "example" + } + }] + }' +---- ++ +3. GET /client/* +[source,bash] +---- +# Example request +curl -X GET "http://localhost:8181/cxs/client/myapp/status" \ + -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" +---- + +All other endpoints are considered private and require either: +* JAAS authentication with admin credentials, or +* Private API key authentication with tenant credentials + +Example private endpoint access: +[source,bash] +---- +# Get segment details +curl -X GET "http://localhost:8181/cxs/segments/important-customers" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Accept: application/json" + +# Create a new segment +curl -X POST "http://localhost:8181/cxs/segments" \ + --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ + -H "Content-Type: application/json" \ + -d '{ + "itemId": "high-value-customers", + "name": "High Value Customers", + "description": "Customers with high purchase value", + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "totalPurchases", + "comparisonOperator": "greaterThan", + "propertyValue": 1000 + } + } + }' +---- diff --git a/manual/src/main/asciidoc/consent-api.adoc b/manual/src/main/asciidoc/consent-api.adoc index 02657c232..ad23f7ac5 100644 --- a/manual/src/main/asciidoc/consent-api.adoc +++ b/manual/src/main/asciidoc/consent-api.adoc @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[[_consent_api]] === Consent API Starting with Apache Unomi 1.3, a new API for consent management is now available. This API @@ -35,6 +36,7 @@ you can simply retrieve a profile with the following request: ---- curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "source": { @@ -126,6 +128,7 @@ You could send it using the following curl request: ---- curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "source":{ diff --git a/manual/src/main/asciidoc/datamodel.adoc b/manual/src/main/asciidoc/datamodel.adoc index 9c4359b5c..2f4d17e0a 100755 --- a/manual/src/main/asciidoc/datamodel.adoc +++ b/manual/src/main/asciidoc/datamodel.adoc @@ -12,6 +12,7 @@ // limitations under the License. // +[[_data_model_overview]] === Data Model Overview Apache Unomi gathers information about users actions, information that is processed and stored by Unomi services. @@ -222,7 +223,7 @@ Inherits all the fields from: <> Event types are completely open, and any new event type will be accepted by Apache Unomi. -Apache Unomi also comes with an extensive list of <> you can find in the reference section of this manual. +Apache Unomi also comes with an extensive list of <<_builtin_event_types,built-in event types>> you can find in the reference section of this manual. === Profile @@ -612,6 +613,7 @@ The visitor’s location is also resolve based on the IP address that was used t } ---- +[[_segment]] === Segment Segments are used to group profiles together, and are based on conditions that are executed on profiles to determine @@ -681,7 +683,7 @@ Here is an example of a simple segment definition registered using the REST API: [source] ---- curl -X POST http://localhost:8181/cxs/segments \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -730,7 +732,7 @@ The result of a condition is always a boolean value of true or false. Apache Unomi provides quite a lot of built-in condition types, including boolean types that make it possible to compose conditions using operators such as `and`, `or` or `not`. Composition is an essential element of building more complex conditions. -For a more complete list of available condition types, see the <> reference section. +For a more complete list of available condition types, see the <<_builtin_condition_types,Built-in condition types>> reference section. ==== Structure definition @@ -875,7 +877,7 @@ calling web hooks, setting profile properties, extracting data from the incoming an IP address), or even pulling and/or pushing data to third-party systems such as a CRM server. Apache Unomi also comes with built-in action types. -You may find the list of built-in action types in the <> section. +You may find the list of built-in action types in the <<_builtin_action_types,Built-in action types>> section. ==== Structure definition diff --git a/manual/src/main/asciidoc/getting-started.adoc b/manual/src/main/asciidoc/getting-started.adoc index 3f59a7ea9..c7acd9121 100644 --- a/manual/src/main/asciidoc/getting-started.adoc +++ b/manual/src/main/asciidoc/getting-started.adoc @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[[_getting_started_with_unomi]] === Getting started with Unomi We will first get you up and running with an example. We will then lift the corner of the cover somewhat and explain @@ -21,6 +22,7 @@ in greater details what just happened. This document assumes working knowledge of https://git-scm.com/[git] to be able to retrieve the code for Unomi and the example. Additionally, you will require a working Java 17 or above install. Refer to http://www.oracle.com/technetwork/java/javase/[http://www.oracle.com/technetwork/java/javase/] for details on how to download and install Java SE 17 or greater. +[[_jdk_compatibility]] ===== JDK compatibility Starting with Java 9, Oracle made some big changes to the Java platform releases. This is why Apache Unomi is focused on @@ -50,8 +52,8 @@ Note for OpenSearch users: ===== Start Unomi -Start Unomi according to the <> or by compiling using the -<>. Once you have Karaf running, you should wait until you see the following messages on the Karaf console: +Start Unomi according to the <<_five_minutes_quickstart,quick start with docker>> or by compiling using the +<<_building,building instructions>>. Once you have Karaf running, you should wait until you see the following messages on the Karaf console: [source] ---- @@ -65,12 +67,54 @@ Initializing profile service endpoint... Initializing cluster service endpoint... ---- -This indicates that all the Unomi services are started and ready to react to requests. You can then: +This indicates that all the Unomi services are started and ready to react to requests. + +Before you can use the API, you need to create a tenant: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/tenants \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "requestedId": "default", + "properties": { + "name": "Default Tenant", + "description": "Default tenant for getting started" + } + }' +---- + +Save the API keys from the response - you'll need them for all subsequent API calls. + +Now you can: 1. Access the RESTful services list: `http://localhost:8181/cxs` -2. Retrieve an initial context: `http://localhost:8181/cxs/context.json` + +2. Retrieve an initial context: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ + -H "Content-Type: application/json" \ + -H "X-Unomi-Api-Key: " \ + -d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "default" + } + }' +---- + +[NOTE] +==== +This request will automatically create a profile and session if they don't exist, and return a cookie with the profile ID. See <<_how_profile_tracking_works,How profile tracking works>> for details. +==== + ++ 3. View the introduction page: `http://localhost:8181` Also now that your service is up and running you can go look at the -<> to learn basic +<<_request_examples,request examples>> to learn basic requests you can do once your server is up and running. diff --git a/manual/src/main/asciidoc/graphql.adoc b/manual/src/main/asciidoc/graphql.adoc index 984a7bbde..54da2adaf 100644 --- a/manual/src/main/asciidoc/graphql.adoc +++ b/manual/src/main/asciidoc/graphql.adoc @@ -35,3 +35,92 @@ Thanks to GraphQL introspection, there is no dedicated documentation per-se as t You can easily view the schema by navigrating to `/graphql-ui`, depending on your setup (localhost, public host, ...), you might need to adjust the URL to point GraphQL UI to the `/graphql` endpoint. + +=== Multi-Tenancy Support + +The GraphQL API includes comprehensive multi-tenancy support, allowing different tenants to have customized GraphQL schemas based on their specific data models and property types. + +==== Overview + +The GraphQL schema provider automatically creates and manages tenant-specific schemas, ensuring: + +* Each tenant has its own GraphQL schema that reflects only their specific property types +* Changes to property types trigger automatic schema updates +* Proper tenant isolation is maintained throughout the GraphQL API + +==== Architecture + +The multi-tenancy support is built on these key components: + +===== Schema Cache + +Each tenant's GraphQL schema is created on demand and cached for performance: + +[source,java] +---- +private final ConcurrentMap tenantSchemas = new ConcurrentHashMap<>(); +---- + +===== Tenant Context Detection + +When processing a GraphQL request, the system automatically detects the current tenant from the execution context: + +[source,java] +---- +String tenantId = executionContextManager.getCurrentContext() != null ? + executionContextManager.getCurrentContext().getTenantId() : null; +---- + +===== Dynamic Schema Creation + +Each tenant's schema is built dynamically based on its specific property types and configurations: + +[source,java] +---- +GraphQL graphQL = graphQLSchemaUpdater.getGraphQLForTenant(tenantId); +---- + +===== Schema Invalidation + +When property types change, the affected tenant's schema is automatically invalidated: + +[source,java] +---- +public void invalidateTenantSchema(String tenantId) { + tenantSchemas.remove(tenantId); +} +---- + +==== Schema Lifecycle + +Tenant schemas follow this lifecycle: + +. *Creation*: Schemas are created on-demand when first requested +. *Caching*: Created schemas are cached for performance +. *Invalidation*: When property types change, the affected schemas are invalidated +. *Regeneration*: Invalid schemas are regenerated on the next request + +==== Performance Considerations + +* Schemas are created lazily on first request to avoid unnecessary overhead +* Schema creation can be resource-intensive, so caching is essential +* Property type changes trigger selective schema invalidation to minimize rebuilding +* Only the affected tenant's schema is invalidated when property types change + +==== Benefits + +This tenant-aware design provides several advantages: + +* *Isolation*: Each tenant has access only to their own data model +* *Customization*: Tenants can define custom property types that appear in their schema +* *Performance*: Schema caching improves response times +* *Consistency*: Changes to property types are immediately reflected in the API + +==== Troubleshooting + +If tenant-specific schemas aren't working as expected: + +* Check that the tenant context is properly set +* Verify property types have correct tenant IDs +* Review logs for schema creation and invalidation events +* Try explicitly invalidating the tenant schema to force regeneration diff --git a/manual/src/main/asciidoc/how-profile-tracking-works.adoc b/manual/src/main/asciidoc/how-profile-tracking-works.adoc index 0296dd3e1..db3e9ebd4 100644 --- a/manual/src/main/asciidoc/how-profile-tracking-works.adoc +++ b/manual/src/main/asciidoc/how-profile-tracking-works.adoc @@ -11,24 +11,1176 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[[_how_profile_tracking_works]] === How profile tracking works -In this section you will learn how Apache Unomi keeps track of visitors. - -==== Steps - -1. A visitor comes to a website -2. The web server resolves a previous request session ID if it exists, or if it doesn't it create a new sessionID -3. A request to Apache Unomi's /cxs/context.json servlet is made passing the web server session ID as a query parameter -4. Unomi uses the sessionID and tries to load an existing session, if none is found a new session is created with the -ID passed by the web server -5. If a session was found, the profile ID is extracted from the session and if it not found, Unomi looks for a cookie -called `context-profile-id` to read the profileID. If no profileID is found or if the session didn't exist, a new -profile ID is created by Apache Unomi -6. If the profile ID existed, the corresponding profile is loaded by Apache Unomi, otherwise a new profile is created -7. If events were passed along with the request to the context.json endpoint, they are processed against the profile -8. The updated profile is sent back as a response to the context.json request. Along with the response - -It is important to note that the profileID is always server-generated. Injecting a custom cookie with a non-valid -profile ID will result in failure to load the profile. Profile ID are UUIDs, which make them (pretty) safe from brute- -forcing. +In this section you will learn how Apache Unomi keeps track of visitors and processes context requests. + +==== Overview + +When a visitor interacts with a website that uses Apache Unomi, a single request to the `/cxs/context.json` or `/cxs/context.js` endpoint can perform multiple operations: + +* Automatically create or load a visitor profile +* Automatically create or load a visitor session +* Process events (if provided) +* Execute rules triggered by those events +* Resolve personalization (if requested) +* Set a cookie with the profile ID for future requests + +All of this happens in a single request-response cycle, making Apache Unomi efficient and easy to integrate. + +[plantuml] +---- +@startuml +title Overview: Single Request Processing Flow + +Client -> ContextEndpoint: POST /cxs/context.json\n+ X-Unomi-Api-Key\n+ sessionId\n+ events (optional)\n+ filters (optional)\n+ personalizations (optional) +activate ContextEndpoint + +ContextEndpoint -> AuthenticationFilter: Resolve tenant +activate AuthenticationFilter +AuthenticationFilter -> TenantService: Lookup tenant by API key +TenantService --> AuthenticationFilter: Tenant ID +AuthenticationFilter --> ContextEndpoint: Tenant context set +deactivate AuthenticationFilter + +ContextEndpoint -> RestServiceUtils: initEventsRequest() +activate RestServiceUtils +RestServiceUtils -> RestServiceUtils: Resolve profile ID\n(from cookie/parameter) +RestServiceUtils -> ProfileService: Load profile +alt Profile not found + RestServiceUtils -> RestServiceUtils: Create new profile\n(generate UUID) +end +RestServiceUtils -> ProfileService: Load session +alt Session not found + RestServiceUtils -> RestServiceUtils: Create new session + RestServiceUtils -> EventService: Send sessionCreated event +end +alt Profile was created + RestServiceUtils -> EventService: Send profileUpdated event +end +RestServiceUtils --> ContextEndpoint: EventsRequestContext +deactivate RestServiceUtils + +alt Events provided + ContextEndpoint -> RestServiceUtils: performEventsRequest() + activate RestServiceUtils + loop For each event + RestServiceUtils -> EventService: Send event + EventService -> RulesService: Execute matching rules + RulesService -> RulesService: Update profile/segments/scores + end + RestServiceUtils --> ContextEndpoint: Updated context + deactivate RestServiceUtils +end + +alt Filters provided + ContextEndpoint -> PersonalizationService: Filter content + PersonalizationService -> PersonalizationService: Evaluate conditions\n(using updated profile) + PersonalizationService --> ContextEndpoint: Filtering results +end + +alt Personalizations provided + ContextEndpoint -> PersonalizationService: Resolve personalization + PersonalizationService -> PersonalizationService: Evaluate filters\n(using updated profile) + PersonalizationService --> ContextEndpoint: Selected content +end + +ContextEndpoint -> RestServiceUtils: finalizeEventsRequest() +activate RestServiceUtils +RestServiceUtils -> ProfileService: Save profile (if updated) +RestServiceUtils -> ProfileService: Save session (if updated) +RestServiceUtils -> HttpUtils: Generate cookie string +RestServiceUtils --> ContextEndpoint: Cookie set in response +deactivate RestServiceUtils + +ContextEndpoint --> Client: ContextResponse\n+ Profile data\n+ Session data\n+ Personalization results\n+ Set-Cookie header +deactivate ContextEndpoint + +@enduml +---- + +==== Detailed Request Flow + +When a request is made to `/cxs/context.json` or `/cxs/context.js`, Apache Unomi processes it through the following steps: + +[IMPORTANT] +==== +**Tenant Resolution (Version 3.1+)** + +Starting with Apache Unomi 3.1, tenant resolution is mandatory for all requests to `/cxs/context.json` and `/cxs/eventcollector` endpoints. Apache Unomi must know which tenant's data to use before processing the request. The tenant is resolved through one of the following methods (in priority order): + +1. **Public API Key Authentication** (Recommended for public endpoints): + * Send the `X-Unomi-Api-Key` header with a public API key + * Apache Unomi automatically resolves the tenant from the API key + * This is the recommended method for public-facing applications + +2. **Basic Authentication with Private API Key**: + * Use Basic Auth with format: `tenantId:privateApiKey` + * The tenant is resolved from the credentials + * This is used for administrative or server-to-server requests + +3. **Tenant ID Header** (When using JAAS authentication): + * Send the `X-Unomi-Tenant-Id` header with the tenant ID + * Used when authenticating via JAAS (e.g., `karaf:karaf`) + * The tenant ID must exist in the system + +If no tenant can be resolved, the request will fail with an `UNAUTHORIZED` (401) error. This ensures that all data operations are scoped to the correct tenant in multi-tenant deployments. +==== + +===== Example: Tenant Resolution + +Here are examples of how to authenticate and resolve the tenant: + +**Example 1: Using Public API Key (Recommended)** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: public-key-abc123..." \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +The tenant is automatically resolved from the public API key. No additional headers needed. + +**Example 2: Using Basic Auth with Private API Key** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +--user "mytenant:private-key-xyz789..." \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +The tenant is resolved from the Basic Auth credentials (`mytenant` is the tenant ID). + +**Example 3: Using JAAS Auth with Tenant Header** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +--user "karaf:karaf" \ +-H "X-Unomi-Tenant-Id: mytenant" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +The tenant is resolved from the `X-Unomi-Tenant-Id` header when using JAAS authentication. + +[plantuml] +---- +@startuml +title Tenant Resolution Flow (Version 3.1+) + +Client -> AuthenticationFilter: Request with authentication +activate AuthenticationFilter + +alt Public API Key (Recommended) + Client -> AuthenticationFilter: X-Unomi-Api-Key header + AuthenticationFilter -> TenantService: getTenantByApiKey(apiKey, PUBLIC) + TenantService --> AuthenticationFilter: Tenant + AuthenticationFilter -> ExecutionContextManager: Set tenant context + AuthenticationFilter --> Client: Request authorized +else Basic Auth with Private Key + Client -> AuthenticationFilter: Basic Auth: tenantId:privateKey + AuthenticationFilter -> TenantService: validateApiKeyWithType(tenantId, key, PRIVATE) + TenantService --> AuthenticationFilter: Valid + AuthenticationFilter -> ExecutionContextManager: Set tenant context + AuthenticationFilter --> Client: Request authorized +else JAAS Auth with Tenant Header + Client -> AuthenticationFilter: JAAS Auth + X-Unomi-Tenant-Id header + AuthenticationFilter -> TenantService: getTenant(tenantId) + TenantService --> AuthenticationFilter: Tenant + AuthenticationFilter -> ExecutionContextManager: Set tenant context + AuthenticationFilter --> Client: Request authorized +else No Tenant Resolved + AuthenticationFilter --> Client: 401 UNAUTHORIZED +end + +deactivate AuthenticationFilter + +note right of AuthenticationFilter + Tenant resolution is mandatory + in version 3.1+ for context.json + and eventcollector endpoints +end note + +@enduml +---- + +===== Step 1: Profile Identification and Creation + +[IMPORTANT] +==== +At least one of the following must be provided in the request: `sessionId`, `profileId` (as a parameter or in a cookie), or `personaId`. If none of these are provided, the request will fail with a `BadRequestException` (400 error). +==== + +Apache Unomi attempts to identify the visitor's profile through the following process: + +1. **Profile ID Resolution**: + * First checks for a `profileId` parameter in the request + * If not found, looks for a cookie named `context-profile-id` (configurable via `org.apache.unomi.profile.cookie.name`) + * Cookie values are validated against a JSON schema - invalid values (e.g., containing script tags) will cause a `400 Bad Request` error + * The resolved profile ID is used to attempt loading the profile from the database + +2. **Session Profile Override** (if session exists): + * If a session is found (see Step 2) and contains a profile ID that differs from the cookie/profileId parameter + * Apache Unomi uses the session's profile ID instead (this handles cases where a user switches accounts) + * The profile is reloaded from the database using the session's profile ID + +3. **Profile Creation**: If no profile ID is found or the profile doesn't exist: + * If a profile ID was provided (from parameter or cookie) but doesn't exist in the database, creates a new profile with that ID + * If no profile ID was found, generates a new UUID as the profile ID and creates a new profile + * Sets the `firstVisit` property to the current timestamp + * Marks the profile for persistence + * Note: The `profileUpdated` event is sent after session creation (if a session is created) to ensure proper event ordering + +This ensures that profiles are always available for processing, even for first-time visitors or after database resets. + +[IMPORTANT] +==== +The profile ID is always server-generated (UUID format). Even if a client sends a custom profile ID in a cookie or parameter, Apache Unomi validates it exists in the database. If it doesn't exist, a new profile is created (potentially with that ID if provided via parameter, or with a newly generated UUID). This makes profile IDs secure and prevents profile ID manipulation. +==== + +===== Example: Profile Identification + +**Example 1: First-Time Visitor (No Cookie)** + +[source,bash] +---- +# Request (no cookie sent) +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +**Response Headers:** +[source,http] +---- +Set-Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890; Path=/; Max-Age=31536000; SameSite=Lax +---- + +**Response Body:** +[source,json] +---- +{ + "profileId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "sessionId": "1234" +} +---- + +Apache Unomi generated a new UUID (`a1b2c3d4-e5f6-7890-abcd-ef1234567890`) and created a new profile. The profile ID is returned in both the response body and the `Set-Cookie` header. + +**Example 2: Returning Visitor (With Cookie)** + +[source,bash] +---- +# Request (browser automatically sends cookie) +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +**Response:** +[source,json] +---- +{ + "profileId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "sessionId": "1234" +} +---- + +Apache Unomi loaded the existing profile using the profile ID from the cookie. + +**Example 3: Using profileId Parameter** + +[source,bash] +---- +# Request with explicit profileId parameter +curl -X POST "http://localhost:8181/cxs/context.json?sessionId=1234&profileId=a1b2c3d4-e5f6-7890-abcd-ef1234567890" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +The `profileId` parameter takes precedence over the cookie value. + +[plantuml] +---- +@startuml +title Profile and Session Identification Flow + +RestServiceUtils -> RestServiceUtils: Get profileId from parameter +alt profileId parameter exists + RestServiceUtils -> ProfileService: load(profileId) +else Check cookie + RestServiceUtils -> HttpServletRequest: getCookie("context-profile-id") + HttpServletRequest --> RestServiceUtils: profileId from cookie + RestServiceUtils -> ProfileService: load(profileId) +end + +alt Profile found + RestServiceUtils -> RestServiceUtils: Use existing profile +else Profile not found + RestServiceUtils -> RestServiceUtils: createNewProfile(profileId or null) + RestServiceUtils -> RestServiceUtils: Generate UUID if needed + RestServiceUtils -> RestServiceUtils: Set firstVisit property + RestServiceUtils -> EventService: Send profileUpdated event + RestServiceUtils -> RestServiceUtils: Mark for persistence +end + +alt sessionId provided + RestServiceUtils -> ProfileService: loadSession(sessionId) + alt Session found + RestServiceUtils -> RestServiceUtils: Check session profile + alt Session profile differs + RestServiceUtils -> ProfileService: load(sessionProfileId) + RestServiceUtils -> RestServiceUtils: Use session's profile + end + else Session not found + RestServiceUtils -> RestServiceUtils: Create new Session(sessionId, profile) + RestServiceUtils -> EventService: Send sessionCreated event + RestServiceUtils -> RestServiceUtils: Mark for persistence + end +end + +@enduml +---- + +===== Step 2: Session Identification and Creation + +If a `sessionId` is provided in the request: + +1. **Session Loading**: Apache Unomi attempts to load the existing session from the database +2. **Profile Association**: If a session is found, it may contain a profile ID that takes precedence over the cookie/profileId parameter +3. **Session Creation**: If no session is found or the session is invalidated: + * Creates a new session with the provided `sessionId` + * Associates the session with the current profile (or anonymous profile if privacy settings require it) + * Sends a `sessionCreated` event (this happens before the `profileUpdated` event if both profile and session are created) + * Marks the session for persistence + * After session creation, if a new profile was created, sends a `profileUpdated` event (non-persistent) to mark the profile creation + +If no `sessionId` is provided, a transient (non-persisted) session may be used for the request. + +===== Example: Session Creation + +**Example: Creating a New Session** + +[source,bash] +---- +# First request with a new sessionId +curl -X POST http://localhost:8181/cxs/context.json?sessionId=abc-123-xyz \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +Apache Unomi will: +1. Check if session `abc-123-xyz` exists +2. If not found, create a new session with ID `abc-123-xyz` +3. Associate it with the current profile +4. Send a `sessionCreated` event +5. Save the session to the database + +**Subsequent Request with Same Session:** + +[source,bash] +---- +# Request with existing sessionId +curl -X POST http://localhost:8181/cxs/context.json?sessionId=abc-123-xyz \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "product-page", + "itemType": "page", + "scope": "mydigital" + } +}' +---- + +Apache Unomi loads the existing session `abc-123-xyz` and uses the profile associated with it. + +===== Step 3: Event Processing + +If events are provided in the request body: + +1. **Event Validation**: Each event is validated against its JSON schema +2. **Event Authorization**: + * Events are checked for tenant authorization (the tenant must be authorized to send the event type) + * Events are filtered based on privacy settings (if the profile has opted out of certain event types) + * Events are checked against IP address restrictions if configured +3. **Event Execution**: For each valid event: + * The event is sent to the event service + * All matching rules are automatically executed (this happens synchronously during the request) + * Rules can trigger actions that modify the profile, session, or trigger other events + * Profile segments and scores are recalculated based on rule execution + * If the profile is updated during rule execution, the updated profile is used for subsequent events in the same request +4. **Profile Updates**: If events cause profile changes, the profile is marked for persistence + +This is where the power of Apache Unomi's rule engine comes into play - events automatically trigger rules, which can perform complex logic, update profiles, add segments, calculate scores, and more. All of this happens within the same request, ensuring that personalization and subsequent operations use the most up-to-date profile state. + +===== Example: Event Processing with Rule Execution + +**Example: Sending a View Event** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "product-page", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "source": { + "itemType": "site", + "scope": "mydigital", + "itemId": "mysite" + }, + "target": { + "itemType": "product", + "scope": "mydigital", + "itemId": "product-123", + "properties": { + "name": "Awesome Product", + "category": "electronics", + "price": 99.99 + } + } + } + ] +}' +---- + +**What Happens:** + +1. The `view` event is validated against its schema +2. The event is checked for tenant authorization +3. The event is sent to the event service +4. All rules matching the `view` event are executed: + * Rules might increment a `pageViewCount` property + * Rules might add the profile to a "product-viewers" segment + * Rules might calculate a score based on product category + * Rules might trigger additional events +5. The profile is updated with any changes from rule execution +6. The updated profile is used for personalization (if requested) + +**Response:** +[source,json] +---- +{ + "profileId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "sessionId": "1234", + "processedEvents": 1 +} +---- + +The `processedEvents` field indicates that 1 event was successfully processed. The profile may have been updated by rules triggered by this event. + +[plantuml] +---- +@startuml +title Event Processing and Rule Execution Flow + +RestServiceUtils -> RestServiceUtils: performEventsRequest(events) +activate RestServiceUtils + +loop For each event in request + RestServiceUtils -> SchemaService: Validate event schema + alt Event valid + RestServiceUtils -> RestServiceUtils: Check tenant authorization + RestServiceUtils -> PrivacyService: Check privacy filters + alt Event authorized and not filtered + RestServiceUtils -> EventService: send(event) + activate EventService + EventService -> RulesService: Find matching rules + activate RulesService + loop For each matching rule + RulesService -> RulesService: Evaluate rule conditions + alt Conditions match + RulesService -> RulesService: Execute rule actions + RulesService -> ProfileService: Update profile properties + RulesService -> ProfileService: Recalculate segments + RulesService -> ProfileService: Recalculate scores + RulesService -> EventService: Trigger additional events (if any) + end + end + deactivate RulesService + EventService --> RestServiceUtils: Event processed + deactivate EventService + RestServiceUtils -> RestServiceUtils: Update profile from event + else Event filtered or not authorized + RestServiceUtils -> RestServiceUtils: Skip event + end + else Event invalid + RestServiceUtils -> RestServiceUtils: Skip event + end +end + +RestServiceUtils -> RestServiceUtils: Mark profile for persistence\nif changes occurred +deactivate RestServiceUtils + +note right of RulesService + Rules execute synchronously + during the request, ensuring + profile updates are immediately + available for personalization +end note + +@enduml +---- + +===== Step 4: Content Filtering and Personalization Resolution + +If filters or personalization requests are provided in the request body: + +1. **Content Filtering** (if `filters` are provided): + * For each filter node, evaluates conditions against the current profile and session + * Returns filtering results indicating which content variants match + * Results are added to the response's `filteringResults` object + +2. **Personalization Resolution** (if `personalizations` are provided): + * For each personalization request: + * Evaluates content filters against the current profile and session (which may have been updated by events in Step 3) + * Filters use conditions to determine which content variants match + * Applies the personalization strategy (e.g., "matching-first", "random", etc.) + * **Content Selection**: Selects the appropriate content based on: + * Filter condition evaluation results + * Personalization strategy + * Fallback content (if no filters match) + * Results are added to both `personalizationResults` (detailed results) and `personalizations` (selected content IDs) in the response + +Note that personalization uses the profile state after event processing, ensuring that events can influence which content is selected. + +===== Example: Personalization Resolution + +**Example: Request with Events and Personalization** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "target": { + "itemType": "page", + "scope": "mydigital", + "itemId": "homepage" + } + } + ], + "personalizations": [ + { + "id": "homepage-hero", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "default-hero" + }, + "contents": [ + { + "id": "premium-user-hero", + "content": "Welcome, Premium Member!", + "filters": [ + { + "condition": { + "type": "profileSegmentCondition", + "parameterValues": { + "segments": ["premium-users"] + } + } + } + ] + }, + { + "id": "new-visitor-hero", + "content": "Welcome! Sign up today!", + "filters": [ + { + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.firstVisit", + "comparisonOperator": "exists" + } + } + } + ] + } + ] + } + ], + "requireSegments": true +}' +---- + +**Processing Flow:** + +1. **Event Processing**: The `view` event is processed first, which may trigger rules that: + * Add the profile to segments + * Update profile properties + * Calculate scores + +2. **Personalization Resolution**: After events are processed, personalization is evaluated: + * For a first-time visitor: The `new-visitor-hero` content matches (because `firstVisit` exists) + * For a premium user: The `premium-user-hero` content matches (if the profile is in the "premium-users" segment) + * The `matching-first` strategy selects the first matching content + +3. **Response**: The response includes the selected content + +**Response:** +[source,json] +---- +{ + "profileId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "sessionId": "1234", + "processedEvents": 1, + "profileSegments": ["new-visitors"], + "personalizations": { + "homepage-hero": "new-visitor-hero" + }, + "personalizationResults": { + "homepage-hero": { + "selectedContentId": "new-visitor-hero", + "strategy": "matching-first" + } + } +} +---- + +The `personalizations` object shows the selected content ID for each personalization request. The `personalizationResults` provides detailed information about the selection process. + +[plantuml] +---- +@startuml +title Personalization Resolution Flow + +ContextEndpoint -> PersonalizationService: personalizeList(profile, session, request) +activate PersonalizationService + +loop For each personalization request + PersonalizationService -> PersonalizationService: Get content variants + + loop For each content variant + PersonalizationService -> PersonalizationService: Evaluate content filters + activate PersonalizationService + + loop For each filter condition + PersonalizationService -> ConditionService: Evaluate condition\n(profile, session, event history) + ConditionService --> PersonalizationService: Match result + end + + alt All filters match + PersonalizationService -> PersonalizationService: Content variant matches + else Filters don't match + PersonalizationService -> PersonalizationService: Skip variant + end + deactivate PersonalizationService + end + + PersonalizationService -> PersonalizationService: Apply strategy\n(matching-first, random, etc.) + alt Content selected + PersonalizationService -> PersonalizationService: Use selected content + else No content matches + PersonalizationService -> PersonalizationService: Use fallback content + end +end + +PersonalizationService --> ContextEndpoint: PersonalizationResult +deactivate PersonalizationService + +note right of PersonalizationService + Personalization uses the profile + state AFTER event processing, + so events can influence + content selection +end note + +@enduml +---- + +===== Step 5: Response Preparation + +The response is built with the following information (based on what was requested in the `ContextRequest`): + +* **Profile Information**: + * Profile ID (always included) + * Profile properties (if `requiredProfileProperties` is specified, with `"*"` for all properties) + * Profile segments (if `requireSegments` is `true`) + * Profile scores (if `requireScores` is `true`) + * Anonymous browsing status + * Consent information +* **Session Information**: + * Session ID (if session exists) + * Session properties (if `requiredSessionProperties` is specified) +* **Event Processing Results**: + * Count of processed events (`processedEvents`) +* **Personalization Results**: + * `personalizations`: Map of personalization ID to selected content IDs + * `personalizationResults`: Detailed results for each personalization request +* **Filtering Results**: + * `filteringResults`: Results of content filtering operations (if `filters` were provided) +* **Tracked Conditions**: + * Conditions that are being tracked for the current source (if not using a Persona) + +===== Step 6: Cookie Setting + +After processing, Apache Unomi sets a cookie in the HTTP response header (`Set-Cookie`) with: + +[NOTE] +==== +Cookies are only set for regular profiles, not for Personas. If a `personaId` is used in the request, no cookie will be set in the response. +==== + +* **Cookie Name**: `context-profile-id` (configurable, default: `context-profile-id`) +* **Cookie Value**: The profile ID (UUID) +* **Cookie Attributes**: + * `Path=/` - Available for all paths on the domain + * `Max-Age=31536000` - Valid for 1 year by default (configurable via `org.apache.unomi.profile.cookie.maxAgeInSeconds`) + * `SameSite=Lax` - CSRF protection + * `Secure` - Set if the request is over HTTPS (configurable) + * `HttpOnly` - Configurable via `org.apache.unomi.profile.cookie.httpOnly` (default: false) + * `Domain` - Configurable via `org.apache.unomi.profile.cookie.domain` + +This cookie allows the browser to automatically send the profile ID on subsequent requests, enabling Apache Unomi to load the existing profile. + +===== Example: Cookie Setting + +**Example: Cookie in Response Headers** + +When you make a request, Apache Unomi sets a cookie in the response. Here's what the cookie looks like: + +**Response Headers:** +[source,http] +---- +HTTP/1.1 200 OK +Content-Type: application/json;charset=UTF-8 +Set-Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890; Path=/; Max-Age=31536000; SameSite=Lax +---- + +**Cookie Breakdown:** +* `context-profile-id` - Cookie name (configurable) +* `a1b2c3d4-e5f6-7890-abcd-ef1234567890` - Profile ID (UUID) +* `Path=/` - Available for all paths on the domain +* `Max-Age=31536000` - Valid for 1 year (31,536,000 seconds) +* `SameSite=Lax` - CSRF protection + +**Subsequent Requests:** + +On the next request, the browser automatically includes the cookie: + +[source,http] +---- +# Browser automatically sends cookie +GET /cxs/context.json?sessionId=1234 HTTP/1.1 +Host: localhost:8181 +X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY +Cookie: context-profile-id=a1b2c3d4-e5f6-7890-abcd-ef1234567890 +---- + +Apache Unomi reads the cookie and loads the existing profile, maintaining continuity across requests. + +===== Step 7: Persistence + +Finally, Apache Unomi persists any changes: + +* **Profile Persistence**: If the profile was created or updated, it is saved to the database +* **Session Persistence**: If the session was created or updated, it is saved to the database + +==== First-Time Visitor Flow + +For a first-time visitor making their first request: + +1. No `context-profile-id` cookie exists +2. No `profileId` parameter is provided +3. Apache Unomi generates a new UUID and creates a new profile +4. If a `sessionId` is provided, a new session is created +5. The profile ID is returned in a cookie +6. The browser stores the cookie +7. On subsequent requests, the browser automatically sends the cookie +8. Apache Unomi loads the existing profile using the cookie value + +===== Complete Example: First-Time Visitor Journey + +**Request 1: First Visit (No Cookie)** + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=visitor-123 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "target": { + "itemType": "page", + "scope": "mydigital", + "itemId": "homepage" + } + } + ], + "requiredProfileProperties": ["properties.firstVisit"] +}' +---- + +**Response Headers:** +[source,http] +---- +Set-Cookie: context-profile-id=7f8e9d0c-1b2a-3456-7890-abcdef123456; Path=/; Max-Age=31536000; SameSite=Lax +---- + +**Response Body:** +[source,json] +---- +{ + "profileId": "7f8e9d0c-1b2a-3456-7890-abcdef123456", + "sessionId": "visitor-123", + "processedEvents": 1, + "profileProperties": { + "firstVisit": "2024-01-15T10:30:00Z" + } +} +---- + +**What Happened:** +* New profile created with UUID `7f8e9d0c-1b2a-3456-7890-abcdef123456` +* `firstVisit` property set to current timestamp +* New session created with ID `visitor-123` +* `sessionCreated` event sent (triggers any matching rules) +* `profileUpdated` event sent (triggers any matching rules) +* `view` event processed (may have triggered rules) +* Cookie set in response header +* Profile and session saved to database + +**Request 2: Second Visit (Cookie Present)** + +[source,bash] +---- +# Browser automatically includes the cookie from Request 1 +curl -X POST http://localhost:8181/cxs/context.json?sessionId=visitor-123 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=7f8e9d0c-1b2a-3456-7890-abcdef123456" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "product-page", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "target": { + "itemType": "product", + "scope": "mydigital", + "itemId": "product-456" + } + } + ], + "requiredProfileProperties": ["properties.firstVisit", "properties.pageViewCount"] +}' +---- + +**Response:** +[source,json] +---- +{ + "profileId": "7f8e9d0c-1b2a-3456-7890-abcdef123456", + "sessionId": "visitor-123", + "processedEvents": 1, + "profileProperties": { + "firstVisit": "2024-01-15T10:30:00Z", + "pageViewCount": 2 + } +} +---- + +**What Happened:** +* Existing profile loaded using cookie value `7f8e9d0c-1b2a-3456-7890-abcdef123456` +* Existing session loaded using `visitor-123` +* `view` event processed, which may have triggered a rule that incremented `pageViewCount` +* Cookie refreshed in response (expiration extended) +* Updated profile saved to database + +The profile is now being tracked across multiple requests, and any rules triggered by events can build up a profile of the visitor's behavior. + +[plantuml] +---- +@startuml +title First-Time Visitor vs Returning Visitor Flow + +== First Request (First-Time Visitor) == + +Client -> ContextEndpoint: POST /context.json\n+ X-Unomi-Api-Key\n+ sessionId=1234 +activate ContextEndpoint + +ContextEndpoint -> RestServiceUtils: initEventsRequest() +RestServiceUtils -> HttpServletRequest: getCookie("context-profile-id") +HttpServletRequest --> RestServiceUtils: null (no cookie) +RestServiceUtils -> RestServiceUtils: profileId = null +RestServiceUtils -> RestServiceUtils: createNewProfile(null) +RestServiceUtils -> RestServiceUtils: Generate UUID: "abc-123-def" +RestServiceUtils -> ProfileService: Create profile with firstVisit +RestServiceUtils -> EventService: Send profileUpdated event +RestServiceUtils -> ProfileService: Create session(sessionId, profile) +RestServiceUtils -> EventService: Send sessionCreated event +RestServiceUtils --> ContextEndpoint: Profile and session created + +ContextEndpoint -> RestServiceUtils: finalizeEventsRequest() +RestServiceUtils -> ProfileService: Save profile +RestServiceUtils -> ProfileService: Save session +RestServiceUtils -> HttpUtils: Generate cookie: "context-profile-id=abc-123-def" +RestServiceUtils -> HttpServletResponse: setHeader("Set-Cookie", cookie) +RestServiceUtils --> ContextEndpoint: Cookie set + +ContextEndpoint --> Client: ContextResponse\n+ Set-Cookie: context-profile-id=abc-123-def +deactivate ContextEndpoint + +Client -> Client: Browser stores cookie + +== Subsequent Request (Returning Visitor) == + +Client -> ContextEndpoint: POST /context.json\n+ X-Unomi-Api-Key\n+ sessionId=1234\n+ Cookie: context-profile-id=abc-123-def +activate ContextEndpoint + +ContextEndpoint -> RestServiceUtils: initEventsRequest() +RestServiceUtils -> HttpServletRequest: getCookie("context-profile-id") +HttpServletRequest --> RestServiceUtils: "abc-123-def" +RestServiceUtils -> ProfileService: load("abc-123-def") +ProfileService --> RestServiceUtils: Existing profile +RestServiceUtils -> ProfileService: loadSession("1234") +ProfileService --> RestServiceUtils: Existing session +RestServiceUtils --> ContextEndpoint: Profile and session loaded + +ContextEndpoint -> RestServiceUtils: finalizeEventsRequest() +RestServiceUtils -> HttpUtils: Generate cookie: "context-profile-id=abc-123-def" +RestServiceUtils -> HttpServletResponse: setHeader("Set-Cookie", cookie) +RestServiceUtils --> ContextEndpoint: Cookie refreshed + +ContextEndpoint --> Client: ContextResponse\n+ Set-Cookie: context-profile-id=abc-123-def\n(refreshed expiration) +deactivate ContextEndpoint + +@enduml +---- + +==== Complete Example: Full Request Flow + +Here's a complete example showing all aspects of a context request: + +**Request:** +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=example-session \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Cookie: context-profile-id=existing-profile-uuid" \ +-H "Content-Type: application/json" \ +-d '{ + "source": { + "itemId": "checkout-page", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "target": { + "itemType": "page", + "scope": "mydigital", + "itemId": "checkout-page" + } + } + ], + "personalizations": [ + { + "id": "checkout-banner", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "default-banner" + }, + "contents": [ + { + "id": "discount-banner", + "content": "Use code SAVE10 for 10% off!", + "filters": [ + { + "condition": { + "type": "profileSegmentCondition", + "parameterValues": { + "segments": ["frequent-buyers"] + } + } + } + ] + } + ] + } + ], + "requiredProfileProperties": ["*"], + "requireSegments": true, + "requireScores": true +}' +---- + +**Processing Steps:** + +1. **Tenant Resolution**: Resolved from `X-Unomi-Api-Key` header +2. **Profile Loading**: Loaded existing profile `existing-profile-uuid` from cookie +3. **Session Loading**: Loaded existing session `example-session` +4. **Event Processing**: + * `view` event validated and processed + * Rules executed (may update profile, segments, scores) + * Profile updated if rules triggered changes +5. **Personalization**: + * Evaluated using updated profile state + * Selected content based on segment membership +6. **Response Building**: Constructed with all requested data +7. **Cookie Setting**: Cookie refreshed in response header +8. **Persistence**: Profile and session saved if updated + +**Response:** +[source,json] +---- +{ + "profileId": "existing-profile-uuid", + "sessionId": "example-session", + "processedEvents": 1, + "profileProperties": { + "firstName": "John", + "lastName": "Doe", + "email": "john.doe@example.com", + "pageViewCount": 15 + }, + "profileSegments": ["frequent-buyers", "premium-customers"], + "profileScores": { + "loyalty": 85, + "engagement": 92 + }, + "personalizations": { + "checkout-banner": "discount-banner" + }, + "personalizationResults": { + "checkout-banner": { + "selectedContentId": "discount-banner", + "strategy": "matching-first" + } + } +} +---- + +**Response Headers:** +[source,http] +---- +Set-Cookie: context-profile-id=existing-profile-uuid; Path=/; Max-Age=31536000; SameSite=Lax +---- + +This example demonstrates how a single request can: +* Load an existing profile and session +* Process events that trigger rules +* Update the profile based on rule execution +* Resolve personalization using the updated profile state +* Return comprehensive profile data +* Maintain tracking via cookies + +==== Summary + +A single request to `/cxs/context.json` or `/cxs/context.js` performs the following operations in sequence: + +1. **Tenant Resolution** (mandatory in 3.1+): Identifies which tenant's data to use +2. **Profile/Session Management**: Creates or loads profiles and sessions automatically +3. **Event Processing**: Processes events and executes matching rules (synchronously) +4. **Content Filtering**: Evaluates filter conditions (if `filters` are provided) +5. **Personalization**: Resolves personalization requests (if `personalizations` are provided) +6. **Response Building**: Constructs the response with requested data (properties, segments, scores, etc.) +7. **Persistence**: Saves any changes to the database (profile and session) +8. **Cookie Setting**: Sets the profile ID cookie for future requests + +This integrated approach ensures that: +* Profiles and sessions are always available, even for first-time visitors +* Events immediately trigger rules, updating profiles in real-time +* Personalization uses the most up-to-date profile state +* The browser automatically tracks the visitor across requests via cookies + +==== Configuration + +The following configuration properties control profile tracking behavior: + +* `org.apache.unomi.profile.cookie.name` - Cookie name (default: `context-profile-id`) +* `org.apache.unomi.profile.cookie.maxAgeInSeconds` - Cookie expiration time (default: `31536000` = 1 year) +* `org.apache.unomi.profile.cookie.httpOnly` - Whether cookie is HTTP-only (default: `false`) +* `org.apache.unomi.profile.cookie.domain` - Cookie domain (optional) + +These can be configured in the `org.apache.unomi.web.cfg` configuration file. diff --git a/manual/src/main/asciidoc/index.adoc b/manual/src/main/asciidoc/index.adoc index 5b0913896..09a9d8843 100644 --- a/manual/src/main/asciidoc/index.adoc +++ b/manual/src/main/asciidoc/index.adoc @@ -12,9 +12,9 @@ // limitations under the License. // -= Apache Unomi 2.x - Documentation += Apache Unomi 3.x - Documentation Apache Software Foundation -:doctype: article +:doctype: book :toc: left :toclevels: 3 :toc-position: left @@ -24,6 +24,9 @@ Apache Software Foundation image::asf_logo_url.png[pdfwidth=35%,align=center] +include::foreword.adoc[] + +[[_whats_new]] == What's new include::whats-new.adoc[] @@ -42,20 +45,24 @@ include::recipes.adoc[] include::request-examples.adoc[] +[[_configuration]] == Configuration include::configuration.adoc[] +[[_json_schemas]] == JSON schemas include::jsonSchema/json-schema.adoc[] +[[_graphql_api]] == GraphQL API include::graphql.adoc[] include::graphql-examples.adoc[] +[[_migrations]] == Migrations include::migrations/migrations.adoc[] @@ -82,6 +89,19 @@ include::clustering.adoc[] == Reference +=== Architecture Overview + +include::architecture.adoc[] + +==== Data Structures +include::data-structures.adoc[] + +==== Event Processing Architecture +include::event-processing.adoc[] + +==== Security Architecture +include::security.adoc[] + include::useful-unomi-urls.adoc[] include::how-profile-tracking-works.adoc[] @@ -90,6 +110,8 @@ include::context-request-flow.adoc[] include::datamodel.adoc[] +include::property-types.adoc[] + include::builtin-event-types.adoc[] include::builtin-condition-types.adoc[] @@ -98,8 +120,13 @@ include::builtin-action-types.adoc[] include::updating-events.adoc[] +include::past-event-conditions.adoc[] + include::web-tracker.adoc[] +include::javascript-tracker-guide.adoc[] + +[[_integration_samples]] == Integration samples include::samples/samples.adoc[] @@ -125,5 +152,3 @@ include::shell-commands.adoc[] include::writing-plugins.adoc[] include::patches.adoc[] - -include::migrate-es7-to-es9.adoc[] diff --git a/manual/src/main/asciidoc/javascript-tracker-guide.adoc b/manual/src/main/asciidoc/javascript-tracker-guide.adoc new file mode 100644 index 000000000..044945bd8 --- /dev/null +++ b/manual/src/main/asciidoc/javascript-tracker-guide.adoc @@ -0,0 +1,1260 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +=== Implementing a JavaScript Tracker + +This guide explains how to implement a basic JavaScript tracker for Apache Unomi from scratch. This will help you understand the underlying concepts and API interactions, whether you're building a custom tracker or integrating Apache Unomi into an existing application. + +[NOTE] +==== +Apache Unomi provides an official web tracker (see <<_unomi_web_tracker_reference,Unomi Web Tracker reference>>) that handles many of these details automatically. This guide is useful if you need a custom implementation or want to understand how the tracker works internally. +==== + +==== Prerequisites + +Before implementing a tracker, you need: + +* An Apache Unomi instance running and accessible +* A tenant created with a public API key (see <<_request_examples,Request examples>> for tenant creation) +* Basic knowledge of JavaScript and HTTP requests +* Understanding of how profile tracking works (see <<_how_profile_tracking_works,How profile tracking works>>) + +==== Basic Tracker Structure + +A basic JavaScript tracker needs to: + +1. **Manage session and profile identifiers** - Generate and store session IDs, handle profile ID cookies +2. **Make context requests** - Call `/cxs/context.json` to retrieve profile/session data +3. **Send events** - Send user interaction events to Apache Unomi +4. **Handle personalization** - Request and apply personalized content +5. **Manage cookies** - Read and write cookies for profile tracking + +Here's a minimal tracker structure: + +[source,javascript] +---- +(function() { + 'use strict'; + + // Configuration + const CONFIG = { + contextServerUrl: 'http://localhost:8181', + apiKey: 'YOUR_PUBLIC_API_KEY', + scope: 'mydigital', + sessionCookieName: 'unomi-session-id', + profileCookieName: 'context-profile-id' + }; + + // Tracker object + const tracker = { + // Session management + getSessionId: function() { /* ... */ }, + setSessionId: function(sessionId) { /* ... */ }, + + // Cookie utilities + getCookie: function(name) { /* ... */ }, + setCookie: function(name, value, days) { /* ... */ }, + + // Context requests + getContext: function(options, callback) { /* ... */ }, + + // Event tracking + trackEvent: function(event, callback) { /* ... */ }, + trackEvents: function(events, callback) { /* ... */ }, + + // Personalization + personalize: function(personalizations, callback) { /* ... */ }, + + // Initialization + init: function() { /* ... */ } + }; + + // Expose tracker globally + window.unomiTracker = tracker; +})(); +---- + +==== Cookie Management + +The tracker needs to read and write cookies to maintain the session ID and handle the profile ID cookie set by Apache Unomi. + +[source,javascript] +---- +// Cookie utilities +getCookie: function(name) { + const value = `; ${document.cookie}`; + const parts = value.split(`; ${name}=`); + if (parts.length === 2) { + return parts.pop().split(';').shift(); + } + return null; +}, + +setCookie: function(name, value, days) { + const expires = new Date(); + expires.setTime(expires.getTime() + (days * 24 * 60 * 60 * 1000)); + document.cookie = `${name}=${value};expires=${expires.toUTCString()};path=/`; +}, + +deleteCookie: function(name) { + document.cookie = `${name}=;expires=Thu, 01 Jan 1970 00:00:00 UTC;path=/;`; +} +---- + +==== Session Management + +The tracker needs to generate and maintain a session ID. If no session ID exists, generate a new one: + +[source,javascript] +---- +// Generate a UUID v4 +// Uses crypto.randomUUID() if available (most secure, modern browsers) +// Falls back to crypto.getRandomValues() (secure, widely supported) +// Falls back to Math.random() only for very old browsers (less secure) +generateUUID: function() { + // Use crypto.randomUUID() if available (Chrome 92+, Firefox 95+, Safari 15.4+, Node.js 19+) + if (typeof crypto !== 'undefined' && crypto.randomUUID) { + return crypto.randomUUID(); + } + + // Use crypto.getRandomValues() if available (widely supported, secure) + if (typeof crypto !== 'undefined' && crypto.getRandomValues) { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = crypto.getRandomValues(new Uint8Array(1))[0] % 16; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } + + // Fallback to Math.random() for very old browsers (not cryptographically secure) + // Note: This is less secure and should only be used as a last resort + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +}, + +// Get or create session ID +getSessionId: function() { + let sessionId = this.getCookie(CONFIG.sessionCookieName); + if (!sessionId) { + sessionId = this.generateUUID(); + this.setSessionId(sessionId); + } + return sessionId; +}, + +setSessionId: function(sessionId) { + this.setCookie(CONFIG.sessionCookieName, sessionId, 1); // 1 day expiration +} +---- + +==== Making Context Requests + +The core of the tracker is making requests to `/cxs/context.json`. This endpoint: + +* Automatically creates or loads profiles and sessions +* Processes events +* Resolves personalization +* Returns profile and session data + +[source,javascript] +---- +// Make a context request +getContext: function(options, callback) { + const sessionId = this.getSessionId(); + const url = `${CONFIG.contextServerUrl}/cxs/context.json?sessionId=${encodeURIComponent(sessionId)}`; + + // Build request payload + const payload = { + source: { + itemId: options.sourceId || window.location.pathname, + itemType: options.sourceType || 'page', + scope: CONFIG.scope + } + }; + + // Add optional parameters + if (options.requiredProfileProperties) { + payload.requiredProfileProperties = options.requiredProfileProperties; + } + if (options.requiredSessionProperties) { + payload.requiredSessionProperties = options.requiredSessionProperties; + } + if (options.requireSegments) { + payload.requireSegments = true; + } + if (options.requireScores) { + payload.requireScores = true; + } + if (options.events) { + payload.events = options.events; + } + if (options.filters) { + payload.filters = options.filters; + } + if (options.personalizations) { + payload.personalizations = options.personalizations; + } + + // Make the request + const xhr = new XMLHttpRequest(); + xhr.open('POST', url, true); + xhr.setRequestHeader('Content-Type', 'application/json'); + xhr.setRequestHeader('X-Unomi-Api-Key', CONFIG.apiKey); + xhr.withCredentials = true; // Important for cookies + + xhr.onreadystatechange = function() { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + try { + const response = JSON.parse(xhr.responseText); + if (callback) { + callback(null, response); + } + } catch (e) { + if (callback) { + callback(new Error('Failed to parse response: ' + e.message), null); + } + } + } else { + if (callback) { + callback(new Error('Request failed with status: ' + xhr.status), null); + } + } + } + }; + + xhr.send(JSON.stringify(payload)); +} +---- + +==== Retrieving Profile and Session Properties + +To retrieve profile and session properties, include them in the context request: + +[source,javascript] +---- +// Get all profile and session properties +tracker.getContext({ + requireSegments: true, + requireScores: true, + requiredProfileProperties: ['*'], // '*' means all properties + requiredSessionProperties: ['*'] +}, function(error, response) { + if (error) { + console.error('Failed to get context:', error); + return; + } + + console.log('Profile ID:', response.profileId); + console.log('Session ID:', response.sessionId); + console.log('Profile Properties:', response.profileProperties); + console.log('Session Properties:', response.sessionProperties); + console.log('Profile Segments:', response.profileSegments); + console.log('Profile Scores:', response.profileScores); +}); +---- + +You can also request specific properties: + +[source,javascript] +---- +// Get specific profile properties +tracker.getContext({ + requiredProfileProperties: ['firstName', 'lastName', 'email', 'pageViewCount'] +}, function(error, response) { + if (error) { + console.error('Failed to get context:', error); + return; + } + + const profile = response.profileProperties || {}; + console.log('User:', profile.firstName, profile.lastName); + console.log('Email:', profile.email); + console.log('Page Views:', profile.pageViewCount); +}); +---- + +==== Sending Events + +Events are sent as part of the context request. You can send a single event or multiple events in one request: + +[source,javascript] +---- +// Build an event object +buildEvent: function(eventType, target, source) { + const event = { + eventType: eventType, + scope: CONFIG.scope + }; + + // Set source if provided, otherwise use default based on event type + if (source) { + event.source = source; + } else if (eventType === 'view') { + // For view events, source should typically be a site + event.source = { + itemType: 'site', + scope: CONFIG.scope, + itemId: window.location.hostname || 'default-site' + }; + } else { + // For other events, source defaults to the current page + event.source = { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname + }; + } + + // Set target if provided + if (target) { + event.target = target; + } + + return event; +}, + +// Track a single event +trackEvent: function(event, callback) { + this.getContext({ + events: [event] + }, callback); +}, + +// Track multiple events (more efficient) +trackEvents: function(events, callback) { + this.getContext({ + events: events + }, callback); +} +---- + +**Example: Tracking a page view** + +[source,javascript] +---- +// Track a page view event +const viewEvent = tracker.buildEvent('view', { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname, + properties: { + pageInfo: { + pageName: document.title, + destinationURL: window.location.href, + referringURL: document.referrer + } + } +}, { + itemType: 'site', + scope: CONFIG.scope, + itemId: window.location.hostname || 'default-site' +}); + +tracker.trackEvent(viewEvent, function(error, response) { + if (error) { + console.error('Failed to track view event:', error); + return; + } + console.log('View event tracked. Processed events:', response.processedEvents); +}); +---- + +**Example: Tracking a click event** + +[source,javascript] +---- +// Track a click on a button +function trackButtonClick(buttonId, buttonText) { + const clickEvent = tracker.buildEvent('click', { + itemType: 'button', + scope: CONFIG.scope, + itemId: buttonId, + properties: { + text: buttonText + } + }); + + tracker.trackEvent(clickEvent, function(error, response) { + if (error) { + console.error('Failed to track click:', error); + return; + } + console.log('Click event tracked'); + }); +} + +// Use it +document.getElementById('myButton').addEventListener('click', function() { + trackButtonClick('myButton', 'Click Me'); +}); +---- + +**Example: Tracking a form submission** + +[source,javascript] +---- +// Track a form submission +function trackFormSubmission(formId, formData) { + const formEvent = { + eventType: 'form', + scope: CONFIG.scope, + source: { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname + }, + target: { + itemType: 'form', + scope: CONFIG.scope, + itemId: formId, + properties: formData + } + }; + + tracker.trackEvent(formEvent, function(error, response) { + if (error) { + console.error('Failed to track form:', error); + return; + } + console.log('Form event tracked'); + }); +} + +// Use it +document.getElementById('myForm').addEventListener('submit', function(e) { + e.preventDefault(); + const formData = { + formName: this.id, + formFields: {} + }; + + // Extract form field values + Array.from(this.elements).forEach(function(element) { + if (element.name) { + formData.formFields[element.name] = element.value; + } + }); + + trackFormSubmission(this.id, formData); +}); +---- + +==== Using Filters + +Filters allow you to evaluate conditions against the profile and session to determine which content should be displayed. Filters return boolean results indicating whether conditions match. + +[source,javascript] +---- +// Request filter evaluation +tracker.getContext({ + filters: [ + { + id: 'premium-user-filter', + filters: [ + { + condition: { + type: 'profileSegmentCondition', + parameterValues: { + segments: ['premium-users'] + } + } + } + ] + }, + { + id: 'new-visitor-filter', + filters: [ + { + condition: { + type: 'profilePropertyCondition', + parameterValues: { + propertyName: 'properties.firstVisit', + comparisonOperator: 'exists' + } + } + } + ] + } + ] +}, function(error, response) { + if (error) { + console.error('Failed to evaluate filters:', error); + return; + } + + const results = response.filteringResults || {}; + if (results['premium-user-filter']) { + console.log('User is a premium user'); + // Show premium content + } + if (results['new-visitor-filter']) { + console.log('User is a new visitor'); + // Show welcome message + } +}); +---- + +==== Using Personalization + +Personalization allows you to request content variants and have Apache Unomi select the most appropriate one based on the profile and session state. + +[source,javascript] +---- +// Request personalization +tracker.getContext({ + personalizations: [ + { + id: 'homepage-hero', + strategy: 'matching-first', + strategyOptions: { + fallback: 'default-hero' + }, + contents: [ + { + id: 'premium-user-hero', + filters: [ + { + condition: { + type: 'profileSegmentCondition', + parameterValues: { + segments: ['premium-users'] + } + } + } + ] + }, + { + id: 'new-visitor-hero', + filters: [ + { + condition: { + type: 'profilePropertyCondition', + parameterValues: { + propertyName: 'properties.firstVisit', + comparisonOperator: 'exists' + } + } + } + ] + }, + { + id: 'default-hero' + } + ] + } + ] +}, function(error, response) { + if (error) { + console.error('Failed to get personalization:', error); + return; + } + + // Get selected content ID(s) + // Note: personalizations is deprecated but still available + // It returns a List, so get the first element for single-selection strategies + const contentIds = response.personalizations['homepage-hero']; + const selectedContentId = contentIds && contentIds.length > 0 ? contentIds[0] : null; + + // Alternatively, use personalizationResults for more detailed information + const personalizationResult = response.personalizationResults['homepage-hero']; + if (personalizationResult && personalizationResult.contentIds && personalizationResult.contentIds.length > 0) { + const selectedId = personalizationResult.contentIds[0]; + console.log('Selected content:', selectedId); + console.log('Strategy used:', personalizationResult.strategy); + applyPersonalizedContent('homepage-hero', selectedId); + } else if (selectedContentId) { + console.log('Selected content:', selectedContentId); + applyPersonalizedContent('homepage-hero', selectedContentId); + } else { + console.log('No content selected, using fallback'); + applyPersonalizedContent('homepage-hero', 'default-hero'); + } +}); + +function applyPersonalizedContent(personalizationId, contentId) { + // Your logic to apply the personalized content + const element = document.getElementById(personalizationId); + if (element) { + // Example: Update content based on selected ID + if (contentId === 'premium-user-hero') { + element.innerHTML = '

Welcome, Premium Member!

'; + } else if (contentId === 'new-visitor-hero') { + element.innerHTML = '

Welcome! Sign up today!

'; + } else { + element.innerHTML = '

Welcome!

'; + } + } +} +---- + +==== Complete Example: Page Load with Personalization + +Here's a complete example that loads context, tracks a page view, and applies personalization on page load: + +[source,javascript] +---- +// Initialize tracker on page load +tracker.init = function() { + const sessionId = this.getSessionId(); + + // Build page view event + const viewEvent = this.buildEvent('view', { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname, + properties: { + pageInfo: { + pageName: document.title, + destinationURL: window.location.href, + referringURL: document.referrer + } + } + }); + + // Request context with event and personalization + this.getContext({ + events: [viewEvent], + requireSegments: true, + requiredProfileProperties: ['firstName', 'lastName', 'pageViewCount'], + personalizations: [ + { + id: 'homepage-hero', + strategy: 'matching-first', + strategyOptions: { + fallback: 'default-hero' + }, + contents: [ + { + id: 'premium-user-hero', + filters: [ + { + condition: { + type: 'profileSegmentCondition', + parameterValues: { + segments: ['premium-users'] + } + } + } + ] + }, + { + id: 'new-visitor-hero', + filters: [ + { + condition: { + type: 'profilePropertyCondition', + parameterValues: { + propertyName: 'properties.firstVisit', + comparisonOperator: 'exists' + } + } + } + ] + }, + { + id: 'default-hero' + } + ] + } + ] + }, function(error, response) { + if (error) { + console.error('Failed to initialize tracker:', error); + return; + } + + // Log profile information + console.log('Profile ID:', response.profileId); + console.log('Session ID:', response.sessionId); + console.log('Profile Segments:', response.profileSegments); + + // Apply personalization + // Get the first content ID from the list (for single-selection strategies like 'matching-first') + // Note: personalizations returns a List, so get the first element + const contentIds = response.personalizations && response.personalizations['homepage-hero']; + const selectedContentId = contentIds && contentIds.length > 0 ? contentIds[0] : 'default-hero'; + applyPersonalizedContent('homepage-hero', selectedContentId); + + // Update UI with profile data + if (response.profileProperties) { + const profile = response.profileProperties; + if (profile.firstName) { + updateWelcomeMessage(profile.firstName, profile.lastName); + } + if (profile.pageViewCount) { + updatePageViewCount(profile.pageViewCount); + } + } + }); +}; + +function applyPersonalizedContent(personalizationId, contentId) { + const element = document.getElementById(personalizationId); + if (!element) return; + + const contentMap = { + 'premium-user-hero': '

Welcome, Premium Member!

Thank you for your loyalty.

', + 'new-visitor-hero': '

Welcome!

Sign up today and get 10% off your first purchase.

', + 'default-hero': '

Welcome!

Discover our amazing products.

' + }; + + element.innerHTML = contentMap[contentId] || contentMap['default-hero']; +} + +function updateWelcomeMessage(firstName, lastName) { + const element = document.getElementById('welcome-message'); + if (element) { + element.textContent = `Welcome back, ${firstName} ${lastName}!`; + } +} + +function updatePageViewCount(count) { + const element = document.getElementById('page-view-count'); + if (element) { + element.textContent = `You've viewed ${count} pages`; + } +} + +// Initialize when DOM is ready +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', function() { + tracker.init(); + }); +} else { + tracker.init(); +} +---- + +==== Advanced Features + +===== Batch Event Tracking + +For better performance, batch multiple events into a single request: + +[source,javascript] +---- +// Track multiple events in one request +const events = [ + tracker.buildEvent('view', { /* ... */ }), + tracker.buildEvent('click', { /* ... */ }), + tracker.buildEvent('form', { /* ... */ }) +]; + +tracker.trackEvents(events, function(error, response) { + if (error) { + console.error('Failed to track events:', error); + return; + } + console.log('Tracked', response.processedEvents, 'events'); +}); +---- + +===== Event Queue + +Implement an event queue to batch events and send them periodically or when a threshold is reached: + +[source,javascript] +---- +// Event queue implementation +const eventQueue = { + queue: [], + maxSize: 10, + flushInterval: 5000, // 5 seconds + + add: function(event) { + this.queue.push(event); + if (this.queue.length >= this.maxSize) { + this.flush(); + } + }, + + flush: function() { + if (this.queue.length === 0) return; + + const events = this.queue.slice(); + this.queue = []; + + tracker.trackEvents(events, function(error, response) { + if (error) { + console.error('Failed to flush events:', error); + // Re-add events to queue on error + eventQueue.queue = events.concat(eventQueue.queue); + } else { + console.log('Flushed', response.processedEvents, 'events'); + } + }); + }, + + start: function() { + setInterval(function() { + eventQueue.flush(); + }, this.flushInterval); + } +}; + +// Start automatic flushing +eventQueue.start(); + +// Add events to queue +eventQueue.add(tracker.buildEvent('click', { /* ... */ })); +---- + +===== Error Handling and Retry Logic + +Implement robust error handling with retry logic: + +[source,javascript] +---- +// Enhanced context request with retry logic +getContextWithRetry: function(options, callback, maxRetries) { + maxRetries = maxRetries || 3; + let retries = 0; + + const attempt = () => { + this.getContext(options, function(error, response) { + if (error && retries < maxRetries) { + retries++; + console.warn('Request failed, retrying...', retries); + setTimeout(attempt, 1000 * retries); // Exponential backoff + } else { + callback(error, response); + } + }); + }; + + attempt(); +} +---- + +===== Profile ID Synchronization + +The profile ID is set by Apache Unomi in a cookie. Make sure to read it from the response: + +[source,javascript] +---- +// Enhanced context request that handles profile ID cookie +getContext: function(options, callback) { + // ... existing code ... + + xhr.onreadystatechange = function() { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + try { + const response = JSON.parse(xhr.responseText); + + // Profile ID cookie is automatically set by Apache Unomi + // but we can verify it matches the response + const cookieProfileId = tracker.getCookie(CONFIG.profileCookieName); + if (response.profileId && cookieProfileId !== response.profileId) { + console.warn('Profile ID mismatch:', { + cookie: cookieProfileId, + response: response.profileId + }); + } + + if (callback) { + callback(null, response); + } + } catch (e) { + if (callback) { + callback(new Error('Failed to parse response: ' + e.message), null); + } + } + } else { + if (callback) { + callback(new Error('Request failed with status: ' + xhr.status), null); + } + } + } + }; + + // ... rest of existing code ... +} +---- + +===== Consent Management + +Handle user consent for tracking: + +[source,javascript] +---- +// Check if user has consented to tracking +hasConsent: function(consentType) { + // Check localStorage or cookie for consent + const consent = localStorage.getItem('unomi-consent-' + consentType); + return consent === 'true'; +}, + +// Request context only if consent is given +getContextWithConsent: function(options, callback) { + if (!this.hasConsent('tracking')) { + console.warn('User has not consented to tracking'); + if (callback) { + callback(new Error('Consent required'), null); + } + return; + } + + this.getContext(options, callback); +} +---- + +===== Cross-Domain Tracking + +For cross-domain tracking, ensure cookies are set with the correct domain: + +[source,javascript] +---- +// Enhanced cookie setting with domain support +setCookie: function(name, value, days, domain) { + const expires = new Date(); + expires.setTime(expires.getTime() + (days * 24 * 60 * 60 * 1000)); + + let cookie = `${name}=${value};expires=${expires.toUTCString()};path=/`; + if (domain) { + cookie += `;domain=${domain}`; + } + + document.cookie = cookie; +} +---- + +===== Performance Optimization + +Optimize performance by: + +* **Debouncing events**: Don't send every single interaction immediately +* **Batching requests**: Combine multiple operations into one request +* **Caching context**: Cache profile data and only refresh when needed + +[source,javascript] +---- +// Debounce function +debounce: function(func, wait) { + let timeout; + return function executedFunction(...args) { + const later = () => { + clearTimeout(timeout); + func(...args); + }; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + }; +}, + +// Debounced event tracking +trackEventDebounced: this.debounce(function(event) { + tracker.trackEvent(event, function(error, response) { + if (error) { + console.error('Failed to track event:', error); + } + }); +}, 1000), // Wait 1 second before sending + +// Context caching +contextCache: { + data: null, + timestamp: null, + ttl: 60000, // 1 minute + + get: function() { + if (this.data && this.timestamp && (Date.now() - this.timestamp) < this.ttl) { + return this.data; + } + return null; + }, + + set: function(data) { + this.data = data; + this.timestamp = Date.now(); + }, + + clear: function() { + this.data = null; + this.timestamp = null; + } +} +---- + +==== Complete Tracker Implementation + +Here's a complete, production-ready tracker implementation combining all the concepts: + +[source,javascript] +---- +(function() { + 'use strict'; + + const CONFIG = { + contextServerUrl: 'http://localhost:8181', + apiKey: 'YOUR_PUBLIC_API_KEY', + scope: 'mydigital', + sessionCookieName: 'unomi-session-id', + profileCookieName: 'context-profile-id', + eventQueueSize: 10, + eventQueueInterval: 5000 + }; + + const tracker = { + // Cookie management + getCookie: function(name) { + const value = `; ${document.cookie}`; + const parts = value.split(`; ${name}=`); + if (parts.length === 2) { + return parts.pop().split(';').shift(); + } + return null; + }, + + setCookie: function(name, value, days, domain) { + const expires = new Date(); + expires.setTime(expires.getTime() + (days * 24 * 60 * 60 * 1000)); + let cookie = `${name}=${value};expires=${expires.toUTCString()};path=/`; + if (domain) { + cookie += `;domain=${domain}`; + } + document.cookie = cookie; + }, + + // UUID generation + // Uses crypto.randomUUID() if available (most secure, modern browsers) + // Falls back to crypto.getRandomValues() (secure, widely supported) + // Falls back to Math.random() only for very old browsers (less secure) + generateUUID: function() { + // Use crypto.randomUUID() if available (Chrome 92+, Firefox 95+, Safari 15.4+, Node.js 19+) + if (typeof crypto !== 'undefined' && crypto.randomUUID) { + return crypto.randomUUID(); + } + + // Use crypto.getRandomValues() if available (widely supported, secure) + if (typeof crypto !== 'undefined' && crypto.getRandomValues) { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = crypto.getRandomValues(new Uint8Array(1))[0] % 16; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } + + // Fallback to Math.random() for very old browsers (not cryptographically secure) + // Note: This is less secure and should only be used as a last resort + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + }, + + // Session management + getSessionId: function() { + let sessionId = this.getCookie(CONFIG.sessionCookieName); + if (!sessionId) { + sessionId = this.generateUUID(); + this.setSessionId(sessionId); + } + return sessionId; + }, + + setSessionId: function(sessionId) { + this.setCookie(CONFIG.sessionCookieName, sessionId, 1); + }, + + // Event building + buildEvent: function(eventType, target, source) { + return { + eventType: eventType, + scope: CONFIG.scope, + source: source || { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname + }, + target: target + }; + }, + + // Context request + getContext: function(options, callback) { + const sessionId = this.getSessionId(); + const url = `${CONFIG.contextServerUrl}/cxs/context.json?sessionId=${encodeURIComponent(sessionId)}`; + + const payload = { + source: { + itemId: options.sourceId || window.location.pathname, + itemType: options.sourceType || 'page', + scope: CONFIG.scope + } + }; + + if (options.requiredProfileProperties) { + payload.requiredProfileProperties = options.requiredProfileProperties; + } + if (options.requiredSessionProperties) { + payload.requiredSessionProperties = options.requiredSessionProperties; + } + if (options.requireSegments) { + payload.requireSegments = true; + } + if (options.requireScores) { + payload.requireScores = true; + } + if (options.events) { + payload.events = options.events; + } + if (options.filters) { + payload.filters = options.filters; + } + if (options.personalizations) { + payload.personalizations = options.personalizations; + } + + const xhr = new XMLHttpRequest(); + xhr.open('POST', url, true); + xhr.setRequestHeader('Content-Type', 'application/json'); + xhr.setRequestHeader('X-Unomi-Api-Key', CONFIG.apiKey); + xhr.withCredentials = true; + + xhr.onreadystatechange = function() { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + try { + const response = JSON.parse(xhr.responseText); + if (callback) { + callback(null, response); + } + } catch (e) { + if (callback) { + callback(new Error('Failed to parse response: ' + e.message), null); + } + } + } else { + if (callback) { + callback(new Error('Request failed with status: ' + xhr.status), null); + } + } + } + }; + + xhr.send(JSON.stringify(payload)); + }, + + // Event tracking + trackEvent: function(event, callback) { + this.getContext({ + events: [event] + }, callback); + }, + + trackEvents: function(events, callback) { + this.getContext({ + events: events + }, callback); + }, + + // Initialization + init: function(config) { + // Merge custom config + if (config) { + Object.assign(CONFIG, config); + } + + // Initialize session + this.getSessionId(); + + // Track page view + const viewEvent = this.buildEvent('view', { + itemType: 'page', + scope: CONFIG.scope, + itemId: window.location.pathname, + properties: { + pageInfo: { + pageName: document.title, + destinationURL: window.location.href, + referringURL: document.referrer + } + } + }); + + this.trackEvent(viewEvent, function(error, response) { + if (error) { + console.error('Failed to track page view:', error); + } else { + console.log('Page view tracked'); + } + }); + } + }; + + // Expose tracker + window.unomiTracker = tracker; + + // Auto-initialize if DOM is ready + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', function() { + tracker.init(); + }); + } else { + tracker.init(); + } +})(); +---- + +==== Best Practices + +1. **Always use HTTPS in production** - Update `contextServerUrl` to use HTTPS +2. **Handle errors gracefully** - Don't break the user experience if tracking fails +3. **Respect user privacy** - Implement consent management +4. **Optimize performance** - Batch events, cache context, debounce rapid interactions +5. **Test thoroughly** - Test with different browsers, network conditions, and privacy settings +6. **Monitor and log** - Log errors and track success rates +7. **Keep session IDs secure** - Use cryptographically secure UUID generation (see UUID generation section below) +8. **Handle cookie restrictions** - Some browsers block third-party cookies + +==== UUID Generation Security + +[IMPORTANT] +==== +**Security Considerations for UUID Generation** + +The tracker uses UUIDs for session IDs, which should be unpredictable to prevent session hijacking. The implementation in this guide uses a secure fallback chain: + +1. **`crypto.randomUUID()`** (Preferred): Available in modern browsers (Chrome 92+, Firefox 95+, Safari 15.4+, Edge 92+). This is the most secure option and generates RFC 4122-compliant UUIDs using cryptographically secure random number generation. + +2. **`crypto.getRandomValues()`** (Fallback): Available in all modern browsers (IE 11+, Chrome 11+, Firefox 21+, Safari 5.1+). Uses the Web Crypto API to generate cryptographically secure random values. This is secure and widely compatible. + +3. **`Math.random()`** (Last Resort): Only used for very old browsers that don't support the Web Crypto API. This is **not cryptographically secure** and can be predictable, making it vulnerable to session hijacking attacks. + +**Recommendations:** +* For production applications, ensure your minimum browser support includes browsers with `crypto.getRandomValues()` support (essentially all browsers from 2013+) +* If you need to support very old browsers (IE 10 and below), consider using a polyfill or warning users about security limitations +* Never use `Math.random()` alone for security-sensitive identifiers like session IDs or authentication tokens +* The profile ID is always generated server-side by Apache Unomi using secure UUID generation, so client-side UUID generation is only needed for session IDs + +**Browser Compatibility:** +* `crypto.randomUUID()`: Chrome 92+, Firefox 95+, Safari 15.4+, Edge 92+ (2021+) +* `crypto.getRandomValues()`: All modern browsers (2013+) +* `Math.random()`: All browsers (but not secure) + +For maximum security and compatibility, the implementation automatically uses the best available method. +==== + +==== Next Steps + +* Learn about <<_how_profile_tracking_works,How profile tracking works>> to understand the full flow +* Review the <<_unomi_web_tracker_reference,official Unomi Web Tracker>> for a production-ready implementation +* Explore <<_request_examples,Request examples>> for more API usage patterns +* Check <<_builtin_event_types,Built-in event types>> for available event types +* Review <<_builtin_condition_types,Built-in condition types>> for filter conditions diff --git a/manual/src/main/asciidoc/jsonSchema/json-schema-api.adoc b/manual/src/main/asciidoc/jsonSchema/json-schema-api.adoc index d489912b1..1b908c206 100644 --- a/manual/src/main/asciidoc/jsonSchema/json-schema-api.adoc +++ b/manual/src/main/asciidoc/jsonSchema/json-schema-api.adoc @@ -16,6 +16,8 @@ The JSON schema endpoints are private, so the user has to be authenticated to manage the JSON schema in Unomi. +IMPORTANT: JSON schema endpoints require tenant authentication using Basic Auth with `tenantId:privateKey`. Only the Tenant API (`/cxs/tenants`) uses system administrator authentication (`karaf:karaf`). + ==== List existing schemas The REST endpoint GET `{{url}}/cxs/jsonSchema` allows to get all ids of available schemas and subschemas. @@ -62,12 +64,14 @@ Example: [source] ---- curl --location --request POST 'http://localhost:8181/cxs/jsonSchema/query' \ --u 'karaf:karaf' +--user 'TENANT_ID:PRIVATE_KEY' \ --header 'Content-Type: text/plain' \ ---header 'Cookie: context-profile-id=0f2fbca8-c242-4e6d-a439-d65fcbf0f0a8' \ --data-raw 'https://unomi.apache.org/schemas/json/event/1-0-0' ---- +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. You can obtain these when creating a tenant via the Tenant API (which requires system administrator authentication). + +[[_create_update_json_schema]] ==== Create / update a JSON schema to validate an event It’s possible to add or update JSON schema by calling the endpoint `POST {{url}}/cxs/jsonSchema` with the JSON schema in the payload of the request. @@ -78,9 +82,8 @@ Example of creation: [source] ---- curl --location --request POST 'http://localhost:8181/cxs/jsonSchema' \ --u 'karaf:karaf' \ +--user 'TENANT_ID:PRIVATE_KEY' \ --header 'Content-Type: application/json' \ ---header 'Cookie: context-profile-id=0f2fbca8-c242-4e6d-a439-d65fcbf0f0a8' \ --data-raw '{ "$id": "https://vendor.test.com/schemas/json/events/dummy/1-0-0", "$schema": "https://json-schema.org/draft/2019-09/schema", @@ -116,12 +119,13 @@ Example: [source] ---- curl --location --request POST 'http://localhost:8181/cxs/jsonSchema/delete' \ --u 'karaf:karaf' \ +--user 'TENANT_ID:PRIVATE_KEY' \ --header 'Content-Type: text/plain' \ ---header 'Cookie: context-profile-id=0f2fbca8-c242-4e6d-a439-d65fcbf0f0a8' \ --data-raw 'https://vendor.test.com/schemas/json/events/dummy/1-0-0' ---- +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. + ==== Error Management When calling an endpoint with invalid data, such as an invalid value for the *sessionId* property in the contextRequest object or eventCollectorRequest object, the server would respond with a 400 error code and the message *Request rejected by the server because: Invalid received data*. diff --git a/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc b/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc index 45ec0d50a..ece986438 100644 --- a/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc +++ b/manual/src/main/asciidoc/jsonSchema/json-schema-develop.adoc @@ -16,7 +16,7 @@ Schemas can be complex to develop, and sometimes, understanding why an event is rejected can be challenging. -This section of the documentation defails mechanisms put in place to facilitate the development when working around JSON Schemas (when creating a new schema, when +This section of the documentation details mechanisms put in place to facilitate the development when working around JSON Schemas (when creating a new schema, when modifying an existing event, ...etc). ==== Logs in debug mode @@ -41,14 +41,14 @@ Doing so will output logs similar to this: ==== validateEvent endpoint -A dedicated Admin endpoint (requires authentication), accessible at: `cxs/jsonSchema/validateEvent`, was created to validate events against JSON Schemas loaded in Apache Unomi. +A dedicated endpoint (requires tenant authentication), accessible at: `cxs/jsonSchema/validateEvent`, was created to validate events against JSON Schemas loaded in Apache Unomi. For example, sending an event not matching a schema: [source] ---- curl --request POST \ --url http://localhost:8181/cxs/jsonSchema/validateEvent \ - --user karaf:karaf \ + --user 'TENANT_ID:PRIVATE_KEY' \ --header 'Content-Type: application/json' \ --data '{ "eventType": "no-event", @@ -81,14 +81,14 @@ towards the incorrect property: ==== validateEvents endpoint -A dedicated Admin endpoint (requires authentication), accessible at: `cxs/jsonSchema/validateEvents`, was created to validate a list of event at once against JSON Schemas loaded in Apache Unomi. +A dedicated endpoint (requires tenant authentication), accessible at: `cxs/jsonSchema/validateEvents`, was created to validate a list of event at once against JSON Schemas loaded in Apache Unomi. For example, sending a list of event not matching a schema: [source] ---- curl --request POST \ --url http://localhost:8181/cxs/jsonSchema/validateEvents \ - --user karaf:karaf \ + --user 'TENANT_ID:PRIVATE_KEY' \ --header 'Content-Type: application/json' \ --data '[{ "eventType": "view", diff --git a/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc b/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc index 8747d18ed..8b58df24e 100644 --- a/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc @@ -35,6 +35,8 @@ Apache Unomi 3.0 supports the following search engine versions: - **Elasticsearch**: Version 9.x (upgraded from 7.17.5) - **OpenSearch**: Version 3.x (new support) +NOTE: OpenSearch support is introduced in Apache Unomi 3.1. If you plan to use OpenSearch, first migrate to 3.0 and then follow the 3.0 → 3.1 migration guide. + ===== Karaf Version Upgrade Apache Unomi 3.0 includes an upgrade of the underlying Karaf framework: @@ -234,7 +236,7 @@ While the mapping system provides backward compatibility, it is recommended to u ===== OpenSearch Security Configuration -When using OpenSearch 3.x with Apache Unomi 3.0, security is enabled by default and requires specific configuration: +When using OpenSearch 3.x with Apache Unomi 3.1, security is enabled by default and requires specific configuration: [source,properties] ---- @@ -259,11 +261,11 @@ When using Docker, you can specify the search engine backend using the distribut - **For OpenSearch**: `UNOMI_DISTRIBUTION=unomi-distribution-opensearch` - **For custom configurations**: `UNOMI_DISTRIBUTION=your-custom-distribution-name` -==== Migrating from Elasticsearch to OpenSearch +==== Elasticsearch to OpenSearch Migration -Apache Unomi 3.0 introduces official support for OpenSearch as an alternative to Elasticsearch. If you want to migrate from Elasticsearch to OpenSearch, you can use the dedicated migration guide. +Apache Unomi 3.1 introduces official support for OpenSearch as an alternative to Elasticsearch. If you want to migrate from Elasticsearch to OpenSearch, you can use the dedicated migration guide located in the 3.0 → 3.1 migration section. -For detailed instructions on migrating your data from Elasticsearch to OpenSearch, please refer to the <>. +For detailed instructions on migrating your data from Elasticsearch to OpenSearch, please refer to the <<_migrating_from_elasticsearch_to_opensearch,Elasticsearch to OpenSearch migration guide>>. ==== Migration Checklist @@ -272,14 +274,14 @@ Before upgrading to Apache Unomi 3.0, complete the following checklist: ===== Pre-Migration Requirements - [ ] **Java 17+**: Ensure Java 17 or later is installed and configured -- [ ] **Search Engine**: Upgrade Elasticsearch to version 9.x OR set up OpenSearch 3.x +- [ ] **Search Engine**: Upgrade Elasticsearch to version 9.x - [ ] **Backup**: Create a complete backup of your current Apache Unomi 2.x installation and data - [ ] **Test Environment**: Test the migration in a non-production environment first ===== Custom Configuration Review - [ ] **QueryBuilder IDs**: Review and update any custom condition definitions that reference old queryBuilder IDs -- [ ] **OpenSearch Security**: If migrating to OpenSearch, configure security settings as required +- [ ] **OpenSearch Security**: If migrating to OpenSearch in 3.1, configure security settings as required - [ ] **Docker Configuration**: Update Docker environment variables if using containerized deployment ===== Post-Migration Verification @@ -291,4 +293,4 @@ Before upgrading to Apache Unomi 3.0, complete the following checklist: ==== Other Breaking Changes -For information about other breaking changes in Apache Unomi 3.0, please refer to the <> section of the documentation. +For information about other breaking changes in Apache Unomi 3.0, please refer to the <<_whats_new,What's new>> section of the documentation. diff --git a/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc b/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc new file mode 100644 index 000000000..146f97510 --- /dev/null +++ b/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc @@ -0,0 +1,319 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +=== Migration Overview + +Apache Unomi 3.1 introduces comprehensive multi-tenancy support, enabling complete data isolation between different tenants. This fundamental architectural change requires a new tenant-based authentication model while keeping all API endpoints unchanged. + +The key innovation in 3.1 is the introduction of **tenant isolation** for all data: +- **Profiles, events, segments, rules, and schemas** are now tenant-specific +- **Complete data separation** between tenants - no cross-tenant data access +- **Tenant-specific API keys** for secure access control +- **Backward compatibility** with system administrator access for management operations + +=== Updating applications consuming Unomi + +==== Authentication Model Changes + +The main change in 3.1 is the introduction of tenant-based authentication. The system must now identify which tenant context to operate in for every request. + +[cols="1,1,1", options="header"] +|=== +|Aspect |Unomi 3.0 |Unomi 3.1 + +|Authentication Method +|System Administrator Authentication (karaf/karaf) +|Tenant-based API Keys + System Administrator Authentication + +|Public API Endpoints +|No authentication required +|Public API Key required (via X-Unomi-Api-Key header) + +|Private API Endpoints +|System Administrator Authentication +|Tenant Authentication (tenantId/privateKey) OR System Administrator Authentication + +|Tenant Administration +|System Administrator Authentication (karaf/karaf) +|System Administrator Authentication (karaf/karaf) +|=== + +==== API Key Types (3.1 Only) + +3.1 introduces two types of API keys per tenant: + +- **Public Key**: Used for public endpoints (event collection via `/context.json`) +- **Private Key**: Used with tenantId for tenant-specific administrative operations + +==== Authentication Requirements by Endpoint Type + +[cols="1,1,1", options="header"] +|=== +|Endpoint Category |3.0 Authentication |3.1 Authentication + +|Event Collection (`/context.json`) +|None +|Public API Key only + +|Administrative Operations +|System Admin (karaf/karaf) +|Tenant Auth (tenantId/privateKey) OR System Admin (karaf/karaf) + +|Tenant Administration (`/cxs/tenants`) +|System Admin (karaf/karaf) +|System Admin (karaf/karaf) +|=== + +==== Authentication Flow (3.1) + +The AuthenticationFilter in 3.1 follows this resolution order: + +1. **Tenant endpoints** (`/cxs/tenants`): Requires system administrator authentication only +2. **Public endpoints** (e.g., `/context.json`): Requires public API key via `X-Unomi-Api-Key` header +3. **Private endpoints**: Tries tenant authentication first, then falls back to system administrator authentication: + - **Tenant Authentication**: Basic Auth with `tenantId:privateKey` + - **System Administrator Authentication**: Basic Auth with `karaf:karaf` (or configured admin credentials) + +==== Code Examples + +===== 3.0 Authentication + +[source,java] +---- +// Global system administrator authentication for all endpoints +RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); + +// Context requests require no authentication +RestAssured.given() + .auth().none() + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); +---- + +===== 3.1 Authentication + +[source,java] +---- +// For public endpoints (event collection) +given() + .header("X-Unomi-Api-Key", publicKey) + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); + +// For private endpoints using tenant authentication +given() + .auth().preemptive().basic(tenantId, privateKey) + .contentType(ContentType.JSON) + .body(payload) + .post("/cxs/profiles"); + +// For private endpoints using system administrator authentication +given() + .auth().preemptive().basic("karaf", "karaf") + .contentType(ContentType.JSON) + .body(payload) + .post("/cxs/profiles"); + +// For tenant administration (system admin only) +given() + .auth().preemptive().basic("karaf", "karaf") + .contentType(ContentType.JSON) + .body(tenantPayload) + .post("/cxs/tenants"); +---- + +==== Implementation Strategy + +===== Client Factory Pattern + +[source,java] +---- +public class UnomiConfiguration { + public UnomiClient createClient(String baseUrl) { + String version = System.getProperty("unomi.version", "3.1"); + + if ("3.1".equals(version)) { + return new UnomiV31Client(baseUrl); + } else { + return new UnomiV30Client(baseUrl); + } + } +} +---- + +===== Version-Specific Authentication + +[source,java] +---- +// 3.0 Client +public void init() { + RestAssured.baseURI = baseUrl; + RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); +} + +// 3.1 Client +public void init() { + RestAssured.baseURI = baseUrl; +} + +public void updateKeys(String publicKey, String privateKey) { + this.publicKey = publicKey; + this.privateKey = privateKey; +} +---- + +==== No API Contract Changes + +All API endpoints remain the same between 3.0 and 3.1. The only differences are in the authentication mechanism and tenant resolution. Request/response payloads are unchanged. + +=== Migrating your existing data + +==== Multi-Tenancy Impact + +When migrating to 3.1, you need to understand that: + +- All data (profiles, events, segments, rules, schemas) becomes tenant-specific +- Each tenant operates in complete isolation with their own data space +- Tenant context must be established for every API operation + +==== Migration Steps + +1. **Understand Multi-Tenancy Impact** + - All data (profiles, events, segments, rules, schemas) becomes tenant-specific + - Each tenant operates in complete isolation with their own data space + - Tenant context must be established for every API operation + +2. **Update Authentication Configuration** + - Remove global system administrator authentication + - Configure tenant-specific public and private API keys + - Implement endpoint-specific authentication logic + +3. **Endpoint-Specific Changes** + - Add `X-Unomi-Api-Key` header with public key for event collection + - Use tenant authentication (tenantId/privateKey) for tenant-specific administrative operations + - Keep system administrator authentication as fallback for administrative operations + - Continue using system administrator authentication for tenant administration + +4. **No API Contract Changes** + - All endpoints remain the same + - Request/response payloads are unchanged + - Only authentication mechanism differs + +==== Benefits of Multi-Tenancy in 3.1 + +- **Data Isolation**: Complete separation ensures tenant data never crosses boundaries +- **Scalability**: Support for multiple customers/organizations in a single Unomi instance +- **Security**: Tenant-specific API keys prevent unauthorized cross-tenant access +- **Compliance**: Easier to meet data privacy regulations with clear tenant boundaries +- **Cost Efficiency**: Shared infrastructure with isolated data reduces operational costs + +=== Migration Checklist + +Before starting the migration, please ensure that: + +- You do have a backup of your data +- You did practice the migration in a staging environment, NEVER migrate a production environment without prior validation +- You verified your applications were operational with Apache Unomi 3.1 (authentication updated, client applications updated, ...) +- You are currently running Apache Unomi 3.0 (or a later 3.0.x version) +- You understand the multi-tenancy impact on your data model +- You have configured tenant-specific API keys for your applications + +=== Migration Process + +The migration from 3.0 to 3.1 is primarily a configuration and authentication update: + +1. **Shutdown your Apache Unomi 3.0 cluster** +2. **Update your client applications** to use the new authentication model +3. **Configure tenant-specific API keys** for your applications +4. **Start your Apache Unomi 3.1 cluster** +5. **Test your applications** with the new authentication model + +=== 3.0 Compatibility Mode + +To facilitate the migration process, Unomi 3.1 includes a **3.0 compatibility mode** that allows 3.0 client applications to work with Unomi 3.1 without immediate code changes. + +==== Enabling 3.0 Compatibility Mode + +To enable 3.0 compatibility mode, set the following system property when starting Unomi 3.1: + +[source,bash] +---- +# Enable 3.0 compatibility mode +-Dunomi.v3_0.compatibility.mode=true +---- + +==== 3.0 Compatibility Mode Behavior + +When 3.0 compatibility mode is enabled: + +- **Public endpoints** (e.g., `/context.json`): No authentication required (same as 3.0) +- **Private endpoints**: JAAS authentication required (same as 3.0) +- **All API endpoints**: Identical behavior to 3.0 +- **Data isolation**: Still enforced through tenant context + +==== Migration Strategy with 3.0 Compatibility Mode + +1. **Phase 1: Enable 3.0 Compatibility Mode** + - Start Unomi 3.1 with `-Dunomi.v3_0.compatibility.mode=true` + - Verify all 3.0 client applications work without changes + - Migrate data to tenant structure + +2. **Phase 2: Gradual Migration** + - Update client applications one by one to use 3.1 authentication + - Test each application with 3.1 authentication + - Keep 3.0 compatibility mode enabled for remaining applications + +3. **Phase 3: Complete Migration** + - Update all client applications to 3.1 authentication + - Disable 3.0 compatibility mode: `-Dunomi.v3_0.compatibility.mode=false` + - Verify all applications work with full 3.1 multi-tenancy + +==== Security Considerations + +- **3.0 compatibility mode** should only be used during migration +- **Production environments** should use full 3.1 authentication for security +- **3.0 compatibility mode** bypasses tenant API key requirements +- **Data isolation** is still enforced through tenant context + +==== Example: Starting Unomi 3.1 with 3.0 Compatibility Mode + +[source,bash] +---- +# Start Unomi 3.1 with 3.0 compatibility mode +./karaf -Dunomi.v3_0.compatibility.mode=true + +# Or set as environment variable +export KARAF_OPTS="-Dunomi.v3_0.compatibility.mode=true" +./karaf +---- + +=== Post Migration + +Once the migration has been completed, you will be able to start Apache Unomi 3.1 with full multi-tenancy support. + +Remember that all data operations now require proper tenant context, either through tenant authentication or system administrator authentication. + +The fundamental difference between Unomi 3.0 and 3.1 is the introduction of **comprehensive multi-tenancy support**: + +- **3.0**: Single-tenant architecture with system administrator authentication for all operations +- **3.1**: Multi-tenant architecture with complete data isolation and tenant-specific authentication +- **API Endpoints**: Identical between versions - no breaking changes to existing integrations +- **Data Model**: All entities (profiles, events, segments, rules, schemas) become tenant-specific in 3.1 +- **Authentication**: New tenant-based authentication model with system administrator authentication as fallback + +The authentication changes in 3.1 are driven by the need to establish tenant context for every operation, ensuring complete data isolation while maintaining backward compatibility for administrative operations. diff --git a/manual/src/main/asciidoc/migrate-es7-to-es9.adoc b/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc similarity index 93% rename from manual/src/main/asciidoc/migrate-es7-to-es9.adoc rename to manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc index d7a9c40ef..4afcdf17e 100644 --- a/manual/src/main/asciidoc/migrate-es7-to-es9.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc @@ -11,7 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -=== Migrate from Elasticsearch 7 to Elasticsearch 9 +[[_migrate_from_elasticsearch_7_to_elasticsearch_9]] +==== Migrate from Elasticsearch 7 to Elasticsearch 9 You can use the *remote reindex* API to upgrade directly from Elasticsearch 7 to Elasticsearch 9. This approach runs both clusters in parallel and uses Elasticsearch's remote reindex feature. @@ -25,7 +26,7 @@ The script migration_es7-es9.sh at the root of the project and handles: * Data reindexing from ES7 to ES9 * Validation and comparison reporting -==== Prerequisites +===== Prerequisites * `bash` shell * `jq` command-line JSON processor @@ -47,7 +48,7 @@ apt-get install jq yum install jq ---- -==== Elasticsearch 9 Remote Reindex Configuration +===== Elasticsearch 9 Remote Reindex Configuration Before running the script, you must configure the remote reindex whitelist on your ES9 cluster. Add this to your `elasticsearch.yml` configuration file: @@ -56,7 +57,7 @@ Before running the script, you must configure the remote reindex whitelist on yo reindex.remote.whitelist: "your-es7-host:9200" ---- -==== Script Configuration +===== Script Configuration The script uses environment variables for configuration. Export variables before running the script: @@ -75,7 +76,7 @@ export INDEX_PREFIX="context-" export BATCH_SIZE="1000" ---- -==== Configuration Variables +===== Configuration Variables [cols="1,3,1", options="header"] |=== @@ -92,7 +93,7 @@ export BATCH_SIZE="1000" | BATCH_SIZE | Reindex batch size | 1000 |=== -== Execution +==== Execution Make the script executable and run it: @@ -102,7 +103,7 @@ chmod +x migration_es7-es9.sh ./migration_es7-es9.sh ---- -==== What the Script Does +===== What the Script Does * Discovers indices matching the configured patterns on ES7 * Collects source statistics (document count, size) for each index @@ -113,7 +114,7 @@ chmod +x migration_es7-es9.sh * Collects destination statistics after migration * Displays a comparison report showing document counts and any mismatches -==== Output +===== Output The script provides detailed logging with timestamps and a final comparison report: diff --git a/manual/src/main/asciidoc/migrations/migrations.adoc b/manual/src/main/asciidoc/migrations/migrations.adoc index 8ab43b63a..6e73631e8 100644 --- a/manual/src/main/asciidoc/migrations/migrations.adoc +++ b/manual/src/main/asciidoc/migrations/migrations.adoc @@ -14,11 +14,25 @@ This section contains information and steps to migrate between major Unomi versions. +=== V2/V3 API Compatibility Guide + +include::v2-v3-compatibility.adoc[] + +=== V2 Compatibility Mode + +include::v2-compatibility-mode.adoc[] + +=== From version 3.0 to 3.1 + +include::migrate-3.0-to-3.1.adoc[] + +include::migrate-elasticsearch-to-opensearch.adoc[] + === From version 2.x to 3.0 include::migrate-2.x-to-3.0.adoc[] -include::migrate-elasticsearch-to-opensearch.adoc[] +include::migrate-es7-to-es9.adoc[] === From version 1.6 to 2.0 diff --git a/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc b/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc new file mode 100644 index 000000000..a1cc67efc --- /dev/null +++ b/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc @@ -0,0 +1,351 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + += Apache Unomi V2 Compatibility Mode + +This document explains how to use the V2 compatibility mode in Apache Unomi V3, which allows V2 client applications to work with Unomi V3 without requiring API keys. + +== Overview + +The V2 compatibility mode is designed to ease the migration from Unomi V2 to V3 by allowing V2 clients to continue working without immediate changes to their authentication logic. This mode provides backward compatibility while still leveraging the multi-tenant architecture of V3. + +=== How It Works + +When V2 compatibility mode is enabled: + +- **Public endpoints** (like `/context.json`) require no authentication (like V2) +- **Protected events** (like `login`, `updateProperties`) require IP + X-Unomi-Peer (like V2) +- **Private endpoints** require system administrator authentication (like V2) +- **A default tenant** is automatically used for all operations +- **No authentication** is required for non-protected events (like V2) + +This allows V2 clients to work with Unomi V3 immediately after migration, giving you time to gradually update client applications to use the new V3 authentication model. + +== Prerequisites + +Before enabling V2 compatibility mode, ensure that: + +1. **Data Migration Completed**: Your V2 data has been migrated to a tenant using the migration scripts +2. **Default Tenant Exists**: A default tenant exists that will be used for all operations +3. **V3 Installation**: Unomi V3 is properly installed and configured + +== Configuration + +=== Enable V2 Compatibility Mode + +1. **Edit the configuration file**: + ```bash + # Edit the configuration file + vi etc/org.apache.unomi.rest.authentication.cfg + ``` + +2. **Enable V2 compatibility mode**: + ```properties + # Enable V2 compatibility mode + v2CompatibilityModeEnabled = true + + # Set the default tenant ID (should match the tenant ID used during migration) + v2CompatibilityDefaultTenantId = your-migration-tenant-id + ``` + +3. **Restart the server** to apply the configuration changes: + ```bash + # Stop the server + ./bin/stop + + # Start the server + ./bin/start + ``` + +=== Configuration Management + +V2 compatibility mode is managed through configuration files only. This approach is safer and prevents accidental changes to authentication settings. + +== Migration Workflow + +=== Step 1: Migrate Data + +First, migrate your V2 data to V3 using the migration scripts: + +```bash +# Run the migration scripts +unomi:migrate-3.1.0-00-tenantDocumentIds +unomi:migrate-3.1.0-10-tenantInitialization +``` + +The `migrate-3.1.0-10-tenantInitialization` script creates a default tenant that will be used for V2 compatibility mode. + +=== Step 2: Enable V2 Compatibility Mode + +Enable V2 compatibility mode by updating the configuration file: + +```bash +# Edit the configuration file +vi etc/org.apache.unomi.rest.authentication.cfg + +# Set v2CompatibilityModeEnabled = true +# Set v2CompatibilityDefaultTenantId = your-tenant-id + +# Restart the server to apply changes +./bin/stop +./bin/start +``` + +=== Step 3: Test V2 Clients + +Your V2 clients should now work without any changes: + +```java +// V2-style authentication still works +RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); + +// Context requests work without API keys +RestAssured.given() + .auth().none() + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); +``` + +=== Step 4: Gradual Migration + +Over time, gradually update your clients to use V3 authentication: + +1. **Update client applications** to use API keys +2. **Test with V3 authentication** while keeping V2 compatibility mode enabled +3. **Disable V2 compatibility mode** once all clients are updated + +== Client Migration Examples + +=== From V2 to V3 (with V2 Compatibility Mode) + +**V2 Client (continues to work)**: +```java +// This continues to work in V2 compatibility mode +RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); + +RestAssured.given() + .auth().none() + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); +``` + +**V3 Client (new implementation)**: +```java +// New V3 client using API keys +given() + .header("X-Unomi-Api-Key", publicKey) + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); +``` + +=== Gradual Migration Strategy + +1. **Phase 1**: Enable V2 compatibility mode, V2 clients continue working +2. **Phase 2**: Develop and test V3 clients alongside V2 clients +3. **Phase 3**: Migrate clients one by one to V3 authentication +4. **Phase 4**: Disable V2 compatibility mode once all clients are migrated + +== Security Considerations + +=== V2 Compatibility Mode Security + +When V2 compatibility mode is enabled: + +- **Public endpoints** are accessible without authentication (same as V2) +- **Protected events** require IP + X-Unomi-Peer authentication (same as V2) +- **Private endpoints** require system administrator authentication (same as V2) +- **All operations** use the default tenant context +- **Non-protected events** require no authentication (same as V2) + +=== Protected Events in V2 Compatibility Mode + +In V2 compatibility mode, protected event types are configured dynamically using the V2 third-party configuration file. By default, the following event types are protected: + +- `login` - User authentication events +- `updateProperties` - Profile property updates + +Additional event types can be configured as protected by editing the V2 third-party configuration file. + +For protected events, clients must: +1. Send the request from an authorized IP address (configured in the V2 third-party configuration) +2. Include the `X-Unomi-Peer` header with the third-party ID (e.g., "provider1") + +All other event types are considered non-protected and require no authentication. + +=== V2 Third-Party Configuration + +The protected events and third-party providers are configured in the original V2 configuration file `etc/org.apache.unomi.thirdparty.cfg`. The system dynamically detects any number of providers using the pattern `thirdparty.{providerName}.{property}`: + +```properties +# Provider 1 Configuration (default provider) +thirdparty.provider1.key=${org.apache.unomi.thirdparty.provider1.key:-670c26d1cc413346c3b2fd9ce65dab41} +thirdparty.provider1.ipAddresses=${org.apache.unomi.thirdparty.provider1.ipAddresses:-127.0.0.1,::1} +thirdparty.provider1.allowedEvents=${org.apache.unomi.thirdparty.provider1.allowedEvents:-login,updateProperties} + +# Additional providers can be added dynamically +thirdparty.myapp.key=${org.apache.unomi.thirdparty.myapp.key:-my-secret-key} +thirdparty.myapp.ipAddresses=${org.apache.unomi.thirdparty.myapp.ipAddresses:-192.168.1.0/24} +thirdparty.myapp.allowedEvents=${org.apache.unomi.thirdparty.myapp.allowedEvents:-login,updateProperties,sessionCreated} +``` + +This uses the exact same configuration format as V2, ensuring complete compatibility with existing V2 setups. The system automatically detects and configures any provider that has a valid key. + +=== Configuration Management + +The V2 third-party configuration supports dynamic updates: + +1. **Edit the configuration file**: + ```bash + # Edit the V2 third-party configuration + vi etc/org.apache.unomi.thirdparty.cfg + ``` + +2. **Update protected events**: + ```properties + # Add more protected event types + thirdparty.provider1.allowedEvents=${org.apache.unomi.thirdparty.provider1.allowedEvents:-login,updateProperties,sessionCreated,profileUpdated} + ``` + +3. **Add additional providers**: + ```properties + # Configure additional providers (any name is supported) + thirdparty.myapp.key=${org.apache.unomi.thirdparty.myapp.key:-your-secret-key-here} + thirdparty.myapp.ipAddresses=${org.apache.unomi.thirdparty.myapp.ipAddresses:-192.168.1.0/24,10.0.0.1} + thirdparty.myapp.allowedEvents=${org.apache.unomi.thirdparty.myapp.allowedEvents:-login,updateProperties} + + thirdparty.analytics.key=${org.apache.unomi.thirdparty.analytics.key:-analytics-secret} + thirdparty.analytics.ipAddresses=${org.apache.unomi.thirdparty.analytics.ipAddresses:-10.0.0.0/8} + thirdparty.analytics.allowedEvents=${org.apache.unomi.thirdparty.analytics.allowedEvents:-login,updateProperties,sessionCreated} + ``` + +4. **Restart the server** to apply changes: + ```bash + ./bin/stop + ./bin/start + ``` + +=== Recommendations + +1. **Use V2 compatibility mode temporarily** during migration +2. **Plan for gradual migration** to V3 authentication +3. **Monitor access patterns** during the transition +4. **Disable V2 compatibility mode** once migration is complete + +== Troubleshooting + +=== Common Issues + +**V2 clients still not working**: +- Check configuration file: `etc/org.apache.unomi.rest.authentication.cfg` +- Verify `v2CompatibilityModeEnabled = true` +- Ensure `v2CompatibilityDefaultTenantId` matches the tenant ID used during migration +- Ensure the tenant exists and is accessible + +**Authentication errors**: +- Verify system administrator credentials (karaf/karaf) +- Check that the server is running properly +- Review logs for authentication errors + +**Tenant context issues**: +- Ensure the default tenant ID matches your migrated tenant +- Verify tenant exists in the tenant index +- Check tenant configuration in the migration scripts + +=== Debugging + +Enable debug logging for authentication: + +```bash +# Enable debug logging +log:set DEBUG org.apache.unomi.rest.authentication +``` + +Check authentication filter logs: + +```bash +# View recent logs +log:display | grep AuthenticationFilter +``` + +== Disabling V2 Compatibility Mode + +Once all clients are migrated to V3 authentication: + +1. **Update configuration**: + ```properties + v2CompatibilityModeEnabled = false + ``` + +2. **Restart the server**: + ```bash + ./bin/stop + ./bin/start + ``` + +3. **Verify all clients work** with V3 authentication + +4. **Monitor for any issues** and address them before final deployment + +== Testing V2 Compatibility Mode + +The existing test framework supports testing V2 compatibility mode using system properties. + +=== Running Tests in V2 Compatibility Mode + +To run tests with V2 compatibility mode enabled: + +```bash +# Enable V2 compatibility mode for tests +mvn test -Dunomi.v2.compatibility.mode=true + +# Or set the property in your test environment +export UNOMI_V2_COMPATIBILITY_MODE=true +mvn test +``` + +=== Test Framework Integration + +The test framework automatically detects V2 compatibility mode and uses the appropriate client: + +- **V2 Compatibility Mode Enabled**: Uses `UnomiV2Client` for all tests +- **V2 Compatibility Mode Disabled**: Uses normal V2/V3 detection logic + +This allows you to test both V2 compatibility mode and normal V3 mode using the same test suite. + +=== Example Test Execution + +```bash +# Test with V2 compatibility mode (server should be configured for V2 compatibility) +mvn test -Dunomi.v2.compatibility.mode=true -Dunomi.url=http://localhost:8181 + +# Test with normal V3 mode +mvn test -Dunomi.url=http://localhost:8181 +``` + +== Conclusion + +The V2 compatibility mode provides a smooth migration path from Unomi V2 to V3, allowing you to: + +- **Maintain existing V2 clients** during migration +- **Gradually migrate** to V3 authentication +- **Leverage V3 features** while maintaining backward compatibility +- **Minimize downtime** during the migration process +- **Test both modes** using the existing test framework + +Use this mode as a temporary solution during your migration journey, and plan to disable it once all clients are updated to use V3 authentication. diff --git a/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc b/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc new file mode 100644 index 000000000..1c7a72988 --- /dev/null +++ b/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc @@ -0,0 +1,230 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +== Apache Unomi V2/V3 API Differences Guide + +This document explains the key differences between Apache Unomi 2.x and 3.x versions from an API perspective. + +== Overview + +Apache Unomi 3.x introduces comprehensive multi-tenancy support, enabling complete data isolation between different tenants. This fundamental architectural change requires a new tenant-based authentication model while keeping all API endpoints unchanged. + +=== Multi-Tenancy in V3 + +The key innovation in V3 is the introduction of **tenant isolation** for all data: + +- **Profiles, events, segments, rules, and schemas** are now tenant-specific +- **Complete data separation** between tenants - no cross-tenant data access +- **Tenant-specific API keys** for secure access control +- **Backward compatibility** with system administrator access for management operations + +This multi-tenancy support necessitates the authentication changes described below, as the system must now identify which tenant context to operate in for every request. + +== Key Differences Between V2 and V3 + +=== Authentication Model + +[cols="1,1,1", options="header"] +|=== +|Aspect |Unomi V2 |Unomi V3 + +|Authentication Method +|System Administrator Authentication (karaf/karaf) +|Tenant-based API Keys + System Administrator Authentication + +|Public API Endpoints +|No authentication required +|Public API Key required (via X-Unomi-Api-Key header) + +|Private API Endpoints +|System Administrator Authentication +|Tenant Authentication (tenantId/privateKey) OR System Administrator Authentication + +|Tenant Administration +|System Administrator Authentication (karaf/karaf) +|System Administrator Authentication (karaf/karaf) +|=== + +=== API Key Types (V3 Only) + +V3 introduces two types of API keys per tenant: + +- **Public Key**: Used for public endpoints (event collection via `/context.json`) +- **Private Key**: Used with tenantId for tenant-specific administrative operations + +=== Authentication Requirements by Endpoint Type + +[cols="1,1,1", options="header"] +|=== +|Endpoint Category |V2 Authentication |V3 Authentication + +|Event Collection (`/context.json`) +|None +|Public API Key only + +|Administrative Operations +|System Admin (karaf/karaf) +|Tenant Auth (tenantId/privateKey) OR System Admin (karaf/karaf) + +|Tenant Administration (`/cxs/tenants`) +|System Admin (karaf/karaf) +|System Admin (karaf/karaf) +|=== + +== Authentication Flow (V3) + +The AuthenticationFilter in V3 follows this resolution order: + +1. **Tenant endpoints** (`/cxs/tenants`): Requires system administrator authentication only +2. **Public endpoints** (e.g., `/context.json`): Requires public API key via `X-Unomi-Api-Key` header +3. **Private endpoints**: Tries tenant authentication first, then falls back to system administrator authentication: + - **Tenant Authentication**: Basic Auth with `tenantId:privateKey` + - **System Administrator Authentication**: Basic Auth with `karaf:karaf` (or configured admin credentials) + +== Code Examples + +=== V2 Authentication + +[source,java] +---- +// Global system administrator authentication for all endpoints +RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); + +// Context requests require no authentication +RestAssured.given() + .auth().none() + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); +---- + +=== V3 Authentication + +[source,java] +---- +// For public endpoints (event collection) +given() + .header("X-Unomi-Api-Key", publicKey) + .contentType(ContentType.JSON) + .body(contextJson) + .post("/context.json"); + +// For private endpoints using tenant authentication +given() + .auth().preemptive().basic(tenantId, privateKey) + .contentType(ContentType.JSON) + .body(payload) + .post("/cxs/profiles"); + +// For private endpoints using system administrator authentication +given() + .auth().preemptive().basic("karaf", "karaf") + .contentType(ContentType.JSON) + .body(payload) + .post("/cxs/profiles"); + +// For tenant administration (system admin only) +given() + .auth().preemptive().basic("karaf", "karaf") + .contentType(ContentType.JSON) + .body(tenantPayload) + .post("/cxs/tenants"); +---- + +== Implementation Strategy + +=== Client Factory Pattern + +[source,java] +---- +public class UnomiConfiguration { + public UnomiClient createClient(String baseUrl) { + String version = System.getProperty("unomi.version", "3"); + + if ("3".equals(version)) { + return new UnomiV3Client(baseUrl); + } else { + return new UnomiV2Client(baseUrl); + } + } +} +---- + +=== Version-Specific Authentication + +[source,java] +---- +// V2 Client +public void init() { + RestAssured.baseURI = baseUrl; + RestAssured.authentication = RestAssured.preemptive() + .basic("karaf", "karaf"); +} + +// V3 Client +public void init() { + RestAssured.baseURI = baseUrl; +} + +public void updateKeys(String publicKey, String privateKey) { + this.publicKey = publicKey; + this.privateKey = privateKey; +} +---- + +== Migration Guidelines + +=== From V2 to V3 + +1. **Understand Multi-Tenancy Impact** + - All data (profiles, events, segments, rules, schemas) becomes tenant-specific + - Each tenant operates in complete isolation with their own data space + - Tenant context must be established for every API operation + +2. **Update Authentication Configuration** + - Remove global system administrator authentication + - Configure tenant-specific public and private API keys + - Implement endpoint-specific authentication logic + +3. **Endpoint-Specific Changes** + - Add `X-Unomi-Api-Key` header with public key for event collection + - Use tenant authentication (tenantId/privateKey) for tenant-specific administrative operations + - Keep system administrator authentication as fallback for administrative operations + - Continue using system administrator authentication for tenant administration + +4. **No API Contract Changes** + - All endpoints remain the same + - Request/response payloads are unchanged + - Only authentication mechanism differs + +=== Benefits of Multi-Tenancy in V3 + +- **Data Isolation**: Complete separation ensures tenant data never crosses boundaries +- **Scalability**: Support for multiple customers/organizations in a single Unomi instance +- **Security**: Tenant-specific API keys prevent unauthorized cross-tenant access +- **Compliance**: Easier to meet data privacy regulations with clear tenant boundaries +- **Cost Efficiency**: Shared infrastructure with isolated data reduces operational costs + +== Conclusion + +The fundamental difference between Unomi V2 and V3 is the introduction of **comprehensive multi-tenancy support**: + +- **V2**: Single-tenant architecture with system administrator authentication for all operations +- **V3**: Multi-tenant architecture with complete data isolation and tenant-specific authentication +- **API Endpoints**: Identical between versions - no breaking changes to existing integrations +- **Data Model**: All entities (profiles, events, segments, rules, schemas) become tenant-specific in V3 +- **Authentication**: New tenant-based authentication model with system administrator authentication as fallback + +The authentication changes in V3 are driven by the need to establish tenant context for every operation, ensuring complete data isolation while maintaining backward compatibility for administrative operations. \ No newline at end of file diff --git a/manual/src/main/asciidoc/multitenancy.adoc b/manual/src/main/asciidoc/multitenancy.adoc new file mode 100644 index 000000000..2d98b1157 --- /dev/null +++ b/manual/src/main/asciidoc/multitenancy.adoc @@ -0,0 +1,848 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + += Apache Unomi Multi-tenancy +:toc: macro +:toclevels: 4 +:toc-title: Table of contents + +toc::[] + +== Overview + +Apache Unomi provides robust multi-tenancy support, allowing multiple organizations to use the same Unomi instance while maintaining complete data isolation. Each tenant gets their own dedicated space with separate data storage, configuration, and API keys. + +== Key Features + +* Complete data isolation between tenants +* Dual API key system (public and private keys) +* Tenant-specific configuration +* Resource quotas and limits +* Migration tools for existing data +* Support for both REST and GraphQL APIs + +== Authentication Methods + +=== Public API Key +Used for public endpoints (e.g., context requests) that are typically accessed from client-side applications. + +[source,http] +---- +X-Unomi-Api-Key: +---- + +=== Private API Key +Used for administrative operations and sensitive endpoints. Requires Basic Authentication using tenant ID and private key. + +[source,bash] +---- +--user "TENANT_ID:PRIVATE_KEY" +---- + +NOTE: curl automatically handles Base64 encoding when using the `--user` option, so you don't need to manually encode the credentials. + +== Getting Started + +=== Creating Your First Tenant + +To create a new tenant, use the Tenant API endpoint: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/tenants \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "requestedId": "my-tenant", + "properties": { + "name": "My Organization", + "description": "My organization description" + } + }' +---- + +The response includes the tenant with automatically generated API keys: + +[source,json] +---- +{ + "itemId": "my-tenant", + "name": "My Organization", + "description": "My organization description", + "apiKeys": [ + { + "type": "PUBLIC", + "key": "abc123...", + "created": "2024-01-01T00:00:00Z" + }, + { + "type": "PRIVATE", + "key": "xyz789...", + "created": "2024-01-01T00:00:00Z" + } + ] +} +---- + +NOTE: Both public and private API keys are automatically generated when creating a tenant. Extract the `key` value from the appropriate API key object in the `apiKeys` array. + +=== Making Your First API Call + +==== Public Endpoint Example (Context Request) + +[source,bash] +---- +curl http://localhost:8181/cxs/context.json \ + -H "X-Unomi-Api-Key: " \ + -H "Content-Type: application/json" \ + -d '{ + "sessionId": "session-123", + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "example" + } + }' +---- + +==== Private Endpoint Example (Profile Management) + +[source,bash] +---- +curl -X GET http://localhost:8181/cxs/profiles \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" +---- + +== Configuration + +=== Basic Setup + +Configure default tenant settings in `etc/org.apache.unomi.tenant.cfg`: + +[source,properties] +---- +# Default tenant ID for backward compatibility +tenant.default.id=default + +# API key validity period +tenant.apikey.validity.period=30 +tenant.apikey.validity.unit=DAYS + +# Maximum number of API calls per tenant per day +tenant.apikey.maxCalls=100000 + +# Enable/disable tenant isolation +tenant.isolation.enabled=true +---- + +=== Security Provider Configuration + +For Elasticsearch: +[source,properties] +---- +tenant.security.provider=elasticsearch +---- + +For OpenSearch: +[source,properties] +---- +tenant.security.provider=opensearch +---- + +== Security Model + +=== Roles and Permissions + +Apache Unomi implements a hierarchical role-based access control (RBAC) system. The main roles are: + +* `ROLE_UNOMI_SYSTEM`: Highest privilege level, used for system operations +* `ROLE_UNOMI_ADMIN`: Administrative access across the platform +* `ROLE_UNOMI_TENANT_USER`: Basic tenant access for public operations +* `ROLE_UNOMI_TENANT_ADMIN`: Extended tenant access for private operations +* `ROLE_UNOMI_TENANT_PUBLIC_PREFIX_*`: Tenant-specific public roles +* `ROLE_UNOMI_TENANT_PRIVATE_PREFIX_*`: Tenant-specific private roles + +Configure system roles in `etc/org.apache.unomi.security.cfg`: + +[source,properties] +---- +# Define system roles +systemRoles=ROLE_UNOMI_SYSTEM,ROLE_UNOMI_ADMIN + +# Enable encryption for sensitive data +enableEncryption=false + +# Operation role mappings +operation.roles.QUERY=ROLE_UNOMI_TENANT_USER,ROLE_UNOMI_TENANT_ADMIN +operation.roles.PROFILE_UPDATE=ROLE_UNOMI_TENANT_ADMIN +operation.roles.SYSTEM_MAINTENANCE=ROLE_UNOMI_SYSTEM +operation.roles.TENANT_MANAGEMENT=ROLE_UNOMI_ADMIN +operation.roles.DECRYPT_PROFILE_DATA=ROLE_UNOMI_TENANT_ADMIN +operation.roles.SEGMENT_UPDATE=ROLE_UNOMI_TENANT_ADMIN +operation.roles.RULE_UPDATE=ROLE_UNOMI_TENANT_ADMIN +---- + +The configuration uses the format `operation.roles.OPERATION_NAME=ROLE1,ROLE2,...` where: +- `OPERATION_NAME` is the uppercase operation identifier +- Multiple roles are comma-separated +- Changes take effect immediately without restart + +=== Operation Configuration + +Operations in Unomi can be customized to require specific roles. This is configured through OSGi configuration files. + +==== Configuration File + +Create or modify the file `etc/org.apache.unomi.security.cfg`: + +[source,properties] +---- +# Define system roles +systemRoles=ROLE_UNOMI_SYSTEM,ROLE_UNOMI_ADMIN + +# Enable encryption for sensitive data +enableEncryption=false + +# Operation role mappings +operation.roles.QUERY=ROLE_UNOMI_TENANT_USER,ROLE_UNOMI_TENANT_ADMIN +operation.roles.PROFILE_UPDATE=ROLE_UNOMI_TENANT_ADMIN +operation.roles.SYSTEM_MAINTENANCE=ROLE_UNOMI_SYSTEM +operation.roles.TENANT_MANAGEMENT=ROLE_UNOMI_ADMIN +operation.roles.DECRYPT_PROFILE_DATA=ROLE_UNOMI_TENANT_ADMIN +operation.roles.SEGMENT_UPDATE=ROLE_UNOMI_TENANT_ADMIN +operation.roles.RULE_UPDATE=ROLE_UNOMI_TENANT_ADMIN +---- + +The configuration uses the format `operation.roles.OPERATION_NAME=ROLE1,ROLE2,...` where: +- `OPERATION_NAME` is the uppercase operation identifier +- Multiple roles are comma-separated +- Changes take effect immediately without restart + +==== Common Operations + +Here are some common operations and their typical role requirements: + +[options="header"] +|=== +|Operation |Description |Default Required Roles +|QUERY |Basic data querying |ROLE_UNOMI_TENANT_USER, ROLE_UNOMI_TENANT_ADMIN +|PROFILE_UPDATE |Update profile data |ROLE_UNOMI_TENANT_ADMIN +|SYSTEM_MAINTENANCE |System-level operations |ROLE_UNOMI_SYSTEM +|TENANT_MANAGEMENT |Tenant administration |ROLE_UNOMI_ADMIN +|DECRYPT_PROFILE_DATA |Access to encrypted profile data |ROLE_UNOMI_TENANT_ADMIN +|SEGMENT_UPDATE |Update user segments |ROLE_UNOMI_TENANT_ADMIN +|RULE_UPDATE |Update business rules |ROLE_UNOMI_TENANT_ADMIN +|=== + +==== Custom Operations + +To define custom operations: + +1. Define the operation name (use uppercase by convention) +2. Add the operation-role mapping to the configuration file +3. Use `securityService.validateTenantOperation()` to enforce the permission + +Example: + +1. Add to `etc/org.apache.unomi.security.cfg`: +[source,properties] +---- +operation.roles.CUSTOM_OPERATION=ROLE_UNOMI_TENANT_ADMIN +---- + +2. Use in your code: +[source,java] +---- +public void performCustomOperation() { + securityService.validateTenantOperation("CUSTOM_OPERATION"); + // Operation implementation +} +---- + +=== Subjects and Authentication + +A Subject represents an authenticated entity in the system. There are three types of subjects: + +1. System Subject: +* Used for system-level operations +* Has full access across all tenants +* Created with `ROLE_UNOMI_SYSTEM` + +2. Admin Subject: +* Used for administrative operations +* Has tenant management capabilities +* Created with `ROLE_UNOMI_ADMIN` + +3. Tenant Subject: +* Represents a tenant-specific user +* Has access only to their tenant's resources +* Created with tenant-specific roles + +Example of subject creation: + +[source,java] +---- +Subject tenantSubject = new Subject(); +tenantSubject.getPrincipals().add(new UserPrincipal("tenant-id")); +tenantSubject.getPrincipals().add(new RolePrincipal("ROLE_UNOMI_TENANT_ADMIN")); +---- + +=== Tenant-Role Relationship + +Each tenant has associated public and private roles: + +1. User Role (`ROLE_UNOMI_TENANT_USER`): +* Basic read-only access to tenant data +* Can perform queries and view profiles + +2. Admin Role (`ROLE_UNOMI_TENANT_ADMIN`): +* Full access to tenant data +* Can perform all tenant operations + +=== Operation Validation + +The security service validates operations based on: + +1. Subject's roles +2. Operation type +3. Tenant context + +Example of operation validation: + +[source,java] +---- +// Validate a tenant operation +securityService.validateTenantOperation("SYSTEM_MAINTENANCE"); + +// Execute with elevated privileges +securityService.executeAsSystemSubject(() -> { + // Perform system operation +}); +---- + +=== Best Practices + +1. Role Assignment: +* Assign minimum required roles +* Use tenant-specific roles when possible +* Avoid using system roles for regular operations + +2. Subject Management: +* Clear subjects after operations +* Use temporary privileged subjects sparingly +* Always validate tenant context + +3. Security Configuration: +* Regularly rotate API keys +* Enable encryption for sensitive data +* Monitor failed authentication attempts + +4. Operation Execution: +* Use `executeAsSystemSubject` for system operations +* Validate operations before execution +* Maintain proper audit trails + +== Tenant Management + +=== Listing Tenants + +[source,bash] +---- +curl -X GET http://localhost:8181/cxs/tenants \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +=== Updating a Tenant + +[source,bash] +---- +curl -X PUT http://localhost:8181/cxs/tenants/my-tenant \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "displayName": "Updated Organization Name", + "description": "Updated description" + }' +---- + +=== Regenerating API Keys + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/tenants/my-tenant/apikeys \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "type": "PUBLIC", + "validityDays": 30 + }' +---- + +NOTE: You can specify `type` as `PUBLIC` or `PRIVATE`, and optionally set `validityDays` for key expiration. If omitted, both keys will be regenerated. + +=== Deleting a Tenant + +[source,bash] +---- +curl -X DELETE http://localhost:8181/cxs/tenants/my-tenant \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +== GraphQL Support + +GraphQL endpoints support both public and private authentication methods: + +[source,bash] +---- +curl -X POST http://localhost:8181/graphql \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ profiles { edges { node { id } } } }" + }' +---- + +=== Tenant-Specific GraphQL Schemas + +The GraphQL API provides tenant-specific schemas, meaning each tenant can have a unique GraphQL schema based on their property types and configurations. This ensures that tenants only see the data and fields relevant to their specific implementation. + +When a tenant accesses the GraphQL API: + +1. The system automatically detects the tenant from the authentication context +2. It retrieves (or creates) a GraphQL schema specific to that tenant +3. The schema only includes property types defined for that tenant +4. Changes to a tenant's property types are automatically reflected in their schema + +For complete details on the GraphQL multi-tenancy implementation, refer to the <<_graphql_api,GraphQL API>> section of the documentation. + +== Monitoring and Management + +=== Monitoring API Usage + +Track tenant API usage: + +[source,bash] +---- +curl -X GET http://localhost:8181/cxs/tenants/my-tenant/apiCalls \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +=== Data Migration + +Migrate data between tenants: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/tenants/source-tenant/migrate/target-tenant \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +== Best Practices + +=== API Key Management +* Rotate keys regularly using the key regeneration endpoint +* Use public keys only for public endpoints +* Never expose private keys in client-side code +* Monitor API key usage and implement rate limiting + +=== Resource Management +* Set appropriate quotas for each tenant +* Monitor resource usage through the monitoring endpoints +* Configure alerts for quota limits +* Regularly review and adjust limits based on usage patterns + +=== Security +* Always use HTTPS in production +* Implement proper key rotation policies +* Conduct regular security audits +* Monitor for suspicious activity patterns +* Keep tenant configurations up to date + +== Troubleshooting + +=== Common Issues + +==== 401 Unauthorized +* Verify API key is correct +* Check if using public key for private endpoint +* Ensure tenant ID matches the API key + +==== 400 Bad Request +* Check if API key header is present +* Verify request format is correct + +==== 404 Not Found +* Verify tenant ID exists +* Check if endpoint path is correct + +=== Logging + +Enable debug logging for tenant-related operations: + +[source,properties] +---- +log4j.logger.org.apache.unomi.tenant=DEBUG +---- + +== Migration Guide + +=== Migrating Existing Data + +To migrate existing data to use multi-tenancy: + +[source,bash] +---- +# Step 1: Create new tenant +curl -X POST http://localhost:8181/cxs/tenants \ + --user karaf:karaf \ + -H "Content-Type: application/json" \ + -d '{ + "requestedId": "new-tenant", + "properties": { + "name": "New Tenant", + "description": "Migrated tenant" + } + }' + +# Step 2: Migrate data +curl -X POST http://localhost:8181/cxs/tenants/migration/default/new-tenant \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +=== Verification + +After migration, verify data integrity: + +[source,bash] +---- +# Check profile count +curl -X GET http://localhost:8181/cxs/tenants/new-tenant/profiles/count \ + --user karaf:karaf \ + -H "Content-Type: application/json" +---- + +== Working with Events and Rules + +=== Creating Custom Event Types + +First, create a JSON schema for your custom event type and deploy it using the JSON schema endpoint: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/jsonSchema \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ +--data-raw '{ + "$id": "https://unomi.apache.org/schemas/json/events/purchaseCompleted/1-0-0", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "self": { + "vendor": "org.apache.unomi", + "name": "purchaseCompleted", + "format": "jsonschema", + "target": "events", + "version": "1-0-0" + }, + "title": "Purchase Completed Event", + "type": "object", + "allOf": [{ "$ref": "https://unomi.apache.org/schemas/json/event/1-0-0" }], + "properties": { + "properties": { + "type": "object", + "properties": { + "orderId": { + "type": "string", + "description": "The unique order identifier" + }, + "amount": { + "type": "number", + "description": "The total purchase amount" + }, + "currency": { + "type": "string", + "description": "The currency code (e.g., USD)" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "productId": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "price": { + "type": "number" + } + }, + "required": ["productId", "quantity", "price"] + } + } + }, + "required": ["orderId", "amount", "currency"] + } + }, + "unevaluatedProperties": false +}' +---- + +You can verify your schema has been deployed by listing all available schemas: + +[source,bash] +---- +curl -X GET http://localhost:8181/cxs/jsonSchema \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" +---- + +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. Only the Tenant API (`/cxs/tenants`) uses system administrator authentication (`karaf:karaf`). + +You can also validate events against your schema using the validation endpoint: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/jsonSchema/validateEvent \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + --data '{ + "eventType": "purchaseCompleted", + "scope": "myapp", + "properties": { + "orderId": "order-123", + "amount": 99.99, + "currency": "USD", + "items": [ + { + "productId": "product-001", + "quantity": 2, + "price": 49.99 + } + ] + } + }' +---- + +=== Sending Custom Events + +Once the event type is defined, you can send events: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json \ + -H "X-Unomi-Api-Key: " \ + -H "Content-Type: application/json" \ + -d '{ + "sessionId": "session-123", + "profileId": "profile-456", + "source": { + "itemId": "checkout-page", + "itemType": "page", + "scope": "myapp" + }, + "events": [{ + "eventType": "purchaseCompleted", + "scope": "myapp", + "properties": { + "orderId": "order-789", + "amount": 99.99, + "currency": "USD", + "items": [ + { + "productId": "product-001", + "quantity": 2, + "price": 49.99 + } + ] + } + }] + }' +---- + +=== Creating Rules for Event Processing + +Create a rule to update profile properties based on purchase events: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/rules \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "metadata": { + "id": "updateTotalPurchases", + "name": "Update total purchases", + "description": "Updates profile properties when a purchase is completed", + "scope": "myapp" + }, + "condition": { + "type": "eventTypeCondition", + "parameterValues": { + "eventTypeId": "purchaseCompleted" + } + }, + "actions": [ + { + "type": "setPropertyAction", + "parameterValues": { + "setPropertyName": "properties.totalPurchases", + "setPropertyValue": "script::profile.properties.totalPurchases != null ? profile.properties.totalPurchases + 1 : 1", + "setPropertyStrategy": "alwaysSet" + } + }, + { + "type": "setPropertyAction", + "parameterValues": { + "setPropertyName": "properties.totalRevenue", + "setPropertyValue": "script::profile.properties.totalRevenue != null ? profile.properties.totalRevenue + event.properties.amount : event.properties.amount", + "setPropertyStrategy": "alwaysSet" + } + } + ] + }' +---- + +=== Testing the Event Processing + +To test that everything works: + +1. Send a purchase event: +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/context.json \ + -H "X-Unomi-Api-Key: " \ + -H "Content-Type: application/json" \ + -d '{ + "sessionId": "session-123", + "profileId": "profile-456", + "source": { + "itemId": "checkout-page", + "itemType": "page", + "scope": "myapp" + }, + "events": [{ + "eventType": "purchaseCompleted", + "scope": "myapp", + "properties": { + "orderId": "order-790", + "amount": 149.99, + "currency": "USD", + "items": [ + { + "productId": "product-002", + "quantity": 1, + "price": 149.99 + } + ] + } + }] + }' +---- + +2. Verify profile properties were updated: +[source,bash] +---- +curl -X GET http://localhost:8181/cxs/profiles/profile-456 \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" +---- + +Expected response will show updated properties: +[source,json] +---- +{ + "itemId": "profile-456", + "properties": { + "totalPurchases": 1, + "totalRevenue": 149.99 + } + // ... other profile properties ... +} +---- + +=== Advanced Rule Examples + +==== Segmenting High-Value Customers + +Create a segment for customers with high total revenue: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/segments \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "metadata": { + "id": "highValueCustomers", + "name": "High Value Customers", + "scope": "myapp" + }, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.totalRevenue", + "comparisonOperator": "greaterThan", + "propertyValueInteger": 1000 + } + } + }' +---- + +==== Tracking Purchase Frequency + +Create a rule to track days between purchases: + +[source,bash] +---- +curl -X POST http://localhost:8181/cxs/rules \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "metadata": { + "id": "trackPurchaseFrequency", + "name": "Track Purchase Frequency", + "scope": "myapp" + }, + "condition": { + "type": "eventTypeCondition", + "parameterValues": { + "eventTypeId": "purchaseCompleted" + } + }, + "actions": [ + { + "type": "setPropertyAction", + "parameterValues": { + "setPropertyName": "properties.lastPurchaseDate", + "setPropertyValue": "script::currentDate", + "setPropertyStrategy": "alwaysSet" + } + }, + { + "type": "setPropertyAction", + "parameterValues": { + "setPropertyName": "properties.daysBetweenPurchases", + "setPropertyValue": "script::profile.properties.lastPurchaseDate != null ? Duration.between(profile.properties.lastPurchaseDate.toInstant(), currentDate.toInstant()).toDays() : null", + "setPropertyStrategy": "alwaysSet" + } + } + ] + }' +---- diff --git a/manual/src/main/asciidoc/privacy.adoc b/manual/src/main/asciidoc/privacy.adoc index 91021cb68..bbde3c398 100644 --- a/manual/src/main/asciidoc/privacy.adoc +++ b/manual/src/main/asciidoc/privacy.adoc @@ -69,12 +69,15 @@ session data will also be detached from the current profile and anonymized. [source] ---- -curl -X DELETE http://localhost:8181/cxs/privacy/profiles/{profileID}?withData=false --user karaf:karaf +curl -X DELETE http://localhost:8181/cxs/privacy/profiles/{profileID}?withData=false \ +--user "TENANT_ID:PRIVATE_KEY" ---- +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. Only the Tenant API (`/cxs/tenants`) uses system administrator authentication (`karaf:karaf`). + where `{profileID}` must be replaced by the actual identifier of a profile and the `withData` specifies whether the data associated with the profile must be anonymized or not === Related -You might also be interested in the <> section that describe how to manage profile consents. +You might also be interested in the <<_consent_api,Consent API>> section that describe how to manage profile consents. diff --git a/manual/src/main/asciidoc/property-types.adoc b/manual/src/main/asciidoc/property-types.adoc new file mode 100644 index 000000000..cccc33b0a --- /dev/null +++ b/manual/src/main/asciidoc/property-types.adoc @@ -0,0 +1,1553 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +[[_property_types]] +== Property Types + +Property types define the structure and metadata for properties that can be used in profiles and sessions within Apache Unomi. They specify the data type, display hints (ranges, constraints), default values, and other metadata that help inform UIs and enable rich querying capabilities. + +IMPORTANT: Property types are primarily designed for **UI purposes** - they inform user interfaces about what properties should be editable, how they should be displayed, and what constraints apply. Property types are **dynamically created and updated** at runtime and do not enforce strict validation on the server side. For actual data validation, Apache Unomi uses <<_json_schemas,JSON Schemas>>, which are used exclusively for validating events sent through public endpoints. + +=== Quick Reference: Property Types vs JSON Schemas + +|=== +| Aspect | Property Types | JSON Schemas + +| **Primary Purpose** | UI metadata and display hints | Server-side validation + +| **Used For Objects** | Profiles (editing), Sessions (editing) | Events (validation only) + +| **Storage** | Stored as PropertyType items, accessible via REST API | Stored in Elasticsearch/OpenSearch + +| **Dynamic Creation** | Yes, can be created/updated at runtime via REST API | Yes, can be added via API but primarily for events + +| **Validation** | No server-side validation (informational only) | Yes, enforces strict validation + +| **Event Support** | Not applicable - property types are for profiles and sessions only | Required for all events sent through public endpoints + +| **Accessibility** | Accessible via `/cxs/profiles/properties/targets/{target}` REST API | Accessible via JSON schema API + +|=== + +=== Overview + +Property types serve as **informational schemas** for properties, primarily used by UIs to: + +* The data type (string, integer, date, boolean, etc.) +* Whether the property can have multiple values +* Default values +* Validation constraints (ranges, date ranges, IP ranges) - primarily for UI hints +* Which object types the property applies to (profiles or sessions) +* How properties should be merged when profiles are combined +* Whether the property is protected (read-only) +* Display metadata (names, descriptions, tags, ranks) for UI organization + +=== Property Type Targets + +Property types are organized by their `target`, which indicates which type of object the property applies to: + +* `profiles` or `profile` - Properties for Profile objects (primary use case: editing profile data in UIs) +* `sessions` or `session` - Properties for Session objects (primary use case: editing session data in UIs) + +The target is typically determined automatically from the file path when property types are loaded from JSON files: + +* Files in `/META-INF/cxs/properties/profiles/...` → target = `"profiles"` +* Files in `/META-INF/cxs/properties/sessions/...` → target = `"sessions"` + +NOTE: Property types are only used for profiles and sessions. Events are immutable records and should not be edited through property type-based UIs. Event validation is handled by JSON schemas, not property types. + +=== Property Type Structure + +A property type is defined using the following structure: + +==== Structure Definition + +Inherits all the fields from: <> + +|=== +| *Field name* | *Type* | *Description* + +| target | String | The type of object this property type applies to (e.g., "profiles", "sessions"). Determined automatically from file path if not explicitly set. The target is extracted from the directory name immediately following "properties" in the file path (e.g., `META-INF/cxs/properties/profiles/...` → `"profiles"`). + +| type / valueTypeId | String | The identifier of the value type constraining values for properties using this PropertyType (e.g., "string", "integer", "date", "boolean", "set"). + +| defaultValue | String | The default value that properties using this PropertyType will use if no value is specified explicitly. + +| multivalued | Boolean | Whether properties using this property type are multi-valued (a vector of values as opposed to a scalar value). + +| protected / protekted | Boolean | Whether properties with this type are marked as protected. Protected properties can be displayed but their value cannot be changed. Note: In JSON files, use `"protected"` (the Java field name is `protekted`). + +| rank | Double | The rank of this PropertyType for ordering purposes. Used to control the display order of properties in UIs. + +| mergeStrategy | String | The identifier of the PropertyMergeStrategyType to be used when profiles with properties using this PropertyType are being merged. + +| dateRanges | List | Optional list of date ranges that can be used for UI categorization and hints. Note: These are informational for UIs, not enforced server-side validation. + +| numericRanges | List | Optional list of numeric ranges that can be used for UI categorization and hints (e.g., age groups). Note: These are informational for UIs, not enforced server-side validation. + +| ipRanges | List | Optional list of IP ranges that can be used for UI categorization and hints. Note: These are informational for UIs, not enforced server-side validation. + +| automaticMappingsFrom | Set | Set of property names from which properties of this type would be automatically initialized from (legacy feature). + +| childPropertyTypes | Set | For nested/complex properties, this contains the child property types that define the structure. + +|=== + +==== Metadata Fields + +Property types inherit from MetadataItem, which provides: + +|=== +| *Field name* | *Type* | *Description* + +| id | String | Unique identifier for the property type (usually matches the property name). + +| name | String | Display name for the property type. + +| description | String | Description of what the property represents. + +| scope | String | The scope for the property type. + +| tags | Set | User-editable tags for categorization. + +| systemTags | Set | System-reserved tags for categorization (e.g., "profileProperties", "sessionProperties", "personalIdentifierProperties"). + +| enabled | Boolean | Whether the property type is enabled. + +| hidden | Boolean | Whether the property type should be hidden in UIs. + +| readOnly | Boolean | Whether the property type definition can be edited. + +|=== + +=== Value Types + +Property types use value types to define the primitive data type. Common value types include: + +* `string` - Text values +* `integer` - Whole numbers +* `long` - Large integers +* `float` - Floating point numbers +* `date` - Date and time values +* `boolean` - True/false values +* `set` - Collections of values (for multi-valued or nested properties) +* `email` - Email addresses +* `geoPoint` - Geographic coordinates + +=== Examples + +==== Profile Property Type Example + +This example shows a profile property type for storing a person's age with numeric ranges: + +[source,json] +---- +{ + "metadata": { + "id": "age", + "name": "Age", + "systemTags": [ + "properties", + "profileProperties", + "personalProfileProperties" + ] + }, + "type": "integer", + "defaultValue": "", + "numericRanges": [ + {"key": "*_10", "to": 10}, + {"key": "10_20", "from": 10, "to": 20}, + {"key": "20_30", "from": 20, "to": 30}, + {"key": "30_40", "from": 30, "to": 40}, + {"key": "40_50", "from": 40, "to": 50}, + {"key": "50_*", "from": 50} + ], + "automaticMappingsFrom": [], + "rank": "501.0" +} +---- + +==== Session Property Type Example + +This example shows a session property type for storing geographic information: + +[source,json] +---- +{ + "metadata": { + "id": "sessionCity", + "name": "City", + "systemTags": [ + "properties", + "sessionProperties", + "geographicSessionProperties" + ] + }, + "type": "string", + "defaultValue": "", + "automaticMappingsFrom": [], + "rank": "3.0" +} +---- + +==== Simple Profile Property Type Example + +This example shows a basic profile property type for storing a first name: + +[source,json] +---- +{ + "metadata": { + "id": "firstName", + "name": "First name", + "systemTags": [ + "properties", + "profileProperties", + "basicProfileProperties", + "personalIdentifierProperties" + ] + }, + "type": "string", + "defaultValue": "", + "automaticMappingsFrom": [ + "j:firstName" + ], + "rank": "101.0" +} +---- + +=== Dynamic Creation and Management + +Property types are designed to be **dynamically created and updated at runtime**. This makes them ideal for scenarios where property definitions need to change without code deployments or server restarts. + +==== Key Characteristics + +* **Runtime Creation**: Property types can be created, updated, and deleted via REST API or GraphQL without restarting the server +* **No Code Deployment Required**: Changes to property types take effect immediately +* **UI-Driven**: Primarily used to inform UIs about property structure and constraints +* **Flexible**: Can be modified to adapt to changing business requirements +* **Multi-tenant**: Property types can be scoped to specific tenants + +==== Use Cases for Dynamic Property Types + +* **Custom Fields**: Allow users to define custom profile or session properties through a UI +* **A/B Testing**: Quickly add or modify property definitions for testing +* **Configuration-Driven UIs**: Build forms and property editors that adapt to property type definitions +* **Integration Flexibility**: Add properties for new integrations without code changes +* **Business Rule Changes**: Update property constraints and metadata as business rules evolve + +=== Creating Property Types + +Property types can be created in several ways: + +==== Via JSON Files in Plugins + +The most common way is to include property type definitions as JSON files in your plugin bundle: + +1. Create JSON files in your plugin's resources directory: + * Profile properties: `META-INF/cxs/properties/profiles/...` + * Session properties: `META-INF/cxs/properties/sessions/...` + +2. The target is automatically determined from the directory structure: + * Files in `properties/profiles/` → target = `"profiles"` + * Files in `properties/sessions/` → target = `"sessions"` + +3. Subdirectories can be used for organization (e.g., `profiles/personal/`, `profiles/contact/`) + +==== Via REST API + +Property types can be created or updated using the REST API: + +[source,http] +---- +POST /cxs/profiles/properties +Content-Type: application/json + +{ + "metadata": { + "id": "customProperty", + "name": "Custom Property", + "systemTags": ["properties", "profileProperties"] + }, + "type": "string", + "defaultValue": "", + "target": "profiles", + "rank": "100.0" +} +---- + +==== Via GraphQL API + +Property types can also be managed through the GraphQL API using property type mutations and queries. + +=== Querying Property Types + +==== Get All Property Types by Target + +Retrieve all property types for a specific target: + +[source,http] +---- +GET /cxs/profiles/properties/targets/profiles +---- + +Response: +[source,json] +---- +[ + { + "metadata": { + "id": "firstName", + "name": "First name", + ... + }, + "type": "string", + ... + }, + ... +] +---- + +==== Get All Property Types + +Retrieve all property types grouped by target: + +[source,http] +---- +GET /cxs/profiles/properties +---- + +Response: +[source,json] +---- +{ + "profiles": [...], + "sessions": [...] + // Note: Event property types are NOT included here as they are not stored as PropertyType items +} +---- + +==== Get Property Type by ID + +Retrieve a specific property type: + +[source,http] +---- +GET /cxs/profiles/properties/firstName +---- + +==== Get Property Types by Tag + +Retrieve property types with specific tags: + +[source,http] +---- +GET /cxs/profiles/properties/tags/profileProperties,personalProfileProperties +---- + +==== Get Property Types by System Tag + +Retrieve property types with specific system tags: + +[source,http] +---- +GET /cxs/profiles/properties/systemTags/profileProperties +---- + +=== Property Type Features + +==== Multi-valued Properties + +Set `multivalued` to `true` to allow a property to contain multiple values: + +[source,json] +---- +{ + "metadata": { + "id": "interests", + "name": "Interests" + }, + "type": "string", + "multivalued": true, + "rank": "200.0" +} +---- + +==== Protected Properties + +Set `protected` to `true` to make a property read-only (note: use `"protected"` in JSON, not `"protekted"`): + +[source,json] +---- +{ + "metadata": { + "id": "systemId", + "name": "System ID" + }, + "type": "string", + "protected": true, + "rank": "1.0" +} +---- + +==== Numeric Ranges + +Define ranges for numeric properties to enable categorization: + +[source,json] +---- +{ + "metadata": { + "id": "income", + "name": "Income" + }, + "type": "integer", + "numericRanges": [ + {"key": "low", "to": 30000}, + {"key": "medium", "from": 30000, "to": 100000}, + {"key": "high", "from": 100000} + ] +} +---- + +==== Date Ranges + +Define ranges for date properties: + +[source,json] +---- +{ + "metadata": { + "id": "registrationDate", + "name": "Registration Date" + }, + "type": "date", + "dateRanges": [ + {"key": "recent", "from": "2024-01-01"}, + {"key": "old", "to": "2023-12-31"} + ] +} +---- + +==== Merge Strategies + +Specify how properties should be merged when profiles are combined: + +[source,json] +---- +{ + "metadata": { + "id": "lastLoginDate", + "name": "Last Login Date" + }, + "type": "date", + "mergeStrategy": "mostRecentMergeStrategy", + "rank": "300.0" +} +---- + +Common merge strategies include: +* `defaultMergeStrategy` - Default merge behavior (typically uses the first non-null value) +* `mostRecentMergeStrategy` - Use the most recent value (for dates and timestamps) +* `oldestMergeStrategy` - Use the oldest value (for dates and timestamps) +* `addMergeStrategy` - Add values together (for numeric types) +* `nonEmptyMergeStrategy` - Prefer non-empty values + +NOTE: The merge strategy identifier in property type JSON files should match the `id` field from the merge strategy definition JSON files located in the `META-INF/cxs/mergers/` directory (this is a file system path within plugin bundles, not an API endpoint). + +==== Nested Properties + +For complex objects, use `childPropertyTypes` to define nested structures: + +[source,json] +---- +{ + "metadata": { + "id": "address", + "name": "Address" + }, + "type": "set", + "childPropertyTypes": [ + { + "metadata": { + "id": "street", + "name": "Street" + }, + "type": "string" + }, + { + "metadata": { + "id": "city", + "name": "City" + }, + "type": "string" + }, + { + "metadata": { + "id": "zipCode", + "name": "Zip Code" + }, + "type": "string" + } + ] +} +---- + +=== Tags and System Tags + +Property types support two types of tags for categorization: `tags` and `systemTags`. Understanding the difference between them is important for organizing and filtering property types in UIs. + +==== Tags vs System Tags + +**Tags** (`tags` field): +* User-editable tags that can be modified through UIs +* Intended for custom categorization and organization +* Can be added, removed, or changed by end users +* Perfect for creating custom groupings for UI display +* Example use cases: grouping properties by department, project, or custom business logic + +**System Tags** (`systemTags` field): +* Reserved tags typically populated through JSON descriptors +* Not meant to be modified by end users +* Used for system-level categorization and filtering +* Help classify properties for internal system logic +* Example: `profileProperties`, `sessionProperties`, `personalIdentifierProperties` + +==== Using Tags to Group Properties in UIs + +Tags provide a flexible way to organize property types for display and editing in user interfaces. Here's how you can leverage tags: + +===== UI Grouping Strategy + +1. **Create Custom Tags**: Assign custom tags to property types based on how you want to group them in your UI +2. **Query by Tags**: Use the REST API to retrieve property types filtered by specific tags +3. **Display in Groups**: Organize properties in tabs, sections, or accordions based on their tags + +===== Example: Grouping Properties by Department + +Suppose you want to group profile properties by department in your UI: + +[source,json] +---- +// Marketing properties +{ + "metadata": { + "id": "marketingSource", + "name": "Marketing Source", + "tags": ["marketing", "acquisition"] + }, + "type": "string" +} + +// Sales properties +{ + "metadata": { + "id": "salesRep", + "name": "Sales Representative", + "tags": ["sales", "account-management"] + }, + "type": "string" +} + +// Support properties +{ + "metadata": { + "id": "supportTier", + "name": "Support Tier", + "tags": ["support", "customer-service"] + }, + "type": "string" +} +---- + +Then in your UI, you can fetch and display properties grouped by tag: + +[source,http] +---- +# Get all marketing properties +GET /cxs/profiles/properties/tags/marketing + +# Get all sales properties +GET /cxs/profiles/properties/tags/sales + +# Get all support properties +GET /cxs/profiles/properties/tags/support +---- + +===== Example: Multi-Tag Filtering + +Properties can have multiple tags, allowing for flexible filtering: + +[source,json] +---- +{ + "metadata": { + "id": "vipCustomer", + "name": "VIP Customer", + "tags": ["customer-tier", "priority", "billing"] + }, + "type": "boolean" +} +---- + +You can query by multiple tags (comma-separated): + +[source,http] +---- +# Get properties with any of these tags +GET /cxs/profiles/properties/tags/customer-tier,priority,billing +---- + +==== Creating and Assigning Tags + +===== Creating Tags + +Tags don't need to be pre-registered. Simply assign a tag string to a property type, and it becomes available for filtering. Tags are created implicitly when first used. + +===== Assigning Tags via JSON Files + +When defining property types in JSON files, include tags in the metadata: + +[source,json] +---- +{ + "metadata": { + "id": "customProperty", + "name": "Custom Property", + "tags": ["my-custom-tag", "another-tag"], + "systemTags": ["properties", "profileProperties"] + }, + "type": "string", + "rank": "100.0" +} +---- + +===== Assigning Tags via REST API + +You can add or modify tags when creating or updating a property type: + +[source,http] +---- +POST /cxs/profiles/properties +Content-Type: application/json + +{ + "metadata": { + "id": "newProperty", + "name": "New Property", + "tags": ["custom-group", "editable"], + "systemTags": ["properties", "profileProperties"] + }, + "type": "string", + "target": "profiles" +} +---- + +===== Updating Tags on Existing Property Types + +To add tags to an existing property type, retrieve it, modify the tags, and save it: + +[source,http] +---- +# 1. Get the existing property type +GET /cxs/profiles/properties/existingProperty + +# 2. Update it with new tags +POST /cxs/profiles/properties +Content-Type: application/json + +{ + "metadata": { + "id": "existingProperty", + "name": "Existing Property", + "tags": ["existing-tag", "new-tag", "another-tag"], + "systemTags": ["properties", "profileProperties"] + }, + "type": "string" +} +---- + +===== Assigning Tags via GraphQL + +Tags can also be managed through the GraphQL API when creating or updating property types. + +==== Tag-Based UI Organization Examples + +===== Example 1: Tabbed Interface + +Organize properties into tabs based on tags: + +[source,typescript] +---- +// Fetch properties grouped by tags +const marketingProps: PropertyType[] = await fetch('/cxs/profiles/properties/tags/marketing') + .then(res => res.json()); +const salesProps: PropertyType[] = await fetch('/cxs/profiles/properties/tags/sales') + .then(res => res.json()); +const supportProps: PropertyType[] = await fetch('/cxs/profiles/properties/tags/support') + .then(res => res.json()); + +// Display in tabs (pseudo-rendering - in real UI framework, this would be JSX/TSX) +console.log('Tabs:'); +console.log(' Tab: Marketing'); +console.log(' PropertyEditor with', marketingProps.length, 'properties'); +console.log(' Tab: Sales'); +console.log(' PropertyEditor with', salesProps.length, 'properties'); +console.log(' Tab: Support'); +console.log(' PropertyEditor with', supportProps.length, 'properties'); +---- + +===== Example 2: Accordion Sections + +Group properties in collapsible sections: + +[source,typescript] +---- +const allProps: PropertyType[] = await fetch('/cxs/profiles/properties/targets/profiles') + .then(res => res.json()); + +// Group properties by tag +function groupByTag(propertyTypes: PropertyType[]): Record { + const grouped: Record = {}; + for (const pt of propertyTypes) { + const tags = pt.metadata.tags || []; + for (const tag of tags) { + if (!grouped[tag]) { + grouped[tag] = []; + } + grouped[tag].push(pt); + } + } + return grouped; +} + +const grouped = groupByTag(allProps); + +// Display in accordion (pseudo-rendering - in real UI framework, this would be JSX/TSX) +console.log('Accordion:'); +for (const [tag, properties] of Object.entries(grouped)) { + console.log(` AccordionItem: ${tag}`); + console.log(` PropertyEditor with ${properties.length} properties`); +} +---- + +===== Example 3: Filterable List + +Allow users to filter properties by tag: + +[source,typescript] +---- +let selectedTag: string | null = null; + +const allProps: PropertyType[] = await fetch('/cxs/profiles/properties/targets/profiles') + .then(res => res.json()); + +// Extract unique tags +function extractUniqueTags(propertyTypes: PropertyType[]): string[] { + const tags: string[] = []; + for (const pt of propertyTypes) { + const propertyTags = pt.metadata.tags || []; + for (const tag of propertyTags) { + if (!tags.includes(tag)) { + tags.push(tag); + } + } + } + return tags; +} + +const uniqueTags = extractUniqueTags(allProps); + +// Filter properties based on selected tag +const filteredProps: PropertyType[] = selectedTag + ? await fetch(`/cxs/profiles/properties/tags/${selectedTag}`).then(res => res.json()) + : allProps; + +// Display (pseudo-rendering - in real UI framework, this would be JSX/TSX) +console.log('Select dropdown:'); +console.log(' Option: All Properties'); +for (const tag of uniqueTags) { + console.log(` Option: ${tag}`); +} + +console.log(`PropertyList with ${filteredProps.length} properties`); +---- + +==== Best Practices for Tags + +1. **Use Descriptive Tag Names**: Choose clear, meaningful tag names (e.g., `marketing`, `sales`, `support` rather than `m`, `s`, `sup`) + +2. **Create Hierarchical Tags**: Use dot notation for hierarchical organization (e.g., `department.marketing`, `department.sales`) + +3. **Use Consistent Tagging**: Establish a tagging convention and apply it consistently across all property types + +4. **Combine Tags and System Tags**: Use system tags for system-level categorization and tags for UI-level organization + +5. **Document Tag Conventions**: Maintain documentation about your tagging conventions so team members use tags consistently + +6. **Avoid Over-Tagging**: Don't assign too many tags to a single property type - 2-4 tags is usually sufficient + +7. **Use Tags for Permissions**: Consider using tags to control which properties are visible or editable by different user roles + +=== System Tags + +System tags are used to categorize property types. Common system tags include: + +* `properties` - General property tag +* `profileProperties` - Properties for profiles +* `sessionProperties` - Properties for sessions +* `basicProfileProperties` - Basic profile information +* `personalProfileProperties` - Personal information +* `contactProfileProperties` - Contact information +* `workProfileProperties` - Work-related information +* `socialProfileProperties` - Social media identifiers +* `systemProfileProperties` - System-managed properties (automatically maintained by Unomi) +* `leadProfileProperties` - Lead management properties +* `personalIdentifierProperties` - Properties that identify individuals (may be subject to privacy regulations) +* `geographicSessionProperties` - Geographic session information +* `technicalSessionProperties` - Technical session information + +=== Validation and Property Types + +IMPORTANT: Property types provide **informational metadata** for UIs but do **not enforce server-side validation**. The ranges, types, and constraints defined in property types are hints for UI components, not strict validation rules. + +==== What Property Types Do + +* **Inform UIs**: Tell user interfaces what properties exist, their types, and how to display them +* **Provide Hints**: Give UI components information about expected data formats and constraints +* **Enable Dynamic Forms**: Allow UIs to generate forms and property editors automatically +* **Organize Display**: Help organize properties in UIs using tags, ranks, and categories + +==== What Property Types Don't Do + +* **Server-Side Validation**: Property types do not validate data on the server +* **Enforce Constraints**: Ranges and constraints are informational, not enforced +* **Prevent Invalid Data**: Invalid data can still be stored if not validated elsewhere +* **Type Safety**: Property types don't guarantee type safety at the storage level + +==== When Validation Happens + +* **JSON Schemas**: Used exclusively for server-side validation of events sent through `/context.json` and `/eventcollector` endpoints +* **Property Types**: Provide no server-side validation - they are informational only +* **UI Validation**: Implement client-side validation in your UI based on property type hints (recommended for profiles and sessions) +* **Application Logic**: Add validation in your application code if needed for profiles and sessions +* **Database Constraints**: Use database-level constraints if your persistence layer supports them + +==== Best Practice: Property Types for Profiles/Sessions, JSON Schemas for Events + +Property types and JSON schemas serve different purposes for different objects: + +**For Profiles and Sessions:** +* Use property types to build UIs for editing profile/session properties +* Implement client-side validation in your UI based on property type hints +* Property types provide no server-side validation - data is stored as-is + +**For Events:** +* Use JSON schemas to validate events sent through `/context.json` or `/eventcollector` +* Event property types are not accessible via REST API (they're generated internally) +* To understand event structure, query actual events or use JSON schemas + +[plantuml] +---- +@startuml +skinparam rectangle { + BackgroundColor #E8F4F8 + BorderColor #4A90A4 +} + +rectangle "Profiles/Sessions Workflow" { + rectangle "Property Type\n(UI hints)" as PT + rectangle "UI Form\n(with client validation)" as UI + rectangle "Profile/Session Update\n(No server validation)" as Update + + PT --> UI + UI --> Update +} + +rectangle "Events Workflow" { + rectangle "JSON Schema\n(validation)" as Schema + rectangle "Event sent via\n/context.json\nor /eventcollector" as Event + rectangle "Server validates\nvia JSON Schema" as Validate + rectangle "Event stored\nif valid" as Store + + Schema --> Event + Event --> Validate + Validate --> Store +} +@enduml +---- + +=== Best Practices + +1. **Use descriptive names**: Property type IDs should be clear and descriptive (e.g., `firstName` rather than `fn`). + +2. **Set appropriate ranks**: Use the `rank` field to control display order in UIs. Lower numbers appear first. + +3. **Use system tags**: Apply appropriate system tags to help categorize and filter property types. + +4. **Organize with subdirectories**: Use subdirectories in your plugin structure to organize related property types. + +5. **Define ranges for categorization**: Use numeric ranges, date ranges, or IP ranges to enable automatic categorization of values. + +6. **Mark sensitive properties**: Use `personalIdentifierProperties` system tag for properties that identify individuals. + +7. **Set merge strategies**: Define appropriate merge strategies for properties that may need to be merged when profiles are combined. + +8. **Use appropriate value types**: Choose the correct value type (string, integer, date, etc.) for your data. + +=== Built-in Property Types + +Apache Unomi comes with a comprehensive set of built-in property types that cover common use cases. These are organized by category and target type. + +==== Built-in Profile Property Types + +===== Basic Profile Properties + +These properties store basic identifying information about profiles: + +* `firstName` (string) - First name of the profile +* `lastName` (string) - Last name of the profile +* `gender` (string) - Gender +* `nationality` (string) - Nationality + +All basic profile properties are tagged with `basicProfileProperties`. `firstName` and `lastName` are also tagged with `personalIdentifierProperties`. + +===== Personal Profile Properties + +These properties store personal information: + +* `age` (integer) - Age with numeric ranges (0-10, 10-20, 20-30, 30-40, 40-50, 50+) +* `birthDate` (date) - Birth date with date ranges +* `kids` (integer) - Number of children +* `maritalStatus` (string) - Marital status + +All personal profile properties are tagged with `personalProfileProperties`. + +===== Contact Profile Properties + +These properties store contact information: + +* `email` (email) - Email address +* `phoneNumber` (string) - Phone number +* `address` (string) - Street address +* `city` (string) - City +* `zipCode` (string) - Zip/postal code +* `countryName` (string) - Country name + +All contact profile properties are tagged with `contactProfileProperties`. Email, address, and phoneNumber are also tagged with `personalIdentifierProperties`. + +===== Work Profile Properties + +These properties store work-related information: + +* `company` (string) - Company name +* `jobTitle` (string) - Job title +* `income` (integer) - Income + +All work profile properties are tagged with `workProfileProperties`. + +===== Social Profile Properties + +These properties store social media identifiers: + +* `facebookId` (string) - Facebook ID +* `twitterId` (string) - Twitter ID +* `linkedInId` (string) - LinkedIn ID +* `googleId` (string) - Google ID + +All social profile properties are tagged with `socialProfileProperties`. + +===== System Profile Properties + +These properties are automatically managed by Unomi and are typically protected (read-only): + +* `firstVisit` (date) - Date of first visit (protected, uses `oldestMergeStrategy`) +* `lastVisit` (date) - Date of last visit (protected, uses `mostRecentMergeStrategy`) +* `previousVisit` (date) - Date of previous visit (protected, uses `mostRecentMergeStrategy`) +* `nbOfVisits` (integer) - Number of visits with ranges (protected, uses `addMergeStrategy`) +* `totalNbOfVisits` (integer) - Total number of visits (protected, uses `addMergeStrategy`) + +All system profile properties are tagged with `systemProfileProperties`. + +===== Lead Profile Properties + +* `leadAssignedTo` (string) - Person assigned to the lead + +All lead profile properties are tagged with `leadProfileProperties`. + +==== Built-in Session Property Types + +===== Geographic Session Properties + +These properties store geographic information about the session: + +* `sessionCity` (string) - City where the session occurred +* `sessionCountryName` (string) - Country name +* `sessionCountryCode` (string) - Country code +* `sessionAdminSubDiv1` (string) - First-level administrative subdivision (e.g., state, province) +* `sessionAdminSubDiv2` (string) - Second-level administrative subdivision (e.g., county) +* `latitude` (string) - Latitude coordinate +* `longitude` (string) - Longitude coordinate + +All geographic session properties are tagged with `geographicSessionProperties`. + +===== Technical Session Properties + +These properties store technical information about the session: + +* `deviceCategory` (string) - Device category (e.g., desktop, mobile, tablet) +* `operatingSystemName` (string) - Operating system name +* `operatingSystemFamily` (string) - Operating system family +* `userAgentName` (string) - User agent name (browser name) +* `userAgentVersion` (string) - User agent version (browser version) +* `pageReferringURL` (string) - URL of the referring page + +All technical session properties are tagged with `technicalSessionProperties`. + +==== Property Type Organization + +Built-in property types are organized in the following directory structure: + +[source] +---- +META-INF/cxs/properties/ + profiles/ + basic/ # Basic identifying information + personal/ # Personal information + contact/ # Contact information + work/ # Work-related information + social/ # Social media identifiers + system/ # System-managed properties + lead/ # Lead management + sessions/ + geographic/ # Geographic information + technical/ # Technical information +---- + +The directory structure helps organize property types, and the target is automatically determined from the parent directory (`profiles` or `sessions`). + +==== Viewing Built-in Property Types + +You can view all built-in property types using the REST API: + +[source,http] +---- +# Get all profile property types +GET /cxs/profiles/properties/targets/profiles + +# Get all session property types +GET /cxs/profiles/properties/targets/sessions + +# Get property types by system tag +GET /cxs/profiles/properties/systemTags/profileProperties +GET /cxs/profiles/properties/systemTags/sessionProperties +---- + +=== Property Types vs JSON Schemas + +Property types and JSON schemas serve complementary but distinct roles in Apache Unomi: + +==== Property Types: UI-Focused Metadata + +Property types are: +* **Primarily for UIs**: Inform user interfaces about property structure, display names, types, and constraints +* **Dynamically managed**: Can be created, updated, and deleted at runtime via REST API or GraphQL +* **Informational**: Provide metadata about properties but don't enforce strict server-side validation +* **Flexible**: Allow UIs to adapt to changing property definitions without code changes +* **Target-specific**: Organized by target (profiles and sessions) + +Use property types when you need to: +* Build dynamic forms and property editors in UIs +* Display property information to users +* Organize properties by tags or categories +* Control property visibility and editability in interfaces +* Provide hints about data types and constraints to UI components + +==== JSON Schemas: Server-Side Validation + +JSON schemas are: +* **For validation**: Enforce strict data validation on the server side +* **Event-only**: Used exclusively to validate events sent through public endpoints (`/context.json` and `/eventcollector`) +* **Standard-based**: Use the JSON Schema standard for validation +* **Persistent**: Stored in Elasticsearch/OpenSearch and loaded at startup +* **Required for events**: Each event type must have an associated JSON schema with target `"events"` + +NOTE: While JSON schemas technically support targets like "profiles", "sessions", "rules", and "segments", in practice they are **only used for event validation**. The validation code only processes schemas with target `"events"` when validating incoming requests through the public endpoints. + +Use JSON schemas when you need to: +* Validate incoming events from external sources +* Ensure event data integrity and type safety +* Enforce business rules and constraints on events +* Validate event structures before processing + +==== How They Work Together + +Property types and JSON schemas complement each other: + +[plantuml] +---- +@startuml +skinparam rectangle { + BackgroundColor #E8F4F8 + BorderColor #4A90A4 +} + +rectangle "Property Types" as PT +note right of PT + • UI metadata (names, descriptions, types) + • Display hints (ranks, tags, protected flags) + • Dynamic and editable at runtime + • Used by GraphQL to generate schema +end note + +rectangle "User Interface" as UI +note right of UI + • Dynamic forms based on property types + • Property editors with type hints + • Grouped displays using tags +end note + +rectangle "JSON Schemas" as JSON +note right of JSON + • Server-side validation for EVENTS ONLY + • Event type checking and constraints + • Event structure validation + • Event data integrity enforcement + • Used when events sent via /context.json + or /eventcollector endpoints +end note + +PT --> UI : UI Generation +UI --> JSON : Data Submission +@enduml +---- + +**Example Workflow:** + +1. **Property Type Definition**: A property type defines `email` as a string with description "Email address" for profiles +2. **UI Generation**: The UI uses the property type to render an email input field with appropriate validation hints for profile editing +3. **User Input**: User enters an email address in the UI +4. **Data Storage**: Data is stored in the profile (property types don't validate, but UI can implement client-side validation) +5. **Event Validation (separate)**: When events are sent through `/context.json` or `/eventcollector`, JSON schemas validate the event structure (not profile properties) + +==== When to Use Each + +**Use Property Types for:** +* Building dynamic property editors +* Organizing properties in UIs (tabs, sections, filters) +* Providing type hints to form builders +* Controlling property visibility and editability +* GraphQL schema generation + +**Use JSON Schemas for:** +* Validating incoming events from external sources (the only actual use case) +* Enforcing event data type constraints +* Validating complex nested event structures +* Ensuring event data integrity on the server + +**Use Both When:** +* You want UI hints for profiles/sessions (property types) AND need to validate events (JSON schemas) +* Building UIs that display profile/session data (property types) while also handling event validation (JSON schemas) + +=== Property Types for Events + +Property types are **not used for events**. Events are immutable records of user interactions and should not be edited through property type-based UIs. + +If you need to understand event structure for display purposes: +* Query actual events to see their structure +* Use JSON schemas (with target `"events"`) to understand the expected structure of events +* JSON schemas define the validation rules for events sent through public endpoints + +Event validation is performed by JSON schemas (with target `"events"`) when events are sent through the public endpoints (`/context.json` and `/eventcollector`). + +=== Building UIs with Property Types + +Property types are designed to enable dynamic UI generation. Here's how to use them effectively: + +==== Step 1: Fetch Property Types + +Retrieve property types for your target (profiles or sessions): + +[source] +---- +# Fetch all profile property types +GET /cxs/profiles/properties/targets/profiles + +# Fetch session property types +GET /cxs/profiles/properties/targets/sessions + +# Fetch by tag for grouping +GET /cxs/profiles/properties/tags/marketing +---- + +TypeScript example: + +[source,typescript] +---- +// Fetch property types from REST API +const propertyTypes: PropertyType[] = await fetch('/cxs/profiles/properties/targets/profiles') + .then(res => res.json()); + +// Or fetch by tag +const marketingProps: PropertyType[] = await fetch('/cxs/profiles/properties/tags/marketing') + .then(res => res.json()); +---- + +==== Step 2: Generate Form Fields + +Use property type metadata to generate appropriate form fields: + +[source,typescript] +---- +interface FormField { + id: string; + name: string; + description?: string; + inputType: string; + multivalued: boolean; + defaultValue: string; + readOnly: boolean; + tags: string[]; +} + +function generateFormField(propertyType: PropertyType): FormField { + // Extract property type information + const id = propertyType.metadata.id; + const name = propertyType.metadata.name; + const description = propertyType.metadata.description; + const valueType = propertyType.type; + const isMultivalued = propertyType.multivalued || false; + const defaultValue = propertyType.defaultValue || ''; + const isProtected = propertyType.protected || false; + + // Determine input type based on value type + let inputType: string; + if (valueType === 'integer' || valueType === 'long') { + inputType = 'number'; + } else if (valueType === 'date') { + inputType = 'date'; + } else if (valueType === 'boolean') { + inputType = 'checkbox'; + } else if (valueType === 'email') { + inputType = 'email'; + } else { + inputType = 'text'; + } + + // Return field configuration + return { + id, + name, + description, + inputType, + multivalued: isMultivalued, + defaultValue, + readOnly: isProtected, + tags: propertyType.metadata.tags || [] + }; +} + +// Generate form fields for all property types +const formFields: FormField[] = propertyTypes.map(generateFormField); +---- + +==== Step 3: Render Dynamic Forms + +Use the generated fields to render your form: + +[source,typescript] +---- +interface PropertyFormProps { + propertyTypes: PropertyType[]; + onSubmit: (values: Record) => void; +} + +function renderPropertyForm({ propertyTypes, onSubmit }: PropertyFormProps): void { + // Initialize form values + const values: Record = {}; + + // Generate form fields + const fields: FormField[] = propertyTypes.map(generateFormField); + + // Group fields by tags + const groupedFields: Record = {}; + for (const field of fields) { + const tag = field.tags[0] || 'other'; + if (!groupedFields[tag]) { + groupedFields[tag] = []; + } + groupedFields[tag].push(field); + } + + // Render form (pseudo-rendering logic) + for (const [tag, tagFields] of Object.entries(groupedFields)) { + // Render fieldset with legend + console.log(`Fieldset: ${tag}`); + + for (const field of tagFields) { + // Render label + console.log(`Label: ${field.name}`); + + if (field.description) { + // Render help-text + console.log(`Help: ${field.description}`); + } + + if (field.multivalued) { + // Render multi-value-input + const currentValue = values[field.id] || []; + // onChange handler would update values[field.id] = newValue + console.log(`Multi-value input: ${field.inputType}`, currentValue); + } else { + // Render single input + const currentValue = values[field.id] ?? field.defaultValue; + // onChange handler would update values[field.id] = newValue + console.log(`Input: ${field.inputType}`, currentValue, `readOnly: ${field.readOnly}`); + } + } + } + + // Render submit button + console.log('Submit button: Save'); +} +---- + +==== Step 4: Handle Numeric Ranges + +For properties with numeric ranges, you can provide range-based inputs: + +[source,typescript] +---- +interface NumericRange { + key: string; + from?: number; + to?: number; +} + +function renderNumericRangeInput( + propertyType: PropertyType, + value: string, + onChange: (value: string) => void +): void { + const numericRanges = propertyType.numericRanges; + + if (numericRanges && numericRanges.length > 0) { + // Render as select dropdown with range options + console.log('Select dropdown:'); + console.log(' Option: Select range'); + + for (const range of numericRanges) { + const label = formatRange(range); + console.log(` Option: ${label} (value: ${range.key})`); + } + } else { + // Fallback to regular number input + console.log(`Number input: ${value}`); + } +} + +function formatRange(range: NumericRange): string { + if (range.from !== undefined && range.to !== undefined) { + return `${range.from} - ${range.to}`; + } else if (range.from !== undefined) { + return `${range.from}+`; + } else if (range.to !== undefined) { + return `up to ${range.to}`; + } else { + return range.key; + } +} +---- + +==== Step 5: Sort by Rank + +Use the `rank` field to control display order: + +[source,typescript] +---- +// Sort property types by rank (lower numbers first) +function sortByRank(propertyTypes: PropertyType[]): PropertyType[] { + return [...propertyTypes].sort((a, b) => { + const rankA = a.rank ?? 999; // Default rank for items without rank + const rankB = b.rank ?? 999; + return rankA - rankB; + }); +} + +const sortedPropertyTypes: PropertyType[] = sortByRank(propertyTypes); +---- + +==== Complete Example: Dynamic Property Editor + +Here's a complete example of a dynamic property editor: + +[source,typescript] +---- +interface PropertyType { + metadata: { + id: string; + name: string; + description?: string; + tags?: string[]; + systemTags?: string[]; + }; + type: string; + defaultValue?: string; + multivalued?: boolean; + protected?: boolean; + rank?: number; + numericRanges?: NumericRange[]; +} + +interface DynamicPropertyEditorState { + propertyTypes: PropertyType[]; + values: Record; + selectedTag: string | null; +} + +async function createDynamicPropertyEditor( + target: string, + onSave: (values: Record) => void +): Promise { + // Initialize state + let propertyTypes: PropertyType[] = []; + const values: Record = {}; + let selectedTag: string | null = null; + + // Fetch property types from REST API + propertyTypes = await fetch(`/cxs/profiles/properties/targets/${target}`) + .then(res => res.json()); + + // Initialize values with defaults + for (const propertyType of propertyTypes) { + if (propertyType.defaultValue != null) { + values[propertyType.metadata.id] = propertyType.defaultValue; + } + } + + // Get unique tags for filtering + const tags: string[] = []; + for (const propertyType of propertyTypes) { + const propertyTags = propertyType.metadata.tags || []; + for (const tag of propertyTags) { + if (!tags.includes(tag)) { + tags.push(tag); + } + } + } + + // Filter property types by selected tag + let filteredTypes = propertyTypes; + if (selectedTag != null) { + filteredTypes = propertyTypes.filter(pt => + (pt.metadata.tags || []).includes(selectedTag!) + ); + } + + // Sort by rank + const sortedTypes = sortByRank(filteredTypes); + + // Render form (pseudo-rendering - in real UI framework, this would be JSX/TSX) + console.log('Rendering form...'); + + // Tag filter dropdown + if (tags.length > 0) { + console.log('Tag filter dropdown:', selectedTag); + console.log(' Option: All properties'); + for (const tag of tags) { + console.log(` Option: ${tag}`); + } + } + + // Property fields + for (const propertyType of sortedTypes) { + const fieldId = propertyType.metadata.id; + const fieldValue = values[fieldId] ?? propertyType.defaultValue ?? ''; + + console.log(`Field: ${propertyType.metadata.name} (${fieldId})`); + + if (propertyType.metadata.description) { + console.log(` Description: ${propertyType.metadata.description}`); + } + + if (propertyType.protected) { + console.log(` Input: text, value: ${fieldValue}, readOnly: true`); + } else { + propertyInput(propertyType, fieldValue, (newValue) => { + values[fieldId] = newValue; + }); + } + } + + // Submit button + console.log('Submit button: Save Properties'); + + return { propertyTypes, values, selectedTag }; +} + +function propertyInput( + propertyType: PropertyType, + value: any, + onChange: (value: any) => void +): void { + // Handle numeric ranges + if (propertyType.numericRanges && propertyType.numericRanges.length > 0) { + console.log(' Select dropdown with numeric ranges'); + console.log(' Option: Select...'); + for (const range of propertyType.numericRanges) { + console.log(` Option: ${formatRange(range)} (value: ${range.key})`); + } + return; + } + + // Handle multivalued properties + if (propertyType.multivalued) { + const valueList: any[] = Array.isArray(value) ? value : (value ? [value] : []); + console.log(' Multi-value input container:'); + for (let index = 0; index < valueList.length; index++) { + const val = valueList[index]; + const inputType = getInputType(propertyType.type); + console.log(` Input[${index}]: ${inputType}, value: ${val}`); + // onChange would update valueList[index] = newVal, then call onChange(valueList) + } + console.log(' Button: Add Value'); + return; + } + + // Regular single-value input + const inputType = getInputType(propertyType.type); + console.log(` Input: ${inputType}, value: ${value}`); +} + +function getInputType(valueType: string): string { + if (valueType === 'integer' || valueType === 'long') { + return 'number'; + } else if (valueType === 'date') { + return 'date'; + } else if (valueType === 'boolean') { + return 'checkbox'; + } else if (valueType === 'email') { + return 'email'; + } else { + return 'text'; + } +} +---- + +=== Related Topics + +* <<_data_model_overview,Data Model Overview>> - Learn about the overall data model +* <<_writing_plugins,Writing Plugins>> - Learn how to create plugins with property types +* <<_json_schemas,JSON Schemas>> - Learn about JSON schema support in Unomi +* <<_graphql_api,GraphQL API>> - Learn about GraphQL property type operations diff --git a/manual/src/main/asciidoc/queries-and-aggregations.adoc b/manual/src/main/asciidoc/queries-and-aggregations.adoc index c7c25b9d7..aa536a03c 100644 --- a/manual/src/main/asciidoc/queries-and-aggregations.adoc +++ b/manual/src/main/asciidoc/queries-and-aggregations.adoc @@ -28,7 +28,7 @@ Here's an example of a query: [source,bash] ---- curl -X POST http://localhost:8181/cxs/query/profile/count \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -80,7 +80,7 @@ Here's an example request that uses the `sum` and `avg` metrics: [source] ---- curl -X POST http://localhost:8181/cxs/query/session/profile.properties.nbOfVisits/sum/avg \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -132,14 +132,14 @@ Aggregations may be of different types. They are listed here below. Date aggregations make it possible to automatically generate "buckets" by time periods. The format is compatible with both ElasticSearch and OpenSearch. For more information about the format, you can refer to: - ElasticSearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-datehistogram-aggregation.html -- OpenSearch documentation: https://opensearch.org/docs/2.11/aggregations/bucket/datehistogram/ +- OpenSearch documentation: https://opensearch.org/docs/latest/aggregations/bucket/datehistogram/ Here's an example of a request to retrieve a histogram of by day of all the session that have been create by newcomers (nbOfVisits=1) [source] ---- curl -X POST http://localhost:8181/cxs/query/session/timeStamp \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -205,7 +205,7 @@ below: [source,shell script] ---- curl -X POST http://localhost:8181/cxs/query/profile/properties.birthDate \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -270,7 +270,9 @@ The resulting JSON response will look something like this: } ---- -You can find more information about the date range formats here: https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket-daterange-aggregation.html +You can find more information about the date range formats here: +- ElasticSearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-daterange-aggregation.html +- OpenSearch documentation: https://opensearch.org/docs/latest/aggregations/bucket/daterange/ ===== Numeric range @@ -282,7 +284,7 @@ Here's an example of a using numeric range to regroup profiles by number of visi [source,shell script] ---- curl -X POST http://localhost:8181/cxs/query/profile/properties.nbOfVisits \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ -d @- <<'EOF' { @@ -346,3 +348,74 @@ This will produce an output that looks like this: } ---- +=== Scroll Queries + +Scroll queries are useful when you need to retrieve large result sets efficiently. Instead of retrieving all results at once, +which could be memory intensive, scroll queries allow you to retrieve results in batches while maintaining consistency +between requests. + +==== Initial Scroll Query + +To start a scroll query, use the search endpoint with the following parameters: + +[source,bash] +---- +# Initial scroll query +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "TENANT_ID:PRIVATE_KEY" \ +-H "Content-Type: application/json" \ +-d @- <<'EOF' +{ + "condition": { // Optional filtering condition + "type": "matchAllCondition", + "parameterValues": {} + }, + "offset": 0, // Starting position + "limit": 50, // Number of items per batch + "sortby": "property:asc", // Optional sorting + "scrollTimeValidity": "1m", // How long to keep scroll context alive + "forceRefresh": false // Whether to force index refresh +} +EOF +---- + +The response will include: +- `scrollIdentifier`: A unique identifier for continuing the scroll +- `list`: The first batch of results +- `totalSize`: Total number of matching items +- `offset`: Current position in the result set +- `pageSize`: Size of the current batch + +==== Continue Scroll Query + +To retrieve subsequent batches, only the scroll identifier and validity are needed: + +[source,bash] +---- +# Retrieve next batch +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "TENANT_ID:PRIVATE_KEY" \ +-H "Content-Type: application/json" \ +-d @- <<'EOF' +{ + "scrollIdentifier": "YOUR_SCROLL_ID", // Required: the scroll ID from previous response + "scrollTimeValidity": "1m" // Required: extends scroll context lifetime +} +EOF +---- + +IMPORTANT: When continuing a scroll query, you do not need to resend other query parameters (condition, sortby, etc.). The scroll context maintains the original query state. + +==== Query Parameters + +Initial query parameters: +- `condition`: The filtering condition (optional) +- `offset`: Starting position in results +- `limit`: Number of results per batch +- `sortby`: Property and direction for sorting (optional) +- `scrollTimeValidity`: How long to keep the scroll context alive (e.g., "1m") +- `forceRefresh`: Whether to force an index refresh before query (default: false) + +Continuation query parameters: +- `scrollIdentifier`: The scroll ID returned from the previous request +- `scrollTimeValidity`: How long to extend the scroll context \ No newline at end of file diff --git a/manual/src/main/asciidoc/recipes.adoc b/manual/src/main/asciidoc/recipes.adoc index 592c31db9..5d21c2d4d 100644 --- a/manual/src/main/asciidoc/recipes.adoc +++ b/manual/src/main/asciidoc/recipes.adoc @@ -18,6 +18,7 @@ In this section of the documentation we provide quick recipes focused on helping you achieve a specific result with Apache Unomi. +[[_enabling_debug_mode]] ==== Enabling debug mode Although the examples provided in this documentation are correct (they will work "as-is"), @@ -70,12 +71,18 @@ therefore to find the source of a schema validation issue it's best to start fro The simplest way to retrieve profile data for the current profile is to simply send a request to the /cxs/context.json endpoint. However you will need to send a body along with that request. Here's an example: +[NOTE] +==== +The `/cxs/context.json` endpoint automatically creates or loads profiles and sessions as needed. See <<_how_profile_tracking_works,How profile tracking works>> for complete details about the automatic profile and session creation process. +==== + Here is an example that will retrieve all the session and profile properties, as well as the profile's segments and scores [source] ---- curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ --data-raw '{ "source": { "itemId":"homepage", @@ -137,7 +144,7 @@ Let's go into more detail about the preferred way to update a profile. Let's con [source] ---- curl -X POST http://localhost:8181/cxs/rules \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ --data-raw '{ "metadata": { @@ -193,7 +200,7 @@ We will start by creating a scope called "example" scope: [source] ---- curl --location --request POST 'http://localhost:8181/cxs/scopes' \ --u 'karaf:karaf' \ +--user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "itemId": "example", @@ -201,12 +208,14 @@ curl --location --request POST 'http://localhost:8181/cxs/scopes' \ }' ---- +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. Only the Tenant API (`/cxs/tenants`) uses system administrator authentication (`karaf:karaf`). + The next step consist in creating a JSON Schema to validate our event. [source] ---- curl --location --request POST 'http://localhost:8181/cxs/jsonSchema' \ --u 'karaf:karaf' \ +--user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "$id": "https://unomi.apache.org/schemas/json/events/contactInfoSubmitted/1-0-0", @@ -260,6 +269,7 @@ Finally, send the `contactInfoSubmitted` event using a request similar to this o ---- curl -X POST http://localhost:8181/cxs/eventcollector \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ --data-raw '{ "sessionId" : "1234", "events":[ @@ -291,7 +301,7 @@ The event we just submitted can be retrieved using the following request: [source] ---- curl -X POST http://localhost:8181/cxs/events/search \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ --data-raw '{ "offset" : 0, @@ -313,11 +323,11 @@ There could be two types of common errors while customizing the above requests: * The schema is invalid * The event is invalid -While first submitting the schema during its creation, Apache Unomi will validate it is syntaxically correct (JSON) +While first submitting the schema during its creation, Apache Unomi will validate it is syntactically correct (JSON) but will not perform any further validation. Since the schema will be processed for the first time when events are submitted, errors might be noticeable at that time. -Those errors are usually self-explanatory, such as this one pointing to an incorrect lcoation for the "firstName" keyword: +Those errors are usually self-explanatory, such as this one pointing to an incorrect location for the "firstName" keyword: [source] ---- 09:35:56.573 WARN [qtp1421852915-83] Unknown keyword firstName - you should define your own Meta Schema. If the keyword is irrelevant for validation, just use a NonValidationKeyword @@ -340,7 +350,7 @@ that looks something like this (and https://unomi.apache.org/rest-api-doc/#17681 [source] ---- curl -X POST http://localhost:8181/cxs/events/search \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ --data-raw '{ "offset" : 0, @@ -372,7 +382,7 @@ on the Apache Unomi server. [source] ---- curl -X POST http://localhost:8181/cxs/rules \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ --data-raw '{ "metadata": { @@ -407,7 +417,7 @@ structure. Here's an example of a profile search with a Query object: [source] ---- curl -X POST http://localhost:8181/cxs/profiles/search \ ---user karaf:karaf \ +--user "TENANT_ID:PRIVATE_KEY" \ -H "Content-Type: application/json" \ --data-raw '{ "text" : "unomi", @@ -449,7 +459,7 @@ on the server and potentially this could affect performance. ==== Getting / updating consents -You can find information on how to retrieve or create/update consents in the <> section. +You can find information on how to retrieve or create/update consents in the <<_consent_api,Consent API>> section. ==== How to send a login event to Unomi @@ -488,6 +498,43 @@ upon successful login using an email, Unomi will look for other profiles that ha a single profile. Because of the merge, this should only be done for authenticated profiles, otherwise this could be a security issue since it could be a way to load data from other profiles by merging their data ! +==== Technical Implementation Details + +===== Profile Selection Strategy + +By default, profiles are merged with the oldest profile (determined by the `firstVisit` property) becoming the master profile. This behavior can be overridden by setting `forceEventProfileAsMaster` to true in the action parameters, which forces the current event's profile to become the master. + +===== Property Merge Strategy + +When merging profile properties, Unomi follows these rules: + +1. Properties from all profiles are combined into the master profile +2. When conflicting values exist for a property, the master profile's value takes precedence +3. Multi-valued properties (arrays/collections) are merged together +4. Profile segments and scores are recalculated after the merge + +A custom property merge strategy can be implemented by: + +1. Creating a class that implements `PropertyMergeStrategyType` +2. Registering it as an OSGi service +3. Configuring it in your merge action + +===== Asynchronous Processing + +To ensure performance, Unomi processes resource-intensive operations asynchronously: + +1. The profile merge itself happens synchronously during event processing +2. Session and event reassignment is scheduled asynchronously via the SchedulerService +3. Property updates triggered by the merge may cause delayed rule evaluation + +===== Result Codes + +The `MergeProfilesOnPropertyAction` returns one of the following result codes: + +* `EventService.NO_CHANGE` - No merge was performed +* `EventService.PROFILE_UPDATED` - A profile was updated during the merge +* `EventService.PROFILE_UPDATED + EventService.SESSION_UPDATED` - Both profile and session were updated + ==== What profile aliases are and how to use them Profile aliases make it possible to reference profiles using multiple identifiers. @@ -546,3 +593,97 @@ Upon merge: | DELETE | /cxs/profiles/PROFILE_ID/aliases/ALIAS_ID | Remove an alias from a profile |=== + +==== Common Issues and Best Practices + +===== Decision Guide for Profile Merging + +[plantuml] +---- +@startuml +title When to Use Profile Merges + +start +:Evaluate Use Case; + +if (Do you need to combine visitor data\nafter authentication?) then (yes) + :Profile merging is appropriate; +else (no) + :Consider alternative solutions; + stop +endif + +if (Is the identifier unique per user?) then (yes) + :Continue; +else (no) + :Choose a more unique identifier; + note right: Email, CRM ID, or verified account ID + stop +endif + +if (Is the merge happening in a\nsecure/authenticated context?) then (yes) + :Continue; +else (no) + :Secure the context first; + note right: Require login or verification + stop +endif + +:Implement profile merging with +MergeProfilesOnPropertyAction; +stop +@enduml +---- + +===== Troubleshooting Common Issues + +====== Merge Not Happening + +If profiles aren't merging as expected, check: + +1. The merge property exists on both profiles with exactly matching values +2. The merge property is stored as a system property (`systemProperties.mergeIdentifier`) +3. The rule containing the merge action is correctly triggered +4. The profiles aren't personas or anonymous profiles (which are skipped) +5. Ensure the `maxProfilesInOneMerge` limit (default 50) is not being exceeded + +====== Data Loss During Merge + +By default, when property values conflict, the master profile's values take precedence. This can appear as data loss from secondary profiles. + +Consider: + +1. Implementing a custom PropertyMergeStrategy for complex merging logic +2. Pre-processing profiles to ensure critical data is preserved +3. Logging pre-merge state for audit purposes + +====== Performance Issues + +Profile merges can impact performance, especially in high-traffic sites: + +1. Monitor Elasticsearch load during merge operations +2. Consider batch processing for bulk merges +3. Adjust the `maxProfilesInOneMerge` parameter based on your system capabilities +4. Schedule large merge operations during off-peak hours + +===== Testing Recommendations + +To properly test profile merges, consider: + +1. *Test Environment*: Set up a separate testing environment that closely mirrors production +2. *Test Scenarios*: + a. Simple merge of two profiles + b. Merge of profiles with conflicting property values + c. Merge with session data and event history + d. Race condition testing (simultaneous merges) +3. *Data Verification*: Verify that all important data is preserved after merges +4. *Performance Testing*: Test merge operations under load to identify bottlenecks + +===== Implementation Best Practices + +1. *Security First*: Always implement merges in authenticated contexts only +2. *Clear Audit Trail*: Maintain logs of merge operations for troubleshooting +3. *Graceful Degradation*: Handle merge failures gracefully without disrupting the user experience +4. *Monitoring*: Set up alerts for unusual merge patterns that might indicate security issues +5. *Data Protection*: Consider regulations like GDPR when merging personal data +6. *Backup Strategy*: Ensure you have a strategy to recover from problematic merges diff --git a/manual/src/main/asciidoc/request-examples.adoc b/manual/src/main/asciidoc/request-examples.adoc index 819e0188d..69af6fe01 100644 --- a/manual/src/main/asciidoc/request-examples.adoc +++ b/manual/src/main/asciidoc/request-examples.adoc @@ -11,30 +11,224 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[[_request_examples]] === Request examples +==== Prerequisites + +Before running any of the examples below, you need to: + +===== 1. Create a tenant + +First, create a tenant that will own all the data: + +[source] +---- +curl -X POST http://localhost:8181/cxs/tenants \ +--user karaf:karaf \ +-H "Content-Type: application/json" \ +-d '{ + "requestedId": "mytenant", + "properties": { + "name": "My Company", + "description": "My tenant description" + } +}' +---- + +The response will include the created tenant with automatically generated API keys: + +[source,json] +---- +{ + "itemId": "mytenant", + "name": "My Company", + "description": "My tenant description", + "apiKeys": [ + { + "type": "PUBLIC", + "key": "public-key-abc123...", + "created": "2024-01-01T00:00:00Z" + }, + { + "type": "PRIVATE", + "key": "private-key-xyz789...", + "created": "2024-01-01T00:00:00Z" + } + ] +} +---- + +After creating the tenant, you will need to use these credentials in the examples: +- Tenant ID: `mytenant` +- Private API Key: Extract the `key` value from the API key with `type: "PRIVATE"` in the response +- Public API Key: Extract the `key` value from the API key with `type: "PUBLIC"` in the response + +===== 2. Create a scope + +Then, create a scope for your digital property (website, mobile app, etc.): + +[source] +---- +curl -X POST http://localhost:8181/cxs/scopes \ +--user "mytenant:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "mydigital", + "name": "My Digital Property", + "description": "My website scope" + } +}' +---- + +TIP: The scope creation response will include a public API key that you should save for use with the public APIs. + +[IMPORTANT] +==== +**Tenant Resolution (Version 3.1+)** + +Starting with Apache Unomi 3.1, tenant resolution is mandatory for all requests to `/cxs/context.json` and `/cxs/eventcollector` endpoints. The tenant must be identified before Apache Unomi can process the request. This is done automatically when you use: + +* **Public API Key** (recommended for public endpoints): The `X-Unomi-Api-Key` header with a public API key automatically resolves the tenant +* **Private API Key**: Basic authentication with `tenantId:privateApiKey` resolves the tenant from the credentials +* **Tenant ID Header**: When using JAAS authentication, the `X-Unomi-Tenant-Id` header can be used + +All examples in this document use the `X-Unomi-Api-Key` header with a public API key, which is the recommended approach for public-facing applications. See <<_how_profile_tracking_works,How profile tracking works>> for complete details about tenant resolution. +==== + +NOTE: In all the examples below, replace: +- `YOUR_TENANT_ID` with `mytenant` +- `YOUR_PRIVATE_API_KEY` with your actual private API key +- `example` scope with `mydigital` +- `YOUR_PUBLIC_API_KEY` with the public key from the scope creation response + +===== 3. Verify the scope + +You can verify the scope was created correctly by retrieving it: + +[source] +---- +curl -X GET http://localhost:8181/cxs/scopes/mydigital \ +--user "mytenant:YOUR_PRIVATE_API_KEY" +---- + +Or list all scopes: + +[source] +---- +curl -X GET http://localhost:8181/cxs/scopes \ +--user "mytenant:YOUR_PRIVATE_API_KEY" +---- + +[TIP] +==== +To get nicely formatted JSON responses, you can pipe the curl output through `jq`: + +[source] +---- +# Install jq if you don't have it: +# macOS: brew install jq +# Ubuntu/Debian: apt-get install jq +# CentOS/RHEL: yum install jq + +# Then add | jq '.' to any curl command: +curl -X GET http://localhost:8181/cxs/scopes \ +--user "mytenant:YOUR_PRIVATE_API_KEY" | jq '.' +---- + +For multi-line curl requests using heredoc syntax (`<<'EOF'`), you can still use jq: + +[source] +---- +# With heredoc +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d @- <<'EOF' | jq '.' +{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "requiredProfileProperties": ["*"] +} +EOF + +# With inline JSON +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20 +}' | jq '.' +---- + +Useful jq options and filters: + +[source] +---- +# Pretty print with specific indentation +curl ... | jq '.' --indent 4 + +# Only show specific fields +curl ... | jq '.metadata' + +# Show array elements on separate lines +curl ... | jq '.[]' + +# Filter and format specific data +curl ... | jq '.profiles[] | {id: .itemId, name: .properties.firstName}' + +# Sort array elements +curl ... | jq '.profiles | sort_by(.properties.lastName)' + +# Count array elements +curl ... | jq '.profiles | length' +---- +==== + ==== Retrieving your first context You can retrieve a context using curl like this : [source] ---- -curl http://localhost:8181/cxs/context.js?sessionId=1234 +curl http://localhost:8181/cxs/context.js?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" ---- This will retrieve a JavaScript script that contains a `cxs` object that contains the context with the current user profile, segments, scores as well as functions that makes it easier to perform further requests (such as collecting events using the cxs.collectEvents() function). +[NOTE] +==== +When you make a request to the `context.js` or `context.json` endpoints, Apache Unomi automatically: + +* Creates or loads a visitor profile (from cookie, parameter, or creates a new one) +* Creates or loads a visitor session (if `sessionId` is provided) +* Processes any events provided in the request (which automatically triggers rule execution) +* Resolves personalization (if requested) +* Sets a cookie with the profile ID in the response + +For detailed information about how profile tracking, session management, event processing, and rule execution work, see the <<_how_profile_tracking_works,How profile tracking works>> section. +==== + ==== Retrieving a context as a JSON object. If you prefer to retrieve a pure JSON object, you can simply use a request formed like this: [source] ---- -curl http://localhost:8181/cxs/context.json?sessionId=1234 +curl http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" ---- +The same automatic profile and session creation behavior applies to `context.json` requests. See <<_how_profile_tracking_works,How profile tracking works>> for complete details. + ==== Accessing profile properties in a context By default, in order to optimize the amount of data sent over the network, Apache Unomi will not send the content of @@ -48,12 +242,13 @@ scores ---- curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "source": { "itemId":"homepage", "itemType":"page", - "scope":"example" + "scope":"mydigital" }, "requiredProfileProperties":["*"], "requiredSessionProperties":["*"], @@ -77,25 +272,26 @@ illustrated in the following example: ---- curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "source":{ "itemId":"homepage", "itemType":"page", - "scope":"example" + "scope":"mydigital" }, "events":[ { "eventType":"view", - "scope": "example", + "scope": "mydigital", "source":{ "itemType": "site", - "scope":"example", + "scope":"mydigital", "itemId": "mysite" }, "target":{ "itemType":"page", - "scope":"example", + "scope":"mydigital", "itemId":"homepage", "properties":{ "pageInfo":{ @@ -123,21 +319,22 @@ respond quickly and minimize network traffic. Here is an example of using this s ---- curl -X POST http://localhost:8181/cxs/eventcollector \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "sessionId" : "1234", "events":[ { "eventType":"view", - "scope": "example", + "scope": "mydigital", "source":{ "itemType": "site", - "scope":"example", + "scope":"mydigital", "itemId": "mysite" }, "target":{ "itemType":"page", - "scope":"example", + "scope":"mydigital", "itemId":"homepage", "properties":{ "pageInfo":{ @@ -156,5 +353,1475 @@ to send additional events. ==== Where to go from here -* You can find more <> that can be used in the same way as the above examples. -* Read the <> documentation that contains a detailed example of how to integrate with Apache Unomi. +* You can find more <<_useful_apache_unomi_urls,useful Apache Unomi URLs>> that can be used in the same way as the above examples. +* Read the <<_twitter_sample,Twitter sample>> documentation that contains a detailed example of how to integrate with Apache Unomi. + +=== Public API Examples + +==== Sending a context request + +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d @- <<'EOF' +{ + "source":{ + "itemId":"homepage", + "itemType":"page", + "scope":"mydigital" + }, + "events":[ + { + "eventType":"view", + "scope": "mydigital", + "source":{ + "itemType": "site", + "scope":"mydigital", + "itemId": "mysite" + }, + "target":{ + "itemType":"page", + "scope":"mydigital", + "itemId":"homepage", + "properties":{ + "pageInfo":{ + "referringURL":"https://apache.org/" + } + } + } + } + ] +} +EOF +---- + +==== Collecting events + +[source] +---- +curl -X POST http://localhost:8181/cxs/eventcollector \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "sessionId" : "1234", + "events":[ + { + "eventType":"contactInfoSubmitted", + "scope": "mydigital", + "source":{ + "itemType": "site", + "scope": "mydigital", + "itemId": "mysite" + }, + "target":{ + "itemType": "form", + "scope": "mydigital", + "itemId": "contactForm" + }, + "properties" : { + "firstName": "John", + "lastName": "Doe", + "email": "john.doe@acme.com" + } + } + ] +}' +---- + +=== Private API Examples + +==== Setting up birthday personalization + +This example shows how to set up and test birthday-based personalization in Unomi. + +===== 1. Creating test profiles + +First, let's create two test profiles - one with today's birth date and another with a different date: + +[source] +---- +# Create a profile with today's birth date +curl -X POST http://localhost:8181/cxs/profiles \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "itemId": "profile-1", + "itemType": "profile", + "scope": "mydigital", + "properties": { + "firstName": "John", + "lastName": "Birthday", + "email": "john.birthday@example.com", + "birthDate": "2000-03-24", + "birthday": "03-24" + } +}' + +# Create a profile with a different birth date +curl -X POST http://localhost:8181/cxs/profiles \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "itemId": "profile-2", + "itemType": "profile", + "scope": "mydigital", + "properties": { + "firstName": "Jane", + "lastName": "Regular", + "email": "jane.regular@example.com", + "birthDate": "1995-12-31", + "birthday": "12-31" + } +}' +---- + +NOTE: The `birthday` property stores just the month and day in `MM-DD` format for easy matching, while `birthDate` stores the full date. + +===== 2. Verifying the profiles + +You can verify that both profiles were created with their birth dates: + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "exists" + } + } +}' +---- + +===== 3. Finding profiles with birthdays today + +To find all profiles whose birthday matches today's date: + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } +}' +---- + +[IMPORTANT] +==== +Replace `03-24` with the current month and day in zero-padded format. For example: +- March 24th: `03-24` +- December 31st: `12-31` + +The format is always `MM-DD` where: +- `MM` is the two-digit month (01-12) +- `DD` is the two-digit day (01-31) +==== + +You can also update the personalization example to use the birthday property: + +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "sessionId": "birthday-session", + "profileId": "profile-1", + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "requiredProfileProperties": ["properties.birthday"], + "personalizations": [ + { + "id": "birthdayMessage", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "Welcome to our site!" + }, + "contents": [ + { + "id": "birthday-content", + "path": "/birthday", + "content": "🎉 Happy Birthday! Enjoy your special day!", + "filters": [ + { + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } + } + ] + } + ] + } + ] +}' +---- + +And similarly for the birthday segment: + +[source] +---- +curl -X POST http://localhost:8181/cxs/segments \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "birthdaySegment", + "name": "Users with Birthday Today", + "scope": "mydigital" + }, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } +}' +---- + +[TIP] +==== +To make this more maintainable in a production environment, you can deploy a custom condition type that handles birthday matching: + +[source] +---- +curl -X POST http://localhost:8181/cxs/definitions \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "birthdayTodayCondition", + "name": "birthdayTodayCondition", + "description": "A condition that matches birthdays on current day", + "systemTags": [ + "profileCondition", + "demographic", + "condition" + ] + }, + "parentCondition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "parameter::monthDay" + } + }, + "parameters": [ + { + "id": "monthDay", + "type": "string", + "multivalued": false, + "defaultValue": "03-24" + } + ] +}' +---- + +After deploying the condition, you can use it in your searches and segments like this: + +[source] +---- +{ + "type": "birthdayTodayCondition", + "parameterValues": { + "monthDay": "03-24" + } +} +---- + +For example, to create a segment using this condition: + +[source] +---- +curl -X POST http://localhost:8181/cxs/segments \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "birthdaySegment", + "name": "Users with Birthday Today", + "scope": "mydigital" + }, + "condition": { + "type": "birthdayTodayCondition", + "parameterValues": { + "monthDay": "03-24" + } + } +}' +---- +==== + +===== 4. Testing personalization + +Now we can test how personalization works for both profiles. We'll use the context.json endpoint to get personalized content: + +For the birthday profile (should show birthday message): +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "sessionId": "birthday-session", + "profileId": "profile-1", + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "requiredProfileProperties": ["properties.birthday"], + "personalizations": [ + { + "id": "birthdayMessage", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "Welcome to our site!" + }, + "contents": [ + { + "id": "birthday-content", + "path": "/birthday", + "content": "🎉 Happy Birthday! Enjoy your special day!", + "filters": [ + { + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } + } + ] + } + ] + } + ] +}' +---- + +For the non-birthday profile (should show welcome message): +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "sessionId": "regular-session", + "profileId": "profile-2", + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "requiredProfileProperties": ["properties.birthday"], + "personalizations": [ + { + "id": "birthdayMessage", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "Welcome to our site!" + }, + "contents": [ + { + "id": "birthday-content", + "path": "/birthday", + "content": "🎉 Happy Birthday! Enjoy your special day!", + "filters": [ + { + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } + } + ] + } + ] + } + ] +}' +---- + +The responses will include a `personalizations` object that contains: +- For profile-1: The birthday message "🎉 Happy Birthday! Enjoy your special day!" +- For profile-2: The fallback message "Welcome to our site!" + +===== 5. Creating a birthday segment + +You can also create a segment to automatically group profiles with birthdays today: + +[source] +---- +curl -X POST http://localhost:8181/cxs/segments \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "birthdaySegment", + "name": "Users with Birthday Today", + "scope": "mydigital" + }, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthday", + "comparisonOperator": "equals", + "propertyValue": "03-24" + } + } +}' +---- + +==== Searching profiles + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset" : 0, + "limit" : 20, + "condition" : { + "type": "profilePropertyCondition", + "parameterValues" : { + "propertyName" : "properties.firstName", + "comparisonOperator" : "equals", + "propertyValue" : "John" + } + } +}' +---- + +==== Creating a segment + +[source] +---- +curl -X POST http://localhost:8181/cxs/segments \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "newSegment", + "name": "New Segment", + "scope": "mydigital" + }, + "condition": { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.age", + "comparisonOperator": "greaterThan", + "propertyValueInteger": 25 + } + } +}' +---- + +==== Setting up product view tracking + +Before using the product view search examples, you need to send product view events to Unomi. Here's how to set it up: + +===== 1. Sending a product view event + +You can use the eventcollector endpoint to send product view events: + +[source] +---- +curl -X POST http://localhost:8181/cxs/eventcollector \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "sessionId": "1234", + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "source": { + "itemType": "site", + "scope": "mydigital", + "itemId": "mysite" + }, + "target": { + "itemType": "product", + "scope": "mydigital", + "itemId": "product-123", + "properties": { + "pageInfo": { + "referringURL": "https://www.google.com" + } + } + } + } + ] +}' +---- + +Key points about the event structure: +1. Use `"eventType": "view"` for view events +2. Set `target.itemType` to `"product"` for product views +3. Include product details in `target.properties` +4. Use consistent `itemId` values to track the same product + +===== 2. Using the context.json endpoint + +For web applications, you can also send product views through the context.json endpoint: + +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234 \ +-H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-d '{ + "source": { + "itemId": "product-page", + "itemType": "page", + "scope": "mydigital" + }, + "events": [ + { + "eventType": "view", + "scope": "mydigital", + "source": { + "itemType": "site", + "scope": "mydigital", + "itemId": "mysite" + }, + "target": { + "itemType": "product", + "scope": "mydigital", + "itemId": "product-123", + "properties": { + "pageInfo": { + "referringURL": "https://www.google.com" + } + } + } + } + ], + "requiredProfileProperties": ["*"] +}' +---- + +===== 3. Product properties schema + +[IMPORTANT] +==== +To simplify product view tracking, you can deploy a custom condition type that combines all the necessary event conditions: + +[source] +---- +curl -X POST http://localhost:8181/cxs/definitions \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "productViewEventCondition", + "name": "productViewEventCondition", + "description": "A condition that matches product view events", + "systemTags": [ + "eventCondition", + "event", + "condition" + ] + }, + "parentCondition": { + "type": "booleanCondition", + "parameterValues": { + "operator": "and", + "subConditions": [ + { + "type": "eventTypeCondition", + "parameterValues": { + "eventTypeId": "view" + } + }, + { + "type": "eventPropertyCondition", + "parameterValues": { + "propertyName": "target.itemType", + "comparisonOperator": "equals", + "propertyValue": "product" + } + }, + { + "type": "eventPropertyCondition", + "parameterValues": { + "propertyName": "target.itemId", + "comparisonOperator": "equals", + "propertyValue": "parameter::productId" + } + } + ] + } + }, + "parameters": [ + { + "id": "productId", + "type": "string", + "multivalued": false + } + ] +}' +---- + +After deploying the condition, you can use it in your searches like this: + +[source] +---- +{ + "type": "productViewEventCondition", + "parameterValues": { + "productId": "product-123" + } +} +---- + +This custom condition type: +1. Is properly tagged with `eventCondition` making it valid for use in `pastEventCondition` +2. Combines all the necessary conditions using `booleanCondition` in its definition +3. Provides a simple parameter interface (just specify the product ID) +==== + +Unomi is flexible with product properties - you don't need to declare a schema beforehand. However, for consistency, you should: +1. Use consistent property names across events +2. Use consistent value types (e.g., always use numbers for prices) +3. Use consistent categories and other enumerated values + +Common product properties to consider: +- `name`: Product name (string) +- `category`: Product category (string) +- `price`: Product price (number) +- `brand`: Product brand (string) +- `sku`: Stock keeping unit (string) +- `color`: Product color (string) +- `size`: Product size (string) +- `inStock`: Stock status (boolean) + +===== 4. Testing the setup + +To verify your events are being recorded, you can: + +1. Send multiple view events for the same product +2. Wait a few seconds for processing +3. Use the profile search example below to check if the views were counted + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20, + "condition": { + "type": "pastEventCondition", + "parameterValues": { + "numberOfDays": 1, + "minimumEventCount": 1, + "eventCondition": { + "type": "productViewEventCondition", + "parameterValues": { + "productId": "product-123" + } + } + } + } +}' +---- + +==== Searching profiles with frequent product views + +This example shows how to find profiles that have viewed a specific product at least 3 times in the last 7 days: + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20, + "condition": { + "type": "pastEventCondition", + "parameterValues": { + "numberOfDays": 7, + "minimumEventCount": 3, + "eventCondition": { + "type": "productViewEventCondition", + "parameterValues": { + "productId": "product-123" + } + } + } + } +}' +---- + +This search will: +1. Look for product view events in the past 7 days +2. Match events that: + - Have type "view" + - Target a product (target.itemType = "product") + - Target the specific product ID (product-123) +3. Return profiles that have at least 3 such events +4. Results are paginated (20 results per page) + +You can adjust: +- `minimumEventCount`: change the minimum number of views required +- `maximumEventCount`: optionally set a maximum number of views +- `numberOfDays`: modify the time period to look back +- `operator`: use "eventsOccurred" (default) or "eventsNotOccurred" +- `productId`: change which product to track + +For example, to find profiles that have viewed products in a specific category, you could create another custom condition type `productCategoryViewEventCondition.json`: + +[source,json] +---- +{ + "metadata": { + "id": "productCategoryViewEventCondition", + "name": "productCategoryViewEventCondition", + "description": "A condition that matches product views in a category", + "systemTags": [ + "eventCondition", + "event", + "condition" + ] + }, + "parentCondition": { + "type": "booleanCondition", + "parameterValues": { + "operator": "and", + "subConditions": [ + { + "type": "eventTypeCondition", + "parameterValues": { + "eventTypeId": "view" + } + }, + { + "type": "eventPropertyCondition", + "parameterValues": { + "propertyName": "target.itemType", + "comparisonOperator": "equals", + "propertyValue": "product" + } + }, + { + "type": "eventPropertyCondition", + "parameterValues": { + "propertyName": "target.properties.category", + "comparisonOperator": "equals", + "propertyValue": "parameter::category" + } + } + ] + } + }, + "parameters": [ + { + "id": "category", + "type": "string", + "multivalued": false + } + ] +} +---- + +Then use it like this: + +[source] +---- +curl -X POST http://localhost:8181/cxs/profiles/search \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "offset": 0, + "limit": 20, + "condition": { + "type": "pastEventCondition", + "parameterValues": { + "numberOfDays": 30, + "minimumEventCount": 5, + "eventCondition": { + "type": "productCategoryViewEventCondition", + "parameterValues": { + "category": "electronics" + } + } + } + } +}' +---- + +=== Setting up Groovy Actions + +==== Deploying a Groovy Action + +This example shows how to deploy and use a Groovy action to automatically extract the birthday (MM-DD) from a full birthDate. + +===== 1. Create the Groovy action file + +First, create a file named `ExtractBirthdayAction.groovy` with this content: + +[source,groovy] +---- +package org.apache.unomi.groovy.actions + +import org.apache.unomi.api.Event +import org.apache.unomi.api.Profile +import org.apache.unomi.api.actions.Action +import org.apache.unomi.api.actions.ActionExecutor +import org.apache.unomi.api.services.EventService + +class ExtractBirthdayAction implements ActionExecutor { + public int execute(Action action, Event event) { + Profile profile = event.getProfile() + def birthDate = profile.getProperty("birthDate") + + if (birthDate != null && birthDate instanceof String && birthDate.length() >= 10) { + try { + // Extract month-day part (e.g., "03-24" from "2000-03-24") + def monthDay = birthDate.substring(5, 10) + + // Only update if different to avoid unnecessary saves + if (monthDay != profile.getProperty("birthday")) { + profile.setProperty("birthday", monthDay) + return EventService.PROFILE_UPDATED + } + } catch (Exception e) { + // Log error or handle invalid date format + } + } + return EventService.NO_CHANGE + } +} +---- + +===== 2. Deploy the Groovy action + +Use the Groovy actions endpoint to deploy the action: + +[source] +---- +curl -X POST http://localhost:8181/cxs/groovyActions \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: multipart/form-data" \ +-F "file=@ExtractBirthdayAction.groovy" +---- + +NOTE: The action ID will be `extractBirthday` (derived from the filename without the .groovy extension). + +===== 3. Create the action definition + +After deploying the Groovy script, create the action definition: + +[source] +---- +curl -X POST http://localhost:8181/cxs/definitions \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "extractBirthdayAction", + "name": "Extract Birthday Action", + "description": "Extracts MM-DD from birthDate and sets it as birthday property", + "systemTags": [ + "profileTags", + "demographic", + "event" + ] + }, + "actionExecutor": "groovy:extractBirthday", + "parameters": [] +}' +---- + +===== 4. Create a rule to trigger the action + +Create a rule that will trigger the Groovy action whenever a profile's birthDate is set or modified: + +[source] +---- +curl -X POST http://localhost:8181/cxs/rules \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "metadata": { + "id": "setBirthdayRule", + "name": "Set Birthday Rule", + "description": "Sets birthday property when birthDate changes", + "scope": "mydigital" + }, + "condition": { + "type": "booleanCondition", + "parameterValues": { + "operator": "and", + "subConditions": [ + { + "type": "profilePropertyCondition", + "parameterValues": { + "propertyName": "properties.birthDate", + "comparisonOperator": "exists" + } + }, + { + "type": "profileUpdatedEventCondition", + "parameterValues": { + "propertyName": "properties.birthDate" + } + } + ] + } + }, + "actions": [ + { + "type": "extractBirthdayAction", + "parameterValues": {} + } + ] +}' +---- + +===== 5. Test with example profiles + +Create test profiles to verify the action works: + +[source] +---- +# Create a profile with birthDate - should trigger the action +curl -X POST http://localhost:8181/cxs/profiles \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "itemId": "test-profile-1", + "properties": { + "firstName": "John", + "lastName": "Doe", + "birthDate": "1990-03-24" + } +}' + +# Verify the birthday property was set +curl -X GET http://localhost:8181/cxs/profiles/test-profile-1 \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" | jq '.' + +# Update an existing profile's birthDate - should trigger the action +curl -X PATCH http://localhost:8181/cxs/profiles/test-profile-1 \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "properties": { + "birthDate": "1990-12-31" + } +}' + +# Verify the birthday property was updated +curl -X GET http://localhost:8181/cxs/profiles/test-profile-1 \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" | jq '.' +---- + +The responses should show: +1. First profile creation: `birthday` property set to "03-24" +2. After update: `birthday` property changed to "12-31" + +===== 6. Remove the Groovy action (if needed) + +To remove the Groovy action: + +[source] +---- +curl -X DELETE http://localhost:8181/cxs/groovyActions/extractBirthday \ +--user "YOUR_TENANT_ID:YOUR_PRIVATE_API_KEY" \ +-H "Content-Type: application/json" +---- + +NOTE: This will only remove the Groovy script. You'll need to separately delete the action definition and rule if desired. + +[TIP] +==== +Best practices for Groovy actions: +1. Always handle potential errors gracefully +2. Check property existence and types +3. Avoid unnecessary profile updates +4. Use meaningful action and rule names +5. Test with various date formats +==== + +==== Using the explain parameter for request tracing + +Apache Unomi provides a powerful request tracing feature through the `explain` query parameter. This feature helps administrators understand how requests are processed internally, including event processing, condition evaluations, and rule executions. + +===== Prerequisites + +To use the explain parameter, you must have one of the following roles: +- ADMINISTRATOR +- TENANT_ADMINISTRATOR + +===== Request examples + +====== Context request with explain + +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234&explain=true \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d @- <<'EOF' +{ + "source": { + "itemId":"homepage", + "itemType":"page", + "scope":"mydigital" + }, + "requiredProfileProperties":["*"], + "requireSegments":true +} +EOF +---- + +====== Event collector request with explain + +[source] +---- +curl -X POST http://localhost:8181/cxs/eventcollector?explain=true \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d @- <<'EOF' +{ + "sessionId": "1234", + "events": [ + { + "eventType":"view", + "scope": "mydigital", + "source":{ + "itemType": "site", + "scope":"mydigital", + "itemId": "mysite" + }, + "target":{ + "itemType":"page", + "scope":"mydigital", + "itemId":"homepage" + } + } + ] +} +EOF +---- + +===== Understanding the trace output + +The explain parameter adds a `requestTracing` field to the response that contains a tree structure of all operations performed during request processing. Here's an example trace output: + +[source,json] +---- +{ + "profileId": "12345", + "sessionId": "1234", + // ... other response fields ... + "requestTracing": { + "operationType": "request-processing", + "description": "Processing context request", + "startTime": 1234567890, + "endTime": 1234567899, + "children": [ + { + "operationType": "event-validation", + "description": "Validating event: view", + "startTime": 1234567891, + "endTime": 1234567892, + "traces": [ + "Validating against schema event/1-0-0", + "Event validation successful" + ] + }, + { + "operationType": "rule-evaluation", + "description": "Evaluating rules for event", + "startTime": 1234567893, + "endTime": 1234567895, + "children": [ + { + "operationType": "condition-evaluation", + "description": "Evaluating condition: matchAll", + "startTime": 1234567894, + "endTime": 1234567894, + "result": true + } + ] + } + ] + } +} +---- + +===== Request processing flow + +The following diagram shows the high-level flow of request processing when explain is enabled: + +[plantuml] +---- +@startuml +participant "Client" as client +participant "ContextJsonEndpoint" as context +participant "EventsCollectorEndpoint" as collector +participant "TracerService" as tracer +participant "RequestTracer" as requestTracer + +alt Context Request + client -> context: POST /context.json?explain=true + activate context + context -> context: Check admin role + context -> tracer: enableTracing() + activate tracer + tracer -> requestTracer: setEnabled(true) + activate requestTracer + context -> context: Process request + context -> requestTracer: startOperation() + requestTracer -> requestTracer: Create trace node + context -> requestTracer: endOperation() + context -> tracer: getTraceNode() + tracer --> context: TraceNode + context --> client: ContextResponse with traces + deactivate requestTracer + deactivate tracer + deactivate context +else Event Collector Request + client -> collector: POST /eventcollector?explain=true + activate collector + collector -> collector: Check admin role + collector -> tracer: enableTracing() + activate tracer + tracer -> requestTracer: setEnabled(true) + activate requestTracer + collector -> collector: Process events + collector -> requestTracer: startOperation() + requestTracer -> requestTracer: Create trace node + collector -> requestTracer: endOperation() + collector -> tracer: getTraceNode() + tracer --> collector: TraceNode + collector --> client: EventCollectorResponse with traces + deactivate requestTracer + deactivate tracer + deactivate collector +end +@enduml +---- + +===== Common trace operations + +The tracing system captures various types of operations: + +1. Request Processing +- Overall request handling +- Parameter validation +- Schema validation + +2. Event Processing +- Event validation +- Event type resolution +- Event property processing + +3. Rule Evaluation +- Condition evaluation +- Action execution +- Score updates + +4. Profile Operations +- Profile merging +- Property updates +- Segment evaluation + +Each operation in the trace contains: +- Operation type +- Description +- Start time +- End time +- Result (if applicable) +- Child operations +- Trace messages + +===== Best practices + +1. Use explain parameter selectively +- Only enable when debugging or troubleshooting +- Disable in production environments +- Consider performance impact + +2. Analyze trace output +- Look for unexpected operations +- Check operation timing +- Review validation results +- Monitor rule evaluations + +3. Security considerations +- Only grant admin access to trusted users +- Monitor explain parameter usage +- Review trace data for sensitive information + +===== Complex personalization example with explain + +This example demonstrates using the explain parameter to understand how personalization filters are evaluated: + +[source] +---- +curl -X POST http://localhost:8181/cxs/context.json?sessionId=1234&explain=true \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ +-H "Content-Type: application/json" \ +-d @- <<'EOF' +{ + "source": { + "itemId": "homepage", + "itemType": "page", + "scope": "mydigital" + }, + "requiredProfileProperties": ["*"], + "requireSegments": true, + "personalizations": [ + { + "id": "homepage-hero", + "strategy": "matching-first", + "strategyOptions": { + "fallback": "default-content" + }, + "contents": [ + { + "id": "premium-user-content", + "filters": [ + { + "condition": { + "type": "profileSegmentCondition", + "parameterValues": { + "segments": ["premium-users"] + } + } + }, + { + "condition": { + "type": "pastEventCondition", + "parameterValues": { + "eventType": "purchase", + "minimumEventCount": 1, + "numberOfDays": 30 + } + } + } + ] + }, + { + "id": "new-visitor-content", + "filters": [ + { + "condition": { + "type": "sessionPropertyCondition", + "parameterValues": { + "propertyName": "duration", + "comparisonOperator": "lessThan", + "propertyValueInteger": 300 + } + } + } + ] + } + ] + } + ] +} +EOF +---- + +The response will include detailed tracing of the personalization evaluation process: + +[source,json] +---- +{ + "profileId": "12345", + "sessionId": "1234", + "requestTracing": { + "operationType": "request-processing", + "description": "Processing context request with personalization", + "startTime": 1234567890, + "endTime": 1234567899, + "children": [ + { + "operationType": "personalization-evaluation", + "description": "Evaluating personalization: homepage-hero", + "startTime": 1234567891, + "endTime": 1234567895, + "children": [ + { + "operationType": "content-filter-evaluation", + "description": "Evaluating filters for content: premium-user-content", + "startTime": 1234567892, + "endTime": 1234567893, + "children": [ + { + "operationType": "condition-evaluation", + "description": "Evaluating segment condition", + "startTime": 1234567892, + "endTime": 1234567892, + "result": false, + "traces": [ + "Profile not in segment: premium-users" + ] + }, + { + "operationType": "condition-evaluation", + "description": "Evaluating past event condition", + "startTime": 1234567892, + "endTime": 1234567893, + "result": false, + "traces": [ + "No purchase events found in last 30 days" + ] + } + ], + "result": false + }, + { + "operationType": "content-filter-evaluation", + "description": "Evaluating filters for content: new-visitor-content", + "startTime": 1234567893, + "endTime": 1234567894, + "children": [ + { + "operationType": "condition-evaluation", + "description": "Evaluating session duration condition", + "startTime": 1234567893, + "endTime": 1234567894, + "result": true, + "traces": [ + "Session duration: 120 seconds, threshold: 300 seconds" + ] + } + ], + "result": true + } + ] + } + ] + }, + "personalizations": { + "homepage-hero": "new-visitor-content" + } +} +---- + +This example demonstrates: + +1. Complex personalization setup +- Multiple content variants +- Different condition types +- Fallback content +- Strategy configuration + +2. Detailed tracing of +- Personalization evaluation flow +- Filter condition evaluation +- Segment membership checks +- Past event queries +- Session property checks + +3. Trace node hierarchy showing +- Parent-child relationships +- Timing information +- Decision points +- Result propagation + +The trace output helps understand: +- Why specific content was selected +- Which conditions failed/passed +- Performance of different operations +- Order of evaluation + +[plantuml] +---- +@startuml +participant "Client" as client +participant "ContextEndpoint" as context +participant "PersonalizationService" as perso +participant "TracerService" as tracer +participant "RequestTracer" as requestTracer + +client -> context: POST /context.json?explain=true +activate context + +context -> tracer: enableTracing() +activate tracer + +context -> perso: filter(profile, session, content) +activate perso + +perso -> requestTracer: startOperation("personalization-evaluation") +activate requestTracer + +loop for each content + perso -> requestTracer: startOperation("content-filter-evaluation") + + loop for each filter + perso -> requestTracer: startOperation("condition-evaluation") + perso -> perso: evaluate condition + perso -> requestTracer: trace(result details) + perso -> requestTracer: endOperation(result) + end + + perso -> requestTracer: endOperation(filter result) +end + +perso -> requestTracer: endOperation(selected content) +deactivate requestTracer + +perso --> context: PersonalizationResult +deactivate perso + +context -> tracer: getTraceNode() +tracer --> context: TraceNode + +context --> client: ContextResponse with traces +deactivate tracer +deactivate context + +@enduml +---- + +This sequence diagram shows the detailed flow of personalization evaluation with tracing enabled, including: +1. Initial request handling +2. Personalization service interaction +3. Filter evaluation loops +4. Trace node creation and updates +5. Result aggregation and response diff --git a/manual/src/main/asciidoc/samples/twitter-sample.adoc b/manual/src/main/asciidoc/samples/twitter-sample.adoc index b522d12cb..7ac2c65ec 100644 --- a/manual/src/main/asciidoc/samples/twitter-sample.adoc +++ b/manual/src/main/asciidoc/samples/twitter-sample.adoc @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[[_twitter_sample]] === Twitter sample ==== Overview @@ -214,6 +215,7 @@ Here is an example of a filter request: ---- curl --location --request POST 'http://localhost:8181/cxs/context.json' \ --header 'Content-Type: application/json' \ +--header 'X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY' \ --header 'Cookie: JSESSIONID=48C8AFB3E18B8E3C93C2F4D5B7BD43B7; context-profile-id=01060c4c-a055-4c8f-9692-8a699d0c434a' \ --data-raw '{ "source": null, @@ -283,6 +285,7 @@ Here is an example of a `personalizations` request: ---- curl --location --request POST 'http://localhost:8181/cxs/context.json' \ --header 'Content-Type: application/json' \ +--header 'X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY' \ --header 'Cookie: JSESSIONID=48C8AFB3E18B8E3C93C2F4D5B7BD43B7; context-profile-id=01060c4c-a055-4c8f-9692-8a699d0c434a' \ --data-raw '{ "source": null, diff --git a/manual/src/main/asciidoc/security.adoc b/manual/src/main/asciidoc/security.adoc new file mode 100644 index 000000000..02eb3bd83 --- /dev/null +++ b/manual/src/main/asciidoc/security.adoc @@ -0,0 +1,82 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +==== Security Architecture + +[plantuml] +---- +@startuml +skinparam componentStyle uml2 +skinparam component { + BackgroundColor<> LightYellow + BackgroundColor<> LightBlue +} + +package "Security Layer" { + [Security Service] <> + [Authentication Filter] <> + [Authorization Filter] <> + [Encryption Service] <> +} + +package "API Layer" { + [Public API] <> + [Private API] <> +} + +package "Core Services" { + [Profile Service] + [Event Service] + [Other Services] +} + +actor "Public Client" as public +actor "Admin Client" as admin + +database "Tenant Store" { + [Tenant Configuration] + [Roles & Permissions] +} + +public --> [Public API] +admin --> [Private API] + +[Public API] --> [Authentication Filter] +[Private API] --> [Authentication Filter] + +[Authentication Filter] --> [Authorization Filter] +[Authorization Filter] --> [Security Service] +[Security Service] --> [Tenant Configuration] +[Security Service] --> [Roles & Permissions] + +[Security Service] --> [Core Services] +[Encryption Service] --> [Profile Service] + +note right of [Security Service] + - JAAS Authentication + - Role-based Authorization + - Tenant Isolation + - Operation Permissions +end note + +note right of [Authentication Filter] + - Basic Auth + - API Key Auth + - Token Auth +end note + +@enduml +---- + +===== Overview \ No newline at end of file diff --git a/manual/src/main/asciidoc/shell-commands.adoc b/manual/src/main/asciidoc/shell-commands.adoc index 515d63cf9..94dd773b8 100644 --- a/manual/src/main/asciidoc/shell-commands.adoc +++ b/manual/src/main/asciidoc/shell-commands.adoc @@ -96,10 +96,11 @@ These commands are available once the application is running. If an argument is |Command|Arguments|Description |rule-list -|[maxEntries] [--csv] +|[--csv] [maxEntries] |Lists all the rules registered in the Apache Unomi server. The maxEntries (defaults to 100) will allow you to specify how many entries need to be retrieved. If the value is inferior to the total value, a message will display the total -value of rules registered in the server. If you add the "--csv" option the list will be output as a CSV formatted table +value of rules registered in the server. If you add the "--csv" option the list will be output as a CSV formatted table. +Note: Options must come before arguments, so use `rule-list --csv 100` not `rule-list 100 --csv`. |rule-view |rule-id |Dumps a single rule in JSON. The rule-id argument can be retrieved from the `rule-list` command output. @@ -131,9 +132,11 @@ created, EXECUTE means the rule's actions are being executed. |event-id |Dumps a single event in JSON. The `event-id` can be retrieved from the event-tail command output. |event-list -|[max-entries] [event-type] [--csv] +|[--csv] [max-entries] [event-type] |List the last events processed by Apache Unomi. The `max-entries` parameter can be used to control how many events are displayed (default is 100). The `event-type` makes it possible to filter the list by event type. The `--csv` argument is used to output the list as a CSV list instead of an ASCII table. +Note: Options must come before arguments, so use `event-list --csv 100 view` not `event-list 100 view --csv`. +Note: Options must come before arguments, so use `event-list --csv 100 view` not `event-list 100 view --csv`. |event-search |profile-id [event-type] [max-entries] |This command makes it possible to search for the last events by `profile-id` and by `event-type`. A `max-entries` @@ -148,7 +151,7 @@ check that everything is properly registered. If you add the "--csv" option the |Dumps a single action in JSON. The action-id argument can be retrieved from the `action-list` command output. |condition-list -|[csv] +|[--csv] |List all the conditions registered in the server. If you add the "--csv" option the list will be output as a CSV formatted table |condition-view |condition-id diff --git a/manual/src/main/asciidoc/tutorial.adoc b/manual/src/main/asciidoc/tutorial.adoc index b239c3ab3..cc5d89abe 100644 --- a/manual/src/main/asciidoc/tutorial.adoc +++ b/manual/src/main/asciidoc/tutorial.adoc @@ -95,7 +95,7 @@ You might notice the `scope` used in the snippet. All events sent to Unomi must [source,shell] ---- curl --location --request POST 'http://localhost:8181/cxs/scopes' \ - --header 'Authorization: Basic a2FyYWY6a2FyYWY=' \ + --user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "itemId": "unomi-tracker-test", @@ -106,7 +106,7 @@ curl --location --request POST 'http://localhost:8181/cxs/scopes' \ }' ---- -The authorization is the default username/password for the REST API, which is `karaf:karaf` and you that should definitely be changed as soon as possible by modifying the `etc/users.properties` file. +NOTE: Replace `TENANT_ID` and `PRIVATE_KEY` with your actual tenant ID and private API key. Only the Tenant API (`/cxs/tenants`) uses system administrator authentication (`karaf:karaf`). The default `karaf:karaf` credentials should be changed as soon as possible by modifying the `etc/users.properties` file. ==== Using tracker in your own JavaScript projects @@ -184,7 +184,7 @@ There are multiple ways to view the events that were received. For example, you [source,shell] ---- curl --location --request POST 'http://localhost:8181/cxs/events/search' \ - --header 'Authorization: Basic a2FyYWY6a2FyYWY=' \ + --user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "sortby" : "timeStamp:desc", @@ -221,7 +221,7 @@ You could also retrieve the profile details using the REST API by using a reques [source,shell] ---- curl --location --request GET 'http://localhost:8181/cxs/profiles/PROFILE_UUID' \ ---header 'Authorization: Basic a2FyYWY6a2FyYWY=' \ +--user "TENANT_ID:PRIVATE_KEY" ---- ==== Adding a rule @@ -233,7 +233,7 @@ In this example we will simply setup a basic rule that will react to the `view` [source,shell] ---- curl --location --request POST 'http://localhost:8181/cxs/rules' \ ---header 'Authorization: Basic a2FyYWY6a2FyYWY=' \ +--user "TENANT_ID:PRIVATE_KEY" \ --header 'Content-Type: application/json' \ --data-raw '{ "metadata": { diff --git a/manual/src/main/asciidoc/updating-events.adoc b/manual/src/main/asciidoc/updating-events.adoc index 7e2d52f1d..3ef158b9e 100644 --- a/manual/src/main/asciidoc/updating-events.adoc +++ b/manual/src/main/asciidoc/updating-events.adoc @@ -19,6 +19,14 @@ This can easily achieved using the `KafkaInjector` module built in within Unomi. But, as streaming system usually operates in https://dzone.com/articles/kafka-clients-at-most-once-at-least-once-exactly-o[at-least-once] semantics, we need to have a way to guarantee we wont have duplicate events in the system. +==== Authentication Requirements +To update events in Unomi, you must authenticate as either: + +* A tenant administrator +* The system administrator + +This authentication requirement ensures proper access control and security when modifying event data. + ==== Solution One of the solutions to this scenario is to have the ability to control and pass in the `eventId` property from outside of Unomi, @@ -30,6 +38,7 @@ Here is an example of a request contains the `itemdId` ---- curl -X POST http://localhost:8181/cxs/context.json \ -H "Content-Type: application/json" \ +--user "TENANT_ID:PRIVATE_KEY" \ -d @- <<'EOF' { "events":[ @@ -45,7 +54,7 @@ curl -X POST http://localhost:8181/cxs/context.json \ } EOF ---- -Make sure to use an authorized third party using `X-Unomi-Peer` requests headers and that the `eventType` is in the list of allowed events +Make sure to authenticate as either a tenant administrator or system administrator using Basic Authentication, and verify that the `eventType` is in the list of allowed events. ==== Defining Rules Another use case we support is the ability to define a rule on the above mentioned events. @@ -56,6 +65,7 @@ this can be achieved by adding `"raiseEventOnlyOnce": false` to the rule definit ---- curl -X POST http://localhost:8181/cxs/context.json \ -H "Content-Type: application/json" \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -d @- <<'EOF' { "metadata": { diff --git a/manual/src/main/asciidoc/whats-new.adoc b/manual/src/main/asciidoc/whats-new.adoc index 03398ff94..febe1479a 100644 --- a/manual/src/main/asciidoc/whats-new.adoc +++ b/manual/src/main/asciidoc/whats-new.adoc @@ -16,7 +16,7 @@ Apache Unomi 3 is a new release focused on integrations of the client to support elasticsearch version 9. It also include the upgrade of the Karaf version. -=== Elasticsearch client upgrade +==== Elasticsearch client upgrade The official client for Elasticsearch has been added to Apache Unomi in version 3.0 in order to replace the old rest-client which is not supported anymore. @@ -25,7 +25,7 @@ The documentation of the client can be found here: https://www.elastic.co/docs/r === Elasticsearch 7 data migration -A procedure to migrate your data from Elasticsearch 7 to Elasticsearch 9 can be found in the <> section +A procedure to migrate your data from Elasticsearch 7 to Elasticsearch 9 can be found in the <<_migrate_from_elasticsearch_7_to_elasticsearch_9,Migrate from Elasticsearch 7 to Elasticsearch 9>> section === Karaf upgrade @@ -34,7 +34,7 @@ This upgrade also brings support for Java 17. ==== OpenSearch Support -Starting with version 3.0.0, Apache Unomi now officially supports OpenSearch 3 as an alternative to Elasticsearch. This addition gives users more flexibility in choosing their search engine backend. Key features include: +Starting with version 3.1.0, Apache Unomi now officially supports OpenSearch 3 as an alternative to Elasticsearch. This addition gives users more flexibility in choosing their search engine backend. Key features include: - Full support for OpenSearch 3 - Seamless integration with existing Unomi features @@ -49,7 +49,7 @@ Users can choose between ElasticSearch and OpenSearch through various configurat 2. Using Maven profiles during build 3. Using Docker environment variables -For detailed configuration instructions, see the <>. +For detailed configuration instructions, see the <<_configuration,configuration section>>. ===== Security Considerations @@ -65,4 +65,4 @@ For users wanting to migrate from ElasticSearch to OpenSearch: - All existing features, including the flattened field type, are supported - Existing queries and aggregations work seamlessly with both backends -For detailed migration instructions, refer to the <>. +For detailed migration instructions, refer to the <<_migrations,migration guide>>. diff --git a/manual/src/main/asciidoc/writing-plugins.adoc b/manual/src/main/asciidoc/writing-plugins.adoc index b485f0cea..b5f0f7a8b 100644 --- a/manual/src/main/asciidoc/writing-plugins.adoc +++ b/manual/src/main/asciidoc/writing-plugins.adoc @@ -248,10 +248,12 @@ A strategy to resolve how to merge properties when merging profile together. ==== PropertyType -Definition for a profile or session property, specifying how possible values are constrained, if the value is +Definition for a profile, session, or event property, specifying how possible values are constrained, if the value is multi-valued (a vector of values as opposed to a scalar value). `PropertyType`s can also be categorized using systemTags or file system structure, using sub-directories to organize definition files. +For detailed information about property types, see <<_property_types,Property Types>>. + ==== Rule `Rule`s are conditional sets of actions to be executed in response to incoming events. Triggering of rules is guarded @@ -283,8 +285,8 @@ Definition for values that can be assigned to properties ("primitive" types). === Custom plugins Apache Unomi is a pluggeable server that may be extended in many ways. This document assumes you are familiar with the -<> . This document is mostly a reference document on the different things that may -be used inside an extension. If you are looking for complete samples, please see the <>. +<<_data_model_overview,Apache Unomi Data Model>> . This document is mostly a reference document on the different things that may +be used inside an extension. If you are looking for complete samples, please see the <<_samples,samples page>>. ==== Creating a plugin @@ -347,7 +349,7 @@ When you deploy a custom bundle with a custom definition (see "Predefined xxx" c definition will automatically be deployed at your bundle start event *if it does not exist*. After that if you redeploy the same bundle, the definition will not be redeployed, but you can redeploy it manually using the command `unomi:deploy-definition <bundleId> <fileName>` If you need to modify an existing -definition when deploying the module, see <>. +definition when deploying the module, see <<_migration_patches,Migration patches>>. ==== Predefined segments @@ -701,9 +703,8 @@ package org.apache.unomi.plugin.elasticsearch; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; - -import java.util.Map; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; public class MyCustomQueryBuilder implements ConditionESQueryBuilder { @@ -744,9 +745,8 @@ package org.apache.unomi.plugin.opensearch; import org.opensearch.client.opensearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.persistence.opensearch.ConditionOSQueryBuilder; -import org.apache.unomi.persistence.opensearch.ConditionOSQueryBuilderDispatcher; - -import java.util.Map; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; public class MyCustomQueryBuilder implements ConditionOSQueryBuilder { @@ -889,7 +889,7 @@ my-custom-plugin/ - 3.0.0 + 3.1.0 ---- @@ -995,7 +995,7 @@ my-custom-plugin/ elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:9.15.0 opensearch: - image: opensearchproject/opensearch:3.0.0 + image: opensearchproject/opensearch:3.4.0 ``` - Test with both search engines to ensure compatibility @@ -1007,6 +1007,7 @@ my-custom-plugin/ - For OpenSearch: `feature:install my-custom-plugin-opensearch` - Document dependencies and requirements clearly +[[_custom_distribution_feature]] ==== Custom Distribution Feature For production deployments, you can create a custom distribution's feature file to automatically include your plugin features in the startup configuration. This approach ensures your plugin is automatically deployed when Apache Unomi starts. @@ -1060,8 +1061,9 @@ For production deployments, you can create a custom distribution's feature file - **Production ready**: No manual feature installation required - **Version control**: Configuration can be versioned and managed with your deployment -**Note**: For more detailed information about custom distributions, including environment-specific examples, see the <> section of the documentation. +**Note**: For more detailed information about custom distributions, including environment-specific examples, see the <<_custom_distribution_feature,Custom Distribution Feature>> section or the <<_configuration,Configuration>> section of the documentation. ++ 6. **Migration from Legacy Implementations** - **DO NOT** use legacy mappings for custom query builders - Rename existing query builders to follow new naming conventions