diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index 8a027bbb95..3a428cb09e 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -68,7 +68,7 @@ jobs: echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props - MAVEN_OPTS="-Xmx3G -Xss2m" mvn clean package -Pprod 2>&1 | tee maven-build.log + MAVEN_OPTS="-Xmx3G -Xss2m -XX:MaxMetaspaceSize=1024M" mvn clean package -Pprod 2>&1 | tee maven-build.log - name: Report failing tests (if any) if: always() diff --git a/flushall_fast_build_and_run.sh b/flushall_fast_build_and_run.sh new file mode 100755 index 0000000000..a616be7b62 --- /dev/null +++ b/flushall_fast_build_and_run.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# Fast build script - skips clean, uses parallel builds, more RAM +# +# This script should be run from the OBP-API root directory: +# cd /path/to/OBP-API +# ./flushall_fast_build_and_run.sh +# +# Options: +# --clean Force a clean build (slower, but useful if you have issues) +# --offline Skip checking remote repos (faster if deps haven't changed) +# +# The http4s server will run in the background on port 8081 +# The Jetty server will run in the foreground on port 8080 + +set -e # Exit on error + +# Parse arguments +DO_CLEAN="" +OFFLINE_FLAG="" +for arg in "$@"; do + case $arg in + --clean) + DO_CLEAN="clean" + echo ">>> Clean build requested" + ;; + --offline) + OFFLINE_FLAG="-o" + echo ">>> Offline mode enabled" + ;; + esac +done + +# Detect CPU cores for parallel builds +if command -v nproc &> /dev/null; then + CORES=$(nproc) +elif command -v sysctl &> /dev/null; then + CORES=$(sysctl -n hw.ncpu) +else + CORES=4 +fi +echo ">>> Using $CORES CPU cores for parallel builds" + +# Common Maven options for better performance +# - More heap memory (4G-8G) +# - More metaspace (2G) +# - Larger stack for Scala compiler +# - Java module opens for compatibility +export MAVEN_OPTS="-Xms4G -Xmx8G -XX:MaxMetaspaceSize=2G -Xss128m \ +--add-opens java.base/java.lang=ALL-UNNAMED \ +--add-opens java.base/java.lang.reflect=ALL-UNNAMED \ +--add-opens java.base/java.util=ALL-UNNAMED \ +--add-opens java.base/java.lang.invoke=ALL-UNNAMED \ +--add-opens java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED" + +echo "==========================================" +echo "Flushing Redis cache..." +echo "==========================================" +redis-cli < http4s-server.log 2>&1 & +HTTP4S_PID=$! +echo "http4s server started with PID: $HTTP4S_PID (port 8081)" +echo "Logs are being written to: http4s-server.log" +echo "" +echo "To stop http4s server later: kill $HTTP4S_PID" +echo "" + +echo "==========================================" +echo "Starting Jetty server (foreground)..." +echo "==========================================" +mvn jetty:run -pl obp-api $OFFLINE_FLAG diff --git a/flushall_http4s_build_and_run.sh b/flushall_http4s_build_and_run.sh new file mode 100755 index 0000000000..024675536e --- /dev/null +++ b/flushall_http4s_build_and_run.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Script to flush Redis, build the project, and run the http4s server +# +# This script should be run from the OBP-API root directory: +# cd /path/to/OBP-API +# ./flushall_http4s_build_and_run.sh +# +# The http4s server will run in the foreground on the port configured +# in your props file (default: 8086) + +set -e # Exit on error + +echo "==========================================" +echo "Flushing Redis cache..." +echo "==========================================" +redis-cli < + + + + + org.tpolecat + doobie-core_${scala.version} + 1.0.0-RC4 + + + org.tpolecat + doobie-hikari_${scala.version} + 1.0.0-RC4 + com.microsoft.sqlserver mssql-jdbc diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index c0f151d836..3629b04162 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -681,6 +681,7 @@ supported_locales = en_GB,es_ES,ro_RO apiOptions.getBranchesIsPublic = true apiOptions.getAtmsIsPublic = true apiOptions.getProductsIsPublic = true +apiOptions.getApiProductsIsPublic = true apiOptions.getTransactionTypesIsPublic = true apiOptions.getCurrentFxRateIsPublic = true @@ -1613,7 +1614,7 @@ ethereum.rpc.url=http://127.0.0.1:8545 ########################################################## -# Redis Logging # +# Redis Logging Log Cache # ########################################################## ## Enable Redis logging (true/false) redis_logging_enabled = false @@ -1701,4 +1702,4 @@ securelogging_mask_email=true # Host and port for http4s server (used by bootstrap.http4s.Http4sServer) # Defaults (if not set) are 127.0.0.1 and 8086 http4s.host=127.0.0.1 -http4s.port=8086 \ No newline at end of file +http4s.port=8086 diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index c2d0e2777d..a6f2d3fe51 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -49,6 +49,8 @@ import code.api.util._ import code.api.util.migration.Migration import code.api.util.migration.Migration.DbFunction import code.apicollection.ApiCollection +import code.apiproduct.ApiProduct +import code.apiproductattribute.ApiProductAttribute import code.featuredapicollection.FeaturedApiCollection import code.apicollectionendpoint.ApiCollectionEndpoint import code.atmattribute.AtmAttribute @@ -1119,6 +1121,8 @@ object ToSchemify { MappedUserRefreshes, ApiCollection, ApiCollectionEndpoint, + ApiProduct, + ApiProductAttribute, FeaturedApiCollection, JsonSchemaValidation, AuthenticationTypeValidation, diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index d26346d1ef..6c7b6c3897 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -2870,6 +2870,26 @@ object SwaggerDefinitionsJSON { created = DateWithDayExampleObject, logo_url = Some(logoURLExample.value) ) + + lazy val consumerJsonV600: ConsumerJsonV600 = ConsumerJsonV600( + consumer_id = consumerIdExample.value, + consumer_key = consumerKeyExample.value, + app_name = appNameExample.value, + app_type = appTypeExample.value, + description = descriptionExample.value, + developer_email = emailExample.value, + company = companyExample.value, + redirect_url = redirectUrlExample.value, + certificate_pem = pem, + certificate_info = Some(certificateInfoJsonV510), + created_by_user = resourceUserJSON, + enabled = true, + created = DateWithDayExampleObject, + logo_url = Some(logoURLExample.value), + active_rate_limits = activeRateLimitsJsonV600, + call_counters = redisCallCountersJsonV600 + ) + lazy val consumerJsonOnlyForPostResponseV510: ConsumerJsonOnlyForPostResponseV510 = ConsumerJsonOnlyForPostResponseV510( consumer_id = consumerIdExample.value, consumer_key = consumerKeyExample.value, @@ -3068,6 +3088,27 @@ object SwaggerDefinitionsJSON { lazy val metricsJsonV510 = MetricsJsonV510( metrics = List(metricJson510) ) + lazy val metricJsonV600 = MetricJsonV600( + user_id = ExampleValue.userIdExample.value, + url = "www.openbankproject.com", + date = DateWithDayExampleObject, + user_name = "OBP", + app_name = "SOFI", + developer_email = ExampleValue.emailExample.value, + implemented_by_partial_function = "getBanks", + implemented_in_version = "v210", + consumer_id = "123", + verb = "get", + correlation_id = "v8ho6h5ivel3uq7a5zcnv0w1", + duration = 39, + source_ip = "2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b", + target_ip = "2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b", + response_body = json.parse("""{"code":401,"message":"OBP-20001: User not logged in. Authentication is required!"}"""), + operation_id = "OBPv4.0.0-getBanks" + ) + lazy val metricsJsonV600 = MetricsJsonV600( + metrics = List(metricJsonV600) + ) lazy val branchJsonPut = BranchJsonPutV210("gh.29.fi", "OBP", addressJsonV140, @@ -5182,6 +5223,44 @@ object SwaggerDefinitionsJSON { lazy val featuredApiCollectionJsonV600 = FeaturedApiCollectionJsonV600(featuredApiCollectionIdExample.value, apiCollectionIdExample.value, 1) lazy val featuredApiCollectionsJsonV600 = FeaturedApiCollectionsJsonV600(List(featuredApiCollectionJsonV600)) + // Api Product (v6.0.0) + lazy val apiProductAttributeResponseJsonV600 = ApiProductAttributeResponseJsonV600( + bank_id = bankIdExample.value, + api_product_code = productCodeExample.value, + api_product_attribute_id = "api-product-attribute-id-123", + name = "OVERDRAFT_LIMIT", + `type` = "STRING", + value = "10000", + is_active = Some(true) + ) + lazy val apiProductAttributeJsonV600 = ApiProductAttributeJsonV600( + name = "OVERDRAFT_LIMIT", + `type` = "STRING", + value = "10000", + is_active = Some(true) + ) + lazy val postPutApiProductJsonV600 = PostPutApiProductJsonV600( + parent_api_product_code = Some(""), + name = "ApiProduct1", + category = Some("category1"), + more_info_url = Some("https://example.com/more-info"), + terms_and_conditions_url = Some("https://example.com/terms"), + description = Some("Description of the product") + ) + lazy val apiProductJsonV600 = ApiProductJsonV600( + api_product_id = "api-product-id-123", + bank_id = bankIdExample.value, + api_product_code = productCodeExample.value, + parent_api_product_code = "", + name = "ApiProduct1", + category = "category1", + more_info_url = "https://example.com/more-info", + terms_and_conditions_url = "https://example.com/terms", + description = "Description of the product", + attributes = Some(List(apiProductAttributeResponseJsonV600)) + ) + lazy val apiProductsJsonV600 = ApiProductsJsonV600(List(apiProductJsonV600)) + lazy val jsonScalaConnectorMethod = JsonConnectorMethod(Some(connectorMethodIdExample.value),"getBank", connectorMethodBodyScalaExample.value, "Scala") lazy val jsonScalaConnectorMethodMethodBody = JsonConnectorMethodMethodBody(connectorMethodBodyScalaExample.value, "Scala") diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index 74313f4ecd..0741f6719e 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -177,6 +177,9 @@ object Redis extends MdcLoggable { jedisConnection.head.del(key).toString }else if (method ==JedisMethod.GET) { jedisConnection.head.get(key) + } else if (method == JedisMethod.SCAN) { + import scala.collection.JavaConverters._ + jedisConnection.head.keys(key).asScala.mkString(",") } else if(method ==JedisMethod.SET && value.isDefined){ if (ttlSeconds.isDefined) {//if set ttl, call `setex` method to set the expired seconds. jedisConnection.head.setex(key, ttlSeconds.get, value.get).toString diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 0f7167cbe1..5a9940ec54 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -221,6 +221,8 @@ object Constant extends MdcLoggable { final val METRICS_STABLE_NAMESPACE = "metrics_stable" final val METRICS_RECENT_NAMESPACE = "metrics_recent" final val ABAC_RULE_NAMESPACE = "abac_rule" + final val CONNECTOR_OUTBOUND_NAMESPACE = "connector_outbound" + final val CONNECTOR_INBOUND_NAMESPACE = "connector_inbound" // List of all versioned cache namespaces final val ALL_CACHE_NAMESPACES = List( @@ -234,7 +236,9 @@ object Constant extends MdcLoggable { CONNECTOR_NAMESPACE, METRICS_STABLE_NAMESPACE, METRICS_RECENT_NAMESPACE, - ABAC_RULE_NAMESPACE + ABAC_RULE_NAMESPACE, + CONNECTOR_OUTBOUND_NAMESPACE, + CONNECTOR_INBOUND_NAMESPACE ) // Cache key prefixes with global namespace and versioning for easy invalidation @@ -266,6 +270,10 @@ object Constant extends MdcLoggable { // ABAC Cache Prefixes (with global namespace and versioning) def ABAC_RULE_PREFIX: String = getVersionedCachePrefix(ABAC_RULE_NAMESPACE) + // Connector Metrics Redis Counter Prefixes (with global namespace and versioning) + def CONNECTOR_OUTBOUND_PREFIX: String = getVersionedCachePrefix(CONNECTOR_OUTBOUND_NAMESPACE) + def CONNECTOR_INBOUND_PREFIX: String = getVersionedCachePrefix(CONNECTOR_INBOUND_NAMESPACE) + // ABAC Policy Constants final val ABAC_POLICY_ACCOUNT_ACCESS = "account-access" diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 14279692fe..379fcfa7fb 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -58,6 +58,7 @@ import code.api.v1_2.ErrorMessage import code.api.v2_0_0.CreateEntitlementJSON import code.api.v2_2_0.OBPAPI2_2_0.Implementations2_2_0 import code.api.v5_1_0.OBPAPI5_1_0 +import code.api.v6_0_0.OBPAPI6_0_0 import code.authtypevalidation.AuthenticationTypeValidationProvider import code.bankconnectors.Connector import code.consumer.Consumers @@ -3403,7 +3404,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ BerlinGroupCheck.validate(body, verb, url, reqHeaders, result) } map { result => - val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root").split(",").toList + val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root,getOAuth2ServerWellKnown").split(",").toList cc.resourceDocument.map(_.partialFunctionName) match { case Some(functionName) if excludeFunctions.exists(_ == functionName) => result case _ => RateLimitingUtil.underCallLimits(result) @@ -3459,7 +3460,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ BerlinGroupCheck.validate(body, verb, url, reqHeaders, result) } map { result => - val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root").split(",").toList + val excludeFunctions = getPropsValue("rate_limiting.exclude_endpoints", "root,getOAuth2ServerWellKnown").split(",").toList cc.resourceDocument.map(_.partialFunctionName) match { case Some(functionName) if excludeFunctions.exists(_ == functionName) => result case _ => RateLimitingUtil.underCallLimits(result) @@ -4973,6 +4974,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ } val getProductsIsPublic = APIUtil.getPropsAsBoolValue("apiOptions.getProductsIsPublic", true) + val getApiProductsIsPublic = APIUtil.getPropsAsBoolValue("apiOptions.getApiProductsIsPublic", true) val createProductEntitlements = canCreateProduct :: canCreateProductAtAnyBank :: Nil @@ -5015,7 +5017,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val allowedAnswerTransactionRequestChallengeAttempts = APIUtil.getPropsAsIntValue("answer_transactionRequest_challenge_allowed_attempts").openOr(3) - lazy val allStaticResourceDocs = (OBPAPI5_1_0.allResourceDocs + lazy val allStaticResourceDocs = (OBPAPI6_0_0.allResourceDocs ++ OBP_UKOpenBanking_200.allResourceDocs ++ OBP_UKOpenBanking_310.allResourceDocs ++ code.api.Polish.v2_1_1_1.OBP_PAPI_2_1_1_1.allResourceDocs diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index 07a31d9291..e9fb9aa5d4 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -661,6 +661,23 @@ object ApiRole extends MdcLoggable{ case class CanMaintainProductCollection(requiresBankId: Boolean = true) extends ApiRole lazy val canMaintainProductCollection = CanMaintainProductCollection() + case class CanCreateApiProduct(requiresBankId: Boolean = true) extends ApiRole + lazy val canCreateApiProduct = CanCreateApiProduct() + case class CanUpdateApiProduct(requiresBankId: Boolean = true) extends ApiRole + lazy val canUpdateApiProduct = CanUpdateApiProduct() + case class CanGetApiProduct(requiresBankId: Boolean = true) extends ApiRole + lazy val canGetApiProduct = CanGetApiProduct() + case class CanDeleteApiProduct(requiresBankId: Boolean = true) extends ApiRole + lazy val canDeleteApiProduct = CanDeleteApiProduct() + case class CanCreateApiProductAttribute(requiresBankId: Boolean = true) extends ApiRole + lazy val canCreateApiProductAttribute = CanCreateApiProductAttribute() + case class CanUpdateApiProductAttribute(requiresBankId: Boolean = true) extends ApiRole + lazy val canUpdateApiProductAttribute = CanUpdateApiProductAttribute() + case class CanGetApiProductAttribute(requiresBankId: Boolean = true) extends ApiRole + lazy val canGetApiProductAttribute = CanGetApiProductAttribute() + case class CanDeleteApiProductAttribute(requiresBankId: Boolean = true) extends ApiRole + lazy val canDeleteApiProductAttribute = CanDeleteApiProductAttribute() + case class CanCreateSystemView(requiresBankId: Boolean = false) extends ApiRole lazy val canCreateSystemView = CanCreateSystemView() case class CanUpdateSystemView(requiresBankId: Boolean = false) extends ApiRole @@ -1024,6 +1041,9 @@ object ApiRole extends MdcLoggable{ case class CanGetSystemConnectorMethodNames(requiresBankId: Boolean = false) extends ApiRole lazy val canGetSystemConnectorMethodNames = CanGetSystemConnectorMethodNames() + case class CanGetConnectorNames(requiresBankId: Boolean = false) extends ApiRole + lazy val canGetConnectorNames = CanGetConnectorNames() + case class CanCreateDynamicResourceDoc(requiresBankId: Boolean = false) extends ApiRole lazy val canCreateDynamicResourceDoc = CanCreateDynamicResourceDoc() diff --git a/obp-api/src/main/scala/code/api/util/ApiTag.scala b/obp-api/src/main/scala/code/api/util/ApiTag.scala index 2f2e43c250..c257b22cd7 100644 --- a/obp-api/src/main/scala/code/api/util/ApiTag.scala +++ b/obp-api/src/main/scala/code/api/util/ApiTag.scala @@ -67,6 +67,8 @@ object ApiTag { val apiTagProduct = ResourceDocTag("Product") val apiTagProductAttribute = ResourceDocTag("Product-Attribute") val apiTagProductCollection = ResourceDocTag("Product-Collection") + val apiTagApiProduct = ResourceDocTag("Api-Product") + val apiTagApiProductAttribute = ResourceDocTag("Api-Product-Attribute") val apiTagOpenData = ResourceDocTag("Open-Data") val apiTagConsumer = ResourceDocTag("Consumer") val apiTagSearchWarehouse = ResourceDocTag("Data-Warehouse") diff --git a/obp-api/src/main/scala/code/api/util/DoobieQueries.scala b/obp-api/src/main/scala/code/api/util/DoobieQueries.scala new file mode 100644 index 0000000000..9046b8d350 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/DoobieQueries.scala @@ -0,0 +1,112 @@ +package code.api.util + +import doobie._ +import doobie.implicits._ + +/** + * Common Doobie queries used across OBP-API. + * + * These replace raw SQL queries that were using Lift's DB.runQuery (via DBUtil.runQuery). + * Doobie provides type-safe query results and proper JDBC type handling for all databases, + * including SQL Server's NVARCHAR type which Lift doesn't handle correctly. + */ +object DoobieQueries { + + /** + * Get distinct providers from the resourceuser table. + * Used by ResourceUser.getDistinctProviders + * + * @return List of distinct provider names, sorted alphabetically + */ + def getDistinctProviders: List[String] = { + val query: ConnectionIO[List[String]] = + sql"""SELECT DISTINCT provider_ FROM resourceuser ORDER BY provider_""" + .query[String] + .to[List] + + DoobieUtil.runQuery(query) + } + + /** + * Get parent IDs by bank ID from an attribute table (no filters). + * Used by AttributeQueryTrait.getParentIdByParams and NewAttributeQueryTrait.getParentIdByParams + * + * @param tableName The attribute table name + * @param parentIdColumn The column name for parent ID + * @param bankIdColumn The column name for bank ID + * @param bankId The bank ID to filter by + * @return List of distinct parent IDs + */ + def getDistinctParentIds( + tableName: String, + parentIdColumn: String, + bankIdColumn: String, + bankId: String + ): List[String] = { + // Note: We use Fragment.const for table/column names since they come from metadata + // and are not user input. The bankId value is safely parameterized. + val query: ConnectionIO[List[String]] = { + val tableF = Fragment.const(tableName) + val parentIdF = Fragment.const(parentIdColumn) + val bankIdF = Fragment.const(bankIdColumn) + + (fr"SELECT DISTINCT attr." ++ parentIdF ++ fr" FROM " ++ tableF ++ fr" attr WHERE attr." ++ bankIdF ++ fr" = $bankId") + .query[String] + .to[List] + } + + DoobieUtil.runQuery(query) + } + + /** + * Get parent IDs with attribute name/value data for filtering. + * Used by AttributeQueryTrait.getParentIdByParams and NewAttributeQueryTrait.getParentIdByParams + * + * @param tableName The attribute table name + * @param parentIdColumn The column name for parent ID + * @param nameColumn The column name for attribute name + * @param valueColumn The column name for attribute value + * @param bankIdColumn The column name for bank ID + * @param bankId The bank ID to filter by + * @param params Map of attribute name -> list of acceptable values + * @return List of (parentId, attributeName, attributeValue) tuples + */ + def getParentIdWithAttributes( + tableName: String, + parentIdColumn: String, + nameColumn: String, + valueColumn: String, + bankIdColumn: String, + bankId: String, + params: Map[String, List[String]] + ): List[(String, String, String)] = { + val paramList = params.toList + + // Build the SQL parameters filter + // e.g., (name = ? AND value = ?) OR (name = ? AND value in (?, ?, ?)) + val sqlParametersFilter = paramList.map { case (name, values) => + if (values.size == 1) { + (fr"(" ++ Fragment.const(nameColumn) ++ fr" = $name AND " ++ Fragment.const(valueColumn) ++ fr" = ${values.head})") + } else { + val valueFragments = values.map(v => fr"$v") + val inClause = valueFragments.reduceLeft((a, b) => a ++ fr"," ++ b) + (fr"(" ++ Fragment.const(nameColumn) ++ fr" = $name AND " ++ Fragment.const(valueColumn) ++ fr" IN (" ++ inClause ++ fr"))") + } + }.reduceOption((a, b) => a ++ fr" OR " ++ b).getOrElse(fr"1=1") + + val query: ConnectionIO[List[(String, String, String)]] = { + val tableF = Fragment.const(tableName) + val parentIdF = Fragment.const(parentIdColumn) + val nameF = Fragment.const(nameColumn) + val valueF = Fragment.const(valueColumn) + val bankIdF = Fragment.const(bankIdColumn) + + (fr"SELECT attr." ++ parentIdF ++ fr", attr." ++ nameF ++ fr", attr." ++ valueF ++ + fr" FROM " ++ tableF ++ fr" attr WHERE attr." ++ bankIdF ++ fr" = $bankId AND (" ++ sqlParametersFilter ++ fr")") + .query[(String, String, String)] + .to[List] + } + + DoobieUtil.runQuery(query) + } +} diff --git a/obp-api/src/main/scala/code/api/util/DoobieTransactor.scala b/obp-api/src/main/scala/code/api/util/DoobieTransactor.scala new file mode 100644 index 0000000000..fce325e9fc --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/DoobieTransactor.scala @@ -0,0 +1,149 @@ +package code.api.util + +import cats.effect.{IO, Resource} +import cats.effect.unsafe.implicits.global +import com.zaxxer.hikari.HikariConfig +import doobie._ +import doobie.hikari.HikariTransactor +import doobie.implicits._ + +import scala.concurrent.{ExecutionContext, Future} + +/** + * Doobie Transactor for OBP-API + * + * Provides a type-safe, functional JDBC layer for raw SQL queries. + * This handles all JDBC types correctly, including SQL Server's NVARCHAR (type -9) + * which Lift's DB.runQuery doesn't handle. + * + * IMPORTANT: This uses a SEPARATE HikariCP connection pool from Lift's pool. + * This ensures complete isolation and safety - Doobie manages its own connections + * without any interference with Lift's connection management. + * + * Benefits over DBUtil.runQuery: + * - Type-safe query results via case classes + * - Type-safe parameters (no SQL injection risk) + * - Proper JDBC type handling for all databases + * - Composable queries using cats-effect IO + * - Complete isolation from Lift's connection pool + * + * Usage: + * {{{ + * import doobie._ + * import doobie.implicits._ + * import code.api.util.DoobieUtil._ + * + * case class TopApi(count: Int, partialFunction: String, version: String) + * + * val query = sql""" + * SELECT count(*), implementedbypartialfunction, implementedinversion + * FROM metric + * WHERE date_c >= $fromDate + * GROUP BY implementedbypartialfunction, implementedinversion + * """.query[TopApi].to[List] + * + * val result: List[TopApi] = DoobieUtil.runQuery(query) + * }}} + */ +object DoobieUtil { + + /** + * Lazy-initialized HikariCP transactor for Doobie. + * + * This creates a separate connection pool from Lift's pool, ensuring + * complete isolation and safety. The pool is initialized on first use + * and kept alive for the lifetime of the application. + */ + private lazy val transactor: Transactor[IO] = { + val (dbUrl, dbUser, dbPassword) = DBUtil.getDbConnectionParameters + + val hikariConfig = new HikariConfig() + hikariConfig.setJdbcUrl(dbUrl) + hikariConfig.setUsername(dbUser) + hikariConfig.setPassword(dbPassword) + + // Pool configuration - conservative settings for safety + hikariConfig.setPoolName("doobie-pool") + hikariConfig.setMaximumPoolSize( + APIUtil.getPropsAsIntValue("doobie.hikari.maximumPoolSize", 10) + ) + hikariConfig.setMinimumIdle( + APIUtil.getPropsAsIntValue("doobie.hikari.minimumIdle", 2) + ) + hikariConfig.setConnectionTimeout( + APIUtil.getPropsAsLongValue("doobie.hikari.connectionTimeout", 30000L) + ) + hikariConfig.setIdleTimeout( + APIUtil.getPropsAsLongValue("doobie.hikari.idleTimeout", 600000L) + ) + hikariConfig.setMaxLifetime( + APIUtil.getPropsAsLongValue("doobie.hikari.maxLifetime", 1800000L) + ) + + // Set driver class based on database type + if (dbUrl.contains("sqlserver")) { + hikariConfig.setDriverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver") + } else if (dbUrl.contains("postgresql")) { + hikariConfig.setDriverClassName("org.postgresql.Driver") + } else if (dbUrl.contains("mysql")) { + hikariConfig.setDriverClassName("com.mysql.cj.jdbc.Driver") + } else if (dbUrl.contains("h2")) { + hikariConfig.setDriverClassName("org.h2.Driver") + } else if (dbUrl.contains("oracle")) { + hikariConfig.setDriverClassName("oracle.jdbc.OracleDriver") + } + // For other databases, HikariCP will try to auto-detect the driver + + // Create the transactor - this allocates the pool immediately + // We use unsafeRunSync here because we need a stable, long-lived transactor + HikariTransactor.fromHikariConfig[IO](hikariConfig) + .allocated + .map(_._1) // Get the transactor, discard the finalizer (pool lives for app lifetime) + .unsafeRunSync() + } + + /** + * Run a Doobie query synchronously using the dedicated Doobie connection pool. + * + * This uses a completely separate HikariCP pool from Lift's pool, + * ensuring no interference between Doobie and Lift's connection management. + * + * @param query The Doobie ConnectionIO query to execute + * @return The query result + */ + def runQuery[A](query: ConnectionIO[A]): A = { + query.transact(transactor).unsafeRunSync() + } + + /** + * Run a Doobie query asynchronously, returning a Future. + * + * @param query The Doobie ConnectionIO query to execute + * @param ec ExecutionContext for the Future + * @return Future containing the query result + */ + def runQueryAsync[A](query: ConnectionIO[A])(implicit ec: ExecutionContext): Future[A] = { + query.transact(transactor).unsafeToFuture() + } + + /** + * Run a Doobie query and return an IO. + * Useful when you want to compose with other cats-effect operations. + * + * @param query The Doobie ConnectionIO query to execute + * @return IO containing the query result + */ + def runQueryIO[A](query: ConnectionIO[A]): IO[A] = { + query.transact(transactor) + } + + /** + * Check if the database is SQL Server (for syntax differences like TOP vs LIMIT) + */ + def isSqlServer: Boolean = DBUtil.isSqlServer + + /** + * Get database URL for checking database type + */ + def dbUrl: String = DBUtil.dbUrl +} diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 1fd2dcda17..1fdf757f13 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -436,6 +436,13 @@ object ErrorMessages { val ApiCollectionEndpointAlreadyExists = "OBP-30085: The ApiCollectionEndpoint is already exists." val ApiCollectionAlreadyExists = "OBP-30086: The ApiCollection is already exists." + val ApiProductNotFound = "OBP-30500: ApiProduct not found. Please specify a valid value for BANK_ID and API_PRODUCT_CODE." + val CreateApiProductError = "OBP-30501: Could not create ApiProduct." + val DeleteApiProductError = "OBP-30502: Could not delete ApiProduct." + val ApiProductAttributeNotFound = "OBP-30503: ApiProductAttribute not found. Please specify a valid value for API_PRODUCT_ATTRIBUTE_ID." + val CreateApiProductAttributeError = "OBP-30504: Could not create ApiProductAttribute." + val DeleteApiProductAttributeError = "OBP-30505: Could not delete ApiProductAttribute." + val FeaturedApiCollectionNotFound = "OBP-30400: FeaturedApiCollection not found. Please specify a valid value for API_COLLECTION_ID." val CreateFeaturedApiCollectionError = "OBP-30401: Could not create FeaturedApiCollection." val UpdateFeaturedApiCollectionError = "OBP-30402: Could not update FeaturedApiCollection." diff --git a/obp-api/src/main/scala/code/api/util/ExampleValue.scala b/obp-api/src/main/scala/code/api/util/ExampleValue.scala index 3ad75af34a..626afd25dc 100644 --- a/obp-api/src/main/scala/code/api/util/ExampleValue.scala +++ b/obp-api/src/main/scala/code/api/util/ExampleValue.scala @@ -292,7 +292,7 @@ object ExampleValue { lazy val sortOrderExample = ConnectorField("1", "The sort order for displaying featured API collections. Lower numbers appear first.") glossaryItems += makeGlossaryItem("FeaturedApiCollection.sortOrder", sortOrderExample) - lazy val operationIdExample = ConnectorField("OBPv4.0.0-getBanks", "A uniquely identify the obp endpoint on OBP instance, you can get it from Get Resource endpoints.") + lazy val operationIdExample = ConnectorField("OBPv6.0.0-getBanks", "A uniquely identify the obp endpoint on OBP instance, you can get it from Get Resource endpoints.") glossaryItems += makeGlossaryItem("ApiCollectionEndpoint.operationId", operationIdExample) lazy val tagNameExample = ConnectorField("BankAccountTag1", "The endpoint tag name") @@ -1026,6 +1026,9 @@ object ExampleValue { lazy val relatesToKycCheckIdExample = ConnectorField(NoExampleProvided,NoDescriptionProvided) glossaryItems += makeGlossaryItem("relates_to_kyc_check_id", relatesToKycCheckIdExample) + lazy val productIdExample = ConnectorField("product-id-example-uuid", "The UUID of the product") + glossaryItems += makeGlossaryItem("product_id", productIdExample) + lazy val productCodeExample = ConnectorField("1234BW", NoDescriptionProvided) glossaryItems += makeGlossaryItem("product_code", productCodeExample) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index faf074d323..177f73d023 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -4763,6 +4763,17 @@ object Glossary extends MdcLoggable { |- ABAC_Object_Properties_Reference - Property reference |""".stripMargin) + glossaryItems += GlossaryItem( + title = "Tenancy-Model-Open-Bank-Project", + description = + s""" + |The Open Bank Project (OBP) supports multi-bank operation within a single deployment, with banks acting as the primary domain and isolation boundary. Integration behaviour can be configured per bank, including connector routing based on bank_id. + | + |For SaaS deployments requiring a "dedicated tenant", OBP typically applies tenancy at the deployment level, using separate runtimes, databases, and secrets to meet regulatory and operational isolation requirements common in banking environments. + | + |Centralised operations across multiple deployments are achieved through automated platform tooling (e.g. CI/CD, configuration management, monitoring, logging, and backups), providing a unified operational experience even when tenants are deployed separately. + |""".stripMargin) + private def getContentFromMarkdownFile(path: String): String = { val source = scala.io.Source.fromFile(path) val lines: String = try source.mkString finally source.close() diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index e9b28d6cea..ca62a40011 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -12,6 +12,8 @@ import code.api.util.APIUtil._ import code.api.util.ErrorMessages.{InsufficientAuthorisationToCreateTransactionRequest, _} import code.api.{APIFailureNewStyle, Constant, JsonResponseException} import code.apicollection.{ApiCollectionTrait, MappedApiCollectionsProvider} +import code.apiproduct.{ApiProductTrait, MappedApiProductsProvider} +import code.apiproductattribute.{ApiProductAttributeTrait, MappedApiProductAttributesProvider} import code.apicollectionendpoint.{ApiCollectionEndpointTrait, MappedApiCollectionEndpointsProvider} import code.featuredapicollection.{FeaturedApiCollectionTrait, MappedFeaturedApiCollectionsProvider} import code.atmattribute.AtmAttribute @@ -3802,6 +3804,76 @@ object NewStyle extends MdcLoggable{ } } + def getApiProductByBankIdAndCode(bankId: String, apiProductCode: String, callContext: Option[CallContext]): OBPReturnType[ApiProductTrait] = { + Future(MappedApiProductsProvider.getApiProductByBankIdAndCode(bankId, apiProductCode)) map { + i => (unboxFullOrFail(i, callContext, s"$ApiProductNotFound Current BANK_ID($bankId) API_PRODUCT_CODE($apiProductCode)"), callContext) + } + } + + def getApiProductsByBankId(bankId: String, callContext: Option[CallContext]): OBPReturnType[List[ApiProductTrait]] = { + Future(MappedApiProductsProvider.getApiProductsByBankId(bankId), callContext) + } + + def createOrUpdateApiProduct( + bankId: String, + apiProductCode: String, + parentApiProductCode: String, + name: String, + category: String, + moreInfoUrl: String, + termsAndConditionsUrl: String, + description: String, + callContext: Option[CallContext] + ): OBPReturnType[ApiProductTrait] = { + Future(MappedApiProductsProvider.createOrUpdateApiProduct( + bankId, apiProductCode, parentApiProductCode, name, category, + moreInfoUrl, termsAndConditionsUrl, description + )) map { + i => (unboxFullOrFail(i, callContext, CreateApiProductError), callContext) + } + } + + def deleteApiProduct(bankId: String, apiProductCode: String, callContext: Option[CallContext]): OBPReturnType[Boolean] = { + Future(MappedApiProductsProvider.deleteApiProduct(bankId, apiProductCode)) map { + i => (unboxFullOrFail(i, callContext, s"$DeleteApiProductError Current BANK_ID($bankId) API_PRODUCT_CODE($apiProductCode)"), callContext) + } + } + + def getApiProductAttributeById(apiProductAttributeId: String, callContext: Option[CallContext]): OBPReturnType[ApiProductAttributeTrait] = { + Future(MappedApiProductAttributesProvider.getApiProductAttributeById(apiProductAttributeId)) map { + i => (unboxFullOrFail(i, callContext, s"$ApiProductAttributeNotFound Current API_PRODUCT_ATTRIBUTE_ID($apiProductAttributeId)"), callContext) + } + } + + def getApiProductAttributesByBankIdAndCode(bankId: String, apiProductCode: String, callContext: Option[CallContext]): OBPReturnType[List[ApiProductAttributeTrait]] = { + Future(MappedApiProductAttributesProvider.getApiProductAttributesByBankIdAndCode(bankId, apiProductCode)) map { + i => (unboxFullOrFail(i, callContext, s"$ApiProductAttributeNotFound Current BANK_ID($bankId) API_PRODUCT_CODE($apiProductCode)"), callContext) + } + } + + def createOrUpdateApiProductAttribute( + bankId: String, + apiProductCode: String, + apiProductAttributeId: Option[String], + name: String, + attributeType: String, + value: String, + isActive: Option[Boolean], + callContext: Option[CallContext] + ): OBPReturnType[ApiProductAttributeTrait] = { + Future(MappedApiProductAttributesProvider.createOrUpdateApiProductAttribute( + bankId, apiProductCode, apiProductAttributeId, name, attributeType, value, isActive + )) map { + i => (unboxFullOrFail(i, callContext, CreateApiProductAttributeError), callContext) + } + } + + def deleteApiProductAttribute(apiProductAttributeId: String, callContext: Option[CallContext]): OBPReturnType[Boolean] = { + Future(MappedApiProductAttributesProvider.deleteApiProductAttribute(apiProductAttributeId)) map { + i => (unboxFullOrFail(i, callContext, s"$DeleteApiProductAttributeError Current API_PRODUCT_ATTRIBUTE_ID($apiProductAttributeId)"), callContext) + } + } + def createApiCollectionEndpoint( apiCollectionId: String, operationId: String, diff --git a/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala b/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala index 2564270ca6..4217387202 100644 --- a/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala +++ b/obp-api/src/main/scala/code/api/util/RateLimitingUtil.scala @@ -230,38 +230,39 @@ object RateLimitingUtil extends MdcLoggable { /** * Increment API call counter for a consumer after successful rate limit check. * Called after the request passes all rate limit checks to update the counters. - * + * + * Counters are ALWAYS incremented regardless of limit value. This provides visibility + * into consumer activity even when rate limiting is disabled (limit = -1), which is + * useful for monitoring which apps are active and verifying the counting infrastructure. + * * @param consumerKey The consumer ID or IP address * @param period The time period (PER_SECOND, PER_MINUTE, etc.) - * @param limit The rate limit value (-1 means disabled) - * @return (TTL in seconds, current counter value) or (-1, -1) on error/disabled + * @param limit The rate limit value (-1 means disabled, but counter still incremented) + * @return (TTL in seconds, current counter value) or (-1, -1) on Redis error */ private def incrementConsumerCounters(consumerKey: String, period: LimitCallPeriod, limit: Long): (Long, Long) = { if (useConsumerLimits) { val key = createUniqueKey(consumerKey, period) - (limit) match { - case -1 => // Limit is disabled for this period - Redis.use(JedisMethod.DELETE, key) + // Always increment counters regardless of limit value. + // This provides visibility into consumer activity even when rate limiting is disabled (limit = -1). + // Useful for monitoring which apps are active and verifying that call counting infrastructure works. + val ttlOpt = Redis.use(JedisMethod.TTL, key).map(_.toInt) + ttlOpt match { + case Some(-2) => // Key does not exist, create it + val seconds = RateLimitingPeriod.toSeconds(period).toInt + Redis.use(JedisMethod.SET, key, Some(seconds), Some("1")) + (seconds, 1) + case Some(ttl) if ttl > 0 => // Key exists with TTL, increment it + val cnt = Redis.use(JedisMethod.INCR, key).map(_.toInt).getOrElse(1) + (ttl, cnt) + case Some(ttl) if ttl <= 0 => // Key expired or has no expiry (shouldn't happen) + logger.warn(s"Unexpected TTL state ($ttl) for consumer $consumerKey, period $period - recreating counter") + val seconds = RateLimitingPeriod.toSeconds(period).toInt + Redis.use(JedisMethod.SET, key, Some(seconds), Some("1")) + (seconds, 1) + case None => // Redis unavailable + logger.error(s"Redis unavailable when incrementing counter for consumer $consumerKey, period $period") (-1, -1) - case _ => // Limit is enabled, increment counter - val ttlOpt = Redis.use(JedisMethod.TTL, key).map(_.toInt) - ttlOpt match { - case Some(-2) => // Key does not exist, create it - val seconds = RateLimitingPeriod.toSeconds(period).toInt - Redis.use(JedisMethod.SET, key, Some(seconds), Some("1")) - (seconds, 1) - case Some(ttl) if ttl > 0 => // Key exists with TTL, increment it - val cnt = Redis.use(JedisMethod.INCR, key).map(_.toInt).getOrElse(1) - (ttl, cnt) - case Some(ttl) if ttl <= 0 => // Key expired or has no expiry (shouldn't happen) - logger.warn(s"Unexpected TTL state ($ttl) for consumer $consumerKey, period $period - recreating counter") - val seconds = RateLimitingPeriod.toSeconds(period).toInt - Redis.use(JedisMethod.SET, key, Some(seconds), Some("1")) - (seconds, 1) - case None => // Redis unavailable - logger.error(s"Redis unavailable when incrementing counter for consumer $consumerKey, period $period") - (-1, -1) - } } } else { (-1, -1) @@ -360,7 +361,7 @@ object RateLimitingUtil extends MdcLoggable { } userAndCallContext._2.map(_.copy(xRateLimitLimit = limit)) .map(_.copy(xRateLimitReset = z._1)) - .map(_.copy(xRateLimitRemaining = limit - z._2)) + .map(_.copy(xRateLimitRemaining = Math.max(0, limit - z._2))) } // Helper function to set rate limit headers for anonymous access def setXRateLimitsAnonymous(id: String, z: (Long, Long), period: LimitCallPeriod): Option[CallContext] = { @@ -370,7 +371,7 @@ object RateLimitingUtil extends MdcLoggable { } userAndCallContext._2.map(_.copy(xRateLimitLimit = limit)) .map(_.copy(xRateLimitReset = z._1)) - .map(_.copy(xRateLimitRemaining = limit - z._2)) + .map(_.copy(xRateLimitRemaining = Math.max(0, limit - z._2))) } // Helper function to create rate limit exceeded response with remaining TTL for authorized users diff --git a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala index efd26036ac..141b4ea11d 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala @@ -16193,6 +16193,8 @@ trait APIMethods400 extends MdcLoggable { |* Terms and Conditions |* License the data under this endpoint is released under | + |The combination of bank_id and product_code is unique. + | |Can filter with attributes name and values. |URL params example: /banks/some-bank-id/products?&limit=50&offset=1 | @@ -16341,6 +16343,8 @@ trait APIMethods400 extends MdcLoggable { |* Attributes |* Fees | + |The combination of bank_id and product_code is unique. + | |${userAuthenticationMessage(!getProductsIsPublic)}""".stripMargin, EmptyBody, productJsonV400, diff --git a/obp-api/src/main/scala/code/api/v5_0_0/APIMethods500.scala b/obp-api/src/main/scala/code/api/v5_0_0/APIMethods500.scala index e6ceaa94f8..58bb1d8904 100644 --- a/obp-api/src/main/scala/code/api/v5_0_0/APIMethods500.scala +++ b/obp-api/src/main/scala/code/api/v5_0_0/APIMethods500.scala @@ -1674,6 +1674,7 @@ trait APIMethods500 { "Create Product", s"""Create or Update Product for the Bank. | + |The combination of bank_id and product_code is unique. If a Product already exists for the bank_id and product_code, it will be updated. | |Typical Super Family values / Asset classes are: | diff --git a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala index 3c2db50a27..4ddd59d100 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala @@ -2,8 +2,8 @@ package code.api.v6_0_0 import scala.language.reflectiveCalls import code.accountattribute.AccountAttributeX -import code.api.Constant -import code.api.{DirectLogin, ObpApiFailure} +import code.api.Constant._ +import code.api.{Constant, DirectLogin, ObpApiFailure} import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._ import code.api.cache.{Caching, Redis} import code.api.util.APIUtil._ @@ -15,13 +15,13 @@ import code.api.util.FutureUtil.EndpointContext import code.api.util.Glossary import code.api.util.JsonSchemaGenerator import code.api.util.NewStyle.HttpCode -import code.api.util.{APIUtil, CallContext, DiagnosticDynamicEntityCheck, ErrorMessages, NewStyle, OBPLimit, RateLimitingUtil} +import code.api.util.{APIUtil, ApiVersionUtils, CallContext, DiagnosticDynamicEntityCheck, ErrorMessages, NewStyle, OBPLimit, RateLimitingUtil} import net.liftweb.json import code.api.util.NewStyle.function.extractQueryParams import code.api.util.newstyle.ViewNewStyle import code.api.v3_0_0.JSONFactory300 import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson -import code.api.v2_0_0.JSONFactory200 +import code.api.v2_0_0.{BasicViewJson, JSONFactory200} import code.api.v3_1_0.{JSONFactory310, PostCustomerNumberJsonV310} import code.api.v1_2_1.{AccountHolderJSON, BankRoutingJsonV121, TransactionDetailsJSON} import code.api.v4_0_0.{BankAttributeBankResponseJsonV400, CallLimitPostJsonV400} @@ -30,11 +30,10 @@ import code.api.v5_0_0.JSONFactory500 import code.api.v5_0_0.{ViewJsonV500, ViewsJsonV500} import code.api.v5_1_0.{JSONFactory510, PostCustomerLegalNameJsonV510} import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo} -import code.api.v6_0_0.JSONFactory600.{AddUserToGroupResponseJsonV600, DynamicEntityDiagnosticsJsonV600, DynamicEntityIssueJsonV600, GroupEntitlementJsonV600, GroupEntitlementsJsonV600, GroupJsonV600, GroupsJsonV600, PostGroupJsonV600, PostGroupMembershipJsonV600, PostResetPasswordUrlJsonV600, PutGroupJsonV600, ReferenceTypeJsonV600, ReferenceTypesJsonV600, ResetPasswordUrlJsonV600, RoleWithEntitlementCountJsonV600, RolesWithEntitlementCountsJsonV600, ScannedApiVersionJsonV600, UpdateViewJsonV600, UserGroupMembershipJsonV600, UserGroupMembershipsJsonV600, ValidateUserEmailJsonV600, ValidateUserEmailResponseJsonV600, ViewJsonV600, ViewPermissionJsonV600, ViewPermissionsJsonV600, ViewsJsonV600, createAbacRuleJsonV600, createAbacRulesJsonV600, createActiveRateLimitsJsonV600, createCallLimitJsonV600, createRedisCallCountersJson, createFeaturedApiCollectionJsonV600, createFeaturedApiCollectionsJsonV600} -import code.api.v6_0_0.{AbacRuleJsonV600, AbacRuleResultJsonV600, AbacRulesJsonV600, CacheConfigJsonV600, CacheInfoJsonV600, CacheNamespaceInfoJsonV600, CreateAbacRuleJsonV600, CreateDynamicEntityRequestJsonV600, CurrentConsumerJsonV600, DynamicEntityDefinitionJsonV600, DynamicEntityDefinitionWithCountJsonV600, DynamicEntitiesWithCountJsonV600, DynamicEntityLinksJsonV600, ExecuteAbacRuleJsonV600, GetOidcClientResponseJsonV600, InMemoryCacheStatusJsonV600, MyDynamicEntitiesJsonV600, PopularApisJsonV600, PostVerifyUserCredentialsJsonV600, RedisCacheStatusJsonV600, RelatedLinkJsonV600, UpdateAbacRuleJsonV600, UpdateDynamicEntityRequestJsonV600, VerifyOidcClientRequestJsonV600, VerifyOidcClientResponseJsonV600} +import code.api.v6_0_0.JSONFactory600.{AddUserToGroupResponseJsonV600, DynamicEntityDiagnosticsJsonV600, DynamicEntityIssueJsonV600, GroupEntitlementJsonV600, GroupEntitlementsJsonV600, GroupJsonV600, GroupsJsonV600, PostGroupJsonV600, PostGroupMembershipJsonV600, PostResetPasswordUrlJsonV600, PutGroupJsonV600, ReferenceTypeJsonV600, ReferenceTypesJsonV600, ResetPasswordUrlJsonV600, RoleWithEntitlementCountJsonV600, RolesWithEntitlementCountsJsonV600, ScannedApiVersionJsonV600, UpdateViewJsonV600, UserGroupMembershipJsonV600, UserGroupMembershipsJsonV600, ValidateUserEmailJsonV600, ValidateUserEmailResponseJsonV600, ViewJsonV600, ViewPermissionJsonV600, ViewPermissionsJsonV600, ViewsJsonV600, createAbacRuleJsonV600, createAbacRulesJsonV600, createActiveRateLimitsJsonV600, createActiveRateLimitsJsonV600FromCallLimit, createCallLimitJsonV600, createConsumerJsonV600, createRedisCallCountersJson, createFeaturedApiCollectionJsonV600, createFeaturedApiCollectionsJsonV600} import code.api.v6_0_0.OBPAPI6_0_0 import code.abacrule.{AbacRuleEngine, MappedAbacRuleProvider} -import code.metrics.APIMetrics +import code.metrics.{APIMetrics, ConnectorCountsRedis} import code.bankconnectors.{Connector, LocalMappedConnectorInternal} import code.bankconnectors.storedprocedure.StoredProcedureUtils import code.bankconnectors.LocalMappedConnectorInternal._ @@ -53,6 +52,7 @@ import code.dynamicEntity.DynamicEntityCommons import code.DynamicData.{DynamicData, DynamicDataProvider} import com.github.dwickern.macros.NameOf.nameOf import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.dto.GetProductsParam import com.openbankproject.commons.model._ import com.openbankproject.commons.model.enums.DynamicEntityOperation._ import com.openbankproject.commons.model.enums.UserAttributeType @@ -221,6 +221,159 @@ trait APIMethods600 { } } + // --- GET Accounts at Bank (v6.0.0 with account_id) --- + staticResourceDocs += ResourceDoc( + getAccountsAtBank, + implementedInApiVersion, + nameOf(getAccountsAtBank), + "GET", + "/banks/BANK_ID/accounts", + "Get Accounts at Bank", + s""" + |Returns the list of accounts at BANK_ID that the user has access to. + |For each account the API returns the account ID and the views available to the user. + |Each account must have at least one private View. + | + |This v6.0.0 version returns `account_id` instead of `id` for consistency with other v6.0.0 endpoints. + | + |Optional request parameters for filtering with attributes: + |URL params example: /banks/some-bank-id/accounts?limit=50&offset=1 + | + |${userAuthenticationMessage(true)} + | + """.stripMargin, + EmptyBody, + BasicAccountsJsonV600(List(BasicAccountJsonV600( + account_id = "8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0", + bank_id = "gh.29.uk", + label = "My Account", + views_available = List(BasicViewJson("owner", "Owner", false)) + ))), + List( + $AuthenticatedUserIsRequired, + $BankNotFound, + UnknownError + ), + List(apiTagAccount, apiTagPrivateData, apiTagPublicData) + ) + + lazy val getAccountsAtBank: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "accounts" :: Nil JsonGet req => { cc => + implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), bank, callContext) <- SS.userBank + (privateViewsUserCanAccessAtOneBank, privateAccountAccess) <- Future { + Views.views.vend.privateViewsUserCanAccessAtBank(u, bankId) + } + params <- Future { + req.params + .filterNot(_._1 == PARAM_TIMESTAMP) + .filterNot(_._1 == PARAM_LOCALE) + } + privateAccountAccess2 <- + if (params.isEmpty || privateAccountAccess.isEmpty) { + Future.successful(privateAccountAccess) + } else { + AccountAttributeX.accountAttributeProvider.vend + .getAccountIdsByParams(bankId, params) + .map { boxedAccountIds => + val accountIds = boxedAccountIds.getOrElse(Nil) + privateAccountAccess.filter(aa => + accountIds.contains(aa.account_id.get) + ) + } + } + (availablePrivateAccounts, callContext2) <- bank.privateAccountsFuture( + privateAccountAccess2, + callContext + ) + } yield { + val accountsJson = availablePrivateAccounts.map { account => + val viewsAvailable = privateViewsUserCanAccessAtOneBank + .filter(v => v.bankId == bankId && v.accountId == account.accountId) + .map(v => BasicViewJson(v.viewId.value, v.name, v.isPublic)) + JSONFactory600.createBasicAccountJsonV600(account, viewsAvailable) + } + (BasicAccountsJsonV600(accountsJson), HttpCode.`200`(callContext2)) + } + } + } + + // --- GET Account by Id (Core) (v6.0.0 with account_id) --- + staticResourceDocs += ResourceDoc( + getCoreAccountByIdV600, + implementedInApiVersion, + nameOf(getCoreAccountByIdV600), + "GET", + "/my/banks/BANK_ID/accounts/ACCOUNT_ID/account", + "Get Account by Id (Core)", + s"""Information returned about the account specified by ACCOUNT_ID: + | + |* Number - The human readable account number given by the bank that identifies the account. + |* Label - A label given by the owner of the account + |* Owners - Users that own this account + |* Type - The type of account + |* Balance - Currency and Value + |* Account Routings - A list that might include IBAN or national account identifiers + |* Account Rules - A list that might include Overdraft and other bank specific rules + |* Tags - A list of Tags assigned to this account + | + |This call returns the owner view and requires access to that view. + | + |This v6.0.0 version returns `account_id` instead of `id` for consistency with other v6.0.0 endpoints. + | + |${userAuthenticationMessage(true)} + | + |""".stripMargin, + EmptyBody, + ModeratedCoreAccountJsonV600( + account_id = "8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0", + bank_id = "gh.29.uk", + label = "My Account", + number = "123456", + product_code = "CURRENT", + balance = AmountOfMoneyJsonV121("EUR", "1000.00"), + account_routings = List(AccountRoutingJsonV121("IBAN", "DE89370400440532013000")), + views_basic = List("owner") + ), + List( + $AuthenticatedUserIsRequired, + $BankAccountNotFound, + UnknownError + ), + apiTagAccount :: apiTagPSD2AIS :: apiTagPsd2 :: Nil + ) + + lazy val getCoreAccountByIdV600: OBPEndpoint = { + case "my" :: "banks" :: BankId(bankId) :: "accounts" :: AccountId(accountId) :: "account" :: Nil JsonGet req => { cc => + implicit val ec = EndpointContext(Some(cc)) + for { + (user @ Full(u), account, callContext) <- SS.userAccount + view <- ViewNewStyle.checkOwnerViewAccessAndReturnOwnerView( + u, + BankIdAccountId(account.bankId, account.accountId), + callContext + ) + moderatedAccount <- NewStyle.function.moderatedBankAccountCore( + account, + view, + user, + callContext + ) + } yield { + val availableViews: List[View] = + Views.views.vend.privateViewsUserCanAccessForAccount( + u, + BankIdAccountId(account.bankId, account.accountId) + ) + ( + JSONFactory600.createModeratedCoreAccountJsonV600(moderatedAccount, availableViews), + HttpCode.`200`(callContext) + ) + } + } + } + staticResourceDocs += ResourceDoc( getConsumerCallCounters, implementedInApiVersion, @@ -596,6 +749,62 @@ trait APIMethods600 { Some(List(canGetCurrentConsumer)) ) + staticResourceDocs += ResourceDoc( + getConsumer, + implementedInApiVersion, + nameOf(getConsumer), + "GET", + "/management/consumers/CONSUMER_ID", + "Get Consumer", + s"""Get the Consumer specified by CONSUMER_ID. + | + |This endpoint returns all consumer fields including: + |- Basic info: consumer_id, app_name, app_type, description, developer_email, company + |- OAuth: consumer_key, redirect_url + |- Status: enabled, created + |- Certificate: certificate_pem, certificate_info (subject, issuer, validity dates, PSD2 roles) + |- Branding: logo_url + |- Creator: created_by_user details + |- Rate limits: active_rate_limits showing current rate limiting configuration + |- Call counters: call_counters showing current API call usage from Redis + | + |Note: consumer_secret is never returned for security reasons. + | + |${userAuthenticationMessage(true)} + | + |""".stripMargin, + EmptyBody, + consumerJsonV600, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + ConsumerNotFoundByConsumerId, + UnknownError + ), + List(apiTagConsumer), + Some(List(canGetConsumers)) + ) + + lazy val getConsumer: OBPEndpoint = { + case "management" :: "consumers" :: consumerId :: Nil JsonGet _ => { + cc => + implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, canGetConsumers, callContext) + consumer <- NewStyle.function.getConsumerByConsumerId(consumerId, callContext) + // Get rate limits and call counters + currentConsumerCallCounters <- Future(RateLimitingUtil.consumerRateLimitState(consumer.consumerId.get).toList) + date = new java.util.Date() + (activeRateLimit, rateLimitIds) <- RateLimitingUtil.getActiveRateLimitsWithIds(consumer.consumerId.get, date) + activeRateLimitsJson = JSONFactory600.createActiveRateLimitsJsonV600FromCallLimit(activeRateLimit, rateLimitIds, date) + callCountersJson = JSONFactory600.createRedisCallCountersJson(currentConsumerCallCounters) + } yield { + (JSONFactory600.createConsumerJsonV600(consumer, None, activeRateLimitsJson, callCountersJson), HttpCode.`200`(callContext)) + } + } + } + staticResourceDocs += ResourceDoc( invalidateCacheNamespace, implementedInApiVersion, @@ -1999,6 +2208,195 @@ trait APIMethods600 { } } + staticResourceDocs += ResourceDoc( + getConnectors, + implementedInApiVersion, + nameOf(getConnectors), + "GET", + "/system/connectors", + "Get Connectors", + s"""Get the list of connectors and their availability for method routing. + | + |Returns a sorted list of all connectors with their availability status for use in Method Routing. + | + |## Response Fields + | + |* **connector_name** - The name of the connector + |* **is_available_in_method_routing** - Whether this connector can be used in Method Routing configuration. + | This depends on the `connector` and `starConnector_supported_types` props settings. + | + |## Available Connectors + | + |The OBP-API supports multiple connectors for accessing banking data: + | + |* **mapped** - Local database connector using Lift Mapper ORM + |* **akka_vDec2018** - Akka-based connector for remote banking systems + |* **rest_vMar2019** - REST connector for external APIs + |* **stored_procedure_vDec2019** - Stored procedure connector for database-native operations + |* **rabbitmq_vOct2024** - RabbitMQ message queue connector + |* **cardano_vJun2025** - Cardano blockchain connector + |* **ethereum_vSept2025** - Ethereum blockchain connector + |* **star** - Star connector (special routing connector) + |* **proxy** - Proxy connector (for testing) + |* **internal** - Internal dynamic connector + | + |## Use Case + | + |Use this endpoint to discover which connectors are available when configuring Method Routing. + |A connector is available for method routing if it matches the `connector` prop setting, + |or if `connector=star` and the connector is listed in `starConnector_supported_types`. + | + |${userAuthenticationMessage(true)} + | + |CanGetConnectorNames entitlement is required. + | + """.stripMargin, + EmptyBody, + ConnectorsJsonV600(List( + ConnectorInfoJsonV600("mapped", true), + ConnectorInfoJsonV600("akka_vDec2018", false), + ConnectorInfoJsonV600("rest_vMar2019", true), + ConnectorInfoJsonV600("stored_procedure_vDec2019", false) + )), + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + UnknownError + ), + List(apiTagConnector, apiTagSystem, apiTagApi), + Some(List(canGetConnectorNames)) + ) + + lazy val getConnectors: OBPEndpoint = { + case "system" :: "connectors" :: Nil JsonGet _ => + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, canGetConnectorNames, callContext) + } yield { + // Get the connector names from the Connector object's nameToConnector map + // Also include "star" which is handled separately in getConnectorInstance + val connectorNames = code.bankconnectors.Connector.nameToConnector.keys.toList :+ "star" + val connectorInfos = connectorNames.map { name => + ConnectorInfoJsonV600( + connector_name = name, + is_available_in_method_routing = NewStyle.function.getConnectorByName(name).isDefined + ) + } + (JSONFactory600.createConnectorsJson(connectorInfos), HttpCode.`200`(callContext)) + } + } + + staticResourceDocs += ResourceDoc( + getTopAPIs, + implementedInApiVersion, + nameOf(getTopAPIs), + "GET", + "/management/metrics/top-apis", + "Get Top APIs", + s"""Get metrics about the most popular APIs. e.g.: total count, response time (in ms), etc. + | + |This v6.0.0 version includes the **operation_id** field which uniquely identifies each API endpoint + |across different API standards (OBP, Berlin Group, UK Open Banking, etc.). + | + |Should be able to filter on the following fields: + | + |eg: /management/metrics/top-apis?from_date=$epochTimeString&to_date=$DefaultToDateString&consumer_id=5 + |&user_id=66214b8e-259e-44ad-8868-3eb47be70646&implemented_by_partial_function=getTransactionsForBankAccount + |&implemented_in_version=v3.0.0&url=/obp/v3.0.0/banks/gh.29.uk/accounts/8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0/owner/transactions + |&verb=GET&anon=false&app_name=MapperPostman + |&exclude_app_names=API-EXPLORER,API-Manager,SOFI,null + | + |1 from_date (defaults to one year ago): eg:from_date=$epochTimeString + | + |2 to_date (defaults to the current date) eg:to_date=$DefaultToDateString + | + |3 consumer_id (if null ignore) + | + |4 user_id (if null ignore) + | + |5 anon (if null ignore) only support two values: true (return where user_id is null) or false (return where user_id is not null) + | + |6 url (if null ignore), note: can not contain '&'. + | + |7 app_name (if null ignore) + | + |8 implemented_by_partial_function (if null ignore) + | + |9 implemented_in_version (if null ignore) + | + |10 verb (if null ignore) + | + |11 correlation_id (if null ignore) + | + |12 duration (if null ignore) non digit chars will be silently omitted + | + |13 exclude_app_names (if null ignore). eg: &exclude_app_names=API-EXPLORER,API-Manager,SOFI,null + | + |14 exclude_url_patterns (if null ignore). You can design your own SQL NOT LIKE pattern. eg: &exclude_url_patterns=%management/metrics%,%management/aggregate-metrics% + | + |15 exclude_implemented_by_partial_functions (if null ignore). eg: &exclude_implemented_by_partial_functions=getMetrics,getConnectorMetrics,getAggregateMetrics + | + |${userAuthenticationMessage(true)} + | + |CanReadMetrics entitlement is required. + | + """.stripMargin, + EmptyBody, + TopApisJsonV600(List( + TopApiJsonV600(1000, "getBanks", "v4.0.0", "OBPv4.0.0-getBanks"), + TopApiJsonV600(500, "getBank", "v4.0.0", "OBPv4.0.0-getBank"), + TopApiJsonV600(250, "getAccountList", "v1.3", "BGv1.3-getAccountList") + )), + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidFilterParameterFormat, + GetTopApisError, + UnknownError + ), + List(apiTagMetric, apiTagApi), + Some(List(canReadMetrics)) + ) + + lazy val getTopAPIs: OBPEndpoint = { + case "management" :: "metrics" :: "top-apis" :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, ApiRole.canReadMetrics, callContext) + httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url) + (obpQueryParams, callContext) <- createQueriesByHttpParamsFuture(httpParams, callContext) + topApis <- APIMetrics.apiMetrics.vend.getTopApisFuture(obpQueryParams) map { + unboxFullOrFail(_, callContext, GetTopApisError) + } + } yield { + // Build lookup map from partialFunctionName -> operationId + // This handles OBP, Berlin Group, UK Open Banking, and other standards correctly + val allDocs = APIUtil.getAllResourceDocs + val lookupMap: Map[String, String] = allDocs.map { doc => + doc.partialFunctionName -> doc.operationId + }.toMap + + // Convert TopApi to TopApiJsonV600 with operation_id + val topApisWithOperationId = topApis.map { api => + val operationId = lookupMap.getOrElse( + api.ImplementedByPartialFunction, + scala.util.Try(APIUtil.buildOperationId(ApiVersionUtils.valueOf(api.implementedInVersion), api.ImplementedByPartialFunction)) + .getOrElse(s"${api.implementedInVersion}-${api.ImplementedByPartialFunction}") + ) + TopApiJsonV600( + count = api.count, + implemented_by_partial_function = api.ImplementedByPartialFunction, + implemented_in_version = api.implementedInVersion, + operation_id = operationId + ) + } + (JSONFactory600.createTopApisJsonV600(topApisWithOperationId), HttpCode.`200`(callContext)) + } + } + } + staticResourceDocs += ResourceDoc( getScannedApiVersions, implementedInApiVersion, @@ -2555,7 +2953,7 @@ trait APIMethods600 { | """.stripMargin, EmptyBody, - metricsJsonV510, + metricsJsonV600, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -2593,7 +2991,12 @@ trait APIMethods600 { } } } yield { - (JSONFactory510.createMetricsJson(metrics), HttpCode.`200`(callContext)) + // Build lookup map from partialFunctionName -> operationId + val allDocs = APIUtil.getAllResourceDocs + val lookupMap: Map[String, String] = allDocs.map { doc => + doc.partialFunctionName -> doc.operationId + }.toMap + (JSONFactory600.createMetricsJsonV600(metrics, lookupMap), HttpCode.`200`(callContext)) } } } @@ -7749,24 +8152,521 @@ trait APIMethods600 { unboxFullOrFail(_, callContext, UnknownError) } } yield { - // Build lookup map from (partialFunctionName, shortVersion) -> operationId + // Build lookup map from partialFunctionName -> operationId // This handles OBP, Berlin Group, UK Open Banking, and other standards correctly val allDocs = APIUtil.getAllResourceDocs - val lookupMap: Map[(String, String), String] = allDocs.map { doc => - // Extract short version (e.g., "v4.0.0" from "OBPv4.0.0" or "v1.3" from "BGv1.3") - val shortVersion = doc.implementedInApiVersion.toString - (doc.partialFunctionName, shortVersion) -> doc.operationId + val lookupMap: Map[String, String] = allDocs.map { doc => + doc.partialFunctionName -> doc.operationId }.toMap // Convert TopApi to operation_id, looking up correct format for each standard val operationIds = topApis.flatMap { api => - lookupMap.get((api.ImplementedByPartialFunction, api.implementedInVersion)) + lookupMap.get(api.ImplementedByPartialFunction) + .orElse( + scala.util.Try(Some(APIUtil.buildOperationId(ApiVersionUtils.valueOf(api.implementedInVersion), api.ImplementedByPartialFunction))) + .getOrElse(None) + ) } (PopularApisJsonV600(operationIds), HttpCode.`200`(callContext)) } } } + staticResourceDocs += ResourceDoc( + getConnectorCallCounts, + implementedInApiVersion, + nameOf(getConnectorCallCounts), + "GET", + "/management/connector/metrics/counts", + "Get Connector Call Counts", + s"""Returns per-hour Redis counters for connector outbound and inbound messages. + | + |This provides real-time visibility into which connector methods are being called + |and how many responses (success/failure) are being received. + | + |Counters automatically reset every hour (rolling window). + |The ttl_seconds field shows when the current hour window resets. + | + |Requires the prop: write_connector_metrics_redis=true + | + |Redis key format: + | + |- Outbound (before connector call): {instance}_{env}_connector_outbound_{version}_{connectorName}_{methodName}_PER_HOUR + |- Inbound success (after connector call): {instance}_{env}_connector_inbound_{version}_{connectorName}_{methodName}_success_PER_HOUR + |- Inbound failure (after connector call): {instance}_{env}_connector_inbound_{version}_{connectorName}_{methodName}_failure_PER_HOUR + | + |For example: obp_dev_connector_outbound_1_star_getBanks_PER_HOUR + | + |Authentication is Required + | + |""".stripMargin, + EmptyBody, + ConnectorCountsJsonV600( + enabled = true, + connector_counts = List( + ConnectorCountJsonV600( + connector_name = "mapped", + method_name = "getBank", + per_hour_outbound_count = 152, + per_hour_inbound_success_count = 150, + per_hour_inbound_failure_count = 2, + ttl_seconds = 2847 + ) + ) + ), + List( + UnknownError + ), + List(apiTagMetric, apiTagApi), + Some(List(canReadMetrics)) + ) + + lazy val getConnectorCallCounts: OBPEndpoint = { + case "management" :: "connector" :: "metrics" :: "counts" :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, canReadMetrics, callContext) + } yield { + val counts = ConnectorCountsRedis.getAllCounts() + val json = ConnectorCountsJsonV600( + enabled = ConnectorCountsRedis.isEnabled, + connector_counts = counts.map(c => ConnectorCountJsonV600( + connector_name = c.connector_name, + method_name = c.method_name, + per_hour_outbound_count = c.per_hour_outbound_count, + per_hour_inbound_success_count = c.per_hour_inbound_success_count, + per_hour_inbound_failure_count = c.per_hour_inbound_failure_count, + ttl_seconds = c.ttl_seconds + )) + ) + (json, HttpCode.`200`(callContext)) + } + } + } + + // Api Product Endpoints (independent of CBS) + + staticResourceDocs += ResourceDoc( + createApiProduct, + implementedInApiVersion, + nameOf(createApiProduct), + "POST", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE", + "Create Api Product", + s"""Create an Api Product for the Bank. + | + |Authentication is Required. + | + |""".stripMargin, + postPutApiProductJsonV600, + apiProductJsonV600, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidJsonFormat, + CreateApiProductError, + UnknownError + ), + List(apiTagApi, apiTagApiProduct), + Some(List(canCreateApiProduct)) + ) + + lazy val createApiProduct: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: Nil JsonPost json -> _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canCreateApiProduct, callContext) + _ <- NewStyle.function.getBank(bankId, callContext) + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the $PostPutApiProductJsonV600", 400, callContext) { + json.extract[PostPutApiProductJsonV600] + } + (apiProduct, callContext) <- NewStyle.function.createOrUpdateApiProduct( + bankId.value, + apiProductCode, + postJson.parent_api_product_code.getOrElse(""), + postJson.name, + postJson.category.getOrElse(""), + postJson.more_info_url.getOrElse(""), + postJson.terms_and_conditions_url.getOrElse(""), + postJson.description.getOrElse(""), + callContext + ) + } yield { + (JSONFactory600.createApiProductJsonV600(apiProduct, None), HttpCode.`201`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + createOrUpdateApiProduct, + implementedInApiVersion, + nameOf(createOrUpdateApiProduct), + "PUT", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE", + "Create or Update Api Product", + s"""Create or Update an Api Product for the Bank. + | + |Authentication is Required. + | + |""".stripMargin, + postPutApiProductJsonV600, + apiProductJsonV600, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidJsonFormat, + CreateApiProductError, + UnknownError + ), + List(apiTagApi, apiTagApiProduct), + Some(List(canUpdateApiProduct)) + ) + + lazy val createOrUpdateApiProduct: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: Nil JsonPut json -> _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canUpdateApiProduct, callContext) + _ <- NewStyle.function.getBank(bankId, callContext) + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the $PostPutApiProductJsonV600", 400, callContext) { + json.extract[PostPutApiProductJsonV600] + } + (apiProduct, callContext) <- NewStyle.function.createOrUpdateApiProduct( + bankId.value, + apiProductCode, + postJson.parent_api_product_code.getOrElse(""), + postJson.name, + postJson.category.getOrElse(""), + postJson.more_info_url.getOrElse(""), + postJson.terms_and_conditions_url.getOrElse(""), + postJson.description.getOrElse(""), + callContext + ) + } yield { + (JSONFactory600.createApiProductJsonV600(apiProduct, None), HttpCode.`201`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + getApiProduct, + implementedInApiVersion, + nameOf(getApiProduct), + "GET", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE", + "Get Api Product", + s"""Get an Api Product by BANK_ID and API_PRODUCT_CODE. + | + |Returns the Api Product with its attributes. + | + |${userAuthenticationMessage(!getApiProductsIsPublic)} + | + |""".stripMargin, + EmptyBody, + apiProductJsonV600, + if (getApiProductsIsPublic) List(ApiProductNotFound, UnknownError) else List(UserHasMissingRoles, ApiProductNotFound, UnknownError), + List(apiTagApi, apiTagApiProduct), + if (getApiProductsIsPublic) None else Some(List(canGetApiProduct)) + ) + + lazy val getApiProduct: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (_, callContext) <- getApiProductsIsPublic match { + case false => authenticatedAccess(cc) + case true => anonymousAccess(cc) + } + _ <- if (!getApiProductsIsPublic) { + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canGetApiProduct, callContext) + } yield callContext + } else { + Future.successful(callContext) + } + _ <- NewStyle.function.getBank(bankId, callContext) + (apiProduct, callContext) <- NewStyle.function.getApiProductByBankIdAndCode(bankId.value, apiProductCode, callContext) + (attributes, callContext) <- NewStyle.function.getApiProductAttributesByBankIdAndCode(bankId.value, apiProductCode, callContext) + } yield { + (JSONFactory600.createApiProductJsonV600(apiProduct, Some(attributes)), HttpCode.`200`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + getApiProducts, + implementedInApiVersion, + nameOf(getApiProducts), + "GET", + "/banks/BANK_ID/api-products", + "Get Api Products", + s"""Get Api Products for the Bank. + | + |${userAuthenticationMessage(!getApiProductsIsPublic)} + | + |""".stripMargin, + EmptyBody, + apiProductsJsonV600, + if (getApiProductsIsPublic) List(UnknownError) else List(UserHasMissingRoles, UnknownError), + List(apiTagApi, apiTagApiProduct), + if (getApiProductsIsPublic) None else Some(List(canGetApiProduct)) + ) + + lazy val getApiProducts: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (_, callContext) <- getApiProductsIsPublic match { + case false => authenticatedAccess(cc) + case true => anonymousAccess(cc) + } + _ <- if (!getApiProductsIsPublic) { + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canGetApiProduct, callContext) + } yield callContext + } else { + Future.successful(callContext) + } + _ <- NewStyle.function.getBank(bankId, callContext) + (apiProducts, callContext) <- NewStyle.function.getApiProductsByBankId(bankId.value, callContext) + } yield { + (JSONFactory600.createApiProductsJsonV600(apiProducts), HttpCode.`200`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + deleteApiProduct, + implementedInApiVersion, + nameOf(deleteApiProduct), + "DELETE", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE", + "Delete Api Product", + s"""Delete an Api Product by BANK_ID and API_PRODUCT_CODE. + | + |Authentication is Required. + | + |""".stripMargin, + EmptyBody, + EmptyBody, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + ApiProductNotFound, + DeleteApiProductError, + UnknownError + ), + List(apiTagApi, apiTagApiProduct), + Some(List(canDeleteApiProduct)) + ) + + lazy val deleteApiProduct: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: Nil JsonDelete _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canDeleteApiProduct, callContext) + _ <- NewStyle.function.getBank(bankId, callContext) + (_, callContext) <- NewStyle.function.deleteApiProduct(bankId.value, apiProductCode, callContext) + } yield { + (Full(true), HttpCode.`204`(callContext)) + } + } + } + + // Api Product Attribute Endpoints + + staticResourceDocs += ResourceDoc( + createApiProductAttribute, + implementedInApiVersion, + nameOf(createApiProductAttribute), + "POST", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE/attribute", + "Create Api Product Attribute", + s"""Create an Api Product Attribute. + | + |Authentication is Required. + | + |""".stripMargin, + apiProductAttributeJsonV600, + apiProductAttributeResponseJsonV600, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidJsonFormat, + ApiProductNotFound, + CreateApiProductAttributeError, + UnknownError + ), + List(apiTagApi, apiTagApiProductAttribute), + Some(List(canCreateApiProductAttribute)) + ) + + lazy val createApiProductAttribute: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: "attribute" :: Nil JsonPost json -> _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canCreateApiProductAttribute, callContext) + _ <- NewStyle.function.getBank(bankId, callContext) + _ <- NewStyle.function.getApiProductByBankIdAndCode(bankId.value, apiProductCode, callContext) + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the $ApiProductAttributeJsonV600", 400, callContext) { + json.extract[ApiProductAttributeJsonV600] + } + (attribute, callContext) <- NewStyle.function.createOrUpdateApiProductAttribute( + bankId.value, + apiProductCode, + None, + postJson.name, + postJson.`type`, + postJson.value, + postJson.is_active, + callContext + ) + } yield { + (JSONFactory600.createApiProductAttributeResponseJsonV600(attribute), HttpCode.`201`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + updateApiProductAttribute, + implementedInApiVersion, + nameOf(updateApiProductAttribute), + "PUT", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE/attributes/API_PRODUCT_ATTRIBUTE_ID", + "Update Api Product Attribute", + s"""Update an Api Product Attribute. + | + |Authentication is Required. + | + |""".stripMargin, + apiProductAttributeJsonV600, + apiProductAttributeResponseJsonV600, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + InvalidJsonFormat, + ApiProductNotFound, + ApiProductAttributeNotFound, + UnknownError + ), + List(apiTagApi, apiTagApiProductAttribute), + Some(List(canUpdateApiProductAttribute)) + ) + + lazy val updateApiProductAttribute: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: "attributes" :: apiProductAttributeId :: Nil JsonPut json -> _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canUpdateApiProductAttribute, callContext) + _ <- NewStyle.function.getBank(bankId, callContext) + _ <- NewStyle.function.getApiProductByBankIdAndCode(bankId.value, apiProductCode, callContext) + postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the $ApiProductAttributeJsonV600", 400, callContext) { + json.extract[ApiProductAttributeJsonV600] + } + (attribute, callContext) <- NewStyle.function.createOrUpdateApiProductAttribute( + bankId.value, + apiProductCode, + Some(apiProductAttributeId), + postJson.name, + postJson.`type`, + postJson.value, + postJson.is_active, + callContext + ) + } yield { + (JSONFactory600.createApiProductAttributeResponseJsonV600(attribute), HttpCode.`200`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + getApiProductAttribute, + implementedInApiVersion, + nameOf(getApiProductAttribute), + "GET", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE/attributes/API_PRODUCT_ATTRIBUTE_ID", + "Get Api Product Attribute", + s"""Get an Api Product Attribute by API_PRODUCT_ATTRIBUTE_ID. + | + |${userAuthenticationMessage(!getApiProductsIsPublic)} + | + |""".stripMargin, + EmptyBody, + apiProductAttributeResponseJsonV600, + if (getApiProductsIsPublic) List(ApiProductAttributeNotFound, UnknownError) else List(UserHasMissingRoles, ApiProductAttributeNotFound, UnknownError), + List(apiTagApi, apiTagApiProductAttribute), + if (getApiProductsIsPublic) None else Some(List(canGetApiProductAttribute)) + ) + + lazy val getApiProductAttribute: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: "attributes" :: apiProductAttributeId :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (_, callContext) <- getApiProductsIsPublic match { + case false => authenticatedAccess(cc) + case true => anonymousAccess(cc) + } + _ <- if (!getApiProductsIsPublic) { + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canGetApiProductAttribute, callContext) + } yield callContext + } else { + Future.successful(callContext) + } + (attribute, callContext) <- NewStyle.function.getApiProductAttributeById(apiProductAttributeId, callContext) + } yield { + (JSONFactory600.createApiProductAttributeResponseJsonV600(attribute), HttpCode.`200`(callContext)) + } + } + } + + staticResourceDocs += ResourceDoc( + deleteApiProductAttribute, + implementedInApiVersion, + nameOf(deleteApiProductAttribute), + "DELETE", + "/banks/BANK_ID/api-products/API_PRODUCT_CODE/attributes/API_PRODUCT_ATTRIBUTE_ID", + "Delete Api Product Attribute", + s"""Delete an Api Product Attribute by API_PRODUCT_ATTRIBUTE_ID. + | + |Authentication is Required. + | + |""".stripMargin, + EmptyBody, + EmptyBody, + List( + $AuthenticatedUserIsRequired, + UserHasMissingRoles, + ApiProductAttributeNotFound, + DeleteApiProductAttributeError, + UnknownError + ), + List(apiTagApi, apiTagApiProductAttribute), + Some(List(canDeleteApiProductAttribute)) + ) + + lazy val deleteApiProductAttribute: OBPEndpoint = { + case "banks" :: BankId(bankId) :: "api-products" :: apiProductCode :: "attributes" :: apiProductAttributeId :: Nil JsonDelete _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement(bankId.value, u.userId, canDeleteApiProductAttribute, callContext) + (_, callContext) <- NewStyle.function.deleteApiProductAttribute(apiProductAttributeId, callContext) + } yield { + (Full(true), HttpCode.`204`(callContext)) + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 17525237d6..fdddf73f5a 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -17,8 +17,8 @@ import code.api.util.APIUtil.stringOrNull import code.api.util.RateLimitingPeriod.LimitCallPeriod import code.api.util._ import code.api.v1_2_1.{AccountHolderJSON, BankRoutingJsonV121, OtherAccountMetadataJSON, TransactionDetailsJSON, TransactionMetadataJSON} -import code.api.v1_4_0.JSONFactory1_4_0.CustomerFaceImageJson -import code.api.v2_0_0.{EntitlementJSONs, JSONFactory200} +import code.api.v1_4_0.JSONFactory1_4_0.{CustomerFaceImageJson, MetaJsonV140, createMetaJson} +import code.api.v2_0_0.{BasicViewJson, EntitlementJSONs, JSONFactory200} import code.api.v2_1_0.CustomerCreditRatingJSON import code.api.v3_0_0.{ CustomerAttributeResponseJsonV300, @@ -27,12 +27,15 @@ import code.api.v3_0_0.{ ViewJSON300, ViewsJSON300 } -import code.api.v3_1_0.{AccountAttributeResponseJson, RateLimit, RedisCallLimitJson} -import code.api.v4_0_0.TransactionAttributeResponseJson -import code.api.v4_0_0.{BankAttributeBankResponseJsonV400, UserAgreementJson} +import code.api.v3_1_0.{AccountAttributeResponseJson, ProductAttributeResponseWithoutBankIdJson, RateLimit, RedisCallLimitJson} +import code.api.v3_1_0.JSONFactory310.createProductAttributesJson +import code.api.v4_0_0.{BankAttributeBankResponseJsonV400, ProductFeeJsonV400, ProductFeeValueJsonV400, TransactionAttributeResponseJson, UserAgreementJson} import code.entitlement.Entitlement +import code.apiproduct.ApiProductTrait +import code.apiproductattribute.ApiProductAttributeTrait import code.featuredapicollection.FeaturedApiCollectionTrait import code.loginattempts.LoginAttempt +import code.model.ModeratedBankAccountCore import code.model.dataAccess.ResourceUser import code.users.UserAgreement import code.util.Helper.MdcLoggable @@ -80,6 +83,26 @@ case class CurrentConsumerJsonV600( call_counters: RedisCallCountersJsonV600 ) +// Full Consumer details for management endpoints (V600) +case class ConsumerJsonV600( + consumer_id: String, + consumer_key: String, + app_name: String, + app_type: String, + description: String, + developer_email: String, + company: String, + redirect_url: String, + certificate_pem: String, + certificate_info: Option[code.api.v5_1_0.CertificateInfoJsonV510], + created_by_user: code.api.v2_1_0.ResourceUserJSON, + enabled: Boolean, + created: Date, + logo_url: Option[String], + active_rate_limits: ActiveRateLimitsJsonV600, + call_counters: RedisCallCountersJsonV600 +) + // OIDC Client Verification models (V600) case class VerifyOidcClientRequestJsonV600( client_id: String, @@ -278,6 +301,66 @@ case class ProvidersJsonV600(providers: List[String]) case class ConnectorMethodNamesJsonV600(connector_method_names: List[String]) +case class ConnectorInfoJsonV600( + connector_name: String, + is_available_in_method_routing: Boolean +) + +case class ConnectorsJsonV600(connectors: List[ConnectorInfoJsonV600]) + +// Basic Account with account_id instead of id for v6.0.0 consistency +case class BasicAccountJsonV600( + account_id: String, + bank_id: String, + label: String, + views_available: List[BasicViewJson] +) + +case class BasicAccountsJsonV600( + accounts: List[BasicAccountJsonV600] +) + +// Moderated Core Account with account_id instead of id for v6.0.0 consistency +case class ModeratedCoreAccountJsonV600( + account_id: String, + bank_id: String, + label: String, + number: String, + product_code: String, + balance: AmountOfMoneyJsonV121, + account_routings: List[AccountRoutingJsonV121], + views_basic: List[String] +) + +case class TopApiJsonV600( + count: Int, + implemented_by_partial_function: String, + implemented_in_version: String, + operation_id: String +) + +case class TopApisJsonV600(top_apis: List[TopApiJsonV600]) + +case class MetricJsonV600( + user_id: String, + url: String, + date: Date, + user_name: String, + app_name: String, + developer_email: String, + implemented_by_partial_function: String, + implemented_in_version: String, + consumer_id: String, + verb: String, + correlation_id: String, + duration: Long, + source_ip: String, + target_ip: String, + response_body: net.liftweb.json.JValue, + operation_id: String +) +case class MetricsJsonV600(metrics: List[MetricJsonV600]) + case class CacheNamespaceJsonV600( prefix: String, description: String, @@ -660,6 +743,62 @@ case class PopularApisJsonV600( operation_ids: List[String] ) +case class ConnectorCountJsonV600( + connector_name: String, + method_name: String, + per_hour_outbound_count: Long, + per_hour_inbound_success_count: Long, + per_hour_inbound_failure_count: Long, + ttl_seconds: Long +) + +case class ConnectorCountsJsonV600( + enabled: Boolean, + connector_counts: List[ConnectorCountJsonV600] +) + +// Api Product (independent of CBS) +case class PostPutApiProductJsonV600( + parent_api_product_code: Option[String], + name: String, + category: Option[String], + more_info_url: Option[String], + terms_and_conditions_url: Option[String], + description: Option[String] +) + +case class ApiProductJsonV600( + api_product_id: String, + bank_id: String, + api_product_code: String, + parent_api_product_code: String, + name: String, + category: String, + more_info_url: String, + terms_and_conditions_url: String, + description: String, + attributes: Option[List[ApiProductAttributeResponseJsonV600]] +) + +case class ApiProductsJsonV600(api_products: List[ApiProductJsonV600]) + +case class ApiProductAttributeJsonV600( + name: String, + `type`: String, + value: String, + is_active: Option[Boolean] +) + +case class ApiProductAttributeResponseJsonV600( + bank_id: String, + api_product_code: String, + api_product_attribute_id: String, + name: String, + `type`: String, + value: String, + is_active: Option[Boolean] +) + object JSONFactory600 extends CustomJsonFormats with MdcLoggable { def createRedisCallCountersJson( @@ -688,6 +827,43 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { ) } + def createConsumerJsonV600( + c: code.model.Consumer, + certificateInfo: Option[code.api.v5_1_0.CertificateInfoJsonV510], + activeRateLimits: ActiveRateLimitsJsonV600, + callCounters: RedisCallCountersJsonV600 + ): ConsumerJsonV600 = { + val resourceUserJSON = code.users.Users.users.vend.getUserByUserId(c.createdByUserId.toString()) match { + case net.liftweb.common.Full(resourceUser) => code.api.v2_1_0.ResourceUserJSON( + user_id = resourceUser.userId, + email = resourceUser.emailAddress, + provider_id = resourceUser.idGivenByProvider, + provider = resourceUser.provider, + username = resourceUser.name + ) + case _ => null + } + + ConsumerJsonV600( + consumer_id = c.consumerId.get, + consumer_key = c.key.get, + app_name = c.name.get, + app_type = c.appType.toString(), + description = c.description.get, + developer_email = c.developerEmail.get, + company = c.company.get, + redirect_url = c.redirectURL.get, + certificate_pem = c.clientCertificate.get, + certificate_info = certificateInfo, + created_by_user = resourceUserJSON, + enabled = c.isActive.get, + created = c.createdAt.get, + logo_url = if (c.logoUrl.get == null || c.logoUrl.get.isEmpty) None else Some(c.logoUrl.get), + active_rate_limits = activeRateLimits, + call_counters = callCounters + ) + } + def createUserInfoJSON( current_user: UserV600, onBehalfOfUser: Option[UserV600] @@ -883,6 +1059,82 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { ConnectorMethodNamesJsonV600(methodNames.sorted) } + def createConnectorsJson( + connectorInfos: List[ConnectorInfoJsonV600] + ): ConnectorsJsonV600 = { + ConnectorsJsonV600(connectorInfos.sortBy(_.connector_name)) + } + + def createBasicAccountJsonV600(account: BankAccount, viewsAvailable: List[BasicViewJson]): BasicAccountJsonV600 = { + BasicAccountJsonV600( + account_id = account.accountId.value, + bank_id = account.bankId.value, + label = account.label, + views_available = viewsAvailable + ) + } + + def createBasicAccountsJsonV600(accounts: List[BasicAccountJsonV600]): BasicAccountsJsonV600 = { + BasicAccountsJsonV600(accounts) + } + + def createModeratedCoreAccountJsonV600( + account: ModeratedBankAccountCore, + availableViews: List[View] + ): ModeratedCoreAccountJsonV600 = { + ModeratedCoreAccountJsonV600( + account_id = account.accountId.value, + bank_id = account.bankId.value, + label = account.label.getOrElse(""), + number = account.number.getOrElse(""), + product_code = account.accountType.getOrElse(""), + balance = AmountOfMoneyJsonV121( + account.currency.getOrElse(""), + account.balance.getOrElse("").toString + ), + account_routings = account.accountRoutings.map(r => + AccountRoutingJsonV121(scheme = r.scheme, address = r.address) + ), + views_basic = availableViews.map(_.viewId.value) + ) + } + + def createTopApisJsonV600( + topApis: List[TopApiJsonV600] + ): TopApisJsonV600 = { + TopApisJsonV600(topApis) + } + + def createMetricJsonV600(metric: code.metrics.APIMetric, lookupMap: Map[String, String]): MetricJsonV600 = { + val operationId = lookupMap.getOrElse( + metric.getImplementedByPartialFunction(), + scala.util.Try(code.api.util.APIUtil.buildOperationId(code.api.util.ApiVersionUtils.valueOf(metric.getImplementedInVersion()), metric.getImplementedByPartialFunction())) + .getOrElse(s"${metric.getImplementedInVersion()}-${metric.getImplementedByPartialFunction()}") + ) + MetricJsonV600( + user_id = metric.getUserId(), + user_name = metric.getUserName(), + developer_email = metric.getDeveloperEmail(), + app_name = metric.getAppName(), + url = metric.getUrl(), + date = metric.getDate(), + consumer_id = metric.getConsumerId(), + verb = metric.getVerb(), + implemented_in_version = metric.getImplementedInVersion(), + implemented_by_partial_function = metric.getImplementedByPartialFunction(), + correlation_id = metric.getCorrelationId(), + duration = metric.getDuration(), + source_ip = metric.getSourceIp(), + target_ip = metric.getTargetIp(), + response_body = net.liftweb.json.parseOpt(metric.getResponseBody()).getOrElse(net.liftweb.json.JString("Not enabled")), + operation_id = operationId + ) + } + + def createMetricsJsonV600(metrics: List[code.metrics.APIMetric], lookupMap: Map[String, String]): MetricsJsonV600 = { + MetricsJsonV600(metrics.map(createMetricJsonV600(_, lookupMap))) + } + def createBankJSON600( bank: Bank, attributes: List[BankAttributeTrait] = Nil @@ -1768,4 +2020,42 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { ) } + def createApiProductAttributeResponseJsonV600( + attribute: ApiProductAttributeTrait + ): ApiProductAttributeResponseJsonV600 = { + ApiProductAttributeResponseJsonV600( + bank_id = attribute.bankId, + api_product_code = attribute.apiProductCode, + api_product_attribute_id = attribute.apiProductAttributeId, + name = attribute.name, + `type` = attribute.attributeType, + value = attribute.value, + is_active = attribute.isActive + ) + } + + def createApiProductJsonV600( + product: ApiProductTrait, + attributes: Option[List[ApiProductAttributeTrait]] + ): ApiProductJsonV600 = { + ApiProductJsonV600( + api_product_id = product.apiProductId, + bank_id = product.bankId, + api_product_code = product.apiProductCode, + parent_api_product_code = product.parentApiProductCode, + name = product.name, + category = product.category, + more_info_url = product.moreInfoUrl, + terms_and_conditions_url = product.termsAndConditionsUrl, + description = product.description, + attributes = attributes.map(_.map(createApiProductAttributeResponseJsonV600)) + ) + } + + def createApiProductsJsonV600( + products: List[ApiProductTrait] + ): ApiProductsJsonV600 = { + ApiProductsJsonV600(products.map(p => createApiProductJsonV600(p, None))) + } + } diff --git a/obp-api/src/main/scala/code/apiproduct/ApiProduct.scala b/obp-api/src/main/scala/code/apiproduct/ApiProduct.scala new file mode 100644 index 0000000000..3a2812e0ad --- /dev/null +++ b/obp-api/src/main/scala/code/apiproduct/ApiProduct.scala @@ -0,0 +1,44 @@ +package code.apiproduct + +import code.util.{MappedUUID, UUIDString} +import net.liftweb.mapper._ + +class ApiProduct extends ApiProductTrait with LongKeyedMapper[ApiProduct] with IdPK with CreatedUpdated { + def getSingleton = ApiProduct + + object ApiProductId extends MappedUUID(this) + object BankId extends UUIDString(this) + object ApiProductCode extends MappedString(this, 50) + object ParentApiProductCode extends MappedString(this, 50) + object Name extends MappedString(this, 256) + object Category extends MappedString(this, 256) + object MoreInfoUrl extends MappedString(this, 2000) + object TermsAndConditionsUrl extends MappedString(this, 2000) + object Description extends MappedString(this, 2000) + + override def apiProductId: String = ApiProductId.get + override def bankId: String = BankId.get + override def apiProductCode: String = ApiProductCode.get + override def parentApiProductCode: String = ParentApiProductCode.get + override def name: String = Name.get + override def category: String = Category.get + override def moreInfoUrl: String = MoreInfoUrl.get + override def termsAndConditionsUrl: String = TermsAndConditionsUrl.get + override def description: String = Description.get +} + +object ApiProduct extends ApiProduct with LongKeyedMetaMapper[ApiProduct] { + override def dbIndexes = UniqueIndex(BankId, ApiProductCode) :: Index(BankId) :: super.dbIndexes +} + +trait ApiProductTrait { + def apiProductId: String + def bankId: String + def apiProductCode: String + def parentApiProductCode: String + def name: String + def category: String + def moreInfoUrl: String + def termsAndConditionsUrl: String + def description: String +} diff --git a/obp-api/src/main/scala/code/apiproduct/ApiProductsProvider.scala b/obp-api/src/main/scala/code/apiproduct/ApiProductsProvider.scala new file mode 100644 index 0000000000..f1d0033815 --- /dev/null +++ b/obp-api/src/main/scala/code/apiproduct/ApiProductsProvider.scala @@ -0,0 +1,99 @@ +package code.apiproduct + +import code.util.Helper.MdcLoggable +import net.liftweb.common.Box +import net.liftweb.mapper.By +import net.liftweb.util.Helpers.tryo + +trait ApiProductsProvider { + def createOrUpdateApiProduct( + bankId: String, + apiProductCode: String, + parentApiProductCode: String, + name: String, + category: String, + moreInfoUrl: String, + termsAndConditionsUrl: String, + description: String + ): Box[ApiProductTrait] + + def getApiProductByBankIdAndCode( + bankId: String, + apiProductCode: String + ): Box[ApiProductTrait] + + def getApiProductsByBankId( + bankId: String + ): List[ApiProductTrait] + + def deleteApiProduct( + bankId: String, + apiProductCode: String + ): Box[Boolean] +} + +object MappedApiProductsProvider extends MdcLoggable with ApiProductsProvider { + + override def createOrUpdateApiProduct( + bankId: String, + apiProductCode: String, + parentApiProductCode: String, + name: String, + category: String, + moreInfoUrl: String, + termsAndConditionsUrl: String, + description: String + ): Box[ApiProductTrait] = { + val existing = ApiProduct.find( + By(ApiProduct.BankId, bankId), + By(ApiProduct.ApiProductCode, apiProductCode) + ) + existing match { + case net.liftweb.common.Full(product) => + tryo( + product + .ParentApiProductCode(parentApiProductCode) + .Name(name) + .Category(category) + .MoreInfoUrl(moreInfoUrl) + .TermsAndConditionsUrl(termsAndConditionsUrl) + .Description(description) + .saveMe() + ) + case _ => + tryo( + ApiProduct + .create + .BankId(bankId) + .ApiProductCode(apiProductCode) + .ParentApiProductCode(parentApiProductCode) + .Name(name) + .Category(category) + .MoreInfoUrl(moreInfoUrl) + .TermsAndConditionsUrl(termsAndConditionsUrl) + .Description(description) + .saveMe() + ) + } + } + + override def getApiProductByBankIdAndCode( + bankId: String, + apiProductCode: String + ): Box[ApiProductTrait] = ApiProduct.find( + By(ApiProduct.BankId, bankId), + By(ApiProduct.ApiProductCode, apiProductCode) + ) + + override def getApiProductsByBankId( + bankId: String + ): List[ApiProductTrait] = ApiProduct.findAll(By(ApiProduct.BankId, bankId)) + + override def deleteApiProduct( + bankId: String, + apiProductCode: String + ): Box[Boolean] = ApiProduct.find( + By(ApiProduct.BankId, bankId), + By(ApiProduct.ApiProductCode, apiProductCode) + ).map(_.delete_!) +} diff --git a/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttribute.scala b/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttribute.scala new file mode 100644 index 0000000000..ec3448eae6 --- /dev/null +++ b/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttribute.scala @@ -0,0 +1,38 @@ +package code.apiproductattribute + +import code.util.{MappedUUID, UUIDString} +import net.liftweb.mapper._ + +class ApiProductAttribute extends ApiProductAttributeTrait with LongKeyedMapper[ApiProductAttribute] with IdPK with CreatedUpdated { + def getSingleton = ApiProductAttribute + + object BankId extends UUIDString(this) + object ApiProductCode extends MappedString(this, 50) + object ApiProductAttributeId extends MappedUUID(this) + object Name extends MappedString(this, 256) + object Type extends MappedString(this, 50) + object Value extends MappedString(this, 2000) + object IsActive extends MappedBoolean(this) + + override def bankId: String = BankId.get + override def apiProductCode: String = ApiProductCode.get + override def apiProductAttributeId: String = ApiProductAttributeId.get + override def name: String = Name.get + override def attributeType: String = Type.get + override def value: String = Value.get + override def isActive: Option[Boolean] = Some(IsActive.get) +} + +object ApiProductAttribute extends ApiProductAttribute with LongKeyedMetaMapper[ApiProductAttribute] { + override def dbIndexes = Index(BankId) :: UniqueIndex(ApiProductAttributeId) :: super.dbIndexes +} + +trait ApiProductAttributeTrait { + def bankId: String + def apiProductCode: String + def apiProductAttributeId: String + def name: String + def attributeType: String + def value: String + def isActive: Option[Boolean] +} diff --git a/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttributesProvider.scala b/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttributesProvider.scala new file mode 100644 index 0000000000..e7fefe9422 --- /dev/null +++ b/obp-api/src/main/scala/code/apiproductattribute/ApiProductAttributesProvider.scala @@ -0,0 +1,110 @@ +package code.apiproductattribute + +import code.util.Helper.MdcLoggable +import net.liftweb.common.Box +import net.liftweb.mapper.By +import net.liftweb.util.Helpers.tryo + +trait ApiProductAttributesProvider { + def getApiProductAttributesByBankIdAndCode( + bankId: String, + apiProductCode: String + ): Box[List[ApiProductAttributeTrait]] + + def getApiProductAttributeById( + apiProductAttributeId: String + ): Box[ApiProductAttributeTrait] + + def createOrUpdateApiProductAttribute( + bankId: String, + apiProductCode: String, + apiProductAttributeId: Option[String], + name: String, + attributeType: String, + value: String, + isActive: Option[Boolean] + ): Box[ApiProductAttributeTrait] + + def deleteApiProductAttribute( + apiProductAttributeId: String + ): Box[Boolean] +} + +object MappedApiProductAttributesProvider extends MdcLoggable with ApiProductAttributesProvider { + + override def getApiProductAttributesByBankIdAndCode( + bankId: String, + apiProductCode: String + ): Box[List[ApiProductAttributeTrait]] = { + tryo( + ApiProductAttribute.findAll( + By(ApiProductAttribute.BankId, bankId), + By(ApiProductAttribute.ApiProductCode, apiProductCode) + ) + ) + } + + override def getApiProductAttributeById( + apiProductAttributeId: String + ): Box[ApiProductAttributeTrait] = { + ApiProductAttribute.find(By(ApiProductAttribute.ApiProductAttributeId, apiProductAttributeId)) + } + + override def createOrUpdateApiProductAttribute( + bankId: String, + apiProductCode: String, + apiProductAttributeId: Option[String], + name: String, + attributeType: String, + value: String, + isActive: Option[Boolean] + ): Box[ApiProductAttributeTrait] = { + apiProductAttributeId match { + case Some(id) => + ApiProductAttribute.find(By(ApiProductAttribute.ApiProductAttributeId, id)) match { + case net.liftweb.common.Full(existing) => + tryo( + existing + .BankId(bankId) + .ApiProductCode(apiProductCode) + .Name(name) + .Type(attributeType) + .Value(value) + .IsActive(isActive.getOrElse(true)) + .saveMe() + ) + case _ => + createNew(bankId, apiProductCode, name, attributeType, value, isActive) + } + case None => + createNew(bankId, apiProductCode, name, attributeType, value, isActive) + } + } + + private def createNew( + bankId: String, + apiProductCode: String, + name: String, + attributeType: String, + value: String, + isActive: Option[Boolean] + ): Box[ApiProductAttributeTrait] = { + tryo( + ApiProductAttribute + .create + .BankId(bankId) + .ApiProductCode(apiProductCode) + .Name(name) + .Type(attributeType) + .Value(value) + .IsActive(isActive.getOrElse(true)) + .saveMe() + ) + } + + override def deleteApiProductAttribute( + apiProductAttributeId: String + ): Box[Boolean] = { + ApiProductAttribute.find(By(ApiProductAttribute.ApiProductAttributeId, apiProductAttributeId)).map(_.delete_!) + } +} diff --git a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnector.scala b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnector.scala index 988ec21681..ef36b37249 100644 --- a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnector.scala +++ b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnector.scala @@ -578,7 +578,7 @@ object LocalMappedConnector extends Connector with MdcLoggable { } //gets a particular bank handled by this connector - override def getBankLegacy(bankId: BankId, callContext: Option[CallContext]): Box[(Bank, Option[CallContext])] = writeMetricEndpointTiming { + override def getBankLegacy(bankId: BankId, callContext: Option[CallContext]): Box[(Bank, Option[CallContext])] = { MappedBank .find(By(MappedBank.permalink, bankId.value)) .map( @@ -587,14 +587,14 @@ object LocalMappedConnector extends Connector with MdcLoggable { .mBankRoutingScheme(APIUtil.ValueOrOBP(bank.bankRoutingScheme)) .mBankRoutingAddress(APIUtil.ValueOrOBPId(bank.bankRoutingAddress, bank.bankId.value)) ).map(bank => (bank, callContext)) - }("getBank") + } override def getBank(bankId: BankId, callContext: Option[CallContext]): Future[Box[(Bank, Option[CallContext])]] = Future { getBankLegacy(bankId, callContext) } - override def getBanksLegacy(callContext: Option[CallContext]): Box[(List[Bank], Option[CallContext])] = writeMetricEndpointTiming { + override def getBanksLegacy(callContext: Option[CallContext]): Box[(List[Bank], Option[CallContext])] = { Full(MappedBank .findAll() .map( @@ -605,7 +605,7 @@ object LocalMappedConnector extends Connector with MdcLoggable { ), callContext ) - }("getBanks") + } override def getBanks(callContext: Option[CallContext]): Future[Box[(List[Bank], Option[CallContext])]] = Future { getBanksLegacy(callContext) diff --git a/obp-api/src/main/scala/code/bankconnectors/akka/AkkaConnector_vDec2018.scala b/obp-api/src/main/scala/code/bankconnectors/akka/AkkaConnector_vDec2018.scala index 0a182dc499..e9248d4c17 100644 --- a/obp-api/src/main/scala/code/bankconnectors/akka/AkkaConnector_vDec2018.scala +++ b/obp-api/src/main/scala/code/bankconnectors/akka/AkkaConnector_vDec2018.scala @@ -2746,7 +2746,8 @@ object AkkaConnector_vDec2018 extends Connector with AkkaConnectorActorInit { ), exampleInboundMessage = ( InBoundGetProducts(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data=List( ProductCommons(bankId=BankId(bankIdExample.value), + data=List( ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -2783,7 +2784,8 @@ object AkkaConnector_vDec2018 extends Connector with AkkaConnectorActorInit { ), exampleInboundMessage = ( InBoundGetProduct(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data= ProductCommons(bankId=BankId(bankIdExample.value), + data= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -5607,7 +5609,8 @@ object AkkaConnector_vDec2018 extends Connector with AkkaConnectorActorInit { status=MessageDocsSwaggerDefinitions.inboundStatus, data=List( ProductCollectionItemsTree(productCollectionItem= ProductCollectionItemCommons(collectionCode=collectionCodeExample.value, memberProductCode=memberProductCodeExample.value), - product= ProductCommons(bankId=BankId(bankIdExample.value), + product= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, diff --git a/obp-api/src/main/scala/code/bankconnectors/generator/ConnectorBuilderUtil.scala b/obp-api/src/main/scala/code/bankconnectors/generator/ConnectorBuilderUtil.scala index 5a1438be1f..0654c465dd 100644 --- a/obp-api/src/main/scala/code/bankconnectors/generator/ConnectorBuilderUtil.scala +++ b/obp-api/src/main/scala/code/bankconnectors/generator/ConnectorBuilderUtil.scala @@ -230,8 +230,7 @@ object ConnectorBuilderUtil { if(doCache && methodName.matches("^(get|check|validate).+")) { signature = signature.replaceFirst("""(\b\S+)\s*:\s*Option\[CallContext\]""", "@CacheKeyOmit callContext: Option[CallContext]") body = - s"""saveConnectorMetric { - | /** + s""" /** | * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" | * is just a temporary value field with UUID values in order to prevent any ambiguity. | * The real value will be assigned by Macro during compile time at this line of a code: @@ -245,7 +244,6 @@ object ConnectorBuilderUtil { | | } | } - | }("$methodName") |""".stripMargin } s""" diff --git a/obp-api/src/main/scala/code/bankconnectors/package.scala b/obp-api/src/main/scala/code/bankconnectors/package.scala index e85f19c3fd..c2d761dea3 100644 --- a/obp-api/src/main/scala/code/bankconnectors/package.scala +++ b/obp-api/src/main/scala/code/bankconnectors/package.scala @@ -6,26 +6,28 @@ import java.util.regex.Pattern import org.apache.pekko.http.scaladsl.model.HttpMethod import code.api.{APIFailureNewStyle, ApiVersionHolder} import code.api.util.{CallContext, FutureUtil, NewStyle} +import code.api.util.APIUtil.{canOpenFuture, fullBoxOrException, getCorrelationId, getPropsAsBoolValue} +import code.api.util.ErrorMessages.{InvalidConnectorResponseForMissingRequiredValues, ServiceIsTooBusy} import code.methodrouting.{MethodRouting, MethodRoutingT} +import code.metrics.{ConnectorMetricsProvider, ConnectorCountsRedis} import code.util.Helper import code.util.Helper.MdcLoggable -import com.openbankproject.commons.model.BankId +import com.openbankproject.commons.model.{AccountId, BankId} import com.openbankproject.commons.util.ReflectUtils.{findMethodByArgs, getConstructorArgs} import com.openbankproject.commons.ExecutionContext.Implicits.global -import net.liftweb.common.{Box, Empty, EmptyBox, Full, ParamFailure} +import net.liftweb.common.{Box, Empty, EmptyBox, Failure, Full, ParamFailure} +import net.liftweb.util.Helpers.now +import net.liftweb.util.ThreadGlobal import net.sf.cglib.proxy.{Enhancer, MethodInterceptor, MethodProxy} import scala.collection.mutable.ArrayBuffer +import scala.collection.GenTraversableOnce +import scala.concurrent.Future import scala.reflect.runtime.universe.{MethodSymbol, Type, typeOf} -import code.api.util.ErrorMessages.{InvalidConnectorResponseForMissingRequiredValues, ServiceIsTooBusy} -import code.api.util.APIUtil.{canOpenFuture, fullBoxOrException} +import scala.util.{Success => TrySuccess, Failure => TryFailure} import com.openbankproject.commons.util.{ApiVersion, ReflectUtils} import com.openbankproject.commons.util.ReflectUtils._ import com.openbankproject.commons.util.Functions.Implicits._ -import net.liftweb.util.ThreadGlobal - -import scala.collection.GenTraversableOnce -import scala.concurrent.Future package object bankconnectors extends MdcLoggable { @@ -58,7 +60,63 @@ package object bankconnectors extends MdcLoggable { } connectorMethodResult } else { + val methodName = method.getName + val argNameToValue: Array[(String, AnyRef)] = method.getParameters.map(_.getName).zip(args) + // TODO: getConnectorNameAndMethodRouting is also called inside invokeMethod. + // Consider refactoring invokeMethod to accept a pre-resolved connectorName to avoid the duplicate lookup. + val (_, connectorName) = getConnectorNameAndMethodRouting(methodName, argNameToValue) + + // Record outbound (before call) + ConnectorCountsRedis.incrementOutbound(connectorName, methodName) + val t0 = System.currentTimeMillis() + val (connectorMethodResult, methodSymbol) = invokeMethod(method, args) + + // Track metrics for Future results + if (connectorMethodResult.isInstanceOf[Future[_]]) { + val future = connectorMethodResult.asInstanceOf[Future[Any]] + future.onComplete { result => + val duration = System.currentTimeMillis() - t0 + val isSuccess = result match { + case TrySuccess(value) => !isFailureBox(value) + case TryFailure(_) => false + } + + // Record inbound + ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) + + // Record detailed metric to DB + if (getPropsAsBoolValue("write_connector_metrics", false)) { + val params = extractKeyParams(args) + // TODO: The correlation_id should be passed down from the REST API layer + // so that one REST call with a correlation_id results in multiple connector + // metric records each sharing the same correlation_id. Currently getCorrelationId() + // relies on Lift's S.containerSession which is unavailable inside this Future. + val correlationId = getCorrelationId() + Future { + ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( + connectorName, methodName, correlationId, now, duration, params, isSuccess) + } + } + } + } else { + // Non-future (legacy Box) result - track synchronously + val duration = System.currentTimeMillis() - t0 + val isSuccess = !isFailureBox(connectorMethodResult) + + ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) + + if (getPropsAsBoolValue("write_connector_metrics", false)) { + val params = extractKeyParams(args) + // TODO: Same as above — correlation_id should come from the REST API layer. + val correlationId = getCorrelationId() + Future { + ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( + connectorName, methodName, correlationId, now, duration, params, isSuccess) + } + } + } + if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) } @@ -363,4 +421,35 @@ package object bankconnectors extends MdcLoggable { logger.error(message) ParamFailure(message, Empty, Empty, APIFailureNewStyle(message, 400, cc.map(_.toLight))) } + + /** + * Extract key parameters (bankId, accountId) from connector method args as a compact JSON string. + * Max 1024 characters to fit in the DB field. + */ + private def extractKeyParams(args: Array[AnyRef]): String = { + try { + val params = scala.collection.mutable.Map[String, String]() + args.foreach { + case bankId: BankId => params("bankId") = bankId.value + case accountId: AccountId => params("accountId") = accountId.value + case _ => // skip other types + } + if (params.isEmpty) "" + else { + val json = params.map { case (k, v) => s""""$k":"$v"""" }.mkString("{", ",", "}") + if (json.length > 1024) json.substring(0, 1024) else json + } + } catch { + case _: Throwable => "" + } + } + + /** + * Check if a connector result value represents a failure (Failure or Empty Box). + */ + private def isFailureBox(value: Any): Boolean = value match { + case _: EmptyBox => true + case (_: EmptyBox, _) => true + case _ => false + } } diff --git a/obp-api/src/main/scala/code/bankconnectors/rabbitmq/RabbitMQConnector_vOct2024.scala b/obp-api/src/main/scala/code/bankconnectors/rabbitmq/RabbitMQConnector_vOct2024.scala index c644945b78..88b7fe1b28 100644 --- a/obp-api/src/main/scala/code/bankconnectors/rabbitmq/RabbitMQConnector_vOct2024.scala +++ b/obp-api/src/main/scala/code/bankconnectors/rabbitmq/RabbitMQConnector_vOct2024.scala @@ -3418,7 +3418,8 @@ trait RabbitMQConnector_vOct2024 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProducts(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data=List( ProductCommons(bankId=BankId(bankIdExample.value), + data=List( ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -3457,7 +3458,8 @@ trait RabbitMQConnector_vOct2024 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProduct(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data= ProductCommons(bankId=BankId(bankIdExample.value), + data= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -6383,7 +6385,8 @@ trait RabbitMQConnector_vOct2024 extends Connector with MdcLoggable { status=MessageDocsSwaggerDefinitions.inboundStatus, data=List( ProductCollectionItemsTree(productCollectionItem= ProductCollectionItemCommons(collectionCode=collectionCodeExample.value, memberProductCode=memberProductCodeExample.value), - product= ProductCommons(bankId=BankId(bankIdExample.value), + product= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, diff --git a/obp-api/src/main/scala/code/bankconnectors/rest/RestConnector_vMar2019.scala b/obp-api/src/main/scala/code/bankconnectors/rest/RestConnector_vMar2019.scala index c5bd84c1bd..2cdf943901 100644 --- a/obp-api/src/main/scala/code/bankconnectors/rest/RestConnector_vMar2019.scala +++ b/obp-api/src/main/scala/code/bankconnectors/rest/RestConnector_vMar2019.scala @@ -3354,7 +3354,8 @@ trait RestConnector_vMar2019 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProducts(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data=List( ProductCommons(bankId=BankId(bankIdExample.value), + data=List( ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -3393,7 +3394,8 @@ trait RestConnector_vMar2019 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProduct(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data= ProductCommons(bankId=BankId(bankIdExample.value), + data= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -6282,7 +6284,8 @@ trait RestConnector_vMar2019 extends Connector with MdcLoggable { status=MessageDocsSwaggerDefinitions.inboundStatus, data=List( ProductCollectionItemsTree(productCollectionItem= ProductCollectionItemCommons(collectionCode=collectionCodeExample.value, memberProductCode=memberProductCodeExample.value), - product= ProductCommons(bankId=BankId(bankIdExample.value), + product= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, diff --git a/obp-api/src/main/scala/code/bankconnectors/storedprocedure/StoredProcedureConnector_vDec2019.scala b/obp-api/src/main/scala/code/bankconnectors/storedprocedure/StoredProcedureConnector_vDec2019.scala index 7531169217..6e260835a8 100644 --- a/obp-api/src/main/scala/code/bankconnectors/storedprocedure/StoredProcedureConnector_vDec2019.scala +++ b/obp-api/src/main/scala/code/bankconnectors/storedprocedure/StoredProcedureConnector_vDec2019.scala @@ -3408,7 +3408,8 @@ trait StoredProcedureConnector_vDec2019 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProducts(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data=List( ProductCommons(bankId=BankId(bankIdExample.value), + data=List( ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -3447,7 +3448,8 @@ trait StoredProcedureConnector_vDec2019 extends Connector with MdcLoggable { exampleInboundMessage = ( InBoundGetProduct(inboundAdapterCallContext=MessageDocsSwaggerDefinitions.inboundAdapterCallContext, status=MessageDocsSwaggerDefinitions.inboundStatus, - data= ProductCommons(bankId=BankId(bankIdExample.value), + data= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, @@ -6367,7 +6369,8 @@ trait StoredProcedureConnector_vDec2019 extends Connector with MdcLoggable { status=MessageDocsSwaggerDefinitions.inboundStatus, data=List( ProductCollectionItemsTree(productCollectionItem= ProductCollectionItemCommons(collectionCode=collectionCodeExample.value, memberProductCode=memberProductCodeExample.value), - product= ProductCommons(bankId=BankId(bankIdExample.value), + product= ProductCommons( + bankId=BankId(bankIdExample.value), code=ProductCode(productCodeExample.value), parentProductCode=ProductCode(parentProductCodeExample.value), name=productNameExample.value, diff --git a/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala b/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala index 9267e18c6a..94455a67ee 100644 --- a/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala @@ -14,13 +14,16 @@ object ConnectorMetrics extends ConnectorMetricsProvider { val cachedAllConnectorMetrics = APIUtil.getPropsValue(s"ConnectorMetrics.cache.ttl.seconds.getAllConnectorMetrics", "7").toInt - override def saveConnectorMetric(connectorName: String, functionName: String, correlationId: String, date: Date, duration: Long): Unit = { + override def saveConnectorMetric(connectorName: String, functionName: String, correlationId: String, date: Date, duration: Long, + requestParams: String, isSuccessful: Boolean): Unit = { MappedConnectorMetric.create .connectorName(connectorName) .functionName(functionName) .date(date) .duration(duration) .correlationId(correlationId) + .requestParams(requestParams) + .isSuccessful(isSuccessful) .save } @@ -71,14 +74,18 @@ class MappedConnectorMetric extends ConnectorMetric with LongKeyedMapper[MappedC object correlationId extends MappedUUID(this) object date extends MappedDateTime(this) object duration extends MappedLong(this) + object requestParams extends MappedString(this, 1024) + object isSuccessful extends MappedBoolean(this) override def getConnectorName(): String = connectorName.get override def getFunctionName(): String = functionName.get override def getCorrelationId(): String = correlationId.get override def getDate(): Date = date.get override def getDuration(): Long = duration.get + override def getRequestParams(): String = requestParams.get + override def getIsSuccessful(): Boolean = isSuccessful.get } object MappedConnectorMetric extends MappedConnectorMetric with LongKeyedMetaMapper[MappedConnectorMetric] { - override def dbIndexes = Index(connectorName) :: Index(functionName) :: Index(date) :: Index(correlationId) :: super.dbIndexes + override def dbIndexes = Index(connectorName) :: Index(functionName) :: Index(date) :: Index(correlationId) :: Index(isSuccessful) :: super.dbIndexes } diff --git a/obp-api/src/main/scala/code/metrics/ConnectorMetricsProvider.scala b/obp-api/src/main/scala/code/metrics/ConnectorMetricsProvider.scala index 40f63c3fb9..a11a198bc2 100644 --- a/obp-api/src/main/scala/code/metrics/ConnectorMetricsProvider.scala +++ b/obp-api/src/main/scala/code/metrics/ConnectorMetricsProvider.scala @@ -31,7 +31,11 @@ object ConnectorMetricsProvider extends SimpleInjector { trait ConnectorMetricsProvider { - def saveConnectorMetric(connectorName: String, functionName: String, correlationId: String, date: Date, duration: Long): Unit + def saveConnectorMetric(connectorName: String, functionName: String, correlationId: String, date: Date, duration: Long): Unit = { + saveConnectorMetric(connectorName, functionName, correlationId, date, duration, "", true) + } + def saveConnectorMetric(connectorName: String, functionName: String, correlationId: String, date: Date, duration: Long, + requestParams: String, isSuccessful: Boolean): Unit def getAllConnectorMetrics(queryParams: List[OBPQueryParam]): List[ConnectorMetric] def bulkDeleteConnectorMetrics(): Boolean @@ -44,5 +48,7 @@ trait ConnectorMetric { def getCorrelationId(): String def getDate(): Date def getDuration(): Long + def getRequestParams(): String + def getIsSuccessful(): Boolean } diff --git a/obp-api/src/main/scala/code/metrics/ConnectorMetricsRedis.scala b/obp-api/src/main/scala/code/metrics/ConnectorMetricsRedis.scala new file mode 100644 index 0000000000..eed109f416 --- /dev/null +++ b/obp-api/src/main/scala/code/metrics/ConnectorMetricsRedis.scala @@ -0,0 +1,188 @@ +package code.metrics + +import code.api.Constant._ +import code.api.JedisMethod +import code.api.cache.Redis +import code.api.util.APIUtil +import code.util.Helper.MdcLoggable + +/** + * Redis-based per-hour counters for connector outbound/inbound message tracking. + * + * Follows the same pattern as RateLimitingUtil.incrementConsumerCounters: + * - Check TTL of key + * - If key doesn't exist (TTL = -2): SET key 1 with TTL = 3600 seconds (1 hour) + * - If key exists (TTL > 0): INCR key + * - Counters automatically expire after 1 hour, giving a rolling per-hour window + * + * Gated by prop: write_connector_metrics_redis (default: false) + */ +object ConnectorCountsRedis extends MdcLoggable { + + private val PER_HOUR_TTL = 3600 // 1 hour in seconds + + def isEnabled: Boolean = APIUtil.getPropsAsBoolValue("write_connector_metrics_redis", false) + + private def outboundKey(connectorName: String, methodName: String): String = + s"${CONNECTOR_OUTBOUND_PREFIX}${connectorName}_${methodName}_PER_HOUR" + + private def inboundKey(connectorName: String, methodName: String, success: Boolean): String = { + val suffix = if (success) "_success" else "_failure" + s"${CONNECTOR_INBOUND_PREFIX}${connectorName}_${methodName}${suffix}_PER_HOUR" + } + + /** + * Increment the per-hour outbound counter for a connector method call. + * Called before invoking the connector method. + */ + def incrementOutbound(connectorName: String, methodName: String): Unit = { + if (!isEnabled) return + try { + incrementCounter(outboundKey(connectorName, methodName)) + } catch { + case e: Throwable => + logger.warn(s"Failed to increment outbound counter for $connectorName.$methodName: ${e.getMessage}") + } + } + + /** + * Increment the per-hour inbound counter for a connector method response. + * Called after receiving the connector method result. + */ + def incrementInbound(connectorName: String, methodName: String, success: Boolean): Unit = { + if (!isEnabled) return + try { + incrementCounter(inboundKey(connectorName, methodName, success)) + } catch { + case e: Throwable => + logger.warn(s"Failed to increment inbound counter for $connectorName.$methodName (success=$success): ${e.getMessage}") + } + } + + /** + * Read the current hour's outbound counter value. + */ + def getOutboundCount(connectorName: String, methodName: String): Long = { + try { + Redis.use(JedisMethod.GET, outboundKey(connectorName, methodName)).map(_.toLong).getOrElse(0L) + } catch { + case _: Throwable => 0L + } + } + + /** + * Read the current hour's inbound counter value (success or failure). + */ + def getInboundCount(connectorName: String, methodName: String, success: Boolean): Long = { + try { + Redis.use(JedisMethod.GET, inboundKey(connectorName, methodName, success)).map(_.toLong).getOrElse(0L) + } catch { + case _: Throwable => 0L + } + } + + /** + * Scan all per-hour outbound/inbound keys for a summary view. + * Returns a list of ConnectorCount entries with current counts and TTL. + */ + def getAllCounts(): List[ConnectorCount] = { + try { + val outboundPrefix = CONNECTOR_OUTBOUND_PREFIX + val inboundPrefix = CONNECTOR_INBOUND_PREFIX + + // Scan for outbound keys + val outboundPattern = s"${outboundPrefix}*_PER_HOUR" + val outboundKeys = Redis.scanKeys(outboundPattern) + + // Build a map of connectorName_methodName -> outbound count + val outboundMap: Map[String, Long] = outboundKeys.flatMap { key => + extractConnectorMethod(key, outboundPrefix, "_PER_HOUR").map { cm => + val count = Redis.use(JedisMethod.GET, key).map(_.toLong).getOrElse(0L) + cm -> count + } + }.toMap + + // Scan for inbound keys + val inboundPattern = s"${inboundPrefix}*_PER_HOUR" + val inboundKeys = Redis.scanKeys(inboundPattern) + + // Build maps for success and failure inbound counts + val inboundSuccessMap = scala.collection.mutable.Map[String, Long]() + val inboundFailureMap = scala.collection.mutable.Map[String, Long]() + + inboundKeys.foreach { key => + val count = Redis.use(JedisMethod.GET, key).map(_.toLong).getOrElse(0L) + if (key.contains("_success_PER_HOUR")) { + extractConnectorMethod(key, inboundPrefix, "_success_PER_HOUR").foreach { cm => + inboundSuccessMap(cm) = count + } + } else if (key.contains("_failure_PER_HOUR")) { + extractConnectorMethod(key, inboundPrefix, "_failure_PER_HOUR").foreach { cm => + inboundFailureMap(cm) = count + } + } + } + + // Combine all connector_method keys + val allKeys = (outboundMap.keySet ++ inboundSuccessMap.keySet ++ inboundFailureMap.keySet).toList + + allKeys.flatMap { cm => + val parts = cm.split("_", 2) + if (parts.length == 2) { + val connectorName = parts(0) + val methodName = parts(1) + val ttl = Redis.use(JedisMethod.TTL, outboundKey(connectorName, methodName)).map(_.toLong).getOrElse(0L) + Some(ConnectorCount( + connector_name = connectorName, + method_name = methodName, + per_hour_outbound_count = outboundMap.getOrElse(cm, 0L), + per_hour_inbound_success_count = inboundSuccessMap.getOrElse(cm, 0L), + per_hour_inbound_failure_count = inboundFailureMap.getOrElse(cm, 0L), + ttl_seconds = ttl + )) + } else None + } + } catch { + case e: Throwable => + logger.warn(s"Failed to get all connector counts: ${e.getMessage}") + List.empty + } + } + + /** + * Core counter increment logic following the RateLimitingUtil pattern. + */ + private def incrementCounter(key: String): Unit = { + val ttlOpt = Redis.use(JedisMethod.TTL, key).map(_.toInt) + ttlOpt match { + case Some(-2) => // Key does not exist, create it + Redis.use(JedisMethod.SET, key, Some(PER_HOUR_TTL), Some("1")) + case Some(ttl) if ttl > 0 => // Key exists with TTL, increment it + Redis.use(JedisMethod.INCR, key) + case Some(ttl) if ttl <= 0 => // Expired or no expiry (shouldn't happen) + Redis.use(JedisMethod.SET, key, Some(PER_HOUR_TTL), Some("1")) + case None => // Redis unavailable + logger.warn(s"Redis unavailable when incrementing connector counter: $key") + } + } + + /** + * Extract connectorName_methodName from a Redis key by stripping prefix and suffix. + */ + private def extractConnectorMethod(key: String, prefix: String, suffix: String): Option[String] = { + if (key.startsWith(prefix) && key.endsWith(suffix)) { + Some(key.stripPrefix(prefix).stripSuffix(suffix)) + } else { + None + } + } +} + +case class ConnectorCount( + connector_name: String, + method_name: String, + per_hour_outbound_count: Long, + per_hour_inbound_success_count: Long, + per_hour_inbound_failure_count: Long, + ttl_seconds: Long +) diff --git a/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala b/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala new file mode 100644 index 0000000000..4cade8d6c3 --- /dev/null +++ b/obp-api/src/main/scala/code/metrics/DoobieMetricsQueries.scala @@ -0,0 +1,396 @@ +package code.metrics + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import doobie._ +import doobie.implicits._ +import doobie.implicits.javasql._ // Provides Meta instances for java.sql.Timestamp +import code.api.util.{DBUtil, DoobieUtil} + +import java.util.Date +import scala.concurrent.{ExecutionContext, Future} + +/** + * Doobie-based query implementations for metrics. + * + * These queries replace the raw SQL queries in MappedMetrics that were using + * Lift's DB.runQuery (via DBUtil.runQuery). Doobie provides: + * - Type-safe query results + * - Type-safe parameters (no SQL injection risk) + * - Proper JDBC type handling for all databases (including SQL Server NVARCHAR) + * + * Usage: + * {{{ + * val result: List[TopApi] = DoobieMetricsQueries.getTopApis(fromDate, toDate, limit, filters) + * }}} + */ +object DoobieMetricsQueries { + + /** + * Get aggregate metrics (count, avg, min, max duration) for the given time range and filters. + * + * @param fromDate Start date for the query + * @param toDate End date for the query + * @param filters Filter options (consumerId, userId, appName, verb, etc.) + * @param isNewVersion Whether to use include patterns (true) or exclude patterns (false) + * @return List of AggregateMetrics (typically a single element) + */ + def getAggregateMetrics( + fromDate: Date, + toDate: Date, + filters: MetricsQueryFilters, + isNewVersion: Boolean + ): List[AggregateMetrics] = { + val query = buildAggregateMetricsQuery(fromDate, toDate, filters, isNewVersion) + DoobieUtil.runQuery(query) + } + + def getAggregateMetricsFuture( + fromDate: Date, + toDate: Date, + filters: MetricsQueryFilters, + isNewVersion: Boolean + )(implicit ec: ExecutionContext): Future[List[AggregateMetrics]] = { + Future { + getAggregateMetrics(fromDate, toDate, filters, isNewVersion) + } + } + + private def buildAggregateMetricsQuery( + fromDate: Date, + toDate: Date, + filters: MetricsQueryFilters, + isNewVersion: Boolean + ): ConnectionIO[List[AggregateMetrics]] = { + // Convert dates to SQL timestamps + val fromTs = new java.sql.Timestamp(fromDate.getTime) + val toTs = new java.sql.Timestamp(toDate.getTime) + + // Build dynamic WHERE conditions + val baseQuery = fr""" + SELECT count(*), avg(duration), min(duration), max(duration) + FROM metric + WHERE date_c >= $fromTs + AND date_c <= $toTs + """ + + val conditions = buildFilterConditions(filters, isNewVersion) + val fullQuery = baseQuery ++ conditions + + fullQuery.query[(Long, Option[Double], Option[Double], Option[Double])].to[List].map { rows => + rows.map { case (count, avgOpt, minOpt, maxOpt) => + AggregateMetrics( + count.toInt, + avgOpt.map(d => BigDecimal(d).setScale(2, BigDecimal.RoundingMode.HALF_UP).toDouble).getOrElse(0.0), + minOpt.getOrElse(0.0), + maxOpt.getOrElse(0.0) + ) + } + } + } + + /** + * Get top APIs by call count for the given time range. + * + * @param fromDate Start date for the query + * @param toDate End date for the query + * @param limit Maximum number of results + * @param filters Filter options + * @return List of TopApi sorted by count descending + */ + def getTopApis( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): List[TopApi] = { + val query = buildTopApisQuery(fromDate, toDate, limit, filters) + DoobieUtil.runQuery(query) + } + + def getTopApisFuture( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + )(implicit ec: ExecutionContext): Future[List[TopApi]] = { + Future { + getTopApis(fromDate, toDate, limit, filters) + } + } + + private def buildTopApisQuery( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): ConnectionIO[List[TopApi]] = { + val fromTs = new java.sql.Timestamp(fromDate.getTime) + val toTs = new java.sql.Timestamp(toDate.getTime) + + val isSqlServer = DBUtil.isSqlServer + + // SQL Server uses TOP, others use LIMIT + val baseQuery = if (isSqlServer) { + fr""" + SELECT TOP($limit) count(*), metric.implementedbypartialfunction, metric.implementedinversion + FROM metric + WHERE date_c >= $fromTs + AND date_c <= $toTs + """ + } else { + fr""" + SELECT count(*), metric.implementedbypartialfunction, metric.implementedinversion + FROM metric + WHERE date_c >= $fromTs + AND date_c <= $toTs + """ + } + + val conditions = buildFilterConditions(filters, isNewVersion = false) + + val groupAndOrder = fr""" + GROUP BY metric.implementedbypartialfunction, metric.implementedinversion + ORDER BY count(*) DESC + """ + + val limitClause = if (isSqlServer) fr"" else fr"LIMIT $limit" + + val fullQuery = baseQuery ++ conditions ++ groupAndOrder ++ limitClause + + fullQuery.query[(Long, String, String)].to[List].map { rows => + rows.map { case (count, partialFunction, version) => + TopApi(count.toInt, partialFunction, version) + } + } + } + + /** + * Get top consumers by API call count for the given time range. + * + * @param fromDate Start date for the query + * @param toDate End date for the query + * @param limit Maximum number of results + * @param filters Filter options + * @return List of TopConsumer sorted by count descending + */ + def getTopConsumers( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): List[TopConsumer] = { + val query = buildTopConsumersQuery(fromDate, toDate, limit, filters) + DoobieUtil.runQuery(query) + } + + def getTopConsumersFuture( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + )(implicit ec: ExecutionContext): Future[List[TopConsumer]] = { + Future { + getTopConsumers(fromDate, toDate, limit, filters) + } + } + + private def buildTopConsumersQuery( + fromDate: Date, + toDate: Date, + limit: Int, + filters: MetricsQueryFilters + ): ConnectionIO[List[TopConsumer]] = { + val fromTs = new java.sql.Timestamp(fromDate.getTime) + val toTs = new java.sql.Timestamp(toDate.getTime) + + val isSqlServer = DBUtil.isSqlServer + + // SQL Server uses TOP, others use LIMIT + val baseQuery = if (isSqlServer) { + fr""" + SELECT TOP($limit) count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, + consumer.developeremail as email, consumer.consumerid as consumerid + FROM metric, consumer + WHERE metric.appname = consumer.name + AND date_c >= $fromTs + AND date_c <= $toTs + """ + } else { + fr""" + SELECT count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, + consumer.developeremail as email, consumer.consumerid as consumerid + FROM metric, consumer + WHERE metric.appname = consumer.name + AND date_c >= $fromTs + AND date_c <= $toTs + """ + } + + val conditions = buildFilterConditions(filters, isNewVersion = false) + + val groupAndOrder = fr""" + GROUP BY appname, consumer.developeremail, consumer.id, consumer.consumerid + ORDER BY count DESC + """ + + val limitClause = if (isSqlServer) fr"" else fr"LIMIT $limit" + + val fullQuery = baseQuery ++ conditions ++ groupAndOrder ++ limitClause + + fullQuery.query[(Long, Long, String, String, String)].to[List].map { rows => + rows.map { case (count, _, appName, email, consumerId) => + TopConsumer(count.toInt, consumerId, appName, email) + } + } + } + + /** + * Build WHERE clause conditions from filter options. + */ + private def buildFilterConditions(filters: MetricsQueryFilters, isNewVersion: Boolean): Fragment = { + val simpleConditions = List( + filters.consumerId.map(v => fr"AND consumerid = $v"), + filters.userId.map(v => fr"AND userid = $v"), + filters.implementedByPartialFunction.map(v => fr"AND implementedbypartialfunction = $v"), + filters.implementedInVersion.map(v => fr"AND implementedinversion = $v"), + filters.url.map(v => fr"AND url = $v"), + filters.appName.map(v => fr"AND appname = $v"), + filters.verb.map(v => fr"AND verb = $v"), + filters.correlationId.map(v => fr"AND correlationid = $v"), + filters.httpStatusCode.map(v => fr"AND httpcode = $v"), + filters.anon.flatMap { + case true => Some(fr"AND userid = 'null'") + case false => Some(fr"AND userid != 'null'") + } + ).flatten + + // Handle exclude/include app names + val appNameCondition = if (isNewVersion) { + filters.includeAppNames.filter(_.nonEmpty).map { names => + buildInClause("appname", names) + } + } else { + filters.excludeAppNames.filter(_.nonEmpty).map { names => + buildNotInClause("appname", names) + } + } + + // Handle exclude/include implemented by partial functions + val partialFunctionCondition = if (isNewVersion) { + filters.includeImplementedByPartialFunctions.filter(_.nonEmpty).map { names => + buildInClause("implementedbypartialfunction", names) + } + } else { + filters.excludeImplementedByPartialFunctions.filter(_.nonEmpty).map { names => + buildNotInClause("implementedbypartialfunction", names) + } + } + + // Handle exclude/include URL patterns + val urlPatternCondition = if (isNewVersion) { + filters.includeUrlPatterns.filter(_.nonEmpty).map { patterns => + buildLikeClause("url", patterns) + } + } else { + filters.excludeUrlPatterns.filter(_.nonEmpty).map { patterns => + buildNotLikeClause("url", patterns) + } + } + + val allConditions = simpleConditions ++ + appNameCondition.toList ++ + partialFunctionCondition.toList ++ + urlPatternCondition.toList + + allConditions.foldLeft(fr"")(_ ++ _) + } + + /** + * Build a NOT IN clause for excluding values. + */ + private def buildNotInClause(column: String, values: List[String]): Fragment = { + if (values.isEmpty) { + fr"" + } else { + val columnFr = Fragment.const(column) + val valueFragments = values.map(v => fr"$v") + val inList = valueFragments.reduceLeft((a, b) => a ++ fr"," ++ b) + fr"AND" ++ columnFr ++ fr"NOT IN (" ++ inList ++ fr")" + } + } + + /** + * Build an IN clause for including values. + */ + private def buildInClause(column: String, values: List[String]): Fragment = { + if (values.isEmpty) { + fr"" + } else { + val columnFr = Fragment.const(column) + val valueFragments = values.map(v => fr"$v") + val inList = valueFragments.reduceLeft((a, b) => a ++ fr"," ++ b) + fr"AND" ++ columnFr ++ fr"IN (" ++ inList ++ fr")" + } + } + + /** + * Build a NOT LIKE clause for excluding URL patterns. + */ + private def buildNotLikeClause(column: String, patterns: List[String]): Fragment = { + if (patterns.isEmpty || (patterns.size == 1 && patterns.head.isEmpty)) { + fr"" + } else { + val columnFr = Fragment.const(column) + val notLikeConditions = patterns.filter(_.nonEmpty).map { pattern => + columnFr ++ fr"NOT LIKE $pattern" + } + if (notLikeConditions.isEmpty) { + fr"" + } else { + fr"AND (" ++ notLikeConditions.reduceLeft((a, b) => a ++ fr"AND" ++ b) ++ fr")" + } + } + } + + /** + * Build a LIKE clause for including URL patterns. + */ + private def buildLikeClause(column: String, patterns: List[String]): Fragment = { + if (patterns.isEmpty || (patterns.size == 1 && patterns.head.isEmpty)) { + fr"" + } else { + val columnFr = Fragment.const(column) + val likeConditions = patterns.filter(_.nonEmpty).map { pattern => + columnFr ++ fr"LIKE $pattern" + } + if (likeConditions.isEmpty) { + fr"" + } else { + fr"AND (" ++ likeConditions.reduceLeft((a, b) => a ++ fr"OR" ++ b) ++ fr")" + } + } + } +} + +/** + * Filter options for metrics queries. + */ +case class MetricsQueryFilters( + consumerId: Option[String] = None, + userId: Option[String] = None, + url: Option[String] = None, + appName: Option[String] = None, + implementedByPartialFunction: Option[String] = None, + implementedInVersion: Option[String] = None, + verb: Option[String] = None, + anon: Option[Boolean] = None, + correlationId: Option[String] = None, + httpStatusCode: Option[Int] = None, + excludeAppNames: Option[List[String]] = None, + includeAppNames: Option[List[String]] = None, + excludeUrlPatterns: Option[List[String]] = None, + includeUrlPatterns: Option[List[String]] = None, + excludeImplementedByPartialFunctions: Option[List[String]] = None, + includeImplementedByPartialFunctions: Option[List[String]] = None +) diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 350a5c4a2c..43d647b3ca 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -467,13 +467,14 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } // Smart caching applied - uses determineMetricsCacheTTL based on query date range + // Uses Doobie for type-safe database queries with proper JDBC type handling (including SQL Server NVARCHAR) override def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] = Future{ - /** + /** * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUU * is just a temporary value field with UUID values in order to prevent any ambiguity. - * The real value will be assigned by Macro during compile time at this line of a code: + * The real value will be assigned by Macro during compile time at this line of a code: * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/t - */ + */ var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) val cacheTTL = determineMetricsCacheTTL(queryParams) CacheKeyFromArguments.buildCacheKey {Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cacheTTL.seconds){ @@ -495,61 +496,32 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val excludeUrlPatterns = queryParams.collect { case OBPExcludeUrlPatterns(value) => value }.headOption val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse(10) - - val excludeUrlPatternsList= excludeUrlPatterns.getOrElse(List("")) - val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeImplementedByPartialFunctionsNumberList = - excludeImplementedByPartialFunctions.getOrElse(List("")).map(i => s"'$i'").mkString(",") - val excludeUrlPatternsQueries: String = extendLikeQuery(excludeUrlPatternsList, false) - - val (dbUrl, _, _) = DBUtil.getDbConnectionParameters + // Build MetricsQueryFilters for Doobie + val filters = MetricsQueryFilters( + consumerId = consumerId, + userId = userId, + url = url, + appName = appName, + implementedByPartialFunction = implementedByPartialFunction, + implementedInVersion = implementedInVersion, + verb = verb, + anon = anon, + correlationId = correlationId, + httpStatusCode = httpStatusCode, + excludeAppNames = excludeAppNames, + excludeUrlPatterns = excludeUrlPatterns, + excludeImplementedByPartialFunctions = excludeImplementedByPartialFunctions + ) val result: Box[List[TopApi]] = tryo { - // MS SQL server has the specific syntax for limiting number of rows - val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" - // TODO Make it work in case of Oracle database - val otherDbLimit = if (dbUrl.contains("sqlserver")) s"" else s"LIMIT $limit" - val sqlQuery: String = - s"""SELECT ${msSqlLimit} count(*), metric.implementedbypartialfunction, metric.implementedinversion - FROM metric - WHERE - date_c >= '${sqlTimestamp(fromDate.get)}' AND - date_c <= '${sqlTimestamp(toDate.get)}' - AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("null")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("null")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("null")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("null")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("null")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("null")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = null) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != null) - AND (${trueOrFalse(httpStatusCode.isEmpty)} or httpcode = ${sqlFriendlyInt(httpStatusCode)}) - AND (${trueOrFalse(excludeUrlPatterns.isEmpty)} or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalse(excludeAppNames.isEmpty)} or appname not in ($excludeAppNamesNumberList)) - AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty)} or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) - GROUP BY metric.implementedbypartialfunction, metric.implementedinversion - ORDER BY count(*) DESC - ${otherDbLimit} - """.stripMargin - - logger.debug(s"getTopApisFuture SQL query: $sqlQuery") - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (_, rows) = DBUtil.runQuery(sqlQuery) - logger.debug(s"getTopApisFuture returned ${rows.length} rows") - if (rows.nonEmpty) { - logger.debug(s"getTopApisFuture first row sample: ${rows.head}") + logger.debug(s"getTopApisFuture using Doobie with filters: $filters, limit: $limit") + val topApis = DoobieMetricsQueries.getTopApis(fromDate.get, toDate.get, limit, filters) + logger.debug(s"getTopApisFuture returned ${topApis.length} rows") + if (topApis.nonEmpty) { + logger.debug(s"getTopApisFuture first row sample: ${topApis.head}") } - val sqlResult = - rows.map { rs => // Map result to case class - TopApi( - tryo(rs(0).toInt).getOrElse(0), // Safe conversion with fallback - rs(1), - rs(2) - ) - } - sqlResult + topApis } result }} diff --git a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala index 896afc7892..1b88ccf93b 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala @@ -31,12 +31,11 @@ import java.util.UUID.randomUUID import code.api.Constant import code.api.cache.Caching -import code.api.util.{APIUtil, DBUtil} +import code.api.util.{APIUtil, DoobieQueries} import code.util.MappedUUID import com.openbankproject.commons.model.{User, UserPrimaryKey} import com.tesobe.CacheKeyFromArguments import net.liftweb.mapper._ -import net.liftweb.mapper.DB import scala.concurrent.duration._ @@ -140,10 +139,8 @@ object ResourceUser extends ResourceUser with LongKeyedMetaMapper[ResourceUser]{ val cacheTTL = APIUtil.getPropsAsIntValue("getDistinctProviders.cache.ttl.seconds", 3600) CacheKeyFromArguments.buildCacheKey { Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cacheTTL.seconds) { - val sql = "SELECT DISTINCT provider_ FROM resourceuser ORDER BY provider_" - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (_, rows) = DBUtil.runQuery(sql) - rows.flatten + // Use Doobie for type-safe query with proper JDBC type handling (including SQL Server NVARCHAR) + DoobieQueries.getDistinctProviders } } } diff --git a/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala b/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala index 3714f3a038..4041854603 100644 --- a/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala +++ b/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala @@ -1,6 +1,6 @@ package code.util -import code.api.util.DBUtil +import code.api.util.DoobieQueries import com.openbankproject.commons.model.BankId import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper} @@ -39,48 +39,20 @@ trait AttributeQueryTrait { self: BaseMetaMapper => */ def getParentIdByParams(bankId: BankId, params: Map[String, List[String]]): List[String] = { if (params.isEmpty) { - val sql = s"SELECT DISTINCT attr.$parentIdColumn FROM $tableName attr where attr.$bankIdColumn = ? " - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (_, list) = DBUtil.runQuery(sql, List(bankId.value)) - list.flatten + // Use Doobie for type-safe query with proper JDBC type handling (including SQL Server NVARCHAR) + DoobieQueries.getDistinctParentIds(tableName, parentIdColumn, bankIdColumn, bankId.value) } else { - val paramList = params.toList - val parameters = paramList.flatMap { kv => - val (name, values) = kv - name :: values - } - - - val sqlParametersFilter = paramList.map { kv => - val (_, values) = kv - if (values.size == 1) { - s"($nameColumn = ? AND $valueColumn = ?)" - } else { - //For lift framework not support in query, here just express in operation: mname = ? and mvalue in (?, ?, ?) - val valueExp = values.map(_ => "?").mkString(", ") - s"( $nameColumn = ? AND $valueColumn in ($valueExp) )" - } - }.mkString(" OR ") - - val sql = - s""" SELECT attr.$parentIdColumn, attr.$nameColumn, attr.$valueColumn - | FROM $tableName attr - | WHERE attr.$bankIdColumn = ? - | AND ($sqlParametersFilter) - |""".stripMargin - - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (columnNames: List[String], list: List[List[String]]) = DBUtil.runQuery(sql, bankId.value :: parameters) - val columnNamesLowerCase = columnNames.map(_.toLowerCase) - val parentIdIndex = columnNamesLowerCase.indexOf(parentIdColumn.toLowerCase) - val nameIndex = columnNamesLowerCase.indexOf(nameColumn.toLowerCase) - val valueIndex = columnNamesLowerCase.indexOf(valueColumn.toLowerCase) + // Use Doobie for type-safe query with proper JDBC type handling (including SQL Server NVARCHAR) + val results: List[(String, String, String)] = DoobieQueries.getParentIdWithAttributes( + tableName, parentIdColumn, nameColumn, valueColumn, bankIdColumn, bankId.value, params + ) - val parentIdToAttributes: Map[String, List[List[String]]] = list.groupBy(_.apply(parentIdIndex)) + // Group by parentId and filter + val parentIdToAttributes: Map[String, List[(String, String, String)]] = results.groupBy(_._1) val parentIdToNameValues: Map[String, Map[String, String]] = parentIdToAttributes.mapValues(rows => { - rows.map { row => - row(nameIndex) -> row(valueIndex) + rows.map { case (_, name, value) => + name -> value }.toMap }) diff --git a/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala b/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala index a0a579574b..23e3f42c2f 100644 --- a/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala +++ b/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala @@ -1,6 +1,6 @@ package code.util -import code.api.util.DBUtil +import code.api.util.DoobieQueries import com.openbankproject.commons.model.BankId import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper} @@ -36,48 +36,20 @@ trait NewAttributeQueryTrait { */ def getParentIdByParams(bankId: BankId, params: Map[String, List[String]]): List[String] = { if (params.isEmpty) { - val sql = s"SELECT DISTINCT attr.$parentIdColumn FROM $tableName attr where attr.$bankIdColumn = ? " - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (_, list) = DBUtil.runQuery(sql, List(bankId.value)) - list.flatten + // Use Doobie for type-safe query with proper JDBC type handling (including SQL Server NVARCHAR) + DoobieQueries.getDistinctParentIds(tableName, parentIdColumn, bankIdColumn, bankId.value) } else { - val paramList = params.toList - val parameters = paramList.flatMap { kv => - val (name, values) = kv - name :: values - } - - - val sqlParametersFilter = paramList.map { kv => - val (_, values) = kv - if (values.size == 1) { - s"($nameColumn = ? AND $valueColumn = ?)" - } else { - //For lift framework not support in query, here just express in operation: mname = ? and mvalue in (?, ?, ?) - val valueExp = values.map(_ => "?").mkString(", ") - s"( $nameColumn = ? AND $valueColumn in ($valueExp) )" - } - }.mkString(" OR ") - - val sql = - s""" SELECT attr.$parentIdColumn, attr.$nameColumn, attr.$valueColumn - | FROM $tableName attr - | WHERE attr.$bankIdColumn = ? - | AND ($sqlParametersFilter) - |""".stripMargin - - // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly - val (columnNames: List[String], list: List[List[String]]) = DBUtil.runQuery(sql, bankId.value :: parameters) - val columnNamesLowerCase = columnNames.map(_.toLowerCase) - val parentIdIndex = columnNamesLowerCase.indexOf(parentIdColumn.toLowerCase) - val nameIndex = columnNamesLowerCase.indexOf(nameColumn.toLowerCase) - val valueIndex = columnNamesLowerCase.indexOf(valueColumn.toLowerCase) + // Use Doobie for type-safe query with proper JDBC type handling (including SQL Server NVARCHAR) + val results: List[(String, String, String)] = DoobieQueries.getParentIdWithAttributes( + tableName, parentIdColumn, nameColumn, valueColumn, bankIdColumn, bankId.value, params + ) - val parentIdToAttributes: Map[String, List[List[String]]] = list.groupBy(_.apply(parentIdIndex)) + // Group by parentId and filter + val parentIdToAttributes: Map[String, List[(String, String, String)]] = results.groupBy(_._1) val parentIdToNameValues: Map[String, Map[String, String]] = parentIdToAttributes.mapValues(rows => { - rows.map { row => - row(nameIndex) -> row(valueIndex) + rows.map { case (_, name, value) => + name -> value }.toMap }) diff --git a/obp-api/src/test/scala/code/api/v1_4_0/ProductsTest.scala b/obp-api/src/test/scala/code/api/v1_4_0/ProductsTest.scala index 800df536f1..88f6a2995f 100644 --- a/obp-api/src/test/scala/code/api/v1_4_0/ProductsTest.scala +++ b/obp-api/src/test/scala/code/api/v1_4_0/ProductsTest.scala @@ -12,9 +12,10 @@ class ProductsTest extends ServerSetup with DefaultUsers with V140ServerSetup { val BankWithLicense = BankId("testBank1") val BankWithoutLicense = BankId("testBank2") + // Have to repeat the constructor parameters from the trait // Have to repeat the constructor parameters from the trait case class ProductImpl(bankId: BankId, - code : ProductCode, + code : ProductCode, parentProductCode : ProductCode, name : String, category: String, diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index f9c676b617..f98b70644f 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -139,7 +139,7 @@ class ApiCollectionEndpointTest extends V400ServerSetup { Then(s"we test the $ApiEndpoint6- OBPv400") val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) - lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv4.0.0-getBanks") + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv6.0.0-getBanks") val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) Then("We should get a 201") @@ -156,7 +156,7 @@ class ApiCollectionEndpointTest extends V400ServerSetup { Then(s"we test the $ApiEndpoint6- OBPv500") val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) - lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv5.0.0-createCustomer") + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv6.0.0-createCustomer") val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) Then("We should get a 201") diff --git a/obp-api/src/test/scala/code/api/v6_0_0/TopApisTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/TopApisTest.scala new file mode 100644 index 0000000000..6f62fffab5 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/TopApisTest.scala @@ -0,0 +1,266 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v6_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.CanReadMetrics +import code.api.util.ErrorMessages.{UserHasMissingRoles, AuthenticatedUserIsRequired} +import code.api.v6_0_0.OBPAPI6_0_0.Implementations6_0_0 +import code.entitlement.Entitlement +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * Test for the getTopAPIs v6.0.0 endpoint. + * + * This endpoint uses Doobie for database access (DoobieMetricsQueries.getTopApis) + * and returns API usage metrics with operation_id fields. + */ +class TopApisTest extends V600ServerSetup { + /** + * Test tags + * Example: To run tests with tag "getPermissions": + * mvn test -D tagsToInclude + * + * This is made possible by the scalatest maven plugin + */ + object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) + object ApiEndpoint1 extends Tag(nameOf(Implementations6_0_0.getTopAPIs)) + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Unauthorized access") { + scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) { + When("We make a request v6.0.0") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET + val response = makeGetRequest(request) + Then("We should get a 401") + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Authorized access without role") { + scenario("We will call the endpoint with user credentials but without proper entitlement", ApiEndpoint1, VersionOfApi) { + When("We make a request v6.0.0") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) + val response = makeGetRequest(request) + Then("error should be " + UserHasMissingRoles + CanReadMetrics) + response.code should equal(403) + response.body.extract[ErrorMessage].message should be (UserHasMissingRoles + CanReadMetrics) + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Authorized access with proper Role") { + scenario("We will call the endpoint with user credentials and proper entitlement", ApiEndpoint1, VersionOfApi) { + // Enable metrics writing so API calls are recorded + setPropsValues("write_metrics" -> "true") + + // Add required entitlement + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + // Make some API calls to generate metrics data + val requestUsers = (v6_0_0_Request / "users" / "current").GET <@ (user1) + makeGetRequest(requestUsers) + makeGetRequest(requestUsers) + makeGetRequest(requestUsers) + + val requestBanks = (v6_0_0_Request / "banks").GET <@ (user1) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + + When("We make a request v6.0.0 to get top APIs") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) + val response = makeGetRequest(request) + + Then("We get successful response") + response.code should equal(200) + + And("The response should have the correct structure") + val topApisJson = response.body.extract[TopApisJsonV600] + topApisJson.top_apis should not be null + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Response structure with operation_id") { + scenario("We verify the response includes operation_id field", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + // Generate some metrics by calling APIs + val requestBanks = (v6_0_0_Request / "banks").GET <@ (user1) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + + When("We make a request v6.0.0 to get top APIs") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) + val response = makeGetRequest(request) + + Then("We get successful response") + response.code should equal(200) + + val topApisJson = response.body.extract[TopApisJsonV600] + + And("Each top API entry should have the required fields including operation_id") + if (topApisJson.top_apis.nonEmpty) { + topApisJson.top_apis.foreach { topApi => + topApi.count should be >= 0 + topApi.implemented_by_partial_function should not be empty + topApi.implemented_in_version should not be empty + topApi.operation_id should not be empty + } + } + } + } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Filter parameters") { + scenario("We test filtering by limit parameter", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + // Generate some metrics + val requestBanks = (v6_0_0_Request / "banks").GET <@ (user1) + makeGetRequest(requestBanks) + val requestUsers = (v6_0_0_Request / "users" / "current").GET <@ (user1) + makeGetRequest(requestUsers) + + When("We request top APIs with limit=1") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + // Generate some metrics + val requestBanks = (v6_0_0_Request / "banks").GET <@ (user1) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + + When("We request top APIs filtered by implemented_by_partial_function=getBanks") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < + topApi.implemented_by_partial_function should equal("getBanks") + } + } + + scenario("We test filtering by verb parameter", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + When("We request top APIs filtered by verb=GET") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + When("We request top APIs excluding certain app names") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + When("We request top APIs with from_date far in the future") + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + // Generate some metrics + val requestBanks = (v6_0_0_Request / "banks").GET <@ (user1) + makeGetRequest(requestBanks) + makeGetRequest(requestBanks) + + When("We request top APIs with multiple filters") + val params = List( + ("verb", "GET"), + ("implemented_by_partial_function", "getBanks"), + ("limit", "10") + ) + val request = (v6_0_0_Request / "management" / "metrics" / "top-apis").GET <@(user1) < + topApi.implemented_by_partial_function should equal("getBanks") + } + topApisJson.top_apis.size should be <= 10 + } + } +}