Skip to content

Commit 02a1bc6

Browse files
simonredfernsimonredfern
authored
Merge pull request #2684 from simonredfern/develop
Using Counters even if no rate limiting for consumer. Using Doobie for some queries. Doobie has own Hikari pool.
2 parents 3364532 + 6ec418f commit 02a1bc6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+3241
-226
lines changed

‎.github/workflows/build_container.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props
6969
7070
echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props
71-
MAVEN_OPTS="-Xmx3G -Xss2m" mvn clean package -Pprod 2>&1 | tee maven-build.log
71+
MAVEN_OPTS="-Xmx3G -Xss2m -XX:MaxMetaspaceSize=1024M" mvn clean package -Pprod 2>&1 | tee maven-build.log
7272
7373
- name: Report failing tests (if any)
7474
if: always()

‎flushall_fast_build_and_run.sh‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/bin/bash
2+
3+
# Fast build script - skips clean, uses parallel builds, more RAM
4+
#
5+
# This script should be run from the OBP-API root directory:
6+
# cd /path/to/OBP-API
7+
# ./flushall_fast_build_and_run.sh
8+
#
9+
# Options:
10+
# --clean Force a clean build (slower, but useful if you have issues)
11+
# --offline Skip checking remote repos (faster if deps haven't changed)
12+
#
13+
# The http4s server will run in the background on port 8081
14+
# The Jetty server will run in the foreground on port 8080
15+
16+
set -e # Exit on error
17+
18+
# Parse arguments
19+
DO_CLEAN=""
20+
OFFLINE_FLAG=""
21+
for arg in "$@"; do
22+
case $arg in
23+
--clean)
24+
DO_CLEAN="clean"
25+
echo ">>> Clean build requested"
26+
;;
27+
--offline)
28+
OFFLINE_FLAG="-o"
29+
echo ">>> Offline mode enabled"
30+
;;
31+
esac
32+
done
33+
34+
# Detect CPU cores for parallel builds
35+
if command -v nproc &> /dev/null; then
36+
CORES=$(nproc)
37+
elif command -v sysctl &> /dev/null; then
38+
CORES=$(sysctl -n hw.ncpu)
39+
else
40+
CORES=4
41+
fi
42+
echo ">>> Using $CORES CPU cores for parallel builds"
43+
44+
# Common Maven options for better performance
45+
# - More heap memory (4G-8G)
46+
# - More metaspace (2G)
47+
# - Larger stack for Scala compiler
48+
# - Java module opens for compatibility
49+
export MAVEN_OPTS="-Xms4G -Xmx8G -XX:MaxMetaspaceSize=2G -Xss128m \
50+
--add-opens java.base/java.lang=ALL-UNNAMED \
51+
--add-opens java.base/java.lang.reflect=ALL-UNNAMED \
52+
--add-opens java.base/java.util=ALL-UNNAMED \
53+
--add-opens java.base/java.lang.invoke=ALL-UNNAMED \
54+
--add-opens java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED"
55+
56+
echo "=========================================="
57+
echo "Flushing Redis cache..."
58+
echo "=========================================="
59+
redis-cli <<EOF
60+
flushall
61+
exit
62+
EOF
63+
64+
if [ $? -eq 0 ]; then
65+
echo "Redis cache flushed successfully"
66+
else
67+
echo "Warning: Failed to flush Redis cache. Continuing anyway..."
68+
fi
69+
70+
echo ""
71+
echo "=========================================="
72+
echo "Fast build: installing commons + packaging http4s runner..."
73+
echo "=========================================="
74+
75+
# Build everything needed in ONE Maven invocation:
76+
# - Install root pom and obp-commons first
77+
# - Then package obp-http4s-runner (which depends on obp-api)
78+
#
79+
# Optimizations:
80+
# - -T 1C: Use 1 thread per CPU core
81+
# - No 'clean' by default (incremental build)
82+
# - -DskipTests: Skip running tests
83+
# - -Dmaven.test.skip: Skip compiling tests too
84+
# - $OFFLINE_FLAG: Skip remote repo checks if --offline passed
85+
86+
mvn install -pl .,obp-commons \
87+
-T 1C \
88+
$OFFLINE_FLAG \
89+
-DskipTests=true \
90+
-Dmaven.test.skip=true \
91+
-Dcheckstyle.skip=true \
92+
-Dspotbugs.skip=true \
93+
-Dpmd.skip=true
94+
95+
mvn package -pl obp-http4s-runner -am \
96+
-T 1C \
97+
$DO_CLEAN \
98+
$OFFLINE_FLAG \
99+
-DskipTests=true \
100+
-Dmaven.test.skip=true \
101+
-Dcheckstyle.skip=true \
102+
-Dspotbugs.skip=true \
103+
-Dpmd.skip=true
104+
105+
echo ""
106+
echo "=========================================="
107+
echo "Starting http4s server in background..."
108+
echo "=========================================="
109+
java -jar obp-http4s-runner/target/obp-http4s-runner.jar > http4s-server.log 2>&1 &
110+
HTTP4S_PID=$!
111+
echo "http4s server started with PID: $HTTP4S_PID (port 8081)"
112+
echo "Logs are being written to: http4s-server.log"
113+
echo ""
114+
echo "To stop http4s server later: kill $HTTP4S_PID"
115+
echo ""
116+
117+
echo "=========================================="
118+
echo "Starting Jetty server (foreground)..."
119+
echo "=========================================="
120+
mvn jetty:run -pl obp-api $OFFLINE_FLAG

‎flushall_http4s_build_and_run.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
# Script to flush Redis, build the project, and run the http4s server
4+
#
5+
# This script should be run from the OBP-API root directory:
6+
# cd /path/to/OBP-API
7+
# ./flushall_http4s_build_and_run.sh
8+
#
9+
# The http4s server will run in the foreground on the port configured
10+
# in your props file (default: 8086)
11+
12+
set -e # Exit on error
13+
14+
echo "=========================================="
15+
echo "Flushing Redis cache..."
16+
echo "=========================================="
17+
redis-cli <<EOF
18+
flushall
19+
exit
20+
EOF
21+
22+
if [ $? -eq 0 ]; then
23+
echo "Redis cache flushed successfully"
24+
else
25+
echo "Warning: Failed to flush Redis cache. Continuing anyway..."
26+
fi
27+
28+
echo ""
29+
echo "=========================================="
30+
echo "Building with Maven..."
31+
echo "=========================================="
32+
export MAVEN_OPTS="-Xms3G -Xmx6G -XX:MaxMetaspaceSize=2G"
33+
mvn -pl obp-http4s-runner -am clean package -DskipTests=true -Dmaven.test.skip=true
34+
35+
echo ""
36+
echo "=========================================="
37+
echo "Starting http4s server (foreground)..."
38+
echo "=========================================="
39+
java \
40+
--add-opens java.base/java.lang=ALL-UNNAMED \
41+
--add-opens java.base/java.lang.reflect=ALL-UNNAMED \
42+
--add-opens java.base/java.util=ALL-UNNAMED \
43+
--add-opens java.base/java.lang.invoke=ALL-UNNAMED \
44+
--add-opens java.base/java.util.jar=ALL-UNNAMED \
45+
--add-opens java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \
46+
-jar obp-http4s-runner/target/obp-http4s-runner.jar

‎obp-api/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,19 @@
445445
</exclusion>
446446
</exclusions>
447447
</dependency>
448+
449+
<!-- Doobie - Pure functional JDBC layer for raw SQL queries -->
450+
<!-- Handles all JDBC types correctly (including SQL Server NVARCHAR) -->
451+
<dependency>
452+
<groupId>org.tpolecat</groupId>
453+
<artifactId>doobie-core_${scala.version}</artifactId>
454+
<version>1.0.0-RC4</version>
455+
</dependency>
456+
<dependency>
457+
<groupId>org.tpolecat</groupId>
458+
<artifactId>doobie-hikari_${scala.version}</artifactId>
459+
<version>1.0.0-RC4</version>
460+
</dependency>
448461
<dependency>
449462
<groupId>com.microsoft.sqlserver</groupId>
450463
<artifactId>mssql-jdbc</artifactId>

‎obp-api/src/main/resources/props/sample.props.template‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -681,6 +681,7 @@ supported_locales = en_GB,es_ES,ro_RO
681681
apiOptions.getBranchesIsPublic = true
682682
apiOptions.getAtmsIsPublic = true
683683
apiOptions.getProductsIsPublic = true
684+
apiOptions.getApiProductsIsPublic = true
684685
apiOptions.getTransactionTypesIsPublic = true
685686
apiOptions.getCurrentFxRateIsPublic = true
686687

@@ -1613,7 +1614,7 @@ ethereum.rpc.url=http://127.0.0.1:8545
16131614

16141615

16151616
##########################################################
1616-
# Redis Logging #
1617+
# Redis Logging Log Cache #
16171618
##########################################################
16181619
## Enable Redis logging (true/false)
16191620
redis_logging_enabled = false
@@ -1701,4 +1702,4 @@ securelogging_mask_email=true
17011702
# Host and port for http4s server (used by bootstrap.http4s.Http4sServer)
17021703
# Defaults (if not set) are 127.0.0.1 and 8086
17031704
http4s.host=127.0.0.1
1704-
http4s.port=8086
1705+
http4s.port=8086

‎obp-api/src/main/scala/bootstrap/liftweb/Boot.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ import code.api.util._
4949
import code.api.util.migration.Migration
5050
import code.api.util.migration.Migration.DbFunction
5151
import code.apicollection.ApiCollection
52+
import code.apiproduct.ApiProduct
53+
import code.apiproductattribute.ApiProductAttribute
5254
import code.featuredapicollection.FeaturedApiCollection
5355
import code.apicollectionendpoint.ApiCollectionEndpoint
5456
import code.atmattribute.AtmAttribute
@@ -1119,6 +1121,8 @@ object ToSchemify {
11191121
MappedUserRefreshes,
11201122
ApiCollection,
11211123
ApiCollectionEndpoint,
1124+
ApiProduct,
1125+
ApiProductAttribute,
11221126
FeaturedApiCollection,
11231127
JsonSchemaValidation,
11241128
AuthenticationTypeValidation,

‎obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2870,6 +2870,26 @@ object SwaggerDefinitionsJSON {
28702870
created = DateWithDayExampleObject,
28712871
logo_url = Some(logoURLExample.value)
28722872
)
2873+
2874+
lazy val consumerJsonV600: ConsumerJsonV600 = ConsumerJsonV600(
2875+
consumer_id = consumerIdExample.value,
2876+
consumer_key = consumerKeyExample.value,
2877+
app_name = appNameExample.value,
2878+
app_type = appTypeExample.value,
2879+
description = descriptionExample.value,
2880+
developer_email = emailExample.value,
2881+
company = companyExample.value,
2882+
redirect_url = redirectUrlExample.value,
2883+
certificate_pem = pem,
2884+
certificate_info = Some(certificateInfoJsonV510),
2885+
created_by_user = resourceUserJSON,
2886+
enabled = true,
2887+
created = DateWithDayExampleObject,
2888+
logo_url = Some(logoURLExample.value),
2889+
active_rate_limits = activeRateLimitsJsonV600,
2890+
call_counters = redisCallCountersJsonV600
2891+
)
2892+
28732893
lazy val consumerJsonOnlyForPostResponseV510: ConsumerJsonOnlyForPostResponseV510 = ConsumerJsonOnlyForPostResponseV510(
28742894
consumer_id = consumerIdExample.value,
28752895
consumer_key = consumerKeyExample.value,
@@ -3068,6 +3088,27 @@ object SwaggerDefinitionsJSON {
30683088
lazy val metricsJsonV510 = MetricsJsonV510(
30693089
metrics = List(metricJson510)
30703090
)
3091+
lazy val metricJsonV600 = MetricJsonV600(
3092+
user_id = ExampleValue.userIdExample.value,
3093+
url = "www.openbankproject.com",
3094+
date = DateWithDayExampleObject,
3095+
user_name = "OBP",
3096+
app_name = "SOFI",
3097+
developer_email = ExampleValue.emailExample.value,
3098+
implemented_by_partial_function = "getBanks",
3099+
implemented_in_version = "v210",
3100+
consumer_id = "123",
3101+
verb = "get",
3102+
correlation_id = "v8ho6h5ivel3uq7a5zcnv0w1",
3103+
duration = 39,
3104+
source_ip = "2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b",
3105+
target_ip = "2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b",
3106+
response_body = json.parse("""{"code":401,"message":"OBP-20001: User not logged in. Authentication is required!"}"""),
3107+
operation_id = "OBPv4.0.0-getBanks"
3108+
)
3109+
lazy val metricsJsonV600 = MetricsJsonV600(
3110+
metrics = List(metricJsonV600)
3111+
)
30713112

30723113
lazy val branchJsonPut = BranchJsonPutV210("gh.29.fi", "OBP",
30733114
addressJsonV140,
@@ -5182,6 +5223,44 @@ object SwaggerDefinitionsJSON {
51825223
lazy val featuredApiCollectionJsonV600 = FeaturedApiCollectionJsonV600(featuredApiCollectionIdExample.value, apiCollectionIdExample.value, 1)
51835224
lazy val featuredApiCollectionsJsonV600 = FeaturedApiCollectionsJsonV600(List(featuredApiCollectionJsonV600))
51845225

5226+
// Api Product (v6.0.0)
5227+
lazy val apiProductAttributeResponseJsonV600 = ApiProductAttributeResponseJsonV600(
5228+
bank_id = bankIdExample.value,
5229+
api_product_code = productCodeExample.value,
5230+
api_product_attribute_id = "api-product-attribute-id-123",
5231+
name = "OVERDRAFT_LIMIT",
5232+
`type` = "STRING",
5233+
value = "10000",
5234+
is_active = Some(true)
5235+
)
5236+
lazy val apiProductAttributeJsonV600 = ApiProductAttributeJsonV600(
5237+
name = "OVERDRAFT_LIMIT",
5238+
`type` = "STRING",
5239+
value = "10000",
5240+
is_active = Some(true)
5241+
)
5242+
lazy val postPutApiProductJsonV600 = PostPutApiProductJsonV600(
5243+
parent_api_product_code = Some(""),
5244+
name = "ApiProduct1",
5245+
category = Some("category1"),
5246+
more_info_url = Some("https://example.com/more-info"),
5247+
terms_and_conditions_url = Some("https://example.com/terms"),
5248+
description = Some("Description of the product")
5249+
)
5250+
lazy val apiProductJsonV600 = ApiProductJsonV600(
5251+
api_product_id = "api-product-id-123",
5252+
bank_id = bankIdExample.value,
5253+
api_product_code = productCodeExample.value,
5254+
parent_api_product_code = "",
5255+
name = "ApiProduct1",
5256+
category = "category1",
5257+
more_info_url = "https://example.com/more-info",
5258+
terms_and_conditions_url = "https://example.com/terms",
5259+
description = "Description of the product",
5260+
attributes = Some(List(apiProductAttributeResponseJsonV600))
5261+
)
5262+
lazy val apiProductsJsonV600 = ApiProductsJsonV600(List(apiProductJsonV600))
5263+
51855264
lazy val jsonScalaConnectorMethod = JsonConnectorMethod(Some(connectorMethodIdExample.value),"getBank", connectorMethodBodyScalaExample.value, "Scala")
51865265
lazy val jsonScalaConnectorMethodMethodBody = JsonConnectorMethodMethodBody(connectorMethodBodyScalaExample.value, "Scala")
51875266

‎obp-api/src/main/scala/code/api/cache/Redis.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ object Redis extends MdcLoggable {
177177
jedisConnection.head.del(key).toString
178178
}else if (method ==JedisMethod.GET) {
179179
jedisConnection.head.get(key)
180+
} else if (method == JedisMethod.SCAN) {
181+
import scala.collection.JavaConverters._
182+
jedisConnection.head.keys(key).asScala.mkString(",")
180183
} else if(method ==JedisMethod.SET && value.isDefined){
181184
if (ttlSeconds.isDefined) {//if set ttl, call `setex` method to set the expired seconds.
182185
jedisConnection.head.setex(key, ttlSeconds.get, value.get).toString

‎obp-api/src/main/scala/code/api/constant/constant.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@ object Constant extends MdcLoggable {
221221
final val METRICS_STABLE_NAMESPACE = "metrics_stable"
222222
final val METRICS_RECENT_NAMESPACE = "metrics_recent"
223223
final val ABAC_RULE_NAMESPACE = "abac_rule"
224+
final val CONNECTOR_OUTBOUND_NAMESPACE = "connector_outbound"
225+
final val CONNECTOR_INBOUND_NAMESPACE = "connector_inbound"
224226

225227
// List of all versioned cache namespaces
226228
final val ALL_CACHE_NAMESPACES = List(
@@ -234,7 +236,9 @@ object Constant extends MdcLoggable {
234236
CONNECTOR_NAMESPACE,
235237
METRICS_STABLE_NAMESPACE,
236238
METRICS_RECENT_NAMESPACE,
237-
ABAC_RULE_NAMESPACE
239+
ABAC_RULE_NAMESPACE,
240+
CONNECTOR_OUTBOUND_NAMESPACE,
241+
CONNECTOR_INBOUND_NAMESPACE
238242
)
239243

240244
// Cache key prefixes with global namespace and versioning for easy invalidation
@@ -266,6 +270,10 @@ object Constant extends MdcLoggable {
266270
// ABAC Cache Prefixes (with global namespace and versioning)
267271
def ABAC_RULE_PREFIX: String = getVersionedCachePrefix(ABAC_RULE_NAMESPACE)
268272

273+
// Connector Metrics Redis Counter Prefixes (with global namespace and versioning)
274+
def CONNECTOR_OUTBOUND_PREFIX: String = getVersionedCachePrefix(CONNECTOR_OUTBOUND_NAMESPACE)
275+
def CONNECTOR_INBOUND_PREFIX: String = getVersionedCachePrefix(CONNECTOR_INBOUND_NAMESPACE)
276+
269277
// ABAC Policy Constants
270278
final val ABAC_POLICY_ACCOUNT_ACCESS = "account-access"
271279

0 commit comments

Comments
 (0)