From e8cc4315542d6f9585145fecc89892c537f991f2 Mon Sep 17 00:00:00 2001 From: overpathz Date: Sun, 5 Jan 2025 15:46:16 +0200 Subject: [PATCH] init --- .mvn/wrapper/maven-wrapper.properties | 19 ++ Dockerfile | 15 + HELP.md | 33 +++ deployment.yml | 41 +++ docker-compose.yml | 132 +++++++++ filebeat/filebeat.docker.yml | 33 +++ init-db/init-payment-intents.sql | 19 ++ init-db/init.sql | 8 + mvnw | 259 ++++++++++++++++++ mvnw.cmd | 149 ++++++++++ pom.xml | 125 +++++++++ prometheus/prometheus.yml | 11 + .../DistributedJobProcessorApplication.java | 15 + .../ClientProviderNetworkException.java | 7 + .../client/ExternalPaymentClient.java | 41 +++ .../config/ExecutorConfig.java | 17 ++ .../config/RestTemplateConfig.java | 36 +++ .../config/RetryConfig.java | 27 ++ .../controller/HelloWorldController.java | 13 + .../entity/PaymentIntent.java | 47 ++++ .../repository/PaymentIntentRepository.java | 32 +++ .../scheduler/PaymentIntentScheduler.java | 33 +++ .../service/BatchUpdater.java | 47 ++++ .../service/PaymentIntentService.java | 112 ++++++++ src/main/resources/application.yml | 49 ++++ ...stributedJobProcessorApplicationTests.java | 13 + 26 files changed, 1333 insertions(+) create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 Dockerfile create mode 100644 HELP.md create mode 100644 deployment.yml create mode 100644 docker-compose.yml create mode 100644 filebeat/filebeat.docker.yml create mode 100644 init-db/init-payment-intents.sql create mode 100644 init-db/init.sql create mode 100755 mvnw create mode 100644 mvnw.cmd create mode 100644 pom.xml create mode 100644 prometheus/prometheus.yml create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplication.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/client/ClientProviderNetworkException.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/client/ExternalPaymentClient.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/config/ExecutorConfig.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/config/RestTemplateConfig.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/config/RetryConfig.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/controller/HelloWorldController.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/entity/PaymentIntent.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/repository/PaymentIntentRepository.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/scheduler/PaymentIntentScheduler.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/service/BatchUpdater.java create mode 100644 src/main/java/com/overpathz/distributedjobprocessor/service/PaymentIntentService.java create mode 100644 src/main/resources/application.yml create mode 100644 src/test/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplicationTests.java diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..d58dfb7 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0a7749e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM maven:3.9.5-eclipse-temurin-21 AS build-env +WORKDIR /app + +COPY pom.xml . +RUN mvn dependency:go-offline + +COPY src ./src +RUN mvn package -DskipTests + +FROM openjdk:21-slim +VOLUME /tmp + +COPY --from=build-env /app/target/distributed-job-processor-0.0.1.jar /app/app.jar + +ENTRYPOINT ["java", "-jar", "/app/app.jar"] diff --git a/HELP.md b/HELP.md new file mode 100644 index 0000000..259c642 --- /dev/null +++ b/HELP.md @@ -0,0 +1,33 @@ +# Getting Started + +### Reference Documentation + +For further reference, please consider the following sections: + +* [Official Apache Maven documentation](https://maven.apache.org/guides/index.html) +* [Spring Boot Maven Plugin Reference Guide](https://docs.spring.io/spring-boot/3.4.1/maven-plugin) +* [Create an OCI image](https://docs.spring.io/spring-boot/3.4.1/maven-plugin/build-image.html) +* [Resilience4J](https://docs.spring.io/spring-cloud-circuitbreaker/reference/spring-cloud-circuitbreaker-resilience4j.html) +* [Spring Data JPA](https://docs.spring.io/spring-boot/3.4.1/reference/data/sql.html#data.sql.jpa-and-spring-data) +* [Spring Data Redis (Access+Driver)](https://docs.spring.io/spring-boot/3.4.1/reference/data/nosql.html#data.nosql.redis) +* [Prometheus](https://docs.spring.io/spring-boot/3.4.1/reference/actuator/metrics.html#actuator.metrics.export.prometheus) +* [Spring Web](https://docs.spring.io/spring-boot/3.4.1/reference/web/servlet.html) + +### Guides + +The following guides illustrate how to use some features concretely: + +* [Accessing Data with JPA](https://spring.io/guides/gs/accessing-data-jpa/) +* [Messaging with Redis](https://spring.io/guides/gs/messaging-redis/) +* [Building a RESTful Web Service](https://spring.io/guides/gs/rest-service/) +* [Serving Web Content with Spring MVC](https://spring.io/guides/gs/serving-web-content/) +* [Building REST services with Spring](https://spring.io/guides/tutorials/rest/) + +### Maven Parent overrides + +Due to Maven's design, elements are inherited from the parent POM to the project POM. +While most of the inheritance is fine, it also inherits unwanted elements like `` and `` from the +parent. +To prevent this, the project POM contains empty overrides for these elements. +If you manually switch to a different parent and actually want the inheritance, you need to remove those overrides. + diff --git a/deployment.yml b/deployment.yml new file mode 100644 index 0000000..b5ff588 --- /dev/null +++ b/deployment.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: distributed-job-processor + labels: + app: distributed-job-processor +spec: + replicas: 3 # Set the number of replicas here + selector: + matchLabels: + app: distributed-job-processor + template: + metadata: + labels: + app: distributed-job-processor + spec: + containers: + - name: distributed-job-processor + image: distributed-job-processor:0.0.1 + ports: + - containerPort: 8080 + env: + - name: SPRING_DATASOURCE_URL + value: "jdbc:postgresql://host.docker.internal:5432/paymentsdb" + - name: SPRING_DATASOURCE_USERNAME + value: "puser" + - name: SPRING_DATASOURCE_PASSWORD + value: "ppass" +--- +apiVersion: v1 +kind: Service +metadata: + name: distributed-job-processor-service +spec: + selector: + app: distributed-job-processor + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: NodePort # Expose on a specific port for testing diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e23e48d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,132 @@ +version: "3.8" + +services: + # 1. POSTGRES DB + db: + image: postgres:15-alpine + container_name: distributed-job-processor-postgres + environment: + POSTGRES_DB: paymentsdb + POSTGRES_USER: puser + POSTGRES_PASSWORD: ppass + ports: + - "5432:5432" + volumes: + - db-data:/var/lib/postgresql/data + - ./init-db/init.sql:/docker-entrypoint-initdb.d/init.sql + - ./init-db/init-payment-intents.sql:/docker-entrypoint-initdb.d/init-payment-intents.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U puser -d paymentsdb"] + interval: 5s + timeout: 5s + retries: 5 + + # 2. DISTRIBUTED-JOB-PROCESSOR Spring Boot App + distributed-job-processor: + build: + context: . + dockerfile: Dockerfile + image: com.overpathz/distributed-job-processor:0.0.1 + container_name: distributed-job-processor-app + depends_on: + db: + condition: service_healthy + ports: + - "8080:8080" + environment: + SPRING_DATASOURCE_URL: "jdbc:postgresql://db:5432/paymentsdb" + SPRING_DATASOURCE_USERNAME: "puser" + SPRING_DATASOURCE_PASSWORD: "ppass" + # ? Optional stuff. Expose Prometheus endpoint on /actuator/prometheus + # (Actuator + Micrometer dependencies required) + + # 3. ELASTICSEARCH (logs from filebeat) + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.9.2 + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false # disable security for local development + - xpack.security.transport.ssl.enabled=false + - xpack.security.http.ssl.enabled=false + - ES_JAVA_OPTS=-Xms512m -Xmx512m # configurable :) + volumes: + - es-data:/usr/share/elasticsearch/data + ports: + - "9200:9200" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"] + interval: 5s + timeout: 5s + retries: 5 + + # 4. KIBANA (logs from elasticsearch) + kibana: + image: docker.elastic.co/kibana/kibana:8.9.2 + container_name: kibana + depends_on: + elasticsearch: + condition: service_healthy + environment: + - SERVER_NAME=kibana + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - XPACK_SECURITY_ENABLED=false + ports: + - "5601:5601" + + # 5. FILEBEAT (collects container logs and sends to Elasticsearch) + filebeat: + image: docker.elastic.co/beats/filebeat:8.9.2 + container_name: filebeat + user: root + depends_on: + elasticsearch: + condition: service_healthy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ./filebeat/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + # If you enable ES security, specify credentials or tokens + + # 6. PROMETHEUS. Scrapes metrics from our docker container + prometheus: + image: prom/prometheus:v2.47.1 + container_name: prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.enable-lifecycle" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9090:9090" + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://localhost:9090/-/healthy || exit 1"] + interval: 5s + timeout: 5s + retries: 5 + + # 7. GRAFANA. Dashboard for metrics from Prometheus + grafana: + image: grafana/grafana:10.1.0 + container_name: grafana + depends_on: + prometheus: + condition: service_healthy + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD: "admin" + GF_SERVER_DOMAIN: "localhost" + GF_SERVER_ROOT_URL: "http://localhost:3000" + volumes: + - grafana-data:/var/lib/grafana + +volumes: + db-data: + name: db-data + es-data: + grafana-data: diff --git a/filebeat/filebeat.docker.yml b/filebeat/filebeat.docker.yml new file mode 100644 index 0000000..f26e778 --- /dev/null +++ b/filebeat/filebeat.docker.yml @@ -0,0 +1,33 @@ +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + templates: + - condition: + contains: + docker.container.name: "distributed-job-processor-app" + config: + - type: container + paths: + - /var/lib/docker/containers/${data.docker.container.id}/*.log + # You can add more parse or multiline settings here if needed + + - type: docker + hints.enabled: true + templates: + - condition: + contains: + docker.container.name: "distributed-job-processor-postgres" + config: + - type: container + paths: + - /var/lib/docker/containers/${data.docker.container.id}/*.log + +# -> logs to Elasticsearch +output.elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + # If security is enabled, add username/password or token + +# (Optional) console output for debugging +# output.console: +# pretty: true diff --git a/init-db/init-payment-intents.sql b/init-db/init-payment-intents.sql new file mode 100644 index 0000000..5815435 --- /dev/null +++ b/init-db/init-payment-intents.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS payment_intent ( + id SERIAL PRIMARY KEY, + amount NUMERIC(10, 2) NOT NULL, + status VARCHAR(50) NOT NULL, + created_at TIMESTAMP NOT NULL + ); + + +DO +$$ +DECLARE +i INT; +BEGIN +FOR i IN 1..1000000 LOOP + INSERT INTO payment_intent (amount, status, created_at) + VALUES ((100 + i % 10), 'NEW', NOW()); +END LOOP; +END +$$; diff --git a/init-db/init.sql b/init-db/init.sql new file mode 100644 index 0000000..5af7bf6 --- /dev/null +++ b/init-db/init.sql @@ -0,0 +1,8 @@ +DO +$$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM pg_database WHERE datname = 'paymentsdb') THEN + CREATE DATABASE paymentsdb; +END IF; +END +$$; \ No newline at end of file diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..19529dd --- /dev/null +++ b/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..249bdf3 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..f05de2a --- /dev/null +++ b/pom.xml @@ -0,0 +1,125 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.4.1 + + + com.overpathz + distributed-job-processor + 0.0.1 + distributed-job-processor + distributed-job-processor + + + + + + + + + + + + + + + 21 + 2024.0.0 + + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-data-jpa + + + io.github.resilience4j + resilience4j-spring-boot3 + + + org.springframework.boot + spring-boot-starter-data-redis + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-starter-circuitbreaker-resilience4j + + + org.postgresql + postgresql + 42.7.4 + + + org.apache.httpcomponents.client5 + httpclient5 + 5.4.1 + + + io.micrometer + micrometer-registry-prometheus + + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.projectlombok + lombok + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..b240b23 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "distributed-job-processor" + static_configs: + - targets: + - "distributed-job-processor-service:8080" + metrics_path: "/actuator/prometheus" + scheme: http \ No newline at end of file diff --git a/src/main/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplication.java b/src/main/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplication.java new file mode 100644 index 0000000..ebf2776 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplication.java @@ -0,0 +1,15 @@ +package com.overpathz.distributedjobprocessor; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@EnableScheduling +public class DistributedJobProcessorApplication { + + public static void main(String[] args) { + SpringApplication.run(DistributedJobProcessorApplication.class, args); + } + +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/client/ClientProviderNetworkException.java b/src/main/java/com/overpathz/distributedjobprocessor/client/ClientProviderNetworkException.java new file mode 100644 index 0000000..db289af --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/client/ClientProviderNetworkException.java @@ -0,0 +1,7 @@ +package com.overpathz.distributedjobprocessor.client; + +public class ClientProviderNetworkException extends RuntimeException { + public ClientProviderNetworkException(String message) { + super(message); + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/client/ExternalPaymentClient.java b/src/main/java/com/overpathz/distributedjobprocessor/client/ExternalPaymentClient.java new file mode 100644 index 0000000..26872aa --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/client/ExternalPaymentClient.java @@ -0,0 +1,41 @@ +package com.overpathz.distributedjobprocessor.client; + +import com.overpathz.distributedjobprocessor.entity.PaymentIntent; +import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +@Slf4j +public class ExternalPaymentClient { + + /* Simulate the network call */ + + private final Map prevTaskIdMap = new ConcurrentHashMap<>(); + + @CircuitBreaker(name = "externalServiceCircuitBreaker", fallbackMethod = "fallbackForCircuitBreaker") + @Retry(name = "externalServiceRetry") + public boolean sendPayment(PaymentIntent intent) { + if (Boolean.TRUE.equals(prevTaskIdMap.get(intent.getId()))) { + log.info("Retry was triggered on payment={}", intent); + } + try { + Thread.sleep(50); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted"); + } + } + + // The fallback method signature must match the original method’s signature + Throwable + public boolean fallbackForCircuitBreaker(PaymentIntent intent, Throwable t) { + // This is called if circuit is open or retries are exhausted + // Return false => treat as failed + return false; + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/config/ExecutorConfig.java b/src/main/java/com/overpathz/distributedjobprocessor/config/ExecutorConfig.java new file mode 100644 index 0000000..6641e42 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/config/ExecutorConfig.java @@ -0,0 +1,17 @@ +package com.overpathz.distributedjobprocessor.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Configuration +public class ExecutorConfig { + + @Bean + public ExecutorService externalCallExecutor() { + // we can tune thread numbers based on our perf tests + return Executors.newFixedThreadPool(10); + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/config/RestTemplateConfig.java b/src/main/java/com/overpathz/distributedjobprocessor/config/RestTemplateConfig.java new file mode 100644 index 0000000..c49553d --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/config/RestTemplateConfig.java @@ -0,0 +1,36 @@ +package com.overpathz.distributedjobprocessor.config; + +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import java.util.concurrent.TimeUnit; + +@Configuration +public class RestTemplateConfig { + + @Bean + public RestTemplate restTemplate() { + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setMaxTotal(200); + connectionManager.setDefaultMaxPerRoute(50); + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(5000, TimeUnit.MILLISECONDS) + .setConnectionRequestTimeout(5000, TimeUnit.MILLISECONDS) + .build(); + + CloseableHttpClient httpClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .setDefaultRequestConfig(requestConfig) + .build(); + + HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient); + return new RestTemplate(requestFactory); + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/config/RetryConfig.java b/src/main/java/com/overpathz/distributedjobprocessor/config/RetryConfig.java new file mode 100644 index 0000000..1732aff --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/config/RetryConfig.java @@ -0,0 +1,27 @@ +package com.overpathz.distributedjobprocessor.config; + +import io.github.resilience4j.retry.RetryRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Slf4j +public class RetryConfig { + + @Bean + public RetryRegistry retryRegistry() { + log.info("Initializing retry registry"); + + RetryRegistry registry = RetryRegistry.ofDefaults(); + + registry.retry("externalServiceRetry").getEventPublisher() + .onRetry(event -> log.info("Retry attempt #{} for '{}'", + event.getNumberOfRetryAttempts(), + event.getName())) + .onSuccess(event -> log.info("Retry successful for '{}'", event.getName())) + .onError(event -> log.error("Retry failed for '{}'", event.getName(), event.getLastThrowable())); + + return registry; + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/controller/HelloWorldController.java b/src/main/java/com/overpathz/distributedjobprocessor/controller/HelloWorldController.java new file mode 100644 index 0000000..08f7610 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/controller/HelloWorldController.java @@ -0,0 +1,13 @@ +package com.overpathz.distributedjobprocessor.controller; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class HelloWorldController { + + @GetMapping + public String helloWorld() { + return "Hello World!"; + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/entity/PaymentIntent.java b/src/main/java/com/overpathz/distributedjobprocessor/entity/PaymentIntent.java new file mode 100644 index 0000000..0caed6d --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/entity/PaymentIntent.java @@ -0,0 +1,47 @@ +package com.overpathz.distributedjobprocessor.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +@Entity +@Table(name = "payment_intent") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PaymentIntent { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private BigDecimal amount; + + private String status; + + @Column(name = "created_at", updatable = false) + private LocalDateTime createdAt = LocalDateTime.now(); + + @Override + public String toString() { + return "PaymentIntent{" + + "id=" + id + + ", amount=" + amount + + ", status='" + status + '\'' + + ", createdAt=" + createdAt + + '}'; + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/repository/PaymentIntentRepository.java b/src/main/java/com/overpathz/distributedjobprocessor/repository/PaymentIntentRepository.java new file mode 100644 index 0000000..5ad5753 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/repository/PaymentIntentRepository.java @@ -0,0 +1,32 @@ +package com.overpathz.distributedjobprocessor.repository; + +import com.overpathz.distributedjobprocessor.entity.PaymentIntent; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface PaymentIntentRepository extends JpaRepository { + + @Query(value = """ + SELECT * + FROM payment_intent + WHERE status = 'NEW' + ORDER BY id + LIMIT :batchSize + FOR UPDATE SKIP LOCKED + """, nativeQuery = true) + List lockBatchForProcessing(@Param("batchSize") int batchSize); + + @Modifying + @Query(""" + UPDATE PaymentIntent p + SET p.status = :status + WHERE p.id IN :ids + """) + int updateStatusForIds(@Param("status") String status, @Param("ids") List ids); +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/scheduler/PaymentIntentScheduler.java b/src/main/java/com/overpathz/distributedjobprocessor/scheduler/PaymentIntentScheduler.java new file mode 100644 index 0000000..e73c717 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/scheduler/PaymentIntentScheduler.java @@ -0,0 +1,33 @@ +package com.overpathz.distributedjobprocessor.scheduler; + +import com.overpathz.distributedjobprocessor.entity.PaymentIntent; +import com.overpathz.distributedjobprocessor.service.PaymentIntentService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@Slf4j +public class PaymentIntentScheduler { + + private final PaymentIntentService paymentIntentService; + private static final int BATCH_SIZE = 1000; + + @Autowired + public PaymentIntentScheduler(PaymentIntentService paymentIntentService) { + this.paymentIntentService = paymentIntentService; + } + + @Scheduled(fixedRate = 5000, initialDelayString = "#{T(java.util.concurrent.ThreadLocalRandom).current().nextLong(1000, 10000)}") + public void processTransactions() { + log.info("Processing batch of transactions.."); + List batch = paymentIntentService.fetchAndLockBatch(BATCH_SIZE); + if (!batch.isEmpty()) { + paymentIntentService.processBatch(batch); + } + log.info("Processed {} transactions.", batch.size()); + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/service/BatchUpdater.java b/src/main/java/com/overpathz/distributedjobprocessor/service/BatchUpdater.java new file mode 100644 index 0000000..cf9271d --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/service/BatchUpdater.java @@ -0,0 +1,47 @@ +package com.overpathz.distributedjobprocessor.service; + +import com.overpathz.distributedjobprocessor.entity.PaymentIntent; +import com.overpathz.distributedjobprocessor.repository.PaymentIntentRepository; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Component +public class BatchUpdater { + private final PaymentIntentRepository paymentIntentRepository; + + public BatchUpdater(PaymentIntentRepository paymentIntentRepository) { + this.paymentIntentRepository = paymentIntentRepository; + } + + @Transactional + public void updateBatchStatus(List batch) { + List processedIds = batch.stream() + .filter(i -> "PROCESSED".equals(i.getStatus())) + .map(PaymentIntent::getId) + .toList(); + + List failedIds = batch.stream() + .filter(i -> "FAILED".equals(i.getStatus())) + .map(PaymentIntent::getId) + .toList(); + + if (!processedIds.isEmpty()) { + paymentIntentRepository.updateStatusForIds("PROCESSED", processedIds); + } + if (!failedIds.isEmpty()) { + paymentIntentRepository.updateStatusForIds("FAILED", failedIds); + } + } + + @Transactional + public void updateBatchStatus(List processedIds, List failedIds) { + if (!processedIds.isEmpty()) { + paymentIntentRepository.updateStatusForIds("PROCESSED", processedIds); + } + if (!failedIds.isEmpty()) { + paymentIntentRepository.updateStatusForIds("FAILED", failedIds); + } + } +} diff --git a/src/main/java/com/overpathz/distributedjobprocessor/service/PaymentIntentService.java b/src/main/java/com/overpathz/distributedjobprocessor/service/PaymentIntentService.java new file mode 100644 index 0000000..12d4781 --- /dev/null +++ b/src/main/java/com/overpathz/distributedjobprocessor/service/PaymentIntentService.java @@ -0,0 +1,112 @@ +package com.overpathz.distributedjobprocessor.service; + +import com.overpathz.distributedjobprocessor.client.ExternalPaymentClient; +import com.overpathz.distributedjobprocessor.entity.PaymentIntent; +import com.overpathz.distributedjobprocessor.repository.PaymentIntentRepository; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +@Service +@Slf4j +public class PaymentIntentService { + private final PaymentIntentRepository paymentIntentRepository; + private final ExternalPaymentClient externalPaymentClient; + private final BatchUpdater batchUpdater; + private final Timer batchProcessingTimer; + private final ExecutorService externalCallExecutor; + + @Autowired + public PaymentIntentService(PaymentIntentRepository paymentIntentRepository, + ExternalPaymentClient externalPaymentClient, + MeterRegistry meterRegistry, + BatchUpdater batchUpdater, ExecutorService externalCallExecutor) { + this.paymentIntentRepository = paymentIntentRepository; + this.externalPaymentClient = externalPaymentClient; + this.batchUpdater = batchUpdater; + + this.batchProcessingTimer = meterRegistry.timer("payment.batch.processing.time"); + this.externalCallExecutor = externalCallExecutor; + } + + /** + * 1) Lock a batch of rows in a short transaction & mark them IN_PROGRESS. + */ + @Transactional + public List fetchAndLockBatch(int batchSize) { + List batch = paymentIntentRepository.lockBatchForProcessing(batchSize); + for (PaymentIntent intent : batch) { + intent.setStatus("IN_PROGRESS"); + } + // They will be persisted as IN_PROGRESS automatically upon commit + return batch; + } + + /** + * 2) Process each row outside of transaction & update status in separate transaction. + */ +// public void processBatch(List batch) { +// log.info("Starting batch processing. New image"); +// long startTime = System.currentTimeMillis(); +// // Call external service outside the transaction +// // Call external service outside the transaction +// for (PaymentIntent intent : batch) { +// boolean success = externalPaymentClient.sendPayment(intent); +// intent.setStatus(success ? "PROCESSED" : "FAILED"); +// } +// // Update statuses in a new short transaction +// long startUpdateBatchStatus = System.currentTimeMillis(); +// batchUpdater.updateBatchStatus(batch); // not good, but leave for demo +// log.info("Update batch status in={} ms", System.currentTimeMillis() - startUpdateBatchStatus); +// log.info("Finishing batch processing. Took {} ms", System.currentTimeMillis() - startTime); +// } + + /** + * Processes each PaymentIntent in parallel using the ExecutorService. + */ + public void processBatch(List batch) { + log.info("Starting batch processing. New image 2"); + log.info("Starting batch processing for {}", batch.size()); + long startTime = System.currentTimeMillis(); + + List> futures = new ArrayList<>(batch.size()); + for (PaymentIntent intent : batch) { + futures.add(externalCallExecutor.submit(() -> { + boolean success = externalPaymentClient.sendPayment(intent); + return new ProcessingResult(intent.getId(), success); + })); + } + + List processedIds = new ArrayList<>(); + List failedIds = new ArrayList<>(); + + for (Future future : futures) { + try { + ProcessingResult result = future.get(); // blocking wait + if (result.success()) { + processedIds.add(result.paymentId()); + } else { + failedIds.add(result.paymentId()); + } + } catch (Exception e) { + // Right now, we are not capturing the ID unless we wrap this logic properly. + // I mean, we can wrap our Future task by some record with id to be able to know id + // when exception is thrown. + } + } + + batchUpdater.updateBatchStatus(processedIds, failedIds); + + log.info("Finishing batch processing. Took {} ms", System.currentTimeMillis() - startTime); + } + + record ProcessingResult(Long paymentId, boolean success) {} +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..359994b --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,49 @@ +spring: + application: + name: distributed-job-processor + datasource: + url: jdbc:postgresql://localhost:5432/paymentsdb + username: postgres + password: postgres + jpa: + hibernate: + ddl-auto: update + show-sql: false + +management: + endpoints: + web: + exposure: + include: "*" + endpoint: + prometheus: + enabled: true + prometheus: + metrics: + export: + enabled: true + +resilience4j: + circuitbreaker: + instances: + externalServiceCircuitBreaker: + sliding-window-type: COUNT_BASED # COUNT_BASED or TIME_BASED + sliding-window-size: 10 # Number of calls to consider for the circuit breaker + failure-rate-threshold: 50 # Percentage of failures to trigger the circuit breaker + wait-duration-in-open-state: 10s # Time the circuit breaker stays open before attempting recovery + permitted-number-of-calls-in-half-open-state: 5 # Calls allowed in half-open state + minimum-number-of-calls: 5 # Minimum calls to evaluate failure rate + automatic-transition-from-open-to-half-open-enabled: true + retry: + instances: + externalServiceRetry: + max-attempts: 3 + wait-duration: 500ms + retry-exceptions: + - com.overpathz.distributedjobprocessor.client.ClientProviderNetworkException + ignore-exceptions: + - java.lang.InterruptedException + +logging: + level: + root: INFO diff --git a/src/test/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplicationTests.java b/src/test/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplicationTests.java new file mode 100644 index 0000000..f18acce --- /dev/null +++ b/src/test/java/com/overpathz/distributedjobprocessor/DistributedJobProcessorApplicationTests.java @@ -0,0 +1,13 @@ +package com.overpathz.distributedjobprocessor; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DistributedJobProcessorApplicationTests { + + @Test + void contextLoads() { + } + +}