Code refactor

adam-pawelek · Feb 10, 2024 · 0fbbc9c · 0fbbc9c
1 parent 1194c03
commit 0fbbc9c
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/config/development.json b/config/development.json
@@ -1,6 +1,6 @@
 {
     "spark_session_configs": {
-        "spark.app.name": "Display Two Random Files from Public S3 Bucket",
+        "spark.app.name": "Distributed-PE-Processor",
         "spark.master": "local",
         "spark.jars.packages": "org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.262,org.postgresql:postgresql:42.5.0",
         "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",

diff --git a/config/e2e.json b/config/e2e.json
@@ -1,6 +1,6 @@
 {
     "spark_session_configs": {
-        "spark.app.name": "Display Two Random Files from Public S3 Bucket",
+        "spark.app.name": "Distributed-PE-Processor",
         "spark.master": "spark://spark-master:7077",
         "spark.jars.packages": "org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.262,org.postgresql:postgresql:42.5.0",
         "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",

diff --git a/config/production.json b/config/production.json
@@ -1,6 +1,6 @@
 {
     "spark_session_configs": {
-        "spark.app.name": "Display Two Random Files from Public S3 Bucket",
+        "spark.app.name": "Distributed-PE-Processor",
         "spark.master": "spark://spark-master:7077",
         "spark.jars.packages": "org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.262,org.postgresql:postgresql:42.5.0",
         "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",

diff --git a/distributed_pe_processor/src/load/db_loader.py b/distributed_pe_processor/src/load/db_loader.py
@@ -32,7 +32,7 @@ def load_to_db(metadata_rdd: pyspark.rdd.RDD, db_connection_data: ConnectionData
             properties=db_connection_data.properties
 
         )
-        logger.info(f"Successfully written DataFrame to PostgreSQL table: {db_connection_data.table_name}")
+        logger.info(f"The DataFrame has been successfully written to the PostgreSQL.: {db_connection_data.table_name}")
 
     except Exception as e:
         logger.error(f"Failed to load data to DB. Error: {str(e)}")