Skip to content

Commit 9b7d80f

Browse files
imtiazahmad007imtiazahmad007
committed
init commit of project 1
1 parent 5f3818e commit 9b7d80f

File tree

6 files changed

+178
-0
lines changed

6 files changed

+178
-0
lines changed

‎project1/.settings/org.eclipse.jdt.core.prefs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
eclipse.preferences.version=1
2+
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
3+
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
4+
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
5+
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
6+
org.eclipse.jdt.core.compiler.compliance=1.8
7+
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
8+
org.eclipse.jdt.core.compiler.debug.localVariable=generate
9+
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
10+
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
11+
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
12+
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
13+
org.eclipse.jdt.core.compiler.release=disabled
14+
org.eclipse.jdt.core.compiler.source=1.8

‎project1/.settings/org.eclipse.m2e.core.prefs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
activeProfiles=
2+
eclipse.preferences.version=1
3+
resolveWorkspaceProjects=true
4+
version=1

‎project1/pom.xml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2+
<modelVersion>4.0.0</modelVersion>
3+
<groupId>com.jobreadyprogrammer</groupId>
4+
<artifactId>project1</artifactId>
5+
<version>0.0.1-SNAPSHOT</version>
6+
<packaging>jar</packaging>
7+
8+
9+
<properties>
10+
11+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
12+
<java.version>1.8</java.version>
13+
14+
<scala.version>2.11</scala.version>
15+
<spark.version>2.3.1</spark.version>
16+
<postgresql.version>42.1.4</postgresql.version>
17+
18+
<maven.compiler.source>1.8</maven.compiler.source>
19+
<maven.compiler.target>1.8</maven.compiler.target>
20+
21+
</properties>
22+
23+
<dependencies>
24+
<!-- Spark -->
25+
<dependency>
26+
<groupId>org.apache.spark</groupId>
27+
<artifactId>spark-core_${scala.version}</artifactId>
28+
<version>${spark.version}</version>
29+
</dependency>
30+
31+
<dependency>
32+
<groupId>org.apache.spark</groupId>
33+
<artifactId>spark-sql_${scala.version}</artifactId>
34+
<version>${spark.version}</version>
35+
</dependency>
36+
37+
<dependency>
38+
<groupId>org.apache.spark</groupId>
39+
<artifactId>spark-mllib_${scala.version}</artifactId>
40+
<version>${spark.version}</version>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>junit</groupId>
45+
<artifactId>junit</artifactId>
46+
<version>4.11</version>
47+
<scope>test</scope>
48+
</dependency>
49+
50+
<dependency>
51+
<groupId>org.postgresql</groupId>
52+
<artifactId>postgresql</artifactId>
53+
<version>${postgresql.version}</version>
54+
</dependency>
55+
56+
57+
</dependencies>
58+
59+
<build>
60+
61+
<plugins>
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-dependency-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<id>copy-dependencies</id>
68+
<phase>prepare-package</phase>
69+
<goals>
70+
<goal>copy-dependencies</goal>
71+
</goals>
72+
<configuration>
73+
<outputDirectory>
74+
${project.build.directory}/libs
75+
</outputDirectory>
76+
</configuration>
77+
</execution>
78+
</executions>
79+
</plugin>
80+
81+
82+
<plugin>
83+
<groupId>org.springframework.boot</groupId>
84+
<artifactId>spring-boot-maven-plugin</artifactId>
85+
<executions>
86+
<execution>
87+
<goals>
88+
<goal>repackage</goal>
89+
</goals>
90+
<configuration>
91+
92+
<mainClass>com.jobreadyprogrammer.spark.Application</mainClass>
93+
94+
</configuration>
95+
</execution>
96+
</executions>
97+
</plugin>
98+
99+
100+
</plugins>
101+
102+
103+
</build>
104+
105+
106+
</project>

‎project1/src/main/java/com/jobreadyprogrammer/spark/Application.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package com.jobreadyprogrammer.spark;
2+
3+
import static org.apache.spark.sql.functions.concat;
4+
import static org.apache.spark.sql.functions.lit;
5+
6+
import java.util.Properties;
7+
8+
import org.apache.spark.sql.Dataset;
9+
import org.apache.spark.sql.Row;
10+
import org.apache.spark.sql.SaveMode;
11+
import org.apache.spark.sql.SparkSession;
12+
13+
public class Application {
14+
15+
public static void main(String args[]) throws InterruptedException {
16+
17+
// Create a session
18+
SparkSession spark = new SparkSession.Builder()
19+
.appName("CSV to DB")
20+
.master("local")
21+
.getOrCreate();
22+
23+
// get data
24+
Dataset<Row> df = spark.read().format("csv")
25+
.option("header", true)
26+
.load("src/main/resources/name_and_comments.txt");
27+
28+
// df.show(3);
29+
30+
// Transformation
31+
df = df.withColumn("full_name",
32+
concat(df.col("last_name"), lit(", "), df.col("first_name")))
33+
.filter(df.col("comment").rlike("\\d+"))
34+
.orderBy(df.col("last_name").asc());
35+
36+
// Write to destination
37+
String dbConnectionUrl = "jdbc:postgresql://localhost/course_data"; // <<- You need to create this database
38+
Properties prop = new Properties();
39+
prop.setProperty("driver", "org.postgresql.Driver");
40+
prop.setProperty("user", "postgres");
41+
prop.setProperty("password", "password"); // <- The password you used while installing Postgres
42+
43+
df.write()
44+
.mode(SaveMode.Overwrite)
45+
.jdbc(dbConnectionUrl, "project1", prop);
46+
}
47+
}

‎project1/src/main/resources/name_and_comments.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
last_name,first_name,comment
2+
Lon,Jim,There are plenty of people in this world.
3+
Ingram,Milford,I've been using the internet for 10.
4+
Gideon,Elmer,Social media has taken over our lives for good.
5+
Dong,Fen,The body is 70% water so make sure to stay hydrated.

‎project1/target/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/classes/
2+
/test-classes/

0 commit comments

Comments
 (0)