v0.1-SNAPSHOT

.gitignore

@@ -15,3 +15,4 @@ project/plugins/project/
 # Scala-IDE specific
 .scala_dependencies
 .worksheet
+/bin/

.travis.yml

@@ -8,4 +8,8 @@ jdk:
   - oraclejdk7
   - oraclejdk8
   - openjdk7
-  - openjdk8
+
+script:
+  - sbt ++$TRAVIS_SCALA_VERSION clean
+  - sbt ++$TRAVIS_SCALA_VERSION test
+  - sbt ++$TRAVIS_SCALA_VERSION one-jar

project/Build.scala

@@ -0,0 +1,21 @@
+import com.github.retronym.SbtOneJar
+import sbt._
+import Keys._
+
+object Build extends Build {
+
+  lazy val project = Project("root", file("."), settings = Seq(
+    name := "postgresql-to-sqlite",
+    organization := "com.github.caiiiycuk",
+    version := "0.0.1-SNAPSHOT",
+    scalaVersion := "2.11.7",
+
+    libraryDependencies ++= Seq(
+      "com.github.scopt" %% "scopt" % "3.3.0",
+      "ch.qos.logback" % "logback-classic" % "1.1.2",
+      "org.xerial" % "sqlite-jdbc" % "3.8.10.2",
+      "org.scalatest" %% "scalatest" % "2.2.4" % "test"
+    )
+  ) ++ SbtOneJar.oneJarSettings)
+
+}

project/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.5

project/plugins.sbt

@@ -0,0 +1,5 @@
+addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")
+
+addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
+
+addSbtPlugin("org.scala-sbt.plugins" % "sbt-onejar" % "0.8")

scalastyle-config.xml

@@ -0,0 +1,98 @@
+<scalastyle>
+ <name>Scalastyle standard configuration</name>
+ <check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true">
+  <parameters>
+   <parameter name="maxFileLength"><![CDATA[800]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
+  <parameters>
+   <parameter name="maxLineLength"><![CDATA[160]]></parameter>
+   <parameter name="tabSize"><![CDATA[4]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
+  <parameters>
+   <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
+  <parameters>
+   <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
+  <parameters>
+   <parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true">
+  <parameters>
+   <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
+  <parameters>
+   <parameter name="maxParameters"><![CDATA[8]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true">
+  <parameters>
+   <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
+  <parameters>
+   <parameter name="regex"><![CDATA[println]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true">
+  <parameters>
+   <parameter name="maxTypes"><![CDATA[30]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="false">
+  <parameters>
+   <parameter name="maximum"><![CDATA[10]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
+  <parameters>
+   <parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
+   <parameter name="doubleLineAllowed"><![CDATA[false]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true">
+  <parameters>
+   <parameter name="maxLength"><![CDATA[50]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true">
+  <parameters>
+   <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true">
+  <parameters>
+   <parameter name="maxMethods"><![CDATA[30]]></parameter>
+  </parameters>
+ </check>
+ <check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="false"></check>
+ <check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check>
+</scalastyle>

src/main/scala/com/github/caiiiycuk/pg2sqlite/Boot.scala

@@ -0,0 +1,48 @@
+package com.github.caiiiycuk.pg2sqlite
+
+import com.github.caiiiycuk.pg2sqlite.command.CommandException
+import com.github.caiiiycuk.pg2sqlite.iterator.LineIterator
+import com.github.caiiiycuk.pg2sqlite.values.ValueParseException
+
+import ch.qos.logback.classic.Level
+
+object Boot extends App with Log {
+
+  val root = org.slf4j.LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger]
+  root.setLevel(Level.INFO)
+
+  val config = Config.parse(args)
+  import config._
+
+  val size = pgdump.length()
+  val connection = Connection.sqlite(sqlite)
+  val iterator = LineIterator(pgdump)
+  val loggedIterator = LoggedIterator(iterator, () => 100.0 * iterator.readed / size)
+  val dumpInserter = new DumpInserter(connection)
+
+  log.info(s"'$pgdump' (${toMb(size)} Mb) -> '$sqlite'")
+
+  val success = try {
+    dumpInserter.insert(loggedIterator)
+    true
+  } catch {
+    case e: CommandException =>
+      log.error(e.getMessage)
+      false
+    case e: ValueParseException =>
+      log.error(e.getMessage)
+      false
+    case e: Throwable =>
+      log.error(e.getMessage, e)
+      false
+  }
+
+  iterator.close
+  connection.close
+
+  if (success) {
+    log.info("Well done...")
+  } else {
+    log.error("Task failed...")
+  }
+}

src/main/scala/com/github/caiiiycuk/pg2sqlite/Config.scala

@@ -0,0 +1,57 @@
+package com.github.caiiiycuk.pg2sqlite
+
+import java.io.File
+
+case class Config(pgdump: File = new File("dump"), sqlite: File = new File("db"), force: Boolean = false)
+
+object Config extends Log {
+  private val parser = new scopt.OptionParser[Config]("postgresql-to-sqlite") {
+    head("postgresql-to-sqlite")
+
+    opt[File]('d', "dump") required () valueName ("<dump file>") action { (v, c) =>
+      c.copy(pgdump = v)
+    } text ("postgresql dump generated by pg_dump")
+
+    opt[File]('o', "out") required () valueName ("<sqlite3 database>") action { (v, c) =>
+      c.copy(sqlite = v)
+    } text ("sqlite3 database to create")
+
+    opt[Boolean]('f', "force") optional () valueName ("<true|false>") action { (v, c) =>
+      c.copy(force = v)
+    } text ("recreate database if exists")
+
+    checkConfig { c =>
+      import c._
+
+      if (!pgdump.exists()) {
+        failure(s"Dump '${pgdump}' does not exists")
+      } else if (sqlite.exists()) {
+        if (force) {
+          sqlite.delete()
+          success
+        } else {
+          failure(s"Database '${sqlite}' already exists")
+        }
+      } else {
+        success
+      }
+    }
+  }
+
+  def parse(args: Array[String]) = {
+    parser.parse(args, Config()) match {
+      case Some(config) =>
+        Option(System.getenv("SQLITE_TMPDIR")) match {
+          case None =>
+            log.warn("You should set SQLITE_TMPDIR environment variable to control where sqlite stores temp files")
+          case _ =>
+        }
+
+        config
+      case _ =>
+        System.exit(1)
+        ???
+    }
+  }
+
+}

src/main/scala/com/github/caiiiycuk/pg2sqlite/Connection.scala

@@ -0,0 +1,87 @@
+package com.github.caiiiycuk.pg2sqlite
+
+import java.sql.DriverManager
+import java.sql.Statement
+import java.sql.PreparedStatement
+import scala.collection.mutable.ListBuffer
+import java.sql.ResultSet
+import scala.annotation.tailrec
+import java.io.File
+
+trait ConnectionHolder {
+  def makeConnection: java.sql.Connection
+  def db: String
+}
+
+object Connection {
+  final val FETCH_SIZE = 8192
+  final val MAX_VARIABLE_NUMBER = 999
+
+  def sqlite(dbFile: File) = {
+    val connectionHolder = new ConnectionHolder {
+      override def makeConnection: java.sql.Connection = {
+        implicit val connection = DriverManager.getConnection(s"jdbc:sqlite:$dbFile")
+
+        connection.setAutoCommit(true)
+        sqlitePragmas()
+
+        connection.setAutoCommit(false)
+        connection
+      }
+
+      override def db = dbFile.toString
+    }
+
+    new Connection(connectionHolder)
+  }
+
+  private def sqlitePragmas()(implicit connection: java.sql.Connection) = {
+    val statment = connection.createStatement()
+    statment.executeUpdate("PRAGMA synchronous = OFF")
+    statment.executeUpdate("PRAGMA journal_mode = OFF")
+    statment.executeUpdate("PRAGMA threads = 64")
+    statment.executeUpdate("PRAGMA max_page_count = 2147483646")
+    statment.executeUpdate("PRAGMA cache_size = 65536")
+    statment.executeUpdate("PRAGMA cache_spill = true")
+    statment.close
+  }
+}
+
+class Connection(connectionHolder: ConnectionHolder) {
+
+  import Connection._
+
+  final val MAX_VARIABLE_NUMBER = Connection.MAX_VARIABLE_NUMBER
+
+  lazy val connection = connectionHolder.makeConnection
+
+  lazy val db = connectionHolder.db
+
+  def withStatement[T](block: (Statement) => T): T = {
+    val statement = connection.createStatement()
+    val t = block(statement)
+    statement.close
+    t
+  }
+
+  def withPreparedStatement[T](sql: String, keepAlive: Boolean = false)(block: (PreparedStatement) => T): T = {
+    val statement = connection.prepareStatement(sql)
+    statement.setFetchSize(FETCH_SIZE)
+
+    val t = block(statement)
+    if (!keepAlive) statement.close
+    t
+  }
+
+  def close = {
+    connection.commit
+    connection.close
+  }
+
+  def execute(sql: String) = {
+    withStatement { statement =>
+      statement.executeUpdate(sql)
+    }
+  }
+
+}

src/main/scala/com/github/caiiiycuk/pg2sqlite/DumpInserter.scala

@@ -0,0 +1,32 @@
+package com.github.caiiiycuk.pg2sqlite
+
+import scala.annotation.tailrec
+import com.github.caiiiycuk.pg2sqlite.command._
+import com.github.caiiiycuk.pg2sqlite.iterator.Line
+import com.github.caiiiycuk.pg2sqlite.schema.Schema
+
+object DumpInserter {
+  val COMMANDS = List(CreateTable, Copy, CreateIndex)
+}
+
+class DumpInserter(connection: Connection) {
+
+  import DumpInserter._
+
+  implicit val schema = new Schema()
+
+  @tailrec
+  final def insert(iterator: Iterator[Line]): Unit = {
+    if (iterator.hasNext) {
+      val head = iterator.next()
+      val fullIterator = Iterator(head) ++ iterator
+
+      COMMANDS.find(_.matchHead(head)).map { command =>
+        command.apply(connection, fullIterator)
+      }
+
+      insert(iterator)
+    }
+  }
+
+}