Skip to content

Commit

Permalink
Added a SQL Server backend
Browse files Browse the repository at this point in the history
  • Loading branch information
milessabin committed Nov 3, 2024
1 parent 49f72ac commit dde325a
Show file tree
Hide file tree
Showing 30 changed files with 6,451 additions and 2 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,11 @@ jobs:

- name: Make target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
run: mkdir -p modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
run: mkdir -p modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target

- name: Compress target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
run: tar cf targets.tar modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
run: tar cf targets.tar modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target

- name: Upload target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
Expand Down
23 changes: 23 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ val kindProjectorVersion = "0.13.3"
val literallyVersion = "1.1.0"
val logbackVersion = "1.5.12"
val log4catsVersion = "2.7.0"
val mssqlDriverVersion = "12.8.1.jre11"
val munitVersion = "1.0.0-M11"
val munitCatsEffectVersion = "2.0.0"
val munitScalaCheckVersion = "1.0.0-M11"
Expand Down Expand Up @@ -82,13 +83,17 @@ lazy val pgUp = taskKey[Unit]("Start Postgres")
lazy val pgStop = taskKey[Unit]("Stop Postgres")
lazy val oracleUp = taskKey[Unit]("Start Oracle")
lazy val oracleStop = taskKey[Unit]("Stop Oracle")
lazy val mssqlUp = taskKey[Unit]("Start SQL Server")
lazy val mssqlStop = taskKey[Unit]("Stop SQL Server")

ThisBuild / allUp := "docker compose up -d --wait --quiet-pull".!
ThisBuild / allStop := "docker compose stop".!
ThisBuild / pgUp := "docker compose up -d --wait --quiet-pull postgres".!
ThisBuild / pgStop := "docker compose stop postgres".!
ThisBuild / oracleUp := "docker compose up -d --wait --quiet-pull oracle".!
ThisBuild / oracleStop := "docker compose stop oracle".!
ThisBuild / mssqlUp := "docker compose up -d --wait --quiet-pull mssql".!
ThisBuild / mssqlStop := "docker compose stop mssql".!

lazy val commonSettings = Seq(
//scalacOptions --= Seq("-Wunused:params", "-Wunused:imports", "-Wunused:patvars", "-Wdead-code", "-Wunused:locals", "-Wunused:privates", "-Wunused:implicits"),
Expand Down Expand Up @@ -133,6 +138,7 @@ lazy val modules: List[CompositeProject] = List(
doobiecore,
doobiepg,
doobieoracle,
doobiemssql,
skunk,
generic,
docs,
Expand Down Expand Up @@ -265,6 +271,22 @@ lazy val doobieoracle = project
)
)

lazy val doobiemssql = project
.in(file("modules/doobie-mssql"))
.enablePlugins(AutomateHeaderPlugin)
.disablePlugins(RevolverPlugin)
.dependsOn(doobiecore % "test->test;compile->compile")
.settings(commonSettings)
.settings(
name := "grackle-doobie-mssql",
Test / fork := true,
Test / parallelExecution := false,
Test / testOptions += Tests.Setup(_ => "docker compose up -d --wait --quiet-pull mssql".!),
libraryDependencies ++= Seq(
"com.microsoft.sqlserver" % "mssql-jdbc" % mssqlDriverVersion
)
)

lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
.crossType(CrossType.Full)
.in(file("modules/skunk"))
Expand Down Expand Up @@ -409,6 +431,7 @@ lazy val unidocs = project
doobiecore,
doobiepg,
doobieoracle,
doobiemssql,
skunk.jvm,
generic.jvm,
)
Expand Down
21 changes: 21 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,24 @@ services:
retries: 10
start_period: 5s
start_interval: 5s

mssql:
image: mcr.microsoft.com/mssql/server:2022-latest
ports:
- "1433:1433"
environment:
SA_PASSWORD: Test_123_Test
MSSQL_PID: Developer
ACCEPT_EULA: Y
MSSQL_TCP_PORT: 1433
volumes:
- ./testdata/mssql/:/grackle-initdb.d/
- ./modules/doobie-mssql/src/test/resources/scripts/:/container-entrypoint-initdb.d/
entrypoint: ["/bin/bash", "/container-entrypoint-initdb.d/entrypoint.sh"]
healthcheck:
test: bash -c "[ -f /tmp/healthy ]"
interval: 10s
timeout: 5s
retries: 10
start_period: 5s
start_interval: 5s
111 changes: 111 additions & 0 deletions modules/doobie-mssql/src/main/scala/DoobieMSSqlMapping.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright (c) 2016-2023 Association of Universities for Research in Astronomy, Inc. (AURA)
// Copyright (c) 2016-2023 Grackle Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grackle.doobie.mssql

import cats.effect.Sync
import cats.syntax.all._
import _root_.doobie.Transactor

import grackle.Mapping
import grackle.Query.OrderSelection
import grackle.doobie._
import grackle.sql._

abstract class DoobieMSSqlMapping[F[_]](
val transactor: Transactor[F],
val monitor: DoobieMonitor[F],
)(
implicit val M: Sync[F]
) extends Mapping[F] with DoobieMSSqlMappingLike[F]

trait DoobieMSSqlMappingLike[F[_]] extends DoobieMappingLike[F] with SqlMappingLike[F] {
import SqlQuery.SqlSelect
import TableExpr.Laterality

def collateToFragment: Fragment =
Fragments.const(" COLLATE DATABASE_DEFAULT")

def aliasDefToFragment(alias: String): Fragment =
Fragments.const(s" $alias")

def offsetToFragment(offset: Fragment): Fragment =
Fragments.const(" OFFSET ") |+| offset |+| Fragments.const(" ROWS")

def limitToFragment(limit: Fragment): Fragment =
Fragments.const(" FETCH FIRST ") |+| limit |+| Fragments.const(" ROWS ONLY")

def likeToFragment(expr: Fragment, pattern: String, caseInsensitive: Boolean): Fragment = {
val casedExpr = if(caseInsensitive) Fragments.const("UPPER(") |+| expr |+| Fragments.const(s")") else expr
val casedPattern = if(caseInsensitive) pattern.toUpperCase else pattern
casedExpr |+| Fragments.const(s" LIKE ") |+| Fragments.bind(stringEncoder, casedPattern)
}

def ascribedNullToFragment(codec: Codec): Fragment =
Fragments.sqlTypeName(codec) match {
case Some(name) if !name.startsWith("_") =>
val convName =
name match {
case "VARCHAR" => "CHAR"
case "NVARCHAR" => "NCHAR"
case "INTEGER" => "INTEGER"
case "BIGINT" => "BIGINT"
case "BOOLEAN" => "BIT"
case "TIMESTAMP" => "DATETIMEOFFSET" // TODO: Probably shouldn't be TIMESTAMP on the LHS
case other => other
}
Fragments.const(s"CAST(NULL AS $convName)")
case _ => Fragments.const("NULL")
}

def collateSelected: Boolean = false

def distinctOnToFragment(dcols: List[Fragment]): Fragment =
Fragments.const("DISTINCT ")

def distinctOrderColumn(owner: ColumnOwner, col: SqlColumn, predCols: List[SqlColumn], orders: List[OrderSelection[_]]): SqlColumn =
SqlColumn.FirstValueColumn(owner, col, predCols, orders)

def encapsulateUnionBranch(s: SqlSelect): SqlSelect =
if(s.orders.isEmpty) s
else s.toSubquery(s.table.name+"_encaps", Laterality.NotLateral)

def mkLateral(inner: Boolean): Laterality =
Laterality.Apply(inner)

def defaultOffsetForSubquery(subquery: SqlQuery): SqlQuery =
subquery match {
case s: SqlSelect if s.orders.nonEmpty && s.offset.isEmpty => s.copy(offset = 0.some)
case _ => subquery
}

def defaultOffsetForLimit(limit: Option[Int]): Option[Int] =
limit.as(0)

def orderToFragment(col: Fragment, ascending: Boolean, nullsLast: Boolean): Fragment = {
val dir = if(ascending) Fragments.empty else Fragments.const(" DESC")
val nulls =
if(nullsLast && ascending)
Fragments.const(" CASE WHEN ") |+| col |+| Fragments.const(" IS NULL THEN 1 ELSE 0 END ASC, ")
else if(!nullsLast && !ascending)
Fragments.const(" CASE WHEN ") |+| col |+| Fragments.const(" IS NULL THEN 0 ELSE 1 END DESC, ")
else
Fragments.empty

nulls |+| col |+| dir
}

def nullsHigh: Boolean = false
}
29 changes: 29 additions & 0 deletions modules/doobie-mssql/src/test/resources/scripts/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/bin/bash

# Start SQL Server
/opt/mssql/bin/sqlservr &

# Wait for SQL Server to start (max 90 seconds)
for i in {1..90}; do
/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Test_123_Test -No -Q 'SELECT 1' &> /dev/null
if [ $? -eq 0 ]; then
echo "SQL Server is up"
break
fi
echo "Waiting for SQL Server to start..."
sleep 1
done

# Initialize the database
if [ ! -f /tmp/healthy ]; then
echo "Intializing the database ..."
/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Test_123_Test -No \
-i <(cat /container-entrypoint-initdb.d/init.sql /grackle-initdb.d/*.sql)

/bin/touch /tmp/healthy

echo "Database initialized"
fi

# Keep the container running
tail -f /dev/null
9 changes: 9 additions & 0 deletions modules/doobie-mssql/src/test/resources/scripts/init.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
USE master;
GO

CREATE DATABASE test
COLLATE Latin1_General_100_CI_AS_SC_UTF8;
GO

USE test;
GO
105 changes: 105 additions & 0 deletions modules/doobie-mssql/src/test/scala/DoobieMSSqlDatabaseSuite.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Copyright (c) 2016-2023 Association of Universities for Research in Astronomy, Inc. (AURA)
// Copyright (c) 2016-2023 Grackle Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grackle.doobie.mssql
package test

import java.sql.{Time, Timestamp}
import java.time.{LocalDate, LocalTime, OffsetDateTime, ZoneId}
import java.util.UUID
import scala.util.Try

import cats.effect.{Resource, Sync, IO}
import cats.syntax.all._
import doobie.{Meta, Transactor}
import doobie.enumerated.JdbcType
import doobie.util.meta.MetaConstructors.Basic
import io.circe.{Decoder => CDecoder, Encoder => CEncoder, Json}
import io.circe.syntax._
import io.circe.parser.parse
import munit.catseffect._

import grackle.doobie.DoobieMonitor
import grackle.doobie.test.DoobieDatabaseSuite

import grackle.sql.test._

trait DoobieMSSqlDatabaseSuite extends DoobieDatabaseSuite {
abstract class DoobieMSSqlTestMapping[F[_]: Sync](transactor: Transactor[F], monitor: DoobieMonitor[F] = DoobieMonitor.noopMonitor[IO])
extends DoobieMSSqlMapping[F](transactor, monitor) with DoobieTestMapping[F] with SqlTestMapping[F] {
def mkTestCodec[T](meta: Meta[T]): TestCodec[T] = (meta, false)

val uuid: TestCodec[UUID] =
mkTestCodec(Meta[String].tiemap(s => Try(UUID.fromString(s)).toEither.leftMap(_.getMessage))(_.toString))

val localTime: TestCodec[LocalTime] = {
mkTestCodec(Meta[Time].timap(t => LocalTime.ofNanoOfDay(t.toLocalTime.toNanoOfDay))(lt => Time.valueOf(lt)))
}

val localDate: TestCodec[LocalDate] =
(Basic.oneObject(JdbcType.Date, None, classOf[LocalDate]), false)

// Forget precise time zone for compatibility with Postgres. Nb. this is specific to this test suite.
val offsetDateTime: TestCodec[OffsetDateTime] =
mkTestCodec(Meta[Timestamp].timap(t => OffsetDateTime.ofInstant(t.toInstant, ZoneId.of("UTC")))(o => Timestamp.from(o.toInstant)))

val nvarchar: TestCodec[String] = mkTestCodec(Meta[String])

val jsonb: TestCodec[Json] =
mkTestCodec(Meta[String].tiemap(s => parse(s).leftMap(_.getMessage))(_.noSpaces))

override def list[T: CDecoder : CEncoder](c: TestCodec[T]): TestCodec[List[T]] = {
def put(ts: List[T]): String = ts.asJson.noSpaces
def get(s: String): Either[String, List[T]] = parse(s).map(_.as[List[T]].toOption.get).leftMap(_.getMessage)

mkTestCodec(Meta[String].tiemap(get)(put))
}
}

case class MSSqlConnectionInfo(host: String, port: Int) {
val driverClassName = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
val databaseName = "test"
val username = "sa"
val password = "Test_123_Test"
val jdbcUrl = s"jdbc:sqlserver://$host:$port;databaseName=$databaseName;user=$username;password=$password;trustServerCertificate=true;sendTimeAsDatetime=false;"
}

object MSSqlConnectionInfo {
val DefaultPort = 1433
}

val msSqlConnectionInfo: MSSqlConnectionInfo =
MSSqlConnectionInfo("localhost", MSSqlConnectionInfo.DefaultPort)

def transactorResource: Resource[IO, Transactor[IO]] = {
val connInfo = msSqlConnectionInfo
import connInfo._

val props = new java.util.Properties()
Resource.pure(
Transactor.fromDriverManager[IO](
driverClassName,
jdbcUrl,
props,
None
)
)
}

val transactorFixture: IOFixture[Transactor[IO]] = ResourceSuiteLocalFixture("mssqlpg", transactorResource)
override def munitFixtures: Seq[IOFixture[_]] = Seq(transactorFixture)

def transactor: Transactor[IO] = transactorFixture()
}
Loading

0 comments on commit dde325a

Please sign in to comment.