Merge tag 'v3.6.2' into debian

Author: df7cb

.travis.sh

@@ -15,8 +15,8 @@ lisp_install() {
 			;;
 
 		sbcl)
-			sbcl_checksum='eb44d9efb4389f71c05af0327bab7cd18f8bb221fb13a6e458477a9194853958'
-			sbcl_version='1.3.18'
+			sbcl_checksum='22ccd9409b2ea16d4be69235c5ad5fde833452955cb24483815312d3b1d7401c'
+			sbcl_version='1.5.2'
 
 			remote_file "/tmp/sbcl-${sbcl_version}.tgz" "http://prdownloads.sourceforge.net/sbcl/sbcl-${sbcl_version}-x86-64-linux-binary.tar.bz2" "$sbcl_checksum"
 			tar --file  "/tmp/sbcl-${sbcl_version}.tgz" --extract --directory '/tmp'

.travis.yml

@@ -21,10 +21,10 @@ before_script:
   - PGUSER=pgloader psql -d pgloader -c "create schema expected;"
   - PGUSER=pgloader psql -d pgloader -c "create schema err;"
   - make --version
-  - make "CL=$LISP"
+  - make "CL=$LISP" clones save
 
 script:
-  - PGUSER=pgloader make "CL=$LISP" check
+  - PGUSER=pgloader make "CL=$LISP" check-saved
 
 notifications:
   email:

Dockerfile

@@ -26,7 +26,7 @@ FROM debian:stable-slim as builder
 
   RUN mkdir -p /opt/src/pgloader/build/bin \
       && cd /opt/src/pgloader \
-      && make
+      && make clones save
 
 FROM debian:stable-slim
 

Dockerfile.ccl

@@ -14,7 +14,6 @@ FROM debian:stable-slim as builder
         make \
         openssl \
         patch \
-        sbcl \
         time \
         unzip \
         wget \
@@ -30,7 +29,7 @@ FROM debian:stable-slim as builder
 
   RUN mkdir -p /opt/src/pgloader/build/bin \
       && cd /opt/src/pgloader \
-      && make CL=ccl DYNSIZE=256
+      && make CL=ccl DYNSIZE=256 clones save
 
 FROM debian:stable-slim
 
@@ -40,6 +39,7 @@ FROM debian:stable-slim
         freetds-dev \
         gawk \
         libsqlite3-dev \
+        libssl1.1 \
         libzip-dev \
         make \
         sbcl \

INSTALL.md

@@ -27,14 +27,45 @@ about how to backport a recent enough SBCL here (1.2.5 or newer).
 
 ### Redhat / CentOS
 
-You will need to install the Steel Bank Common Lisp package (sbcl) from EPEL, as 
-well as the freetds-devel package for some shared libraries. With RHEL/CentOS 6, 
-if the packaged version isn't >=1.3.6, you'll need to build it from source. With 
-v7, after installing freetds, you also need to create a softlink from the versioned
-shared library `libsybdb.so.5` to `libsybdb.so`. 
-
-The above steps are prepared for you with `boostrap-centos.sh` and `bootstrap-centos7.sh` respectively. 
-Please report to us if your standard RHEL/CentOS installation required additional steps.
+To build and install pgloader the Steel Bank Common Lisp package (sbcl) from EPEL,
+and the freetds packages are required.
+
+With RHEL/CentOS 6, if the packaged version of sbcl isn't >=1.3.6, you'll need
+to build it from source.
+
+It is recommended to build the RPM yourself, see below, to ensure that all installed
+files are properly tracked and that you can safely update to newer versions of
+pgloader as they're released.
+
+To do an adhoc build and install run `boostrap-centos.sh` for CentOS 6 or
+`bootstrap-centos7.sh` for CentOS 7 to install the required dependencies.
+[Build pgloader](INSTALL.md#building-pgloader).
+
+#### rpmbuild
+
+The spec file in the root of the pgloader repository can be used to build your
+own RPM. For production deployments it is recommended that you build this RPM on
+a dedicated build box and then copy the RPM to your production environment for
+use; it is considered bad practice to have compilers and build tools present in
+production environments.
+
+1. Install the [EPEL repo](https://fedoraproject.org/wiki/EPEL#Quickstart).
+
+1. Install rpmbuild dependencies:
+
+        sudo yum -y install yum-utils rpmdevtools @"Development Tools"
+
+1. Install pgloader build dependencies:
+
+        sudo yum-builddep pgloader.spec
+
+1. Download pgloader source:
+
+        spectool -g -R pgloader.spec
+
+1. Build the source and binary RPMs (see `rpmbuild --help` for other build options):
+
+        rpmbuild -ba pgloader.spec
 
 ### Mac OS X
 

Makefile

@@ -1,6 +1,6 @@
 # pgloader build tool
 APP_NAME   = pgloader
-VERSION    = 3.6.1
+VERSION    = 3.6.2
 
 # use either sbcl or ccl
 CL	   = sbcl
@@ -24,7 +24,7 @@ QLDIR      = $(BUILDDIR)/quicklisp
 MANIFEST   = $(BUILDDIR)/manifest.ql
 LATEST     = $(BUILDDIR)/pgloader-latest.tgz
 
-BUNDLEDIST = 2019-01-07
+BUNDLEDIST = 2020-02-18
 BUNDLENAME = pgloader-bundle-$(VERSION)
 BUNDLEDIR  = $(BUILDDIR)/bundle/$(BUNDLENAME)
 BUNDLE     = $(BUILDDIR)/$(BUNDLENAME).tgz
@@ -103,7 +103,6 @@ $(LIBS): $(QLDIR)/setup.lisp
 	$(CL) $(CL_OPTS) --load $(QLDIR)/setup.lisp                   \
              --eval '(push :pgloader-image *features*)'               \
              --eval '(setf *print-circle* t *print-pretty* t)'        \
-             --eval '(ql:quickload "pgloader")'                       \
              --eval '(push "$(PWD)/" ql:*local-project-directories*)' \
              --eval '(ql:quickload "pgloader")'                       \
              --eval '(quit)'
@@ -173,7 +172,7 @@ test: $(PGLOADER)
 save: ./src/save.lisp $(LISP_SRC)
 	$(CL) $(CL_OPTS) --load ./src/save.lisp
 
-check-saved: save
+check-saved:
 	$(MAKE) PGLOADER=$(realpath $(PGLOADER)) CL=$(CL) -C test regress
 
 clean-bundle:

README.md

@@ -1,8 +1,9 @@
 # PGLoader
 
 [![Build Status](https://travis-ci.org/dimitri/pgloader.svg?branch=master)](https://travis-ci.org/dimitri/pgloader)
-
 [![Join the chat at https://gitter.im/dimitri/pgloader](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dimitri/pgloader?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Docker Build Status](https://img.shields.io/docker/cloud/build/dimitri/pgloader.svg)](https://cloud.docker.com/repository/docker/dimitri/pgloader)
+[![Read The Docs Status](https://readthedocs.org/projects/pgloader/badge/?version=latest&style=plastic)](http://pgloader.readthedocs.io/en/latest/)
 
 pgloader is a data loading tool for PostgreSQL, using the `COPY` command.
 
@@ -89,8 +90,12 @@ it builds aginst current set of dependencies versions.
 
     $ apt-get install sbcl unzip libsqlite3-dev make curl gawk freetds-dev libzip-dev
     $ cd /path/to/pgloader
-	$ make pgloader
-	$ ./build/bin/pgloader --help
+    $ make pgloader
+    $ ./build/bin/pgloader --help
+
+### Building from sources on RedHat/CentOS
+
+See "Redhat / CentOS" in [INSTALL.md](INSTALL.md#redhat--centos)
 
 ### Building from sources on macOS
 
@@ -178,9 +183,3 @@ indexes, foreign keys, comments) and parallel loading of the corrected data:
 
     $ createdb pagila
     $ pgloader mysql://user@localhost/sakila postgresql:///pagila
-
-See the documentation file `pgloader.1.md` for details. You can compile that
-file into a manual page or an HTML page thanks to the `ronn` application:
-
-    $ apt-get install ruby-ronn
-	$ make docs

docs/index.rst

@@ -147,7 +147,7 @@ Schema only, Data only
 
 Repeatable (DROP+CREATE)
     By default, pgloader issues DROP statements in the target PostgreSQL
-    database before issing any CREATE statement, so that you can repeat the
+    database before issuing any CREATE statement, so that you can repeat the
     migration as many times as necessary until migration specifications and
     rules are bug free.
 

docs/ref/dbf.rst

@@ -1,13 +1,19 @@
 Loading DBF data
 =================
 
-This command instructs pgloader to load data from a `DBF` file. Here's an
-example::
+This command instructs pgloader to load data from a `DBF` file. A default
+set of casting rules are provided and might be overloaded and appended to by
+the command.
+
+Here's an example with a remote HTTP source and some user defined casting
+rules::
 
     LOAD DBF
 	    FROM http://www.insee.fr/fr/methodes/nomenclatures/cog/telechargement/2013/dbf/reg2013.dbf
         INTO postgresql://user@localhost/dbname
-        WITH truncate, create table;
+        WITH truncate, create table
+        CAST column reg2013.region to integer,
+             column reg2013.tncc to smallint;
 
 The `dbf` format command accepts the following clauses and options.
 
@@ -51,3 +57,16 @@ When loading from a `DBF` file, the following options are supported:
 
     This options expects as its value the possibly qualified name of the
     table to create.
+
+Default DB3 Casting Rules
+-------------------------
+
+When migrating from DB3 the following Casting Rules are provided::
+
+  type C to text using db3-trim-string
+  type M to text using db3-trim-string
+  type N to numeric using db3-numeric-to-pgsql-integer
+  type I to numeric using db3-numeric-to-pgsql-numeric
+  type L to boolean using logical-to-boolean
+  type D to date using db3-date-to-pgsql-date
+

docs/ref/mysql.rst

@@ -103,7 +103,7 @@ MySQL Database Migration Options: WITH
 --------------------------------------
 
 When loading from a `MySQL` database, the following options are supported,
-and the default *WITH* clause is: *no truncate*, *create schema*, *create
+and the default *WITH* clause is: *no truncate*, *create
 tables*, *include drop*, *create indexes*, *reset sequences*, *foreign
 keys*, *downcase identifiers*, *uniquify index names*.
 
@@ -567,6 +567,12 @@ Numbers::
 
   type tinyint to boolean when (= 1 precision) using tinyint-to-boolean
 
+  type bit when (= 1 precision) to boolean drop typemod using bits-to-boolean
+  type bit to bit drop typemod using bits-to-hex-bitstring
+
+  type bigint when signed to bigint drop typemod
+  type bigint when (< 19 precision) to numeric drop typemod
+
   type tinyint when unsigned to smallint   drop typemod
   type smallint when unsigned to integer  drop typemod
   type mediumint when unsigned to integer  drop typemod
@@ -595,12 +601,12 @@ Texts::
 
 Binary::
 
-  type binary     to bytea
-  type varbinary  to bytea
-  type tinyblob   to bytea
-  type blob       to bytea
-  type mediumblob to bytea
-  type longblob   to bytea
+  type binary     to bytea using byte-vecotr-to-bytea
+  type varbinary  to bytea using byte-vecotr-to-bytea
+  type tinyblob   to bytea using byte-vecotr-to-bytea
+  type blob       to bytea using byte-vecotr-to-bytea
+  type mediumblob to bytea using byte-vecotr-to-bytea
+  type longblob   to bytea using byte-vecotr-to-bytea
 
 Date::
   
@@ -638,7 +644,9 @@ Date::
 
 Geometric::
 
-  type point to point using pgloader.transforms::convert-mysql-point
+  type geometry   to point using convert-mysql-point
+  type point      to point using convert-mysql-point
+  type linestring to path using convert-mysql-linestring
 
 Enum types are declared inline in MySQL and separately with a `CREATE TYPE`
 command in PostgreSQL, so each column of Enum Type is converted to a type

docs/ref/pgsql-redshift.rst

@@ -8,7 +8,7 @@ Redshift database by parsing the output of the `SELECT version()` SQL query.
 Redhift as a data source
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-Redshit is a variant of PostgreSQL version 8.0.2, which allows pgloader to
+Redshift is a variant of PostgreSQL version 8.0.2, which allows pgloader to
 work with only a very small amount of adaptation in the catalog queries
 used. In other words, migrating from Redshift to PostgreSQL works just the
 same as when migrating from a PostgreSQL data source, including the

pgloader.asd

@@ -119,6 +119,7 @@
                                 ((:file "api")
                                  (:file "methods")
                                  (:file "md-methods")
+                                 (:file "matviews")
                                  (:file "casting-rules")
                                  (:file "files-and-pathnames")
                                  (:file "project-fields")))
@@ -130,22 +131,32 @@
                                  ;; (:file "csv-database")
                                  (:file "csv")))
 
-                       (:file "fixed"
-                              :depends-on ("common" "csv"))
+                       (:module "fixed"
+                                :depends-on ("common")
+                                :serial t
+                                :components
+                                ((:file "fixed-guess")
+                                 (:file "fixed")))
 
                        (:file "copy"
                               :depends-on ("common" "csv"))
 
                        (:module "db3"
+                                :serial t
                                 :depends-on ("common" "csv")
                                 :components
-                                ((:file "db3-schema")
-                                 (:file "db3" :depends-on ("db3-schema"))))
+                                ((:file "db3-cast-rules")
+                                 (:file "db3-connection")
+                                 (:file "db3-schema")
+                                 (:file "db3")))
 
                        (:module "ixf"
+                                :serial t
                                 :depends-on ("common")
                                 :components
-                                ((:file "ixf-schema")
+                                ((:file "ixf-cast-rules")
+                                 (:file "ixf-connection")
+                                 (:file "ixf-schema")
                                  (:file "ixf" :depends-on ("ixf-schema"))))
 
                                         ;(:file "syslog") ; experimental...
@@ -155,6 +166,7 @@
                                 :depends-on ("common")
                                 :components
                                 ((:file "sqlite-cast-rules")
+                                 (:file "sqlite-connection")
                                  (:file "sqlite-schema")
                                  (:file "sqlite")))
 
@@ -163,6 +175,7 @@
                                 :depends-on ("common")
                                 :components
                                 ((:file "mssql-cast-rules")
+                                 (:file "mssql-connection")
                                  (:file "mssql-schema")
                                  (:file "mssql")
                                  (:file "mssql-index-filters")))
@@ -180,7 +193,6 @@
                                 :serial t
                                 :depends-on ("common")
                                 :components ((:file "pgsql-cast-rules")
-                                             (:file "pgsql-schema")
                                              (:file "pgsql")))))
 
              ;; package pgloader.copy