diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 424b55b..3eb2623 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1 /home/runner/go/bin/golangci-lint run - name: Test - run: go test -v -race -cover -covermode=atomic -coverprofile=coverage.out ./... + run: go test -v -test.timeout 0 -race -cover -covermode=atomic -coverprofile=coverage.out ./... - name: Test Examples run: | pushd examples && \ diff --git a/decompression.go b/decompression.go new file mode 100644 index 0000000..1b5f619 --- /dev/null +++ b/decompression.go @@ -0,0 +1,87 @@ +package embeddedpostgres + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/xi2/xz" +) + +func defaultTarReader(xzReader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { + tarReader := tar.NewReader(xzReader) + + return func() (*tar.Header, error) { + return tarReader.Next() + }, func() io.Reader { + return tarReader + } +} + +func decompressTarXz(tarReader func(*xz.Reader) (func() (*tar.Header, error), func() io.Reader), path, extractPath string) error { + tarFile, err := os.Open(path) + if err != nil { + return errorUnableToExtract(path, extractPath) + } + + defer func() { + if err := tarFile.Close(); err != nil { + panic(err) + } + }() + + xzReader, err := xz.NewReader(tarFile, 0) + if err != nil { + return errorUnableToExtract(path, extractPath) + } + + readNext, reader := tarReader(xzReader) + + for { + header, err := readNext() + + if err == io.EOF { + return nil + } + + if err != nil { + return errorExtractingPostgres(err) + } + + targetPath := filepath.Join(extractPath, header.Name) + + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + return errorExtractingPostgres(err) + } + + switch header.Typeflag { + case tar.TypeReg: + outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return errorExtractingPostgres(err) + } + + if _, err := io.Copy(outFile, reader()); err != nil { + return errorExtractingPostgres(err) + } + + if err := outFile.Close(); err != nil { + return errorExtractingPostgres(err) + } + case tar.TypeSymlink: + if err := os.RemoveAll(targetPath); err != nil { + return errorExtractingPostgres(err) + } + + if err := os.Symlink(header.Linkname, targetPath); err != nil { + return errorExtractingPostgres(err) + } + } + } +} + +func errorUnableToExtract(cacheLocation, binariesPath string) error { + return fmt.Errorf("unable to extract postgres archive %s to %s, if running parallel tests, configure RuntimePath to isolate testing directories", cacheLocation, binariesPath) +} diff --git a/decompression_test.go b/decompression_test.go new file mode 100644 index 0000000..cbd0cb8 --- /dev/null +++ b/decompression_test.go @@ -0,0 +1,134 @@ +package embeddedpostgres + +import ( + "archive/tar" + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/xi2/xz" +) + +func Test_decompressTarXz(t *testing.T) { + tempDir, err := ioutil.TempDir("", "temp_tar_test") + if err != nil { + panic(err) + } + + archive, cleanUp := createTempXzArchive() + defer cleanUp() + + err = decompressTarXz(defaultTarReader, archive, tempDir) + + assert.NoError(t, err) + + expectedExtractedFileLocation := filepath.Join(tempDir, "dir1", "dir2", "some_content") + assert.FileExists(t, expectedExtractedFileLocation) + + fileContentBytes, err := ioutil.ReadFile(expectedExtractedFileLocation) + assert.NoError(t, err) + + assert.Equal(t, "b33r is g00d", string(fileContentBytes)) +} + +func Test_decompressTarXz_ErrorWhenFileNotExists(t *testing.T) { + err := decompressTarXz(defaultTarReader, "/does-not-exist", "/also-fake") + + assert.EqualError(t, err, "unable to extract postgres archive /does-not-exist to /also-fake, if running parallel tests, configure RuntimePath to isolate testing directories") +} + +func Test_decompressTarXz_ErrorWhenErrorDuringRead(t *testing.T) { + tempDir, err := ioutil.TempDir("", "temp_tar_test") + if err != nil { + panic(err) + } + + archive, cleanUp := createTempXzArchive() + defer cleanUp() + + err = decompressTarXz(func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { + return func() (*tar.Header, error) { + return nil, errors.New("oh noes") + }, nil + }, archive, tempDir) + + assert.EqualError(t, err, "unable to extract postgres archive: oh noes") +} + +func Test_decompressTarXz_ErrorWhenFailedToReadFileToCopy(t *testing.T) { + tempDir, err := ioutil.TempDir("", "temp_tar_test") + if err != nil { + panic(err) + } + + archive, cleanUp := createTempXzArchive() + defer cleanUp() + + blockingFile := filepath.Join(tempDir, "blocking") + + if err = ioutil.WriteFile(blockingFile, []byte("wazz"), 0000); err != nil { + panic(err) + } + + fileBlockingExtractTarReader := func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { + shouldReadFile := true + + return func() (*tar.Header, error) { + if shouldReadFile { + shouldReadFile = false + + return &tar.Header{ + Typeflag: tar.TypeReg, + Name: "blocking", + }, nil + } + + return nil, io.EOF + }, func() io.Reader { + open, _ := os.Open("file_not_exists") + return open + } + } + + err = decompressTarXz(fileBlockingExtractTarReader, archive, tempDir) + + assert.Regexp(t, "^unable to extract postgres archive:.+$", err) +} + +func Test_decompressTarXz_ErrorWhenFileToCopyToNotExists(t *testing.T) { + tempDir, err := ioutil.TempDir("", "temp_tar_test") + if err != nil { + panic(err) + } + + archive, cleanUp := createTempXzArchive() + defer cleanUp() + + fileBlockingExtractTarReader := func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { + shouldReadFile := true + + return func() (*tar.Header, error) { + if shouldReadFile { + shouldReadFile = false + + return &tar.Header{ + Typeflag: tar.TypeReg, + Name: "some_dir/wazz/dazz/fazz", + }, nil + } + + return nil, io.EOF + }, func() io.Reader { + open, _ := os.Open("file_not_exists") + return open + } + } + + err = decompressTarXz(fileBlockingExtractTarReader, archive, tempDir) + + assert.Regexp(t, "^unable to extract postgres archive:.+$", err) +} diff --git a/embedded_postgres.go b/embedded_postgres.go index ce99589..d90ca03 100644 --- a/embedded_postgres.go +++ b/embedded_postgres.go @@ -10,8 +10,6 @@ import ( "path/filepath" "runtime" "strings" - - "github.com/mholt/archiver/v3" ) // EmbeddedPostgres maintains all configuration and runtime functions for maintaining the lifecycle of one Postgres process. @@ -102,9 +100,8 @@ func (ep *EmbeddedPostgres) Start() error { } } - if err := archiver.NewTarXz().Unarchive(cacheLocation, ep.config.binariesPath); err != nil { - return fmt.Errorf(`unable to extract postgres archive %s to %s -if running parallel tests, configure RuntimePath to isolate testing directories`, cacheLocation, ep.config.binariesPath) + if err := decompressTarXz(defaultTarReader, cacheLocation, ep.config.binariesPath); err != nil { + return err } } diff --git a/embedded_postgres_test.go b/embedded_postgres_test.go index a1b8cf3..1f63ed0 100644 --- a/embedded_postgres_test.go +++ b/embedded_postgres_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/mholt/archiver/v3" "github.com/stretchr/testify/assert" ) @@ -99,8 +98,7 @@ func Test_ErrorWhenUnableToUnArchiveFile_WrongFormat(t *testing.T) { } } - assert.EqualError(t, err, fmt.Sprintf(`unable to extract postgres archive %s to %s -if running parallel tests, configure RuntimePath to isolate testing directories`, jarFile, filepath.Join(filepath.Dir(jarFile), "extracted"))) + assert.EqualError(t, err, fmt.Sprintf(`unable to extract postgres archive %s to %s, if running parallel tests, configure RuntimePath to isolate testing directories`, jarFile, filepath.Join(filepath.Dir(jarFile), "extracted"))) } func Test_ErrorWhenUnableToInitDatabase(t *testing.T) { @@ -549,7 +547,7 @@ func Test_PrefetchedBinaries(t *testing.T) { } cacheLocation, _ := database.cacheLocator() - if err := archiver.NewTarXz().Unarchive(cacheLocation, binTempDir); err != nil { + if err := decompressTarXz(defaultTarReader, cacheLocation, binTempDir); err != nil { panic(err) } diff --git a/examples/go.sum b/examples/go.sum index 40db2bf..5262b09 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -1,26 +1,13 @@ -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= -github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -31,12 +18,6 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= -github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= -github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -49,8 +30,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/go.mod b/go.mod index 7a45066..253b3f0 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,8 @@ module github.com/fergusstrange/embedded-postgres go 1.13 -// To avoid CVE CVE-2021-29482 -replace github.com/ulikunitz/xz => github.com/ulikunitz/xz v0.5.8 - require ( github.com/lib/pq v1.8.0 - github.com/mholt/archiver/v3 v3.5.0 github.com/stretchr/testify v1.6.1 + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 ) diff --git a/go.sum b/go.sum index efa86cc..88e402e 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,12 @@ -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= -github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= -github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= -github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/platform-test/go.sum b/platform-test/go.sum index efa86cc..88e402e 100644 --- a/platform-test/go.sum +++ b/platform-test/go.sum @@ -1,33 +1,12 @@ -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= -github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= -github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= -github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/prepare_database_test.go b/prepare_database_test.go index 8f02d0a..20b0144 100644 --- a/prepare_database_test.go +++ b/prepare_database_test.go @@ -103,6 +103,7 @@ func Test_defaultCreateDatabase_ErrorWhenSQLOpenError(t *testing.T) { func Test_defaultCreateDatabase_ErrorWhenQueryError(t *testing.T) { database := NewDatabase(DefaultConfig(). + Port(9831). Database("b33r")) if err := database.Start(); err != nil { t.Fatal(err) @@ -114,7 +115,7 @@ func Test_defaultCreateDatabase_ErrorWhenQueryError(t *testing.T) { } }() - err := defaultCreateDatabase(5432, "postgres", "postgres", "b33r") + err := defaultCreateDatabase(9831, "postgres", "postgres", "b33r") assert.EqualError(t, err, `unable to connect to create database with custom name b33r with the following error: pq: database "b33r" already exists`) } diff --git a/remote_fetch.go b/remote_fetch.go index eb52ea4..5ae9704 100644 --- a/remote_fetch.go +++ b/remote_fetch.go @@ -10,8 +10,6 @@ import ( "os" "path/filepath" "strings" - - "github.com/mholt/archiver/v3" ) // RemoteFetchStrategy provides a strategy to fetch a Postgres binary so that it is available for use. @@ -45,65 +43,54 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS } }() - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errorFetchingPostgres(err) - } - - zipFile := archiver.NewZip() + return decompressResponse(resp, cacheLocator, downloadURL) + } +} - if err := zipFile.Open(bytes.NewReader(bodyBytes), resp.ContentLength); err != nil { - return errorFetchingPostgres(err) - } +func decompressResponse(resp *http.Response, cacheLocator CacheLocator, downloadURL string) error { + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errorFetchingPostgres(err) + } - defer func() { - if err := zipFile.Close(); err != nil { - log.Fatal(err) - } - }() + zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), resp.ContentLength) + if err != nil { + return errorFetchingPostgres(err) + } - for { - downloadedArchive, err := zipFile.Read() + for _, file := range zipReader.File { + if !file.FileHeader.FileInfo().IsDir() && strings.HasSuffix(file.FileHeader.Name, ".txz") { + archiveReader, err := file.Open() if err != nil { - return errorExtractingBinary(downloadURL) + return errorExtractingPostgres(err) } - if header, ok := downloadedArchive.Header.(zip.FileHeader); !ok || !strings.HasSuffix(header.Name, ".txz") { - continue + archiveBytes, err := ioutil.ReadAll(archiveReader) + if err != nil { + return errorExtractingPostgres(err) } - downloadedArchiveBytes, err := ioutil.ReadAll(downloadedArchive) - if err == nil { - cacheLocation, _ := cacheLocator() + cacheLocation, _ := cacheLocator() - if err := createArchiveFile(cacheLocation, downloadedArchiveBytes); err != nil { - return fmt.Errorf("unable to extract postgres archive to %s", cacheLocation) - } + if err := os.MkdirAll(filepath.Dir(cacheLocation), 0755); err != nil { + return errorExtractingPostgres(err) + } - break + if err := ioutil.WriteFile(cacheLocation, archiveBytes, file.FileHeader.Mode()); err != nil { + return errorExtractingPostgres(err) } - } - return nil + return nil + } } -} -func errorExtractingBinary(downloadURL string) error { return fmt.Errorf("error fetching postgres: cannot find binary in archive retrieved from %s", downloadURL) } -func errorFetchingPostgres(err error) error { - return fmt.Errorf("error fetching postgres: %s", err) +func errorExtractingPostgres(err error) error { + return fmt.Errorf("unable to extract postgres archive: %s", err) } -func createArchiveFile(archiveLocation string, archiveBytes []byte) error { - if err := os.MkdirAll(filepath.Dir(archiveLocation), 0755); err != nil { - return err - } - - if err := ioutil.WriteFile(archiveLocation, archiveBytes, 0666); err != nil { - return err - } - - return nil +func errorFetchingPostgres(err error) error { + return fmt.Errorf("error fetching postgres: %s", err) } diff --git a/remote_fetch_test.go b/remote_fetch_test.go index c4c3bd7..02b3ff8 100644 --- a/remote_fetch_test.go +++ b/remote_fetch_test.go @@ -1,6 +1,7 @@ package embeddedpostgres import ( + "archive/zip" "io/ioutil" "net/http" "net/http/httptest" @@ -8,7 +9,6 @@ import ( "path/filepath" "testing" - "github.com/mholt/archiver/v3" "github.com/stretchr/testify/assert" ) @@ -64,7 +64,7 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzipSubFile(t *testing.T) { err := remoteFetchStrategy() - assert.EqualError(t, err, "error fetching postgres: creating reader: zip: not a valid zip file") + assert.EqualError(t, err, "error fetching postgres: zip: not a valid zip file") } func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzip(t *testing.T) { @@ -81,19 +81,15 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzip(t *testing.T) { err := remoteFetchStrategy() - assert.EqualError(t, err, "error fetching postgres: creating reader: zip: not a valid zip file") + assert.EqualError(t, err, "error fetching postgres: zip: not a valid zip file") } func Test_defaultRemoteFetchStrategy_ErrorWhenNoSubTarArchive(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - zip := archiver.NewZip() - defer func() { - if err := zip.Close(); err != nil { - panic(err) - } - }() - if err := zip.Create(w); err != nil { - panic(err) + MyZipWriter := zip.NewWriter(w) + + if err := MyZipWriter.Close(); err != nil { + t.Error(err) } })) defer server.Close() @@ -136,7 +132,7 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotExtractSubArchive(t *testing err := remoteFetchStrategy() - assert.EqualError(t, err, "unable to extract postgres archive to "+dirBlockingExtract) + assert.Regexp(t, "^unable to extract postgres archive:.+$", err) } func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateCacheDirectory(t *testing.T) { @@ -171,7 +167,7 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateCacheDirectory(t *test err := remoteFetchStrategy() - assert.EqualError(t, err, "unable to extract postgres archive to "+cacheLocation) + assert.Regexp(t, "^unable to extract postgres archive:.+$", err) } func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateSubArchiveFile(t *testing.T) { @@ -203,7 +199,7 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateSubArchiveFile(t *test err := remoteFetchStrategy() - assert.EqualError(t, err, "unable to extract postgres archive to "+cacheLocation) + assert.Regexp(t, "^unable to extract postgres archive:.+$", err) } func Test_defaultRemoteFetchStrategy(t *testing.T) { diff --git a/test_util_test.go b/test_util_test.go index a7d548f..69b80da 100644 --- a/test_util_test.go +++ b/test_util_test.go @@ -1,60 +1,37 @@ package embeddedpostgres import ( + "encoding/base64" "io/ioutil" "os" - "path/filepath" "testing" - - "github.com/mholt/archiver/v3" ) func createTempXzArchive() (string, func()) { - tempDir, err := ioutil.TempDir("", "remote_fetch_test") - if err != nil { - panic(err) - } - - tempFile, err := ioutil.TempFile(tempDir, "remote_fetch_test") - if err != nil { - panic(err) - } - - tarFile := filepath.Join(tempDir, "remote_fetch_test.txz") - if err := archiver.NewTarXz().Archive([]string{tempFile.Name()}, tarFile); err != nil { - panic(err) - } - - return tarFile, func() { - if err := os.RemoveAll(tempDir); err != nil { - panic(err) - } - } + return writeFileWithBase64Content("remote_fetch_test*.txz", "/Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4Av/AKZdADIaSqdFdWDG5Dyin7tszujmfm9YJn6/1REVUfqW8HwXvgwbrrcDDc4Q2ql+L+ybLTxJ+QNhhaKnawviRjKhUOT3syXi2Ye8k4QMkeurnnCu4a8eoCV+hqNFWkk8/w8MzyMzQZ2D3wtvoaZV/KqJ8jyLbNVj+vsKrzqg5vbSGz5/h7F37nqN1V8ZsdCnKnDMZPzovM8RwtelDd0g3fPC0dG/W9PH4wAAAAC2dqs1k9ZA0QABwgGAGAAAIQZ5XbHEZ/sCAAAAAARZWg==") } func createTempZipArchive() (string, func()) { - tempDir, err := ioutil.TempDir("", "remote_fetch_test") - if err != nil { - panic(err) - } + return writeFileWithBase64Content("remote_fetch_test*.zip", "UEsDBBQACAAIAExBSlMAAAAAAAAAAAAAAAAaAAkAcmVtb3RlX2ZldGNoX3Rlc3Q4MDA0NjE5MDVVVAUAAfCfYmEBAAD//1BLBwgAAAAABQAAAAAAAABQSwMEFAAIAAAATEFKUwAAAAAAAAAAAAAAABUACQByZW1vdGVfZmV0Y2hfdGVzdC50eHpVVAUAAfCfYmH9N3pYWgAABObWtEYCACEBFgAAAHQv5aPgBf8Abl0AORlJ/tq+A8rMBye1kCuXLnw2aeeO0gdfXeVHCWpF8/VeZU/MTVkdLzI+XgKLEMlHJukIdxP7iSAuKts+v7aDrJu68RHNgIsXGrGouAjf780FXjTUjX4vXDh08vNY1yOBayt9z9dKHdoG9AeAIgAAAAAOKMpgA1Mm3wABigGADAAAjIVdpbHEZ/sCAAAAAARZWlBLBwhkmQgRsAAAALAAAABQSwECFAMUAAgACABMQUpTAAAAAAUAAAAAAAAAGgAJAAAAAAAAAAAAgIEAAAAAcmVtb3RlX2ZldGNoX3Rlc3Q4MDA0NjE5MDVVVAUAAfCfYmFQSwECFAMUAAgAAABMQUpTZJkIEbAAAACwAAAAFQAJAAAAAAAAAAAApIFWAAAAcmVtb3RlX2ZldGNoX3Rlc3QudHh6VVQFAAHwn2JhUEsFBgAAAAACAAIAnQAAAFIBAAAAAA==") +} - tempFile, err := ioutil.TempFile(tempDir, "remote_fetch_test") +func writeFileWithBase64Content(filename, base64Content string) (string, func()) { + tempFile, err := ioutil.TempFile("", filename) if err != nil { panic(err) } - tarFile := filepath.Join(tempDir, "remote_fetch_test.txz") - if err := archiver.NewTarXz().Archive([]string{tempFile.Name()}, tarFile); err != nil { + byteContent, err := base64.StdEncoding.DecodeString(base64Content) + if err != nil { panic(err) } - jarFile := filepath.Join(tempDir, "remote_fetch_test.zip") - if err := archiver.NewZip().Archive([]string{tempFile.Name(), tarFile}, jarFile); err != nil { + if err := ioutil.WriteFile(tempFile.Name(), byteContent, 0666); err != nil { panic(err) } - return jarFile, func() { - if err := os.RemoveAll(tempDir); err != nil { + return tempFile.Name(), func() { + if err := os.RemoveAll(tempFile.Name()); err != nil { panic(err) } }