-
Notifications
You must be signed in to change notification settings - Fork 89
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Remove archiver xz dependencies (#50)
* Remove extra debug log * Try removing dependencies on chunky decompression libs * Try removing dependencies on chunky decompression libs * Move around error messaging * Resolve some of the broken tests * Resolve last batch of failing tests * Improve coverage and clean up decompression code * Move to using a non default port for validating create default test database * Try rolling back all changes to see if issues are around new decompression code. * Put back code without timeout on tests now * Fix param to go 1.13 standard * Add a little more coverage and strategy to improve * Add further decompression tests Co-authored-by: Nathan Franke <natfra@pm.me>
- Loading branch information
1 parent
9654f19
commit b5602a5
Showing
13 changed files
with
281 additions
and
170 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package embeddedpostgres | ||
|
|
||
| import ( | ||
| "archive/tar" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/xi2/xz" | ||
| ) | ||
|
|
||
| func defaultTarReader(xzReader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { | ||
| tarReader := tar.NewReader(xzReader) | ||
|
|
||
| return func() (*tar.Header, error) { | ||
| return tarReader.Next() | ||
| }, func() io.Reader { | ||
| return tarReader | ||
| } | ||
| } | ||
|
|
||
| func decompressTarXz(tarReader func(*xz.Reader) (func() (*tar.Header, error), func() io.Reader), path, extractPath string) error { | ||
| tarFile, err := os.Open(path) | ||
| if err != nil { | ||
| return errorUnableToExtract(path, extractPath) | ||
| } | ||
|
|
||
| defer func() { | ||
| if err := tarFile.Close(); err != nil { | ||
| panic(err) | ||
| } | ||
| }() | ||
|
|
||
| xzReader, err := xz.NewReader(tarFile, 0) | ||
| if err != nil { | ||
| return errorUnableToExtract(path, extractPath) | ||
| } | ||
|
|
||
| readNext, reader := tarReader(xzReader) | ||
|
|
||
| for { | ||
| header, err := readNext() | ||
|
|
||
| if err == io.EOF { | ||
| return nil | ||
| } | ||
|
|
||
| if err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
|
|
||
| targetPath := filepath.Join(extractPath, header.Name) | ||
|
|
||
| if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
|
|
||
| switch header.Typeflag { | ||
| case tar.TypeReg: | ||
| outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) | ||
| if err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
|
|
||
| if _, err := io.Copy(outFile, reader()); err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
|
|
||
| if err := outFile.Close(); err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
| case tar.TypeSymlink: | ||
| if err := os.RemoveAll(targetPath); err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
|
|
||
| if err := os.Symlink(header.Linkname, targetPath); err != nil { | ||
| return errorExtractingPostgres(err) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func errorUnableToExtract(cacheLocation, binariesPath string) error { | ||
| return fmt.Errorf("unable to extract postgres archive %s to %s, if running parallel tests, configure RuntimePath to isolate testing directories", cacheLocation, binariesPath) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| package embeddedpostgres | ||
|
|
||
| import ( | ||
| "archive/tar" | ||
| "errors" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/xi2/xz" | ||
| ) | ||
|
|
||
| func Test_decompressTarXz(t *testing.T) { | ||
| tempDir, err := ioutil.TempDir("", "temp_tar_test") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| archive, cleanUp := createTempXzArchive() | ||
| defer cleanUp() | ||
|
|
||
| err = decompressTarXz(defaultTarReader, archive, tempDir) | ||
|
|
||
| assert.NoError(t, err) | ||
|
|
||
| expectedExtractedFileLocation := filepath.Join(tempDir, "dir1", "dir2", "some_content") | ||
| assert.FileExists(t, expectedExtractedFileLocation) | ||
|
|
||
| fileContentBytes, err := ioutil.ReadFile(expectedExtractedFileLocation) | ||
| assert.NoError(t, err) | ||
|
|
||
| assert.Equal(t, "b33r is g00d", string(fileContentBytes)) | ||
| } | ||
|
|
||
| func Test_decompressTarXz_ErrorWhenFileNotExists(t *testing.T) { | ||
| err := decompressTarXz(defaultTarReader, "/does-not-exist", "/also-fake") | ||
|
|
||
| assert.EqualError(t, err, "unable to extract postgres archive /does-not-exist to /also-fake, if running parallel tests, configure RuntimePath to isolate testing directories") | ||
| } | ||
|
|
||
| func Test_decompressTarXz_ErrorWhenErrorDuringRead(t *testing.T) { | ||
| tempDir, err := ioutil.TempDir("", "temp_tar_test") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| archive, cleanUp := createTempXzArchive() | ||
| defer cleanUp() | ||
|
|
||
| err = decompressTarXz(func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { | ||
| return func() (*tar.Header, error) { | ||
| return nil, errors.New("oh noes") | ||
| }, nil | ||
| }, archive, tempDir) | ||
|
|
||
| assert.EqualError(t, err, "unable to extract postgres archive: oh noes") | ||
| } | ||
|
|
||
| func Test_decompressTarXz_ErrorWhenFailedToReadFileToCopy(t *testing.T) { | ||
| tempDir, err := ioutil.TempDir("", "temp_tar_test") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| archive, cleanUp := createTempXzArchive() | ||
| defer cleanUp() | ||
|
|
||
| blockingFile := filepath.Join(tempDir, "blocking") | ||
|
|
||
| if err = ioutil.WriteFile(blockingFile, []byte("wazz"), 0000); err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| fileBlockingExtractTarReader := func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { | ||
| shouldReadFile := true | ||
|
|
||
| return func() (*tar.Header, error) { | ||
| if shouldReadFile { | ||
| shouldReadFile = false | ||
|
|
||
| return &tar.Header{ | ||
| Typeflag: tar.TypeReg, | ||
| Name: "blocking", | ||
| }, nil | ||
| } | ||
|
|
||
| return nil, io.EOF | ||
| }, func() io.Reader { | ||
| open, _ := os.Open("file_not_exists") | ||
| return open | ||
| } | ||
| } | ||
|
|
||
| err = decompressTarXz(fileBlockingExtractTarReader, archive, tempDir) | ||
|
|
||
| assert.Regexp(t, "^unable to extract postgres archive:.+$", err) | ||
| } | ||
|
|
||
| func Test_decompressTarXz_ErrorWhenFileToCopyToNotExists(t *testing.T) { | ||
| tempDir, err := ioutil.TempDir("", "temp_tar_test") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| archive, cleanUp := createTempXzArchive() | ||
| defer cleanUp() | ||
|
|
||
| fileBlockingExtractTarReader := func(reader *xz.Reader) (func() (*tar.Header, error), func() io.Reader) { | ||
| shouldReadFile := true | ||
|
|
||
| return func() (*tar.Header, error) { | ||
| if shouldReadFile { | ||
| shouldReadFile = false | ||
|
|
||
| return &tar.Header{ | ||
| Typeflag: tar.TypeReg, | ||
| Name: "some_dir/wazz/dazz/fazz", | ||
| }, nil | ||
| } | ||
|
|
||
| return nil, io.EOF | ||
| }, func() io.Reader { | ||
| open, _ := os.Open("file_not_exists") | ||
| return open | ||
| } | ||
| } | ||
|
|
||
| err = decompressTarXz(fileBlockingExtractTarReader, archive, tempDir) | ||
|
|
||
| assert.Regexp(t, "^unable to extract postgres archive:.+$", err) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.