Feature parity with V1, but still some ways to go

Readme.md

@@ -5,10 +5,15 @@ Allows you to easily execute SQL against structured text like CSV or TSV.
 Example session:
 ![textql_usage_session](https://raw.github.com/dinedal/textql/master/textql_usage.gif)
 
+## Major changes!
+
+Since there has been some time since the initial release of textql, I've made some improvements as well as made the project much more modular. There's also been a additional performance tweaks and added functionality, but this comes at the cost of breaking the original command line flags and changing the install command.
+
 ## Key differences between textql and sqlite importing
 
 - sqlite import will not accept stdin, breaking unix pipes. textql will happily do so.
 - textql supports quote escaped delimiters, sqlite does not.
+- textql leverages the sqlite in memory database feature as much as possible and only touches disk if asked.
 
 ## Is it any good?
 
@@ -23,7 +28,7 @@ Example session:
 You may need to `export CC=clang` on OS X.
 
 ```bash
-go get -u github.com/dinedal/textql
+go get -u github.com/dinedal/textql/...
 ```
 
 ## Usage
@@ -43,6 +48,6 @@ go get -u github.com/dinedal/textql
 
 ## License
 
-New MIT License - Copyright (c) 2014, Paul Bergeron
+New MIT License - Copyright (c) 2014, 2015 Paul Bergeron
 
 See LICENSE for details

cmd/textql.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/dinedal/textql/inputs"
+	"github.com/dinedal/textql/outputs"
+	"github.com/dinedal/textql/storage"
+	"github.com/dinedal/textql/util"
+)
+
+func main() {
+	commands := flag.String("sql", "", "SQL Command(s) to run on the data")
+	sourceFile := flag.String("source", "stdin", "Source file to load, or defaults to stdin")
+	delimiter := flag.String("dlm", ",", "Input delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
+	header := flag.Bool("header", false, "Treat file as having the first row as a header row")
+	outputHeader := flag.Bool("output-header", false, "Display column names in output")
+	outputDelimiter := flag.String("output-dlm", ",", "Output delimiter between fields -output-dlm=tab for tab, -dlm=0x## to specify a character code in hex")
+	outputFile := flag.String("output-file", "stdout", "Filename to write output to, if empty no output is written")
+	tableName := flag.String("table-name", "", "Override the default table name (input file name or stdin)")
+	saveTo := flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
+	console := flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
+	flag.Parse()
+
+	if *console {
+		if *sourceFile == "stdin" {
+			log.Fatalln("Can not open console with pipe input, read a file instead")
+		}
+		_, sqlite3ConsolePathErr := exec.LookPath("sqlite3")
+		if sqlite3ConsolePathErr != nil {
+			log.Fatalln("Console requested but unable to locate `sqlite3` program on $PATH")
+		}
+	}
+
+	fp := util.OpenFileOrStdDev(sourceFile)
+
+	opts := &inputs.CSVInputOptions{
+		HasHeader: *header,
+		Seperator: util.DetermineSeparator(delimiter),
+		ReadFrom:  fp,
+	}
+
+	input := inputs.NewCSVInput(opts)
+
+	storage_opts := &storage.SQLite3Options{}
+
+	storage := storage.NewSQLite3Storage(storage_opts)
+
+	if (*tableName) != "" {
+		storage.LoadInput(input, *tableName)
+	} else {
+		storage.LoadInput(input, path.Base(input.Name()))
+	}
+
+	queryResults := storage.ExecuteSQLStrings(strings.Split(*commands, ";"))
+
+	if (*outputFile) != "" {
+		displayOpts := &outputs.CSVOutputOptions{
+			WriteHeader: *outputHeader,
+			Seperator:   util.DetermineSeparator(outputDelimiter),
+			WriteTo:     util.OpenFileOrStdDev(outputFile),
+		}
+
+		outputer := outputs.NewCSVOutput(displayOpts)
+		outputer.Show(queryResults)
+	}
+
+	if (*saveTo) != "" {
+		storage.SaveTo(*util.CleanPath(saveTo))
+	}
+
+	if *console {
+		var args []string
+
+		if *outputHeader {
+			args = []string{"-header"}
+		} else {
+			args = []string{}
+		}
+
+		if (*saveTo) != "" {
+			args = append(args, *util.CleanPath(saveTo))
+		} else {
+			tempFile, err := ioutil.TempFile(os.TempDir(), "textql")
+			if err != nil {
+				log.Fatalln(err)
+			}
+			defer os.Remove(tempFile.Name())
+			tempFile.Close()
+			storage.SaveTo(tempFile.Name())
+			args = append(args, tempFile.Name())
+		}
+
+		cmd := exec.Command("sqlite3", args...)
+
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd_err := cmd.Run()
+
+		log.Println("ok whut")
+
+		if cmd.Process != nil {
+			cmd.Process.Release()
+		}
+
+		if cmd_err != nil {
+			log.Fatalln(cmd_err)
+		}
+	} else {
+		storage.Close()
+	}
+}

inputs/csv.go

@@ -2,13 +2,10 @@ package inputs
 
 import (
 	"encoding/csv"
-	"encoding/hex"
 	"io"
 	"log"
 	"os"
 	"strconv"
-	"strings"
-	"unicode/utf8"
 )
 
 type csvInput struct {
@@ -103,19 +100,3 @@ func (this *csvInput) readHeader() {
 func (this *csvInput) Header() []string {
 	return this.header
 }
-
-func (this CSVInputOptions) SetSeperator(delimiter *string) {
-	if (*delimiter) == "tab" {
-		this.Seperator = '\t'
-	} else if strings.Index((*delimiter), "0x") == 0 {
-		dlm, hex_err := hex.DecodeString((*delimiter)[2:])
-
-		if hex_err != nil {
-			log.Fatalln(hex_err)
-		}
-
-		this.Seperator, _ = utf8.DecodeRuneInString(string(dlm))
-	} else {
-		this.Seperator, _ = utf8.DecodeRuneInString(*delimiter)
-	}
-}

outputs/csv.go

@@ -40,6 +40,12 @@ func (this *csvOutput) Show(queryResults []*sql.Rows) {
 			log.Fatalln(colsErr)
 		}
 
+		if this.options.WriteHeader {
+			if err := this.writer.Write(cols); err != nil {
+				log.Fatalln(err)
+			}
+		}
+
 		rawResult := make([][]byte, len(cols))
 		result := make([]string, len(cols))
 

storage/sqlite.go

@@ -20,9 +20,7 @@ type sqlite3Storage struct {
 	connId  int
 }
 
-type SQLite3Options struct {
-	//	SaveToPath string
-}
+type SQLite3Options struct{}
 
 var (
 	sqlite3conn []*sqlite3.SQLiteConn = []*sqlite3.SQLiteConn{}
@@ -62,23 +60,23 @@ func (this *sqlite3Storage) open() {
 	this.db = db
 }
 
-func (this *sqlite3Storage) LoadInput(input inputs.Input) {
-	this.createTable("tbl", input.Header(), true)
+func (this *sqlite3Storage) LoadInput(input inputs.Input, name string) {
+	this.createTable(name, input.Header(), false)
 
 	tx, tx_err := this.db.Begin()
 
 	if tx_err != nil {
 		log.Fatalln(tx_err)
 	}
 
-	stmt := this.createLoadStmt("tbl", len(input.Header()), tx)
+	stmt := this.createLoadStmt(name, len(input.Header()), tx)
 
 	row := input.ReadRecord()
 	for {
 		if row == nil {
 			break
 		}
-		this.loadRow("tbl", len(input.Header()), row, tx, stmt, true)
+		this.loadRow(name, len(input.Header()), row, tx, stmt, true)
 		row = input.ReadRecord()
 	}
 	stmt.Close()
@@ -131,7 +129,7 @@ func (this *sqlite3Storage) createLoadStmt(tableName string, colCount int, db *s
 	}
 	var buffer bytes.Buffer
 
-	buffer.WriteString("INSERT INTO " + (tableName) + " VALUES (")
+	buffer.WriteString("INSERT INTO [" + (tableName) + "] VALUES (")
 	// Don't write the comma for the last column
 	for i := 1; i <= colCount; i++ {
 		buffer.WriteString("?")
@@ -219,3 +217,7 @@ func (this *sqlite3Storage) SaveTo(path string) {
 		log.Fatalln(backup_close_error)
 	}
 }
+
+func (this *sqlite3Storage) Close() {
+	this.db.Close()
+}

storage/storage.go

@@ -7,7 +7,8 @@ import (
 )
 
 type Storage interface {
-	LoadInput(*inputs.Input)
+	LoadInput(*inputs.Input, string)
 	SaveTo(string)
 	ExecuteSQLStrings([]string) []*sql.Rows
+	Close()
 }

util/file_helpers.go

@@ -8,15 +8,18 @@ import (
 	"strings"
 )
 
-func OpenFileOrStdin(path *string) *os.File {
+func OpenFileOrStdDev(path *string) *os.File {
 	var fp *os.File
 	var err error
 
 	if (*path) == "stdin" {
 		fp = os.Stdin
 		err = nil
+	} else if (*path) == "stdout" {
+		fp = os.Stdout
+		err = nil
 	} else {
-		fp, err = os.Open(*cleanPath(path))
+		fp, err = os.Open(*CleanPath(path))
 	}
 
 	if err != nil {
@@ -26,7 +29,7 @@ func OpenFileOrStdin(path *string) *os.File {
 	return fp
 }
 
-func cleanPath(path *string) *string {
+func CleanPath(path *string) *string {
 	var result string
 	usr, err := user.Current()
 	if err != nil {

util/seperator_helpers.go

@@ -0,0 +1,27 @@
+package util
+
+import (
+	"encoding/hex"
+	"log"
+	"strings"
+	"unicode/utf8"
+)
+
+func DetermineSeparator(delimiter *string) rune {
+	var separator rune
+
+	if (*delimiter) == "tab" {
+		separator = '\t'
+	} else if strings.Index((*delimiter), "0x") == 0 {
+		dlm, hex_err := hex.DecodeString((*delimiter)[2:])
+
+		if hex_err != nil {
+			log.Fatalln(hex_err)
+		}
+
+		separator, _ = utf8.DecodeRuneInString(string(dlm))
+	} else {
+		separator, _ = utf8.DecodeRuneInString(*delimiter)
+	}
+	return separator
+}