Skip to content

Commit

Permalink
Fix test (#24)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ginokent
ginokent authored Feb 7, 2021
1 parent 44ee946 commit 4b29c1e
Show file tree
Hide file tree
Showing 4 changed files with 450 additions and 323 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ ROOT_DIR := $(shell git rev-parse --show-toplevel)
MAIN_DIR := ${ROOT_DIR}
COVERAGE_FILE := ${ROOT_DIR}/coverage.txt
COVERAGE_HTML := ${ROOT_DIR}/coverage.html
TEST_CMD := GOTEST=true go test -v -race -cover -coverprofile=${COVERAGE_FILE} ./...
TEST_CMD := GOTEST=true OUTPUT_FILE=/dev/null go test -v -race -cover -coverprofile=${COVERAGE_FILE} ./...

OPEN_CMD := $(shell if command -v explorer.exe 1>/dev/null; then echo "explorer.exe"; elif uname -s | grep -q Darwin; then echo "open"; else echo "echo"; fi)

Expand Down
109 changes: 98 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,105 @@ BigQuery table schema struct generator

## generate

```console
$ cd /path/to/your/golang-project-repository
#### How to generate

$ ## Create a directory where bqtableschema will generate the code.
$ mkdir -p bqtableschema
$ cd bqtableschema
```bash
cd /path/to/your/golang-project-repository

$ ## Set the required environment variables.
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/serviceaccount/keyfile.json
$ export GCLOUD_PROJECT_ID=bigquery-public-data ## ref. https://console.cloud.google.com/bigquery?p=bigquery-public-data&page=project
$ export BIGQUERY_DATASET=hacker_news ## ref. https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=hacker_news&page=dataset
# Create a directory where bqtableschema will generate the code.
mkdir -p bqtableschema
cd bqtableschema

$ ## generate
$ go run github.com/djeeno/bqtableschema
# Set the required environment variables.
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/serviceaccount/keyfile.json
# Set GCP Project ID ref. https://console.cloud.google.com/bigquery?p=bigquery-public-data&page=project
export GCLOUD_PROJECT_ID=bigquery-public-data
# Set BigQuery Dataset name ref. https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=hacker_news&page=dataset
export BIGQUERY_DATASET=hacker_news
# Set output file
export OUTPUT_FILE=bqtableschema.generated.go

# generate
go run github.com/djeeno/bqtableschema
```

Example generated file content:

```go
// Code generated by go run github.com/djeeno/bqtableschema; DO NOT EDIT.

//go:generate go run github.com/djeeno/bqtableschema

package bqtableschema

import "time"

// Comments is BigQuery Table `bigquery-public-data:hacker_news.comments` schema struct.
// Description:
type Comments struct {
Id int64 `bigquery:"id"`
By string `bigquery:"by"`
Author string `bigquery:"author"`
Time int64 `bigquery:"time"`
Time_ts time.Time `bigquery:"time_ts"`
Text string `bigquery:"text"`
Parent int64 `bigquery:"parent"`
Deleted bool `bigquery:"deleted"`
Dead bool `bigquery:"dead"`
Ranking int64 `bigquery:"ranking"`
}

// Full is BigQuery Table `bigquery-public-data:hacker_news.full` schema struct.
// Description: A full daily update of all the stories and comments in Hacker News.
type Full struct {
Title string `bigquery:"title"`
Url string `bigquery:"url"`
Text string `bigquery:"text"`
Dead bool `bigquery:"dead"`
By string `bigquery:"by"`
Score int64 `bigquery:"score"`
Time int64 `bigquery:"time"`
Timestamp time.Time `bigquery:"timestamp"`
Type string `bigquery:"type"`
Id int64 `bigquery:"id"`
Parent int64 `bigquery:"parent"`
Descendants int64 `bigquery:"descendants"`
Ranking int64 `bigquery:"ranking"`
Deleted bool `bigquery:"deleted"`
}

// Full_201510 is BigQuery Table `bigquery-public-data:hacker_news.full_201510` schema struct.
// Description:
type Full_201510 struct {
By string `bigquery:"by"`
Score int64 `bigquery:"score"`
Time int64 `bigquery:"time"`
Title string `bigquery:"title"`
Type string `bigquery:"type"`
Url string `bigquery:"url"`
Text string `bigquery:"text"`
Parent int64 `bigquery:"parent"`
Deleted bool `bigquery:"deleted"`
Dead bool `bigquery:"dead"`
Descendants int64 `bigquery:"descendants"`
Id int64 `bigquery:"id"`
Ranking int64 `bigquery:"ranking"`
}

// Stories is BigQuery Table `bigquery-public-data:hacker_news.stories` schema struct.
// Description:
type Stories struct {
Id int64 `bigquery:"id"`
By string `bigquery:"by"`
Score int64 `bigquery:"score"`
Time int64 `bigquery:"time"`
Time_ts time.Time `bigquery:"time_ts"`
Title string `bigquery:"title"`
Url string `bigquery:"url"`
Text string `bigquery:"text"`
Deleted bool `bigquery:"deleted"`
Dead bool `bigquery:"dead"`
Descendants int64 `bigquery:"descendants"`
Author string `bigquery:"author"`
}
```
59 changes: 33 additions & 26 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,13 @@ const (

var (
// optValue
optValueProjectID string
optValueDataset string
optValueKeyFile string
optValueOutputPath string
optValueProjectID = flag.String(optNameProjectID, defaultValueEmpty, "")
optValueDataset = flag.String(optNameDataset, defaultValueEmpty, "")
optValueKeyFile = flag.String(optNameKeyFile, defaultValueEmpty, "path to service account json key file")
optValueOutputPath = flag.String(optNameOutputFile, defaultValueEmpty, "path to output the generated code")
)

func main() {
flag.StringVar(&optValueProjectID, optNameProjectID, defaultValueEmpty, "")
flag.StringVar(&optValueDataset, optNameDataset, defaultValueEmpty, "")
flag.StringVar(&optValueKeyFile, optNameKeyFile, defaultValueEmpty, "path to service account json key file")
flag.StringVar(&optValueOutputPath, optNameOutputFile, defaultValueEmpty, "path to output the generated code")
flag.Parse()

ctx := context.Background()

Expand All @@ -62,31 +57,36 @@ func main() {

// Run is effectively a `main` function.
// It is separated from the `main` function because of addressing an issue where` defer` is not executed when `os.Exit` is executed.
func Run(ctx context.Context) error {
func Run(ctx context.Context) (err error) {
flag.Parse()

filePath, err := getOptOrEnvOrDefault(optNameOutputFile, optValueOutputPath, envNameOutputFile, defaultValueOutputFile)
var keyfile string
keyfile, err = getOptOrEnvOrDefault(optNameKeyFile, *optValueKeyFile, envNameGoogleApplicationCredentials, "")
if err != nil {
return fmt.Errorf("getOptOrEnvOrDefault: %w", err)
}

keyfile, err := getOptOrEnvOrDefault(optNameKeyFile, optValueKeyFile, envNameGoogleApplicationCredentials, "")
var project string
project, err = getOptOrEnvOrDefault(optNameProjectID, *optValueProjectID, envNameGCloudProjectID, "")
if err != nil {
return fmt.Errorf("getOptOrEnvOrDefault: %w", err)
}

project, err := getOptOrEnvOrDefault(optNameProjectID, optValueProjectID, envNameGCloudProjectID, "")
var dataset string
dataset, err = getOptOrEnvOrDefault(optNameDataset, *optValueDataset, envNameBigQueryDataset, "")
if err != nil {
return fmt.Errorf("getOptOrEnvOrDefault: %w", err)
}

dataset, err := getOptOrEnvOrDefault(optNameDataset, optValueDataset, envNameBigQueryDataset, "")
var filePath string
filePath, err = getOptOrEnvOrDefault(optNameOutputFile, *optValueOutputPath, envNameOutputFile, defaultValueOutputFile)
if err != nil {
return fmt.Errorf("getOptOrEnvOrDefault: %w", err)
}

// set GOOGLE_APPLICATION_CREDENTIALS for Google Cloud SDK
if os.Getenv(envNameGoogleApplicationCredentials) != keyfile {
if err := os.Setenv(envNameGoogleApplicationCredentials, keyfile); err != nil {
if err = os.Setenv(envNameGoogleApplicationCredentials, keyfile); err != nil {
return fmt.Errorf("os.Setenv: %w", err)
}
}
Expand All @@ -96,7 +96,7 @@ func Run(ctx context.Context) error {
return fmt.Errorf("bigquery.NewClient: %w", err)
}
defer func() {
if err := client.Close(); err != nil {
if err = client.Close(); err != nil {
warnln("client.Close: " + err.Error())
}
}()
Expand All @@ -107,7 +107,7 @@ func Run(ctx context.Context) error {
}

// NOTE(djeeno): output
if err := ioutil.WriteFile(filePath, generatedCode, 0644); err != nil {
if err = ioutil.WriteFile(filePath, generatedCode, 0644); err != nil {
return fmt.Errorf("ioutil.WriteFile: %w", err)
}

Expand All @@ -132,7 +132,9 @@ package bqtableschema
var tail string
var importPackages []string
for _, table := range tables {
structCode, pkgs, err := generateTableSchemaCode(ctx, table)
var structCode string
var pkgs []string
structCode, pkgs, err = generateTableSchemaCode(ctx, table)
if err != nil {
warnln("generateTableSchemaCode: " + err.Error())
continue
Expand Down Expand Up @@ -206,7 +208,8 @@ func generateTableSchemaCode(ctx context.Context, table *bigquery.Table) (genera
}
structName := capitalizeInitial(table.TableID)

md, err := table.Metadata(ctx)
var md *bigquery.TableMetadata
md, err = table.Metadata(ctx)
if err != nil {
return "", nil, fmt.Errorf("table.Metadata: %w", err)
}
Expand All @@ -219,7 +222,8 @@ func generateTableSchemaCode(ctx context.Context, table *bigquery.Table) (genera
schemas := []*bigquery.FieldSchema(md.Schema)

for _, schema := range schemas {
goTypeStr, pkg, err := bigqueryFieldTypeToGoType(schema.Type)
var goTypeStr, pkg string
goTypeStr, pkg, err = bigqueryFieldTypeToGoType(schema.Type)
if err != nil {
return "", nil, fmt.Errorf("bigqueryFieldTypeToGoType: %w", err)
}
Expand All @@ -236,7 +240,8 @@ func generateTableSchemaCode(ctx context.Context, table *bigquery.Table) (genera
func getAllTables(ctx context.Context, client *bigquery.Client, datasetID string) (tables []*bigquery.Table, err error) {
tableIterator := client.Dataset(datasetID).Tables(ctx)
for {
table, err := tableIterator.Next()
var table *bigquery.Table
table, err = tableIterator.Next()
if err != nil {
if err == iterator.Done {
break
Expand All @@ -248,21 +253,23 @@ func getAllTables(ctx context.Context, client *bigquery.Client, datasetID string
return tables, nil
}

func readFile(path string) ([]byte, error) {
file, err := os.Open(path)
func readFile(path string) (content []byte, err error) {
var file *os.File
file, err = os.Open(path)
if err != nil {
return nil, fmt.Errorf("os.Open: %w", err)
}

bytea, err := ioutil.ReadAll(file)
var bytea []byte
bytea, err = ioutil.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("ioutil.ReadAll: %w", err)
}

return bytea, nil
}

func getOptOrEnvOrDefault(optName, optValue, envName, defaultValue string) (string, error) {
func getOptOrEnvOrDefault(optName, optValue, envName, defaultValue string) (value string, err error) {
if optName == "" {
return "", fmt.Errorf("optName is empty")
}
Expand Down Expand Up @@ -298,7 +305,7 @@ func infoln(content string) {
}

func warnln(content string) {
log.Println("WARNING: " + content)
log.Println("WARN: " + content)
}

func errorln(content string) {
Expand Down
Loading

0 comments on commit 4b29c1e

Please sign in to comment.