Merge pull request #76 from vijeyash1/main

git insertion issue patch

client/pkg/clickhouse/db_client.go

@@ -1,20 +1,24 @@
 package clickhouse
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
 	"log"
 	"time"
 
-	"github.com/ClickHouse/clickhouse-go"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/kube-tarian/kubviz/client/pkg/config"
 	"github.com/kube-tarian/kubviz/model"
 )
 
 type DBClient struct {
-	conn *sql.DB
-	conf *config.Config
+	splconn driver.Conn
+	conn    *sql.DB
+	conf    *config.Config
 }
 type DBInterface interface {
 	InsertRakeesMetrics(model.RakeesMetrics)
@@ -33,12 +37,23 @@ type DBInterface interface {
 }
 
 func NewDBClient(conf *config.Config) (DBInterface, error) {
+	ctx := context.Background()
 	log.Println("Connecting to Clickhouse DB and creating schemas...")
-	conn, err := sql.Open("clickhouse", DbUrl(conf))
+	splconn, err := clickhouse.Open(&clickhouse.Options{
+		Addr:  []string{fmt.Sprintf("%s:%d", conf.DBAddress, conf.DbPort)},
+		Debug: true,
+		Debugf: func(format string, v ...any) {
+			fmt.Printf(format, v)
+		},
+		Settings: clickhouse.Settings{
+			"allow_experimental_object_type": 1,
+		},
+	})
 	if err != nil {
 		return nil, err
 	}
-	if err := conn.Ping(); err != nil {
+
+	if err := splconn.Ping(ctx); err != nil {
 		if exception, ok := err.(*clickhouse.Exception); ok {
 			fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
 		} else {
@@ -48,11 +63,22 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
 	}
 	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerTable, gitTable}
 	for _, table := range tables {
-		if _, err = conn.Exec(string(table)); err != nil {
+		if err = splconn.Exec(context.Background(), string(table)); err != nil {
 			return nil, err
 		}
 	}
-	return &DBClient{conn: conn, conf: conf}, nil
+	stdconn := clickhouse.OpenDB(&clickhouse.Options{
+		Addr: []string{fmt.Sprintf("%s:%d", conf.DBAddress, conf.DbPort)},
+	})
+	if err := stdconn.Ping(); err != nil {
+		if exception, ok := err.(*clickhouse.Exception); ok {
+			fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		} else {
+			fmt.Println(err)
+		}
+		return nil, err
+	}
+	return &DBClient{splconn: splconn, conn: stdconn, conf: conf}, nil
 }
 
 func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
@@ -203,29 +229,32 @@ func (c *DBClient) InsertKubvizEvent(metrics model.Metrics) {
 	}
 }
 func (c *DBClient) InsertGitEvent(event string) {
-	var (
-		tx, _   = c.conn.Begin()
-		stmt, _ = tx.Prepare(fmt.Sprintf("INSERT INTO git_json FORMAT JSONAsObject %v", event))
-	)
-	defer stmt.Close()
+	ctx := context.Background()
+	batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO git_json")
+	if err != nil {
+		log.Fatal(err)
+	}
 
-	if _, err := stmt.Exec(); err != nil {
+	if err = batch.Append(event); err != nil {
 		log.Fatal(err)
 	}
-	if err := tx.Commit(); err != nil {
+
+	if err = batch.Send(); err != nil {
 		log.Fatal(err)
 	}
 }
 func (c *DBClient) InsertContainerEvent(event string) {
-	var (
-		tx, _   = c.conn.Begin()
-		stmt, _ = tx.Prepare(fmt.Sprintf("INSERT INTO container_bridge FORMAT JSONAsObject %v", event))
-	)
-	defer stmt.Close()
-	if _, err := stmt.Exec(); err != nil {
+	ctx := context.Background()
+	batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO container_bridge")
+	if err != nil {
 		log.Fatal(err)
 	}
-	if err := tx.Commit(); err != nil {
+
+	if err = batch.Append(event); err != nil {
+		log.Fatal(err)
+	}
+
+	if err = batch.Send(); err != nil {
 		log.Fatal(err)
 	}
 }

go.mod

@@ -3,7 +3,8 @@ module github.com/kube-tarian/kubviz
 go 1.20
 
 require (
-	github.com/ClickHouse/clickhouse-go v1.5.1
+	github.com/ClickHouse/clickhouse-go v1.5.4
+	github.com/ClickHouse/clickhouse-go/v2 v2.10.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v23.0.5+incompatible
 	github.com/genuinetools/reg v0.16.1
@@ -23,13 +24,17 @@ require (
 )
 
 require (
+	github.com/ClickHouse/ch-go v0.52.1 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.6.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.1 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
@@ -44,6 +49,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/invopop/yaml v0.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
@@ -52,13 +58,19 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/paulmach/orb v0.9.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/perimeterx/marshmallow v1.1.4 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/cobra v1.6.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.13.0 // indirect
+	go.opentelemetry.io/otel/trace v1.13.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -70,9 +82,9 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
 	github.com/corneliusweig/tabwriter v0.0.0-20190512204542-5f8a091e83b5
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -86,8 +98,8 @@ require (
 	github.com/nats-io/nats-server/v2 v2.6.6 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/peterhellberg/link v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.9.0 // indirect