Skip to content

Commit a4e4ca0

Browse files
author
saul-data
committed
interim working
1 parent 4a7dcf2 commit a4e4ca0

27 files changed

+401
-92
lines changed

‎.devcontainer/Dockerfile.dev‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ RUN go install github.com/go-delve/delve/cmd/dlv@latest
55
RUN mkdir -p /go/bin/dlv-dap
66
RUN cp -r /go/bin/dlv /go/bin/dlv-dap
77
RUN go install honnef.co/go/tools/cmd/staticcheck@latest
8+
RUN go install github.com/davidrjenni/reftools/cmd/fillstruct@latest
89
# RUN go get github.com/uudashr/gopkgs/v2/cmd/gopkgs
910
# RUN go get -u github.com/ramya-rao-a/go-outline
1011

‎.devcontainer/docker-compose.yaml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,14 @@ version: '3'
22

33
volumes:
44
node_modules:
5-
5+
66
services:
7+
8+
redis-service:
9+
image: eqalpha/keydb:alpine_x86_64_v6.2.2
10+
ports:
11+
- 6379
12+
713
postgres:
814
image: timescale/timescaledb:2.5.1-pg14
915
ports:
@@ -62,6 +68,7 @@ services:
6268
- ../:/appdev
6369
environment:
6470
DP_CODE_FOLDER: "/appdev/code-files/"
71+
DP_DFS_CODE_FOLDER: "/appdev/dfs-code-files/"
6572
secret_db_host: postgres
6673
secret_db_user: postgres
6774
secret_db_pwd: "Hello123!"
@@ -91,7 +98,7 @@ services:
9198
DP_WORKER_PORT: "9005"
9299
DP_WORKER_LANGUAGES: "Python"
93100
DP_WORKER_LOAD_PACKAGES: "Python"
94-
DP_CODE_FILE_STORAGE: "LocalFile" #Database, LocalFile, S3
101+
DP_CODE_FILE_STORAGE: "Database" #Database, LocalFile, S3
95102
DP_SYNC_FILE_BATCHES: "100" # 100 files synced at once
96103

97104
networks:

‎app/mainapp/code_editor/distribute_filesystem/code_files_to_db.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package distributefilesystem
2+
3+
import (
4+
"crypto/md5"
5+
"dataplane/mainapp/code_editor/filesystem"
6+
"dataplane/mainapp/config"
7+
"dataplane/mainapp/database"
8+
"dataplane/mainapp/database/models"
9+
"fmt"
10+
"log"
11+
"os"
12+
13+
"gorm.io/gorm"
14+
)
15+
16+
/*
17+
Move code files from shared storage to distributed storage
18+
0. On start - if there is no default file_system, move files to DB
19+
20+
1. Get all the files and locations in the db - tables: code_files, code_folders
21+
22+
2. Compare to all the files - Table: code_file_store
23+
24+
3. For all the missing files, collect each file and store in table code_file_store
25+
*/
26+
27+
func MoveCodeFilesToDB(db *gorm.DB) error {
28+
29+
var existingFile []*models.CodeFiles
30+
31+
query := `
32+
SELECT f.file_id, f.environment_id
33+
FROM code_files f
34+
WHERE NOT EXISTS
35+
(
36+
SELECT file_id
37+
FROM code_files_store cs
38+
WHERE f.file_id = cs.file_id
39+
)
40+
`
41+
db.Raw(query).Scan(&existingFile)
42+
43+
log.Println("Number of code files to move:", len(existingFile))
44+
45+
for _, x := range existingFile {
46+
// log.Println(x.FileID, x.EnvironmentID)
47+
48+
// Open each file
49+
fileLoc, _ := filesystem.FileConstructByID(db, x.FileID, x.EnvironmentID, "pipelines")
50+
51+
dat, err := os.ReadFile(config.CodeDirectory + fileLoc)
52+
if err != nil {
53+
log.Println("Read file error:", err)
54+
} else {
55+
56+
md5byte := md5.Sum(dat)
57+
md5string := fmt.Sprintf("%x", md5byte)
58+
59+
codefile := models.CodeFilesStore{
60+
FileID: x.FileID,
61+
FileStore: dat,
62+
ChecksumMD5: md5string,
63+
EnvironmentID: x.EnvironmentID,
64+
}
65+
66+
errdb := database.DBConn.Create(&codefile).Error
67+
if errdb != nil {
68+
log.Println("Create file in database:", err)
69+
} else {
70+
log.Println("DF add: ", md5string, config.CodeDirectory+fileLoc)
71+
}
72+
}
73+
74+
}
75+
return nil
76+
}

‎app/mainapp/code_editor/distribute_filesystem/deploy_files_to_db.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package distributefilesystem
2+
3+
import (
4+
"crypto/md5"
5+
"dataplane/mainapp/code_editor/filesystem"
6+
"dataplane/mainapp/config"
7+
"dataplane/mainapp/database"
8+
"dataplane/mainapp/database/models"
9+
"fmt"
10+
"log"
11+
"os"
12+
13+
"gorm.io/gorm"
14+
)
15+
16+
/*
17+
Move code files from shared storage to distributed storage
18+
0. On start - if there is no default file_system, move files to DB
19+
20+
1. Get all the files and locations in the db - tables: code_files, code_folders
21+
22+
2. Compare to all the files - Table: code_file_store
23+
24+
3. For all the missing files, collect each file and store in table code_file_store
25+
*/
26+
27+
func DeployFilesToDB(db *gorm.DB) error {
28+
29+
var existingFile []*models.DeployCodeFiles
30+
31+
query := `
32+
SELECT f.file_id, f.environment_id, f.version
33+
FROM deploy_code_files f
34+
WHERE NOT EXISTS
35+
(
36+
SELECT file_id
37+
FROM deploy_files_store cs
38+
WHERE f.file_id = cs.file_id and f.version = cs.version
39+
)
40+
`
41+
db.Raw(query).Scan(&existingFile)
42+
43+
log.Println("Number of deploy files to move:", len(existingFile))
44+
45+
for _, x := range existingFile {
46+
// log.Println(x.FileID, x.EnvironmentID)
47+
48+
// Open each file
49+
fileLoc, _ := filesystem.DeployFileConstructByID(db, x.FileID, x.EnvironmentID, "deployments", x.Version)
50+
51+
dat, err := os.ReadFile(config.CodeDirectory + fileLoc)
52+
if err != nil {
53+
log.Println("Read file error:", err)
54+
} else {
55+
56+
md5byte := md5.Sum(dat)
57+
md5string := fmt.Sprintf("%x", md5byte)
58+
59+
codefile := models.DeployFilesStore{
60+
FileID: x.FileID,
61+
Version: x.Version,
62+
FileStore: dat,
63+
ChecksumMD5: md5string,
64+
EnvironmentID: x.EnvironmentID,
65+
}
66+
67+
errdb := database.DBConn.Create(&codefile).Error
68+
if errdb != nil {
69+
log.Println("Create file in database:", err)
70+
} else {
71+
log.Println("DF add: ", md5string, config.CodeDirectory+fileLoc)
72+
}
73+
}
74+
75+
}
76+
return nil
77+
}

‎app/mainapp/code_editor/distribute_filesystem/implementation.MD‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
### Notes and checklist on implementation
2+
3+
1. Make sure new platform gets a default storage of database
4+
2. If default storage is not found in platform table - run migration to database
5+
3. Both code files and deployment files

‎app/mainapp/code_editor/filesystem/deploy_file_construct_id.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package filesystem
2+
3+
import (
4+
"dataplane/mainapp/database"
5+
"dataplane/mainapp/database/models"
6+
7+
"gorm.io/gorm"
8+
)
9+
10+
/*
11+
subbfolder is the folder in code files.
12+
*/
13+
func DeployFileConstructByID(db *gorm.DB, id string, environmentID string, subfolder string, version string) (string, error) {
14+
var currentFile models.DeployCodeFiles
15+
16+
db.Select("file_name", "folder_id").Where("file_id=? and environment_id = ?", id, environmentID).First(&currentFile)
17+
18+
fileName := currentFile.FileName
19+
folderID := currentFile.FolderID
20+
21+
// Folder
22+
folderPath, _ := DeployFolderConstructByID(database.DBConn, folderID, environmentID, subfolder, version)
23+
24+
return folderPath + fileName, nil
25+
}

‎app/mainapp/code_editor/filesystem/file_construct_id.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ import (
77
"gorm.io/gorm"
88
)
99

10+
/*
11+
subbfolder is the folder in code files.
12+
*/
1013
func FileConstructByID(db *gorm.DB, id string, environmentID string, subfolder string) (string, error) {
1114
var currentFile models.CodeFiles
1215

‎app/mainapp/code_editor/filesystem/file_create.go‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"errors"
88
"log"
99
"os"
10+
"time"
1011

1112
"github.com/google/uuid"
1213
)
@@ -27,24 +28,27 @@ func CreateFile(input models.CodeFiles, Folder string, Content []byte) (models.C
2728
var existingFile models.CodeFiles
2829
database.DBConn.Where("environment_id = ? and node_id =? and file_name = ?", input.EnvironmentID, input.NodeID, input.FileName).First(&existingFile)
2930

30-
if _, err := os.Stat(config.CodeDirectory + Folder); os.IsNotExist(err) {
31+
// -------- if LocalFile --------
32+
if config.FSCodeFileStorage == "LocalFile" {
33+
if _, err := os.Stat(config.CodeDirectory + Folder); os.IsNotExist(err) {
3134

32-
if config.Debug == "true" {
33-
log.Println("Directory doesnt exists: ", config.CodeDirectory+Folder)
34-
return input, returnpath, errors.New("Directory doesnt exists")
35-
}
35+
if config.Debug == "true" {
36+
log.Println("Directory doesnt exists: ", config.CodeDirectory+Folder)
37+
return input, returnpath, errors.New("Directory doesnt exists")
38+
}
3639

37-
} else {
40+
} else {
3841

39-
err := os.WriteFile(createFile, Content, 0644)
40-
if err != nil {
41-
return input, returnpath, errors.New("Failed to write file")
42-
}
42+
err := os.WriteFile(createFile, Content, 0644)
43+
if err != nil {
44+
return input, returnpath, errors.New("Failed to write file")
45+
}
4346

44-
if config.Debug == "true" {
45-
log.Println("Created file: ", createFile)
46-
}
47+
if config.Debug == "true" {
48+
log.Println("Created file: ", createFile)
49+
}
4750

51+
}
4852
}
4953

5054
// Create record if doesnt exist
@@ -58,6 +62,8 @@ func CreateFile(input models.CodeFiles, Folder string, Content []byte) (models.C
5862
}
5963

6064
}
65+
} else {
66+
database.DBConn.Model(&models.CodeFiles{}).Where("file_id = ?", existingFile.FileID).Update("updated_at", time.Now().UTC())
6167
}
6268

6369
return input, returnpath, nil

‎app/mainapp/code_editor/filesystem/file_create_processor.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ print("Node id: ` + node.NodeID + `")`
3030
}
3131

3232
// Folder excludes code directory
33+
3334
_, filepath, err = CreateFile(input, Folder, []byte(content))
3435
if err != nil {
3536
return "", err

‎app/mainapp/config/config.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ var MQDebug string = "false"
3434
var CodeDirectory string
3535
var DPDatabase string = ""
3636

37+
// File storage
38+
var FSCodeFileStorage string
39+
var FSCodeFileBatches int
40+
var FSCodeDirectory string
41+
42+
// Available storage methods: Database, LocalFile, S3
43+
3744
func LoadConfig() {
3845

3946
// Clean tasks set
@@ -67,4 +74,16 @@ func LoadConfig() {
6774

6875
DPDatabase = os.Getenv("DP_DATABASE")
6976

77+
/* --- CODE FILE FS ---- */
78+
FSCodeFileStorage = os.Getenv("DP_CODE_FILE_STORAGE")
79+
if FSCodeFileStorage == "" {
80+
FSCodeFileStorage = "Database"
81+
}
82+
83+
FSCodeFileBatches, _ = strconv.Atoi(os.Getenv("DP_SYNC_FILE_BATCHES"))
84+
if FSCodeFileBatches == 0 {
85+
FSCodeFileBatches = 100
86+
}
87+
88+
FSCodeDirectory = os.Getenv("DP_DFS_CODE_FOLDER")
7089
}

0 commit comments

Comments
 (0)