Skip to content

Commit c23ad31

Browse files
authored
chore(blob): make file blob paths more descriptive (#263)
This commit: - Updates the converted file and chunk blob paths to be more descriptive - From `{kbUID}/{fileUID}/converted-file/{convFileUID}` - To `kb-{kbUID}/file-{fileUID}/converted-file/{convFileUID}` <!-- CURSOR_SUMMARY --> --- > [!NOTE] > Switch MinIO blob paths to `kb-{kbUID}/file-{fileUID}/...`, add legacy cleanup, and change `DeleteKnowledgeBase` to return an error with handler updated accordingly. > > - **Storage (MinIO)**: > - **Blob paths**: Change base paths to `kb-{kbUID}/file-{fileUID}/converted-file` and `.../chunk` via `fileBasePath()`; update `convertedFileBasePath()` and `chunkBasePath()`. > - **Deletion**: `DeleteKnowledgeBase(ctx, kbUID)` now returns `error` and deletes both new (`kb-{kbUID}`) and legacy (`{kbUID}`) prefixes using `collectErrors`. > - **Handler**: > - Update `DeleteCatalog` to call `MinIO().DeleteKnowledgeBase` with direct `error` handling (no channel). > > <sup>Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit c712a94. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot).</sup> <!-- /CURSOR_SUMMARY -->
1 parent 13b766e commit c23ad31

File tree

2 files changed

+25
-10
lines changed

2 files changed

+25
-10
lines changed

pkg/handler/knowledgebase.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -392,8 +392,7 @@ func (ph *PublicHandler) DeleteCatalog(ctx context.Context, req *artifactpb.Dele
392392
logger.Info("DeleteCatalog starts in background", zap.String("catalog_id", kb.UID.String()))
393393
allPass := true
394394
// delete files in minIO
395-
err = <-ph.service.MinIO().DeleteKnowledgeBase(ctx, kb.UID)
396-
if err != nil {
395+
if err := ph.service.MinIO().DeleteKnowledgeBase(ctx, kb.UID); err != nil {
397396
logger.Error("failed to delete files in minIO in background", zap.Error(err))
398397
allPass = false
399398
}

pkg/minio/knowledgebase.go

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ type KnowledgeBaseI interface {
2525
// SaveTextChunks saves batch of chunks(text files) to MinIO.
2626
SaveTextChunks(_ context.Context, kbUID, fileUID uuid.UUID, chunks map[ChunkUIDType]ChunkContentType) (destinations map[string]string, _ error)
2727
// DeleteKnowledgeBase deletes all files in the knowledge base.
28-
DeleteKnowledgeBase(_ context.Context, kbUID uuid.UUID) chan error
28+
DeleteKnowledgeBase(_ context.Context, kbUID uuid.UUID) error
2929
// DeleteConvertedFileByFileUID deletes a converted file by file UID.
3030
DeleteConvertedFileByFileUID(ctx context.Context, kbUID, fileUID uuid.UUID) error
3131
// DeleteTextChunksByFileUID deletes all the chunks extracted from a file.
@@ -35,8 +35,12 @@ type KnowledgeBaseI interface {
3535
const convertedFileDir = "converted-file"
3636
const chunkDir = "chunk"
3737

38+
func fileBasePath(kbUID, fileUID uuid.UUID) string {
39+
return filepath.Join("kb-"+kbUID.String(), "file-"+fileUID.String())
40+
}
41+
3842
func convertedFileBasePath(kbUID, fileUID uuid.UUID) string {
39-
return filepath.Join(kbUID.String(), fileUID.String(), convertedFileDir)
43+
return filepath.Join(fileBasePath(kbUID, fileUID), convertedFileDir)
4044
}
4145

4246
// SaveConvertedFile saves a converted file to MinIO with the appropriate MIME
@@ -62,7 +66,7 @@ type ChunkUIDType string
6266
type ChunkContentType []byte
6367

6468
func chunkBasePath(kbUID, fileUID uuid.UUID) string {
65-
return filepath.Join(kbUID.String(), fileUID.String(), chunkDir)
69+
return filepath.Join(fileBasePath(kbUID, fileUID), chunkDir)
6670
}
6771

6872
// SaveTextChunks saves batch of chunks(text files) to MinIO.
@@ -121,11 +125,23 @@ func (m *Minio) SaveTextChunks(ctx context.Context, kbUID, fileUID uuid.UUID, ch
121125
return destinations, nil
122126
}
123127

124-
// Delete all files in the knowledge base
125-
func (m *Minio) DeleteKnowledgeBase(ctx context.Context, kbUID uuid.UUID) chan error {
126-
// List all objects in the knowledge base
127-
err := m.DeleteFilesWithPrefix(ctx, config.Config.Minio.BucketName, kbUID.String())
128-
return err
128+
// DeleteKnowledgeBase removes all the blobs in the knowledge base.
129+
func (m *Minio) DeleteKnowledgeBase(ctx context.Context, kbUID uuid.UUID) error {
130+
bucket := config.Config.Minio.BucketName
131+
legacyPrefix := kbUID.String()
132+
133+
err := m.collectErrors(m.DeleteFilesWithPrefix(ctx, bucket, "kb-"+legacyPrefix))
134+
if err != nil {
135+
return fmt.Errorf("deleting catalog files: %w", err)
136+
}
137+
138+
// Catalogs might have blobs in legacy paths.
139+
err = m.collectErrors(m.DeleteFilesWithPrefix(ctx, bucket, legacyPrefix))
140+
if err != nil {
141+
return fmt.Errorf("deleting legacy catalog files: %w", err)
142+
}
143+
144+
return nil
129145
}
130146

131147
func (m *Minio) collectErrors(errChan chan error) error {

0 commit comments

Comments
 (0)