-
Notifications
You must be signed in to change notification settings - Fork 107
/
store.go
649 lines (545 loc) · 15.4 KB
/
store.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
// Package store contains primitives for representing and changing the
// osbuild-composer state.
package store
import (
"crypto/rand"
"strings"
// The use of SHA1 is valid here
/* #nosec G505 */
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
"log"
"sort"
"sync"
"time"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/distrofactory"
"github.com/osbuild/images/pkg/manifest"
"github.com/osbuild/osbuild-composer/internal/jsondb"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/target"
"github.com/google/uuid"
)
// StoreDBName is the name under which to save the store to the underlying jsondb
const StoreDBName = "state"
// A Store contains all the persistent state of osbuild-composer, and is serialized
// on every change, and deserialized on start.
//
// blueprints contain the most recent blueprint, using the name as the key
//
// blueprintsCommits contains the order of the commits to a blueprint name as a string of hashes
// with the most recent one last.
//
// blueprintsChanges contains the blueprint change, using the blueprint name string and
// the hash string from blueprintsCommits
type Store struct {
blueprints map[string]blueprint.Blueprint
workspace map[string]blueprint.Blueprint
composes map[uuid.UUID]Compose
sources map[string]SourceConfig
blueprintsChanges map[string]map[string]blueprint.Change
blueprintsCommits map[string][]string
mu sync.RWMutex // protects all fields
stateDir *string
db *jsondb.JSONDatabase
}
type SourceConfig struct {
Name string `json:"name" toml:"name"`
Type string `json:"type" toml:"type"`
URL string `json:"url" toml:"url"`
CheckGPG bool `json:"check_gpg" toml:"check_gpg"`
CheckSSL bool `json:"check_ssl" toml:"check_ssl"`
System bool `json:"system" toml:"system"`
Distros []string `json:"distros" toml:"distros"`
RHSM bool `json:"rhsm" toml:"rhsm"`
CheckRepoGPG bool `json:"check_repogpg" toml:"check_repogpg"`
GPGKeys []string `json:"gpgkeys"`
ModuleHotfixes *bool `json:"module_hotfixes,omitempty"`
}
type NotFoundError struct {
message string
}
func (e *NotFoundError) Error() string {
return e.message
}
type NoLocalTargetError struct {
message string
}
func (e *NoLocalTargetError) Error() string {
return e.message
}
func New(stateDir *string, df *distrofactory.Factory, log *log.Logger) *Store {
var storeStruct storeV0
var db *jsondb.JSONDatabase
if stateDir != nil {
db = jsondb.New(*stateDir, 0600)
_, err := db.Read(StoreDBName, &storeStruct)
if err != nil && log != nil {
log.Fatalf("cannot read state: %v", err)
}
}
store := newStoreFromV0(storeStruct, df, log)
store.stateDir = stateDir
store.db = db
return store
}
func randomSHA1String() (string, error) {
// The use of SHA1 is accepted here
/* #nosec G401 */
hash := sha1.New()
data := make([]byte, 20)
n, err := rand.Read(data)
if err != nil {
return "", err
} else if n != 20 {
return "", errors.New("randomSHA1String: short read from rand")
}
_, err = hash.Write(data)
if err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func (s *Store) change(f func() error) error {
s.mu.Lock()
defer s.mu.Unlock()
result := f()
if s.stateDir != nil {
err := s.db.Write(StoreDBName, s.toStoreV0())
if err != nil {
panic(err)
}
}
return result
}
func (s *Store) ListBlueprints() []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0, len(s.blueprints))
for name := range s.blueprints {
if len(name) == 0 {
continue
}
names = append(names, name)
}
sort.Strings(names)
return names
}
func (s *Store) GetBlueprint(name string) (*blueprint.Blueprint, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
bp, inWorkspace := s.workspace[name]
if !inWorkspace {
var ok bool
bp, ok = s.blueprints[name]
if !ok {
return nil, false
}
}
return &bp, inWorkspace
}
func (s *Store) GetBlueprintCommitted(name string) *blueprint.Blueprint {
s.mu.RLock()
defer s.mu.RUnlock()
bp, ok := s.blueprints[name]
if !ok {
return nil
}
return &bp
}
// GetBlueprintChange returns a specific change to a blueprint
// If the blueprint or change do not exist then an error is returned
func (s *Store) GetBlueprintChange(name string, commit string) (*blueprint.Change, error) {
s.mu.RLock()
defer s.mu.RUnlock()
if _, ok := s.blueprintsChanges[name]; !ok {
return nil, errors.New("Unknown blueprint")
}
change, ok := s.blueprintsChanges[name][commit]
if !ok {
return nil, errors.New("Unknown commit")
}
return &change, nil
}
// GetBlueprintChanges returns the list of changes, oldest first
func (s *Store) GetBlueprintChanges(name string) []blueprint.Change {
s.mu.RLock()
defer s.mu.RUnlock()
var changes []blueprint.Change
for _, commit := range s.blueprintsCommits[name] {
changes = append(changes, s.blueprintsChanges[name][commit])
}
return changes
}
func (s *Store) PushBlueprint(bp blueprint.Blueprint, commitMsg string) error {
return s.change(func() error {
// Make sure the blueprint has default values and that the version is valid
err := bp.Initialize()
if err != nil {
return err
}
commit, err := randomSHA1String()
if err != nil {
return err
}
timestamp := time.Now().Format("2006-01-02T15:04:05Z")
change := blueprint.Change{
Commit: commit,
Message: commitMsg,
Timestamp: timestamp,
Blueprint: bp,
}
delete(s.workspace, bp.Name)
if s.blueprintsChanges[bp.Name] == nil {
s.blueprintsChanges[bp.Name] = make(map[string]blueprint.Change)
}
s.blueprintsChanges[bp.Name][commit] = change
// Keep track of the order of the commits
s.blueprintsCommits[bp.Name] = append(s.blueprintsCommits[bp.Name], commit)
if old, ok := s.blueprints[bp.Name]; ok {
if bp.Version == "" || bp.Version == old.Version {
bp.BumpVersion(old.Version)
}
}
s.blueprints[bp.Name] = bp
return nil
})
}
func (s *Store) PushBlueprintToWorkspace(bp blueprint.Blueprint) error {
return s.change(func() error {
if len(bp.Name) == 0 {
return fmt.Errorf("empty blueprint name not allowed")
}
// Make sure the blueprint has default values and that the version is valid
err := bp.Initialize()
if err != nil {
return err
}
s.workspace[bp.Name] = bp
return nil
})
}
// DeleteBlueprint will remove the named blueprint from the store
// if the blueprint does not exist it will return an error
// The workspace copy is deleted unconditionally, it will not return an error if it does not exist.
func (s *Store) DeleteBlueprint(name string) error {
return s.change(func() error {
delete(s.workspace, name)
if _, ok := s.blueprints[name]; !ok {
return fmt.Errorf("Unknown blueprint: %s", name)
}
delete(s.blueprints, name)
return nil
})
}
// DeleteBlueprintFromWorkspace deletes the workspace copy of a blueprint
// if the blueprint doesn't exist in the workspace it returns an error
func (s *Store) DeleteBlueprintFromWorkspace(name string) error {
return s.change(func() error {
if _, ok := s.workspace[name]; !ok {
return fmt.Errorf("Unknown blueprint: %s", name)
}
delete(s.workspace, name)
return nil
})
}
// TagBlueprint will tag the most recent commit
// It will return an error if the blueprint doesn't exist
func (s *Store) TagBlueprint(name string) error {
return s.change(func() error {
_, ok := s.blueprints[name]
if !ok {
return errors.New("Unknown blueprint")
}
_, ok = s.blueprintsCommits[name]
if !ok || len(s.blueprintsCommits[name]) == 0 {
return errors.New("No commits for blueprint")
}
_, ok = s.blueprintsChanges[name]
if !ok {
return errors.New("No changes for blueprint")
}
latest := s.blueprintsCommits[name][len(s.blueprintsCommits[name])-1]
// If the most recent commit already has a revision, don't bump it
if s.blueprintsChanges[name][latest].Revision != nil {
return nil
}
// Get the latest revision for this blueprint (or 0 if there is none)
// blueprintsCommits has the most recent commit at the end, so start there
var revision int
for i := len(s.blueprintsCommits[name]) - 1; i >= 0; i-- {
commit := s.blueprintsCommits[name][i]
change := s.blueprintsChanges[name][commit]
if change.Revision != nil && *change.Revision > revision {
revision = *change.Revision
break
}
}
// Bump the revision (if there was none it will start at 1) of the latest commit
revision++
change := s.blueprintsChanges[name][latest]
change.Revision = &revision
s.blueprintsChanges[name][latest] = change
return nil
})
}
func (s *Store) GetCompose(id uuid.UUID) (Compose, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
compose, exists := s.composes[id]
return compose, exists
}
// GetAllComposes creates a deep copy of all composes present in this store
// and returns them as a dictionary with compose UUIDs as keys
func (s *Store) GetAllComposes() map[uuid.UUID]Compose {
s.mu.RLock()
defer s.mu.RUnlock()
composes := make(map[uuid.UUID]Compose)
for id, singleCompose := range s.composes {
newCompose := singleCompose.DeepCopy()
composes[id] = newCompose
}
return composes
}
func (s *Store) PushCompose(composeID uuid.UUID,
manifest manifest.OSBuildManifest,
imageType distro.ImageType,
bp *blueprint.Blueprint,
size uint64,
targets []*target.Target,
jobId uuid.UUID,
packages []rpmmd.PackageSpec) error {
if _, exists := s.GetCompose(composeID); exists {
panic("a compose with this id already exists")
}
if targets == nil {
targets = []*target.Target{}
}
// FIXME: handle or comment this possible error
_ = s.change(func() error {
s.composes[composeID] = Compose{
Blueprint: bp,
ImageBuild: ImageBuild{
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
Size: size,
JobID: jobId,
},
Packages: packages,
}
return nil
})
return nil
}
// PushTestCompose is used for testing
// Set testSuccess to create a fake successful compose, otherwise it will create a failed compose
// It does not actually run a compose job
func (s *Store) PushTestCompose(composeID uuid.UUID,
manifest manifest.OSBuildManifest,
imageType distro.ImageType,
bp *blueprint.Blueprint,
size uint64,
targets []*target.Target,
testSuccess bool,
packages []rpmmd.PackageSpec) error {
if targets == nil {
targets = []*target.Target{}
}
var status common.ImageBuildState
if testSuccess {
status = common.IBFinished
} else {
status = common.IBFailed
}
// FIXME: handle or comment this possible error
_ = s.change(func() error {
s.composes[composeID] = Compose{
Blueprint: bp,
ImageBuild: ImageBuild{
QueueStatus: status,
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
JobStarted: time.Now(),
Size: size,
},
Packages: packages,
}
return nil
})
return nil
}
// DeleteCompose deletes the compose from the state file and also removes all files on disk that are
// associated with this compose
func (s *Store) DeleteCompose(id uuid.UUID) error {
return s.change(func() error {
if _, exists := s.composes[id]; !exists {
return &NotFoundError{}
}
delete(s.composes, id)
return nil
})
}
// PushSource stores a SourceConfig in store.Sources
func (s *Store) PushSource(key string, source SourceConfig) {
// FIXME: handle or comment this possible error
_ = s.change(func() error {
s.sources[key] = source
return nil
})
}
// DeleteSourceByName removes a SourceConfig from store.Sources using the .Name field
func (s *Store) DeleteSourceByName(name string) {
// FIXME: handle or comment this possible error
_ = s.change(func() error {
for key := range s.sources {
if s.sources[key].Name == name {
delete(s.sources, key)
return nil
}
}
return nil
})
}
// DeleteSourceByID removes a SourceConfig from store.Sources using the ID
func (s *Store) DeleteSourceByID(key string) {
// FIXME: handle or comment this possible error
_ = s.change(func() error {
delete(s.sources, key)
return nil
})
}
// ListSourcesByName returns the repo source names
// Name is different than Id, it can be a full description of the repo
func (s *Store) ListSourcesByName() []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0, len(s.sources))
for _, source := range s.sources {
names = append(names, source.Name)
}
sort.Strings(names)
return names
}
// ListSourcesById returns the repo source id
// Id is a short identifier for the repo, not a full name description
func (s *Store) ListSourcesById() []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0, len(s.sources))
for name := range s.sources {
names = append(names, name)
}
sort.Strings(names)
return names
}
func (s *Store) GetSource(name string) *SourceConfig {
s.mu.RLock()
defer s.mu.RUnlock()
source, ok := s.sources[name]
if !ok {
return nil
}
return &source
}
// GetAllSourcesByName returns the sources using the repo name as the key
func (s *Store) GetAllSourcesByName() map[string]SourceConfig {
s.mu.RLock()
defer s.mu.RUnlock()
sources := make(map[string]SourceConfig)
for _, v := range s.sources {
sources[v.Name] = v
}
return sources
}
// GetAllSourcesByID returns the sources using the repo id as the key
func (s *Store) GetAllSourcesByID() map[string]SourceConfig {
s.mu.RLock()
defer s.mu.RUnlock()
sources := make(map[string]SourceConfig)
for k, v := range s.sources {
sources[k] = v
}
return sources
}
// GetAllDistroSources returns the sources using the repo id as the key
// skipping sources that cannot be used with the selected distribution
func (s *Store) GetAllDistroSources(distro string) map[string]SourceConfig {
s.mu.RLock()
defer s.mu.RUnlock()
sources := make(map[string]SourceConfig)
for k, v := range s.sources {
found := false
for _, d := range v.Distros {
if d == distro {
found = true
}
}
if len(v.Distros) == 0 || found {
sources[k] = v
}
}
return sources
}
func NewSourceConfig(repo rpmmd.RepoConfig, system bool) SourceConfig {
sc := SourceConfig{
Name: repo.Name,
System: system,
RHSM: repo.RHSM,
GPGKeys: repo.GPGKeys,
ModuleHotfixes: repo.ModuleHotfixes,
}
if repo.CheckGPG != nil {
sc.CheckGPG = *repo.CheckGPG
}
if repo.CheckRepoGPG != nil {
sc.CheckRepoGPG = *repo.CheckRepoGPG
}
if repo.IgnoreSSL != nil {
sc.CheckSSL = !*repo.IgnoreSSL
} else {
// default should be true to maintain backwards compatibility
// and current behaviour
sc.CheckSSL = true
}
if len(repo.BaseURLs) != 0 {
sc.URL = strings.Join(repo.BaseURLs, ",")
sc.Type = "yum-baseurl"
} else if repo.Metalink != "" {
sc.URL = repo.Metalink
sc.Type = "yum-metalink"
} else if repo.MirrorList != "" {
sc.URL = repo.MirrorList
sc.Type = "yum-mirrorlist"
}
return sc
}
func (s *SourceConfig) RepoConfig(name string) rpmmd.RepoConfig {
var repo rpmmd.RepoConfig
repo.Name = name
repo.IgnoreSSL = common.ToPtr(!s.CheckSSL)
repo.CheckGPG = common.ToPtr(s.CheckGPG)
repo.RHSM = s.RHSM
repo.CheckRepoGPG = common.ToPtr(s.CheckRepoGPG)
repo.GPGKeys = s.GPGKeys
repo.ModuleHotfixes = s.ModuleHotfixes
var urls []string
if s.URL != "" {
urls = []string{s.URL}
}
if s.Type == "yum-baseurl" {
repo.BaseURLs = urls
} else if s.Type == "yum-metalink" {
repo.Metalink = s.URL
} else if s.Type == "yum-mirrorlist" {
repo.MirrorList = s.URL
}
return repo
}