From b836f1f84d81bf7322acc3c33aaeb76b656facdf Mon Sep 17 00:00:00 2001 From: jnbdz Date: Fri, 6 Sep 2024 16:43:39 -0400 Subject: [PATCH] Continued with the clean up. Also, added some TODOs and unit tests. --- entity/build/build.go | 46 +++++---- entity/build/build_test.go | 31 +++++- entity/entity.go | 7 +- entity/errors.go | 8 ++ entity/get/get.go | 169 ++++++++++++++++++++------------ entity/validation/validation.go | 3 + errtypes/errtypes.go | 8 -- 7 files changed, 178 insertions(+), 94 deletions(-) create mode 100644 entity/errors.go delete mode 100644 errtypes/errtypes.go diff --git a/entity/build/build.go b/entity/build/build.go index 7496d7f..e6174e4 100644 --- a/entity/build/build.go +++ b/entity/build/build.go @@ -3,7 +3,6 @@ package build import ( "errors" "fmt" - "github.com/AmadlaOrg/hery/errtypes" "github.com/google/uuid" "path/filepath" "strings" @@ -19,10 +18,11 @@ import ( // IBuild to help with mocking and to gather metadata from remote and local sources. type IBuild interface { - MetaFromRemote(paths storage.AbsPaths, entityUri string) (entity.Entity, error) + Meta(paths storage.AbsPaths, entityUri string) (entity.Entity, error) metaFromLocalWithVersion(entityUri, entityVersion string) (entity.Entity, error) metaFromRemoteWithoutVersion(entityUri string) (entity.Entity, error) metaFromRemoteWithVersion(entityUri, entityVersion string) (entity.Entity, error) + constructOrigin(entityUri, name, version string) string } // SBuild struct implements the MetaBuilder interface. @@ -39,9 +39,9 @@ var ( uuidNew = uuid.New ) -// MetaFromRemote gathers as many details about an Entity as possible from git and from the URI passed to populate the +// Meta gathers as many details about an Entity as possible from git and from the URI passed to populate the // Entity struct. It also validates values that are passed to it. -func (s *SBuild) MetaFromRemote(paths storage.AbsPaths, entityUri string) (entity.Entity, error) { +func (s *SBuild) Meta(paths storage.AbsPaths, entityUri string) (entity.Entity, error) { var ( entityVals = entity.Entity{ Have: false, @@ -55,8 +55,8 @@ func (s *SBuild) MetaFromRemote(paths storage.AbsPaths, entityUri string) (entit } dir, err := s.Entity.FindEntityDir(paths, entityVals) - if !errors.Is(err, errtypes.NotFoundError) && - !errors.Is(err, errtypes.MultipleFoundError) && + if !errors.Is(err, entity.ErrorNotFound) && + !errors.Is(err, entity.ErrorMultipleFound) && err != nil { return entityVals, err } else if err == nil { @@ -74,7 +74,7 @@ func (s *SBuild) MetaFromRemote(paths storage.AbsPaths, entityUri string) (entit } } - if errors.Is(err, errtypes.NotFoundError) || entityVersion == "latest" { + if errors.Is(err, entity.ErrorNotFound) || entityVersion == "latest" { entityVals, err = s.metaFromRemoteWithVersion(entityUri, entityVersion) if err != nil { return entityVals, err @@ -115,6 +115,17 @@ func (s *SBuild) metaFromLocalWithVersion(entityUri, entityVersion string) (enti return entityVals, fmt.Errorf("error extracting repo url: %v", err) } + // TODO: Get hash + // entityVals.Hash + + entityVals.Have = true + entityVals.Exist = true + entityVals.IsPseudoVersion = false + entityVals.Name = filepath.Base(entityUriWithoutVersion) + entityVals.Version = entityVersion + entityVals.Entity = entityUri + entityVals.Origin = s.constructOrigin(entityVals.Entity, entityVals.Name, entityVals.Version) + return entityVals, nil } @@ -156,11 +167,7 @@ func (s *SBuild) metaFromRemoteWithoutVersion(entityUri string) (entity.Entity, entityVals.Name = filepath.Base(entityUri) entityVals.Version = entityVersion entityVals.Entity = fmt.Sprintf("%s@%s", entityUri, entityVersion) - entityVals.Origin = strings.Replace( - entityVals.Entity, - fmt.Sprintf("%s@%s", entityVals.Name, entityVals.Version), - "", - 1) + entityVals.Origin = s.constructOrigin(entityVals.Entity, entityVals.Name, entityVals.Version) return entityVals, nil } @@ -210,11 +217,16 @@ func (s *SBuild) metaFromRemoteWithVersion(entityUri, entityVersion string) (ent entityVals.Name = filepath.Base(entityUriWithoutVersion) entityVals.Version = entityVersion entityVals.Entity = entityUri - entityVals.Origin = strings.Replace( - entityVals.Entity, - fmt.Sprintf("%s@%s", entityVals.Name, entityVals.Version), - "", - 1) + entityVals.Origin = s.constructOrigin(entityVals.Entity, entityVals.Name, entityVals.Version) return entityVals, nil } + +// constructOrigin +func (s *SBuild) constructOrigin(entityUri, name, version string) string { + return strings.Replace( + entityUri, + fmt.Sprintf("%s@%s", name, version), + "", + 1) +} diff --git a/entity/build/build_test.go b/entity/build/build_test.go index 39a9551..66480d1 100644 --- a/entity/build/build_test.go +++ b/entity/build/build_test.go @@ -14,7 +14,7 @@ import ( "testing" ) -func TestMetaFromRemote(t *testing.T) { +func TestMeta(t *testing.T) { // Mocking UUID generation for consistent results originalUUIDNew := uuidNew defer func() { uuidNew = originalUUIDNew }() // Restore original function after test @@ -194,7 +194,7 @@ func TestMetaFromRemote(t *testing.T) { EntityVersionValidation: mockEntityVersionVal, } - metaFromRemote, err := mockBuilder.MetaFromRemote(test.inputPaths, test.inputEntityUri) + metaFromRemote, err := mockBuilder.Meta(test.inputPaths, test.inputEntityUri) if test.hasError { assert.Error(t, err) } else { @@ -641,3 +641,30 @@ func TestMetaFromRemoteWithVersion(t *testing.T) { }) } } + +func TestConstructOrigin(t *testing.T) { + mockBuilder := SBuild{} + + tests := []struct { + name string + inputEntityUri string + inputName string + inputVersion string + expected string + }{ + { + name: "Valid origin", + inputEntityUri: "github.com/AmadlaOrg/Entity", + inputName: "github.com/AmadlaOrg/Entity@latest", + inputVersion: "latest", + expected: "github.com/AmadlaOrg/Entity", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mockBuilder.constructOrigin(tt.inputEntityUri, tt.inputName, tt.inputVersion) + assert.Equal(t, tt.expected, got) + }) + } +} diff --git a/entity/entity.go b/entity/entity.go index 73a4acc..adae6d2 100644 --- a/entity/entity.go +++ b/entity/entity.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/AmadlaOrg/hery/entity/version" versionValidationPkg "github.com/AmadlaOrg/hery/entity/version/validation" - "github.com/AmadlaOrg/hery/errtypes" "github.com/AmadlaOrg/hery/storage" "os" "path/filepath" @@ -30,7 +29,7 @@ func (s *SEntity) FindEntityDir(paths storage.AbsPaths, entityVals Entity) (stri // Check if the directory exists if _, err := os.Stat(exactPath); os.IsNotExist(err) { return "", errors.Join( - errtypes.NotFoundError, + ErrorNotFound, fmt.Errorf("no matching directory found for exact version: %s", exactPath)) } else if err != nil { return "", err @@ -52,13 +51,13 @@ func (s *SEntity) FindEntityDir(paths storage.AbsPaths, entityVals Entity) (stri if len(matches) == 0 { return "", errors.Join( - errtypes.NotFoundError, + ErrorNotFound, fmt.Errorf("no matching directories found for pattern: %s", pattern)) } if len(matches) > 1 { return "", errors.Join( - errtypes.MultipleFoundError, + ErrorMultipleFound, fmt.Errorf("multiple matching directories found for pattern: %s", pattern)) } diff --git a/entity/errors.go b/entity/errors.go new file mode 100644 index 0000000..f8dc2a2 --- /dev/null +++ b/entity/errors.go @@ -0,0 +1,8 @@ +package entity + +import "errors" + +var ( + ErrorNotFound = errors.New("not found") + ErrorMultipleFound = errors.New("multiple found") +) diff --git a/entity/get/get.go b/entity/get/get.go index b632fb5..a2440eb 100644 --- a/entity/get/get.go +++ b/entity/get/get.go @@ -59,7 +59,7 @@ func (s *SGet) GetInTmp(collectionName string, entities []string) (storage.AbsPa func (s *SGet) Get(collectionName string, storagePaths *storage.AbsPaths, entities []string) error { entityBuilds := make([]entity.Entity, len(entities)) for i, e := range entities { - entityMeta, err := s.Build.MetaFromRemote(*storagePaths, e) + entityMeta, err := s.Build.Meta(*storagePaths, e) if err != nil { return err } @@ -74,7 +74,8 @@ func (s *SGet) Get(collectionName string, storagePaths *storage.AbsPaths, entiti return s.download(collectionName, storagePaths, entityBuilds) } -// download retrieves entities in parallel. +// download retrieves entities in parallel using concurrency and calls on the functions to set up, validate and +// collect sub entities func (s *SGet) download(collectionName string, storagePaths *storage.AbsPaths, entitiesMeta []entity.Entity) error { var wg sync.WaitGroup wg.Add(len(entitiesMeta)) @@ -91,83 +92,40 @@ func (s *SGet) download(collectionName string, storagePaths *storage.AbsPaths, e go func(entityMeta entity.Entity) { defer wg.Done() - // Create the directory if it does not exist - err := os.MkdirAll(entityMeta.AbsPath, os.ModePerm) + // 1. Add repository in the collection directory + err := s.addRepo(entityMeta) if err != nil { errCh <- err return } - // Download the Entity with `git clone` - if err := s.Git.FetchRepo(entityMeta.RepoUrl, entityMeta.AbsPath); err != nil { - errCh <- fmt.Errorf("error fetching repo: %v", err) - return - } - - // Changes the repository to the tag (version) that was pass - if !entityMeta.IsPseudoVersion { - if err := s.Git.CheckoutTag(entityMeta.AbsPath, entityMeta.Version); err != nil { - errCh <- fmt.Errorf("error checking out version: %v", err) - return - } - } - - read, err := s.HeryExt.Read(entityMeta.AbsPath, collectionName) + // 2. Gather the `.hery` configuration file content + heryContent, err := s.HeryExt.Read(entityMeta.AbsPath, collectionName) if err != nil { errCh <- fmt.Errorf("error reading yaml: %v", err) return } + // 3. Validate the content of hery file content to make sure it does not cause is issue later in the code + // + // -- This follows the Fail Fast principal -- + // err = s.EntityValidation.Entity(collectionName, entityMeta.AbsPath) if err != nil { errCh <- fmt.Errorf("error validating entity: %v", err) return } - var subEntitiesMeta []entity.Entity - for key, value := range read { - if key == "_entity" { - entityPath, ok := value.(string) - if !ok { - errCh <- fmt.Errorf("error converting yaml entity to string: %v", value) - return - } - subEntityMeta, err := s.Build.MetaFromRemote(*storagePaths, entityPath) - if err != nil { - errCh <- fmt.Errorf("error fetching sub entity meta: %v", err) - return - } - subEntitiesMeta = append(subEntitiesMeta, subEntityMeta) - } else if key == "_self" { - selfMap, ok := value.(map[string]interface{}) - if !ok { - errCh <- fmt.Errorf("error converting yaml entity to string: %v", value) - return - } - for selfKey, selfValue := range selfMap { - if selfKey == "_entity" { - entityPath, ok := selfValue.(string) - if !ok { - errCh <- fmt.Errorf("error converting yaml entity to string: %v", selfValue) - return - } - subEntityMeta, err := s.Build.MetaFromRemote(*storagePaths, entityPath) - if err != nil { - errCh <- fmt.Errorf("error fetching sub entity meta: %v", err) - return - } - subEntitiesMeta = append(subEntitiesMeta, subEntityMeta) - } - } - } - } - - if len(subEntitiesMeta) > 0 { - err = s.download(collectionName, storagePaths, subEntitiesMeta) - if err != nil { - errCh <- fmt.Errorf("error downloading sub entities: %v", err) - return - } + // 4. The reference to the other entities are found in the hery file content + // + // This function gathers the `_entity` properties that have the entity URIs that are used to pull the entity + // repositories. + // + // TODO: Add limit on how many times this can be called since we don't want infinite loop (maybe add a counter) + err = s.collectSubEntities(collectionName, storagePaths, heryContent) + if err != nil { + errCh <- fmt.Errorf("error collecting sub entities: %v", err) + return } }(entityMeta) @@ -187,3 +145,88 @@ func (s *SGet) download(collectionName string, storagePaths *storage.AbsPaths, e return combinedErr } + +// addRepo does all the tasks required to setup a new entity (or entity with a different version) +// TODO: Add a timer limit (maybe go-git has something for that) so that it does not get +// TODO: Make sure we have clear error. Because it seems it just hangs without clear error. +// TODO: Might want to add hashing of the entity once it was downloaded to have verification that nothing was corrupted for Fail Fast principal +func (s *SGet) addRepo(entityMeta entity.Entity) error { + // 1. Create the directory if it does not exist + err := os.MkdirAll(entityMeta.AbsPath, os.ModePerm) + if err != nil { + return err + } + + // 2. Download the Entity with `git clone` + if err := s.Git.FetchRepo(entityMeta.RepoUrl, entityMeta.AbsPath); err != nil { + return fmt.Errorf("error fetching repo: %v", err) + } + + // 3. Changes the repository to the tag (version) that was pass + if !entityMeta.IsPseudoVersion { + if err := s.Git.CheckoutTag(entityMeta.AbsPath, entityMeta.Version); err != nil { + return fmt.Errorf("error checking out version: %v", err) + } + } + + return nil +} + +// collectSubEntities Calls on download function with the entity URIs that were found in the `_entity` +// +// For the `_self` contains the initial configuration of the setup of the entity `.hery` configuration. It might contain +// `_entity` and this function also pulls those sub entities. +// +// download function is call because inside any entities there might be again sub entities. +func (s *SGet) collectSubEntities( + collectionName string, + storagePaths *storage.AbsPaths, + henryContent map[string]interface{}) error { + + // 1. Loops through the properties found in the `.hery` configuration file + // found in the `_entity` or the `_entity` in `_self` + var subEntitiesMeta []entity.Entity + for key, value := range henryContent { + if key == "_entity" { + entityPath, ok := value.(string) + if !ok { + return fmt.Errorf("error converting yaml entity to string: %v", value) + } + subEntityMeta, err := s.Build.Meta(*storagePaths, entityPath) + if err != nil { + return fmt.Errorf("error fetching sub entity meta: %v", err) + } + subEntitiesMeta = append(subEntitiesMeta, subEntityMeta) + } else if key == "_self" { + selfMap, ok := value.(map[string]interface{}) + if !ok { + return fmt.Errorf("error converting yaml entity to string: %v", value) + } + for selfKey, selfValue := range selfMap { + if selfKey == "_entity" { + entityPath, ok := selfValue.(string) + if !ok { + return fmt.Errorf("error converting yaml entity to string: %v", selfValue) + } + subEntityMeta, err := s.Build.Meta(*storagePaths, entityPath) + if err != nil { + return fmt.Errorf("error fetching sub entity meta: %v", err) + } + subEntitiesMeta = append(subEntitiesMeta, subEntityMeta) + } + } + } + } + + // 2. If sub entities found then send it to download function + // TODO: Add check that this entity does not already in `Have == true` + // TODO: Or maybe add that logic at a higher level so that it is not added to the `subEntitiesMeta` list + if len(subEntitiesMeta) > 0 { + err := s.download(collectionName, storagePaths, subEntitiesMeta) + if err != nil { + return fmt.Errorf("error downloading sub entities: %v", err) + } + } + + return nil +} diff --git a/entity/validation/validation.go b/entity/validation/validation.go index e7cae8f..9b0d44c 100644 --- a/entity/validation/validation.go +++ b/entity/validation/validation.go @@ -47,6 +47,9 @@ func Schema() *jsonschema.Schema { }*/ // Entity validates the YAML content against the JSON schema +// TODO: Make sure that YAML standard is valid first +// TODO: Since JSON-Schema cannot merge by-it-self the schemas you will need to add code for that +// TODO: Make sure it validates properly with both the based schema found in `.schema` and the entity's own `schema.json` func (s *SValidation) Entity(collectionName, entityPath string) error { schemaPath := filepath.Join(entityPath, fmt.Sprintf(".%s", collectionName), "schema.json") schema, err := s.Schema.Load(schemaPath) diff --git a/errtypes/errtypes.go b/errtypes/errtypes.go deleted file mode 100644 index 61636df..0000000 --- a/errtypes/errtypes.go +++ /dev/null @@ -1,8 +0,0 @@ -package errtypes - -import "errors" - -var ( - NotFoundError = errors.New("not found") - MultipleFoundError = errors.New("multiple found") -)