Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fetch apps from publish run #379

Merged
merged 2 commits into from
Feb 29, 2024
Merged

Fetch apps from publish run #379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
68d8ca1
offline-update: Fetch CI target only if is needed
mike-sul Feb 29, 2024
d81c4d1
offline-update: Fetch apps from publish run if present
mike-sul Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions client/foundries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,11 @@ type DeltaStats struct {
Size int `json:"size"`
}

type FetchedApps struct {
Uri string `json:"uri"`
Shortlist string `json:"shortlist"`
}

type TufCustom struct {
HardwareIds []string `json:"hardwareIds,omitempty"`
Tags []string `json:"tags,omitempty"`
Expand All @@ -271,6 +276,7 @@ type TufCustom struct {
CreatedAt string `json:"createdAt,omitempty"`
UpdatedAt string `json:"updatedAt,omitempty"`
LmpVer string `json:"lmp-ver,omitempty"`
FetchedApps *FetchedApps `json:"fetched-apps,omitempty"`
}

type Target struct {
Expand Down
55 changes: 42 additions & 13 deletions subcommands/targets/offline-update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"strconv"
Expand All @@ -27,6 +28,7 @@ type (
ostreeVersion int
hardwareID string
buildTag string
fetchedApps *client.FetchedApps
}
)

Expand Down Expand Up @@ -91,22 +93,15 @@ func doOfflineUpdate(cmd *cobra.Command, args []string) {
// Get the wave/prod/CI specific target with the specified tag to check if it is really present
if len(ouWave) > 0 {
fmt.Printf("Getting Wave Target details; target: %s, wave: %s...\n", targetName, ouWave)
_, targetGetErr = getWaveTargetMeta(factory, targetName, ouWave)
targetCustomData, targetGetErr = getWaveTargetMeta(factory, targetName, ouWave)
} else if ouProd {
fmt.Printf("Getting production Target details; target: %s, tag: %s...\n", targetName, ouTag)
_, targetGetErr = getProdTargetMeta(factory, targetName, ouTag)
targetCustomData, targetGetErr = getProdTargetMeta(factory, targetName, ouTag)
} else {
fmt.Printf("Getting CI Target details; target: %s, tag: %s...\n", targetName, ouTag)
_, targetGetErr = getCiTargetMeta(factory, targetName, ouTag)
targetCustomData, targetGetErr = getCiTargetMeta(factory, targetName, ouTag)
}
subcommands.DieNotNil(targetGetErr)
// Get the specified target from the list of factory targets to obtain the "original" tag/branch that produced
// the target, so we can find out the correct app bundle fetch URL.
targetCustomData, targetGetErr = api.TargetGet(factory, targetName)
subcommands.DieNotNil(targetGetErr)
// Get the target info in order to deduce the ostree and app download URLs
ti, err := getTargetInfo(targetCustomData)
subcommands.DieNotNil(err)
vkhoroz marked this conversation as resolved.
Show resolved Hide resolved

fmt.Printf("Refreshing and downloading TUF metadata for Target %s to %s...\n", targetName, path.Join(dstDir, "tuf"))
subcommands.DieNotNil(downloadTufRepo(factory, targetName, ouTag, ouProd, ouWave, ouExpiresIn, path.Join(dstDir, "tuf")), "Failed to download TUF metadata:")
Expand All @@ -121,11 +116,31 @@ Notice that multiple targets in the same directory is only supported in LmP >= v
}
}

// Get the target info in order to deduce the ostree and app download URLs
ti, err := getTargetInfo(targetCustomData)
subcommands.DieNotNil(err)

fmt.Printf("Downloading an ostree repo from the Target's OE build %d...\n", ti.ostreeVersion)
subcommands.DieNotNil(downloadOstree(factory, ti.ostreeVersion, ti.hardwareID, dstDir), "Failed to download Target's ostree repo:")
if !ouNoApps {
fmt.Printf("Downloading Apps fetched by the `assemble-system-image` run; build number: %d, tag: %s...\n", ti.version, ti.buildTag)
err = downloadApps(factory, targetName, ti.version, ti.buildTag, path.Join(dstDir, "apps"))
if (len(ouWave) > 0 || ouProd) && ti.fetchedApps == nil {
// Get the specified target from the list of factory targets to obtain the "original" tag/branch that produced
// the target, so we can find out the correct app bundle fetch URL.
targetCustomData, targetGetErr = api.TargetGet(factory, targetName)
subcommands.DieNotNil(targetGetErr)

// Get the target info again in order to extract the "original" tag/branch and deduce the app download URLs
ti, err = getTargetInfo(targetCustomData)
subcommands.DieNotNil(err)
}

vkhoroz marked this conversation as resolved.
Show resolved Hide resolved
if ti.fetchedApps == nil {
fmt.Printf("Downloading Apps fetched by the `assemble-system-image` run; build number: %d, tag: %s...\n", ti.version, ti.buildTag)
err = downloadApps(factory, targetName, ti.version, ti.buildTag, path.Join(dstDir, "apps"))
} else {
fmt.Printf("Downloading Apps fetched by the `publish-compose-apps` run; apps: %s, uri: %s...\n", ti.fetchedApps.Shortlist, ti.fetchedApps.Uri)
err = downloadAppsArchive(ti.fetchedApps.Uri, path.Join(dstDir, "apps"))
}
vkhoroz marked this conversation as resolved.
Show resolved Hide resolved
if herr := client.AsHttpError(err); herr != nil && herr.Response.StatusCode == 404 {
fmt.Println("WARNING: The Target Apps were not fetched by the `assemble` run, make sure that App preloading is enabled if needed. The update won't include any Apps!")
} else {
Expand Down Expand Up @@ -153,7 +168,10 @@ func getTargetInfo(targetFile *tuf.FileMeta) (*ouTargetInfo, error) {
return nil, err
}
info.hardwareID = custom.HardwareIds[0]
info.buildTag = custom.Tags[0] // See the assemble.py script in ci-scripts https://github.com/foundriesio/ci-scripts/blob/18b4fb154c37b6ad1bc6e7b7903a540b7a758f5d/assemble.py#L300
info.fetchedApps = custom.FetchedApps
if info.fetchedApps == nil {
info.buildTag = custom.Tags[0] // See the assemble.py script in ci-scripts https://github.com/foundriesio/ci-scripts/blob/18b4fb154c37b6ad1bc6e7b7903a540b7a758f5d/assemble.py#L300
}
info.ostreeVersion = info.version
if len(custom.OrigUri) > 0 {
indx := strings.LastIndexByte(custom.OrigUri, '/')
Expand Down Expand Up @@ -313,8 +331,19 @@ func downloadApps(factory string, targetName string, targetVer int, tag string,
})
}

func downloadAppsArchive(uri string, dstDir string) error {
resp, err := api.RawGet(uri, nil)
return processDownloadResponse(uri, resp, err, func(r io.Reader) error {
return untar(r, dstDir)
})
}

func downloadItem(factory string, targetVer int, runName string, artifactPath string, storeHandler func(r io.Reader) error) error {
resp, err := api.JobservRunArtifact(factory, targetVer, runName, artifactPath)
return processDownloadResponse(artifactPath, resp, err, storeHandler)
}

func processDownloadResponse(artifactPath string, resp *http.Response, err error, storeHandler func(r io.Reader) error) error {
if err != nil {
return err
}
Expand Down
Loading