From e3e59c8386f128ab878c63450ce36e1ac181a094 Mon Sep 17 00:00:00 2001 From: John-Philip Johansson Date: Thu, 2 Apr 2020 17:08:38 +0200 Subject: [PATCH 1/5] rpi: Move env vars to a separate `~/.retro-cloud.dev` Moving all the environmental variables from .bashrc makes it easier to copy Azure details to: * VM * Docker host (my dev machine) * Docker container (to not have to setup Azure again) * CI artifacts Also keeps the users .bashrc file cleaner. It can also allow the .env file to become a "template" to be filled in. Making it easier to see what is expected, and allow for the scripts to fill it in as it runs. Which would make it much easier to recover from failures (today it's almost impossible as the script must complete without errors to save any Azure details). Save it in the users HOME path (and not in eg. /etc/profile.d/) because authenticating to the created resources requires the users SSH key, so it's not re-usable by different users. --- raspberry-pi/create-vm.ps1 | 29 ++++++++++++++++++----------- raspberry-pi/mount-vm-share.sh | 10 +++++----- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/raspberry-pi/create-vm.ps1 b/raspberry-pi/create-vm.ps1 index 3eca2e9..fd5ec17 100644 --- a/raspberry-pi/create-vm.ps1 +++ b/raspberry-pi/create-vm.ps1 @@ -229,18 +229,25 @@ ssh "$($username)@$ip" "mkdir -p $sharePath" ################################### $currentActivity = "Persist resource values" -ProgressHelper $currentActivity "Saving configuration variables locally (~/.bashrc)" +$envVarFile="$HOME/.retro-cloud.env" +ProgressHelper $currentActivity "Saving configuration variables locally in $envVarFile" + +Add-Content "$envVarFile" '# RETRO-CLOUD: The environment variables below are from raspberry-pi/create-vm.ps1' +Add-Content "$envVarFile" "export RETROCLOUD_VM_IP=$ip" +Add-Content "$envVarFile" "export RETROCLOUD_VM_USER=$username" +Add-Content "$envVarFile" "export RETROCLOUD_VM_SHARE=$sharePath" +Add-Content "$envVarFile" '# These are mostly useful for troubleshooting.' +Add-Content "$envVarFile" "export RETROCLOUD_AZ_RESOURCE_GROUP=$rg" +Add-Content "$envVarFile" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_NAME=$storageAccountName" +Add-Content "$envVarFile" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_KEY=$storageAccountKey" +Add-Content "$envVarFile" "export RETROCLOUD_AZ_FILE_SHARE_NAME=$fileShareName" +Add-Content "$envVarFile" "export RETROCLOUD_AZ_FILE_SHARE_URL=$smbPath" + +ProgressHelper $currentActivity "Add $envVarFile to be loaded by ~/.bashrc" Add-Content "$HOME/.bashrc" "" -Add-Content "$HOME/.bashrc" '# RETRO-CLOUD: The environment variables below were set by the retro-cloud setup script.' -Add-Content "$HOME/.bashrc" "export RETROCLOUD_VM_IP=$ip" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_VM_USER=$username" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_VM_SHARE=$sharePath" -Add-Content "$HOME/.bashrc" '# RETRO-CLOUD: These are mostly useful for troubleshooting.' -Add-Content "$HOME/.bashrc" "export RETROCLOUD_AZ_RESOURCE_GROUP=$rg" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_NAME=$storageAccountName" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_KEY=$storageAccountKey" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_AZ_FILE_SHARE_NAME=$fileShareName" -Add-Content "$HOME/.bashrc" "export RETROCLOUD_AZ_FILE_SHARE_URL=$smbPath" +Add-Content "$HOME/.bashrc" "# RETRO-CLOUD CONFIG START" +Add-Content "$HOME/.bashrc" ". $envVarFile" +Add-Content "$HOME/.bashrc" "# RETRO-CLOUD CONFIG END" ProgressHelper $currentActivity "Passing configuration variables to VM ($username@${ip}:/home/$username/.bashrc)" ssh "$($username)@$ip" "echo '' | sudo tee -a ~/.bashrc > /dev/null" diff --git a/raspberry-pi/mount-vm-share.sh b/raspberry-pi/mount-vm-share.sh index 676d294..51faffa 100644 --- a/raspberry-pi/mount-vm-share.sh +++ b/raspberry-pi/mount-vm-share.sh @@ -54,11 +54,11 @@ ln -s "$gamelists" "$HOME/.emulationstation" ln -s "$downloadedMedia" "$HOME/.emulationstation" ln -s "$roms" "$HOME/RetroPie" -echo 'Add folder paths as environment variables' -echo "" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "# RETRO-CLOUD: The environment variables below are from raspberry-pi/mount-vm-share.sh" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "# RETRO-CLOUD: These are mostly useful for troubleshooting." | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_RPI_MOUNT_POINT=$mntPath" | sudo tee -a "$HOME/.bashrc" > /dev/null +envVarFile="$HOME/.retro-cloud.env" +echo "Add folder paths as environment variables in $envVarFile" +echo "# RETRO-CLOUD: The environment variables below are from raspberry-pi/mount-vm-share.sh" | sudo tee -a "$envVarFile" > /dev/null +echo "# RETRO-CLOUD: These are mostly useful for troubleshooting." | sudo tee -a "$envVarFile" > /dev/null +echo "export RETROCLOUD_RPI_MOUNT_POINT=$mntPath" | sudo tee -a "$envVarFile" > /dev/null echo 'Done!' echo "VM share mounted on $mntPath" From fa5d98752ddd1afde0e73e1ee2c5af8c9cfb086f Mon Sep 17 00:00:00 2001 From: John-Philip Johansson Date: Thu, 2 Apr 2020 13:19:49 +0100 Subject: [PATCH 2/5] rpi: Copy env var file `~/.retro-cloud.dev` to vm Easier to send a file than creating one, and allows the VM to gain the same benefits as RPI. --- raspberry-pi/create-vm.ps1 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/raspberry-pi/create-vm.ps1 b/raspberry-pi/create-vm.ps1 index fd5ec17..ac714b8 100644 --- a/raspberry-pi/create-vm.ps1 +++ b/raspberry-pi/create-vm.ps1 @@ -233,11 +233,14 @@ $envVarFile="$HOME/.retro-cloud.env" ProgressHelper $currentActivity "Saving configuration variables locally in $envVarFile" Add-Content "$envVarFile" '# RETRO-CLOUD: The environment variables below are from raspberry-pi/create-vm.ps1' +Add-Content "$envVarFile" '# These are needed by the RetroPie.' Add-Content "$envVarFile" "export RETROCLOUD_VM_IP=$ip" Add-Content "$envVarFile" "export RETROCLOUD_VM_USER=$username" +Add-Content "$envVarFile" '# These are needed by both the RetroPie and VM.' Add-Content "$envVarFile" "export RETROCLOUD_VM_SHARE=$sharePath" Add-Content "$envVarFile" '# These are mostly useful for troubleshooting.' Add-Content "$envVarFile" "export RETROCLOUD_AZ_RESOURCE_GROUP=$rg" +Add-Content "$envVarFile" '# These are needed by the VM.' Add-Content "$envVarFile" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_NAME=$storageAccountName" Add-Content "$envVarFile" "export RETROCLOUD_AZ_STORAGE_ACCOUNT_KEY=$storageAccountKey" Add-Content "$envVarFile" "export RETROCLOUD_AZ_FILE_SHARE_NAME=$fileShareName" @@ -249,14 +252,13 @@ Add-Content "$HOME/.bashrc" "# RETRO-CLOUD CONFIG START" Add-Content "$HOME/.bashrc" ". $envVarFile" Add-Content "$HOME/.bashrc" "# RETRO-CLOUD CONFIG END" -ProgressHelper $currentActivity "Passing configuration variables to VM ($username@${ip}:/home/$username/.bashrc)" -ssh "$($username)@$ip" "echo '' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo '# RETRO-CLOUD: The environment variables below were set by the retro-cloud setup script.' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo 'export RETROCLOUD_AZ_STORAGE_ACCOUNT_NAME=$storageAccountName' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo 'export RETROCLOUD_AZ_STORAGE_ACCOUNT_KEY=$storageAccountKey' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo 'export RETROCLOUD_AZ_FILE_SHARE_NAME=$fileShareName' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo 'export RETROCLOUD_AZ_FILE_SHARE_URL=$smbPath' | sudo tee -a ~/.bashrc > /dev/null" -ssh "$($username)@$ip" "echo 'export RETROCLOUD_VM_SHARE=$sharePath' | sudo tee -a ~/.bashrc > /dev/null" +$vmEnvVarFile="/home/$username/.retro-cloud.env" +ProgressHelper $currentActivity "Passing configuration variables to VM (${username}@${ip}:${vmEnvVarFile})" +scp "$envVarFile" "${username}@${ip}:${vmEnvVarFile}" +ssh "${username}@${ip}" "echo '' | sudo tee -a ~/.bashrc > /dev/null" +ssh "${username}@${ip}" "echo '# RETRO-CLOUD CONFIG START' | sudo tee -a ~/.bashrc > /dev/null" +ssh "${username}@${ip}" "echo '. $vmEnvVarFile' | sudo tee -a ~/.bashrc > /dev/null" +ssh "${username}@${ip}" "echo '# RETRO-CLOUD CONFIG END' | sudo tee -a ~/.bashrc > /dev/null" ProgressHelper "Done" " " From ae8a10ad70e0a61b8452d3ed78caeb4451c7cad4 Mon Sep 17 00:00:00 2001 From: John-Philip Johansson Date: Thu, 2 Apr 2020 13:14:34 +0100 Subject: [PATCH 3/5] vm: Move env vars to a separate `~/.retro-cloud.dev` Keeps the users .bashrc file cleaner. It can also allow the .env file to become a "template" to be filled in. Making it easier to see what is expected, and allow for the scripts to fill it in as it runs. Which would make it much easier to recover from failures (today it's almost impossible as the script must complete without errors to save any Azure details). Save it in the users HOME path (and not in eg. /etc/profile.d/) because authenticating to the created resources requires the users SSH key, so it's not re-usable by different users. --- virtual-machine/create-vm-share.sh | 11 +++++------ virtual-machine/mount-az-share.sh | 7 +++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/virtual-machine/create-vm-share.sh b/virtual-machine/create-vm-share.sh index c45fe89..017ac4e 100644 --- a/virtual-machine/create-vm-share.sh +++ b/virtual-machine/create-vm-share.sh @@ -19,11 +19,10 @@ cache="$RETROCLOUD_VM_SHARE/cache" sudo mkdir -p "$cache" echo 'Add folder paths as environment variables' -echo "" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "#RETRO-CLOUD: The environment variables below are from virtual-machine/create-vm-share.sh" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER=$gamelists" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_MEDIAFOLDER=$downloadedMedia" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_ROMS=$roms" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_CACHEFOLDER=$cache" | sudo tee -a "$HOME/.bashrc" > /dev/null +echo "# RETRO-CLOUD: The environment variables below are from virtual-machine/create-vm-share.sh" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER=$gamelists" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_SKYSCRAPER_MEDIAFOLDER=$downloadedMedia" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_ROMS=$roms" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_SKYSCRAPER_CACHEFOLDER=$cache" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null echo 'Done!' diff --git a/virtual-machine/mount-az-share.sh b/virtual-machine/mount-az-share.sh index c0ff482..3432720 100644 --- a/virtual-machine/mount-az-share.sh +++ b/virtual-machine/mount-az-share.sh @@ -51,11 +51,10 @@ sudo mount -a echo 'Add folder paths as environment variables' -echo "" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "#RETRO-CLOUD: The environment variables below are from virtual-machine/mount-az-share.sh" | sudo tee -a "$HOME/.bashrc" > /dev/null +echo "# RETRO-CLOUD: The environment variables below are from virtual-machine/mount-az-share.sh" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null # Note: RETROCLOUD_VM_SHARE and RETROCLOUD_VM_MOUNT_POINT are currently the same. -echo "export RETROCLOUD_VM_MOUNT_POINT=$mntPath" | sudo tee -a "$HOME/.bashrc" > /dev/null -echo "export RETROCLOUD_AZ_CREDENTIALS=$smbCredentialFile" | sudo tee -a "$HOME/.bashrc" > /dev/null +echo "export RETROCLOUD_VM_MOUNT_POINT=$mntPath" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_AZ_CREDENTIALS=$smbCredentialFile" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null echo 'Done!' echo "File share mounted on $mntPath" From fb5e6f5b83eeb2432db0f05898919df7dc6627d1 Mon Sep 17 00:00:00 2001 From: John-Philip Johansson Date: Thu, 2 Apr 2020 13:31:47 +0100 Subject: [PATCH 4/5] ci: Print rpi env var files Output both RPi and VM `~/.retro-cloud.dev`. --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0356000..c42878c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,19 @@ jobs: bash -i ssh-vm.sh "bash -i ~/tmp/test-gamelist.sh" bash -i ssh-vm.sh "rm ~/tmp/test-gamelist.sh ~/tmp/test-gamelist.xml ~/tmp/test-gamelist-screenscraper-failed.xml" + - run: + name: Print RaspberryPi ~/.retro-cloud.env + command: cat ~/.retro-cloud.env + when: always + + - run: + name: Print VM ~/.retro-cloud.env + command: | + echo 'ssh $RETROCLOUD_VM_USER@$RETROCLOUD_VM_IP "cat ~/.retro-cloud.env"' > print-vm-bashrc.sh + bash -i print-vm-bashrc.sh + rm print-vm-bashrc.sh + when: always + - run: name: Print RaspberryPi ~/.bashrc command: cat ~/.bashrc From 0f7f71b55866b33e56169f64434b16a4344e8c6a Mon Sep 17 00:00:00 2001 From: John-Philip Johansson Date: Thu, 2 Apr 2020 17:20:26 +0200 Subject: [PATCH 5/5] vm: Rename VM env vars Make them more consistent with other env vars. --- virtual-machine/create-vm-share.sh | 8 ++++---- virtual-machine/install-skyscraper.sh | 8 ++++---- virtual-machine/local/run-skyscraper.sh | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/virtual-machine/create-vm-share.sh b/virtual-machine/create-vm-share.sh index 017ac4e..4d0d107 100644 --- a/virtual-machine/create-vm-share.sh +++ b/virtual-machine/create-vm-share.sh @@ -20,9 +20,9 @@ sudo mkdir -p "$cache" echo 'Add folder paths as environment variables' echo "# RETRO-CLOUD: The environment variables below are from virtual-machine/create-vm-share.sh" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER=$gamelists" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_MEDIAFOLDER=$downloadedMedia" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null -echo "export RETROCLOUD_ROMS=$roms" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null -echo "export RETROCLOUD_SKYSCRAPER_CACHEFOLDER=$cache" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_VM_GAMELISTS=$gamelists" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_VM_DOWNLOADEDMEDIA=$downloadedMedia" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_VM_ROMS=$roms" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null +echo "export RETROCLOUD_VM_SKYSCRAPER_CACHE=$cache" | sudo tee -a "$HOME/.retro-cloud.env" > /dev/null echo 'Done!' diff --git a/virtual-machine/install-skyscraper.sh b/virtual-machine/install-skyscraper.sh index 72f12c7..ea0fd40 100644 --- a/virtual-machine/install-skyscraper.sh +++ b/virtual-machine/install-skyscraper.sh @@ -23,10 +23,10 @@ mkdir -p "$HOME/.skyscraper" cp -v .skyscraper/config.ini "$HOME/.skyscraper/config.ini" # If these variables aren't available, make sure this script is running in interactive mode (https://stackoverflow.com/a/43660876) and mount-az-share.sh. -sed -i -e "s+RETROCLOUD_INPUTFOLDER+$RETROCLOUD_ROMS+g" "$HOME/.skyscraper/config.ini" -sed -i -e "s+RETROCLOUD_GAMELISTFOLDER+$RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER+g" "$HOME/.skyscraper/config.ini" -sed -i -e "s+RETROCLOUD_MEDIAFOLDER+$RETROCLOUD_SKYSCRAPER_MEDIAFOLDER+g" "$HOME/.skyscraper/config.ini" -sed -i -e "s+RETROCLOUD_CACHEFOLDER+$RETROCLOUD_SKYSCRAPER_CACHEFOLDER+g" "$HOME/.skyscraper/config.ini" +sed -i -e "s+RETROCLOUD_INPUTFOLDER+$RETROCLOUD_VM_ROMS+g" "$HOME/.skyscraper/config.ini" +sed -i -e "s+RETROCLOUD_GAMELISTFOLDER+$RETROCLOUD_VM_GAMELISTS+g" "$HOME/.skyscraper/config.ini" +sed -i -e "s+RETROCLOUD_MEDIAFOLDER+$RETROCLOUD_VM_DOWNLOADEDMEDIA+g" "$HOME/.skyscraper/config.ini" +sed -i -e "s+RETROCLOUD_CACHEFOLDER+$RETROCLOUD_VM_SKYSCRAPER_CACHE+g" "$HOME/.skyscraper/config.ini" echo 'Copy run script to user root' cp -v local/run-skyscraper.sh "$HOME/run-skyscraper.sh" diff --git a/virtual-machine/local/run-skyscraper.sh b/virtual-machine/local/run-skyscraper.sh index 644dab2..6bfa2f0 100644 --- a/virtual-machine/local/run-skyscraper.sh +++ b/virtual-machine/local/run-skyscraper.sh @@ -102,7 +102,7 @@ echo 'Create cache and generate gamelists and artwork' for platform in "${platforms[@]}" do # If the are no games for this platform, skip (because starting Skyscraper for no games is really slow) - if ! [ -d "$RETROCLOUD_ROMS/$platform" ]; then + if ! [ -d "$RETROCLOUD_VM_ROMS/$platform" ]; then echo "No games found for $platform. Skipping." continue fi @@ -127,6 +127,6 @@ do # This step is needed because the AZ mounted path is reflected in the gamelists, and they're symlinked on the rpi to look local. echo "Fixing paths in gamelists for $platform" # Need to sudo because Skyscraper creates the gamelists.xml without write access on them. - sudo sed -i -e "s+$RETROCLOUD_ROMS+/home/pi/RetroPie/roms+g" "$RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER/$platform/gamelist.xml" - sudo sed -i -e "s+$RETROCLOUD_SKYSCRAPER_MEDIAFOLDER+/home/pi/.emulationstation/downloaded_media+g" "$RETROCLOUD_SKYSCRAPER_GAMELISTFOLDER/$platform/gamelist.xml" + sudo sed -i -e "s+$RETROCLOUD_VM_ROMS+/home/pi/RetroPie/roms+g" "$RETROCLOUD_VM_GAMELISTS/$platform/gamelist.xml" + sudo sed -i -e "s+$RETROCLOUD_VM_DOWNLOADEDMEDIA+/home/pi/.emulationstation/downloaded_media+g" "$RETROCLOUD_VM_GAMELISTS/$platform/gamelist.xml" done