Merge pull request #232 from ipvych/settings

Combine conty build settings in a single file

Author: Kron4ek

README.md

@@ -394,7 +394,7 @@ There are a few ways to update Conty and get the latest packages, use whichever
 
 ### Manual
 
-1. Obtain Arch Linux bootstrap by using `create-arch-bootstrap.sh`. Before running it, you can edit the script if you want , for example, to include a different set of packages inside the container, or to include additional locales. Make sure you have enough free disk space, i recommend at least 10 GB of free space. Root rights are required for this step.
+1. Obtain Arch Linux bootstrap by using `create-arch-bootstrap.sh`. Before running it, you can edit variables in `settings.sh` if you want, for example, to include a different set of packages inside the container, or to include additional locales. Make sure you have enough free disk space, i recommend at least 10 GB of free space. Root rights are required for this step.
 
     ```
     # ./create-arch-bootstrap.sh
@@ -404,7 +404,7 @@ There are a few ways to update Conty and get the latest packages, use whichever
     ```
     # ./enter-chroot.sh
     ```
-3. Now use `create-conty.sh` to create a SquashFS (or DwarFS) image and create a ready-to-use Conty executable. Root rights are not needed for this step. By default a SquashFS image with zstd compression (level 19) will be created, however, if you want, you can edit the script and enable DwarFS, select a different compression algorithm and/or compression level.
+3. Now use `create-conty.sh` to create a SquashFS (or DwarFS) image and create a ready-to-use Conty executable. Root rights are not needed for this step. By default a SquashFS image with zstd compression (level 19) will be created, however, if you want, you can edit variables in `settings.sh` and enable DwarFS, select a different compression algorithm and/or compression level.
 
     ```
     $ ./create-conty.sh
@@ -416,7 +416,7 @@ For the sake of convenience, there are pre-compiled binaries (utils.tar.gz) of b
 
 This repository has GitHub workflows that allows you to make GitHub automatically generate a new Conty binary of your specification, every week or at any time you want.
 
-To start, first fork this repository. Then, you may edit the `create-arch-bootstrap.sh` inside the new repository, to build the packages you want. Then go to the Actions tab.
+To start, first fork this repository. Then, you may edit the `settings.sh` inside the new repository, to build the packages you want & change compression settings. Then go to the Actions tab.
 
 In the Actions tab, go to the Conty CI section in the left-hand menu. Choose "Run Workflow". This will make GitHub make you a new Conty binary. [By default](https://github.com/Kron4ek/Conty/blob/master/.github/workflows/conty.yml#L5), it will also generate a new Conty binary every Friday (you can use a [cron time expression](https://crontab.cronhub.io/) to change the schedule).
 

create-arch-bootstrap.sh

@@ -2,112 +2,25 @@
 
 # Dependencies: curl tar gzip grep coreutils
 # Root rights are required
+source settings.sh
 
-########################################################################
-
-# Package groups
-audio_pkgs="alsa-lib lib32-alsa-lib alsa-plugins lib32-alsa-plugins libpulse \
-	lib32-libpulse alsa-tools alsa-utils pipewire lib32-pipewire pipewire-pulse pipewire-jack lib32-pipewire-jack"
-
-core_pkgs="xorg-xwayland qt6-wayland wayland \
-	lib32-wayland qt5-wayland xorg-server-xephyr gamescope"
-
-video_pkgs="mesa lib32-mesa vulkan-radeon lib32-vulkan-radeon \
-	vulkan-intel lib32-vulkan-intel \
-	vulkan-icd-loader lib32-vulkan-icd-loader vulkan-mesa-layers \
-	lib32-vulkan-mesa-layers libva-mesa-driver lib32-libva-mesa-driver \
-	libva-intel-driver lib32-libva-intel-driver intel-media-driver \
-	mesa-utils vulkan-tools libva-utils lib32-mesa-utils"
-
-wine_pkgs="wine-staging winetricks-git wine-nine wineasio \
-	freetype2 lib32-freetype2 libxft lib32-libxft \
-	flex lib32-flex fluidsynth lib32-fluidsynth \
-	libxrandr lib32-libxrandr xorg-xrandr libldap lib32-libldap \
-	mpg123 lib32-mpg123 libxcomposite lib32-libxcomposite \
-	libxi lib32-libxi libxinerama lib32-libxinerama libxss lib32-libxss \
-	libxslt lib32-libxslt openal lib32-openal \
-	krb5 lib32-krb5 libpulse lib32-libpulse alsa-plugins \
-	lib32-alsa-plugins alsa-lib lib32-alsa-lib gnutls lib32-gnutls \
-	giflib lib32-giflib gst-libav gst-plugin-pipewire gst-plugins-ugly \
-	gst-plugins-bad gst-plugins-bad-libs \
-	gst-plugins-base-libs lib32-gst-plugins-base-libs gst-plugins-base lib32-gst-plugins-base \
-	gst-plugins-good lib32-gst-plugins-good gstreamer lib32-gstreamer \
-	libpng lib32-libpng v4l-utils lib32-v4l-utils \
-	libgpg-error lib32-libgpg-error libjpeg-turbo lib32-libjpeg-turbo \
-	libgcrypt lib32-libgcrypt ncurses lib32-ncurses ocl-icd lib32-ocl-icd 
-	libxcrypt-compat lib32-libxcrypt-compat libva lib32-libva sqlite lib32-sqlite \
-	gtk3 lib32-gtk3 vulkan-icd-loader lib32-vulkan-icd-loader \
-	sdl2 lib32-sdl2 vkd3d lib32-vkd3d libgphoto2 \
-	openssl-1.1 lib32-openssl-1.1 libnm lib32-libnm \
-	cabextract wget gamemode lib32-gamemode mangohud lib32-mangohud"
-
-devel_pkgs="base-devel git meson mingw-w64-gcc cmake"
-
-gaming_pkgs="lutris python-protobuf steam steam-native-runtime steamtinkerlaunch \
-	minigalaxy gamehub legendary prismlauncher bottles playonlinux obs-studio \
-	retroarch retroarch-assets-ozone libretro-beetle-psx-hw sunshine \
-	libretro-blastem libretro-bsnes libretro-dolphin duckstation \
-	libretro-gambatte libretro-melonds libretro-mgba libretro-nestopia \
-	libretro-parallel-n64 libretro-pcsx2 libretro-picodrive libretro-ppsspp \
-	libretro-retrodream libretro-yabause pcsx2-avx-git"
-
-extra_pkgs="nano ttf-dejavu ttf-liberation firefox mpv geany pcmanfm \
-	htop qbittorrent speedcrunch gpicview file-roller openbox lxterminal \
-	yt-dlp minizip nautilus genymotion jre17-openjdk gnome-themes-extra"
-
-# Packages to install
-# You can add packages that you want and remove packages that you don't need
-# Apart from packages from the official Arch repos, you can also specify
-# packages from the Chaotic-AUR repo
-export packagelist="${audio_pkgs} ${core_pkgs} ${video_pkgs} ${wine_pkgs} ${devel_pkgs} ${gaming_pkgs} ${extra_pkgs}"
-
-# If you want to install AUR packages, specify them in this variable
-export aur_packagelist="faugus-launcher-git"
-
-# ALHP is a repository containing packages from the official Arch Linux
-# repos recompiled with -O3, LTO and optimizations for modern CPUs for
-# better performance
-#
-# When this repository is enabled, most of the packages from the official
-# Arch Linux repos will be replaced with their optimized versions from ALHP
-#
-# Set this variable to true, if you want to enable this repository
-enable_alhp_repo="false"
-
-# Feature levels for ALHP. Available feature levels are 2 and 3
-# For level 2 you need a CPU with SSE4.2 instructions
-# For level 3 you need a CPU with AVX2 instructions
-alhp_feature_level="2"
-
-########################################################################
+check_command_available() {
+	for cmd in "$@"; do
+		if ! command -v "$cmd" >&-; then
+			echo "$cmd is required!"
+			exit 1
+		fi
+	done
+}
+check_command_available curl gzip grep sha256sum
 
 if [ $EUID != 0 ]; then
 	echo "Root rights are required!"
-
-	exit 1
-fi
-
-if ! command -v curl 1>/dev/null; then
-	echo "curl is required!"
-	exit 1
-fi
-
-if ! command -v gzip 1>/dev/null; then
-	echo "gzip is required!"
-	exit 1
-fi
-
-if ! command -v grep 1>/dev/null; then
-	echo "grep is required!"
-	exit 1
-fi
-
-if ! command -v sha256sum 1>/dev/null; then
-	echo "sha256sum is required!"
 	exit 1
 fi
 
 script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+bootstrap="${script_dir}"/root.x86_64
 
 mount_chroot () {
 	# First unmount just in case
@@ -124,6 +37,7 @@ mount_chroot () {
 
 	rm -f "${bootstrap}"/etc/resolv.conf
 	cp /etc/resolv.conf "${bootstrap}"/etc/resolv.conf
+	cp "${script_dir}"/settings.sh "${bootstrap}"/conty_settings.sh
 
 	mkdir -p "${bootstrap}"/run/shm
 }
@@ -146,8 +60,9 @@ run_in_chroot () {
 }
 
 install_packages () {
+	source /conty_settings.sh
 	echo "Checking if packages are present in the repos, please wait..."
-	for p in ${packagelist}; do
+	for p in "${PACKAGES[@]}"; do
 		if pacman -Sp "${p}" &>/dev/null; then
 			good_pkglist="${good_pkglist} ${p}"
 		else
@@ -200,54 +115,15 @@ generate_pkg_licenses_file () {
 }
 
 generate_localegen () {
-	cat <<EOF > locale.gen
-ar_EG.UTF-8 UTF-8
-en_US.UTF-8 UTF-8
-en_GB.UTF-8 UTF-8
-en_CA.UTF-8 UTF-8
-en_SG.UTF-8 UTF-8
-es_MX.UTF-8 UTF-8
-zh_CN.UTF-8 UTF-8
-fr_FR.UTF-8 UTF-8
-ru_RU.UTF-8 UTF-8
-ru_UA.UTF-8 UTF-8
-es_ES.UTF-8 UTF-8
-de_DE.UTF-8 UTF-8
-pt_BR.UTF-8 UTF-8
-it_IT.UTF-8 UTF-8
-id_ID.UTF-8 UTF-8
-ja_JP.UTF-8 UTF-8
-bg_BG.UTF-8 UTF-8
-pl_PL.UTF-8 UTF-8
-da_DK.UTF-8 UTF-8
-ko_KR.UTF-8 UTF-8
-tr_TR.UTF-8 UTF-8
-hu_HU.UTF-8 UTF-8
-cs_CZ.UTF-8 UTF-8
-bn_IN UTF-8
-hi_IN UTF-8
-EOF
+	printf '%s\n' "${LOCALES[@]}" > locale.gen
 }
 
 generate_mirrorlist () {
-	cat <<EOF > mirrorlist
-Server = https://mirror1.sl-chat.ru/archlinux/\$repo/os/\$arch
-Server = https://mirror3.sl-chat.ru/archlinux/\$repo/os/\$arch
-Server = https://us.mirrors.cicku.me/archlinux/\$repo/os/\$arch
-Server = https://mirror.osbeck.com/archlinux/\$repo/os/\$arch
-Server = https://md.mirrors.hacktegic.com/archlinux/\$repo/os/\$arch
-Server = https://geo.mirror.pkgbuild.com/\$repo/os/\$arch
-Server = https://mirror.qctronics.com/archlinux/\$repo/os/\$arch
-Server = https://arch.mirror.constant.com/\$repo/os/\$arch
-Server = https://america.mirror.pkgbuild.com/\$repo/os/\$arch
-Server = https://mirror.tmmworkshop.com/archlinux/\$repo/os/\$arch
-EOF
+	printf '%s\n' "$MIRRORLIST" > mirrorlist
 }
 
 cd "${script_dir}" || exit 1
 
-bootstrap="${script_dir}"/root.x86_64
-
 curl -#LO 'https://cdn-mirror.chaotic.cx/chaotic-aur/chaotic-keyring.pkg.tar.zst'
 curl -#LO 'https://cdn-mirror.chaotic.cx/chaotic-aur/chaotic-mirrorlist.pkg.tar.zst'
 
@@ -257,29 +133,16 @@ if [ ! -s chaotic-keyring.pkg.tar.zst ] || [ ! -s chaotic-mirrorlist.pkg.tar.zst
 	exit 1
 fi
 
-bootstrap_urls=("arch.hu.fo" \
-		"mirror.cyberbits.eu" \
-		"mirror.osbeck.com" \
-		"mirror.lcarilla.de" \
-		"mirror.moson.org" \
-  		"mirror.f4st.host")
+for link in "${BOOTSTRAP_DOWNLOAD_URLS[@]}"; do
+	echo "Downloading Arch Linux bootstrap from $link"
+	curl -#LO "$link"
+	curl -#LO "$BOOTSTRAP_SHA256SUM_FILE_URL"
 
-echo "Downloading Arch Linux bootstrap"
-
-for link in "${bootstrap_urls[@]}"; do
-	curl -#LO "https://${link}/archlinux/iso/latest/archlinux-bootstrap-x86_64.tar.zst"
-	curl -#LO "https://${link}/archlinux/iso/latest/sha256sums.txt"
-
-	if [ -s sha256sums.txt ]; then
-		grep bootstrap-x86_64 sha256sums.txt > sha256.txt
-
-		echo "Verifying the integrity of the bootstrap"
-		if sha256sum -c sha256.txt &>/dev/null; then
-			bootstrap_is_good=1
-			break
-		fi
+	echo "Verifying the integrity of the bootstrap"
+	if sha256sum --ignore-missing -c sha256sums.txt &>/dev/null; then
+		bootstrap_is_good=1
+		break
 	fi
-
 	echo "Download failed, trying again with different mirror"
 done
 
@@ -317,11 +180,10 @@ mv mirrorlist "${bootstrap}"/etc/pacman.d/mirrorlist
 } >> "${bootstrap}"/etc/pacman.conf
 
 run_in_chroot pacman-key --init
-echo "keyserver hkps://keyserver.ubuntu.com" >> "${bootstrap}"/etc/pacman.d/gnupg/gpg.conf
 run_in_chroot pacman-key --populate archlinux
 
 # Add Chaotic-AUR repo
-run_in_chroot pacman-key --recv-key 3056513887B78AEB
+run_in_chroot pacman-key --recv-key 3056513887B78AEB --keyserver keyserver.ubuntu.com
 run_in_chroot pacman-key --lsign-key 3056513887B78AEB
 
 mv chaotic-keyring.pkg.tar.zst chaotic-mirrorlist.pkg.tar.zst "${bootstrap}"/opt
@@ -344,8 +206,8 @@ sed -i 's/#NoExtract   =/NoExtract   = usr\/lib\/firmware\/nvidia\/\* usr\/share
 run_in_chroot pacman -Sy archlinux-keyring --noconfirm
 run_in_chroot pacman -Su --noconfirm
 
-if [ "${enable_alhp_repo}" = "true" ]; then
-	if [ "${alhp_feature_level}" -gt 2 ]; then
+if [ -n "$ENABLE_ALHP_REPO" ]; then
+	if [ "${ALHP_FEATURE_LEVEL}" -gt 2 ]; then
 		alhp_feature_level=3
 	else
 		alhp_feature_level=2
@@ -379,12 +241,12 @@ if [ -f "${bootstrap}"/opt/pacman_failed.txt ]; then
 	exit 1
 fi
 
-if [ -n "${aur_packagelist}" ]; then
+if [ "${#AUR_PACKAGES[@]}" -ne 0 ]; then
 	run_in_chroot pacman --noconfirm --needed -S base-devel yay
 	run_in_chroot useradd -m -G wheel aur
 	echo "%wheel ALL=(ALL:ALL) NOPASSWD: ALL" >> "${bootstrap}"/etc/sudoers
 
-	for p in ${aur_packagelist}; do
+	for p in "${AUR_PACKAGES[@]}"; do
 		aur_pkgs="${aur_pkgs} aur/${p}"
 	done
 	export aur_pkgs

create-conty.sh

@@ -3,37 +3,13 @@
 # Dependencies: sed, squashfs-tools or dwarfs
 
 script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
-
-# Enable this variable to use the system-wide mksquashfs/mkdwarfs instead
-# of those provided by the Conty project
-USE_SYS_UTILS=0
-
-# Supported compression algorithms: lz4, zstd, gzip, xz, lzo
-# These are the algorithms supported by the integrated squashfuse
-# However, your squashfs-tools (mksquashfs) may not support some of them
-squashfs_compressor="zstd"
-squashfs_compressor_arguments=(-b 1M -comp ${squashfs_compressor} -Xcompression-level 19)
-
-# Uncomment these variables if your mksquashfs does not support zstd or
-# if you want faster compression/decompression (at the cost of compression ratio)
-#squashfs_compressor="lz4"
-#squashfs_compressor_arguments=(-b 256K -comp "${squashfs_compressor}" -Xhc)
-
-# Use DwarFS instead of SquashFS
-dwarfs="false"
-dwarfs_compressor_arguments=(-l7 -C zstd:level=19 --metadata-compression null \
-                            -S 21 -B 1 --order nilsimsa \
-                            -W 12 -w 4 --no-create-timestamp)
-
-# Set to true to use an existing image if it exists
-# Otherwise the script will always create a new image
-use_existing_image="false"
-
 image_path="${script_dir}"/image
 bootstrap="${script_dir}"/root.x86_64
 
+source "${bootstrap}"/settings.sh
+
 launch_wrapper () {
-	if [ "${USE_SYS_UTILS}" != 0 ]; then
+	if [ -n "${USE_SYS_UTILS}" ]; then
 		if ! command -v "${1}" 1>/dev/null; then
 			echo "Please install $(echo "${1}" | tail -c +3) and run the script again"
 			exit 1
@@ -52,12 +28,12 @@ fi
 
 cd "${script_dir}" || exit 1
 
-if [ "${dwarfs}" = "true" ]; then
+if [ -n "$USE_DWARFS" ]; then
 	utils="utils_dwarfs.tar.gz"
-	compressor_command=(mkdwarfs -i "${bootstrap}" -o "${image_path}" "${dwarfs_compressor_arguments[@]}")
+	compressor_command=(mkdwarfs -i "${bootstrap}" -o "${image_path}" "${DWARFS_COMPRESSOR_ARGUMENTS[@]}")
 else
 	utils="utils.tar.gz"
-	compressor_command=(mksquashfs "${bootstrap}" "${image_path}" "${squashfs_compressor_arguments[@]}")
+	compressor_command=(mksquashfs "${bootstrap}" "${image_path}" "${SQUASHFS_COMPRESSOR_ARGUMENTS[@]}")
 fi
 
 if [ ! -f "${utils}" ] || [ "$(wc -c < "${utils}")" -lt 100000 ]; then
@@ -91,28 +67,23 @@ if [ $? != 0 ]; then
 fi
 
 # Check if selected compression algorithm is supported by mksquashfs
-if [ "${USE_SYS_UTILS}" != 0 ] && [ "${dwarfs}" != "true" ] && command -v grep 1>/dev/null; then
-	# mksquashfs writes its output to stderr instead of stdout
-	mksquashfs &>mksquashfs_out.txt
-
-	if ! grep -q "${squashfs_compressor}" mksquashfs_out.txt; then
+if [ -n "${USE_SYS_UTILS}" ] && [ -z "$USE_DWARFS" ] && command -v grep 1>/dev/null; then
+	if ! mksquashfs 2>&1 | grep -q "${SQUASHFS_COMPRESSOR}"; then
 		echo "Seems like your mksquashfs doesn't support the selected"
-		echo "compression algorithm (${squashfs_compressor})."
+		echo "compression algorithm (${SQUASHFS_COMPRESSOR})."
 		echo
 		echo "Choose another algorithm and run the script again"
 
 		exit 1
 	fi
-
-	rm -f mksquashfs_out.txt
 fi
 
 echo
 echo "Creating Conty..."
 echo
 
 # Create the image
-if [ ! -f "${image_path}" ] || [ "${use_existing_image}" != "true" ]; then
+if [ ! -f "${image_path}" ] || [ -z "${USE_EXISTING_IMAGE}" ]; then
 	if [ ! -d "${bootstrap}" ]; then
 		echo "Distro bootstrap is required!"
 		echo "Use the create-arch-bootstrap.sh script to get it"