Jenkins: Add dependency management for MSYS environment

.ci/build.sh

@@ -34,7 +34,7 @@
 #
 
 # Define common functions.
-alias is_windows='[ ! -z "$MSYSTEM" ]'
+alias is_windows='[ -n "$MSYSTEM" ]'
 alias is_mac='uname -s | grep -q Darwin'
 
 make_tar() {
@@ -62,7 +62,7 @@ make_tar() {
 	fi
 
 	# Make tar verbose if requested.
-	[ ! -z "$VERBOSE" ] && local compression_flag="$compression_flag -v"
+	[ -n "$VERBOSE" ] && local compression_flag="$compression_flag -v"
 
 	# tar is notorious for having many diverging implementations. For instance,
 	# the flags we use to strip UID/GID metadata can be --owner/group (GNU),
@@ -136,7 +136,7 @@ done
 cmake_flags_extra=
 
 # Check if mandatory arguments were specified.
-if [ -z "$package_name" -a -z "$tarball_name" ] || [ ! -z "$package_name" -a -z "$arch" ]
+if [ -z "$package_name" -a -z "$tarball_name" ] || [ -n "$package_name" -a -z "$arch" ]
 then
 	echo '[!] Usage: build.sh -b {package_name} {architecture} [-t] [cmake_flags...]'
 	echo '           build.sh -s {source_tarball_name}'
@@ -147,7 +147,7 @@ fi
 cd "$(dirname "$0")/.."
 
 # Make source tarball if requested.
-if [ ! -z "$tarball_name" ]
+if [ -n "$tarball_name" ]
 then
 	echo [-] Making source tarball [$tarball_name]
 
@@ -211,6 +211,106 @@ then
 	fi
 	echo [-] Using MSYSTEM [$MSYSTEM]
 
+	# Update keyring, as the package signing keys sometimes change.
+	echo [-] Updating package databases and keyring
+	yes | pacman -Sy --needed msys2-keyring
+
+	# Query installed packages.
+	pacman -Qe > pacman.txt
+
+	# Download the specified versions of architecture-specific dependencies.
+	echo -n [-] Downloading dependencies:
+	pkg_dir="/var/cache/pacman/pkg"
+	repo_base="https://repo.msys2.org/mingw/$(echo $MSYSTEM | tr '[:upper:]' '[:lower:]')"
+	pkgs=""
+	while IFS=" " read pkg version
+	do
+		prefixed_pkg="$MINGW_PACKAGE_PREFIX-$pkg"
+		installed_version=$(grep -E "^$prefixed_pkg " pacman.txt | cut -d " " -f 2)
+		if [ "$installed_version" != "$version" ] # installed_version will be empty if not installed
+		then
+			echo -n " [$pkg"
+
+			# Download package if not already present in the local cache.
+			pkg_tar="$prefixed_pkg-$version-any.pkg.tar"
+			if [ -s "$pkg_dir/$pkg_tar.xz" ]
+			then
+				pkg_fn="$pkg_tar.xz"
+				pkg_dest="$pkg_dir/$pkg_fn"
+			else
+				pkg_fn="$pkg_tar.zst"
+				pkg_dest="$pkg_dir/$pkg_fn"
+				if [ ! -s "$pkg_dest" ]
+				then
+					if ! wget -qO "$pkg_dest" "$repo_base/$pkg_fn"
+					then
+						rm -f "$pkg_dest"
+						pkg_fn="$pkg_tar.xz"
+						pkg_dest="$pkg_dir/$pkg_fn"
+						wget -qO "$pkg_dest" "$repo_base/$pkg_fn" || rm -f "$pkg_dest"
+					fi
+					if [ -s "$pkg_dest" ]
+					then
+						wget -qO "$pkg_dest.sig" "$repo_base/$pkg_fn.sig" || rm -f "$pkg_dest.sig"
+						[ ! -s "$pkg_dest.sig" ] && rm -f "$pkg_dest.sig"
+					fi
+				fi
+			fi
+
+			# Check if the cached package is valid.
+			if [ -s "$pkg_dest" ]
+			then
+				# Add cached zst package.
+				pkgs="$pkgs $pkg_fn"
+			else
+				# Not valid, remove if it exists.
+				rm -f "$pkg_dest" "$pkg_dest.sig"
+				echo -n " FAIL"
+			fi
+			echo -n "]"
+		fi
+	done < <(cat .ci/dependencies_msys.txt | tr -d '\r')
+	[ -z "$pkgs" ] && echo -n none required
+	echo
+
+	# Install the downloaded architecture-specific dependencies.
+	echo [-] Installing dependencies through pacman
+	if [ -n "$pkgs" ]
+	then
+		pushd "$pkg_dir"
+		yes | pacman -U --needed $pkgs
+		if [ $? -ne 0 ]
+		then
+			# Install packages individually if installing them all together failed.
+			for pkg in pkgs
+			do
+				yes | pacman -U --needed "$pkg"
+			done
+		fi
+		popd
+
+		# Query installed packages again.
+		pacman -Qe > pacman.txt
+	fi
+
+	# Install the latest versions for any missing packages (if the specified version couldn't be installed).
+	pkgs="make git"
+	while IFS=" " read pkg version
+	do
+		prefixed_pkg="$MINGW_PACKAGE_PREFIX-$pkg"
+		grep -qE "^$prefixed_pkg " pacman.txt || pkgs="$pkgs $prefixed_pkg"
+	done < <(cat .ci/dependencies_msys.txt | tr -d '\r')
+	rm -f pacman.txt
+	yes | pacman -S --needed $pkgs
+	if [ $? -ne 0 ]
+	then
+		# Install packages individually if installing them all together failed.
+		for pkg in pkgs
+		do
+			yes | pacman -S --needed "$pkg"
+		done
+	fi
+
 	# Point CMake to the toolchain file.
 	cmake_flags_extra="$cmake_flags_extra -D \"CMAKE_TOOLCHAIN_FILE=cmake/$toolchain.cmake\""
 elif is_mac
@@ -227,7 +327,7 @@ else
 	esac
 
 	# Establish general dependencies.
-	pkgs="cmake pkg-config git imagemagick wget p7zip-full wayland-protocols tar gzip"
+	pkgs="cmake pkg-config git imagemagick wget p7zip-full wayland-protocols tar gzip file"
 	if [ "$(dpkg --print-architecture)" = "$arch_deb" ]
 	then
 		pkgs="$pkgs build-essential"
@@ -324,16 +424,16 @@ if [ "$CI" = "true" ]
 then
 	# Backup strategy when running under Jenkins.
 	[ -z "$git_hash" ] && git_hash=$(echo $GIT_COMMIT | cut -c 1-8)
-elif [ ! -z "$git_hash" ]
+elif [ -n "$git_hash" ]
 then
 	# Append + to denote a dirty tree.
 	git diff --quiet 2> /dev/null || git_hash="$git_hash+"
 fi
-[ ! -z "$git_hash" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_GIT_HASH=$git_hash\""
+[ -n "$git_hash" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_GIT_HASH=$git_hash\""
 
 # Add copyright year.
 year=$(date +%Y)
-[ ! -z "$year" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_COPYRIGHT_YEAR=$year\""
+[ -n "$year" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_COPYRIGHT_YEAR=$year\""
 
 # Run CMake.
 echo [-] Running CMake with flags [$cmake_flags $cmake_flags_extra]
@@ -448,7 +548,8 @@ else
 		# Build SDL2 without sound systems.
 		sdl_ss=OFF
 	else
-		# Build FAudio 22.03 manually to remove the dependency on GStreamer.
+		# Build FAudio 22.03 manually to remove the dependency on GStreamer. This is a temporary
+		# workaround until a newer version of FAudio trickles down to Debian repos.
 		if [ -d "FAudio-22.03" ]
 		then
 			rm -rf FAudio-22.03/build
@@ -563,7 +664,7 @@ else
 	project_version=$(grep -oP '#define\s+EMU_VERSION\s+"\K([^"]+)' "build/src/include/$project_lower/version.h" 2> /dev/null)
 	[ -z "$project_version" ] && project_version=unknown
 	build_num=$(grep -oP '#define\s+EMU_BUILD_NUM\s+\K([0-9]+)' "build/src/include/$project_lower/version.h" 2> /dev/null)
-	[ ! -z "$build_num" -a "$build_num" != "0" ] && project_version="$project_version-b$build_num"
+	[ -n "$build_num" -a "$build_num" != "0" ] && project_version="$project_version-b$build_num"
 
 	# Download appimage-builder if necessary.
 	[ ! -e "appimage-builder.AppImage" ] && wget -qO appimage-builder.AppImage \

.ci/dependencies_msys.txt

@@ -0,0 +1,20 @@
+zlib 1.2.11-9
+binutils 2.37-4
+headers-git 9.0.0.6357.eac8c38c1-1
+crt-git 9.0.0.6357.eac8c38c1-2
+libwinpthread-git 9.0.0.6357.eac8c38c1-1
+winpthreads-git 9.0.0.6357.eac8c38c1-1
+winstorecompat-git 9.0.0.6357.eac8c38c1-1
+gcc-libs 11.2.0-4
+gcc 11.2.0-4
+libgccjit 11.2.0-4
+tools-git 9.0.0.6357.eac8c38c1-1
+make 4.3-1
+pkgconf 1.8.0-2
+openal 1.21.1-3
+libpng 1.6.37-6
+freetype 2.11.1-1
+SDL2 2.0.18-2
+rtmidi 4.0.0-1
+cmake 3.22.1-1
+qt5-static 5.15.2-4

.gitignore

@@ -27,6 +27,7 @@ Makefile
 # Build scripts
 /archive_tmp
 /static2dll.*
+/pacman.txt
 /VERSION
 *.zip
 *.tar