From 68bd1c20e83192405ebaa294d1f799f6f7c332dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Bernon?= Date: Wed, 10 Mar 2021 18:38:53 +0100 Subject: [PATCH] build: Use proton sdk docker image for building. --- Makefile | 2 +- Vagrantfile | 17 +-- build-mingw-w64.sh | 229 ---------------------------------------- build/makefile_base.mak | 25 ++--- docker/Makefile | 2 +- docker/README.md | 81 ++++++++++++++ vagrant-user-setup.sh | 33 ------ 7 files changed, 93 insertions(+), 296 deletions(-) delete mode 100755 build-mingw-w64.sh create mode 100644 docker/README.md delete mode 100755 vagrant-user-setup.sh diff --git a/Makefile b/Makefile index 3e1bad50..41ea3d4c 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ endif protonsdk_version := 0.20210126.1-0 CONFIGURE_CMD := ../proton/configure.sh \ - --steam-runtime-image=steam-proton-dev \ + --steam-runtime-image=registry.gitlab.steamos.cloud/proton/soldier/sdk:$(protonsdk_version) \ --build-name="$(_build_name)" # make doesn't handle spaces well... replace them with underscores in paths diff --git a/Vagrantfile b/Vagrantfile index 073764c1..67675742 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -77,15 +77,7 @@ Vagrant.configure(2) do |config| #install host build-time dependencies apt-get update apt-get install -y ccache texinfo gpgv2 gnupg2 git docker-ce docker-ce-cli containerd.io \ - fontforge-nox python-debian python-pip meson libmpc-dev libmpc-dev:i386 \ - gcc g++ gcc-i686-linux-gnu g++-i686-linux-gnu binutils-i686-linux-gnu \ - gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 \ - g++-mingw-w64-i686 g++-mingw-w64-x86-64 - - update-alternatives --set x86_64-w64-mingw32-gcc `which x86_64-w64-mingw32-gcc-posix` - update-alternatives --set x86_64-w64-mingw32-g++ `which x86_64-w64-mingw32-g++-posix` - update-alternatives --set i686-w64-mingw32-gcc `which i686-w64-mingw32-gcc-posix` - update-alternatives --set i686-w64-mingw32-g++ `which i686-w64-mingw32-g++-posix` + fontforge-nox python-debian python-pip #install adobe font devkit to build source san hans pip install afdko @@ -93,18 +85,11 @@ Vagrant.configure(2) do |config| #allow vagrant user to run docker adduser vagrant docker - #add steamrt docker - docker pull registry.gitlab.steamos.cloud/steamrt/soldier/sdk - docker image tag registry.gitlab.steamos.cloud/steamrt/soldier/sdk steam-proton-dev - #allow user to run stuff in steamrt sysctl kernel.unprivileged_userns_clone=1 mkdir -p /etc/sysctl.d/ echo kernel.unprivileged_userns_clone=1 > /etc/sysctl.d/docker_user.conf - # the script below will set up the steam-runtime docker containers - sudo -u vagrant /home/vagrant/proton/vagrant-user-setup.sh - #ensure we use only the mingw-w64 that we built apt-get remove -y '*mingw-w64*' SHELL diff --git a/build-mingw-w64.sh b/build-mingw-w64.sh deleted file mode 100755 index 836e2a38..00000000 --- a/build-mingw-w64.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/bin/bash - -set -e - -# We need two cross-compilers to build Proton: -# -# 64-bit cross-compiler: -# Build (where the compiler is built): 64-bit linux (our VM) -# Host (where the compiler is run): 64-bit linux (64-bit Steam runtime) -# Target (what the compiler outputs): 64-bit win32 (PE files to be run) -# -# 32-bit cross-compiler: -# Build (where the compiler is built): 64-bit linux (our VM) -# Host (where the compiler is run): 64-bit linux (64-bit Steam runtime) -# Target (what the compiler outputs): 32-bit win32 (PE files to be run) - -if [ -z "$1" ]; then - echo "Makes a local build of mingw-w64 in this directory and installs it to the given path." - echo "" - echo "Note: Requires a system mingw-w64 compiler to be present already on your build machine, for us to bootstrap with." - echo "" - echo "usage:" - echo -e "\t$0 " - exit 1 -fi - -if [ -z "$MAKEFLAGS" ]; then - JOBS=-j$(($(nproc) - 1)) -fi - -DST_DIR="$1" - -BINUTILS_VER=2.35 -BINUTILS_SRCTARBALL=binutils-$BINUTILS_VER.tar.xz -BINUTILS_URL="https://ftp.gnu.org/gnu/binutils/$BINUTILS_SRCTARBALL" -BINUTILS_SRCDIR=binutils-$BINUTILS_VER - -GCC_VER=9.3.0 -GCC_SRCTARBALL=gcc-$GCC_VER.tar.xz -GCC_URL="https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VER/$GCC_SRCTARBALL" -GCC_SRCDIR=gcc-$GCC_VER - -MINGW_W64_GITVER=v8.0.0 -MINGW_W64_GITURL="git://git.code.sf.net/p/mingw-w64/mingw-w64" -MINGW_W64_SRCDIR=mingw-w64-git - -ISL_VER=0.22.1 -ISL_SRCTARBALL=isl-$ISL_VER.tar.bz2 -ISL_URL="http://isl.gforge.inria.fr/isl-$ISL_VER.tar.bz2" -ISL_SRCDIR=isl-$ISL_VER - -function setup_src { - if [ ! -e "$BINUTILS_SRCTARBALL" ]; then - wget -O "$BINUTILS_SRCTARBALL" "$BINUTILS_URL" - fi - - if [ ! -e "$BINUTILS_SRCDIR" ]; then - tar -xf "$BINUTILS_SRCTARBALL" - fi - - if [ ! -e "$ISL_SRCTARBALL" ]; then - wget -O "$ISL_SRCTARBALL" "$ISL_URL" - fi - - if [ ! -e "$ISL_SRCDIR" ]; then - tar -xf "$ISL_SRCTARBALL" - fi - - if [ ! -e "$GCC_SRCTARBALL" ]; then - wget -O "$GCC_SRCTARBALL" "$GCC_URL" - fi - - if [ ! -e "$GCC_SRCDIR" ]; then - tar -xf "$GCC_SRCTARBALL" - ln -s ../$ISL_SRCDIR $GCC_SRCDIR/isl - for f in $(dirname $0)/mingw-w64-patches/gcc-*; do - patch -d "$GCC_SRCDIR" -p1 < $f - done - fi - - if [ ! -e "$MINGW_W64_SRCDIR" ]; then - #shallow clone, we just want this one version - git clone -b $MINGW_W64_GITVER --depth 1 \ - "$MINGW_W64_GITURL" "$MINGW_W64_SRCDIR" - fi -} - -function build_arch { - BUILD_ARCH=$(gcc -dumpmachine) #machine which is building the compiler - HOST_ARCH=$1 #machine which will run the compiler - WIN32_TARGET_ARCH=$2 #machine which we are building for - NEWPATH=$DST_DIR/bin:$PATH - - mkdir -p build-$WIN32_TARGET_ARCH/ - pushd build-$WIN32_TARGET_ARCH/ - - mkdir -p binutils/ - pushd binutils/ - if [ ! -e Makefile ]; then - ../../$BINUTILS_SRCDIR/configure \ - --prefix=$DST_DIR/ \ - --build=$BUILD_ARCH \ - --host=$HOST_ARCH \ - --target=$WIN32_TARGET_ARCH \ - --enable-lto \ - --enable-plugins \ - --enable-deterministic-archives \ - --disable-multilib \ - --disable-nls \ - --disable-werror \ - $BINUTILS_EXTRA_CONFIGURE - fi - make $JOBS configure-host - make $JOBS LDFLAGS=-all-static - make $JOBS install - popd - - mkdir -p mingw-w64-headers/ - pushd mingw-w64-headers/ - if [ ! -e Makefile ]; then - PATH=$NEWPATH:$PATH ../../$MINGW_W64_SRCDIR/mingw-w64-headers/configure \ - --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \ - --host=$WIN32_TARGET_ARCH \ - --enable-sdk=all \ - --enable-secure-api \ - $MINGW_W64_HEADERS_EXTRA_CONFIGURE - fi - PATH=$NEWPATH:$PATH make $JOBS install - popd - - mkdir -p gcc/ - pushd gcc/ - if [ ! -e Makefile ]; then - #arguments mostly taken from Arch AUR mingw-w64-gcc PKGBUILD, - #except "--disable-dw2-exceptions" swapped for "--disable-sjlj-exceptions --with-dwarf2" - #for performance reasons on 32-bit - LDFLAGS=-static PATH=$NEWPATH:$PATH ../../$GCC_SRCDIR/configure \ - --prefix=$DST_DIR/ \ - --build=$BUILD_ARCH \ - --host=$HOST_ARCH \ - --target=$WIN32_TARGET_ARCH \ - --enable-static \ - --enable-shared \ - --enable-languages=c,lto,c++ \ - --disable-multilib \ - --enable-threads=posix \ - --enable-fully-dynamic-string \ - --enable-libstdcxx-time=yes \ - --enable-libstdcxx-filesystem-ts=yes \ - --enable-cloog-backend=isl \ - --enable-libgomp \ - --enable-lto \ - --disable-sjlj-exceptions \ - --with-dwarf2 \ - $GCC_EXTRA_CONFIGURE - fi - PATH=$NEWPATH make $JOBS all-gcc - PATH=$NEWPATH make $JOBS install-gcc - popd - - mkdir -p mingw-w64-crt/ - pushd mingw-w64-crt/ - if [ ! -e Makefile ]; then - PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-crt/configure \ - --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \ - --host=$WIN32_TARGET_ARCH \ - --enable-wildcard \ - $MINGW_W64_CRT_EXTRA_CONFIGURE - fi - PATH=$NEWPATH make $JOBS - PATH=$NEWPATH make $JOBS install - popd - - mkdir -p mingw-w64-winpthreads/ - pushd mingw-w64-winpthreads/ - if [ ! -e Makefile ]; then - PATH=$NEWPATH ../../$MINGW_W64_SRCDIR/mingw-w64-libraries/winpthreads/configure \ - --prefix=$DST_DIR/$WIN32_TARGET_ARCH/ \ - --host=$WIN32_TARGET_ARCH \ - --enable-static \ - --enable-shared \ - $MINGW_W64_WINPTHREADS_EXTRA_CONFIGURE - fi - PATH=$NEWPATH make $JOBS - PATH=$NEWPATH make $JOBS install - popd - - pushd gcc/ - #next step requires libgcc in default library location, but - #"canadian" build doesn't handle that(?), so install it explicitly - PATH=$NEWPATH make configure-target-libgcc - PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS - PATH=$NEWPATH make -C $WIN32_TARGET_ARCH/libgcc $JOBS install - - #install libstdc++ and other stuff - PATH=$NEWPATH make $JOBS - PATH=$NEWPATH make $JOBS install - - #libstdc++ requires that libstdc++ is installed in order to find gettimeofday(???) - #so, rebuild libstdc++ after installing it above - PATH=$NEWPATH make $JOBS -C $WIN32_TARGET_ARCH/libstdc++-v3/ distclean - PATH=$NEWPATH make $JOBS - PATH=$NEWPATH make $JOBS install - popd - - mkdir -p mingw-w64-tools/widl - pushd mingw-w64-tools/widl/ - if [ ! -e Makefile ]; then - PATH=$NEWPATH ../../../$MINGW_W64_SRCDIR/mingw-w64-tools/widl/configure \ - --prefix=$DST_DIR/ \ - --target=$WIN32_TARGET_ARCH \ - --program-prefix="${WIN32_TARGET_ARCH}-" - fi - PATH=$NEWPATH make $JOBS - PATH=$NEWPATH make $JOBS install - popd - - popd -} - -setup_src - -mkdir -p $DST_DIR - -MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib64 --enable-lib32" build_arch x86_64-linux-gnu i686-w64-mingw32 - -MINGW_W64_CRT_EXTRA_CONFIGURE="--disable-lib32 --enable-lib64" build_arch x86_64-linux-gnu x86_64-w64-mingw32 - -echo "Done!" diff --git a/build/makefile_base.mak b/build/makefile_base.mak index 474d7cfe..6299ec51 100644 --- a/build/makefile_base.mak +++ b/build/makefile_base.mak @@ -40,20 +40,12 @@ else DOCKER_CCACHE_FLAG = -e CCACHE_DISABLE=1 endif -ifneq ($(filter default undefined,$(origin CC)),) - CC = $(CCACHE_BIN) gcc -endif -ifneq ($(filter default undefined,$(origin CXX)),) - CXX = $(CCACHE_BIN) g++ -endif -ifneq ($(filter default undefined,$(origin CROSSCC)),) - CROSSCC32 = $(CCACHE_BIN) i686-w64-mingw32-gcc - CROSSCC64 = $(CCACHE_BIN) x86_64-w64-mingw32-gcc -endif -ifneq ($(filter default undefined,$(origin CROSSCXX)),) - CROSSCXX32 = $(CCACHE_BIN) i686-w64-mingw32-g++ - CROSSCXX64 = $(CCACHE_BIN) x86_64-w64-mingw32-g++ -endif +CC := $(CCACHE_BIN) x86_64-linux-gnu-gcc +CXX := $(CCACHE_BIN) x86_64-linux-gnu-g++ +CROSSCC32 := $(CCACHE_BIN) i686-w64-mingw32-gcc +CROSSCC64 := $(CCACHE_BIN) x86_64-w64-mingw32-gcc +CROSSCXX32 := $(CCACHE_BIN) i686-w64-mingw32-g++ +CROSSCXX64 := $(CCACHE_BIN) x86_64-w64-mingw32-g++ export CC export CXX @@ -63,8 +55,8 @@ ifeq ($(ENABLE_CCACHE),1) else endif -CC32 := $(CC) -m32 -mstackrealign -CXX32 := $(CXX) -m32 -mstackrealign +CC32 := $(CCACHE_BIN) i686-linux-gnu-gcc -mstackrealign +CXX32 := $(CCACHE_BIN) i686-linux-gnu-g++ -mstackrealign PKG_CONFIG32 := i686-linux-gnu-pkg-config cc-option = $(shell if test -z "`echo 'void*p=1;' | \ @@ -1527,6 +1519,7 @@ vkd3d-proton: vkd3d32 vkd3d64 mediaconv32: SHELL = $(CONTAINER_SHELL) mediaconv32: $(MAKEFILE_DEP) gstreamer32 | $(MEDIACONV_OBJ32) cd $(abspath $(MEDIACONV)) && \ + CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER="i686-linux-gnu-gcc" \ PKG_CONFIG_ALLOW_CROSS=1 \ PKG_CONFIG_PATH=$(abspath $(TOOLS_DIR32))/lib/pkgconfig \ cargo build --target i686-unknown-linux-gnu --target-dir $(abspath $(MEDIACONV_OBJ32)) $(CARGO_BUILD_ARG) diff --git a/docker/Makefile b/docker/Makefile index ac02be7d..d80e9f40 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,7 +2,7 @@ STEAMRT_VERSION = 0.20210126.1 STEAMRT_URLBASE = registry.gitlab.steamos.cloud PROTONSDK_URLBASE = $(STEAMRT_URLBASE)/proton/soldier/sdk -PROTONSDK_VERSION = $(STEAMRT_VERSION)-0 +PROTONSDK_VERSION = $(STEAMRT_VERSION)-0-dev # this is just for building toolchain, as we do static builds it should # not have any impact on the end result, but changing it will invalidate diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..a9583107 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,81 @@ +Proton (soldier) SDK +==================== + +These are the build rules that are used to create docker images to build +Proton. The automated creation of the official images lives in +, but this can be +used to create local images too. + +Local usage +----------- + +The `protonsdk_version` make variable will override which image is used +to build Proton, but may not trigger a full rebuild, so building from +scratch may be necessary. + +Building Proton with a locally build docker image, instead of using the +official images, can be done by using `protonsdk_version=local`. This +may be used for instance to test changes to the docker image recipes. + +Or, it is also possible to build the docker images first by invoking +`make protonsdk` and it will tag the images with the `protonsdk_version` +variable value. + +Official images +--------------- + +To update the official Proton SDK images: + +1) Update the image build rules, `STEAMRT_VERSION` and + `PROTONSDK_VERSION` version numbers in this folder, test locally, + commit and push the changes. + +2) Update `.gitlab-ci.yml` in the + [Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk) + repository to point to the new commit, commit and push to trigger a + new build of "-dev" images. + +3) Once the images are satifying, tag the version in Proton SDK + repository and push the tag, this will trigger a new build of the + images and version them with the same tag as the Git tag. + +4) Once the images have been published, update the default + `protonsdk_version` version number in `Makefile` to use the newly + built images by default. + +Any change or addition to GPL-ed source first requires to update or add +the corresponding source to . +The `SOURCES_URLBASE` variable must be used to download the sources +from there, and its sha256 must be added to validate the sources in the +same way the existing code does. + +Technical details +----------------- + +The images are built incrementally, with intermediate images created +for each component of the toolchain, then assembled together in a single +`proton` image. + +The reason behind this is to optimize the use of docker cache, so that +components that didn't change do not require to be rebuilt every time, +saving a lot of time in the build process, when only small additions are +made. + +It also lets us build several components of the toolchain separately, +in parallel. This is especially the case when building the images on +the CI. + +Because the `ARG` parameter in Dockerfile doesn't behave nicely with +docker caches it is avoided, and `Dockerfile.in` are used with variable +replacements done using `sed`. + +The CI uses Kaniko instead of Docker, +with a bit of script conversion to generate commands usable there. More +details are available in +[Proton SDK](https://gitlab.steamos.cloud/proton/soldier/sdk). + +The `build-base` images are there to create a common ground to build the +other elements of the toolchain. They are based on fairly recent (more +than what `steamrt` provides), but they are only used temporarily to +build a static version of the Binutils, MinGW and GCC binaries, which +are then copied over the `steamrt` base image. diff --git a/vagrant-user-setup.sh b/vagrant-user-setup.sh deleted file mode 100755 index f0b73e8c..00000000 --- a/vagrant-user-setup.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -set -e - -#build and install recent mingw-w64 -if [ ! -e "$HOME/.local/bin/x86_64-w64-mingw32-gcc" ]; then - mkdir -p $HOME/mingw-w64-build/ - (cd $HOME/mingw-w64-build && $HOME/proton/build-mingw-w64.sh "$HOME/.local/") - #clean up the build tree, this takes up like 6GB - rm -rf $HOME/mingw-w64-build/ -fi - -#install rust with x86_64 and i686 targets -RUST_VERSION="1.44.1" -RUST_CARGO_HOME=$HOME/rust/cargo -RUST_RUSTUP_HOME=$HOME/rust/rustup -if [ ! -e "$RUST_CARGO_HOME" ]; then - - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > $HOME/rustup-install.sh - - #64-bit toolchain - CARGO_HOME=$RUST_CARGO_HOME RUSTUP_HOME=$RUST_RUSTUP_HOME \ - sh $HOME/rustup-install.sh -y --no-modify-path \ - --default-host "x86_64-unknown-linux-gnu" \ - --default-toolchain "$RUST_VERSION-x86_64-unknown-linux-gnu" \ - -t "i686-unknown-linux-gnu" - - cat > $HOME/.local/bin/cargo <