From 2b491aeb9a3731df3697dcc53147ae188db52459 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Tue, 21 Jan 2025 13:36:14 -0500 Subject: [PATCH] ci: build multiarch images --- .github/workflows/main.yml | 188 ++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 86 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7778bea..296ebbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,9 @@ on: - "*" workflow_dispatch: +env: + OCI_EXE: docker + jobs: base: name: base @@ -24,8 +27,6 @@ jobs: run: make base - name: test - env: - OCI_EXE: docker run: make base.test - name: archive base @@ -60,24 +61,23 @@ jobs: submodules: "recursive" fetch-depth: 1 + - name: set host_arch + run: | + echo "HOST_ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')" >> $GITHUB_ENV + - name: build multiarch run: | - host_arch=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') - make base-${host_arch} + make base-${{ env.HOST_ARCH }} - name: test multiarch - env: - OCI_EXE: docker run: | - host_arch=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') mkdir -p bin - make base-${host_arch}.test + make base-${{ env.HOST_ARCH }}.test - name: archive base run: | - host_arch=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') mkdir -p cache - docker export dockcross/base:latest-${host_arch} | xz -e9 -T0 > ./cache/base.tar.xz + docker export dockcross/base:latest-${{ env.HOST_ARCH }} | xz -e9 -T0 > ./cache/base.tar.xz - name: save base uses: actions/upload-artifact@v4 @@ -93,19 +93,18 @@ jobs: docker image push dockcross/base --all-tags image: - name: ${{ matrix.arch_name.image }} - needs: base - runs-on: ubuntu-24.04 - env: - OCI_EXE: docker + name: ${{ matrix.arch_name.image }}_on_${{ matrix.os }} + needs: [base, base-multiarch] + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-22.04, ubuntu-22.04-arm] arch_name: # Android images - { image: "android-arm", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7 COMP=ndk", ninja: "no", @@ -127,7 +126,7 @@ jobs: } - { image: "android-arm64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv8 COMP=ndk", ninja: "no", @@ -149,7 +148,7 @@ jobs: } - { image: "android-x86", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-32 COMP=ndk", ninja: "no", @@ -171,7 +170,7 @@ jobs: } - { image: "android-x86_64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64 COMP=ndk", ninja: "no", @@ -194,7 +193,7 @@ jobs: # Linux arm64/armv8 images - { image: "linux-arm64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv8", ninja: "yes", @@ -216,7 +215,7 @@ jobs: } - { image: "linux-arm64-lts", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv8", ninja: "yes", @@ -238,7 +237,7 @@ jobs: } - { image: "linux-arm64-musl", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv8", ninja: "yes", @@ -260,7 +259,7 @@ jobs: } - { image: "linux-arm64-full", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv8", ninja: "yes", @@ -283,7 +282,7 @@ jobs: # Linux mipel images - { image: "linux-mipsel-lts", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -306,7 +305,7 @@ jobs: # Linux armv5 images - { image: "linux-armv5", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -328,7 +327,7 @@ jobs: } - { image: "linux-armv5-musl", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -350,7 +349,7 @@ jobs: } - { image: "linux-armv5-uclibc", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -373,7 +372,7 @@ jobs: # Linux armv6 images - { image: "linux-armv6", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -395,7 +394,7 @@ jobs: } - { image: "linux-armv6-lts", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -417,7 +416,7 @@ jobs: } - { image: "linux-armv6-musl", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -440,7 +439,7 @@ jobs: # Linux armv7 images - { image: "linux-armv7", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7", ninja: "yes", @@ -462,7 +461,7 @@ jobs: } - { image: "linux-armv7a", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7-neon", ninja: "yes", @@ -484,7 +483,7 @@ jobs: } - { image: "linux-armv7a-lts", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7-neon", ninja: "yes", @@ -506,7 +505,7 @@ jobs: } - { image: "linux-armv7-lts", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7", ninja: "yes", @@ -528,7 +527,7 @@ jobs: } - { image: "linux-armv7l-musl", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=armv7", ninja: "yes", @@ -551,7 +550,7 @@ jobs: # Linux x86 images - { image: "linux-x86", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-32-sse41-popcnt", ninja: "yes", @@ -574,7 +573,7 @@ jobs: # Linux i686 images - { image: "linux-i686", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -597,7 +596,7 @@ jobs: # Linux x86_64 images - { image: "linux-x64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern", ninja: "yes", @@ -619,7 +618,7 @@ jobs: } - { image: "linux-x64-tinycc", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern", ninja: "yes", @@ -641,7 +640,7 @@ jobs: } - { image: "linux-x64-clang", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern COMP=clang", ninja: "yes", @@ -663,7 +662,7 @@ jobs: } - { image: "linux-x86_64-full", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern", ninja: "yes", @@ -686,7 +685,7 @@ jobs: # Linux riscv images - { image: "linux-riscv32", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -708,7 +707,7 @@ jobs: } - { image: "linux-riscv64", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -731,7 +730,7 @@ jobs: # Linux s390x images - { image: "linux-s390x", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -754,7 +753,7 @@ jobs: # Linux mips images - { image: "linux-mips", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -776,7 +775,7 @@ jobs: } - { image: "linux-mips-uclibc", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -799,7 +798,7 @@ jobs: # Linux mips images - { image: "linux-mips-lts", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -844,7 +843,7 @@ jobs: # Linux ppc64le images - { image: "linux-ppc64le", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=ppc-64", ninja: "yes", @@ -867,7 +866,7 @@ jobs: # Linux ppc64le-lts images - { image: "linux-ppc64le-lts", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=ppc-64", ninja: "yes", @@ -890,7 +889,7 @@ jobs: # Linux m68k images - { image: "linux-m68k-uclibc", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -913,7 +912,7 @@ jobs: # Linux xtensa images - { image: "linux-xtensa-uclibc", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "yes", @@ -936,7 +935,7 @@ jobs: # Windows x86_x64 images - { image: "windows-static-x64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern COMP=mingw", ninja: "no", @@ -958,7 +957,7 @@ jobs: } - { image: "windows-static-x64-posix", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern COMP=mingw", ninja: "no", @@ -980,7 +979,7 @@ jobs: } - { image: "windows-shared-x64", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1002,7 +1001,7 @@ jobs: } - { image: "windows-shared-x64-posix", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1025,7 +1024,7 @@ jobs: # Windows x86 images - { image: "windows-static-x86", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-32-sse41-popcnt COMP=mingw", ninja: "no", @@ -1047,7 +1046,7 @@ jobs: } - { image: "windows-shared-x86", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1070,7 +1069,7 @@ jobs: # Windows arm images - { image: "windows-armv7", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "ARCH=armv7-neon COMP=clang", ninja: "yes", @@ -1092,7 +1091,7 @@ jobs: } - { image: "windows-arm64", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "ARCH=armv8 COMP=clang", ninja: "yes", @@ -1115,7 +1114,7 @@ jobs: # manylinux_2_28 images - { image: "manylinux_2_28-x64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern", ninja: "yes", @@ -1138,7 +1137,7 @@ jobs: # manylinux2014 images - { image: "manylinux2014-x86", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-32-sse41-popcnt", ninja: "yes", @@ -1160,7 +1159,7 @@ jobs: } - { image: "manylinux2014-x64", - arm_host: "", + multiarch: "", stockfish: "yes", stockfish_arg: "ARCH=x86-64-modern", ninja: "yes", @@ -1182,7 +1181,7 @@ jobs: } - { image: "manylinux2014-aarch64", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1205,7 +1204,7 @@ jobs: # web-wasm images - { image: "web-wasm", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1227,7 +1226,7 @@ jobs: } - { image: "web-wasi", - arm_host: "yes", + multiarch: "yes", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1249,7 +1248,7 @@ jobs: } - { image: "web-wasi-threads", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1272,7 +1271,7 @@ jobs: # Bare metal images - { image: "bare-armv7emhf-nano_newlib", - arm_host: "", + multiarch: "", stockfish: "no", stockfish_arg: "", ninja: "no", @@ -1303,6 +1302,9 @@ jobs: submodules: "recursive" fetch-depth: 1 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - uses: actions/download-artifact@v4 with: name: cache @@ -1310,27 +1312,41 @@ jobs: - name: load base run: xz -d -k < ./cache/base.tar.xz | docker import - dockcross/base:latest - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 + with: + name: cache-${{ matrix.os }} + path: ./cache + + - name: load multiarch base + run: xz -d -k < ./cache/base.tar.xz | docker import - dockcross/base:latest + + - name: load base + run: xz -d -k < ./cache/base.tar.xz | docker import - dockcross/base:latest + + - name: set host_arch + run: | + echo "HOST_ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')" >> $GITHUB_ENV - name: build env: BUILD_CMD: build --cache-from type=gha --cache-to type=gha,mode=max run: | - if test "${{ matrix.arch_name.arm_host }}" = "yes"; then - export BUILD_CMD="buildx build --platform linux/amd64,linux/arm64" + if test "${{ matrix.arch_name.multiarch }}" = "yes"; then + make ${{ matrix.arch_name.image }} + elif test "${{ env.HOST_ARCH }}" = "amd64"; then + make ${{ matrix.arch_name.image }} fi - make ${{ matrix.arch_name.image }} - name: basic test - run: make ${{ matrix.arch_name.image }}.test + run: | + if test "${{ matrix.arch_name.multiarch }}" = "yes" || test "${{ env.HOST_ARCH }}" = "amd64"; then + make ${{ matrix.arch_name.image }}.test + fi # Updated on 19/05/2022 - name: stockfish build test - if: ${{ matrix.arch_name.stockfish == 'yes' }} + if: ${{ matrix.arch_name.stockfish == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch sf_15 https://github.com/official-stockfish/Stockfish.git cd Stockfish/src @@ -1341,7 +1357,7 @@ jobs: # Updated on 12/09/2022 - name: ninja build test - if: ${{ matrix.arch_name.ninja == 'yes' }} + if: ${{ matrix.arch_name.ninja == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch v1.11.1 https://github.com/ninja-build/ninja.git cd ninja @@ -1349,7 +1365,7 @@ jobs: # Updated on 12/09/2022 - name: openssl build test - if: ${{ matrix.arch_name.openssl == 'yes' }} + if: ${{ matrix.arch_name.openssl == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch OpenSSL_1_1_1q https://github.com/openssl/openssl.git cd openssl @@ -1362,7 +1378,7 @@ jobs: # Updated on 12/09/2022 - name: C build test - if: ${{ matrix.arch_name.C == 'yes' }} + if: ${{ matrix.arch_name.C == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone https://github.com/TheAlgorithms/C.git cd C @@ -1373,7 +1389,7 @@ jobs: # Updated on 12/09/2022 No longer maintained - name: C-Plus-Plus build test - if: ${{ matrix.arch_name.C-Plus-Plus == 'yes' }} + if: ${{ matrix.arch_name.C-Plus-Plus == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone https://github.com/TheAlgorithms/C-Plus-Plus.git cd C-Plus-Plus @@ -1384,7 +1400,7 @@ jobs: # Updated on 03/08/2024 - name: llama.cpp build test - if: ${{ matrix.arch_name.llama_cpp == 'yes' }} + if: ${{ matrix.arch_name.llama_cpp == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone https://github.com/ggerganov/llama.cpp.git cd llama.cpp @@ -1395,7 +1411,7 @@ jobs: # Updated on 12/09/2022 - name: fmt build test - if: ${{ matrix.arch_name.fmt == 'yes' }} + if: ${{ matrix.arch_name.fmt == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch 9.1.0 https://github.com/fmtlib/fmt.git cd fmt @@ -1405,7 +1421,7 @@ jobs: # Updated on 19/05/2022 - name: cpython build test - if: ${{ matrix.arch_name.cpython == 'yes' }} + if: ${{ matrix.arch_name.cpython == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch v3.11.2 https://github.com/python/cpython.git cd cpython @@ -1416,7 +1432,7 @@ jobs: # Updated on 12/09/2022 (4.2.0 branch not working) - name: raylib build test - if: ${{ matrix.arch_name.raylib == 'yes' }} + if: ${{ matrix.arch_name.raylib == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone https://github.com/raysan5/raylib.git cd raylib @@ -1427,7 +1443,7 @@ jobs: # Updated on 31/3/2022 - name: mbedtls build test - if: ${{ matrix.arch_name.mbedtls == 'yes' }} + if: ${{ matrix.arch_name.mbedtls == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone --depth 1 --branch archive/baremetal https://github.com/Mbed-TLS/mbedtls.git cd mbedtls @@ -1438,7 +1454,7 @@ jobs: # Updated on 20/6/2022 - name: libopencm3 build test - if: ${{ matrix.arch_name.libopencm3 == 'yes' }} + if: ${{ matrix.arch_name.libopencm3 == 'yes' && (matrix.arch_name.multiarch == 'yes' || env.HOST_ARCH == 'amd64') }} run: | git clone https://github.com/libopencm3/libopencm3.git cd libopencm3