Merge branch 'master' into fix_error

Update CI tests

.github/workflows/main.yml

@@ -18,26 +18,47 @@ jobs:
     name: base
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          fetch-depth: 1
+
       - name: build
-        run: make base
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 120
+          retry_wait_seconds: 40
+          max_attempts: 3
+          retry_on: error
+          command: make base
+
       - name: test
         run: make base.test
+
       - name: archive base
         run: |
           mkdir -p cache
           docker save dockcross/base:latest | xz -e9 -T0 > ./cache/base.tar.xz
+
       - name: save base
         uses: actions/upload-artifact@v3
         with:
           name: cache
           path: ./cache
           retention-days: 1
+
       - name: deploy
         if: github.ref == 'refs/heads/master'
-        run: |
-          docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
-          docker image push dockcross/base --all-tags
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 30
+          retry_wait_seconds: 30
+          max_attempts: 4
+          retry_on: error
+          command: |
+            docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+            docker image push dockcross/base --all-tags
 
   image:
     name: ${{ matrix.arch_name.image }}
@@ -833,39 +854,56 @@ jobs:
               cpython_arg: "",
             }
     steps:
-      - uses: actions/checkout@v3
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          fetch-depth: 1
+
       - uses: actions/download-artifact@v3
         with:
           name: cache
           path: ./cache
+
       - name: load base
         run: xz -d -k < ./cache/base.tar.xz | docker load
+
       - name: build
-        run: make ${{ matrix.arch_name.image }}
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 360
+          retry_wait_seconds: 20
+          max_attempts: 3
+          retry_on: error
+          command: make ${{ matrix.arch_name.image }}
+
       - name: basic test
         run: make ${{ matrix.arch_name.image }}.test
-      # Updated on 28/08/2021
+
+      # Updated on 19/05/2022
       - name: stockfish build test
         if: ${{ matrix.arch_name.stockfish == 'yes' }}
         run: |
-          git clone --depth 1 --branch sf_14 https://github.com/official-stockfish/Stockfish.git
+          git clone --depth 1 --branch sf_15 https://github.com/official-stockfish/Stockfish.git
           cd Stockfish/src
           ./../../tools/dockcross-make-builder.sh ${{ matrix.arch_name.image }} net
           ./../../tools/dockcross-make-builder.sh ${{ matrix.arch_name.image }} build ${{ matrix.arch_name.stockfish_arg }} -j2
           cd ../..
           rm -rf Stockfish
-      # Updated on 28/08/2021
+      # Updated on 19/05/2022
+
       - name: ninja build test
         if: ${{ matrix.arch_name.ninja == 'yes' }}
         run: |
-          git clone --depth 1 --branch v1.10.2 https://github.com/ninja-build/ninja.git
+          git clone --depth 1 --branch v1.11.0 https://github.com/ninja-build/ninja.git
           cd ninja
           ./../tools/dockcross-cmake-builder.sh ${{ matrix.arch_name.image }} ${{ matrix.arch_name.ninja_arg }}
-      # Updated on 28/08/2021
+
+      # Updated on 19/05/2022
       - name: openssl build test
         if: ${{ matrix.arch_name.openssl == 'yes' }}
         run: |
-          git clone --depth 1 --branch OpenSSL_1_1_1k https://github.com/openssl/openssl.git
+          git clone --depth 1 --branch OpenSSL_1_1_1o https://github.com/openssl/openssl.git
           cd openssl
           wget https://raw.githubusercontent.com/mavlink/MAVSDK/main/third_party/openssl/dockcross-android.patch
           patch -p 0 < dockcross-android.patch
@@ -873,47 +911,58 @@ jobs:
           ./../tools/dockcross-make-builder.sh ${{ matrix.arch_name.image }} -j2
           cd ..
           rm -rf openssl
-      # Updated on 28/08/2021
+
+      # Updated on 19/05/2022
       - name: C build test
         if: ${{ matrix.arch_name.C == 'yes' }}
         run: |
           git clone https://github.com/TheAlgorithms/C.git
           cd C
-          git checkout cc241f58c253c533ac94e07151ef91a5ef7e5719
+          git checkout 2314a195862243e09c485a66194866517a6f8c31
           ./../tools/dockcross-cmake-builder.sh ${{ matrix.arch_name.image }} ${{ matrix.arch_name.C_arg }}
           cd ..
           rm -rf C
-      # Updated on 28/08/2021
+
+      # Updated on 19/05/2022
       - name: C-Plus-Plus build test
         if: ${{ matrix.arch_name.C-Plus-Plus == 'yes' }}
         run: |
           git clone https://github.com/TheAlgorithms/C-Plus-Plus.git
           cd C-Plus-Plus
-          git checkout c3b07aed2240e5364e1a49d091a00b61f520e653
+          git checkout 37a29008e6b77921a97a89ca8b7fd9e700aeccd0
           ./../tools/dockcross-cmake-builder.sh ${{ matrix.arch_name.image }} ${{ matrix.arch_name.C-Plus-Plus_arg }}
           cd ..
           rm -rf C-Plus-Plus
-      # Updated on 28/08/2021
+
+      # Updated on 19/05/2022
       - name: fmt build test
         if: ${{ matrix.arch_name.fmt == 'yes' }}
         run: |
-          git clone --depth 1 --branch 8.0.1 https://github.com/fmtlib/fmt.git
+          git clone --depth 1 --branch 8.1.1 https://github.com/fmtlib/fmt.git
           cd fmt
           ./../tools/dockcross-cmake-builder.sh ${{ matrix.arch_name.image }} ${{ matrix.arch_name.fmt_arg }} -DFMT_DOC=OFF
           cd ..
           rm -rf fmt
-      # Updated on 05/10/2021
+
+      # Updated on 19/05/2022
       - name: cpython build test
         if: ${{ matrix.arch_name.cpython == 'yes' }}
         run: |
-          git clone --depth 1 --branch v3.9.2 https://github.com/python/cpython.git
+          git clone --depth 1 --branch v3.9.13 https://github.com/python/cpython.git
           cd cpython
           ./../tools/dockcross-command.sh ${{ matrix.arch_name.image }} ./configure ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no --disable-ipv6 ${{ matrix.arch_name.cpython_arg }} --build=x86_64-linux-gnu --enable-shared
           ./../tools/dockcross-make-builder.sh ${{ matrix.arch_name.image }} -j2
           cd ..
           rm -rf cpython
+
       - name: deploy
         if: github.ref == 'refs/heads/master'
-        run: |
-          docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
-          docker image push dockcross/${{ matrix.arch_name.image }} --all-tags
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 30
+          retry_wait_seconds: 30
+          max_attempts: 4
+          retry_on: error
+          command: |
+            docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+            docker image push dockcross/${{ matrix.arch_name.image }} --all-tags

CONTRIBUTING.md

@@ -0,0 +1,287 @@
+# Contributing
+
+## Getting started
+
+## How to add a new image ? (With crosstool-ng)
+
+In this part, we will see how to add a new image, we will take example with `linux-arm64` for a raspberry pi 4, with [crosstool-ng](https://github.com/crosstool-ng/crosstool-ng).
+
+### Build and config crosstool-ng
+
+To start, you need to download the source code of crosstool-ng:
+
+```bash
+git clone --recurse-submodules --remote-submodules https://github.com/crosstool-ng/crosstool-ng.git
+```
+
+Go to crosstool-ng folder:
+
+```bash
+cd crosstool-ng
+```
+
+Change git branch:
+
+```bash
+git checkout crosstool-ng-1.25.0
+```
+
+Once in the **crosstool-ng** folder, you must first run the **bootstrap** script:
+
+```bash
+./bootstrap
+```
+
+Then run the **configure** script:
+
+*Note: ***--enable-local*** Do a portable install of crosstool-ng.*:
+
+```bash
+./configure --enable-local
+```
+
+Finally, launch the building of crosstool-ng:
+
+```bash
+make -j$(nproc)
+```
+
+Once the crosstool-ng build is complete, you can run this command to test crosstool-ng:
+
+```bash
+./ct-ng --version
+```
+
+Before starting the configuration of the toolchains, i recommend you to use one of the examples from crosstool-ng and then make your changes, the command to display the examples:
+
+```bash
+./ct-ng list-samples
+```
+
+We will take the example of **aarch64-rpi4-linux-gnu**, a **.config** file will be created:
+
+```bash
+./ct-ng aarch64-rpi4-linux-gnu
+```
+
+We will configure the toolchains according to our needs:
+
+```bash
+./ct-ng menuconfig
+```
+
+Once the modifications are made, we will display the name of the toolchains, it will be useful later:
+
+```bash
+./ct-ng show-tuple
+```
+
+### Configuring docker image
+
+You must create a file with the **same** name of the docker image (**linux-arm64**).
+
+Copy the **.config** of crosstool-ng to this file (**linux-arm64**) and rename it to **crosstool-ng.config**.
+
+You need to create a file named **Toolchain.cmake** in **linux-arm64**.
+
+Copy text to **Toolchain.cmake** file:
+
+```cmake
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR ARM64)
+
+set(cross_triple $ENV{CROSS_TRIPLE})
+set(cross_root $ENV{CROSS_ROOT})
+
+set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_CXX_COMPILER $ENV{CXX})
+set(CMAKE_Fortran_COMPILER $ENV{FC})
+
+set(CMAKE_CXX_FLAGS "-I ${cross_root}/include/")
+
+set(CMAKE_FIND_ROOT_PATH ${cross_root} ${cross_root}/${cross_triple})
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)
+set(CMAKE_SYSROOT ${cross_root}/${cross_triple}/sysroot)
+
+set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-arm64)
+```
+
+Then ou must change these lines according to the targeted architecture, here **ARM64**:
+
+```cmake
+set(CMAKE_SYSTEM_PROCESSOR ARM64)
+set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-arm64)
+```
+
+Then you must create a file named **Dockerfile.in** in the image folder (**linux-arm64**).
+
+Copy text to **Dockerfile.in** file:
+
+```docker
+FROM dockcross/base:latest
+
+LABEL maintainer="Matt McCormick matt.mccormick@kitware.com"
+
+# This is for 64-bit ARM Linux machine
+
+# Crosstool-ng crosstool-ng-1.25.0 2022-05-13
+ENV CT_VERSION crosstool-ng-1.25.0
+
+#include "common.crosstool"
+
+# The cross-compiling emulator
+RUN apt-get update \
+&& apt-get install -y \
+  qemu-user \
+  qemu-user-static \
+&& apt-get clean --yes
+
+# The CROSS_TRIPLE is a configured alias of the "aarch64-unknown-linux-gnu" target.
+ENV CROSS_TRIPLE aarch64-unknown-linux-gnu
+
+ENV CROSS_ROOT ${XCC_PREFIX}/${CROSS_TRIPLE}
+ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
+    AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
+    CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
+    CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
+    CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
+    LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld \
+    FC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gfortran
+
+ENV QEMU_LD_PREFIX "${CROSS_ROOT}/${CROSS_TRIPLE}/sysroot"
+ENV QEMU_SET_ENV "LD_LIBRARY_PATH=${CROSS_ROOT}/lib:${QEMU_LD_PREFIX}"
+
+COPY Toolchain.cmake ${CROSS_ROOT}/
+ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake
+
+ENV PKG_CONFIG_PATH /usr/lib/aarch64-linux-gnu/pkgconfig
+
+# Linux kernel cross compilation variables
+ENV PATH ${PATH}:${CROSS_ROOT}/bin
+ENV CROSS_COMPILE ${CROSS_TRIPLE}-
+ENV ARCH arm64
+
+#include "common.label-and-env"
+```
+
+Then ou must change these lines according to the targeted architecture.
+
+Here you have to change the value according to the name of the toolchain (./ct-ng show-tuple):
+
+```docker
+ENV CROSS_TRIPLE aarch64-unknown-linux-gnu
+```
+
+These lines also need to be changed:
+
+```docker
+LABEL maintainer="Matt McCormick matt.mccormick@kitware.com"
+ENV PKG_CONFIG_PATH /usr/lib/aarch64-linux-gnu/pkgconfig
+ENV ARCH arm64
+```
+
+Once this part is finished, there must be 3 files in the **linux-arm64** folder:
+
+- **crosstool-ng.config**, the configuration of the toolchain/crosstool-ng.
+- **Dockerfile.in**, the docker file.
+- **Toolchain.cmake**, the CMake file for the toolchains.
+
+### Makefile
+
+For this last part, we will see how to add the image to the [Makefile](Makefile) and to a github action.
+
+You need to add the image/folder name (**linux-arm64**) to the **STANDARD_IMAGES** variable in the [Makefile](Makefile):
+
+```make
+# These images are built using the "build implicit rule"
+STANDARD_IMAGES = android-arm android-arm64 android-x86 android-x86_64 \
+ linux-x86 linux-x64 linux-x64-clang linux-arm64 linux-arm64-musl linux-arm64-full \
+ linux-armv5 linux-armv5-musl linux-armv5-uclibc linux-m68k-uclibc linux-s390x linux-x64-tinycc \
+ linux-armv6 linux-armv6-lts linux-armv6-musl linux-arm64-lts \
+ linux-armv7l-musl linux-armv7 linux-armv7a linux-armv7-lts linux-x86_64-full \
+ linux-mips linux-ppc64le linux-riscv64 linux-riscv32 linux-xtensa-uclibc \
+ web-wasi \
+ windows-static-x86 windows-static-x64 windows-static-x64-posix windows-armv7 \
+ windows-shared-x86 windows-shared-x64 windows-shared-x64-posix windows-arm64
+```
+
+You need to add the image/folder name (**linux-arm64**) to the **GEN_IMAGES** variable in the [Makefile](Makefile):
+
+```make
+# Generated Dockerfiles.
+GEN_IMAGES = android-arm android-arm64 \
+ linux-x86 linux-x64 linux-x64-clang linux-arm64 linux-arm64-musl linux-arm64-full \
+ manylinux2014-x64 manylinux2014-x86 \
+ manylinux2014-aarch64 linux-arm64-lts \
+ web-wasm web-wasi linux-mips windows-arm64 windows-armv7 \
+ windows-static-x86 windows-static-x64 windows-static-x64-posix \
+ windows-shared-x86 windows-shared-x64 windows-shared-x64-posix \
+ linux-armv7 linux-armv7a linux-armv7l-musl linux-armv7-lts linux-x86_64-full \
+ linux-armv6 linux-armv6-lts linux-armv6-musl \
+ linux-armv5 linux-armv5-musl linux-armv5-uclibc linux-ppc64le linux-s390x \
+ linux-riscv64 linux-riscv32 linux-m68k-uclibc linux-x64-tinycc linux-xtensa-uclibc
+```
+
+### Image building and testing
+
+You can now start building the image:
+
+```bash
+make linux-arm64
+```
+
+When finished, you can test it:
+
+```bash
+make linux-arm64.test
+```
+
+If you want to go a little further in the tests:
+
+```bash
+docker run --rm linux-arm64 > ./linux-arm64
+chmod +x ./linux-arm64
+```
+
+And then run the commands to build a project (you must be in the directory of your project to build):
+
+```bash
+./linux-arm64 make
+```
+
+With CMake + Ninja:
+
+```bash
+./linux-arm64 cmake -Bbuild -S. -GNinja
+./linux-arm64 ninja -Cbuild
+```
+
+### CI (github action)
+
+To finish, you have to add to [Github Action](.github/workflows/main.yml) the image/folder name:
+
+```yml
+          # Linux arm64/armv8 images
+          - {
+              image: "linux-arm64",
+              stockfish: "yes",
+              stockfish_arg: "ARCH=armv8",
+              ninja: "yes",
+              ninja_arg: "",
+              openssl: "yes",
+              openssl_arg: "linux-aarch64",
+              C: "yes",
+              C_arg: "",
+              C-Plus-Plus: "yes",
+              C-Plus-Plus_arg: "",
+              fmt: "yes",
+              fmt_arg: "",
+              cpython: "yes",
+              cpython_arg: "--host=aarch64-unknown-linux-gnu --target=aarch64-unknown-linux-gnu",
+            }
+```
+
+You can disable and enable the build of certain tests which can cause problems with certain CPU architectures (eg. OpenSSL with Risc-V...).

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015, 2016, 2017, 2018 Steeve Morin, Rob Burns, Matthew McCormick, Jean-Christophe-Fillion-Robin
+Copyright (c) 2015, 2016, 2017, 2018, 2021 Steeve Morin, Rob Burns, Matthew McCormick, Jean-Christophe-Fillion-Robin, Bensuperpc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -9,32 +9,37 @@ Cross compiling toolchains in Docker images.
 
 ## Features
 
--   Pre-built and configured toolchains for cross compiling.
--   Most images also contain an emulator for the target system.
--   Clean separation of build tools, source code, and build artifacts.
--   Commands in the container are run as the calling user, so that any created files have the expected ownership, (i.e. not root).
--   Make variables **CC**, **CXX**, **LD**, **AS** etc) are set to point to the appropriate tools in the container.
--   Recent [CMake](https://cmake.org) and ninja are precompiled.
--   [Conan.io](https://www.conan.io) can be used as a package manager.
--   Toolchain files configured for CMake.
--   Current directory is mounted as the container\'s workdir, `/work`.
--   Works with the [Docker for Mac](https://docs.docker.com/docker-for-mac/) and [Docker for Windows](https://docs.docker.com/docker-for-windows/).
--   Support using alternative container executor by setting **OCI_EXE** environment variable. By default, it searches for [docker](https://www.docker.com) and [podman](https://podman.io) executable.
--   [crosstool-ng](https://github.com/crosstool-ng/crosstool-ng) and [buildroot](https://github.com/buildroot/buildroot) configuration files.
+- Pre-built and configured toolchains for cross compiling.
+- Most images also contain an emulator for the target system.
+- Clean separation of build tools, source code, and build artifacts.
+- Commands in the container are run as the calling user, so that any created files have the expected ownership, (i.e. not root).
+- Make variables **CC**, **CXX**, **LD**, **AS** etc) are set to point to the appropriate tools in the container.
+- Recent [CMake](https://cmake.org) and ninja are precompiled.
+- [Conan.io](https://www.conan.io) can be used as a package manager.
+- Toolchain files configured for CMake.
+- Current directory is mounted as the container\'s workdir, `/work`.
+- Works with the [Docker for Mac](https://docs.docker.com/docker-for-mac/) and [Docker for Windows](https://docs.docker.com/docker-for-windows/).
+- Support using alternative container executor by setting **OCI_EXE** environment variable. By default, it searches for [docker](https://www.docker.com) and [podman](https://podman.io) executable.
+- [crosstool-ng](https://github.com/crosstool-ng/crosstool-ng) and [buildroot](https://github.com/buildroot/buildroot) configuration files.
 
 ## Examples
 
-1.  `dockcross make`: Build the *Makefile* in the current directory.
-2.  `dockcross cmake -Bbuild -H. -GNinja`: Run CMake with a build directory `./build` for a *CMakeLists.txt* file in the current directory and generate `ninja` build configuration files.
-3.  `dockcross ninja -Cbuild`: Run ninja in the `./build` directory.
-4.  `dockcross bash -c '$CC test/C/hello.c -o hello'`: Build the *hello.c* file with the compiler identified with the `CC` environmental variable in the build environment.
-5.  `dockcross bash`: Run an interactive shell in the build environment.
+1. `dockcross make`: Build the *Makefile* in the current directory.
+2. `dockcross cmake -Bbuild -S. -GNinja`: Run CMake with a build directory `./build` for a *CMakeLists.txt* file in the current directory and generate `ninja` build configuration files.
+3. `dockcross ninja -Cbuild`: Run ninja in the `./build` directory.
+4. `dockcross bash -c '$CC test/C/hello.c -o hello'`: Build the *hello.c* file with the compiler identified with the `CC` environmental variable in the build environment.
+5. `dockcross bash`: Run an interactive shell in the build environment.
 
 Note that commands are executed verbatim. If any shell processing for environment variable expansion or redirection is required, please use
+
 ```bash
 bash -c "<command args>"
 ```
 
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).
+
 ## Installation
 
 This image does not need to be run manually. Instead, there is a helper script to execute build commands on source code existing on the local host filesystem. This script is bundled with the image.
@@ -546,5 +551,6 @@ The key difference is that [dockbuild](https://github.com/dockbuild/dockbuild#re
 \-\--
 
 Credits:
+
 - [sdt/docker-raspberry-pi-cross-compiler](https://github.com/sdt/docker-raspberry-pi-cross-compiler), who invented the base of the **dockcross** script.
 - [https://github.com/steeve/cross-compiler](https://github.com/steeve/cross-compiler), 

imagefiles/dockcross.sh

@@ -24,10 +24,10 @@ has() {
 
 # If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative")
 if [ -z "$OCI_EXE" ]; then
-    if which docker >/dev/null 2>/dev/null; then
-        OCI_EXE=docker
-    elif which podman >/dev/null 2>/dev/null; then
+    if which podman >/dev/null 2>/dev/null; then
         OCI_EXE=podman
+    elif which docker >/dev/null 2>/dev/null; then
+        OCI_EXE=docker
     else
         die "Cannot find a container executor. Search for docker and podman."
     fi
@@ -190,7 +190,7 @@ MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
 # CYGWIN
 CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "")
 
-if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" ]; then
+if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then
     USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
 fi