Merge remote-tracking branch 'origin/master' into improve_scripts

Fix manylinux2014x86 and manylinux2014x64, update to devtoolset-10

Makefile

@@ -226,7 +226,7 @@ purge: clean
 # Check bash syntax
 bash-check:
 	find . -type f \( -name "*.sh" -o -name "*.bash" \) -print0 | xargs -0 -P"$(shell nproc)" -I{} \
-		$(SHELLCHECK) --check-sourced --color=auto --format=gcc --severity=error --shell=bash --enable=all "{}"
+		$(SHELLCHECK) --check-sourced --color=auto --format=gcc --severity=warning --shell=bash --enable=all "{}"
 
 #
 # testing implicit rule

common/common-manylinux.crosstool

@@ -18,7 +18,7 @@ ENV MANYLINUX_DEPS="glibc-devel glibc-devel.i686 libstdc++-devel glib2-devel lib
 
 ENV PYTHON_COMPILE_DEPS="python-devel zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel"
 
-ENV TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran"
+ENV TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran"
 
 # Install dependent packages required for `ct-ng`.
 RUN \
@@ -62,7 +62,7 @@ RUN \
   rm -rf /dockcross/crosstool /dockcross/install-crosstool-ng-toolchain.sh && \
   # Remove sudo provided by devtoolset since it doesn't work with
   # our sudo wrapper calling gosu.
-  rm -f /opt/rh/devtoolset-9/root/usr/bin/sudo
+  rm -f /opt/rh/devtoolset-10/root/usr/bin/sudo
 
 # Restore our default workdir (from "dockcross/base").
 WORKDIR /work

common/common.manylinux

@@ -23,6 +23,7 @@ RUN \
   rm -f /opt/rh/devtoolset-7/root/usr/bin/sudo && \
   rm -f /opt/rh/devtoolset-8/root/usr/bin/sudo && \
   rm -f /opt/rh/devtoolset-9/root/usr/bin/sudo && \
+  rm -f /opt/rh/devtoolset-10/root/usr/bin/sudo && \
   /buildscripts/install-python-packages.sh && \
   rm -rf /buildscripts
 

imagefiles/build-and-install-cmake.sh

@@ -4,6 +4,9 @@ set -ex
 
 WRAPPER=""
 
+CMAKE_URL="https://gitlab.kitware.com/cmake/cmake.git"
+CMAKE_MIRROR_URL="https://github.com/Kitware/CMake.git"
+
 while [ $# -gt 0 ]; do
   case "$1" in
     -32)
@@ -22,22 +25,24 @@ if ! command -v git &> /dev/null; then
 	exit 1
 fi
 
-if [[ "${CMAKE_VERSION}" == "" ]]; then
+if [[ -z "${CMAKE_VERSION}" ]]; then
   echo >&2 'error: CMAKE_VERSION env. variable must be set to a non-empty value'
   exit 1
 fi
 
 cd /usr/src
 
-git clone https://gitlab.kitware.com/cmake/cmake.git CMake -b v$CMAKE_VERSION --depth 1
+# If the first link doesn't work, it will use the mirror on github
+git clone "$CMAKE_URL" CMake -b v$CMAKE_VERSION --depth 1 \
+  || git clone "$CMAKE_MIRROR_URL" CMake -b v$CMAKE_VERSION --depth 1
 
 mkdir /usr/src/CMake-build
 cd /usr/src/CMake-build
 
 ${WRAPPER} /usr/src/CMake/bootstrap \
-  --parallel=$(nproc) \
+  --parallel="$(nproc)" \
   -- -DCMAKE_USE_OPENSSL=OFF
-${WRAPPER} make -j$(nproc)
+${WRAPPER} make -j"$(nproc)"
 
 
 mkdir /usr/src/CMake-ssl-build
@@ -50,10 +55,10 @@ ${WRAPPER} /usr/src/CMake-build/bin/cmake \
   -DCMAKE_USE_OPENSSL:BOOL=ON \
   -DOPENSSL_ROOT_DIR:PATH=/usr/local/ssl \
   ../CMake
-${WRAPPER} make -j$(nproc) install
+${WRAPPER} make -j"$(nproc)" install
 
 # Cleanup install tree
-cd /usr/src/cmake-$CMAKE_VERSION
+cd "/usr/src/cmake-$CMAKE_VERSION"
 rm -rf doc man
 
 # Install files

imagefiles/build-and-install-curl.sh

@@ -18,7 +18,7 @@ function do_curl_build {
     # We do this shared to avoid obnoxious linker issues where git couldn't
     # link properly. If anyone wants to make this build statically go for it.
     LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static > /dev/null
-    make -j$(nproc) > /dev/null
+    make -j"$(nproc)" > /dev/null
     make install > /dev/null
 }
 

imagefiles/build-and-install-git.sh

@@ -12,7 +12,7 @@ if ! command -v tar &> /dev/null; then
 	exit 1
 fi
 
-if [[ "${GIT_VERSION}" == "" ]]; then
+if [[ -z "${GIT_VERSION}" ]]; then
   echo >&2 'error: GIT_VERSION env. variable must be set to a non-empty value'
   exit 1
 fi
@@ -37,7 +37,7 @@ rm -f git-${GIT_VERSION}.tar.gz
 
 pushd git-${GIT_VERSION}
 ./configure --prefix=/usr/local --with-curl
-make -j$(nproc)
+make -j"$(nproc)"
 make install
 popd
 

imagefiles/build-and-install-openssl.sh

@@ -53,13 +53,13 @@ OPENSSL_HASH=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5
 OPENSSL_DOWNLOAD_URL=http://www.openssl.org/source/
 
 # a recent enough perl is needed to build openssl
-PERL_ROOT=perl-5.32.1
+PERL_ROOT=perl-5.32.2
 PERL_HASH=03b693901cd8ae807231b1787798cf1f2e0b8a56218d07b7da44f784a7caeb2c
 PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0
 
 function do_perl_build {
     ${WRAPPER} sh Configure -des -Dprefix=/opt/perl > /dev/null
-    ${WRAPPER} make -j$(nproc) > /dev/null
+    ${WRAPPER} make -j"$(nproc)" > /dev/null
     ${WRAPPER} make install > /dev/null
 }
 
@@ -84,7 +84,7 @@ function build_perl {
 
 function do_openssl_build {
     ${WRAPPER} ./config no-shared -fPIC $CONFIG_FLAG --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null
-    ${WRAPPER} make -j$(nproc) > /dev/null
+    ${WRAPPER} make -j"$(nproc)" > /dev/null
     ${WRAPPER} make install_sw > /dev/null
 }
 

imagefiles/build-and-install-python.sh

@@ -26,12 +26,12 @@ done
 
 # Download
 wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
-tar xvf Python-${PYTHON_VERSION}.tgz
+tar xvf "Python-${PYTHON_VERSION}.tgz"
 # Configure, build and install
-cd Python-${PYTHON_VERSION}
+cd "Python-${PYTHON_VERSION}"
 # Disable --enable-shared --enable-optimizations --prefix=/usr/local/python-${PYTHON_VERSION}
 ./configure --with-ensurepip=install
-make -j$(nproc) 
+make -j"$(nproc)"
 make install #altinstall
 
 ln -s /usr/local/bin/python3 /usr/local/bin/python
@@ -39,4 +39,4 @@ ln -s /usr/local/bin/pip3 /usr/local/bin/pip
 
 # Clean
 cd ..
-rm -rf Python-${PYTHON_VERSION}
+rm -rf "Python-${PYTHON_VERSION}"

imagefiles/entrypoint.sh

@@ -37,8 +37,8 @@ if [[ -n $BUILDER_UID ]] && [[ -n $BUILDER_GID ]]; then
     fi
 
     # Enable passwordless sudo capabilities for the user
-    chown root:$BUILDER_GID $(which gosu)
+    chown root:$BUILDER_GID "$(which gosu)"
-    chmod +s $(which gosu); sync
+    chmod +s "$(which gosu)"; sync
 
     # Execute project specific pre execution hook
     if [[ -e /work/.dockcross ]]; then

imagefiles/install-cmake-binary.sh

@@ -28,7 +28,7 @@ if ! command -v tar &> /dev/null; then
 	exit 1
 fi
 
-if [[ "${CMAKE_VERSION}" == "" ]]; then
+if [[ -z "${CMAKE_VERSION}" ]]; then
   echo >&2 'error: CMAKE_VERSION env. variable must be set to a non-empty value'
   exit 1
 fi

imagefiles/install-crosstool-ng-toolchain.sh

@@ -89,7 +89,7 @@ BOOTSTRAP_PREFIX="${CTNG}/prefix"
 ./bootstrap
 ./configure \
   --prefix "${BOOTSTRAP_PREFIX}"
-make -j$(nproc)
+make -j"$(nproc)"
 make install
 
 ##

imagefiles/install-gosu-binary.sh

@@ -14,7 +14,7 @@ if ! command -v gpg &> /dev/null; then
 fi
 
 GOSU_VERSION=1.12
-dpkgArch=$(if test $(uname -m) = "x86_64"; then echo amd64; else echo i386; fi)
+dpkgArch=$(if test "$(uname -m)" = "x86_64"; then echo amd64; else echo i386; fi)
 url="https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${dpkgArch}"
 url_key="https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${dpkgArch}.asc"
 

imagefiles/install-python-packages.sh

@@ -22,8 +22,13 @@ cd /tmp
 
 # Todo: Need to update base image from Debian Stretch for the required Python
 # 3.6 or later
-curl -# -LO https://bootstrap.pypa.io/pip/get-pip.py
+curl --connect-timeout 30 \
-#curl -# -LO https://bootstrap.pypa.io/pip/2.7/get-pip.py
+        --max-time 10 \
+        --retry 5 \
+        --retry-delay 10 \
+        --retry-max-time 30 \
+        -# -LO https://bootstrap.pypa.io/pip/get-pip.py
+
 ${PYTHON} get-pip.py --ignore-installed
 rm get-pip.py
 

manylinux2014-x64/Dockerfile.in

@@ -16,7 +16,7 @@ RUN echo $'#!/bin/bash\n\
 LD_PRELOAD=/usr/lib64/libcurl.so.4 /usr/bin/yum "$@"' > /usr/local/bin/yum && chmod a+x /usr/local/bin/yum
 
 ENV CROSS_TRIPLE x86_64-linux-gnu
-ENV CROSS_ROOT /opt/rh/devtoolset-9/root/usr/bin
+ENV CROSS_ROOT /opt/rh/devtoolset-10/root/usr/bin
 ENV AS=${CROSS_ROOT}/as \
     AR=${CROSS_ROOT}/ar \
     CC=${CROSS_ROOT}/gcc \

manylinux2014-x64/Toolchain.cmake

@@ -5,7 +5,7 @@ set(CMAKE_SYSTEM_PROCESSOR x86_64)
 
 set(MANYLINUX2014 TRUE)
 
-set(CMAKE_C_COMPILER /opt/rh/devtoolset-9/root/usr/bin/gcc)
+set(CMAKE_C_COMPILER /opt/rh/devtoolset-10/root/usr/bin/gcc)
-set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-9/root/usr/bin/g++)
+set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-10/root/usr/bin/g++)
 set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})
-set(CMAKE_Fortran_COMPILER /opt/rh/devtoolset-9/root/usr/bin/gfortran)
+set(CMAKE_Fortran_COMPILER /opt/rh/devtoolset-10/root/usr/bin/gfortran)

manylinux2014-x86/Dockerfile.in

@@ -16,7 +16,7 @@ RUN echo $'#!/bin/bash\n\
 LD_PRELOAD=/usr/lib/libcurl.so.4 /usr/bin/yum "$@"' > /usr/local/bin/yum && chmod a+x /usr/local/bin/yum
 
 ENV CROSS_TRIPLE i686-linux-gnu
-ENV CROSS_ROOT /opt/rh/devtoolset-9/root/usr/bin
+ENV CROSS_ROOT /opt/rh/devtoolset-10/root/usr/bin
 ENV AS=${CROSS_ROOT}/as \
     AR=${CROSS_ROOT}/ar \
     CC=${CROSS_ROOT}/gcc \

manylinux2014-x86/Toolchain.cmake

@@ -5,7 +5,7 @@ set(CMAKE_SYSTEM_PROCESSOR i686)
 
 set(MANYLINUX2014 TRUE)
 
-set(CMAKE_C_COMPILER /opt/rh/devtoolset-9/root/usr/bin/gcc)
+set(CMAKE_C_COMPILER /opt/rh/devtoolset-10/root/usr/bin/gcc)
-set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-9/root/usr/bin/g++)
+set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-10/root/usr/bin/g++)
 set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})
-set(CMAKE_Fortran_COMPILER /opt/rh/devtoolset-9/root/usr/bin/gfortran)
+set(CMAKE_Fortran_COMPILER /opt/rh/devtoolset-10/root/usr/bin/gfortran)

tools/container-diff.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# More info: https://github.com/GoogleContainerTools/container-diff
-
-if (( $# < 1 || $# > 2  )); then
-    echo "Need 1 or 2 arguments: <img1> <img2>"
-    exit 1
-fi
-
-if (( $# == 1 )); then
-    container-diff analyze $1 --type=history 2>&1 | tee -a analyze-history.txt
-    container-diff analyze $1 --type=file 2>&1 | tee -a analyze-file.txt
-    container-diff analyze $1 --type=size 2>&1 | tee -a analyze-size.txt
-    container-diff analyze $1 --type=apt 2>&1 | tee -a analyze-apt.txt
-    container-diff analyze $1 --type=pip 2>&1 | tee -a analyze-pip.txt
-
-fi
-
-if (( $# == 2 )); then
-    container-diff diff $1 $2 --type=history 2>&1 | tee -a diff-history.txt
-    container-diff diff $1 $2 --type=file 2>&1 | tee -a diff-file.txt
-    container-diff diff $1 $2 --type=size 2>&1 | tee -a diff-size.txt
-    container-diff diff $1 $2 --type=apt 2>&1 | tee -a diff-apt.txt
-    container-diff diff $1 $2 --type=pip 2>&1 | tee -a diff-pip.txt
-fi

tools/dockcross-builder.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+if (( $# >= 1 )); then
+    image=$1
+    build_file=build-${image%:*}
+    shift 1
+
+    cmake_arg=$*
+    echo "cmake arg: $cmake_arg"
+
+    echo "Pulling dockcross/$image"
+    docker pull dockcross/"$image"
+
+    echo "Make script dockcross-$image"
+    docker run --rm dockcross/"$image" > ./dockcross-"$image"
+    chmod +x ./dockcross-"$image"
+
+    echo "Build $build_file"
+    ./dockcross-"$image" cmake -B "$build_file" -S . -G Ninja "$cmake_arg"
+    ./dockcross-"$image" ninja -C "$build_file"
+else
+    echo "Usage: ${0##*/} <docker imag (ex: linux-x64/linux-x64-clang/linux-arm64/windows-shared-x64/windows-static-x64...)> <cmake arg.>"
+    exit 1
+fi

tools/docker-container-diff.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+if (( $# < 1 || $# > 2  )); then
+    echo "Need 1 or 2 arguments: <img1> <img2>"
+    echo "For local image: daemon://<img>"
+    echo "For remote image: remote://<img>"
+    echo "More info: https://github.com/GoogleContainerTools/container-diff"
+    exit 1
+fi
+
+if (( $# == 1 )); then
+    container-diff analyze "$1" --type=history 2>&1 | tee -a analyze-history.txt
+    container-diff analyze "$1" --type=file 2>&1 | tee -a analyze-file.txt
+    container-diff analyze "$1" --type=size 2>&1 | tee -a analyze-size.txt
+    container-diff analyze "$1" --type=apt 2>&1 | tee -a analyze-apt.txt
+    container-diff analyze "$1" --type=pip 2>&1 | tee -a analyze-pip.txt
+
+fi
+
+if (( $# == 2 )); then
+    container-diff diff "$1" "$2" --type=history 2>&1 | tee -a diff-history.txt
+    container-diff diff "$1" "$2" --type=file 2>&1 | tee -a diff-file.txt
+    container-diff diff "$1" "$2" --type=size 2>&1 | tee -a diff-size.txt
+    container-diff diff "$1" "$2" --type=apt 2>&1 | tee -a diff-apt.txt
+    container-diff diff "$1" "$2" --type=pip 2>&1 | tee -a diff-pip.txt
+fi