ENH: Compile git with SSL support

Older versions of git included in older linux distributions are not able
to download source from Github. A newer version is required with a newer
OpenSSL. This requires to also build curl with the same OpenSSL.

CMake is downloaded precompiled if available (64bits system) or compiled
from source otherwise.

browser-asmjs/Dockerfile.in

@@ -1,4 +1,4 @@
-FROM trzeci/emscripten-slim:sdk-tag-1.37.29-64bit
+FROM trzeci/emscripten-slim:sdk-tag-1.37.36-64bit
 MAINTAINER Matt McCormick "matt.mccormick@kitware.com"
 
 # Revert back to "/bin/sh" as default shell

common.debian

@@ -13,7 +13,7 @@ RUN apt-get update --yes && apt-get install --no-install-recommends --yes \
   ca-certificates \
   curl \
   file \
-  git \
+  gettext \
   gzip \
   zip \
   make \
@@ -31,7 +31,8 @@ RUN apt-get update --yes && apt-get install --no-install-recommends --yes \
   pax \
   vim \
   wget \
-  xz-utils && \
+  xz-utils \
+  zlib1g-dev && \
   apt-get clean --yes
 
 ENV GOSU_VERSION 1.10

common.docker

@@ -1,21 +1,38 @@
-WORKDIR /usr/share
-RUN git clone "https://github.com/nojhan/liquidprompt.git" && \
-  cd liquidprompt && \
-  git checkout v_1.11
-COPY imagefiles/.bashrc /root/
-
 WORKDIR /usr/src
 
-COPY imagefiles/install-openssl.sh imagefiles/install-cmake.sh /dockcross/
+ARG GIT_VERSION=2.16.2
+ARG CMAKE_VERSION=3.11.0
+
+COPY imagefiles/build-and-install-git.sh \
+  imagefiles/utils.sh \
+  imagefiles/build-and-install-openssl.sh \
+  imagefiles/build-and-install-openssh.sh \
+  imagefiles/build-and-install-cmake.sh \
+  imagefiles/install-cmake-binary.sh \
+  imagefiles/build-and-install-curl.sh \
+  /dockcross/
+
 RUN \
   if [ "$DEFAULT_DOCKCROSS_IMAGE" = "dockcross/manylinux-x86" ]; then \
-    /dockcross/install-openssl.sh -32 && \
+    /dockcross/build-and-install-openssl.sh -32 && \
-    /dockcross/install-cmake.sh -32 || exit 1; \
+    /dockcross/build-and-install-openssh.sh && \
+    /dockcross/build-and-install-curl.sh && \
+    /dockcross/build-and-install-git.sh && \
+    /dockcross/build-and-install-cmake.sh -32 || exit 1; \
   else \
-    /dockcross/install-openssl.sh && \
+    /dockcross/build-and-install-openssl.sh && \
-    /dockcross/install-cmake.sh || exit 1; \
+    /dockcross/build-and-install-openssh.sh && \
+    /dockcross/build-and-install-curl.sh && \
+    /dockcross/build-and-install-git.sh && \
+    /dockcross/install-cmake-binary.sh || exit 1; \
   fi; \
-  rm /dockcross/install-openssl.sh /dockcross/install-cmake.sh
+  rm /dockcross/build-and-install-git.sh \
+  /dockcross/utils.sh \
+  /dockcross/build-and-install-openssl.sh \
+  /dockcross/build-and-install-openssh.sh \
+  /dockcross/build-and-install-cmake.sh \
+  /dockcross/install-cmake-binary.sh \
+  /dockcross/build-and-install-curl.sh
 
 COPY imagefiles/cmake.sh /usr/local/bin/cmake
 COPY imagefiles/ccmake.sh /usr/local/bin/ccmake
@@ -38,6 +55,12 @@ RUN if [ -e /opt/python/cp35-cp35m/bin/python ]; then \
 
 RUN $([ -e /opt/python/cp35-cp35m/bin/python ] && echo "/opt/python/cp35-cp35m/bin/python" || echo "python") -m pip install --ignore-installed conan
 
+WORKDIR /usr/share
+RUN git clone "https://github.com/nojhan/liquidprompt.git" && \
+  cd liquidprompt && \
+  git checkout v_1.11
+COPY imagefiles/.bashrc /root/
+
 RUN echo "root:root" | chpasswd
 WORKDIR /work
 ENTRYPOINT ["/dockcross/entrypoint.sh"]

common.manylinux

@@ -2,6 +2,7 @@ ENV GOSU_VERSION 1.10
 RUN set -x \
 	&& yum -y install epel-release \
 	&& yum -y install gpg \
+	&& yum -y install zlib-devel gettext \
 	&& dpkgArch=$(if test $(uname -m) = "x86_64"; then echo amd64; else echo i386; fi) \
 	&& curl -o /usr/bin/gosu -LO "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
 	&& curl -o /tmp/gosu.asc -LO "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc" \

imagefiles/build-and-install-cmake.sh

@@ -1,17 +1,4 @@
 #!/bin/bash
-
-#
-# Configure, build and install CMake
-#
-# Usage:
-#
-#  install-cmake.sh [-32]
-#
-# Options:
-#
-#  -32                 Build CMake as a 32-bit executable
-#
-# Notes:
 #
 #  * build directory is /usr/src/CMake
 #
@@ -19,20 +6,14 @@
 #
 #  * after installation, archive, source and build directories are removed
 #
-
+set -ex
-set -e
-set -o pipefail
 
 WRAPPER=""
-CONFIG_FLAG=""
-SUFFIX=64
 
 while [ $# -gt 0 ]; do
   case "$1" in
     -32)
       WRAPPER="linux32"
-      CONFIG_FLAG="-m32"
-      SUFFIX=32
       ;;
     *)
       echo "Usage: Usage: ${0##*/} [-32]"
@@ -42,27 +23,43 @@ while [ $# -gt 0 ]; do
   shift
 done
 
+if ! command -v git &> /dev/null; then
+	echo >&2 'error: "git" not found!'
+	exit 1
+fi
+
+if [[ "${CMAKE_VERSION}" == "" ]]; then
+  echo >&2 'error: CMAKE_VERSION env. variable must be set to a non-empty value'
+  exit 1
+fi
+
 cd /usr/src
 
-# Download
+git clone git://cmake.org/cmake.git CMake
-CMAKE_REV=v3.10.1
+
-curl -# -o CMake.tar.gz -LO https://github.com/kitware/cmake/archive/$CMAKE_REV.tar.gz
+(cd CMake && git checkout v$CMAKE_VERSION)
-mkdir CMake
-tar -xzvf ./CMake.tar.gz --strip-components=1 -C ./CMake
 
 mkdir /usr/src/CMake-build
+cd /usr/src/CMake-build
 
-pushd /usr/src/CMake-build
-
-NUM_PROCESSOR=$(grep -c processor /proc/cpuinfo)
-
-# Configure boostrap
 ${WRAPPER} /usr/src/CMake/bootstrap \
-  --parallel=$NUM_PROCESSOR \
+  --parallel=$(grep -c processor /proc/cpuinfo)
-  --prefix=/usr
+${WRAPPER} make -j$(grep -c processor /proc/cpuinfo)
 
-# Build and Install
+mkdir /usr/src/CMake-ssl-build
-${WRAPPER} make install -j$NUM_PROCESSOR
+cd /usr/src/CMake-ssl-build
+${WRAPPER} /usr/src/CMake-build/bin/cmake \
+  -DCMAKE_BUILD_TYPE:STRING=Release \
+  -DBUILD_TESTING:BOOL=ON \
+  -DCMAKE_INSTALL_PREFIX:PATH=/usr/src/cmake-$CMAKE_VERSION \
+  -DCMAKE_USE_OPENSSL:BOOL=ON \
+  -DOPENSSL_ROOT_DIR:PATH=/usr/local/ssl \
+  ../CMake
+${WRAPPER} make -j$(grep -c processor /proc/cpuinfo) install
+
+cd /usr/src/cmake-$CMAKE_VERSION
+rm -rf doc man
+find . -type f -exec install -D "{}" "/usr/{}" \;
 
 # Test
 ctest -R CMake.FileDownload
@@ -88,8 +85,5 @@ EOF
 
 # Execute test script
 cmake -P cmake-test-https-download.cmake
-rm cmake-test-https-download.cmake
 
-popd
+rm -rf /usr/src/CMake*
-
-rm -rf CMake*

imagefiles/build-and-install-curl.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -ex
+
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+source $MY_DIR/utils.sh
+
+#
+# Function 'do_curl_build' and 'build_curl'
+# copied from https://github.com/pypa/manylinux/tree/master/docker/build_scripts
+#
+
+CURL_ROOT=curl_7.52.1
+CURL_HASH=a8984e8b20880b621f61a62d95ff3c0763a3152093a9f9ce4287cfd614add6ae
+
+# We had to switch to a debian mirror because we can't use TLS until we
+# bootstrap it with this curl + openssl
+CURL_DOWNLOAD_URL=http://deb.debian.org/debian/pool/main/c/curl
+
+function do_curl_build {
+    # We do this shared to avoid obnoxious linker issues where git couldn't
+    # link properly. If anyone wants to make this build statically go for it.
+    LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static > /dev/null
+    make > /dev/null
+    make install > /dev/null
+}
+
+
+function build_curl {
+    local curl_fname=$1
+    check_var ${curl_fname}
+    local curl_sha256=$2
+    check_var ${curl_sha256}
+    check_var ${CURL_DOWNLOAD_URL}
+    # Can't use curl here because we don't have it yet...we are building it.
+    wget -q ${CURL_DOWNLOAD_URL}/${curl_fname}.orig.tar.gz
+    check_sha256sum ${curl_fname}.orig.tar.gz ${curl_sha256}
+    tar -zxf ${curl_fname}.orig.tar.gz
+    (cd curl-* && do_curl_build)
+    rm -rf curl_*
+}
+
+cd /usr/src
+build_curl $CURL_ROOT $CURL_HASH
+
+(cat /etc/ld.so.conf.d/usr-local.conf 2> /dev/null | grep -q "^/usr/local/lib$") ||
+  echo '/usr/local/lib' >> /etc/ld.so.conf.d/usr-local.conf
+ldconfig
+

imagefiles/build-and-install-git.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -ex
+
+if ! command -v curl &> /dev/null; then
+	echo >&2 'error: "curl" not found!'
+	exit 1
+fi
+
+if ! command -v tar &> /dev/null; then
+	echo >&2 'error: "tar" not found!'
+	exit 1
+fi
+
+if [[ "${GIT_VERSION}" == "" ]]; then
+  echo >&2 'error: GIT_VERSION env. variable must be set to a non-empty value'
+  exit 1
+fi
+
+(cat /etc/ld.so.conf.d/usr-local.conf 2> /dev/null | grep -q "^/usr/local/lib$") ||
+  echo '/usr/local/lib' >> /etc/ld.so.conf.d/usr-local.conf
+ldconfig
+
+cd /usr/src
+
+url="https://www.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.gz"
+echo "Downloading $url"
+curl -# -LO $url
+
+tar xvzf git-${GIT_VERSION}.tar.gz
+rm -f git-${GIT_VERSION}.tar.gz
+
+pushd git-${GIT_VERSION}
+./configure --prefix=/usr/local --with-curl
+make
+make install
+popd
+
+ldconfig
+
+rm -rf git-${GIT_VERSION}
+
+# turn the detached message off
+git config --global advice.detachedHead false

imagefiles/build-and-install-openssh.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+
+OPENSSH_ROOT=V_7_6_P1
+
+cd /usr/src
+curl -LO https://github.com/openssh/openssh-portable/archive/${OPENSSH_ROOT}.tar.gz
+tar -xvf ${OPENSSH_ROOT}.tar.gz
+rm -f ${OPENSSH_ROOT}.tar.gz
+
+OPENSSH_SRC_DIR=openssh-portable-${OPENSSH_ROOT}
+cd ${OPENSSH_SRC_DIR}
+
+autoreconf
+
+./configure --prefix=/usr/local
+
+make -j1 install
+
+cd /usr/src
+rm -rf ${OPENSSH_SRC_DIR}

imagefiles/build-and-install-openssl.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Configure, build and install OpenSSL
+#
+# Usage:
+#
+#  build-and-install-openssl.sh [-32]
+#
+# Options:
+#
+#  -32              Build OpenSSL as a 32-bit library
+#
+# Notes:
+#
+#  * build directory is /usr/src/openssl-$OPENSSL_VERSION
+#
+#  * install directory is /usr
+#
+#  * after installation, build directory and archive are removed
+#
+
+set -ex
+set -o pipefail
+
+WRAPPER=""
+CONFIG_FLAG="-fPIC"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -32)
+      WRAPPER="linux32"
+      CONFIG_FLAG="-m32"
+      ;;
+    *)
+      echo "Usage: Usage: ${0##*/} [-32]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+source $MY_DIR/utils.sh
+
+#
+# Function 'do_openssl_build' and 'build_openssl'
+# copied from https://github.com/pypa/manylinux/tree/master/docker/build_scripts
+#
+
+OPENSSL_ROOT=openssl-1.0.2o
+# Hash from https://www.openssl.org/source/openssl-1.0.2o.tar.gz.sha256
+# Matches hash at https://github.com/Homebrew/homebrew-core/blob/1766321103d9780f6e38d3ac7681b8fa42cdca86/Formula/openssl.rb#L11
+OPENSSL_HASH=ec3f5c9714ba0fd45cb4e087301eb1336c317e0d20b575a125050470e8089e4d
+
+# XXX: the official https server at www.openssl.org cannot be reached
+# with the old versions of openssl and curl in Centos 5.11 hence the fallback
+# to the ftp mirror:
+OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source
+
+function do_openssl_build {
+    ${WRAPPER} ./config no-ssl2 no-shared -fPIC $CONFIG_FLAG --prefix=/usr/local/ssl > /dev/null
+    ${WRAPPER} make > /dev/null
+    ${WRAPPER} make install_sw > /dev/null
+}
+
+function build_openssl {
+    local openssl_fname=$1
+    check_var ${openssl_fname}
+    local openssl_sha256=$2
+    check_var ${openssl_sha256}
+    check_var ${OPENSSL_DOWNLOAD_URL}
+    # Can't use curl here because we don't have it yet
+    curl -# -LO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz
+    check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256}
+    tar -xzf ${openssl_fname}.tar.gz
+    (cd ${openssl_fname} && do_openssl_build)
+    rm -rf ${openssl_fname} ${openssl_fname}.tar.gz /usr/ssl/man
+}
+
+cd /usr/src
+build_openssl $OPENSSL_ROOT $OPENSSL_HASH

imagefiles/install-cmake-binary.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+if ! command -v curl &> /dev/null; then
+	echo >&2 'error: "curl" not found!'
+	exit 1
+fi
+
+if ! command -v tar &> /dev/null; then
+	echo >&2 'error: "tar" not found!'
+	exit 1
+fi
+
+if [[ "${CMAKE_VERSION}" == "" ]]; then
+  echo >&2 'error: CMAKE_VERSION env. variable must be set to a non-empty value'
+  exit 1
+fi
+
+cd /tmp
+
+filename=cmake-${CMAKE_VERSION}-Centos5-x86_64
+url=https://github.com/dockbuild/CMake/releases/download/v${CMAKE_VERSION}/${filename}.tar.gz
+echo "Downloading $url"
+curl -# -LO $url
+
+tar -xzvf ${filename}.tar.gz
+rm -f ${filename}.tar.gz
+
+cd ${filename}
+
+rm -rf doc man
+rm -rf bin/cmake-gui
+
+find . -type f -exec install -D "{}" "/usr/{}" \;

imagefiles/install-openssl.sh

@@ -1,83 +0,0 @@
-#!/bin/bash
-
-#
-# Configure, build and install OpenSSL
-#
-# Usage:
-#
-#  install-openssl.sh [-32]
-#
-# Options:
-#
-#  -32              Build OpenSSL as a 32-bit library
-#
-# Notes:
-#
-#  * build directory is /usr/src/openssl-$OPENSSL_VERSION
-#
-#  * install directory is /usr
-#
-#  * after installation, build directory and archive are removed
-#
-
-set -e
-set -o pipefail
-
-WRAPPER=""
-CONFIG_FLAG="-fPIC"
-SUFFIX=64
-
-while [ $# -gt 0 ]; do
-  case "$1" in
-    -32)
-      WRAPPER="linux32"
-      CONFIG_FLAG="-m32"
-      SUFFIX=32
-      ;;
-    *)
-      echo "Usage: Usage: ${0##*/} [-32]"
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-OPENSSL_VERSION=1.0.2j
-OPENSSL_SHA256=e7aff292be21c259c6af26469c7a9b3ba26e9abaaffd325e3dccc9785256c431
-
-cd /usr/src
-
-# Download
-if [ ! -f ./openssl-$OPENSSL_VERSION.tar.gz ]; then
-  curl -# -LO https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz
-else
-  rm -rf ./openssl-$OPENSSL_VERSION
-fi
-
-# Verify
-sha256_openssl=`sha256sum ./openssl-$OPENSSL_VERSION.tar.gz | awk '{ print $1 }'`
-if [ "$sha256_openssl" != "$OPENSSL_SHA256" ]
-then
-  echo "SHA256 mismatch. Problem downloading OpenSSL."
-  echo "  current [$sha256_openssl]"
-  echo "  expected[$OPENSSL_SHA256]"
-  exit 1
-fi
-
-# Extract
-tar -xzvf openssl-$OPENSSL_VERSION.tar.gz
-
-pushd openssl-$OPENSSL_VERSION
-
-# Configure
-${WRAPPER} ./config --prefix=/usr $CONFIG_FLAG
-
-# Build & Install
-${WRAPPER} make install
-
-popd
-
-# Clean
-rm -rf ./openssl-$OPENSSL_VERSION*
-rm -rf /usr/ssl/man
-

imagefiles/utils.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Copied from https://github.com/pypa/manylinux/blob/master/docker/build_scripts/build_utils.sh
+function check_var {
+    if [ -z "$1" ]; then
+        echo "required variable not defined"
+        exit 1
+    fi
+}
+
+# Copied from https://github.com/pypa/manylinux/blob/master/docker/build_scripts/build_utils.sh
+function check_sha256sum {
+    local fname=$1
+    check_var ${fname}
+    local sha256=$2
+    check_var ${sha256}
+
+    echo "${sha256}  ${fname}" > ${fname}.sha256
+    sha256sum -c ${fname}.sha256
+    rm -f ${fname}.sha256
+}
+

linux-x86/Dockerfile

@@ -34,11 +34,6 @@ ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
     CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++
 
 
-COPY imagefiles/install-openssl.sh /dockcross/
-RUN \
-  /dockcross/install-openssl.sh -32 && \
-  rm /dockcross/install-openssl.sh
-
 ENV DEFAULT_DOCKCROSS_IMAGE dockcross/linux-x86
 
 # Note: Toolchain file support is currently in debian Experimental:

manylinux-x64/Dockerfile.in

@@ -3,10 +3,10 @@ MAINTAINER Matt McCormick "matt.mccormick@kitware.com"
 
 ENV DEFAULT_DOCKCROSS_IMAGE dockcross/manylinux-x64
 
-#include "common.docker"
-
 #include "common.manylinux"
 
+#include "common.docker"
+
 ENV CROSS_TRIPLE x86_64-linux-gnu
 ENV CROSS_ROOT /opt/rh/devtoolset-2/root/usr/bin
 ENV AS=${CROSS_ROOT}/as \

manylinux-x86/Dockerfile.in

@@ -3,10 +3,10 @@ MAINTAINER Matt McCormick "matt.mccormick@kitware.com"
 
 ENV DEFAULT_DOCKCROSS_IMAGE dockcross/manylinux-x86
 
-#include "common.docker"
-
 #include "common.manylinux"
 
+#include "common.docker"
+
 ENV CROSS_TRIPLE i686-linux-gnu
 ENV CROSS_ROOT /opt/rh/devtoolset-2/root/usr/bin
 ENV AS=${CROSS_ROOT}/as \

windows-x64/Dockerfile

@@ -72,7 +72,6 @@ ENV CMAKE_TOOLCHAIN_FILE /usr/src/mxe/usr/x86_64-w64-mingw32.static/share/cmake/
 RUN echo 'set(CMAKE_CROSSCOMPILING_EMULATOR "/usr/bin/wine")' >> ${CMAKE_TOOLCHAIN_FILE}
 RUN cd /usr/local/bin \
   && rm cmake \
-  && rm cpack \
   && ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static-cmake cmake \
   && ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static-cpack cpack
 

windows-x86/Dockerfile

@@ -72,7 +72,6 @@ ENV CMAKE_TOOLCHAIN_FILE /usr/src/mxe/usr/i686-w64-mingw32.static/share/cmake/mx
 RUN echo 'set(CMAKE_CROSSCOMPILING_EMULATOR "/usr/bin/wine")' >> ${CMAKE_TOOLCHAIN_FILE}
 RUN cd /usr/local/bin \
   && rm cmake \
-  && rm cpack \
   && ln -s /usr/src/mxe/usr/bin/i686-w64-mingw32.static-cmake cmake \
   && ln -s /usr/src/mxe/usr/bin/i686-w64-mingw32.static-cpack cpack