ci: Optimize parallel build

* style: Rename TASK to STEP
* do not pull base image after it was copied from node0 and loaded.
  It caused the current Dockerfile to be out-of-sync with the current image.
* do not pull images after loading the downloaded and up-to-date
  base.tar. It was overwriting the base.tar downloaded and loaded.
* save debian:jessie into base.tar
* use "time" command to help identify long running operation
* rename marker file from "BASE_READY" to "BASE_AVAILABLE"
* build base image only if needed.

.circleci-matrix.yml

@@ -14,42 +14,45 @@ env:
   - IMAGE=windows-x86
 command:
   - |
-    if [[ $TASK == "test" ]]; then
+    if [[ $STEP == "test" ]]; then
       if [[ $CIRCLE_NODE_INDEX == 0 ]]; then
-        if [[ -e ~/docker/base.tar ]]; then docker load -i ~/docker/base.tar; fi
-        docker pull dockcross/base
-        make base.test
-        mkdir -p ~/docker; docker save dockcross/base > ~/docker/base.tar
-        touch ~/BASE_READY
+        if [[ ! -f ~/BASE_AVAILABLE ]]; then
+          if [[ -e ~/docker/base.tar ]]; then time docker load -i ~/docker/base.tar; fi
+          time docker pull dockcross/base
+          time make base.test
+          mkdir -p ~/docker; time docker save -o ~/docker/base.tar debian:jessie dockcross/base
+          touch ~/BASE_AVAILABLE
         else
-        if [[ ! -f ~/BASE_LOADED ]]; then
+          echo "Base image already available"
+        fi
+      else
+        if [[ ! -f ~/BASE_DOWNLOADED ]]; then
           echo "Waiting for node0"
           while true; do
             sleep 5
-            ssh -q node0 [[ -f ~/BASE_READY ]]
+            ssh -q node0 [[ -f ~/BASE_AVAILABLE ]]
             exit_code=$?
             if [[ $exit_code -eq 0 ]]; then break; elif [[ $exit_code -eq 1 ]]; then echo -n "."; else exit $exit_code;fi
           done
           echo ""
           echo "Copying base image from node0"
-          mkdir -p ~/docker; scp node0:~/docker/base.tar ~/docker/base.tar
-          docker load -i ~/docker/base.tar
-          docker pull dockcross/base
-          touch ~/BASE_LOADED
+          mkdir -p ~/docker; time scp node0:~/docker/base.tar ~/docker/base.tar
+          touch ~/BASE_DOWNLOADED
         else
-          echo "Base image already loaded"
+          echo "Base image already downloaded"
         fi
       fi
-      if [[ -e ~/docker/$IMAGE.tar ]]; then docker load -i ~/docker/$IMAGE.tar; fi
-      docker pull dockcross/$IMAGE
-      make $IMAGE.test
-      mkdir -p ~/docker; docker save dockcross/$IMAGE > ~/docker/$IMAGE.tar
+      if [[ -e ~/docker/$IMAGE.tar ]]; then time docker load -i ~/docker/$IMAGE.tar; fi
+      time docker pull dockcross/$IMAGE
+      time docker load -i ~/docker/base.tar
+      time make $IMAGE.test
+      mkdir -p ~/docker; time docker save dockcross/$IMAGE > ~/docker/$IMAGE.tar
     fi
   - |
-    if [[ $TASK == "deploy" ]]; then
+    if [[ $STEP == "deployment" ]]; then
       docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
       if [[ $CIRCLE_NODE_INDEX == 0 ]] && [[ ! -f ~/BASE_PUSHED ]]; then
-        docker push dockcross/base
+        time docker push dockcross/base
         touch ~/BASE_PUSHED
       fi
       docker push dockcross/$IMAGE

circle.yml

@@ -21,7 +21,7 @@ test:
         parallel: true
         timeout: 3000
         environment:
-          TASK: test
+          STEP: test
 
 deployment:
   hub:
@@ -30,4 +30,4 @@ deployment:
       - circleci-matrix:
           parallel: true
           environment:
-            TASK: deploy
+            STEP: deployment