diff --git a/imagefiles/entrypoint.sh b/imagefiles/entrypoint.sh
index f93051f..7f6031b 100755
--- a/imagefiles/entrypoint.sh
+++ b/imagefiles/entrypoint.sh
@@ -50,8 +50,10 @@ if [[ -n $BUILDER_UID ]] && [[ -n $BUILDER_GID ]]; then
        gosu $BUILDER_UID:$BUILDER_GID /work/.dockcross
     fi
 
-    # Run the command as the specified user/group.
-    exec gosu $BUILDER_UID:$BUILDER_GID "$@"
+    # Run the command as the specified user.  The group will be BUILDER_GID,
+    # as pulled from /etc/passwd, but will have additional groups from
+    # /etc/group.  This is necessary for manylinux to install python packages.
+    exec gosu $BUILDER_UID "$@"
 else
     # Just run the command as root.
     exec "$@"
diff --git a/manylinux-common/install-python-packages.sh b/manylinux-common/install-python-packages.sh
index cc2c90e..7809a7d 100755
--- a/manylinux-common/install-python-packages.sh
+++ b/manylinux-common/install-python-packages.sh
@@ -4,3 +4,41 @@ for PIP in /opt/python/*/bin/pip; do
   $PIP install --disable-pip-version-check --upgrade pip
   $PIP install scikit-build==0.8.1
 done
+
+# pip is installing everything as root.  We need to make sure whatever uid/gid
+# is used when the image is run is able to update the python install.  We don't
+# want to limit which uid/gid can be used, so instead we're going to:
+#  - create a custom group
+#  - change the group of the python directories to the new group
+#  - change the permissions of the python directories to allow the group to write
+
+# This GID isn't used in the manylinux image, but just to be safe, we'll verify
+# it's still not in use during the image build.
+BUILDER_GID=200
+if id -g "$BUILDER_GID" 2> /dev/null; then
+  echo "Error: builder gid $BUILDER_GID already exists!  Aborting"
+  exit 1
+fi
+
+# Add the builder group
+groupadd -g $BUILDER_GID builder
+
+# Update the permissions for all python directories so they're owned by the
+# builder group, and the builder group can write to them.
+for DIR in /opt/python/*/lib/python*/site-packages; do
+  chown -R :$BUILDER_GID $DIR
+  chmod g+w $DIR
+done
+for DIR in /opt/python/*/bin; do
+  chown -R :$BUILDER_GID $DIR
+  chmod g+w $DIR
+done
+for DIR in /opt/python/*; do
+  mkdir $DIR/man
+  chown -R :$BUILDER_GID $DIR/man
+  chmod g+w $DIR/man
+done
+for DIR in /opt/python/*/share; do
+  chown -R :$BUILDER_GID $DIR
+  chmod g+w $DIR
+done
diff --git a/manylinux-common/pre_exec.sh b/manylinux-common/pre_exec.sh
index 6b4abb8..57d8304 100755
--- a/manylinux-common/pre_exec.sh
+++ b/manylinux-common/pre_exec.sh
@@ -1,15 +1,5 @@
 #!/usr/bin/env bash
 
-for DIR in /opt/python/*/lib/python*/site-packages; do
-  chown -R $BUILDER_UID:$BUILDER_GID $DIR
-done
-for DIR in /opt/python/*/bin; do
-  chown -R $BUILDER_UID:$BUILDER_GID $DIR
-done
-for DIR in /opt/python/*; do
-  mkdir $DIR/man
-  chown -R $BUILDER_UID:$BUILDER_GID $DIR/man
-done
-for DIR in /opt/python/*/share; do
-  chown -R $BUILDER_UID:$BUILDER_GID $DIR
-done
+# Add the BUILDER_USER to the builder group so `pip install` doesn't get access
+# denied errors.
+usermod -a -G builder $BUILDER_USER