Turns out I didn't need to do all that

bash/.bashrc

@@ -133,58 +133,15 @@ if [ -f "$HOME/.private_functions" ] && [ -r "$HOME/.private_functions" ]; then
   . "$HOME/.private_functions"
 fi
 
+# Source bash completion, this file will then lazy load all other completions
 source_completions() {
-  # Source completions if they are available
-  completion_dirs=(
-    "/usr/share/bash-completion/completions"
-    "/etc/bash_completion.d"
-  )
-  cache_dir="${XDG_CACHE_HOME}/bash-completion-sourcing"
-
-  if [ "${BUILD_CACHE}" ]; then
-    mkdir -p "${cache_dir}"
-  fi
-
-  # Base completion sourcing, later completions will fail without this
   if [ -r "/usr/share/bash-completion/bash_completion" ]; then
     . "/usr/share/bash-completion/bash_completion"
   elif [ -r "/etc/bash_completion" ]; then
     . "/etc/bash_completion"
   fi
-
-  # Completions from packages
-  export TIMEFORMAT="%R"
-  for completion_dir in "${completion_dirs[@]}"; do
-    if [ -d "${completion_dir}" ] && [ -r "${completion_dir}" ]; then
-      for completion_file in "${completion_dir}"/*; do
-        if [ "$BUILD_CACHE" ]; then
-          echo "Sourcing ${completion_file}"
-          . "${completion_file}"
-          timeavg=$(time (. "${completion_file}") 2>&1 1>/dev/null)
-          timeavg=$(echo $timeavg | cut -d '.' -f 2)
-          if [ "${timeavg}" -gt 100 ]; then
-            echo "This ones slow, adding to cache"
-            echo "${timeavg}" > "${cache_dir}/$(basename ${completion_file})"
-          elif [ "${timeavg}" -gt 10 ]; then
-            echo "This ones a little slow"
-            echo "${timeavg}"
-          fi
-        else
-          if ! [ -e "${cache_dir}/$(basename ${completion_file})" ]; then
-            . "${completion_file}"
-          else
-            :
-          fi
-        fi
-      done
-    fi
-  done
-  unset TIMEFORMAT
 }
-
-if [ "$BASH_COMPLETION_ENABLED" ]; then
-  source_completions
-fi
+source_completions
 
 # Preserve the usefulness of set -x by attempting to hide everything that
 # happens in prompt_command