gpu does not work on capacity scheduler on dataproc 2.0 ; use fair ; …

…also protect against race condition on removing the .building files

gpu/install_gpu_driver.sh

@@ -687,7 +687,7 @@ function install_nvidia_nccl() {
       popd
       tar xzvf "${local_tarball}"
       gcloud storage cp "${local_tarball}" "${gcs_tarball}"
-      gcloud storage rm "${gcs_tarball}.building"
+      if gcloud storage ls "${gcs_tarball}.building" ; then gcloud storage rm "${gcs_tarball}.building" ; fi
       rm "${local_tarball}"
     fi
   }
@@ -813,7 +813,7 @@ function install_pytorch() {
     tar czf "${local_tarball}" .
     popd
     gcloud storage cp "${local_tarball}" "${gcs_tarball}"
-    gcloud storage rm "${gcs_tarball}.building"
+    if gcloud storage ls "${gcs_tarball}.building" ; then gcloud storage rm "${gcs_tarball}.building" ; fi
   fi
   touch "${workdir}/complete/pytorch"
 }
@@ -1014,7 +1014,7 @@ function build_driver_from_github() {
         "${workdir}/open-gpu-kernel-modules/kernel-open/"*.log \
         $(find /lib/modules/${uname_r}/ -iname 'nvidia*.ko')
       gcloud storage cp "${local_tarball}" "${gcs_tarball}"
-      gcloud storage rm "${gcs_tarball}.building"
+      if gcloud storage ls "${gcs_tarball}.building" ; then gcloud storage rm "${gcs_tarball}.building" ; fi
       rm "${local_tarball}"
       make clean
       popd
@@ -1160,7 +1160,7 @@ function install_nvidia_userspace_runfile() {
         /var/log/nvidia-installer.log \
         $(find /lib/modules/${uname_r}/ -iname 'nvidia*.ko')
       gcloud storage cp "${local_tarball}" "${gcs_tarball}"
-      gcloud storage rm "${gcs_tarball}.building"
+      if gcloud storage ls "${gcs_tarball}.building" ; then gcloud storage rm "${gcs_tarball}.building" ; fi
     fi
   fi
 
@@ -1374,7 +1374,12 @@ function configure_yarn_resources() {
 # This configuration should be applied only if GPU is attached to the node
 function configure_yarn_nodemanager() {
   if [[ "${gpu_count}" == "0" ]] ; then return ; fi
-
+  # CapacityScheduler does not permit use of gpu resources ; switch to FairScheduler
+  if version_lt "${DATAPROC_IMAGE_VERSION}" 2.1 ; then
+    set_hadoop_property 'yarn-site.xml' \
+      'yarn.resourcemanager.scheduler.class' \
+      'org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler'
+  fi
   set_hadoop_property 'yarn-site.xml' 'yarn.nodemanager.resource-plugins' 'yarn.io/gpu'
   set_hadoop_property 'yarn-site.xml' \
     'yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices' 'auto'