Step4-SimulateStop.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit"
  },
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Verify Elastic Pytorch training fault tolerance\n",
    "In this notebook we will \n",
    "- Simulate SpotVM node eviction\n",
    "- Verify ElasticJob Fault Tolerance\n",
    "- Scale number of Nodes and verify Training job autoscaling\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "resource_group = \"elastic-lab\"   # replace with values from Step1 or use this default\n",
    "aks_name = \"elasticaks\"          # replace with values from Step1 or use this default\n",
    "aks_spot_nodepool = \"spotgpu\"    # replace with values from Step1 or use this default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "NAME                                 STATUS   ROLES   AGE   VERSION\n",
      "aks-cpuworkers-40607851-vmss000000   Ready    agent   45h   v1.18.17\n",
      "aks-cpuworkers-40607851-vmss000003   Ready    agent   45h   v1.18.17\n",
      "aks-nodepool1-40607851-vmss000000    Ready    agent   46h   v1.18.17\n",
      "aks-spotgpu-40607851-vmss000000      Ready    agent   38h   v1.18.17\n",
      "aks-spotgpu-40607851-vmss000002      Ready    agent   38h   v1.18.17\n",
      "NAME                                          READY   STATUS    RESTARTS   AGE     IP            NODE                                 NOMINATED NODE   READINESS GATES\n",
      "elastic-job-k8s-controller-5b9bc6b79c-xvdsw   1/1     Running   0          42h     10.244.13.2   aks-cpuworkers-40607851-vmss000003   <none>           <none>\n",
      "etcd                                          1/1     Running   0          43h     10.244.12.2   aks-cpuworkers-40607851-vmss000000   <none>           <none>\n",
      "imagenet-worker-0                             1/1     Running   0          2m35s   10.244.16.9   aks-spotgpu-40607851-vmss000000      <none>           <none>\n",
      "imagenet-worker-1                             1/1     Running   0          2m35s   10.244.18.6   aks-spotgpu-40607851-vmss000002      <none>           <none>\n",
      "imagenet-worker-2                             1/1     Running   0          2m35s   10.244.18.7   aks-spotgpu-40607851-vmss000002      <none>           <none>\n"
     ]
    }
   ],
   "source": [
    "# Get currently running pods on the spotvm nodes\n",
    "!kubectl get nodes\n",
    "!kubectl get pods -n elastic-job -o wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run commands to get all variables for REST API and strip beginning and end quotes\n",
    "subscription_id = !az account show --query id \n",
    "subscription_id = str(subscription_id[0][1:-1])\n",
    "node_rg = !az aks show --resource-group {resource_group} --name {aks_name} --query nodeResourceGroup --only-show-errors\n",
    "node_rg = str(node_rg[0][1:-1])\n",
    "vmss_name= !az vmss list -g MC_elastic-lab_elasticaks_eastus2 --query '[].name' -o tsv | grep spot\n",
    "vmss_name = vmss_name[0]\n",
    "instance_id = 1 ## Fill it in if different node !!"
   ]
  },
  {
   "source": [
    "For information on SpotVM eviction see [Simulate an eviction docs](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/use-spot#simulate-an-eviction)\n",
    "Use REST API to simulate SpotVM eviction (it can also be tested using Azure API Console https://docs.microsoft.com/en-us/rest/api/compute/virtualmachinescalesetvms/simulateeviction#code-try-0)\n",
    "\n",
    "Response HTTP Code should be 204 , accepting the event"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\u001b[32mRequest URL: 'https://management.azure.com/subscriptions/f869415f-5cff-46a3-b728-20659d14d62d/resourceGroups/MC_elastic-lab_elasticaks_eastus2/providers/Microsoft.Compute/virtualMachineScaleSets/aks-spotgpu-40607851-vmss/virtualMachines/2/simulateEviction?api-version=2020-12-01'\u001b[0m\n",
      "\u001b[32mRequest method: 'POST'\u001b[0m\n",
      "\u001b[32mRequest headers:\u001b[0m\n",
      "\u001b[32m    'User-Agent': 'python/3.6.10 (Linux-5.4.72-microsoft-standard-WSL2-x86_64-with-debian-bullseye-sid) AZURECLI/2.19.1 (DEB)'\u001b[0m\n",
      "\u001b[32m    'Accept-Encoding': 'gzip, deflate'\u001b[0m\n",
      "\u001b[32m    'Accept': '*/*'\u001b[0m\n",
      "\u001b[32m    'Connection': 'keep-alive'\u001b[0m\n",
      "\u001b[32m    'x-ms-client-request-id': 'd2d5b243-406e-446b-949b-58f94664cc1e'\u001b[0m\n",
      "\u001b[32m    'CommandName': 'rest'\u001b[0m\n",
      "\u001b[32m    'ParameterSetName': '--verbose --method --uri'\u001b[0m\n",
      "\u001b[32m    'Authorization': 'Bearer eyJ0eXAiOiJKV...'\u001b[0m\n",
      "\u001b[32m    'Content-Length': '0'\u001b[0m\n",
      "\u001b[32mRequest body:\u001b[0m\n",
      "\u001b[32mNone\u001b[0m\n",
      "\u001b[32mResponse status: 204\u001b[0m\n",
      "\u001b[32mResponse headers:\u001b[0m\n",
      "\u001b[32m    'Cache-Control': 'no-cache'\u001b[0m\n",
      "\u001b[32m    'Pragma': 'no-cache'\u001b[0m\n",
      "\u001b[32m    'Expires': '-1'\u001b[0m\n",
      "\u001b[32m    'Strict-Transport-Security': 'max-age=31536000; includeSubDomains'\u001b[0m\n",
      "\u001b[32m    'x-ms-request-id': '224764e3-0229-4672-a920-ccdc0707c88a'\u001b[0m\n",
      "\u001b[32m    'Server': 'Microsoft-HTTPAPI/2.0, Microsoft-HTTPAPI/2.0'\u001b[0m\n",
      "\u001b[32m    'x-ms-ratelimit-remaining-subscription-writes': '1199'\u001b[0m\n",
      "\u001b[32m    'x-ms-correlation-request-id': 'c05d0472-0524-4638-b440-80d423a7dfbe'\u001b[0m\n",
      "\u001b[32m    'x-ms-routing-request-id': 'CANADACENTRAL:20210525T201336Z:c05d0472-0524-4638-b440-80d423a7dfbe'\u001b[0m\n",
      "\u001b[32m    'X-Content-Type-Options': 'nosniff'\u001b[0m\n",
      "\u001b[32m    'Date': 'Tue, 25 May 2021 20:13:36 GMT'\u001b[0m\n",
      "\u001b[32mResponse content:\u001b[0m\n",
      "\u001b[32m\u001b[0m\n",
      "\u001b[32mCommand ran in 0.520 seconds (init: 0.138, invoke: 0.381)\u001b[0m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az rest --verbose --method POST \\\n",
    "   --uri 'https://management.azure.com/subscriptions/{subscription_id}/resourceGroups/{node_rg}/providers/Microsoft.Compute/virtualMachineScaleSets/{vmss_name}/virtualMachines/{instance_id}/simulateEviction?api-version=2020-12-01'"
   ]
  },
  {
   "source": [
    "Eviction event is sent and it might take a minute or so to remove a node, run the cell few times until you see node removed and some of the worker Pods in `Pending` state "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "NAME                                 STATUS     ROLES   AGE   VERSION\n",
      "aks-cpuworkers-40607851-vmss000000   Ready      agent   46h   v1.18.17\n",
      "aks-cpuworkers-40607851-vmss000003   Ready      agent   45h   v1.18.17\n",
      "aks-nodepool1-40607851-vmss000000    Ready      agent   46h   v1.18.17\n",
      "aks-spotgpu-40607851-vmss000002      NotReady   agent   38h   v1.18.17\n",
      "NAME                                          READY   STATUS    RESTARTS   AGE     IP            NODE                                 NOMINATED NODE   READINESS GATES\n",
      "elastic-job-k8s-controller-5b9bc6b79c-xvdsw   1/1     Running   0          43h     10.244.13.2   aks-cpuworkers-40607851-vmss000003   <none>           <none>\n",
      "etcd                                          1/1     Running   0          43h     10.244.12.2   aks-cpuworkers-40607851-vmss000000   <none>           <none>\n",
      "imagenet-worker-0                             0/1     Pending   0          3m33s   <none>        <none>                               <none>           <none>\n",
      "imagenet-worker-1                             1/1     Running   0          34m     10.244.18.6   aks-spotgpu-40607851-vmss000002      <none>           <none>\n",
      "imagenet-worker-2                             1/1     Running   0          34m     10.244.18.7   aks-spotgpu-40607851-vmss000002      <none>           <none>\n"
     ]
    }
   ],
   "source": [
    "# Verify the node was removed from pool and workers on the node stoped\n",
    "!kubectl get nodes\n",
    "!kubectl get pods -n elastic-job -o wide"
   ]
  },
  {
   "source": [
    "Once Node is evicted, workers on the removed node is deleted, and the rest of the running workers will detect that the process group has changed and rendezevous server will adjust the training accoringly.\n",
    "\n",
    "Notice restart_count, group_rank and and process epoch in the logs:"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "=> using checkpoint from rank: 1, max_epoch: 0\n",
      "=> checkpoint broadcast size is: 93588276\n",
      "=> done broadcasting checkpoint\n",
      "=> done restoring from previous checkpoint\n",
      "=> start_epoch: 1, best_acc1: 0.6399999856948853\n",
      "Epoch: [1][  0/782]\tTime  4.830 ( 4.830)\tData  1.852 ( 1.852)\tLoss 4.6512e+00 (4.6512e+00)\tAcc@1   6.25 (  6.25)\tAcc@5  17.19 ( 17.19)\n",
      "Epoch: [1][ 10/782]\tTime  2.197 ( 2.201)\tData  1.704 ( 1.494)\tLoss 4.1309e+00 (4.3900e+00)\tAcc@1  12.50 (  8.95)\tAcc@5  34.38 ( 25.85)\n",
      "Epoch: [1][ 20/782]\tTime  1.499 ( 2.043)\tData  1.100 ( 1.457)\tLoss 4.3086e+00 (4.4244e+00)\tAcc@1   6.25 (  8.71)\tAcc@5  25.00 ( 24.48)\n",
      "Epoch: [1][ 30/782]\tTime  1.606 ( 1.939)\tData  1.210 ( 1.380)\tLoss 4.4815e+00 (4.4541e+00)\tAcc@1   4.69 (  8.11)\tAcc@5  23.44 ( 23.94)\n",
      "Epoch: [1][ 40/782]\tTime  1.820 ( 1.890)\tData  1.234 ( 1.336)\tLoss 4.4205e+00 (4.4695e+00)\tAcc@1  12.50 (  8.08)\tAcc@5  25.00 ( 23.86)\n",
      "=> checkpoint broadcast size is: 93588276\n",
      "/opt/conda/conda-bld/pytorch_1587428398394/work/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n",
      "=> done broadcasting checkpoint\n",
      "=> done restoring from previous checkpoint\n",
      "=> start_epoch: 1, best_acc1: 0.6399999856948853\n",
      "Epoch: [1][  0/782]\tTime  4.831 ( 4.831)\tData  1.363 ( 1.363)\tLoss 4.6879e+00 (4.6879e+00)\tAcc@1   6.25 (  6.25)\tAcc@5  14.06 ( 14.06)\n",
      "Epoch: [1][ 10/782]\tTime  2.197 ( 2.201)\tData  1.820 ( 1.477)\tLoss 4.3488e+00 (4.4434e+00)\tAcc@1   7.81 (  9.23)\tAcc@5  25.00 ( 24.43)\n",
      "Epoch: [1][ 20/782]\tTime  1.496 ( 2.043)\tData  1.098 ( 1.453)\tLoss 4.6689e+00 (4.5220e+00)\tAcc@1   6.25 (  8.18)\tAcc@5  18.75 ( 23.14)\n",
      "Epoch: [1][ 30/782]\tTime  1.606 ( 1.939)\tData  1.221 ( 1.400)\tLoss 4.7405e+00 (4.5395e+00)\tAcc@1   4.69 (  7.76)\tAcc@5  18.75 ( 22.68)\n",
      "Epoch: [1][ 40/782]\tTime  1.815 ( 1.890)\tData  1.431 ( 1.378)\tLoss 4.6776e+00 (4.5438e+00)\tAcc@1   3.12 (  7.70)\tAcc@5  17.19 ( 22.64)\n"
     ]
    }
   ],
   "source": [
    " !kubectl logs -ljob-name=imagenet -n elastic-job --since 10m"
   ]
  },
  {
   "source": [
    "Scale  Up the Spot Node Pool and note the new workers starting to run on the newly added nodes, and Rendezevous process adjusting accordingly"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\u001b[33mThe behavior of this command has been altered by the following extension: aks-preview\u001b[0m\n",
      "{\n",
      "  \"agentPoolType\": \"VirtualMachineScaleSets\",\n",
      "  \"availabilityZones\": null,\n",
      "  \"count\": 2,\n",
      "  \"enableAutoScaling\": null,\n",
      "  \"enableEncryptionAtHost\": false,\n",
      "  \"enableFips\": false,\n",
      "  \"enableNodePublicIp\": false,\n",
      "  \"gpuInstanceProfile\": null,\n",
      "  \"id\": \"/subscriptions/f869415f-5cff-xxxx-xxx-20659d14d62d/resourcegroups/elastic-lab/providers/Microsoft.ContainerService/managedClusters/elasticaks/agentPools/spotgpu\",\n",
      "  \"kubeletConfig\": null,\n",
      "  \"kubeletDiskType\": \"OS\",\n",
      "  \"linuxOsConfig\": null,\n",
      "  \"maxCount\": null,\n",
      "  \"maxPods\": 110,\n",
      "  \"minCount\": null,\n",
      "  \"mode\": \"User\",\n",
      "  \"name\": \"spotgpu\",\n",
      "  \"nodeImageVersion\": \"AKSUbuntu-1804gpu-2021.05.01\",\n",
      "  \"nodeLabels\": {\n",
      "    \"kubernetes.azure.com/scalesetpriority\": \"spot\"\n",
      "  },\n",
      "  \"nodePublicIpPrefixId\": null,\n",
      "  \"nodeTaints\": [\n",
      "    \"kubernetes.azure.com/scalesetpriority=spot:NoSchedule\"\n",
      "  ],\n",
      "  \"orchestratorVersion\": \"1.18.17\",\n",
      "  \"osDiskSizeGb\": 128,\n",
      "  \"osDiskType\": \"Managed\",\n",
      "  \"osSku\": \"Ubuntu\",\n",
      "  \"osType\": \"Linux\",\n",
      "  \"podSubnetId\": null,\n",
      "  \"powerState\": {\n",
      "    \"code\": \"Running\"\n",
      "  },\n",
      "  \"provisioningState\": \"Succeeded\",\n",
      "  \"proximityPlacementGroupId\": null,\n",
      "  \"resourceGroup\": \"elastic-lab\",\n",
      "  \"scaleSetEvictionPolicy\": \"Delete\",\n",
      "  \"scaleSetPriority\": \"Spot\",\n",
      "  \"spotMaxPrice\": -1.0,\n",
      "  \"tags\": null,\n",
      "  \"type\": \"Microsoft.ContainerService/managedClusters/agentPools\",\n",
      "  \"upgradeSettings\": {\n",
      "    \"maxSurge\": null\n",
      "  },\n",
      "  \"vmSize\": \"Standard_NC12\",\n",
      "  \"vnetSubnetId\": null\n",
      "}\n",
      "\u001b[K\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az aks nodepool scale --cluster-name {aks_name} \\\n",
    "                      --name {aks_spot_nodepool} \\\n",
    "                      --resource-group {resource_group} \\\n",
    "                      --node-count 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "pod/imagenet-worker-0 condition met\n",
      "pod/imagenet-worker-1 condition met\n",
      "pod/imagenet-worker-2 condition met\n",
      "NAME                                 STATUS   ROLES   AGE   VERSION\n",
      "aks-cpuworkers-40607851-vmss000000   Ready    agent   46h   v1.18.17\n",
      "aks-cpuworkers-40607851-vmss000003   Ready    agent   46h   v1.18.17\n",
      "aks-nodepool1-40607851-vmss000000    Ready    agent   47h   v1.18.17\n",
      "aks-spotgpu-40607851-vmss000004      Ready    agent   12m   v1.18.17\n",
      "aks-spotgpu-40607851-vmss000005      Ready    agent   12m   v1.18.17\n",
      "NAME                                          READY   STATUS    RESTARTS   AGE   IP            NODE                                 NOMINATED NODE   READINESS GATES\n",
      "elastic-job-k8s-controller-5b9bc6b79c-xvdsw   1/1     Running   0          43h   10.244.13.2   aks-cpuworkers-40607851-vmss000003   <none>           <none>\n",
      "etcd                                          1/1     Running   0          43h   10.244.12.2   aks-cpuworkers-40607851-vmss000000   <none>           <none>\n",
      "imagenet-worker-0                             1/1     Running   0          18m   10.244.20.2   aks-spotgpu-40607851-vmss000005      <none>           <none>\n",
      "imagenet-worker-1                             1/1     Running   0          11m   10.244.21.2   aks-spotgpu-40607851-vmss000004      <none>           <none>\n",
      "imagenet-worker-2                             1/1     Running   0          11m   10.244.21.3   aks-spotgpu-40607851-vmss000004      <none>           <none>\n"
     ]
    }
   ],
   "source": [
    "# wait for node being added and all pods to start \n",
    "!kubectl wait -l job-name=imagenet pods --for condition=ready \n",
    "# Verify the node was added to the pool and all workers are running\n",
    "!kubectl get nodes\n",
    "!kubectl get pods -n elastic-job -o wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[INFO] 2021-05-24 05:27:12,911 launch: Running torchelastic.distributed.launch with args: ['/opt/conda/lib/python3.7/site-packages/torchelastic/distributed/launch.py', '--rdzv_backend=etcd', '--rdzv_endpoint=etcd-service:2379', '--rdzv_id=imagenet', '--nnodes=1:3', '--nproc_per_node=1', '/workspace/examples/imagenet/main.py', '--arch=resnet18', '--epochs=3', '--batch-size=64', '--workers=0', '/workspace/data/tiny-imagenet-200', '--checkpoint-file=/mnt/blob/data/checkpoint.pth.tar']\nINFO 2021-05-24 05:27:12,921 Etcd machines: ['http://0.0.0.0:2379']\n[INFO] 2021-05-24 05:27:12,937 launch: Using nproc_per_node=1.\n[INFO] 2021-05-24 05:27:13,674 api: [default] starting workers for function: wrapper_fn\n[INFO] 2021-05-24 05:27:13,674 api: [default] Rendezvous'ing worker group\nINFO 2021-05-24 05:27:13,674 Attempting to join next rendezvous\nINFO 2021-05-24 05:27:13,678 Observed existing rendezvous state: {'status': 'final', 'version': '9', 'participants': [0, 1], 'keep_alives': ['/torchelastic/p2p/run_imagenet/rdzv/v_9/rank_1', '/torchelastic/p2p/run_imagenet/rdzv/v_9/rank_0'], 'num_workers_waiting': 0}\nINFO 2021-05-24 05:27:13,687 Added self to waiting list. Rendezvous full state: {\"status\": \"final\", \"version\": \"9\", \"participants\": [0, 1], \"keep_alives\": [\"/torchelastic/p2p/run_imagenet/rdzv/v_9/rank_1\", \"/torchelastic/p2p/run_imagenet/rdzv/v_9/rank_0\"], \"num_workers_waiting\": 1}\nINFO 2021-05-24 05:27:25,372 Keep-alive key /torchelastic/p2p/run_imagenet/rdzv/v_9/rank_1 is not renewed.\nINFO 2021-05-24 05:27:25,372 Rendevous version 9 is incomplete. \nINFO 2021-05-24 05:27:25,372 Attempting to destroy it.\nINFO 2021-05-24 05:27:25,375 Rendezvous attempt failed, will retry. Reason: Key not found : /torchelastic/p2p/run_imagenet/rdzv/active_version\nINFO 2021-05-24 05:27:26,377 Attempting to join next rendezvous\nINFO 2021-05-24 05:27:26,380 Observed existing rendezvous state: {'status': 'joinable', 'version': '10', 'participants': [0]}\nINFO 2021-05-24 05:27:26,403 Joined rendezvous version 10 as rank 1. Full state: {'status': 'joinable', 'version': '10', 'participants': [0, 1]}\nINFO 2021-05-24 05:27:26,403 Waiting for remaining peers.\nINFO 2021-05-24 05:27:26,462 All peers arrived. Confirming membership.\nINFO 2021-05-24 05:27:26,480 Waiting for confirmations from all peers.\nINFO 2021-05-24 05:27:26,545 Rendezvous version 10 is complete. Final state: {'status': 'final', 'version': '10', 'participants': [0, 1, 2], 'keep_alives': ['/torchelastic/p2p/run_imagenet/rdzv/v_10/rank_1', '/torchelastic/p2p/run_imagenet/rdzv/v_10/rank_2', '/torchelastic/p2p/run_imagenet/rdzv/v_10/rank_0'], 'num_workers_waiting': 0}\nINFO 2021-05-24 05:27:26,545 Creating EtcdStore as the c10d::Store implementation\n[INFO] 2021-05-24 05:27:26,568 api: [default] Rendezvous complete for workers.\nResult:\n\trestart_count=0\n\tgroup_rank=1\n\tgroup_world_size=3\n\trank stride=1\n\tassigned global_ranks=[1]\n\tmaster_addr=imagenet-worker-1\n\tmaster_port=53663\n\n[INFO] 2021-05-24 05:27:26,568 api: [default] Starting worker group\n=> set cuda device = 0\n=> creating model: resnet18\n=> loading checkpoint file: /mnt/blob/data/checkpoint.pth.tar\n=> loaded checkpoint file: /mnt/blob/data/checkpoint.pth.tar\n=> using checkpoint from rank: 2, max_epoch: 0\n=> checkpoint broadcast size is: 93588276\n=> done broadcasting checkpoint\n=> done restoring from previous checkpoint\n=> start_epoch: 1, best_acc1: 0.6399999856948853\nEpoch: [1][  0/521]\tTime  4.900 ( 4.900)\tData  0.122 ( 0.122)\tLoss 4.3889e+00 (4.3889e+00)\tAcc@1  10.94 ( 10.94)\tAcc@5  18.75 ( 18.75)\nEpoch: [1][ 10/521]\tTime  2.104 ( 2.290)\tData  0.154 ( 0.117)\tLoss 4.8242e+00 (4.4380e+00)\tAcc@1   7.81 (  9.80)\tAcc@5  17.19 ( 24.43)\nEpoch: [1][ 20/521]\tTime  2.126 ( 2.237)\tData  0.119 ( 0.114)\tLoss 4.2107e+00 (4.4472e+00)\tAcc@1   7.81 (  8.41)\tAcc@5  26.56 ( 23.51)\nEpoch: [1][ 30/521]\tTime  2.410 ( 2.198)\tData  0.105 ( 0.112)\tLoss 4.6429e+00 (4.4383e+00)\tAcc@1  10.94 (  8.92)\tAcc@5  23.44 ( 23.64)\nEpoch: [1][ 40/521]\tTime  2.281 ( 2.156)\tData  0.122 ( 0.112)\tLoss 4.2133e+00 (4.4300e+00)\tAcc@1  10.94 (  8.69)\tAcc@5  26.56 ( 23.97)\n"
     ]
    }
   ],
   "source": [
    "# Verify new Rendezevous group was created and workers readjusted\n",
    "!kubectl logs imagenet-worker-0 -n elastic-job "
   ]
  },
  {
   "source": [
    "## Summary\n",
    "\n",
    "This Lab has demonstrated Torch Elastic capabilities and ability to run Fault tolerant elastic training.\n",
    "\n",
    "Great job for completeing it."
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ]
}