adaptive_bound_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "sys.path.append(os.path.abspath(os.path.join(os.getcwd(), 'code')))\n",
    "\n",
    "import torch\n",
    "from skip_block import SkipBlock\n",
    "import torch.nn.functional as F\n",
    "from typing import Optional\n",
    "\n",
    "\n",
    "EPS = 1e-8\n",
    "DEBUG = True\n",
    "INIT_COEF = 4\n",
    "LR = 0.7\n",
    "DECAY = 0.9\n",
    "DECAY_STEP = 1\n",
    "EPOCHS = 20\n",
    "\n",
    "\n",
    "def eliding_w(input_dim: int, target: int) -> torch.Tensor:\n",
    "    w = torch.zeros(input_dim, input_dim)\n",
    "    for i in range(input_dim):\n",
    "        w[i][i] += 1\n",
    "        w[i][target] -= 1\n",
    "    return w\n",
    "\n",
    "\n",
    "def skip_application(skip_dim: int) -> torch.Tensor:\n",
    "    w = torch.zeros(skip_dim, 2*skip_dim)\n",
    "    w[:, :skip_dim] = w[:, skip_dim:skip_dim*2] = torch.eye(skip_dim)\n",
    "    return w\n",
    "\n",
    "\n",
    "def dp_verify(model: torch.nn.Module, input: torch.Tensor, eps: float, true_output: int) -> bool:\n",
    "    input = input.unsqueeze(0)\n",
    "    shallowuni = Deeppoly(input-eps, input+eps)\n",
    "    shallowuni.forward(model)\n",
    "    if shallowuni.append_and_verify(true_output):\n",
    "        return True\n",
    "    if shallowuni.alpha_buffer_idx == 0:\n",
    "        return False\n",
    "    for i in range(shallowuni.alpha_buffer_idx):\n",
    "        shallowuni.alpha_buffer[i].requires_grad = True\n",
    "\n",
    "    if DEBUG:\n",
    "        torch.autograd.set_detect_anomaly(True)\n",
    "\n",
    "    optimizer = torch.optim.Adam(shallowuni.alpha_buffer, lr=LR)\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=DECAY_STEP, gamma=DECAY)\n",
    "    for epoch in range(EPOCHS):\n",
    "        optimizer.zero_grad()\n",
    "        shallowuni.rewind()\n",
    "        shallowuni.forward(model)\n",
    "        if shallowuni.append_and_verify(true_output):\n",
    "            return True\n",
    "        loss = torch.mean(torch.relu(shallowuni.upper_bounds[-1]).flatten())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        print(sum(shallowuni.alpha_buffer[-1].grad[0]))\n",
    "        if scheduler.get_last_lr()[0] > 0.1:\n",
    "            scheduler.step()\n",
    "    return False\n",
    "    \n",
    "\n",
    "\n",
    "def backsubstitute(W_l0: torch.Tensor, W_u0: torch.Tensor, W_l: torch.Tensor, W_u: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:\n",
    "    W_l_pos = W_l.clamp_min(0)\n",
    "    W_l_neg = W_l.clamp_max(0)\n",
    "    W_u_pos = W_u.clamp_min(0)\n",
    "    W_u_neg = W_u.clamp_max(0)\n",
    "\n",
    "    W_l_tilde = W_l0 @ W_l_pos + W_u0 @ W_l_neg\n",
    "    W_u_tilde = W_u0 @ W_u_pos + W_l0 @ W_u_neg\n",
    "    return W_l_tilde, W_u_tilde\n",
    "\n",
    "\n",
    "class Deeppoly():\n",
    "    def __init__(self, init_lower_bound: torch.Tensor, init_upper_bound: torch.Tensor):\n",
    "        # numerical boundaries\n",
    "        self.lower_bounds: list[torch.Tensor] = [init_lower_bound]\n",
    "        self.upper_bounds: list[torch.Tensor] = [init_upper_bound]\n",
    "        # the lock mechanism will be buggy if there's skip in skip, which is not the case for our nets\n",
    "        # the lock mechanism is to ensure that the linear layer absorbed does not cause problem for skip\n",
    "        self.lock_idx : Optional[int] = None\n",
    "        # linear constraint expressions can be stored as matrices multipliers (x' \\leq Ax + b)\n",
    "        # x' <> x (A| 0)\n",
    "        #         (b| 1)\n",
    "        self.lower_const_weights: list[torch.Tensor] = []\n",
    "        self.upper_const_weights: list[torch.Tensor] = []\n",
    "\n",
    "        self.alpha_buffer: list[torch.nn.Parameter] = []\n",
    "        self.alpha_buffer_idx: int = 0\n",
    "\n",
    "        self.forward_net = {\n",
    "            torch.nn.Sequential: self._sequential,\n",
    "            SkipBlock: self._skip,\n",
    "            torch.nn.ReLU6: self._relu6,\n",
    "        }\n",
    "\n",
    "        self.forward_layer = {\n",
    "            torch.nn.Conv2d: self._conv2d,\n",
    "            torch.nn.Linear: self._linear,\n",
    "            torch.nn.ReLU: self._relu,\n",
    "            torch.nn.Flatten: self._flatten,\n",
    "        }\n",
    "\n",
    "\n",
    "    def rewind(self):\n",
    "        self.lower_bounds = self.lower_bounds[:1]\n",
    "        self.upper_bounds = self.upper_bounds[:1]\n",
    "        self.lower_const_weights = []\n",
    "        self.upper_const_weights = []\n",
    "        self.lock_idx = None\n",
    "        self.alpha_buffer_idx = 0\n",
    "        # self.alpha_buffer = []\n",
    "\n",
    "\n",
    "    def lock(self):\n",
    "        self.lock_idx = len(self.upper_bounds)-1\n",
    "\n",
    "\n",
    "    def unlock(self):\n",
    "        self.lock_idx = None\n",
    "\n",
    "\n",
    "    def _prepare_input(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        return torch.cat((x.flatten(start_dim=1), torch.ones(x.shape[0], 1)), dim=1).unsqueeze(1)\n",
    "\n",
    "\n",
    "    def _get_output(self, x: torch.Tensor, shape: tuple[int, ...]) -> torch.Tensor:\n",
    "        return x[:, :, :-1].view(shape)\n",
    "\n",
    "\n",
    "    def _get_bounds(self, lower_bound: torch.Tensor, upper_bound: torch.Tensor, W_l: torch.Tensor, W_u: torch.Tensor, outshape: tuple[int, ...]) -> tuple[torch.Tensor, torch.Tensor]:\n",
    "        lower_bound = self._prepare_input(lower_bound)\n",
    "        upper_bound = self._prepare_input(upper_bound)\n",
    "        W_l_pos = W_l.clamp_min(0)\n",
    "        W_l_neg = W_l.clamp_max(0)\n",
    "        W_u_pos = W_u.clamp_min(0)\n",
    "        W_u_neg = W_u.clamp_max(0)\n",
    "        return self._get_output(lower_bound @ W_l_pos + upper_bound @ W_l_neg, outshape), \\\n",
    "                self._get_output(upper_bound @ W_u_pos + lower_bound @ W_u_neg, outshape)\n",
    "\n",
    "\n",
    "    def forward(self, model: torch.nn.Module):\n",
    "        if type(model) in self.forward_net:\n",
    "            self.forward_net[type(model)](model)\n",
    "        else:\n",
    "            raise NotImplementedError(f\"{type(model)} is not supported\")\n",
    "\n",
    "\n",
    "    def append_constraints(self, bounds: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]):\n",
    "        l_bound, u_bound, l_const_weight, u_const_weight = bounds\n",
    "        if DEBUG:\n",
    "            assert not torch.isnan(l_bound).any(), \"NaN detected in lower bound\"\n",
    "            assert not torch.isnan(u_bound).any(), \"NaN detected in upper bound\"\n",
    "            assert not torch.isnan(l_const_weight).any(), \"NaN detected in lower constraint weights\"\n",
    "            assert not torch.isnan(u_const_weight).any(), \"NaN detected in upper constraint weights\"\n",
    "        if DEBUG:\n",
    "            assert (l_bound > u_bound).sum() == 0\n",
    "        idx = len(self.upper_bounds)-1\n",
    "        if idx > 0 and self.lock_idx != idx and torch.allclose(self.lower_const_weights[-1], self.upper_const_weights[-1]):\n",
    "            W_l, W_u = backsubstitute(self.lower_const_weights[-1], self.upper_const_weights[-1], l_const_weight, u_const_weight)\n",
    "            new_bounds = self._get_bounds(self.lower_bounds[-2], self.upper_bounds[-2], W_l, W_u, l_bound.shape)\n",
    "            self.lower_bounds[-1] = torch.max(new_bounds[0], l_bound)\n",
    "            self.upper_bounds[-1] = torch.min(new_bounds[1], u_bound)\n",
    "            self.lower_const_weights[-1] = W_l\n",
    "            self.upper_const_weights[-1] = W_u\n",
    "        else:\n",
    "            self.lower_bounds.append(l_bound)\n",
    "            self.upper_bounds.append(u_bound)\n",
    "            self.lower_const_weights.append(l_const_weight)\n",
    "            self.upper_const_weights.append(u_const_weight)\n",
    "        self._upd_from_back()\n",
    "\n",
    "\n",
    "    def append_and_verify(self, true_output: int) -> bool:\n",
    "        last_layer = eliding_w(self.upper_bounds[-1].shape[1], true_output)\n",
    "        self.append_constraints(self._affine(self.lower_bounds[-1], self.upper_bounds[-1], last_layer))\n",
    "        n_batch = self.upper_bounds[-1].shape[0]\n",
    "        return any([all(self.upper_bounds[-1][b] <= 0) for b in range(n_batch)])\n",
    "\n",
    "\n",
    "    def _upd_from_back(self):\n",
    "        W_l = self.lower_const_weights[-1]\n",
    "        W_u = self.upper_const_weights[-1]\n",
    "        out_shape = self.lower_bounds[-1].shape\n",
    "        for i in range(len(self.upper_bounds)-3, -1, -1):\n",
    "            W_l, W_u = backsubstitute(self.lower_const_weights[i], self.upper_const_weights[i], W_l, W_u)\n",
    "            new_bounds = self._get_bounds(self.lower_bounds[i], self.upper_bounds[i], W_l, W_u, out_shape)\n",
    "            if DEBUG:\n",
    "                assert (new_bounds[0] > new_bounds[1]).sum() == 0\n",
    "            if DEBUG:\n",
    "                assert (self.lower_bounds[-1] > self.upper_bounds[-1]).sum() == 0, (self.lower_bounds[-1] - self.upper_bounds[-1]).max()\n",
    "            self.lower_bounds[-1] = torch.max(new_bounds[0], self.lower_bounds[-1])\n",
    "            self.upper_bounds[-1] = torch.min(new_bounds[1], self.upper_bounds[-1])\n",
    "            if DEBUG:\n",
    "                assert (self.lower_bounds[-1] > self.upper_bounds[-1]).sum() == 0, (self.lower_bounds[-1] - self.upper_bounds[-1]).max()\n",
    "\n",
    "\n",
    "    def _sequential(self, net: torch.nn.Sequential) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        for i, module in enumerate(net.children()):\n",
    "            if type(module) in self.forward_layer:\n",
    "                self.append_constraints(self.forward_layer[type(module)](self.lower_bounds[-1], self.upper_bounds[-1], module))\n",
    "                if DEBUG:\n",
    "                    assert (self.lower_bounds[-1] > self.upper_bounds[-1]).sum() == 0\n",
    "            else:\n",
    "                self.forward_net[type(module)](module)\n",
    "\n",
    "\n",
    "    def _relu(self, l_bound: torch.Tensor, u_bound: torch.Tensor, layer: torch.nn.ReLU) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        n_batch = u_bound.shape[0]\n",
    "        l_flatten = l_bound.flatten(start_dim=1)\n",
    "        u_flatten = u_bound.flatten(start_dim=1)\n",
    "        l_new = l_bound.clamp(min=0)\n",
    "        u_new = u_bound.clamp(min=0)\n",
    "\n",
    "        above_diag = torch.ones_like(u_flatten) * (l_flatten >= 0)\n",
    "        above_diag = torch.concat([above_diag, torch.zeros(n_batch, 1)], dim=1)\n",
    "        u_above_weight = torch.diag_embed(above_diag)\n",
    "        l_above_weight = torch.diag_embed(above_diag)\n",
    "\n",
    "        dominator = torch.where(\n",
    "            (u_flatten > 0) & (l_flatten < 0), (u_flatten - l_flatten + EPS) ** -1,\n",
    "            0.0\n",
    "        )\n",
    "        upper_slope = u_flatten * dominator\n",
    "        upper_slope = torch.concat([upper_slope, torch.zeros(n_batch, 1)], dim=1)\n",
    "        u_cross_weight = torch.diag_embed(upper_slope)\n",
    "        u_cross_weight[:, -1, :-1] = -l_flatten * upper_slope[:, :-1]\n",
    "\n",
    "        l_cross_weight = torch.zeros_like(l_above_weight)\n",
    "        if self.alpha_buffer_idx == len(self.alpha_buffer):\n",
    "            # lower_slope = torch.nn.Parameter(INIT_COEF * torch.where(u_flatten.abs() > l_flatten.abs(), 1.0, -1.0))\n",
    "            lower_slope = torch.nn.Parameter(torch.randn(u_flatten.size()) * torch.where(u_flatten.abs() > l_flatten.abs(), 1.0, -1.0))\n",
    "            self.alpha_buffer.append(lower_slope)\n",
    "        else:\n",
    "            lower_slope = self.alpha_buffer[self.alpha_buffer_idx]\n",
    "\n",
    "        l_cross_weight[:, :-1, :-1] = torch.diag_embed(torch.where(((u_flatten > 0) & (l_flatten < 0)), torch.sigmoid(lower_slope), 0.0))\n",
    "        self.alpha_buffer_idx += 1\n",
    "\n",
    "        l_weight = l_above_weight + l_cross_weight\n",
    "        u_weight = u_above_weight + u_cross_weight\n",
    "        l_weight[:, -1, -1] = u_weight[:, -1, -1] = 1\n",
    "        if DEBUG:\n",
    "            assert not torch.isnan(l_new).any(), \"NaN detected in lower bound\"\n",
    "            assert not torch.isnan(u_new).any(), \"NaN detected in upper bound\"\n",
    "            assert not torch.isnan(l_weight).any(), \"NaN detected in lower constraint weights\"\n",
    "            assert not torch.isnan(u_weight).any(), \"NaN detected in upper constraint weights\"\n",
    "        return l_new, u_new, l_weight, u_weight\n",
    "\n",
    "\n",
    "    def _relu6(self, module: torch.nn.ReLU6) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        def six_minus(t: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "            l_bound, u_bound = 6 - t[1], 6 - t[0]\n",
    "            l_const_weight = torch.zeros_like(t[2])\n",
    "            l_const_weight[:, :-1, :-1] = -t[3][:, :-1, :-1]\n",
    "            l_const_weight[:, -1, :-1] = 6-t[3][:, -1, :-1]\n",
    "            l_const_weight[:, -1, -1] = 1\n",
    "            u_const_weight = torch.zeros_like(t[3])\n",
    "            u_const_weight[:, :-1, :-1] = -t[2][:, :-1, :-1]\n",
    "            u_const_weight[:, -1, :-1] = 6-t[2][:, -1, :-1]\n",
    "            u_const_weight[:, -1, -1] = 1\n",
    "            if DEBUG:\n",
    "                assert not torch.isnan(l_bound).any(), \"NaN detected in lower bound\"\n",
    "                assert not torch.isnan(u_bound).any(), \"NaN detected in upper bound\"\n",
    "                assert not torch.isnan(l_const_weight).any(), \"NaN detected in lower constraint weights\"\n",
    "                assert not torch.isnan(u_const_weight).any(), \"NaN detected in upper constraint weights\"\n",
    "            return l_bound, u_bound, l_const_weight, u_const_weight\n",
    "\n",
    "        self.append_constraints(six_minus(self._relu(self.lower_bounds[-1], self.upper_bounds[-1], None)))\n",
    "        self.append_constraints(six_minus(self._relu(self.lower_bounds[-1], self.upper_bounds[-1], None)))\n",
    "\n",
    "\n",
    "    def _conv2d(\n",
    "        self, l_bound: torch.Tensor, u_bound: torch.Tensor, layer: torch.nn.Conv2d\n",
    "    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        if DEBUG:\n",
    "            assert (l_bound > u_bound).sum() == 0\n",
    "        # shadow the setting and weight of conv2d layer\n",
    "        w_kernel = layer.weight\n",
    "        b = layer.bias\n",
    "        stride = layer.stride\n",
    "        padding = layer.padding\n",
    "        assert layer.dilation == (1, 1), \"Dilation is not supported\"\n",
    "        assert layer.padding_mode == \"zeros\", \"Padding mode other than zeros is not supported\"\n",
    "        _, in_channel, in_height, in_width = u_bound.shape\n",
    "\n",
    "        # Masking positive and negative weights separately\n",
    "        w_kernel_postive = w_kernel.clamp(min=0)\n",
    "        w_kernel_negative = w_kernel.clamp(max=0)\n",
    "        if DEBUG:\n",
    "            assert torch.allclose(\n",
    "                F.conv2d(l_bound, w_kernel, b, stride, padding),\n",
    "                layer(l_bound),\n",
    "            ), (F.conv2d(l_bound, w_kernel, b, stride, padding) - layer(l_bound)).abs().max()\n",
    "\n",
    "        # conv is basically shifting affine, therefore we can directly reuse F.conv2d with\n",
    "        # bias terms are assembled in the conv2d's bias argument\n",
    "        l_new = F.conv2d(\n",
    "            l_bound, w_kernel_postive, b, stride, padding\n",
    "        ) + F.conv2d(u_bound, w_kernel_negative, None, stride, padding)\n",
    "        u_new = F.conv2d(\n",
    "            u_bound, w_kernel_postive, b, stride, padding\n",
    "        ) + F.conv2d(l_bound, w_kernel_negative, None, stride, padding)\n",
    "        if DEBUG:\n",
    "            assert (l_new > u_new).sum() == 0\n",
    "\n",
    "        _, out_channel, out_height, out_width = u_new.shape\n",
    "\n",
    "        # calculate the linear constraint\n",
    "        conv_mat = torch.zeros(\n",
    "            in_channel * in_height * in_width + 1,\n",
    "            out_channel * out_height * out_width + 1,\n",
    "        )\n",
    "        input = torch.eye(in_channel * in_height * in_width).reshape(-1, in_channel, in_height, in_width)\n",
    "        conv_mat[:-1, :-1] = layer(input).flatten(start_dim=1)\n",
    "        conv_mat[-1, :-1] = b.repeat(out_height * out_width, 1).T.flatten()\n",
    "        conv_mat[:-1, :-1] -= conv_mat[-1, None, :-1]\n",
    "        conv_mat[-1, -1] = 1\n",
    "        if DEBUG:\n",
    "            input = u_bound\n",
    "            output = layer(l_bound)\n",
    "            output_prime = self._get_output(self._prepare_input(l_bound) @ conv_mat, output.shape)\n",
    "            assert torch.allclose(output, output_prime, atol=1), (output - output_prime).abs().max()\n",
    "            l_new_prime = self._get_output(self._prepare_input(l_bound) @ conv_mat.clamp_min(0) + self._prepare_input(u_bound) @ conv_mat.clamp_max(0), l_new.shape)\n",
    "            u_new_prime = self._get_output(self._prepare_input(u_bound) @ conv_mat.clamp_min(0) + self._prepare_input(l_bound) @ conv_mat.clamp_max(0), u_new.shape)\n",
    "            assert torch.allclose(l_new, l_new_prime, atol=1), (l_new - l_new_prime).abs().max()\n",
    "            assert torch.allclose(u_new, u_new_prime, atol=1), (u_new - u_new_prime).abs().max()\n",
    "        return l_new, u_new, conv_mat, conv_mat\n",
    "\n",
    "\n",
    "    def _flatten(self, l_bound: torch.Tensor, u_bound: torch.Tensor, layer: torch.nn.Flatten) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        n_batch = u_bound.shape[0]\n",
    "        l_new = layer(l_bound)\n",
    "        u_new = layer(u_bound)\n",
    "        \n",
    "        l_const_weight = torch.eye(l_new.shape[1]+1).repeat(n_batch, 1, 1)\n",
    "        u_const_weight = torch.eye(u_new.shape[1]+1).repeat(n_batch, 1, 1)\n",
    "\n",
    "        return l_new, u_new, l_const_weight, u_const_weight\n",
    "\n",
    " \n",
    "    def _skip(self, net: torch.nn.Module) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        self.lock()\n",
    "        upd_from = len(self.upper_bounds)-1\n",
    "        skipdim = self.upper_bounds[-1].shape[1]\n",
    "        for i, module in enumerate(net.path.children()):\n",
    "            if type(module) in self.forward_layer:\n",
    "                self.append_constraints(self.forward_layer[type(module)](self.lower_bounds[-1], self.upper_bounds[-1], module))\n",
    "            else:\n",
    "                self.forward_net[type(module)](module)\n",
    "        def carry(bound0: torch.Tensor, bound1: torch.Tensor, const_weights: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:\n",
    "            new_bound = torch.cat([bound1, bound0[:, -skipdim:]], dim=1)\n",
    "            new_weights = torch.zeros(const_weights.shape[0], bound0.shape[1]+1, new_bound.shape[1]+1)\n",
    "            new_weights[:, :const_weights.shape[1]-1, :const_weights.shape[2]-1] = const_weights[:, :-1, :-1]\n",
    "            new_weights[:, -1, :-skipdim-1] = const_weights[:, -1, :-1]\n",
    "            new_weights[:, -skipdim-1:, -skipdim-1:] += torch.eye(skipdim+1)\n",
    "            return new_bound, new_weights\n",
    "        for i in range(upd_from+1, len(self.upper_bounds)):\n",
    "            self.lower_bounds[i], self.lower_const_weights[i-1] = carry(self.lower_bounds[i-1], self.lower_bounds[i], self.lower_const_weights[i-1])\n",
    "            self.upper_bounds[i], self.upper_const_weights[i-1] = carry(self.upper_bounds[i-1], self.upper_bounds[i], self.upper_const_weights[i-1])\n",
    "        self.unlock()\n",
    "        self.append_constraints(self._affine(self.lower_bounds[-1], self.upper_bounds[-1], skip_application(skipdim)))\n",
    "\n",
    " \n",
    "    def _linear(self, l_bound: torch.Tensor, u_bound: torch.Tensor, layer: torch.nn.Linear) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        return self._affine(l_bound, u_bound, layer.weight, layer.bias)\n",
    "    \n",
    "\n",
    "    def _affine(self, l_bound: torch.Tensor, u_bound: torch.Tensor, w: torch.nn.Module, b: Optional[torch.nn.Module] = None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:\n",
    "        n_batch = u_bound.shape[0]\n",
    "        w = w.T\n",
    "\n",
    "        w_postive = w.clamp(min=0)\n",
    "        w_negative = w.clamp(max=0)\n",
    "\n",
    "        l_new = l_bound @ w_postive + u_bound @ w_negative\n",
    "        u_new = u_bound @ w_postive + l_bound @ w_negative\n",
    "        const_weight = torch.zeros(n_batch, w.shape[0]+1, w.shape[1]+1)\n",
    "        const_weight[:, :-1, :-1] = w\n",
    "        const_weight[:, -1, -1] = 1\n",
    "\n",
    "        if b is not None:\n",
    "            const_weight[:, -1, :-1] = b\n",
    "            l_new += b\n",
    "            u_new += b\n",
    "\n",
    "        return l_new, u_new, const_weight, const_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**can modify dp_verify here**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VERBOSE = True\n",
    "def dp_verify(model: torch.nn.Module, input: torch.Tensor, eps: float, true_output: int) -> bool:\n",
    "    input = input.unsqueeze(0)\n",
    "    shallowuni = Deeppoly(input-eps, input+eps)\n",
    "    shallowuni.forward(model)\n",
    "    if shallowuni.append_and_verify(true_output):\n",
    "        return True\n",
    "    if shallowuni.alpha_buffer_idx == 0:\n",
    "        return False\n",
    "    for i in range(shallowuni.alpha_buffer_idx):\n",
    "        shallowuni.alpha_buffer[i].requires_grad = True\n",
    "\n",
    "    if DEBUG:\n",
    "        torch.autograd.set_detect_anomaly(True)\n",
    "\n",
    "    optimizer = torch.optim.Adam(shallowuni.alpha_buffer, lr=LR)\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=DECAY_STEP, gamma=DECAY)\n",
    "    for epoch in range(EPOCHS):\n",
    "        optimizer.zero_grad()\n",
    "        shallowuni.rewind()\n",
    "        shallowuni.forward(model)\n",
    "        if shallowuni.append_and_verify(true_output):\n",
    "            return True\n",
    "        loss = torch.mean(torch.relu(shallowuni.upper_bounds[-1]).flatten())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if VERBOSE:\n",
    "            print(f\"Epoch {epoch+1}, Loss: {loss.item()}\")\n",
    "            print(sum(shallowuni.alpha_buffer[-1].grad))\n",
    "        if scheduler.get_last_lr()[0] > 0.1:\n",
    "            scheduler.step()\n",
    "    return False\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case Recording\n",
    "\n",
    "**| model | testing_case | ground_truth || testing_result | others**\n",
    "\n",
    "- conv_linear | img_mnist_0.072113.txt | not verified ||\n",
    "- conv_linear | img_mnist_0.079101.txt | verified     ||\n",
    "- fc_linear | img_mnist_0.077340.txt | not verified   ||\n",
    "- fc_linear | img_mnist_0.082864.txt | verified       ||\n",
    "- fc6_d | img_mnist_0.089335.txt | not verified       ||\n",
    "- fc6_d | img_mnist_0.050621.txt | verified           ||\n",
    "- fc6_w | img_mnist_0.124546.txt | not verified       ||\n",
    "- fc6_w | img_mnist_0.042779.txt | verified           ||\n",
    "- fc_dw | img_mnist_0.071216.txt | not verified       ||\n",
    "- fc_dw | img_mnist_0.030104.txt | verified           ||\n",
    "- fc6_base | img_mnist_0.074530.txt | not verified    ||\n",
    "- fc6_base | img_mnist_0.044957.txt | verified        ||\n",
    "- fc_base | img_mnist_0.048508.txt | not verified     ||    not | no gradient problem\n",
    "- fc_base | img_mnist_0.048839.txt | verified         ||    not | no gradient problem\n",
    "- conv6_base | img_cifar10_0.003450.txt | not verified||\n",
    "- conv6_base | img_cifar10_0.003959.txt | verified    ||\n",
    "- fc6_dw | img_mnist_0.082344.txt | not verified      ||\n",
    "- fc6_dw | img_mnist_0.034540.txt | verified          ||\n",
    "- fc_d | img_mnist_0.059808.txt | not verified        ||\n",
    "- fc_d | img_mnist_0.040164.txt | verified            ||\n",
    "- fc_w | img_mnist_0.110494.txt | not verified        ||\n",
    "- fc_w | img_mnist_0.042224.txt | verified            ||\n",
    "- conv_d | img_mnist_0.103841.txt | not verified      ||    \n",
    "- conv_d | img_mnist_0.078071.txt | verified          ||    \n",
    "- conv_base | img_mnist_0.075798.txt | not verified   ||    not verified (√)\n",
    "- conv_base | img_mnist_0.055929.txt | verified       ||    verified (√)\n",
    "- skip6 | img_mnist_0.102399.txt | not verified       ||\n",
    "- skip6 | img_mnist_0.074744.txt | verified           ||\n",
    "- skip_large | img_mnist_0.131450.txt | not verified  ||\n",
    "- skip_large | img_mnist_0.041155.txt | verified      ||\n",
    "- skip | img_mnist_0.139199.txt | not verified        ||  \n",
    "- skip | img_mnist_0.078863.txt | verified            ||\n",
    "- skip6_large | img_mnist_0.068828.txt | not verified ||\n",
    "- skip6_large | img_mnist_0.019448.txt | verified     ||"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "img_mnist_0.048508.txt\n",
      "Sequential(\n",
      "  (0): Flatten(start_dim=1, end_dim=-1)\n",
      "  (1): Linear(in_features=784, out_features=50, bias=True)\n",
      "  (2): ReLU()\n",
      "  (3): Linear(in_features=50, out_features=50, bias=True)\n",
      "  (4): ReLU()\n",
      "  (5): Linear(in_features=50, out_features=50, bias=True)\n",
      "  (6): ReLU()\n",
      "  (7): Linear(in_features=50, out_features=10, bias=True)\n",
      ")\n",
      "Epoch 1, Loss: 18.246410369873047\n",
      "tensor([ 0.0000e+00, -7.4184e-03,  0.0000e+00,  0.0000e+00,  4.3374e-03,\n",
      "        -2.3402e-02,  3.5677e-04,  0.0000e+00,  2.8486e-02,  0.0000e+00,\n",
      "         0.0000e+00, -6.3798e-02,  4.0010e-01, -2.0677e-02, -1.3955e-01,\n",
      "        -3.0210e-02, -4.2865e-02,  7.6041e-03, -4.6965e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.1901e-03, -2.0857e-02,\n",
      "         1.5447e-02,  4.6962e-04, -9.5991e-03,  1.7386e-02, -5.2021e-02,\n",
      "        -2.8613e-02,  1.6409e-02,  0.0000e+00, -7.9364e-03, -5.4014e-02,\n",
      "         5.6496e-04, -2.2199e-03,  0.0000e+00, -2.1288e-01,  0.0000e+00,\n",
      "         1.3848e-02,  2.9042e-02,  1.0694e-03,  0.0000e+00,  0.0000e+00,\n",
      "         4.1653e-01,  1.3638e-02,  3.7498e-02, -7.8138e-03, -2.4383e-02])\n",
      "Epoch 2, Loss: 13.963589668273926\n",
      "tensor([ 0.0000e+00, -1.5720e-02,  0.0000e+00,  0.0000e+00, -5.7171e-04,\n",
      "        -3.5070e-02,  2.0149e-04,  0.0000e+00,  1.4832e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  3.4169e-01, -2.9799e-02, -9.4939e-02,\n",
      "        -3.2721e-02, -2.9307e-02,  4.0998e-03, -3.3528e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -4.2403e-03, -3.2633e-02,\n",
      "         8.9061e-03,  1.7031e-04, -1.1577e-02,  2.0441e-02,  0.0000e+00,\n",
      "        -3.7504e-02,  4.9612e-03,  0.0000e+00, -5.9452e-03,  0.0000e+00,\n",
      "         4.4051e-04, -1.8364e-03,  0.0000e+00, -1.9470e-01,  0.0000e+00,\n",
      "         6.4363e-03,  3.0908e-02, -7.8976e-04,  0.0000e+00,  0.0000e+00,\n",
      "         4.8655e-01,  1.3500e-02,  1.8193e-02, -5.1776e-03, -3.0377e-02])\n",
      "Epoch 3, Loss: 11.37093734741211\n",
      "tensor([ 0.0000e+00, -1.2479e-02,  0.0000e+00,  0.0000e+00, -4.8129e-03,\n",
      "        -4.0165e-02, -2.1431e-05,  0.0000e+00,  1.0884e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  2.4721e-01, -2.8047e-02, -5.5540e-02,\n",
      "        -2.8108e-02, -2.3146e-02,  2.3764e-03, -2.3677e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -3.2577e-03, -3.3272e-02,\n",
      "         2.5656e-03,  3.0802e-05, -9.5369e-03,  2.4679e-03,  0.0000e+00,\n",
      "        -3.6190e-02,  1.1654e-03,  0.0000e+00, -4.1210e-03,  0.0000e+00,\n",
      "         9.0296e-05, -1.1149e-03,  0.0000e+00, -1.4594e-01,  0.0000e+00,\n",
      "         2.8614e-03,  2.5443e-02, -1.7406e-03,  0.0000e+00,  0.0000e+00,\n",
      "         4.3339e-01,  2.8085e-03,  9.3054e-03, -3.2453e-03, -3.0689e-02])\n",
      "Epoch 4, Loss: 9.651741981506348\n",
      "tensor([ 0.0000e+00, -1.0011e-02,  0.0000e+00,  0.0000e+00, -4.6688e-03,\n",
      "        -2.9932e-02, -2.2049e-05,  0.0000e+00,  5.2346e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.6734e-01, -2.2957e-02, -3.4266e-02,\n",
      "        -2.0336e-02, -2.0454e-02,  1.5498e-03, -1.6923e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.4235e-03, -2.8269e-02,\n",
      "         1.0324e-03,  2.4059e-05, -6.7852e-03,  2.3971e-04,  0.0000e+00,\n",
      "        -2.8258e-02,  7.6176e-04,  0.0000e+00, -3.0776e-03,  0.0000e+00,\n",
      "         1.2364e-04, -7.1495e-04,  0.0000e+00, -1.0264e-01,  0.0000e+00,\n",
      "         5.6375e-04,  1.5954e-02, -1.2491e-03,  0.0000e+00,  0.0000e+00,\n",
      "         3.5652e-01,  2.8124e-03,  4.8606e-03, -2.2512e-03, -2.3873e-02])\n",
      "Epoch 5, Loss: 8.533369064331055\n",
      "tensor([ 0.0000e+00, -7.4104e-03,  0.0000e+00,  0.0000e+00, -8.0060e-03,\n",
      "        -2.4644e-02, -4.1672e-05,  0.0000e+00,  3.5908e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.1793e-01, -1.6834e-02, -2.5127e-02,\n",
      "        -1.4769e-02, -1.6521e-02,  1.0670e-03, -1.1635e-02,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.6700e-03, -2.2852e-02,\n",
      "        -5.1179e-05,  8.4571e-05, -4.7583e-03, -4.4203e-03,  0.0000e+00,\n",
      "         0.0000e+00,  4.3708e-04,  0.0000e+00, -2.4090e-03,  0.0000e+00,\n",
      "         9.0666e-05, -5.9900e-04,  0.0000e+00, -7.0798e-02,  0.0000e+00,\n",
      "        -4.4305e-05,  1.1314e-02, -2.0356e-03,  0.0000e+00,  0.0000e+00,\n",
      "         2.6667e-01,  9.2937e-04,  3.6710e-03, -1.5302e-03, -1.9414e-02])\n",
      "Epoch 6, Loss: 7.851622581481934\n",
      "tensor([ 0.0000e+00, -5.4028e-03,  0.0000e+00,  0.0000e+00, -9.6104e-03,\n",
      "        -1.8950e-02, -3.8616e-05,  0.0000e+00,  2.6005e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  8.3268e-02, -1.2151e-02, -1.8837e-02,\n",
      "        -1.0687e-02, -1.3609e-02,  7.7160e-04, -8.3393e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.1860e-03, -1.6163e-02,\n",
      "        -1.2403e-04,  1.1386e-04, -3.3739e-03, -5.6465e-03,  0.0000e+00,\n",
      "         0.0000e+00,  2.0477e-04,  0.0000e+00, -1.5907e-03,  0.0000e+00,\n",
      "         7.9676e-05, -4.7153e-04,  0.0000e+00, -5.0324e-02,  0.0000e+00,\n",
      "        -5.2174e-04,  8.1199e-03, -2.2940e-03,  0.0000e+00,  0.0000e+00,\n",
      "         1.9366e-01,  6.4461e-04,  2.7764e-03, -1.1142e-03, -1.4187e-02])\n",
      "Epoch 7, Loss: 7.412049293518066\n",
      "tensor([ 0.0000e+00, -3.9868e-03,  0.0000e+00,  0.0000e+00, -1.1676e-02,\n",
      "        -1.4350e-02, -4.3800e-05,  0.0000e+00,  2.0352e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  6.2485e-02, -8.8847e-03, -1.4709e-02,\n",
      "         0.0000e+00, -1.0513e-02,  5.8785e-04, -6.1545e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -8.5205e-04, -1.2148e-02,\n",
      "        -1.8925e-04,  8.9094e-05, -2.4657e-03, -6.2338e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.6521e-04,  0.0000e+00, -1.1831e-03,  0.0000e+00,\n",
      "         5.7463e-05, -3.9022e-04,  0.0000e+00, -3.7425e-02,  0.0000e+00,\n",
      "        -4.1933e-04,  5.9614e-03, -2.9618e-03,  0.0000e+00,  0.0000e+00,\n",
      "         1.4589e-01,  4.8201e-04,  2.1768e-03, -8.3305e-04, -1.0864e-02])\n",
      "Epoch 8, Loss: 7.134335517883301\n",
      "tensor([ 0.0000e+00, -3.0271e-03,  0.0000e+00,  0.0000e+00, -1.2294e-02,\n",
      "        -1.1165e-02, -3.9761e-05,  0.0000e+00,  1.7289e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  4.8388e-02, -6.7112e-03, -1.1865e-02,\n",
      "         0.0000e+00, -8.2143e-03,  4.6832e-04, -4.8080e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -6.4820e-04, -9.4081e-03,\n",
      "        -1.8537e-04,  7.4071e-05, -1.8779e-03, -5.9471e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.1501e-04,  0.0000e+00, -8.9416e-04,  0.0000e+00,\n",
      "         5.5311e-05, -3.0019e-04,  0.0000e+00, -2.8879e-02,  0.0000e+00,\n",
      "        -3.8965e-04,  4.5446e-03, -3.2481e-03,  0.0000e+00,  0.0000e+00,\n",
      "         1.1252e-01,  4.1849e-04,  1.7660e-03, -6.5200e-04, -8.3673e-03])\n",
      "Epoch 9, Loss: 6.947489261627197\n",
      "tensor([ 0.0000e+00, -2.3790e-03,  0.0000e+00,  0.0000e+00, -1.2486e-02,\n",
      "        -8.8260e-03, -3.9337e-05,  0.0000e+00,  1.4517e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  3.8894e-02, -5.2360e-03, -9.8418e-03,\n",
      "         0.0000e+00, -6.5660e-03,  3.8840e-04, -3.8661e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -5.0932e-04, -7.4129e-03,\n",
      "        -1.8214e-04,  6.2411e-05, -1.4811e-03, -5.8752e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.7426e-04,  0.0000e+00, -7.2781e-04,  0.0000e+00,\n",
      "         4.9471e-05, -2.4827e-04,  0.0000e+00, -2.3063e-02,  0.0000e+00,\n",
      "        -3.9976e-04,  3.6110e-03, -3.3468e-03,  0.0000e+00,  0.0000e+00,\n",
      "         8.9009e-02,  3.6366e-04,  1.4899e-03, -5.3591e-04, -6.6034e-03])\n",
      "Epoch 10, Loss: 6.817782402038574\n",
      "tensor([ 0.0000e+00, -1.9364e-03,  0.0000e+00,  0.0000e+00, -1.1684e-02,\n",
      "        -7.1201e-03, -4.2615e-05,  0.0000e+00,  1.3316e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  3.2190e-02, -4.1983e-03, -8.3756e-03,\n",
      "         0.0000e+00, -5.4579e-03,  3.3269e-04, -3.1925e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -4.1302e-04, -5.9330e-03,\n",
      "        -1.8314e-04,  4.0780e-05, -1.2096e-03, -6.8311e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.4045e-04,  0.0000e+00, -6.4272e-04,  0.0000e+00,\n",
      "         4.4465e-05, -2.0664e-04,  0.0000e+00, -1.9000e-02,  0.0000e+00,\n",
      "         7.2975e-06,  2.9342e-03, -3.3453e-03,  0.0000e+00,  0.0000e+00,\n",
      "         7.1346e-02,  1.7400e-04,  1.3194e-03, -4.4780e-04, -5.3069e-03])\n",
      "Epoch 11, Loss: 6.712267875671387\n",
      "tensor([ 0.0000e+00, -1.6200e-03,  0.0000e+00,  0.0000e+00, -1.1117e-02,\n",
      "        -5.9365e-03, -4.4661e-05,  0.0000e+00,  1.2226e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  2.7463e-02, -3.4881e-03, -7.3435e-03,\n",
      "         0.0000e+00, -4.5468e-03,  2.9253e-04, -2.7456e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -3.4729e-04, -4.9512e-03,\n",
      "        -1.6195e-04,  3.6575e-05, -1.0186e-03, -7.1730e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.4402e-04,  0.0000e+00, -5.7289e-04,  0.0000e+00,\n",
      "         3.9027e-05, -1.8734e-04,  0.0000e+00, -1.6166e-02,  0.0000e+00,\n",
      "        -3.8456e-05,  2.4775e-03, -3.3143e-03,  0.0000e+00,  0.0000e+00,\n",
      "         5.9733e-02,  1.2009e-04,  1.1903e-03, -3.8717e-04, -4.4024e-03])\n",
      "Epoch 12, Loss: 6.645247459411621\n",
      "tensor([ 0.0000e+00, -1.3872e-03,  0.0000e+00,  0.0000e+00, -9.4854e-03,\n",
      "        -5.0490e-03, -3.4087e-05,  0.0000e+00,  1.1640e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  2.4027e-02, -2.9720e-03, -6.5669e-03,\n",
      "         0.0000e+00, -3.9347e-03,  2.6100e-04, -2.4090e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -3.0039e-04, -4.2392e-03,\n",
      "        -1.4831e-04,  3.2745e-05, -8.8021e-04, -7.2949e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.3462e-04,  0.0000e+00, -4.9584e-04,  0.0000e+00,\n",
      "         3.6538e-05, -1.6395e-04,  0.0000e+00, -1.4143e-02,  0.0000e+00,\n",
      "        -6.9760e-05,  2.1422e-03, -3.0564e-03,  0.0000e+00,  0.0000e+00,\n",
      "         5.1468e-02,  1.1386e-04,  1.0777e-03, -3.4170e-04, -3.7637e-03])\n",
      "Epoch 13, Loss: 6.59475564956665\n",
      "tensor([ 0.0000e+00, -1.2165e-03,  0.0000e+00,  0.0000e+00, -8.3831e-03,\n",
      "        -4.3963e-03, -3.3982e-05,  0.0000e+00,  1.1342e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  2.1475e-02, -2.5865e-03, -5.9917e-03,\n",
      "         0.0000e+00, -3.4555e-03,  2.3892e-04, -2.1723e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.6552e-04, -3.7014e-03,\n",
      "        -1.4627e-04,  3.0631e-05, -7.7816e-04, -7.6297e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.3085e-04,  0.0000e+00, -4.4554e-04,  0.0000e+00,\n",
      "         3.4125e-05, -1.4583e-04,  0.0000e+00, -1.2595e-02,  0.0000e+00,\n",
      "        -6.9751e-05,  1.9230e-03, -2.7930e-03,  0.0000e+00,  0.0000e+00,\n",
      "         4.5178e-02,  9.2738e-05,  9.8975e-04, -3.0435e-04, -3.2810e-03])\n",
      "Epoch 14, Loss: 6.556332588195801\n",
      "tensor([ 0.0000e+00, -1.0735e-03,  0.0000e+00,  0.0000e+00, -6.9668e-03,\n",
      "        -3.8169e-03, -3.7400e-05,  0.0000e+00,  1.1077e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.9421e-02, -2.3085e-03, -5.4909e-03,\n",
      "         0.0000e+00, -3.1587e-03,  2.2073e-04, -1.9724e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.4184e-04, -3.0991e-03,\n",
      "        -1.1747e-04,  2.8306e-05, -7.0509e-04, -7.9182e-03,  0.0000e+00,\n",
      "         0.0000e+00,  3.7377e-05,  0.0000e+00, -4.0084e-04,  0.0000e+00,\n",
      "         3.2498e-05, -1.3196e-04,  0.0000e+00, -1.1497e-02,  0.0000e+00,\n",
      "        -6.9653e-05,  1.7423e-03, -2.2965e-03,  0.0000e+00,  0.0000e+00,\n",
      "         4.0230e-02,  3.1691e-05,  9.3765e-04, -2.7498e-04, -2.7523e-03])\n",
      "Epoch 15, Loss: 6.526633262634277\n",
      "tensor([ 0.0000e+00, -9.7744e-04,  0.0000e+00,  0.0000e+00, -6.1622e-03,\n",
      "        -3.4796e-03, -3.4230e-05,  0.0000e+00,  1.0498e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.7866e-02, -2.0969e-03, -5.1199e-03,\n",
      "         0.0000e+00, -2.8933e-03,  2.0706e-04, -1.8361e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.2047e-04, -2.8134e-03,\n",
      "        -1.1377e-04,  2.6341e-05, -6.4558e-04, -8.2120e-03,  0.0000e+00,\n",
      "         0.0000e+00,  4.4967e-05,  0.0000e+00, -3.5850e-04,  0.0000e+00,\n",
      "         3.1332e-05, -1.1379e-04,  0.0000e+00, -1.0610e-02,  0.0000e+00,\n",
      "        -5.4335e-05,  1.6057e-03, -2.0728e-03,  0.0000e+00,  0.0000e+00,\n",
      "         3.6650e-02,  3.9822e-05,  8.8509e-04, -2.5580e-04, -2.4726e-03])\n",
      "Epoch 16, Loss: 6.503363609313965\n",
      "tensor([ 0.0000e+00, -9.0656e-04,  0.0000e+00,  0.0000e+00, -5.6425e-03,\n",
      "        -3.2061e-03, -3.6130e-05,  0.0000e+00,  9.8286e-04,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.6680e-02, -1.9334e-03, -4.8060e-03,\n",
      "         0.0000e+00, -2.6535e-03,  1.9624e-04, -1.7208e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -2.0409e-04, -2.6008e-03,\n",
      "        -1.0786e-04,  2.4753e-05, -5.9992e-04, -8.5059e-03,  0.0000e+00,\n",
      "         0.0000e+00,  6.4885e-05,  0.0000e+00, -3.3626e-04,  0.0000e+00,\n",
      "         2.9264e-05, -1.0481e-04,  0.0000e+00, -9.9091e-03,  0.0000e+00,\n",
      "        -6.3039e-05,  1.5021e-03, -1.8841e-03,  0.0000e+00,  0.0000e+00,\n",
      "         3.3857e-02,  3.7227e-05,  8.4815e-04, -2.4069e-04, -2.2768e-03])\n",
      "Epoch 17, Loss: 6.484923362731934\n",
      "tensor([ 0.0000e+00, -8.4869e-04,  0.0000e+00,  0.0000e+00, -4.9022e-03,\n",
      "        -2.9823e-03, -3.1396e-05,  0.0000e+00,  9.6411e-04,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.5760e-02, -1.8039e-03, -4.5749e-03,\n",
      "         0.0000e+00, -2.4857e-03,  1.8687e-04, -1.6480e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.9222e-04, -2.4271e-03,\n",
      "        -1.0893e-04,  2.3643e-05, -5.6259e-04, -8.4412e-03,  0.0000e+00,\n",
      "         0.0000e+00,  3.2465e-05,  0.0000e+00, -3.1923e-04,  0.0000e+00,\n",
      "         2.8081e-05, -9.8488e-05,  0.0000e+00, -9.3522e-03,  0.0000e+00,\n",
      "        -8.3602e-05,  1.4224e-03, -1.7105e-03,  0.0000e+00,  0.0000e+00,\n",
      "         3.1665e-02,  5.9101e-05,  8.1957e-04, -2.2864e-04, -2.1230e-03])\n",
      "Epoch 18, Loss: 6.470113277435303\n",
      "tensor([ 0.0000e+00, -8.0196e-04,  0.0000e+00,  0.0000e+00, -4.4048e-03,\n",
      "        -2.8014e-03, -2.6396e-05,  0.0000e+00,  9.3562e-04,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.4968e-02, -1.7016e-03, -4.4052e-03,\n",
      "         0.0000e+00, -2.4002e-03,  1.7973e-04, -1.5842e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.8210e-04, -2.2720e-03,\n",
      "        -9.6545e-05,  2.3364e-05, -5.3389e-04, -7.5850e-03,  0.0000e+00,\n",
      "         0.0000e+00,  6.7007e-06,  0.0000e+00, -2.9750e-04,  0.0000e+00,\n",
      "         2.6968e-05, -9.3432e-05,  0.0000e+00, -8.9122e-03,  0.0000e+00,\n",
      "        -1.1898e-04,  1.3358e-03, -1.5482e-03,  0.0000e+00,  0.0000e+00,\n",
      "         2.9808e-02,  9.3532e-05,  8.2195e-04, -2.1744e-04, -1.9959e-03])\n",
      "Epoch 19, Loss: 6.458132266998291\n",
      "tensor([ 0.0000e+00, -7.6261e-04,  0.0000e+00,  0.0000e+00, -3.9642e-03,\n",
      "        -2.6489e-03, -2.3021e-05,  0.0000e+00,  9.0756e-04,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.4408e-02, -1.6202e-03, -4.2675e-03,\n",
      "         0.0000e+00, -2.2954e-03,  1.7392e-04, -1.5216e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.7395e-04, -2.1426e-03,\n",
      "        -1.0566e-04,  2.2286e-05, -5.1074e-04, -7.7604e-03,  0.0000e+00,\n",
      "         0.0000e+00, -6.1056e-06,  0.0000e+00, -2.8006e-04,  0.0000e+00,\n",
      "         2.6587e-05, -9.1023e-05,  0.0000e+00, -8.5435e-03,  0.0000e+00,\n",
      "        -1.1584e-04,  1.2772e-03, -1.3874e-03,  0.0000e+00,  0.0000e+00,\n",
      "         2.8410e-02,  8.8004e-05,  7.9709e-04, -2.0961e-04, -1.8936e-03])\n",
      "Epoch 20, Loss: 6.448392391204834\n",
      "tensor([ 0.0000e+00, -7.3414e-04,  0.0000e+00,  0.0000e+00, -3.3023e-03,\n",
      "        -2.5416e-03, -2.0519e-05,  0.0000e+00,  9.1313e-04,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.3939e-02, -1.5544e-03, -4.1680e-03,\n",
      "         0.0000e+00, -2.2173e-03,  1.6979e-04, -1.4783e-03,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.6725e-04, -2.0524e-03,\n",
      "        -9.9366e-05,  2.1493e-05, -4.9191e-04, -7.9157e-03,  0.0000e+00,\n",
      "         0.0000e+00, -3.7455e-05,  0.0000e+00, -2.6820e-04,  0.0000e+00,\n",
      "         2.6855e-05, -8.9158e-05,  0.0000e+00, -8.2442e-03,  0.0000e+00,\n",
      "        -1.1637e-04,  1.2339e-03, -1.2909e-03,  0.0000e+00,  0.0000e+00,\n",
      "         2.7312e-02,  8.3785e-05,  7.7715e-04, -2.0321e-04, -1.8125e-03])\n",
      "img_mnist_0.048839.txt\n",
      "Sequential(\n",
      "  (0): Flatten(start_dim=1, end_dim=-1)\n",
      "  (1): Linear(in_features=784, out_features=50, bias=True)\n",
      "  (2): ReLU()\n",
      "  (3): Linear(in_features=50, out_features=50, bias=True)\n",
      "  (4): ReLU()\n",
      "  (5): Linear(in_features=50, out_features=50, bias=True)\n",
      "  (6): ReLU()\n",
      "  (7): Linear(in_features=50, out_features=10, bias=True)\n",
      ")\n",
      "Epoch 1, Loss: 8.991628646850586\n",
      "tensor([ 0.0000e+00, -1.2809e-01,  0.0000e+00,  0.0000e+00, -2.7390e-02,\n",
      "         3.9585e-02,  5.7831e-02,  0.0000e+00,  2.4932e-02,  4.8828e-02,\n",
      "         0.0000e+00, -2.3140e-03,  3.4937e-02,  0.0000e+00, -8.0912e-02,\n",
      "        -1.7164e-01,  1.9145e-02,  0.0000e+00, -8.1007e-05,  0.0000e+00,\n",
      "        -3.6966e-03, -1.6869e-02,  0.0000e+00,  0.0000e+00, -1.7066e-01,\n",
      "         4.8712e-03, -5.5523e-03,  0.0000e+00, -6.0117e-03,  0.0000e+00,\n",
      "        -2.2647e-02,  3.0715e-02,  1.5784e-02, -2.9260e-02,  0.0000e+00,\n",
      "         4.9049e-03, -1.5102e-01,  0.0000e+00,  5.3867e-02,  0.0000e+00,\n",
      "        -2.6516e-02, -2.9707e-03, -6.7826e-02,  0.0000e+00,  2.6615e-02,\n",
      "         2.4491e-02,  6.2011e-02,  2.1503e-01,  6.1583e-02,  0.0000e+00])\n",
      "Epoch 2, Loss: 4.8113813400268555\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -2.7640e-02,\n",
      "         6.4102e-03,  5.8099e-02,  0.0000e+00,  1.4434e-02,  4.8783e-02,\n",
      "         0.0000e+00, -5.4204e-03,  9.7608e-03,  0.0000e+00, -8.5928e-02,\n",
      "         0.0000e+00,  1.0153e-02,  0.0000e+00, -4.2750e-04,  0.0000e+00,\n",
      "        -6.8248e-03, -1.1121e-02,  0.0000e+00,  0.0000e+00, -8.5911e-02,\n",
      "         3.0246e-03, -6.1183e-03,  0.0000e+00, -1.1436e-02,  0.0000e+00,\n",
      "        -1.7354e-02,  2.1581e-02,  7.4353e-03, -1.4482e-02,  0.0000e+00,\n",
      "         7.1394e-03,  0.0000e+00,  0.0000e+00,  2.9777e-02,  0.0000e+00,\n",
      "         0.0000e+00, -1.0152e-04, -4.9663e-02,  0.0000e+00,  1.4064e-02,\n",
      "         1.7655e-02,  1.3940e-02,  1.4060e-01,  7.5204e-02,  0.0000e+00])\n",
      "Epoch 3, Loss: 2.7143518924713135\n",
      "tensor([ 0.0000,  0.0000,  0.0000,  0.0000, -0.0230,  0.0009,  0.0444,  0.0000,\n",
      "         0.0089,  0.0431,  0.0000, -0.0024,  0.0076,  0.0000, -0.0739,  0.0000,\n",
      "         0.0045,  0.0000, -0.0001,  0.0000, -0.0027, -0.0058,  0.0000,  0.0000,\n",
      "         0.0000,  0.0014, -0.0048,  0.0000, -0.0054,  0.0000,  0.0000,  0.0135,\n",
      "         0.0054, -0.0053,  0.0000,  0.0078,  0.0000,  0.0000,  0.0174,  0.0000,\n",
      "         0.0000, -0.0010, -0.0260,  0.0000,  0.0075,  0.0000,  0.0015,  0.0445,\n",
      "         0.0823,  0.0000])\n",
      "Epoch 4, Loss: 1.745591402053833\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -6.0588e-03,\n",
      "        -4.4520e-03,  2.8037e-02,  0.0000e+00,  4.0186e-03,  1.9026e-02,\n",
      "         0.0000e+00, -6.0857e-04,  0.0000e+00,  0.0000e+00, -4.0385e-02,\n",
      "         0.0000e+00,  9.5173e-04,  0.0000e+00, -9.4339e-05,  0.0000e+00,\n",
      "        -6.8837e-04, -3.8754e-03,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         7.3908e-04, -3.1899e-03,  0.0000e+00, -4.2578e-03,  0.0000e+00,\n",
      "         0.0000e+00,  9.4916e-03,  5.6576e-03, -6.7645e-04,  0.0000e+00,\n",
      "         6.2142e-03,  0.0000e+00,  0.0000e+00,  1.0079e-02,  0.0000e+00,\n",
      "         0.0000e+00,  1.2648e-03, -2.0530e-02,  0.0000e+00,  1.1748e-03,\n",
      "         0.0000e+00,  1.9596e-03,  1.5753e-02,  6.9544e-02,  0.0000e+00])\n",
      "Epoch 5, Loss: 1.2478663921356201\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -4.9477e-03,\n",
      "        -4.1141e-03,  1.7393e-02,  0.0000e+00,  2.2661e-03,  1.3943e-02,\n",
      "         0.0000e+00, -3.5802e-04,  0.0000e+00,  0.0000e+00, -2.7578e-02,\n",
      "         0.0000e+00,  6.5262e-04,  0.0000e+00,  3.2154e-05,  0.0000e+00,\n",
      "        -5.3340e-04, -2.7016e-03,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         5.4404e-04, -2.2462e-03,  0.0000e+00, -3.3194e-03,  0.0000e+00,\n",
      "         0.0000e+00,  6.6616e-03,  3.6532e-03, -5.2616e-04,  0.0000e+00,\n",
      "         4.6387e-03,  0.0000e+00,  0.0000e+00,  6.0224e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.7682e-03, -1.5618e-02,  0.0000e+00,  7.3113e-04,\n",
      "         0.0000e+00,  1.5184e-03,  2.6978e-03,  5.2185e-02,  0.0000e+00])\n",
      "Epoch 6, Loss: 0.9397454261779785\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -3.6619e-03,\n",
      "        -2.5705e-03,  4.7248e-03,  0.0000e+00,  1.7827e-03,  3.9566e-03,\n",
      "         0.0000e+00, -2.7102e-04,  0.0000e+00,  0.0000e+00, -2.0436e-02,\n",
      "         0.0000e+00, -5.7492e-04,  0.0000e+00,  2.1180e-05,  0.0000e+00,\n",
      "        -3.2196e-04, -2.0502e-03,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         4.2279e-04,  0.0000e+00,  0.0000e+00, -2.5062e-03,  0.0000e+00,\n",
      "         0.0000e+00,  4.8525e-03,  7.7925e-04,  0.0000e+00,  0.0000e+00,\n",
      "         1.9695e-03,  0.0000e+00,  0.0000e+00,  3.7903e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.2660e-03, -8.6525e-03,  0.0000e+00,  5.2498e-04,\n",
      "         0.0000e+00,  1.3091e-03,  2.1014e-03,  3.6346e-02,  0.0000e+00])\n",
      "Epoch 7, Loss: 0.795185387134552\n",
      "tensor([ 0.0000,  0.0000,  0.0000,  0.0000, -0.0028, -0.0027,  0.0024,  0.0000,\n",
      "         0.0014,  0.0030,  0.0000, -0.0001,  0.0000,  0.0000,  0.0000,  0.0000,\n",
      "        -0.0003,  0.0000,  0.0000,  0.0000, -0.0003, -0.0016,  0.0000,  0.0000,\n",
      "         0.0000,  0.0003,  0.0000,  0.0000, -0.0019,  0.0000,  0.0000,  0.0002,\n",
      "         0.0006,  0.0000,  0.0000,  0.0014,  0.0000,  0.0000,  0.0023,  0.0000,\n",
      "         0.0000,  0.0012, -0.0048,  0.0000,  0.0003,  0.0000,  0.0010,  0.0043,\n",
      "         0.0272,  0.0000])\n",
      "Epoch 8, Loss: 0.7244824171066284\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -2.4069e-03,\n",
      "        -2.7612e-03,  3.0197e-03,  0.0000e+00,  1.1256e-03,  1.6040e-03,\n",
      "         0.0000e+00, -8.4400e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -2.1593e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -2.1062e-04, -1.2228e-03,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         2.9083e-04,  0.0000e+00,  0.0000e+00, -1.6900e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.4462e-04,  2.0777e-04,  0.0000e+00,  0.0000e+00,\n",
      "         1.1555e-03,  0.0000e+00,  0.0000e+00,  1.8179e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.7685e-04, -3.2742e-03,  0.0000e+00,  3.5686e-04,\n",
      "         0.0000e+00,  8.3222e-04, -1.0392e-02,  1.9882e-02,  0.0000e+00])\n",
      "Epoch 9, Loss: 0.6789038777351379\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -2.0070e-03,\n",
      "        -2.6562e-03,  2.4589e-03,  0.0000e+00,  9.3297e-04,  1.3178e-03,\n",
      "         0.0000e+00, -7.2250e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -2.4533e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.7652e-04, -9.7334e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         2.4663e-04,  0.0000e+00,  0.0000e+00, -1.3586e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.1568e-04,  1.5763e-04,  0.0000e+00,  0.0000e+00,\n",
      "         9.2190e-04,  0.0000e+00,  0.0000e+00,  1.6544e-03,  0.0000e+00,\n",
      "         0.0000e+00, -4.5643e-05, -2.7815e-03,  0.0000e+00,  3.0366e-04,\n",
      "         0.0000e+00,  7.0035e-04, -1.1226e-02,  1.5979e-02,  0.0000e+00])\n",
      "Epoch 10, Loss: 0.6474916338920593\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.7357e-03,\n",
      "        -2.6544e-03,  2.2666e-03,  0.0000e+00,  8.0661e-04,  1.1238e-03,\n",
      "         0.0000e+00, -6.5598e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.1397e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.5454e-04, -8.4470e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         2.1473e-04,  0.0000e+00,  0.0000e+00, -1.1226e-03,  0.0000e+00,\n",
      "         0.0000e+00,  1.0336e-04,  1.4344e-04,  0.0000e+00,  0.0000e+00,\n",
      "         7.8903e-04,  0.0000e+00,  0.0000e+00,  1.3958e-03,  0.0000e+00,\n",
      "         0.0000e+00, -2.4922e-04, -2.3876e-03,  0.0000e+00,  2.6553e-04,\n",
      "         0.0000e+00,  6.1436e-04, -1.0893e-02,  1.2451e-02,  0.0000e+00])\n",
      "Epoch 11, Loss: 0.6251881718635559\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.5167e-03,\n",
      "        -2.6533e-03,  1.9924e-03,  0.0000e+00,  7.1350e-04,  9.9577e-04,\n",
      "         0.0000e+00, -5.8908e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.1283e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.4230e-04, -7.4907e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.8397e-04,  0.0000e+00,  0.0000e+00, -9.4972e-04,  0.0000e+00,\n",
      "         0.0000e+00,  9.3196e-05,  1.2807e-04,  0.0000e+00,  0.0000e+00,\n",
      "         6.7964e-04,  0.0000e+00,  0.0000e+00,  1.2061e-03,  0.0000e+00,\n",
      "         0.0000e+00, -2.9129e-04, -2.1002e-03,  0.0000e+00,  2.3797e-04,\n",
      "         0.0000e+00,  5.5788e-04, -1.0528e-02,  1.0325e-02,  0.0000e+00])\n",
      "Epoch 12, Loss: 0.6088606715202332\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.3662e-03,\n",
      "        -2.6617e-03,  1.7415e-03,  0.0000e+00,  6.4831e-04,  8.6828e-04,\n",
      "         0.0000e+00, -5.3594e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.1551e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.3190e-04, -6.7079e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.6709e-04,  0.0000e+00,  0.0000e+00, -8.2538e-04,  0.0000e+00,\n",
      "         0.0000e+00,  8.4815e-05,  1.1318e-04,  0.0000e+00,  0.0000e+00,\n",
      "         5.8231e-04,  0.0000e+00,  0.0000e+00,  1.0894e-03,  0.0000e+00,\n",
      "         0.0000e+00, -2.8594e-04, -1.9100e-03,  0.0000e+00,  2.1924e-04,\n",
      "         0.0000e+00,  5.0957e-04, -1.0003e-02,  8.8474e-03,  0.0000e+00])\n",
      "Epoch 13, Loss: 0.5966212153434753\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.2401e-03,\n",
      "        -2.6746e-03,  1.6001e-03,  0.0000e+00,  5.9380e-04,  8.0167e-04,\n",
      "         0.0000e+00, -4.9900e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.2610e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.2256e-04, -6.1159e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.5418e-04,  0.0000e+00,  0.0000e+00, -7.3208e-04,  0.0000e+00,\n",
      "         0.0000e+00,  7.8553e-05,  1.0461e-04,  0.0000e+00,  0.0000e+00,\n",
      "         5.2013e-04,  0.0000e+00,  0.0000e+00,  9.9602e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.5572e-04, -1.7402e-03,  0.0000e+00,  2.0507e-04,\n",
      "         0.0000e+00,  4.7931e-04, -9.6319e-03,  7.7290e-03,  0.0000e+00])\n",
      "Epoch 14, Loss: 0.5872665643692017\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.1475e-03,\n",
      "        -2.7065e-03,  1.4820e-03,  0.0000e+00,  5.4780e-04,  7.4131e-04,\n",
      "         0.0000e+00, -4.7141e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.1483e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.1503e-04, -5.6915e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.4393e-04,  0.0000e+00,  0.0000e+00, -6.6117e-04,  0.0000e+00,\n",
      "         0.0000e+00,  7.4866e-05,  9.7706e-05,  0.0000e+00,  0.0000e+00,\n",
      "         4.6806e-04,  0.0000e+00,  0.0000e+00,  9.1707e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.6006e-04, -1.6236e-03,  0.0000e+00,  1.9415e-04,\n",
      "         0.0000e+00,  4.4916e-04, -9.5258e-03,  6.8101e-03,  0.0000e+00])\n",
      "Epoch 15, Loss: 0.5799870491027832\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.0750e-03,\n",
      "        -2.7531e-03,  1.3904e-03,  0.0000e+00,  5.1847e-04,  6.9864e-04,\n",
      "         0.0000e+00, -4.4944e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.0742e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.0992e-04, -5.3282e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.3580e-04,  0.0000e+00,  0.0000e+00, -6.0629e-04,  0.0000e+00,\n",
      "         0.0000e+00,  7.1101e-05,  9.2369e-05,  0.0000e+00,  0.0000e+00,\n",
      "         4.2954e-04,  0.0000e+00,  0.0000e+00,  8.6562e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.6193e-04, -1.5325e-03,  0.0000e+00,  1.8562e-04,\n",
      "         0.0000e+00,  4.3119e-04, -9.3418e-03,  6.1985e-03,  0.0000e+00])\n",
      "Epoch 16, Loss: 0.5742278099060059\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.0174e-03,\n",
      "        -2.8053e-03,  1.3186e-03,  0.0000e+00,  4.9504e-04,  6.6519e-04,\n",
      "         0.0000e+00, -4.3204e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.0184e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.0578e-04, -5.0395e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.2927e-04,  0.0000e+00,  0.0000e+00, -5.6323e-04,  0.0000e+00,\n",
      "         0.0000e+00,  6.8157e-05,  8.8207e-05,  0.0000e+00,  0.0000e+00,\n",
      "         3.9954e-04,  0.0000e+00,  0.0000e+00,  8.2502e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.6194e-04, -1.4599e-03,  0.0000e+00,  1.7888e-04,\n",
      "         0.0000e+00,  4.1694e-04, -9.2171e-03,  5.7225e-03,  0.0000e+00])\n",
      "Epoch 17, Loss: 0.5696083307266235\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -9.6984e-04,\n",
      "        -2.8635e-03,  1.2604e-03,  0.0000e+00,  4.7599e-04,  6.2841e-04,\n",
      "         0.0000e+00, -4.1932e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -3.8781e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.0243e-04, -4.8077e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.2399e-04,  0.0000e+00,  0.0000e+00, -5.2900e-04,  0.0000e+00,\n",
      "         0.0000e+00,  6.5835e-05,  8.4355e-05,  0.0000e+00,  0.0000e+00,\n",
      "         3.7168e-04,  0.0000e+00,  0.0000e+00,  7.9260e-04,  0.0000e+00,\n",
      "         0.0000e+00, -4.0032e-04, -1.4081e-03,  0.0000e+00,  1.7349e-04,\n",
      "         0.0000e+00,  4.0452e-04, -9.2395e-03,  5.3196e-03,  0.0000e+00])\n",
      "Epoch 18, Loss: 0.5658694505691528\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -9.3243e-04,\n",
      "        -2.9224e-03,  1.2145e-03,  0.0000e+00,  4.5998e-04,  6.0757e-04,\n",
      "         0.0000e+00, -4.0808e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -3.8539e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -9.9673e-05, -4.6200e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.1967e-04,  0.0000e+00,  0.0000e+00, -5.0149e-04,  0.0000e+00,\n",
      "         0.0000e+00,  6.3991e-05,  8.1773e-05,  0.0000e+00,  0.0000e+00,\n",
      "         3.5300e-04,  0.0000e+00,  0.0000e+00,  7.6599e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.9830e-04, -1.3605e-03,  0.0000e+00,  1.6915e-04,\n",
      "         0.0000e+00,  3.9522e-04, -9.1997e-03,  5.0187e-03,  0.0000e+00])\n",
      "Epoch 19, Loss: 0.5628060102462769\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -9.0187e-04,\n",
      "        -2.9823e-03,  1.1771e-03,  0.0000e+00,  4.4755e-04,  5.9078e-04,\n",
      "         0.0000e+00, -3.9896e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -3.8137e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -9.6022e-05, -4.4665e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.1612e-04,  0.0000e+00,  0.0000e+00, -4.7919e-04,  0.0000e+00,\n",
      "         0.0000e+00,  6.2775e-05,  7.9571e-05,  0.0000e+00,  0.0000e+00,\n",
      "         3.3794e-04,  0.0000e+00,  0.0000e+00,  7.3867e-04,  0.0000e+00,\n",
      "         0.0000e+00, -4.0304e-04, -1.3215e-03,  0.0000e+00,  1.6563e-04,\n",
      "         0.0000e+00,  3.8763e-04, -9.1968e-03,  4.7582e-03,  0.0000e+00])\n",
      "Epoch 20, Loss: 0.5602751970291138\n",
      "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -8.7672e-04,\n",
      "        -3.0437e-03,  1.1469e-03,  0.0000e+00,  4.3717e-04,  5.7717e-04,\n",
      "         0.0000e+00, -3.9153e-05,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -3.8017e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -9.4186e-05, -4.3399e-04,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.1318e-04,  0.0000e+00,  0.0000e+00, -4.6094e-04,  0.0000e+00,\n",
      "         0.0000e+00,  6.1580e-05,  7.7885e-05,  0.0000e+00,  0.0000e+00,\n",
      "         3.2569e-04,  0.0000e+00,  0.0000e+00,  7.2147e-04,  0.0000e+00,\n",
      "         0.0000e+00, -3.9850e-04, -1.2896e-03,  0.0000e+00,  1.6276e-04,\n",
      "         0.0000e+00,  3.8146e-04, -9.1922e-03,  4.5648e-03,  0.0000e+00])\n",
      "{'fc_base/img_mnist_0.048508.txt': False, 'fc_base/img_mnist_0.048839.txt': False}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from utils.loading import parse_spec\n",
    "from networks import get_network\n",
    "\n",
    "model = \"fc_base\"\n",
    "dataset = \"mnist\"\n",
    "\n",
    "if dataset == \"mnist\":\n",
    "    in_ch, in_dim, num_class = 1, 28, 10\n",
    "elif dataset == \"cifar10\":\n",
    "    in_ch, in_dim, num_class = 3, 32, 10\n",
    "else:\n",
    "    raise ValueError(f\"Unknown dataset: {dataset}\")\n",
    "\n",
    "resuls = {}\n",
    "for _, _, files in os.walk(f\"C:/Users/TTTeq/Documents/CodeFolder/CP/VCS/RTAI/rtai-project-21/test_cases/{model}/\"):\n",
    "    for file in files:\n",
    "        file_path = os.path.join(f\"C:/Users/TTTeq/Documents/CodeFolder/CP/VCS/RTAI/rtai-project-21/test_cases/{model}/\", file)\n",
    "        true_label, dataset, image, eps = parse_spec(file_path)\n",
    "        print(file_path.split(\"/\")[-1])\n",
    "        net = get_network(\n",
    "            model,\n",
    "            in_ch=in_ch,\n",
    "            in_dim=in_dim,\n",
    "            num_class=num_class,\n",
    "            weight_path=f\"models/{dataset}_{model}.pt\",\n",
    "        ).to(\"cpu\")\n",
    "        \n",
    "        print(net)\n",
    "        resuls[f\"{model}/{file}\"] = dp_verify(net, image, eps, true_label)\n",
    "print(resuls)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rtai-project",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}